├── ext ├── jerasure │ ├── NEWS │ ├── .gitignore │ ├── AUTHORS │ ├── ChangeLog │ ├── .gitattributes │ ├── Makefile.am │ ├── Examples │ │ ├── .gitignore │ │ └── Makefile.am │ ├── src │ │ ├── Makefile.am │ │ └── timing.c │ ├── m4 │ │ ├── ltversion.m4 │ │ └── ax_require_defined.m4 │ ├── include │ │ └── timing.h │ ├── configure.ac │ ├── COPYING │ └── License.txt ├── gf-complete │ ├── AUTHORS │ ├── NEWS │ ├── ChangeLog │ ├── autogen.sh │ ├── manual │ │ ├── image1.png │ │ ├── image2.png │ │ ├── image3.png │ │ ├── image4.png │ │ ├── image5.png │ │ ├── image6.png │ │ └── image7.png │ ├── test │ │ └── Makefile.am │ ├── Makefile.am │ ├── README │ ├── README.txt │ ├── include │ │ ├── gf_cpu.h │ │ ├── gf_method.h │ │ ├── gf_rand.h │ │ ├── gf_w64.h │ │ ├── gf_w16.h │ │ ├── gf_w32.h │ │ └── gf_w4.h │ ├── m4 │ │ └── ltversion.m4 │ ├── examples │ │ ├── Makefile.am │ │ ├── gf_example_1.c │ │ ├── gf_example_4.c │ │ ├── gf_example_7.c │ │ ├── gf_example_5.c │ │ └── gf_example_3.c │ ├── src │ │ ├── Makefile.am │ │ └── gf_rand.c │ ├── COPYING │ ├── License.txt │ └── tools │ │ ├── Makefile.am │ │ ├── gf_div.c │ │ └── gf_mult.c └── dns │ ├── doc │ └── Makefile │ ├── contrib │ └── Makefile │ ├── regress │ ├── Makefile │ ├── 00-spf_xtoi.c │ ├── 12-segfault-in-dns_res_frame_init.c │ └── 14-dns_resconf_search-fqdn.c │ ├── LICENSE │ └── src │ ├── fcrd.spf │ ├── cache.h │ └── zone.h ├── webui ├── .gitignore ├── files │ └── sorttable │ │ ├── arrow-up.gif │ │ ├── sorttable.js │ │ ├── arrow-down.gif │ │ └── arrow-none.gif └── chart.py ├── src ├── cc │ ├── common │ │ ├── .gitignore │ │ ├── nofilelimit.h │ │ ├── rusage.h │ │ ├── time.h │ │ ├── MemLock.h │ │ ├── hsieh_hash.h │ │ ├── Version.h │ │ ├── kfsatomic.h │ │ ├── time.cc │ │ ├── juliantime.h │ │ ├── AverageFilter.h │ │ └── Show.h │ ├── devtools │ │ ├── README │ │ └── CMakeLists.txt │ ├── fuse │ │ └── CMakeLists.txt │ ├── qcdio │ │ ├── qcdefs.h │ │ └── qcdebug.h │ ├── meta │ │ ├── layoutmanager_instance.cc │ │ ├── AuditLog.h │ │ ├── kfstypes.h │ │ └── IdempotentRequestTracker.h │ ├── libclient │ │ ├── utils.h │ │ ├── kfsglob.h │ │ └── ECMethodDef.h │ ├── access │ │ └── CMakeLists.txt │ ├── kfsio │ │ ├── blockname.h │ │ ├── NetForwarder.h │ │ ├── KfsCallbackObj.cc │ │ ├── FileHandle.h │ │ ├── event.h │ │ ├── ProcessRestarter.h │ │ ├── Base64.h │ │ ├── NetManagerWatcher.h │ │ ├── HttpChunkedDecoder.h │ │ └── ZlibInflate.h │ ├── tools │ │ ├── kfspwd.cc │ │ ├── kfsrmdir.cc │ │ ├── kfsmkdirs.cc │ │ ├── kfscd.cc │ │ ├── kfsappend.cc │ │ ├── utils.cc │ │ ├── Trash.h │ │ └── kfschgrp.cc │ ├── qcrs │ │ ├── rs.h │ │ └── encode.c │ ├── s3io │ │ └── CMakeLists.txt │ ├── emulator │ │ ├── emulator_setup.h │ │ └── CMakeLists.txt │ ├── chunk │ │ ├── utils.h │ │ └── IOMethodDef.h │ └── qfsc │ │ └── CMakeLists.txt ├── java │ ├── qfs-access-pre-9 │ │ ├── pom.xml │ │ └── src │ │ │ ├── test │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── quantcast │ │ │ └── qfs │ │ │ └── access │ │ │ ├── KfsTest.java │ │ │ ├── BufferPool.java │ │ │ ├── KfsFileAttr.java │ │ │ ├── Positionable.java │ │ │ ├── KfsAccessBase.java │ │ │ ├── KfsDelegation.java │ │ │ ├── KfsInputChannelBase.java │ │ │ ├── KfsOutputChannelBase.java │ │ │ ├── KfsOutputChannel.java │ │ │ ├── KfsInputChannel.java │ │ │ └── KfsAccess.java │ ├── hadoop-qfs │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── quantcast │ │ │ └── qfs │ │ │ └── hadoop │ │ │ └── CloseableIterator.java │ ├── qfs-access │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── quantcast │ │ │ │ └── qfs │ │ │ │ └── access │ │ │ │ ├── KfsDelegation.java │ │ │ │ ├── Positionable.java │ │ │ │ ├── KfsInputChannel.java │ │ │ │ ├── KfsOutputChannel.java │ │ │ │ └── BufferPool.java │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── quantcast │ │ │ └── qfs │ │ │ └── access │ │ │ └── TestQfsAccess.java │ ├── hadoop-qfs-2 │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── quantcast │ │ │ │ └── qfs │ │ │ │ └── hadoop │ │ │ │ └── QuantcastFileSystem2.java │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── quantcast │ │ │ └── qfs │ │ │ └── hadoop │ │ │ └── TestQuantcastFileSystem2.java │ └── README ├── go │ ├── go.sum │ ├── go.mod │ └── README.md └── test-scripts │ ├── setdebugloglevel.gdb │ ├── CMakeLists.txt │ ├── logcat.sh │ ├── rwtest.sh │ ├── chunkchksum_check_expected.sh │ ├── allocatesend.pl │ ├── debugchunkcore.sh │ └── allocaterecv.pl ├── .gitignore ├── CHANGELOG.md ├── wiki ├── images │ ├── Misc │ │ └── new_logo.png │ ├── Benchmarking │ │ ├── landing.png │ │ ├── read-write.png │ │ ├── throughputs.png │ │ ├── mstress-results.png │ │ └── write-read-sort.png │ ├── S3-Guide │ │ ├── webui_default.png │ │ └── webui_object_store_mode.png │ ├── Administrator's Guide │ │ └── qfs-webui.png │ ├── Architecture │ │ └── qfs_architecture.png │ ├── Deployment-Guide │ │ ├── chunkserver-node.png │ │ ├── metaserver-node.png │ │ ├── qfs-rack-capacity.png │ │ ├── qfs-chunk-server-rack.png │ │ ├── qfs-meta-server-rack.png │ │ ├── qfs-cluster-in-a-rack-rs.png │ │ ├── qfs-communication-flows.png │ │ ├── qfs-cluster-in-a-rack-repl.png │ │ └── qfs-rack-placement-groups-rs.png │ └── QFS-Kerberos-Security-Design │ │ └── Write-Path-with-Replication.png ├── makefile ├── External-Resources.md └── README ├── .ratignore ├── .codecov.yml ├── contrib └── plugins │ └── CMakeLists.txt ├── examples ├── sampleservers │ └── sample_qfs_tool.cfg ├── cc │ └── CMakeLists.txt └── python │ └── qfssample.cfg ├── travis └── before_install.sh ├── benchmarks └── mstress │ ├── mstress_install.sh │ └── mstress_cleanup.py └── scripts └── rat.sh /ext/jerasure/NEWS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ext/gf-complete/AUTHORS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ext/gf-complete/NEWS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ext/jerasure/.gitignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ext/jerasure/AUTHORS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ext/jerasure/ChangeLog: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ext/gf-complete/ChangeLog: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /webui/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /src/cc/common/.gitignore: -------------------------------------------------------------------------------- 1 | Version.cc 2 | Version.cc.* 3 | -------------------------------------------------------------------------------- /src/java/qfs-access-pre-9/pom.xml: -------------------------------------------------------------------------------- 1 | ../qfs-access/pom.xml -------------------------------------------------------------------------------- /src/java/qfs-access-pre-9/src/test: -------------------------------------------------------------------------------- 1 | ../../qfs-access/src/test -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | .vscode 3 | 4 | # Mac OS X 5 | .DS_Store 6 | 7 | -------------------------------------------------------------------------------- /ext/jerasure/.gitattributes: -------------------------------------------------------------------------------- 1 | *.ac eol=lf 2 | *.am eol=lf 3 | *.sh eol=lf 4 | -------------------------------------------------------------------------------- /ext/gf-complete/autogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | autoreconf --force --install -I m4 3 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changes 2 | 3 | 4 | -------------------------------------------------------------------------------- /wiki/images/Misc/new_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Misc/new_logo.png -------------------------------------------------------------------------------- /ext/gf-complete/manual/image1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/ext/gf-complete/manual/image1.png -------------------------------------------------------------------------------- /ext/gf-complete/manual/image2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/ext/gf-complete/manual/image2.png -------------------------------------------------------------------------------- /ext/gf-complete/manual/image3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/ext/gf-complete/manual/image3.png -------------------------------------------------------------------------------- /ext/gf-complete/manual/image4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/ext/gf-complete/manual/image4.png -------------------------------------------------------------------------------- /ext/gf-complete/manual/image5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/ext/gf-complete/manual/image5.png -------------------------------------------------------------------------------- /ext/gf-complete/manual/image6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/ext/gf-complete/manual/image6.png -------------------------------------------------------------------------------- /ext/gf-complete/manual/image7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/ext/gf-complete/manual/image7.png -------------------------------------------------------------------------------- /webui/files/sorttable/arrow-up.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/webui/files/sorttable/arrow-up.gif -------------------------------------------------------------------------------- /webui/files/sorttable/sorttable.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/webui/files/sorttable/sorttable.js -------------------------------------------------------------------------------- /webui/files/sorttable/arrow-down.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/webui/files/sorttable/arrow-down.gif -------------------------------------------------------------------------------- /webui/files/sorttable/arrow-none.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/webui/files/sorttable/arrow-none.gif -------------------------------------------------------------------------------- /wiki/images/Benchmarking/landing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Benchmarking/landing.png -------------------------------------------------------------------------------- /wiki/images/Benchmarking/read-write.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Benchmarking/read-write.png -------------------------------------------------------------------------------- /wiki/images/S3-Guide/webui_default.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/S3-Guide/webui_default.png -------------------------------------------------------------------------------- /wiki/images/Benchmarking/throughputs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Benchmarking/throughputs.png -------------------------------------------------------------------------------- /wiki/makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all 2 | all: 3 | git pull -s subtree git@github.com:quantcast/qfs.git master 4 | git push origin master 5 | -------------------------------------------------------------------------------- /wiki/images/Benchmarking/mstress-results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Benchmarking/mstress-results.png -------------------------------------------------------------------------------- /wiki/images/Benchmarking/write-read-sort.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Benchmarking/write-read-sort.png -------------------------------------------------------------------------------- /wiki/images/Administrator's Guide/qfs-webui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Administrator's Guide/qfs-webui.png -------------------------------------------------------------------------------- /wiki/images/Architecture/qfs_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Architecture/qfs_architecture.png -------------------------------------------------------------------------------- /wiki/images/Deployment-Guide/chunkserver-node.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Deployment-Guide/chunkserver-node.png -------------------------------------------------------------------------------- /wiki/images/Deployment-Guide/metaserver-node.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Deployment-Guide/metaserver-node.png -------------------------------------------------------------------------------- /wiki/images/S3-Guide/webui_object_store_mode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/S3-Guide/webui_object_store_mode.png -------------------------------------------------------------------------------- /wiki/images/Deployment-Guide/qfs-rack-capacity.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Deployment-Guide/qfs-rack-capacity.png -------------------------------------------------------------------------------- /wiki/images/Deployment-Guide/qfs-chunk-server-rack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Deployment-Guide/qfs-chunk-server-rack.png -------------------------------------------------------------------------------- /wiki/images/Deployment-Guide/qfs-meta-server-rack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Deployment-Guide/qfs-meta-server-rack.png -------------------------------------------------------------------------------- /ext/jerasure/Makefile.am: -------------------------------------------------------------------------------- 1 | # Jerasure Automake file 2 | 3 | ACLOCAL_AMFLAGS = -I m4 4 | 5 | SUBDIRS = src Examples 6 | 7 | EXTRA_DIST = Manual.pdf PERF.txt 8 | -------------------------------------------------------------------------------- /src/java/qfs-access-pre-9/src/main/java/com/quantcast/qfs/access/KfsTest.java: -------------------------------------------------------------------------------- 1 | ../../../../../../../../qfs-access/src/main/java/com/quantcast/qfs/access/KfsTest.java -------------------------------------------------------------------------------- /wiki/images/Deployment-Guide/qfs-cluster-in-a-rack-rs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Deployment-Guide/qfs-cluster-in-a-rack-rs.png -------------------------------------------------------------------------------- /wiki/images/Deployment-Guide/qfs-communication-flows.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Deployment-Guide/qfs-communication-flows.png -------------------------------------------------------------------------------- /wiki/images/Deployment-Guide/qfs-cluster-in-a-rack-repl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Deployment-Guide/qfs-cluster-in-a-rack-repl.png -------------------------------------------------------------------------------- /src/go/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= 2 | golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 3 | -------------------------------------------------------------------------------- /src/java/qfs-access-pre-9/src/main/java/com/quantcast/qfs/access/BufferPool.java: -------------------------------------------------------------------------------- 1 | ../../../../../../../../qfs-access/src/main/java/com/quantcast/qfs/access/BufferPool.java -------------------------------------------------------------------------------- /src/java/qfs-access-pre-9/src/main/java/com/quantcast/qfs/access/KfsFileAttr.java: -------------------------------------------------------------------------------- 1 | ../../../../../../../../qfs-access/src/main/java/com/quantcast/qfs/access/KfsFileAttr.java -------------------------------------------------------------------------------- /src/java/qfs-access-pre-9/src/main/java/com/quantcast/qfs/access/Positionable.java: -------------------------------------------------------------------------------- 1 | ../../../../../../../../qfs-access/src/main/java/com/quantcast/qfs/access/Positionable.java -------------------------------------------------------------------------------- /wiki/images/Deployment-Guide/qfs-rack-placement-groups-rs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/Deployment-Guide/qfs-rack-placement-groups-rs.png -------------------------------------------------------------------------------- /src/java/qfs-access-pre-9/src/main/java/com/quantcast/qfs/access/KfsAccessBase.java: -------------------------------------------------------------------------------- 1 | ../../../../../../../../qfs-access/src/main/java/com/quantcast/qfs/access/KfsAccessBase.java -------------------------------------------------------------------------------- /src/java/qfs-access-pre-9/src/main/java/com/quantcast/qfs/access/KfsDelegation.java: -------------------------------------------------------------------------------- 1 | ../../../../../../../../qfs-access/src/main/java/com/quantcast/qfs/access/KfsDelegation.java -------------------------------------------------------------------------------- /src/java/qfs-access-pre-9/src/main/java/com/quantcast/qfs/access/KfsInputChannelBase.java: -------------------------------------------------------------------------------- 1 | ../../../../../../../../qfs-access/src/main/java/com/quantcast/qfs/access/KfsInputChannelBase.java -------------------------------------------------------------------------------- /src/java/qfs-access-pre-9/src/main/java/com/quantcast/qfs/access/KfsOutputChannelBase.java: -------------------------------------------------------------------------------- 1 | ../../../../../../../../qfs-access/src/main/java/com/quantcast/qfs/access/KfsOutputChannelBase.java -------------------------------------------------------------------------------- /wiki/images/QFS-Kerberos-Security-Design/Write-Path-with-Replication.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantcast/qfs/HEAD/wiki/images/QFS-Kerberos-Security-Design/Write-Path-with-Replication.png -------------------------------------------------------------------------------- /ext/jerasure/Examples/.gitignore: -------------------------------------------------------------------------------- 1 | /cauchy_[0-9][0-9] 2 | /decoder 3 | /encoder 4 | /jerasure_[0-9][0-9] 5 | /liberation_[0-9][0-9] 6 | /reed_sol_[0-9][0-9] 7 | /reed_sol_test_gf 8 | /reed_sol_time_gf -------------------------------------------------------------------------------- /ext/gf-complete/test/Makefile.am: -------------------------------------------------------------------------------- 1 | # GF-Complete 'test' AM file 2 | 3 | AM_CPPFLAGS = -I$(top_builddir)/include -I$(top_srcdir)/include 4 | AM_CFLAGS = -O3 -fPIC 5 | 6 | bin_PROGRAMS = gf_unit 7 | 8 | gf_unit_SOURCES = gf_unit.c 9 | #gf_unit_LDFLAGS = -lgf_complete 10 | gf_unit_LDADD = ../src/libgf_complete.la 11 | 12 | -------------------------------------------------------------------------------- /.ratignore: -------------------------------------------------------------------------------- 1 | # Individual files 2 | .gitignore 3 | .ratignore 4 | README.md 5 | CHANGELOG.md 6 | tags 7 | sorttable.js 8 | # Directories 9 | ext 10 | wiki 11 | build 12 | 13 | # Generated/auto-generated files 14 | src/cc/common/Version.cc 15 | src/go/go.sum 16 | src/go/README.md 17 | 18 | # Specific gitignore files 19 | src/cc/common/.gitignore 20 | webui/.gitignore 21 | -------------------------------------------------------------------------------- /ext/gf-complete/Makefile.am: -------------------------------------------------------------------------------- 1 | # Top-level GF-Complete AM file 2 | # Distributes headers 3 | 4 | SUBDIRS = src tools test examples 5 | ACLOCAL_AMFLAGS = -I m4 6 | 7 | include_HEADERS = include/gf_complete.h include/gf_method.h include/gf_rand.h include/gf_general.h 8 | 9 | # display the output of failed TESTS after a failed make check 10 | # export VERBOSE = true 11 | -------------------------------------------------------------------------------- /ext/dns/doc/Makefile: -------------------------------------------------------------------------------- 1 | 2 | 3 | all: dns.pdf # spf.pdf 4 | 5 | %.pdf: %.tex 6 | @echo Pass 1 7 | @pdflatex $< >/dev/null /dev/null /dev/null 12 | #include 13 | #include 14 | 15 | /* These are all pretty self-explanatory */ 16 | uint32_t MOA_Random_32(); 17 | uint64_t MOA_Random_64(); 18 | void MOA_Random_128(uint64_t *x); 19 | uint32_t MOA_Random_W(int w, int zero_ok); 20 | void MOA_Fill_Random_Region (void *reg, int size); /* reg should be aligned to 4 bytes, but 21 | size can be anything. */ 22 | void MOA_Seed(uint32_t seed); 23 | -------------------------------------------------------------------------------- /ext/jerasure/include/timing.h: -------------------------------------------------------------------------------- 1 | // Timing measurement utilities. 2 | 3 | #ifndef JERASURE_INCLUDED__TIMING_H 4 | #define JERASURE_INCLUDED__TIMING_H 5 | 6 | // Define USE_CLOCK to use clock(). Otherwise use gettimeofday(). 7 | #define USE_CLOCK 8 | 9 | #ifdef USE_CLOCK 10 | #include 11 | #else 12 | #include 13 | #endif 14 | 15 | struct timing { 16 | #ifdef USE_CLOCK 17 | clock_t clock; 18 | #else 19 | struct timeval tv; 20 | #endif 21 | }; 22 | 23 | // Get the current time as a double in seconds. 24 | double 25 | timing_now( 26 | void); 27 | 28 | // Set *t to the current time. 29 | void 30 | timing_set( 31 | struct timing * t); 32 | 33 | // Get *t as a double in seconds. 34 | double 35 | timing_get( 36 | struct timing * t); 37 | 38 | // Return *t2 - *t1 as a double in seconds. 39 | double 40 | timing_delta( 41 | struct timing * t1, 42 | struct timing * t2); 43 | #endif 44 | -------------------------------------------------------------------------------- /ext/dns/contrib/Makefile: -------------------------------------------------------------------------------- 1 | # Conditional compiler flag gmake routine, from Linux. 2 | cc-option = $(shell if $(CC) $(CFLAGS) $(1) -S -o /dev/null -xc /dev/null \ 3 | > /dev/null 2>&1; then echo "$(1)"; else echo "$(2)"; fi;) 4 | 5 | override CPPFLAGS := $(CPPFLAGS) -I../src -DSOCKET_DEBUG -DSOCKET_MAIN 6 | override CFLAGS := $(CFLAGS) -std=gnu99 -Wall -Wno-unused-function -Wno-unused-parameter $(call cc-option, -Wno-initializer-overrides,) -O2 -g -fstack-protector 7 | override LIBS := -lssl -lcrypto 8 | 9 | ifeq ($(shell uname -s), Darwin) 10 | override CFLAGS += $(CFLAGS) -Wno-deprecated-declarations 11 | endif 12 | 13 | all: socket 14 | 15 | socket: socket.c ../src/dns.c 16 | $(CC) $(CFLAGS) $(CPPFLAGS) -o $@ $^ $(LIBS) 17 | 18 | socket.S: socket.c 19 | $(CC) $(CFLAGS) -S $(CPPFLAGS) -o $@ $^ 20 | 21 | .PHONY: clean 22 | 23 | clean: 24 | rm -f socket socket.S socket.exe 25 | rm -fr *.dSYM 26 | 27 | -------------------------------------------------------------------------------- /src/test-scripts/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # $Id$ 3 | # 4 | # Created 2010/07/16 5 | # Author: Mike Ovsiannikov 6 | # 7 | # Copyright 2010-2011,2016 Quantcast Corporation. All rights reserved. 8 | # 9 | # This file is part of Kosmos File System (KFS). 10 | # 11 | # Licensed under the Apache License, Version 2.0 12 | # (the "License"); you may not use this file except in compliance with 13 | # the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, software 18 | # distributed under the License is distributed on an "AS IS" BASIS, 19 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | # implied. See the License for the specific language governing 21 | # permissions and limitations under the License. 22 | # 23 | # 24 | 25 | ADD_TEST(kfstest ${CMAKE_CURRENT_SOURCE_DIR}/qfstest.sh ${PROJECT_BINARY_DIR}) 26 | -------------------------------------------------------------------------------- /src/cc/devtools/README: -------------------------------------------------------------------------------- 1 | // Copyright 2012,2016 Quantcast Corporation. All rights reserved. 2 | // 3 | // This file is part of Kosmos File System (KFS). 4 | // 5 | // Licensed under the Apache License, Version 2.0 6 | // (the "License"); you may not use this file except in compliance with 7 | // the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 14 | // implied. See the License for the specific language governing 15 | // permissions and limitations under the License. 16 | // 17 | 18 | The files in this directory are meant for filesystem developers' internal use. 19 | Please refrain from executing the corresponding binaries without understanding 20 | the code. 21 | -------------------------------------------------------------------------------- /.codecov.yml: -------------------------------------------------------------------------------- 1 | # 2 | # $Id$ 3 | # 4 | # Copyright 2017 Quantcast Corporation. All rights reserved. 5 | # 6 | # This file is part of Quantcast File System. 7 | # 8 | # Licensed under the Apache License, Version 2.0 9 | # (the "License"); you may not use this file except in compliance with 10 | # the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | # implied. See the License for the specific language governing 18 | # permissions and limitations under the License. 19 | # 20 | # "Codecov" configuration. 21 | # 22 | 23 | coverage: 24 | 25 | status: 26 | 27 | project: 28 | default: 29 | enabled: true 30 | target: 43% 31 | # threshold: 1% 32 | 33 | patch: 34 | default: 35 | enabled: true 36 | target: 43% 37 | # threshold: 1% 38 | -------------------------------------------------------------------------------- /contrib/plugins/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # $Id$ 3 | # 4 | # Created 2016/02/17 5 | # Author: Mehmet Can Kurt 6 | # 7 | # Copyright 2016-2017 Quantcast Corporation. All rights reserved. 8 | # 9 | # This file is part of Quantcast File System. 10 | # 11 | # Licensed under the Apache License, Version 2.0 12 | # (the "License"); you may not use this file except in compliance with 13 | # the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, software 18 | # distributed under the License is distributed on an "AS IS" BASIS, 19 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | # implied. See the License for the specific language governing 21 | # permissions and limitations under the License. 22 | 23 | add_library (qfsmonitorplugin SHARED qfs_monitor.cc) 24 | add_dependencies (qfsmonitorplugin qcdio-shared) 25 | target_link_libraries (qfsmonitorplugin qcdio-shared) 26 | set_target_properties (qfsmonitorplugin PROPERTIES OUTPUT_NAME "qfs_monitor") 27 | -------------------------------------------------------------------------------- /src/test-scripts/logcat.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # $Id$ 5 | # 6 | # Copyright 2012,2016 Quantcast Corporation. All rights reserved. 7 | # 8 | # This file is part of Kosmos File System (KFS). 9 | # 10 | # Licensed under the Apache License, Version 2.0 11 | # (the "License"); you may not use this file except in compliance with 12 | # the License. You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | # implied. See the License for the specific language governing 20 | # permissions and limitations under the License. 21 | # 22 | # 23 | 24 | logd=${1-.} 25 | if [ -f "$logd"/current ]; then 26 | true 27 | else 28 | "Usage: $0 log-dir [file-count]" 29 | exit 1 30 | fi 31 | 32 | { 33 | 34 | echo "$logd"/@*.s | tr ' ' '\n' | sort | tail -n ${2-20} | xargs gunzip -c 35 | cat "$logd"/current 36 | 37 | } | sed -e 's/^@[0-9a-fA-F]* //' 38 | -------------------------------------------------------------------------------- /ext/dns/regress/Makefile: -------------------------------------------------------------------------------- 1 | 2 | all: rfc4408-tests tests 3 | 4 | CPPFLAGS += -I/usr/local/libyaml/include -I../src -DDNS_DEBUG=0 -DSPF_DEBUG=1 -DVM_MAXCODE=3182 5 | CFLAGS += -std=gnu99 -g 6 | LDFLAGS += -L/usr/local/libyaml/lib 7 | 8 | rfc4408-tests: rfc4408-tests.c ../src/cache.c ../src/zone.c ../src/spf.c ../src/dns.c 9 | $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $^ -lyaml $(LIBS) 10 | 11 | %.c: %.rl 12 | ragel -C -o $@ $< 13 | 14 | TESTS = \ 15 | 00-spf_xtoi \ 16 | 12-segfault-in-dns_res_frame_init \ 17 | 14-dns_resconf_search-fqdn \ 18 | 15-dns_ai_nextaf-null-deref 19 | 20 | 00-spf_xtoi: 00-spf_xtoi.c ../src/spf.c 21 | 12-segfault-in-dns_res_frame_init: 12-segfault-in-dns_res_frame_init.c 22 | 14-dns_resconf_search-fqdn: 14-dns_resconf_search-fqdn.c 23 | 15-dns_ai_nextaf-null-deref: 15-dns_ai_nextaf-null-deref.c 24 | 25 | ${TESTS}: ../src/dns.c 26 | ${TESTS}: 27 | $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $@.c ../src/dns.c $(LIBS) 28 | 29 | tests: ${TESTS} 30 | 31 | check: ${TESTS} 32 | @for T in ${TESTS}; do ./$$T; done 33 | 34 | clean: 35 | rm -f rfc4408-tests ${TESTS} fpack 36 | rm -fr *.dSYM 37 | 38 | -------------------------------------------------------------------------------- /ext/dns/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2008-2015 William Ahern 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to 5 | deal in the Software without restriction, including without limitation the 6 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | sell copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | IN THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /src/test-scripts/rwtest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # $Id: #1 $ 4 | # 5 | # Created 2010/5/25 6 | # Author: Mike Ovsiannikov 7 | # 8 | # Copyright 2010,2016 Quantcast Corporation. All rights reserved. 9 | # 10 | # This file is part of Kosmos File System (KFS). 11 | # 12 | # Licensed under the Apache License, Version 2.0 13 | # (the "License"); you may not use this file except in compliance with 14 | # the License. You may obtain a copy of the License at 15 | # 16 | # http://www.apache.org/licenses/LICENSE-2.0 17 | # 18 | # Unless required by applicable law or agreed to in writing, software 19 | # distributed under the License is distributed on an "AS IS" BASIS, 20 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | # implied. See the License for the specific language governing 22 | # permissions and limitations under the License. 23 | # 24 | # 25 | 26 | if [ ! -x rwtest ]; then 27 | gcc -Wall -g3 -DIOV_MAX=256 rwtest.c -o rwtest || exit 28 | fi 29 | tf="rwtest.$$.file" 30 | rm -rf "$tf"; 31 | while date; do 32 | xfs_io -f -c 'resvsp 0 10m' -c 'truncate 10m' "$tf" 33 | ./rwtest "$tf" || break 34 | rm "$tf" 35 | done > "rwtest.$$.log" 2>&1 & 36 | -------------------------------------------------------------------------------- /ext/gf-complete/examples/Makefile.am: -------------------------------------------------------------------------------- 1 | # GF-Complete 'examples' AM file 2 | 3 | 4 | bin_PROGRAMS = gf_example_1 gf_example_2 gf_example_3 gf_example_4 \ 5 | gf_example_5 gf_example_6 gf_example_7 6 | 7 | gf_example_1_SOURCES = gf_example_1.c 8 | #gf_example_1_LDFLAGS = -lgf_complete 9 | gf_example_1_LDADD = ../src/libgf_complete.la 10 | 11 | gf_example_2_SOURCES = gf_example_2.c 12 | #gf_example_2_LDFLAGS = -lgf_complete 13 | gf_example_2_LDADD = ../src/libgf_complete.la 14 | 15 | gf_example_3_SOURCES = gf_example_3.c 16 | #gf_example_3_LDFLAGS = -lgf_complete 17 | gf_example_3_LDADD = ../src/libgf_complete.la 18 | 19 | gf_example_4_SOURCES = gf_example_4.c 20 | #gf_example_4_LDFLAGS = -lgf_complete 21 | gf_example_4_LDADD = ../src/libgf_complete.la 22 | 23 | gf_example_5_SOURCES = gf_example_5.c 24 | #gf_example_5_LDFLAGS = -lgf_complete 25 | gf_example_5_LDADD = ../src/libgf_complete.la 26 | 27 | gf_example_6_SOURCES = gf_example_6.c 28 | #gf_example_6_LDFLAGS = -lgf_complete 29 | gf_example_6_LDADD = ../src/libgf_complete.la 30 | 31 | gf_example_7_SOURCES = gf_example_7.c 32 | #gf_example_7_LDFLAGS = -lgf_complete 33 | gf_example_7_LDADD = ../src/libgf_complete.la 34 | 35 | 36 | -------------------------------------------------------------------------------- /src/cc/fuse/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # $Id$ 3 | # 4 | # Created 2006 5 | # Author: Sriram Rao (Kosmix Corp) 6 | # 7 | # Copyright 2008-2012,2016 Quantcast Corporation. All rights reserved. 8 | # Copyright 2006 Kosmix Corp. 9 | # 10 | # This file is part of Kosmos File System (KFS). 11 | # 12 | # Licensed under the Apache License, Version 2.0 13 | # (the "License"); you may not use this file except in compliance with 14 | # the License. You may obtain a copy of the License at 15 | # 16 | # http://www.apache.org/licenses/LICENSE-2.0 17 | # 18 | # Unless required by applicable law or agreed to in writing, software 19 | # distributed under the License is distributed on an "AS IS" BASIS, 20 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | # implied. See the License for the specific language governing 22 | # permissions and limitations under the License. 23 | # 24 | # 25 | 26 | # 27 | # Build the fuse binary; build it only on demand 28 | # 29 | 30 | add_executable (qfs_fuse kfs_fuse_main.cc) 31 | target_link_libraries (qfs_fuse kfsClient ${FUSE_LIBRARIES}) 32 | 33 | # 34 | install (TARGETS qfs_fuse 35 | RUNTIME DESTINATION bin 36 | LIBRARY DESTINATION lib 37 | ARCHIVE DESTINATION lib-static) 38 | -------------------------------------------------------------------------------- /src/cc/qcdio/qcdefs.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2024/09/30 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2024 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Quantcast File System (QFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | 24 | #ifndef QC_DEFS_H 25 | #define QC_DEFS_H 26 | 27 | #if !defined(QC_GCC_13_PRAGMA) && defined(__GNUC__) && !defined(__clang__) 28 | #if 13 <= __GNUC__ 29 | # define QC_GCC_13_PRAGMA(x) _Pragma(#x) 30 | #endif 31 | #endif 32 | 33 | #ifndef QC_GCC_13_PRAGMA 34 | # define QC_GCC_13_PRAGMA(x) 35 | #endif 36 | 37 | #endif /* QC_DEFS_H */ 38 | -------------------------------------------------------------------------------- /examples/sampleservers/sample_qfs_tool.cfg: -------------------------------------------------------------------------------- 1 | # 2 | # $Id# 3 | # 4 | # Copyright 2008-2017 Quantcast Corporation. All rights reserved. 5 | # 6 | # This file is part of Quantcast File System (QFS). 7 | # 8 | # Licensed under the Apache License, Version 2.0 9 | # (the "License"); you may not use this file except in compliance with 10 | # the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | # implied. See the License for the specific language governing 18 | # permissions and limitations under the License. 19 | # 20 | # The metaserver configuration. 21 | # Additional information can be found in 22 | # https://github.com/quantcast/qfs/wiki/Configuration-Reference. 23 | 24 | fs.default = qfs://localhost:20000 25 | #fs.msgLogWriter.logLevel=ERROR 26 | # fs.trash.minPathDepth = 5 27 | # fs.trash.current = Current 28 | # fs.trash.homesPrefix = /user 29 | # fs.trash.interval = 60 30 | # fs.trash.trash = .Trash 31 | # dfs.force.remove = true 32 | #fs.euser = 3008 33 | #fs.egroup = 505 34 | -------------------------------------------------------------------------------- /src/cc/common/nofilelimit.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2012/10/25 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2012,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \file nofilelimit.h 24 | // \brief function to set process max number of open files. 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | 29 | #ifndef KFS_NOFILELIMIT_H 30 | #define KFS_NOFILELIMIT_H 31 | 32 | namespace KFS { 33 | 34 | int SetMaxNoFileLimit(); 35 | 36 | } 37 | 38 | #endif /* KFS_NOFILELIMIT_H */ 39 | -------------------------------------------------------------------------------- /examples/cc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # $Id$ 3 | # 4 | # Author: Thilee Subramaniam 5 | # 6 | # Copyright 2012,2016 Quantcast Corporation. All rights reserved. 7 | # 8 | # This file is part of Kosmos File System (KFS). 9 | # 10 | # Licensed under the Apache License, Version 2.0 11 | # (the "License"); you may not use this file except in compliance with 12 | # the License. You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | # implied. See the License for the specific language governing 20 | # permissions and limitations under the License. 21 | # 22 | # Build the C++ example binary 23 | # 24 | 25 | add_executable (qfssample qfssample_main.cc) 26 | 27 | if (USE_STATIC_LIB_LINKAGE) 28 | add_dependencies (qfssample kfsClient) 29 | target_link_libraries (qfssample kfsClient) 30 | else (USE_STATIC_LIB_LINKAGE) 31 | target_link_libraries (qfssample kfsClient-shared) 32 | add_dependencies (qfssample kfsClient-shared) 33 | endif (USE_STATIC_LIB_LINKAGE) 34 | 35 | install (TARGETS qfssample 36 | RUNTIME DESTINATION bin/examples) 37 | -------------------------------------------------------------------------------- /src/cc/meta/layoutmanager_instance.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2012/02/23 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2012,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \file LayoutManagerInstance.cc 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #include "LayoutManager.h" 28 | 29 | namespace KFS 30 | { 31 | 32 | /* static */ LayoutManager& 33 | LayoutManager::Instance() 34 | { 35 | static LayoutManager sLayoutManager; 36 | return sLayoutManager; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /webui/chart.py: -------------------------------------------------------------------------------- 1 | # 2 | # $Id$ 3 | # 4 | # Copyright 2011,2016 Quantcast Corporation. All rights reserved. 5 | # 6 | # Author: Kate Labeeva 7 | # 8 | # This file is part of Kosmos File System (KFS). 9 | # 10 | # Licensed under the Apache License, Version 2.0 11 | # (the "License"); you may not use this file except in compliance with 12 | # the License. You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | # implied. See the License for the specific language governing 20 | # permissions and limitations under the License. 21 | # 22 | # 23 | 24 | 25 | kDeltaPrefix = "D-" 26 | 27 | 28 | class ChartServerData: 29 | def __init__(self, serverName, serverArray): 30 | self.serverName = serverName 31 | self.serverArray = serverArray 32 | 33 | 34 | class ChartData: 35 | def __init__(self): 36 | self.headers = None 37 | self.serverArray = [] 38 | 39 | 40 | class ChartHTML: 41 | def __init__(self, chartData): 42 | self.chartData = chartData 43 | 44 | def printToHTML(self, buffer): 45 | print("TBD") 46 | -------------------------------------------------------------------------------- /examples/python/qfssample.cfg: -------------------------------------------------------------------------------- 1 | # 2 | # $Id$ 3 | # 4 | # Author: Thilee Subramaniam 5 | # 6 | # Copyright 2012,2016 Quantcast Corporation. All rights reserved. 7 | # 8 | # This file is part of Kosmos File System (KFS). 9 | # 10 | # Licensed under the Apache License, Version 2.0 11 | # (the "License"); you may not use this file except in compliance with 12 | # the License. You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | # implied. See the License for the specific language governing 20 | # permissions and limitations under the License. 21 | # 22 | # NOTE: The python support for QFS is EXPERIMENTAL at this stage. The 23 | # python extension module has not been tested on large scale 24 | # deploymentsi yet. Please excercise caution while using the 25 | # python wrapper. 26 | 27 | # This config file can be used by the python client to connect to the servers 28 | # setup locally. The metaserver host and port match the config file: 29 | # examples/sampleservers/sample_setup.cfg 30 | 31 | metaServer.name = localhost 32 | metaServer.port = 20000 33 | -------------------------------------------------------------------------------- /ext/dns/regress/00-spf_xtoi.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | #include "spf.c" 7 | 8 | int main(void) { 9 | unsigned char txt[2] = ""; 10 | unsigned i, n, exp; 11 | 12 | for (i = 0; i < 256; i++) { 13 | txt[0] = i; 14 | n = spf_xtoi((char *)txt); 15 | 16 | switch (i) { 17 | case '0': 18 | case '1': 19 | case '2': 20 | case '3': 21 | case '4': 22 | case '5': 23 | case '6': 24 | case '7': 25 | case '8': 26 | case '9': 27 | exp = i - '0'; 28 | break; 29 | case 'A': 30 | case 'B': 31 | case 'C': 32 | case 'D': 33 | case 'E': 34 | case 'F': 35 | exp = 10 + (i - 'A'); 36 | break; 37 | case 'a': 38 | case 'b': 39 | case 'c': 40 | case 'd': 41 | case 'e': 42 | case 'f': 43 | exp = 10 + (i - 'a'); 44 | break; 45 | default: 46 | exp = 0; 47 | break; 48 | } 49 | 50 | if (n != exp) 51 | errx(1, "expected %u, got %u", exp, n); 52 | } 53 | 54 | warnx("OK"); 55 | 56 | for (i = 0; i < 1U << 20; i++) { 57 | unsigned r = (unsigned)random(); 58 | char txt[20]; 59 | unsigned n; 60 | 61 | if (snprintf(txt, sizeof txt, "%x", r) < 0) 62 | err(1, "snprintf"); 63 | 64 | n = spf_xtoi(txt); 65 | if (r != n) 66 | errx(1, "expected 0x%x, got 0x%x (\"%s\")", r, n, txt); 67 | } 68 | 69 | return 0; 70 | } 71 | -------------------------------------------------------------------------------- /src/cc/qcdio/qcdebug.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2008/11/01 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2008-2012,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // Debug definitions. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef QCDEBUG_H 28 | #define QCDEBUG_H 29 | 30 | #include 31 | 32 | #define QCASSERT(a) assert(a) 33 | 34 | #ifdef NDEBUG 35 | # define QCVERIFY(a) if (!(a)) assert(0) 36 | #else 37 | # define QCVERIFY(a) QCASSERT(a) 38 | #endif 39 | 40 | #endif /* QCDEBUG_H */ 41 | -------------------------------------------------------------------------------- /src/java/hadoop-qfs/src/main/java/com/quantcast/qfs/hadoop/CloseableIterator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed under the Apache License, Version 2.0 4 | * (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | * implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | * 15 | * Implements the Hadoop FS interfaces to allow applications to store files in 16 | * Quantcast File System (QFS). This is an extension of KFS. 17 | */ 18 | 19 | package com.quantcast.qfs.hadoop; 20 | 21 | import java.io.Closeable; 22 | import java.util.Iterator; 23 | 24 | /** 25 | * A {@link Iterator} which is also {@link Closeable}. 26 | * 27 | * A Iterator that provides a close() to release resources. 28 | * 29 | * This can be useful if the Iterator holds resources that should be 30 | * released when the Iterator is no longer needed. 31 | */ 32 | 33 | public interface CloseableIterator extends Iterator, Closeable { 34 | } 35 | -------------------------------------------------------------------------------- /src/java/qfs-access/src/main/java/com/quantcast/qfs/access/KfsDelegation.java: -------------------------------------------------------------------------------- 1 | /** 2 | * $Id$ 3 | * 4 | * Created 2013/12/15 5 | * 6 | * @author: Mike Ovsiannikov 7 | * 8 | * Copyright 2013,2016 Quantcast Corporation. All rights reserved. 9 | * 10 | * This file is part of Quantcast File System 11 | * 12 | * Licensed under the Apache License, Version 2.0 13 | * (the "License"); you may not use this file except in compliance with 14 | * the License. You may obtain a copy of the License at 15 | * 16 | * http://www.apache.org/licenses/LICENSE-2.0 17 | * 18 | * Unless required by applicable law or agreed to in writing, software 19 | * distributed under the License is distributed on an "AS IS" BASIS, 20 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | * implied. See the License for the specific language governing 22 | * permissions and limitations under the License. 23 | * 24 | * \brief Class to return create delegation token method result. 25 | */ 26 | 27 | package com.quantcast.qfs.access; 28 | 29 | public class KfsDelegation 30 | { 31 | public KfsDelegation() {} 32 | public boolean delegationAllowedFlag; 33 | public long issuedTime; 34 | public long tokenValidForSec; 35 | public long delegationValidForSec; 36 | public String token; 37 | public String key; 38 | } 39 | -------------------------------------------------------------------------------- /ext/gf-complete/src/Makefile.am: -------------------------------------------------------------------------------- 1 | # GF-Complete 'core' AM file 2 | # Creates the library 3 | 4 | AUTOMAKE_OPTIONS = subdir-objects 5 | 6 | AM_CPPFLAGS = -I$(top_builddir)/include -I$(top_srcdir)/include 7 | 8 | # avoid using SIMD_FLAGS for code that calls strcmp as new gcc 9 | # versions will use SIMD for the strcmp implementation. Instead 10 | # we create a static library just for gf_method that is not compiled 11 | # with SIMD_FLAGS, this static library will get linked into gf_complete.so 12 | noinst_LTLIBRARIES = libgf_util.la 13 | libgf_util_la_SOURCES = gf_method.c 14 | libgf_util_la_CFLAGS = -O3 -fPIC -Wsign-compare 15 | 16 | # we narrowly use SIMD_FLAGS for code that needs it 17 | lib_LTLIBRARIES = libgf_complete.la 18 | libgf_complete_la_SOURCES = gf.c gf_wgen.c gf_w4.c gf_w8.c gf_w16.c gf_w32.c \ 19 | gf_w64.c gf_w128.c gf_rand.c gf_general.c gf_cpu.c 20 | libgf_complete_la_CFLAGS = -O3 $(SIMD_FLAGS) -fPIC -Wsign-compare 21 | libgf_complete_la_LIBADD = libgf_util.la 22 | 23 | if HAVE_NEON 24 | libgf_complete_la_SOURCES += neon/gf_w4_neon.c \ 25 | neon/gf_w8_neon.c \ 26 | neon/gf_w16_neon.c \ 27 | neon/gf_w32_neon.c \ 28 | neon/gf_w64_neon.c 29 | endif 30 | 31 | libgf_complete_la_LDFLAGS = -version-info 1:0:0 32 | 33 | -------------------------------------------------------------------------------- /src/java/hadoop-qfs-2/src/main/java/com/quantcast/qfs/hadoop/QuantcastFileSystem2.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed under the Apache License, Version 2.0 4 | * (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | * implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | * 15 | * Extends QuantcastFileSystem as needed to be compatible with Hadoop 2.x. 16 | */ 17 | 18 | package com.quantcast.qfs.hadoop; 19 | 20 | import java.io.IOException; 21 | import java.net.URI; 22 | 23 | import org.apache.hadoop.fs.FileStatus; 24 | import org.apache.hadoop.fs.Path; 25 | 26 | public class QuantcastFileSystem2 extends QuantcastFileSystem { 27 | 28 | public QuantcastFileSystem2() { 29 | super(); 30 | } 31 | 32 | QuantcastFileSystem2(IFSImpl fsimpl, URI uri) { 33 | super(fsimpl, uri); 34 | } 35 | 36 | public FileStatus[] listStatus(Path path) throws IOException { 37 | return listStatusInternal(path); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/cc/common/rusage.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2012/05/03 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2012,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \file rusage.h 24 | // \brief getrusage wrapper 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #ifndef RUSAGE_H 29 | #define RUSAGE_H 30 | 31 | #include 32 | 33 | namespace KFS 34 | { 35 | using std::ostream; 36 | 37 | ostream& showrusage(ostream& inStream, const char* inSeparatorPtr, 38 | const char* inDelimiterPtr, bool inSelfFlag); 39 | } 40 | 41 | #endif /* RUSAGE_H */ 42 | -------------------------------------------------------------------------------- /ext/gf-complete/include/gf_w64.h: -------------------------------------------------------------------------------- 1 | /* 2 | * GF-Complete: A Comprehensive Open Source Library for Galois Field Arithmetic 3 | * James S. Plank, Ethan L. Miller, Kevin M. Greenan, 4 | * Benjamin A. Arnold, John A. Burnum, Adam W. Disney, Allen C. McBride. 5 | * 6 | * gf_w64.h 7 | * 8 | * Defines and data structures for 64-bit Galois fields 9 | */ 10 | 11 | #ifndef GF_COMPLETE_GF_W64_H 12 | #define GF_COMPLETE_GF_W64_H 13 | 14 | #include 15 | 16 | #define GF_FIELD_WIDTH (64) 17 | #define GF_FIRST_BIT (1ULL << 63) 18 | 19 | #define GF_BASE_FIELD_WIDTH (32) 20 | #define GF_BASE_FIELD_SIZE (1ULL << GF_BASE_FIELD_WIDTH) 21 | #define GF_BASE_FIELD_GROUP_SIZE GF_BASE_FIELD_SIZE-1 22 | 23 | struct gf_w64_group_data { 24 | uint64_t *reduce; 25 | uint64_t *shift; 26 | uint64_t *memory; 27 | }; 28 | 29 | struct gf_split_4_64_lazy_data { 30 | uint64_t tables[16][16]; 31 | uint64_t last_value; 32 | }; 33 | 34 | struct gf_split_8_64_lazy_data { 35 | uint64_t tables[8][(1<<8)]; 36 | uint64_t last_value; 37 | }; 38 | 39 | struct gf_split_16_64_lazy_data { 40 | uint64_t tables[4][(1<<16)]; 41 | uint64_t last_value; 42 | }; 43 | 44 | struct gf_split_8_8_data { 45 | uint64_t tables[15][256][256]; 46 | }; 47 | 48 | void gf_w64_neon_split_init(gf_t *gf); 49 | 50 | #endif /* GF_COMPLETE_GF_W64_H */ 51 | -------------------------------------------------------------------------------- /src/cc/common/time.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2010/10/27 5 | // Author: Dan Adkins 6 | // 7 | // Copyright 2010,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \file time.cc 24 | // \brief time related functions 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #ifndef COMMON_TIME_H 29 | #define COMMON_TIME_H 30 | 31 | #include 32 | 33 | namespace KFS { 34 | 35 | bool getcurrenttime(int64_t* sec, int64_t* usec); 36 | int64_t microseconds(); 37 | int64_t cputime(int64_t *user, int64_t *sys); 38 | 39 | } // namespace KFS 40 | 41 | #endif // COMMON_TIME_H 42 | -------------------------------------------------------------------------------- /ext/jerasure/m4/ax_require_defined.m4: -------------------------------------------------------------------------------- 1 | # =========================================================================== 2 | # http://www.gnu.org/software/autoconf-archive/ax_require_defined.html 3 | # =========================================================================== 4 | # 5 | # SYNOPSIS 6 | # 7 | # AX_REQUIRE_DEFINED(MACRO) 8 | # 9 | # DESCRIPTION 10 | # 11 | # AX_REQUIRE_DEFINED is a simple helper for making sure other macros have 12 | # been defined and thus are available for use. This avoids random issues 13 | # where a macro isn't expanded. Instead the configure script emits a 14 | # non-fatal: 15 | # 16 | # ./configure: line 1673: AX_CFLAGS_WARN_ALL: command not found 17 | # 18 | # It's like AC_REQUIRE except it doesn't expand the required macro. 19 | # 20 | # Here's an example: 21 | # 22 | # AX_REQUIRE_DEFINED([AX_CHECK_LINK_FLAG]) 23 | # 24 | # LICENSE 25 | # 26 | # Copyright (c) 2014 Mike Frysinger 27 | # 28 | # Copying and distribution of this file, with or without modification, are 29 | # permitted in any medium without royalty provided the copyright notice 30 | # and this notice are preserved. This file is offered as-is, without any 31 | # warranty. 32 | 33 | #serial 1 34 | 35 | AC_DEFUN([AX_REQUIRE_DEFINED], [dnl 36 | m4_ifndef([$1], [m4_fatal([macro ]$1[ is not defined; is a m4 file missing?])]) 37 | ])dnl AX_REQUIRE_DEFINED 38 | -------------------------------------------------------------------------------- /src/java/README: -------------------------------------------------------------------------------- 1 | Background 2 | ========== 3 | 4 | QFS uses Java for the following needs: 5 | 6 | 1. To let Java applications use QFS. This is enabled by the QFS Access library. 7 | Java applications can use the APIs in qfs-access-.jar. 8 | 9 | 2. To let Apache Hadoop use QFS as a backing store. This is enabled by the 10 | Hadoop QFS plugin. Apache Hadoop users can simply drop in the 11 | hadoop--qfs-.jar to their Hadoop setup to use QFS. 12 | 13 | The following Apache Hadoop versions are supported: 14 | * Apache Hadoop branch1 (hadoop-1.0.X, hadoop-1.1.X) 15 | * Apache Hadoop trunk (hadoop-0.23.X, hadoop-2.X.X) 16 | 17 | 18 | Compiling 19 | ========= 20 | 21 | To compile the Hadoop QFS plugin for Apache Hadoop 'branch1', one requires the 22 | hadoop-core-.jar. To do the same for Apache Hadoop 'trunk', one requires 23 | hadoop-common-.jar. The javabuild.sh script is provided to handle these 24 | dependencies. 25 | 26 | The script uses Apache Maven to obtain the dependent JARs and compile the 27 | qfs-access and hadoop-qfs sub modules. In most cases, doing one of the 28 | following is sufficient: 29 | 30 | * Run './javabuild.sh' to build only the qfs-access. 31 | * Run './javabuild.sh ' (eg: './javabuild.sh 2.0.2-alpha') to 32 | build qfs-access as well as hadoop-qfs JARs. 33 | 34 | The build classes and JARs are created under /build/java. 35 | -------------------------------------------------------------------------------- /src/test-scripts/chunkchksum_check_expected.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # $Id$ 4 | # 5 | # Created 2010 6 | # Author: Mike Ovsiannikov 7 | # 8 | # Copyright 2010,2016 Quantcast Corporation. All rights reserved. 9 | # 10 | # This file is part of Kosmos File System (KFS). 11 | # 12 | # Licensed under the Apache License, Version 2.0 13 | # (the "License"); you may not use this file except in compliance with 14 | # the License. You may obtain a copy of the License at 15 | # 16 | # http://www.apache.org/licenses/LICENSE-2.0 17 | # 18 | # Unless required by applicable law or agreed to in writing, software 19 | # distributed under the License is distributed on an "AS IS" BASIS, 20 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | # implied. See the License for the specific language governing 22 | # permissions and limitations under the License. 23 | # 24 | # 25 | 26 | # /mnt/data*/qmr_kfs/kfs-sns/chunk/data/ 27 | 28 | if [ $# -lt 1 ]; then 29 | echo "Usage: $0 []" 30 | exit 1 31 | fi 32 | 33 | chunkDir=$1 34 | shift 35 | 36 | grep 'Checksum mismatch for chunk=' ${1+"$@"} | \ 37 | grep ChunkManager.cc | \ 38 | sed -e 's/^.* chunk=//' | \ 39 | tr ':=' ' ' | \ 40 | awk -v p="$chunkDir" \ 41 | '{ 42 | printf("od -w4 -t u4 -j %d -N %d %s/*.%s.* | grep -n %s\n", \ 43 | $3/65536*4+40, ($5+65535)/65536*4, p, $1, $7); 44 | }' | \ 45 | sh -x 46 | 47 | -------------------------------------------------------------------------------- /src/cc/libclient/utils.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2006/08/31 5 | // Author: Sriram Rao 6 | // 7 | // Copyright 2008-2012,2016 Quantcast Corporation. All rights reserved. 8 | // Copyright 2006-2008 Kosmix Corp. 9 | // 10 | // This file is part of Kosmos File System (KFS). 11 | // 12 | // Licensed under the Apache License, Version 2.0 13 | // (the "License"); you may not use this file except in compliance with 14 | // the License. You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | // implied. See the License for the specific language governing 22 | // permissions and limitations under the License. 23 | // 24 | // \brief utils.h: Miscellaneous utility functions. 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #ifndef LIBKFSCLIENT_UTILS_H 29 | #define LIBKFSCLIENT_UTILS_H 30 | 31 | #include 32 | 33 | namespace KFS { 34 | namespace client { 35 | 36 | void GetTimeval(const char* s, struct timeval& tv, bool hexFormatFlag); 37 | void Sleep(int secs); 38 | 39 | } 40 | } 41 | 42 | #endif // LIBKFSCLIENT_UTILS_H 43 | -------------------------------------------------------------------------------- /src/java/qfs-access/src/main/java/com/quantcast/qfs/access/Positionable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * $Id$ 3 | * 4 | * Created 2007/09/13 5 | * 6 | * @author: Sriram Rao (Kosmix Corp.) 7 | * 8 | * Copyright 2007 Kosmix Corp. 9 | * 10 | * This file is part of Kosmos File System (KFS). 11 | * 12 | * Licensed under the Apache License, Version 2.0 13 | * (the "License"); you may not use this file except in compliance with 14 | * the License. You may obtain a copy of the License at 15 | * 16 | * http://www.apache.org/licenses/LICENSE-2.0 17 | * 18 | * Unless required by applicable law or agreed to in writing, software 19 | * distributed under the License is distributed on an "AS IS" BASIS, 20 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | * implied. See the License for the specific language governing 22 | * permissions and limitations under the License. 23 | * 24 | * \brief Interface for positioning a file pointer. 25 | */ 26 | 27 | package com.quantcast.qfs.access; 28 | 29 | import java.io.IOException; 30 | 31 | public interface Positionable 32 | { 33 | /* 34 | * Position a file pointer at the specified offset from the 35 | * beginning of a file. 36 | */ 37 | long seek(long offset) throws IOException; 38 | 39 | /* 40 | * Return the current position of the file pointer---the position 41 | * is from the beginning of the file. 42 | */ 43 | long tell() throws IOException; 44 | } 45 | -------------------------------------------------------------------------------- /src/test-scripts/allocatesend.pl: -------------------------------------------------------------------------------- 1 | eval 'exec perl -wS $0 ${1+"$@"}' 2 | if 0; 3 | # 4 | # $Id$ 5 | # 6 | # Created 2010 7 | # Author: Mike Ovsiannikov 8 | # 9 | # Copyright 2010,2016 Quantcast Corporation. All rights reserved. 10 | # 11 | # This file is part of Kosmos File System (KFS). 12 | # 13 | # Licensed under the Apache License, Version 2.0 14 | # (the "License"); you may not use this file except in compliance with 15 | # the License. You may obtain a copy of the License at 16 | # 17 | # http://www.apache.org/licenses/LICENSE-2.0 18 | # 19 | # Unless required by applicable law or agreed to in writing, software 20 | # distributed under the License is distributed on an "AS IS" BASIS, 21 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 22 | # implied. See the License for the specific language governing 23 | # permissions and limitations under the License. 24 | # 25 | # 26 | 27 | my $numcmd = shift || 10; 28 | my $fid = shift || 8; # 68030 29 | 30 | my $seqf=999000000; 31 | my $seq=$seqf; 32 | my $end=$seq + $numcmd; 33 | for (; $seq < $end; $seq++) { 34 | print " 35 | ALLOCATE\r 36 | Cseq: $seq\r 37 | Version: KFS/1.0\r 38 | Client-Protocol-Version: 100\r 39 | Client-host: somehostname\r 40 | Pathname: /sort/job/1/fanout/27/file.27\r 41 | File-handle: $fid\r 42 | Chunk-offset: 0\r 43 | Chunk-append: 1\r 44 | Space-reserve: 0\r 45 | Max-appenders: 640000000\r 46 | \r 47 | "; 48 | } 49 | printf("==== test done ===\r\n\r\n"); 50 | -------------------------------------------------------------------------------- /ext/dns/regress/12-segfault-in-dns_res_frame_init.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | #include "dns.h" 8 | 9 | int main(void) { 10 | struct dns_resolv_conf *resconf = NULL; 11 | struct dns_hosts *hosts = NULL; 12 | struct dns_hints *hints = NULL; 13 | struct dns_resolver *res = NULL; 14 | int error, status = 1; 15 | 16 | if (!(resconf = dns_resconf_local(&error))) 17 | goto error; 18 | if (!(hosts = dns_hosts_local(&error))) 19 | goto error; 20 | if (!(hints = dns_hints_local(resconf, &error))) 21 | goto error; 22 | 23 | /* arrange for dns_so_init to fail */ 24 | struct rlimit nofile = { 0, 0 }; 25 | if (0 != setrlimit(RLIMIT_NOFILE, &nofile)) 26 | goto syerr; 27 | 28 | /* bug #12 caused dns_res_open to segfault when dns_so_init failed */ 29 | if ((res = dns_res_open(resconf, hosts, hints, NULL, dns_opts(), &error))) { 30 | warnx("expected dns_res_open to fail"); 31 | goto epilog; 32 | } else if (error != EMFILE) { 33 | warnx("expected dns_res_open to fail with EMFILE, got %d (%s)", error, dns_strerror(error)); 34 | goto epilog; 35 | } 36 | 37 | warnx("OK"); 38 | status = 0; 39 | 40 | goto epilog; 41 | syerr: 42 | error = errno; 43 | error: 44 | warnx("%s", dns_strerror(error)); 45 | 46 | goto epilog; 47 | epilog: 48 | dns_res_close(res); 49 | dns_hints_close(hints); 50 | dns_hosts_close(hosts); 51 | dns_resconf_close(resconf); 52 | 53 | return status; 54 | } 55 | -------------------------------------------------------------------------------- /travis/before_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # $Id$ 4 | # 5 | # Copyright 2016-2017 Quantcast Corporation. All rights reserved. 6 | # 7 | # This file is part of Quantcast File System. 8 | # 9 | # Licensed under the Apache License, Version 2.0 10 | # (the "License"); you may not use this file except in compliance with 11 | # the License. You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | # implied. See the License for the specific language governing 19 | # permissions and limitations under the License. 20 | 21 | ################################################################################ 22 | # The following is executed on .travis.yml's before_install section 23 | ################################################################################ 24 | 25 | set -ex 26 | 27 | if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then 28 | brew update || true 29 | brew install git || true 30 | brew install cmake || true 31 | brew install maven || true 32 | brew install boost || true 33 | brew cask install osxfuse || true 34 | fi 35 | 36 | # use docker to build on linux; pull the corresponding docker image 37 | if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then 38 | docker pull $DISTRO:$VER 39 | fi 40 | -------------------------------------------------------------------------------- /src/cc/libclient/kfsglob.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2012/08/18 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2012,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief Kfs glob() equivalent. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef LIBCLIENT_KFSGLOB_H 28 | #define LIBCLIENT_KFSGLOB_H 29 | 30 | #include 31 | 32 | namespace KFS 33 | { 34 | class KfsClient; 35 | 36 | int KfsGlob( 37 | KfsClient& inClient, 38 | const char* inGlobPtr, 39 | int inGlobFlags, 40 | int (*inErrorHandlerPtr)(const char* inErrPathPtr, int inError), 41 | glob_t* inResultPtr); 42 | 43 | } 44 | 45 | #endif /* LIBCLIENT_KFSGLOB_H */ 46 | -------------------------------------------------------------------------------- /src/cc/common/MemLock.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2011/10/11 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2011-2012,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \file MemLock.h 24 | // \brief Process common memory locking interface. 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #ifndef MEM_LOCK_H 29 | #define MEM_LOCK_H 30 | 31 | #include 32 | #include 33 | 34 | namespace KFS 35 | { 36 | using std::string; 37 | 38 | int 39 | LockProcessMemory( 40 | int64_t inMaxLockedMemorySize, 41 | int64_t inMaxHeapSize = 0, 42 | int64_t inMaxStlPoolSize = 0, 43 | string* outErrMsgPtr = 0); 44 | } 45 | 46 | #endif /* MEM_LOCK_H */ 47 | -------------------------------------------------------------------------------- /src/java/qfs-access-pre-9/src/main/java/com/quantcast/qfs/access/KfsOutputChannel.java: -------------------------------------------------------------------------------- 1 | /** 2 | * $Id$ 3 | * 4 | * Created 2007/09/11 5 | * 6 | * @author: Sriram Rao (Kosmix Corp.) 7 | * 8 | * Copyright 2008-2012,2016 Quantcast Corporation. All rights reserved. 9 | * Copyright 2007 Kosmix Corp. 10 | * 11 | * This file is part of Kosmos File System (KFS). 12 | * 13 | * Licensed under the Apache License, Version 2.0 14 | * (the "License"); you may not use this file except in compliance with 15 | * the License. You may obtain a copy of the License at 16 | * 17 | * http://www.apache.org/licenses/LICENSE-2.0 18 | * 19 | * Unless required by applicable law or agreed to in writing, software 20 | * distributed under the License is distributed on an "AS IS" BASIS, 21 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 22 | * implied. See the License for the specific language governing 23 | * permissions and limitations under the License. 24 | * 25 | * \brief An output channel pre java 9 style cleanup. 26 | */ 27 | package com.quantcast.qfs.access; 28 | 29 | final public class KfsOutputChannel extends KfsOutputChannelBase { 30 | 31 | KfsOutputChannel(KfsAccessBase kfsAccess, int fd, boolean append) { 32 | super(kfsAccess, fd, append); 33 | } 34 | 35 | @Override 36 | protected void finalize() throws Throwable { 37 | try { 38 | state.run(); 39 | } finally { 40 | super.finalize(); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/cc/common/hsieh_hash.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2008/07/17 5 | // Author: Sriram Rao 6 | // 7 | // Copyright 2008,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief String hash using Hsieh hash function. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef COMMON_HSIEH_HASH_H 28 | #define COMMON_HSIEH_HASH_H 29 | 30 | #include 31 | #include 32 | #include 33 | namespace KFS 34 | { 35 | std::size_t HsiehHash(const char * data, std::size_t len); 36 | struct Hsieh_hash_fcn { 37 | std::size_t operator()(const char *data, std::size_t len) const; 38 | std::size_t operator()(const std::string &data) const; 39 | }; 40 | } 41 | 42 | #endif // COMMON_HSIEH_HASH_H 43 | -------------------------------------------------------------------------------- /ext/gf-complete/examples/gf_example_1.c: -------------------------------------------------------------------------------- 1 | /* 2 | * GF-Complete: A Comprehensive Open Source Library for Galois Field Arithmetic 3 | * James S. Plank, Ethan L. Miller, Kevin M. Greenan, 4 | * Benjamin A. Arnold, John A. Burnum, Adam W. Disney, Allen C. McBride. 5 | * 6 | * gf_example_1.c 7 | * 8 | * Demonstrates using the procedures for examples in GF(2^w) for w <= 32. 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "gf_complete.h" 19 | #include "gf_rand.h" 20 | 21 | void usage(char *s) 22 | { 23 | fprintf(stderr, "usage: gf_example_1 w - w must be between 1 and 32\n"); 24 | exit(1); 25 | } 26 | 27 | int main(int argc, char **argv) 28 | { 29 | uint32_t a, b, c; 30 | int w; 31 | gf_t gf; 32 | 33 | if (argc != 2) usage(NULL); 34 | w = atoi(argv[1]); 35 | if (w <= 0 || w > 32) usage("Bad w"); 36 | 37 | /* Get two random numbers in a and b */ 38 | 39 | MOA_Seed(time(0)); 40 | a = MOA_Random_W(w, 0); 41 | b = MOA_Random_W(w, 0); 42 | 43 | /* Create the proper instance of the gf_t object using defaults: */ 44 | 45 | gf_init_easy(&gf, w); 46 | 47 | /* And multiply a and b using the galois field: */ 48 | 49 | c = gf.multiply.w32(&gf, a, b); 50 | printf("%u * %u = %u\n", a, b, c); 51 | 52 | /* Divide the product by a and b */ 53 | 54 | printf("%u / %u = %u\n", c, a, gf.divide.w32(&gf, c, a)); 55 | printf("%u / %u = %u\n", c, b, gf.divide.w32(&gf, c, b)); 56 | 57 | exit(0); 58 | } 59 | -------------------------------------------------------------------------------- /src/java/qfs-access-pre-9/src/main/java/com/quantcast/qfs/access/KfsInputChannel.java: -------------------------------------------------------------------------------- 1 | /** 2 | * $Id$ 3 | * 4 | * Created 2007/09/11 5 | * 6 | * @author: Sriram Rao (Kosmix Corp.) 7 | * 8 | * Copyright 2008-2012,2016 Quantcast Corporation. All rights reserved. 9 | * Copyright 2007 Kosmix Corp. 10 | * 11 | * This file is part of Kosmos File System (KFS). 12 | * 13 | * Licensed under the Apache License, Version 2.0 14 | * (the "License"); you may not use this file except in compliance with 15 | * the License. You may obtain a copy of the License at 16 | * 17 | * http://www.apache.org/licenses/LICENSE-2.0 18 | * 19 | * Unless required by applicable law or agreed to in writing, software 20 | * distributed under the License is distributed on an "AS IS" BASIS, 21 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 22 | * implied. See the License for the specific language governing 23 | * permissions and limitations under the License. 24 | * 25 | * \brief An input channel that does buffered I/O. This is to reduce 26 | * the overhead of JNI calls. 27 | */ 28 | package com.quantcast.qfs.access; 29 | 30 | /* A byte channel interface with seek support */ 31 | final public class KfsInputChannel extends KfsInputChannelBase { 32 | 33 | KfsInputChannel(KfsAccessBase ka, int fd) { 34 | super(ka, fd); 35 | } 36 | 37 | @Override 38 | protected void finalize() throws Throwable { 39 | try { 40 | state.release(); 41 | } finally { 42 | super.finalize(); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/cc/access/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # $Id$ 3 | # 4 | # Created 2006 5 | # Author: Sriram Rao (Kosmix Corp) 6 | # 7 | # Copyright 2008-2012,2016 Quantcast Corporation. All rights reserved. 8 | # Copyright 2006 Kosmix Corp. 9 | # 10 | # This file is part of Kosmos File System (KFS). 11 | # 12 | # Licensed under the Apache License, Version 2.0 13 | # (the "License"); you may not use this file except in compliance with 14 | # the License. You may obtain a copy of the License at 15 | # 16 | # http://www.apache.org/licenses/LICENSE-2.0 17 | # 18 | # Unless required by applicable law or agreed to in writing, software 19 | # distributed under the License is distributed on an "AS IS" BASIS, 20 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | # implied. See the License for the specific language governing 22 | # permissions and limitations under the License. 23 | # 24 | # 25 | 26 | # Take all the .cc files and build a library out of them 27 | add_library (qfs_access SHARED qfs_access_jni.cc) 28 | add_dependencies (qfs_access kfsClient-shared) 29 | target_link_libraries (qfs_access kfsClient-shared) 30 | 31 | IF (CMAKE_SYSTEM_NAME STREQUAL "SunOS") 32 | # On Solaris, cmake uses gcc to link rather than g++. This 33 | # prevents linking with libstdc++ which prevents Java from loading 34 | # qfs_access. To fix, set the compiler to be C++ 35 | set (CMAKE_C_COMPILER "${CMAKE_CXX_COMPILER}") 36 | ENDIF (CMAKE_SYSTEM_NAME STREQUAL "SunOS") 37 | 38 | # 39 | install (TARGETS qfs_access 40 | LIBRARY DESTINATION lib 41 | ARCHIVE DESTINATION lib-static) 42 | -------------------------------------------------------------------------------- /ext/jerasure/configure.ac: -------------------------------------------------------------------------------- 1 | # Jerasure autoconf template 2 | 3 | AC_PREREQ([2.71]) 4 | AC_INIT([Jerasure],[2.0],[],[],[https://bitbucket.org/jimplank/jerasure]) 5 | AC_CONFIG_SRCDIR([src/jerasure.c]) 6 | AC_CONFIG_HEADERS([include/config.h]) 7 | 8 | AC_CONFIG_AUX_DIR([build-aux]) 9 | AC_CONFIG_MACRO_DIR([m4]) 10 | 11 | # This prevents './configure; make' from trying to run autotools. 12 | AM_MAINTAINER_MODE([disable]) 13 | 14 | AM_INIT_AUTOMAKE([1.13 -Wall -Wno-extra-portability]) 15 | 16 | # Package default C compiler flags. 17 | dnl This must be before LT_INIT and AC_PROG_CC. 18 | : ${CFLAGS='-g -O3 -Wall'} 19 | 20 | LT_INIT([disable-static]) 21 | 22 | # Checks for programs. 23 | AC_PROG_CC 24 | 25 | # Checks for libraries. 26 | AC_CHECK_LIB([gf_complete], [gf_init_easy], [], 27 | [AC_MSG_FAILURE( 28 | [You need to have gf_complete installed. 29 | gf_complete is available from http://web.eecs.utk.edu/~plank/plank/papers/CS-13-703.html]) 30 | ]) 31 | 32 | # Checks for header files. 33 | AC_CHECK_HEADERS([stddef.h stdint.h stdlib.h string.h sys/time.h unistd.h]) 34 | AC_CHECK_HEADERS([gf_complete.h gf_general.h gf_method.h gf_rand.h]) 35 | 36 | # Checks for typedefs, structures, and compiler characteristics. 37 | AC_TYPE_UINT32_T 38 | AC_TYPE_UINT64_T 39 | AX_EXT 40 | 41 | # Checks for library functions. 42 | AC_FUNC_MALLOC 43 | AC_CHECK_FUNCS([bzero getcwd gettimeofday mkdir strchr strdup strrchr]) 44 | 45 | AC_CONFIG_FILES([Examples/Makefile 46 | Makefile 47 | src/Makefile]) 48 | AC_OUTPUT 49 | -------------------------------------------------------------------------------- /src/cc/common/Version.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: Version.h 192 2008-10-22 05:33:26Z sriramsrao $ 3 | // 4 | // \brief Header file for getting KFS version #'s related to builds. 5 | // 6 | // Created 2008/10/20 7 | // Author: Sriram Rao 8 | // 9 | // Copyright 2008-2010,2016 Quantcast Corporation. All rights reserved. 10 | // 11 | // This file is part of Kosmos File System (KFS). 12 | // 13 | // Licensed under the Apache License, Version 2.0 14 | // (the "License"); you may not use this file except in compliance with 15 | // the License. You may obtain a copy of the License at 16 | // 17 | // http://www.apache.org/licenses/LICENSE-2.0 18 | // 19 | // Unless required by applicable law or agreed to in writing, software 20 | // distributed under the License is distributed on an "AS IS" BASIS, 21 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 22 | // implied. See the License for the specific language governing 23 | // permissions and limitations under the License. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef COMMON_KFSVERSION_H 28 | #define COMMON_KFSVERSION_H 29 | 30 | #include 31 | 32 | // Store build version informantion in executables. 33 | // Build info can be retrieved with strings | awk 34 | // See buildversgit.sh 35 | namespace KFS { 36 | extern const std::string KFS_BUILD_VERSION_STRING; 37 | extern const std::string KFS_SOURCE_REVISION_STRING; 38 | extern const std::string KFS_BUILD_INFO_STRING; 39 | } 40 | 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /src/cc/libclient/ECMethodDef.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2014/08/18 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2014,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // Erasure code method definitions. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef KFS_LIBCLIENT_ECMETHOD_DEF_H 28 | #define KFS_LIBCLIENT_ECMETHOD_DEF_H 29 | 30 | #include "ECMethod.h" 31 | 32 | #define KFS_MAKE_REGISTERED_EC_METHOD_NAME(inType) \ 33 | ECMethod_KFS_EC_METHOD_##inType 34 | #define KFS_DECLARE_EC_METHOD_PTR(inType) \ 35 | ECMethod* const KFS_MAKE_REGISTERED_EC_METHOD_NAME(inType) 36 | #define KFS_REGISTER_EC_METHOD(inType, inMethodPtr) \ 37 | extern KFS_DECLARE_EC_METHOD_PTR(inType); \ 38 | KFS_DECLARE_EC_METHOD_PTR(inType) = inMethodPtr 39 | 40 | #endif /* KFS_LIBCLIENT_ECMETHOD_DEF_H */ 41 | -------------------------------------------------------------------------------- /ext/dns/regress/14-dns_resconf_search-fqdn.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | 7 | #include "dns.h" 8 | 9 | #define countof(a) (sizeof (a) / sizeof *(a)) 10 | 11 | static void 12 | search_add(struct dns_resolv_conf *resconf, const char *dn) 13 | { 14 | size_t i; 15 | 16 | for (i = 0; i < countof(resconf->search); i++) { 17 | if (*resconf->search[i]) 18 | continue; 19 | dns_strlcpy(resconf->search[i], dn, sizeof resconf->search[i]); 20 | break; 21 | } 22 | } 23 | 24 | static void 25 | search_check(struct dns_resolv_conf *resconf, const char *host, const char **expect) 26 | { 27 | dns_resconf_i_t search = 0; 28 | char dn[DNS_D_MAXNAME + 1]; 29 | 30 | while (dns_resconf_search(dn, sizeof dn, host, strlen(host), resconf, &search)) { 31 | if (*expect) { 32 | if (strcmp(dn, *expect)) 33 | errx(1, "expected domain %s, but got %s", *expect, dn); 34 | expect++; 35 | } else { 36 | errx(1, "unexpected domain (%s)", dn); 37 | } 38 | } 39 | } 40 | 41 | int 42 | main(void) 43 | { 44 | struct dns_resolv_conf resconf = { 0 }; 45 | 46 | search_add(&resconf, "foo.local"); 47 | search_add(&resconf, "bar.local"); 48 | 49 | resconf.options.ndots = 1; 50 | search_check(&resconf, "host", (const char *[]){ "host.foo.local.", "host.bar.local.", "host.", NULL }); 51 | 52 | resconf.options.ndots = 0; 53 | search_check(&resconf, "host", (const char *[]){ "host.", "host.foo.local.", "host.bar.local.", NULL }); 54 | 55 | search_check(&resconf, "host.", (const char *[]){ "host.", NULL }); 56 | 57 | warnx("OK"); 58 | 59 | return 0; 60 | } 61 | -------------------------------------------------------------------------------- /src/cc/kfsio/blockname.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2016/8/9 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2014,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // 24 | // Chunk / object store block's file name / key generation delcartion. 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #ifndef KFSIO_BLOCK_NAME_H 29 | #define KFSIO_BLOCK_NAME_H 30 | 31 | #include "common/kfstypes.h" 32 | 33 | #include 34 | 35 | namespace KFS 36 | { 37 | using std::string; 38 | 39 | bool 40 | AppendChunkFileNameOrObjectStoreBlockKey( 41 | string& inName, 42 | int64_t inFileSystemId, 43 | kfsFileId_t inFileId, 44 | kfsChunkId_t inId, 45 | kfsSeq_t inVersion, 46 | string& ioFileSystemIdSuffix); 47 | 48 | } // namespace KFS 49 | 50 | #endif /* KFSIO_BLOCK_NAME_H */ 51 | -------------------------------------------------------------------------------- /src/cc/tools/kfspwd.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2007/09/20 5 | // Author: Sriram Rao 6 | // 7 | // Copyright 2008,2016 Quantcast Corporation. All rights reserved. 8 | // Copyright 2007-2008 Kosmix Corp. 9 | // 10 | // This file is part of Kosmos File System (KFS). 11 | // 12 | // Licensed under the Apache License, Version 2.0 13 | // (the "License"); you may not use this file except in compliance with 14 | // the License. You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | // implied. See the License for the specific language governing 22 | // permissions and limitations under the License. 23 | // 24 | // \brief Tool that prints the cwd. 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #include "kfsshell.h" 29 | #include "libclient/KfsClient.h" 30 | 31 | #include 32 | 33 | namespace KFS { 34 | namespace tools { 35 | 36 | using std::cout; 37 | using std::vector; 38 | using std::string; 39 | 40 | int 41 | handlePwd(KfsClient *kfsClient, const vector &args) 42 | { 43 | if ((args.size() >= 1) && (args[0] == "--help")) { 44 | cout << "Usage: pwd " << "\n"; 45 | return 0; 46 | } 47 | 48 | cout << kfsClient->GetCwd() << "\n"; 49 | return 0; 50 | } 51 | 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/cc/tools/kfsrmdir.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2006/09/21 5 | // Author: Sriram Rao 6 | // 7 | // Copyright 2008,2016 Quantcast Corporation. All rights reserved. 8 | // Copyright 2006-2008 Kosmix Corp. 9 | // 10 | // This file is part of Kosmos File System (KFS). 11 | // 12 | // Licensed under the Apache License, Version 2.0 13 | // (the "License"); you may not use this file except in compliance with 14 | // the License. You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | // implied. See the License for the specific language governing 22 | // permissions and limitations under the License. 23 | // 24 | // \brief Tool that implements rmdir 25 | //---------------------------------------------------------------------------- 26 | 27 | #include "kfsshell.h" 28 | #include "libclient/KfsClient.h" 29 | 30 | #include 31 | 32 | namespace KFS { 33 | namespace tools { 34 | 35 | using std::cout; 36 | using std::vector; 37 | using std::string; 38 | 39 | int 40 | handleRmdir(KfsClient *client, const vector &args) 41 | { 42 | if ((args.size() < 1) || (args[0] == "--help") || (args[0] == "")) { 43 | cout << "Usage: rmdir " << "\n"; 44 | return 0; 45 | } 46 | 47 | return doRmdir(client, args[0].c_str()); 48 | } 49 | 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/cc/qcrs/rs.h: -------------------------------------------------------------------------------- 1 | /*---------------------------------------------------------- -*- Mode: C -*----- 2 | * $Id$ 3 | * 4 | * Created 2010/07/24 5 | * Author: Dan Adkins 6 | * 7 | * Copyright 2010-2011,2016 Quantcast Corporation. All rights reserved. 8 | * 9 | * This file is part of Kosmos File System (KFS). 10 | * 11 | * Licensed under the Apache License, Version 2.0 12 | * (the "License"); you may not use this file except in compliance with 13 | * the License. You may obtain a copy of the License at 14 | * 15 | * http://www.apache.org/licenses/LICENSE-2.0 16 | * 17 | * Unless required by applicable law or agreed to in writing, software 18 | * distributed under the License is distributed on an "AS IS" BASIS, 19 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | * implied. See the License for the specific language governing 21 | * permissions and limitations under the License. 22 | * 23 | * \file rs.h 24 | * \brief Reed Solomon encoder and decoder public interface. 25 | * 26 | *------------------------------------------------------------------------------ 27 | */ 28 | 29 | #ifndef RS_H 30 | #define RS_H 31 | 32 | #ifdef __cplusplus 33 | extern "C" { 34 | #endif 35 | 36 | #define RS_LIB_MAX_DATA_BLOCKS 64 37 | #define RS_LIB_MAX_RECOVERY_BLOCKS 3 38 | 39 | void rs_encode(int nblocks, int blocksize, void **data); 40 | void rs_decode1(int nblocks, int blocksize, int x, void **data); 41 | void rs_decode2(int nblocks, int blocksize, int x, int y, void **data); 42 | void rs_decode3(int nblocks, int blocksize, int x, int y, int z, void **data); 43 | 44 | #ifdef __cplusplus 45 | } 46 | #endif 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /src/cc/tools/kfsmkdirs.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2006/09/21 5 | // Author: Sriram Rao 6 | // 7 | // Copyright 2008,2016 Quantcast Corporation. All rights reserved. 8 | // Copyright 2006-2008 Kosmix Corp. 9 | // 10 | // This file is part of Kosmos File System (KFS). 11 | // 12 | // Licensed under the Apache License, Version 2.0 13 | // (the "License"); you may not use this file except in compliance with 14 | // the License. You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | // implied. See the License for the specific language governing 22 | // permissions and limitations under the License. 23 | // 24 | // \brief Tool that implements mkdir -p 25 | //---------------------------------------------------------------------------- 26 | 27 | #include "kfsshell.h" 28 | #include "libclient/KfsClient.h" 29 | 30 | #include 31 | 32 | namespace KFS { 33 | namespace tools { 34 | 35 | using std::cout; 36 | using std::vector; 37 | using std::string; 38 | 39 | int 40 | handleMkdirs(KfsClient* client, const vector& args) 41 | { 42 | if ((args.size() < 1) || (args[0] == "--help") || (args[0] == "")) { 43 | cout << "Usage: mkdir " << "\n"; 44 | return 0; 45 | } 46 | 47 | return doMkdirs(client, args[0].c_str()); 48 | } 49 | 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/cc/kfsio/NetForwarder.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2015/04/18 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2015 Quantcast Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // Network byte stream forwader / tcp proxy. 24 | // 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | namespace KFS 29 | { 30 | 31 | class NetManager; 32 | class Properties; 33 | struct ServerLocation; 34 | 35 | class NetForwarder 36 | { 37 | public: 38 | NetForwarder( 39 | NetManager& inNetManager); 40 | ~NetForwarder(); 41 | int Start( 42 | const char* inParametersPrefixPtr, 43 | const Properties& inParameters); 44 | void SetParameters( 45 | const char* inParametersPrefixPtr, 46 | const Properties& inParameters); 47 | void Shutdown(); 48 | private: 49 | class Impl; 50 | Impl& mImpl; 51 | }; 52 | 53 | } // namespace KFS 54 | -------------------------------------------------------------------------------- /src/cc/meta/AuditLog.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2012/04/10 5 | // Author: Mike Ovsiannikov. 6 | // 7 | // Copyright 2012,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // 24 | // \file AuditLog.h 25 | // \brief Kfs meta server audit log interface. Writes every client request into 26 | // audit log file. 27 | // 28 | //---------------------------------------------------------------------------- 29 | 30 | #ifndef AUDIT_LOG_H 31 | #define AUDIT_LOG_H 32 | 33 | namespace KFS 34 | { 35 | 36 | struct MetaRequest; 37 | class Properties; 38 | 39 | class AuditLog 40 | { 41 | public: 42 | static void Log(const MetaRequest& inOp); 43 | static void SetParameters(const Properties& inProps); 44 | static void Stop(); 45 | static void PrepareToFork(); 46 | static void ForkDone(); 47 | static void ChildAtFork(); 48 | static bool Init(); 49 | }; 50 | 51 | }; 52 | 53 | #endif /* AUDIT_LOG_H */ 54 | -------------------------------------------------------------------------------- /src/cc/kfsio/KfsCallbackObj.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2013/04/19 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2013 Quantcast Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // 24 | //---------------------------------------------------------------------------- 25 | 26 | #include "KfsCallbackObj.h" 27 | 28 | #include 29 | 30 | namespace KFS 31 | { 32 | 33 | inline static const ObjectMethodBase* 34 | MakeNullObjMethod() 35 | { 36 | static const ObjectMethod sObjMethod(0, 0); 37 | return &sObjMethod; 38 | } 39 | 40 | const ObjectMethodBase* const kNullObjMethod = MakeNullObjMethod(); 41 | 42 | /* virtual */ 43 | KfsCallbackObj::~KfsCallbackObj() 44 | { 45 | if (mObjMeth == kNullObjMethod) { 46 | abort(); // Catch double delete. 47 | return; 48 | } 49 | if (mObjMeth) { 50 | mObjMeth->~ObjectMethodBase(); 51 | } 52 | mObjMeth = kNullObjMethod; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/java/qfs-access-pre-9/src/main/java/com/quantcast/qfs/access/KfsAccess.java: -------------------------------------------------------------------------------- 1 | /** 2 | * $Id$ 3 | * 4 | * Created 2025/04/20 5 | * 6 | * @author: Mike Ovsiannikov (Quantcast Corporation) 7 | * 8 | * Copyright 2025 Quantcast Corporation. All rights reserved. Copyright 2007 9 | * Kosmix Corp. 10 | * 11 | * This file is part of Kosmos File System (KFS). 12 | * 13 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 14 | * use this file except in compliance with the License. You may obtain a copy of 15 | * the License at 16 | * 17 | * http://www.apache.org/licenses/LICENSE-2.0 18 | * 19 | * Unless required by applicable law or agreed to in writing, software 20 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 21 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 22 | * License for the specific language governing permissions and limitations under 23 | * the License. 24 | * 25 | * \brief Java wrappers to get to the KFS client. 26 | */ 27 | package com.quantcast.qfs.access; 28 | 29 | import java.io.IOException; 30 | 31 | final public class KfsAccess extends KfsAccessBase { 32 | 33 | public KfsAccess(String configFn) throws IOException { 34 | super(configFn); 35 | } 36 | 37 | public KfsAccess(String metaServerHost, 38 | int metaServerPort) throws IOException { 39 | super(metaServerHost, metaServerPort); 40 | } 41 | 42 | @Override 43 | protected void finalize() throws Throwable { 44 | try { 45 | kfs_destroy(); 46 | } finally { 47 | super.finalize(); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/java/qfs-access/src/test/java/com/quantcast/qfs/access/TestQfsAccess.java: -------------------------------------------------------------------------------- 1 | /** 2 | * $Id$ 3 | * 4 | * 5 | * Copyright 2012-2017 Quantcast Corporation. All rights reserved. 6 | * 7 | * This file is part of Quantcast File System (QFS). 8 | * 9 | * Licensed under the Apache License, Version 2.0 10 | * (the "License"); you may not use this file except in compliance with 11 | * the License. You may obtain a copy of the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, software 16 | * distributed under the License is distributed on an "AS IS" BASIS, 17 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | * implied. See the License for the specific language governing 19 | * permissions and limitations under the License. 20 | * 21 | */ 22 | 23 | package com.quantcast.qfs.access; 24 | 25 | import junit.framework.Test; 26 | import junit.framework.TestCase; 27 | import junit.framework.TestSuite; 28 | 29 | /** 30 | * Unit test for simple App. 31 | */ 32 | public class TestQfsAccess extends TestCase 33 | { 34 | /** 35 | * Create the test case 36 | * 37 | * @param testName name of the test case 38 | */ 39 | public TestQfsAccess( String testName ) 40 | { 41 | super( testName ); 42 | } 43 | 44 | /** 45 | * @return the suite of tests being tested 46 | */ 47 | public static Test suite() 48 | { 49 | return new TestSuite( TestQfsAccess.class ); 50 | } 51 | 52 | /** 53 | * Rigourous Test :-) 54 | */ 55 | public void testApp() 56 | { 57 | assertTrue( true ); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /ext/gf-complete/COPYING: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013, James S. Plank, Ethan L. Miller, Kevin M. Greenan, 2 | Benjamin A. Arnold, John A. Burnum, Adam W. Disney, Allen C. McBride 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions 7 | are met: 8 | 9 | - Redistributions of source code must retain the above copyright 10 | notice, this list of conditions and the following disclaimer. 11 | 12 | - Redistributions in binary form must reproduce the above copyright 13 | notice, this list of conditions and the following disclaimer in 14 | the documentation and/or other materials provided with the 15 | distribution. 16 | 17 | - Neither the name of the University of Tennessee nor the names of its 18 | contributors may be used to endorse or promote products derived 19 | from this software without specific prior written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 28 | OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 | AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY 31 | WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 | POSSIBILITY OF SUCH DAMAGE. 33 | -------------------------------------------------------------------------------- /ext/gf-complete/License.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013, James S. Plank, Ethan L. Miller, Kevin M. Greenan, 2 | Benjamin A. Arnold, John A. Burnum, Adam W. Disney, Allen C. McBride 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions 7 | are met: 8 | 9 | - Redistributions of source code must retain the above copyright 10 | notice, this list of conditions and the following disclaimer. 11 | 12 | - Redistributions in binary form must reproduce the above copyright 13 | notice, this list of conditions and the following disclaimer in 14 | the documentation and/or other materials provided with the 15 | distribution. 16 | 17 | - Neither the name of the University of Tennessee nor the names of its 18 | contributors may be used to endorse or promote products derived 19 | from this software without specific prior written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 28 | OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 | AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY 31 | WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 | POSSIBILITY OF SUCH DAMAGE. 33 | -------------------------------------------------------------------------------- /src/java/qfs-access/src/main/java/com/quantcast/qfs/access/KfsInputChannel.java: -------------------------------------------------------------------------------- 1 | /** 2 | * $Id$ 3 | * 4 | * Created 2025/04/20 5 | * 6 | * @author: Mike Ovsiannikov (Quantcast Corporation) 7 | * 8 | * Copyright 2025 Quantcast Corporation. All rights reserved. 9 | * Copyright 2007 Kosmix Corp. 10 | * 11 | * This file is part of Kosmos File System (KFS). 12 | * 13 | * Licensed under the Apache License, Version 2.0 14 | * (the "License"); you may not use this file except in compliance with 15 | * the License. You may obtain a copy of the License at 16 | * 17 | * http://www.apache.org/licenses/LICENSE-2.0 18 | * 19 | * Unless required by applicable law or agreed to in writing, software 20 | * distributed under the License is distributed on an "AS IS" BASIS, 21 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 22 | * implied. See the License for the specific language governing 23 | * permissions and limitations under the License. 24 | * 25 | * \brief Input channel java 9 style cleanup. 26 | */ 27 | package com.quantcast.qfs.access; 28 | 29 | import java.io.IOException; 30 | import java.lang.ref.Cleaner; 31 | 32 | final public class KfsInputChannel extends KfsInputChannelBase { 33 | 34 | final private Cleaner.Cleanable cleanable; 35 | 36 | KfsInputChannel(KfsAccessBase ka, int fd) { 37 | super(ka, fd); 38 | cleanable = registerCleanup(); 39 | } 40 | 41 | private Cleaner.Cleanable registerCleanup() { 42 | return KfsAccess.registerCleanup(this, state); 43 | } 44 | 45 | @Override 46 | public void close() throws IOException { 47 | try { 48 | super.close(); 49 | } finally { 50 | cleanable.clean(); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/cc/kfsio/FileHandle.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2008/05/13 5 | // 6 | // Author: Sriram Rao 7 | // 8 | // Copyright 2008,2016 Quantcast Corporation. All rights reserved. 9 | // 10 | // This file is part of Kosmos File System (KFS). 11 | // 12 | // Licensed under the Apache License, Version 2.0 13 | // (the "License"); you may not use this file except in compliance with 14 | // the License. You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | // implied. See the License for the specific language governing 22 | // permissions and limitations under the License. 23 | // 24 | // \brief A ref-counted file-id object. 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #ifndef _LIBKFSIO_FILEHANDLE_H 29 | #define _LIBKFSIO_FILEHANDLE_H 30 | 31 | #include 32 | 33 | namespace KFS 34 | { 35 | struct FileHandle_t 36 | { 37 | FileHandle_t() : mFd(-1) { } 38 | FileHandle_t(int fd) : mFd(fd) { } 39 | ~FileHandle_t() { 40 | if (mFd < 0) 41 | return; 42 | close(mFd); 43 | mFd = -1; 44 | } 45 | void Close() { 46 | if (mFd < 0) 47 | return; 48 | close(mFd); 49 | mFd = -1; 50 | } 51 | int mFd; // the underlying file pointer 52 | }; 53 | 54 | typedef boost::shared_ptr FileHandlePtr; 55 | } 56 | 57 | #endif // _LIBKFSIO_FILEHANDLE_H 58 | -------------------------------------------------------------------------------- /src/cc/tools/kfscd.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2007/09/20 5 | // Author: Sriram Rao 6 | // 7 | // Copyright 2008,2016 Quantcast Corporation. All rights reserved. 8 | // Copyright 2007-2008 Kosmix Corp. 9 | // 10 | // This file is part of Kosmos File System (KFS). 11 | // 12 | // Licensed under the Apache License, Version 2.0 13 | // (the "License"); you may not use this file except in compliance with 14 | // the License. You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | // implied. See the License for the specific language governing 22 | // permissions and limitations under the License. 23 | // 24 | // \brief Tool that changes current working directory. 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #include "kfsshell.h" 29 | #include "libclient/KfsClient.h" 30 | 31 | #include 32 | 33 | namespace KFS { 34 | namespace tools { 35 | 36 | using std::cout; 37 | using std::vector; 38 | using std::string; 39 | 40 | int 41 | handleCd(KfsClient *kfsClient, const vector &args) 42 | { 43 | if (args.size() != 1 || args[0] == "--help") { 44 | cout << "Usage: cd " << "\n"; 45 | return -EINVAL; 46 | } 47 | const int res = kfsClient->Cd(args[0].c_str()); 48 | if (res < 0) { 49 | cout << "cd failed: " << ErrorCodeToStr(res) << "\n"; 50 | } 51 | return res; 52 | } 53 | 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/java/qfs-access/src/main/java/com/quantcast/qfs/access/KfsOutputChannel.java: -------------------------------------------------------------------------------- 1 | /** 2 | * $Id$ 3 | * 4 | * Created 2025/04/20 5 | * 6 | * @author: Mike Ovsiannikov (Quantcast Corporation) 7 | * 8 | * Copyright 2025 Quantcast Corporation. All rights reserved. 9 | * Copyright 2007 Kosmix Corp. 10 | * 11 | * This file is part of Kosmos File System (KFS). 12 | * 13 | * Licensed under the Apache License, Version 2.0 14 | * (the "License"); you may not use this file except in compliance with 15 | * the License. You may obtain a copy of the License at 16 | * 17 | * http://www.apache.org/licenses/LICENSE-2.0 18 | * 19 | * Unless required by applicable law or agreed to in writing, software 20 | * distributed under the License is distributed on an "AS IS" BASIS, 21 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 22 | * implied. See the License for the specific language governing 23 | * permissions and limitations under the License. 24 | * 25 | * \brief Input channel java 9 style cleanup. 26 | */ 27 | package com.quantcast.qfs.access; 28 | 29 | import java.io.IOException; 30 | import java.lang.ref.Cleaner; 31 | 32 | final public class KfsOutputChannel extends KfsOutputChannelBase { 33 | 34 | final private Cleaner.Cleanable cleanable; 35 | 36 | KfsOutputChannel(KfsAccessBase ka, int fd, boolean append) { 37 | super(ka, fd, append); 38 | cleanable = registerCleanup(); 39 | } 40 | 41 | private Cleaner.Cleanable registerCleanup() { 42 | return KfsAccess.registerCleanup(this, state); 43 | } 44 | 45 | @Override 46 | public void close() throws IOException { 47 | try { 48 | super.close(); 49 | } finally { 50 | cleanable.clean(); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/cc/s3io/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # $Id$ 3 | # 4 | # Created 2015/09/09 5 | # Author: Mike Ovsiannikov 6 | # 7 | # Copyright 2015,2016 Quantcast Corporation. All rights reserved. 8 | # 9 | # This file is part of Kosmos File System (KFS). 10 | # 11 | # Licensed under the Apache License, Version 2.0 12 | # (the "License"); you may not use this file except in compliance with 13 | # the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, software 18 | # distributed under the License is distributed on an "AS IS" BASIS, 19 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | # implied. See the License for the specific language governing 21 | # permissions and limitations under the License. 22 | # 23 | # 24 | 25 | set (sources 26 | s3ion.cc 27 | ) 28 | 29 | # 30 | # Build a static and a dynamically linked libraries. Both libraries 31 | # should have the same root name, but installed in different places 32 | # 33 | add_library (qfss3io STATIC ${sources}) 34 | add_library (qfss3io-shared SHARED ${sources}) 35 | set_target_properties (qfss3io-shared PROPERTIES OUTPUT_NAME "qfss3io") 36 | set_target_properties (qfss3io PROPERTIES OUTPUT_NAME "qfss3io") 37 | 38 | # 39 | # Since the objects have to be built twice, set this up so they don't 40 | # clobber each other. 41 | set_target_properties (qfss3io PROPERTIES CLEAN_DIRECT_OUTPUT 1) 42 | set_target_properties (qfss3io-shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) 43 | 44 | target_link_libraries (qfss3io 45 | kfsIO 46 | ) 47 | target_link_libraries (qfss3io-shared 48 | kfsIO-shared 49 | ) 50 | 51 | install (TARGETS qfss3io qfss3io-shared 52 | LIBRARY DESTINATION lib 53 | ARCHIVE DESTINATION lib/static) 54 | -------------------------------------------------------------------------------- /src/cc/emulator/emulator_setup.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2008/08/29 5 | // 6 | // Author: Sriram Rao 7 | // 8 | // Copyright 2008-2012,2016 Quantcast Corporation. All rights reserved. 9 | // 10 | // This file is part of Kosmos File System (KFS). 11 | // 12 | // Licensed under the Apache License, Version 2.0 13 | // (the "License"); you may not use this file except in compliance with 14 | // the License. You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | // implied. See the License for the specific language governing 22 | // permissions and limitations under the License. 23 | // 24 | // \brief Setup code to get an emulator up. 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #ifndef EMULATOR_EMULATORSETUP_H 29 | #define EMULATOR_EMULATORSETUP_H 30 | 31 | #include 32 | #include 33 | 34 | namespace KFS 35 | { 36 | using std::string; 37 | using std::ostream; 38 | 39 | class LayoutEmulator; 40 | // pass an optional argument that enables changing the degree of replication for a file. 41 | int EmulatorSetup( 42 | LayoutEmulator& emulator, 43 | string& logdir, 44 | string& cpdir, 45 | string& networkFn, 46 | string& chunkmapFn, 47 | int16_t minReplicasPerFile, 48 | bool addChunksToReplicationChecker, 49 | int64_t chunkServerTotalSpace); 50 | } 51 | 52 | #endif // EMULATOR_EMULATORSETUP_H 53 | -------------------------------------------------------------------------------- /benchmarks/mstress/mstress_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # $Id$ 4 | # 5 | # Copyright 2012,2016 Quantcast Corporation. All rights reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 8 | # use this file except in compliance with the License. You may obtain a copy 9 | # of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 16 | # License for the specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | # 20 | # To run mstress, the participating client hosts and the master host should all 21 | # have the mstress files in the same path. 22 | # 23 | # This script, run with a comma-separated list of hostnames, will copy the 24 | # tar + gz bundle of the mstress directory to the home directory of the hosts 25 | # and untar + unzip them for usage. 26 | # 27 | 28 | TAR=${TAR:-"tar"} 29 | 30 | if [ -z "$BOOTSTRAP" ] 31 | then 32 | tarfile="mstress.tgz" 33 | target="mstress-tarball" 34 | else 35 | tarfile="mstress-bootstrap.tgz" 36 | target="mstress-bootstrap" 37 | fi 38 | 39 | if [ $# -lt 1 ] 40 | then 41 | echo "Usage: $0 " 42 | echo " This copies the mstress bundle to master and client hosts." 43 | exit 44 | fi 45 | 46 | if [ ! -f "$tarfile" ] 47 | then 48 | [ -d build ] || mkdir build 49 | (cd build && cmake ../../.. && make "$target" && cp "benchmarks/$tarfile" ..) || exit 1 50 | fi 51 | 52 | while [ $# -ne 0 ] 53 | do 54 | echo "Deploying mstress tarball to $1" 55 | ssh $1 "$TAR xzv || echo >&2 failed to untar on '$1'" < "$tarfile" 56 | shift 57 | done 58 | -------------------------------------------------------------------------------- /src/go/README.md: -------------------------------------------------------------------------------- 1 | # Go QFS Bindings 2 | 3 | These are Go bindings to interact with [Quantcast File System](https://github.com/quantcast/qfs). 4 | It includes many of the common functions exported by the QFS API. 5 | 6 | ## Build and Test 7 | 8 | Several incantations of the compiler flags and the linker are required to get 9 | qfs to compile against the c library. We may be to remove this necessity with 10 | static linking but time is short. 11 | 12 | Some incantation such as this should work: 13 | 14 | Should be the path to your qfs build or production qfs. 15 | 16 | Here are the environment variables I've used to compile these (Mac OS X specific): 17 | 18 | export QFS_RELEASE=/Users/sday/c/qfs/build/release 19 | export DYLD_LIBRARY_PATH=$QFS_RELEASE/lib/ 20 | export CGO_CFLAGS=-I$QFS_RELEASE/include/ 21 | export CGO_LDFLAGS=-L$QFS_RELEASE/lib 22 | 23 | Note that these are Mac OSX specific, but are nearly identical for a linux 24 | system. Just changing DYLD_LIBRARY_PATH to LD_LIBRARY_PATH should work. 25 | DYLD_LIBRARY_PATH/LD_LIBRARY_PATH may not be necessary with proper linking. 26 | 27 | This let's one run the standard go commands: 28 | 29 | go build 30 | go install goget.corp.qc/go/qfs 31 | 32 | Even testing works, with the sample server in the main project, assuming 33 | QFS_SOURCE environment variable is set to the source checkout of qfs: 34 | 35 | python $QFS_SOURCE/examples/sampleservers/sample_setup.py -a install 36 | python $QFS_SOURCE/examples/sampleservers/sample_setup.py -a start 37 | go test -qfs.addr localhost:20000 38 | 39 | ## Caveats 40 | 41 | - The locking may be slightly course-grained. 42 | - There are several easy performance enhancements that can be gained by 43 | eliminating redundant calls to QFS client library and through aggressive 44 | caching. 45 | -------------------------------------------------------------------------------- /ext/dns/src/fcrd.spf: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env PATH=. spf -f 2 | # 3 | # Bytecode to do forward-confirmed reverse DNS the hard way (not using the 4 | # handy iterator addrinfo/nextent ops, and of course not using the actual 5 | # fcrd/fcrdx ops). 6 | # 7 | 8 | # Using a hotmail.com address which had--as of the time this was 9 | # written--multiple PTR records. Multiple PTR records is illegal, but 10 | # Microsoft tends not to care about such things. (They also do illegal CNAME 11 | # chaining, even with their MX records!) 12 | "64.4.32.7 13 | dup 14 | 'i 15 | setenv 16 | "in-addr 17 | 'v 18 | setenv 19 | "%{ir}.%{v}.arpa. 20 | expand 21 | 22 | # 23 | # Resolve IN PTR 24 | # 25 | ## debug 26 | dup 27 | ">>> Querying 28 | swap 29 | cat 30 | " IN PTR 31 | cat 32 | puts 33 | ## query 34 | $DNS_T_PTR 35 | submit 36 | fetch 37 | " 38 | $DNS_S_AN 39 | $DNS_T_PTR 40 | grep 41 | ## OUTER LOOP 42 | L0 43 | next 44 | dup 45 | not 46 | J7 47 | 48 | # 49 | # Resolve the PTR CNAME at [-1] 50 | # 51 | ## debug 52 | dup 53 | ">>> Querying 54 | swap 55 | cat 56 | " IN A 57 | cat 58 | puts 59 | ## query 60 | $DNS_T_A 61 | submit 62 | fetch 63 | " 64 | $DNS_S_AN 65 | $DNS_T_A 66 | grep 67 | ## INNER LOOP 68 | L1 69 | next 70 | dup 71 | not 72 | J2 73 | 74 | # 75 | # Print PTR ([-3]) and A records ([-1]) 76 | # 77 | dup 78 | -4 79 | load 80 | qname 81 | " -- 82 | cat 83 | swap 84 | cat 85 | swap 86 | 0 87 | load 88 | cmp 89 | not 90 | not 91 | J4 92 | " (CONFIRMED) 93 | true 94 | J5 95 | L4 96 | " 97 | L5 98 | cat 99 | puts 100 | ## JUMP TO INNER LOOP 101 | true 102 | J1 103 | 104 | ## JUMP TO OUTER LOOP 105 | L2 106 | pop 107 | pop 108 | pop 109 | true 110 | J0 111 | 112 | # 113 | # HALT 114 | # 115 | L7 116 | pop 117 | pop 118 | pop 119 | pop 120 | halt 121 | -------------------------------------------------------------------------------- /src/java/hadoop-qfs-2/src/test/java/com/quantcast/qfs/hadoop/TestQuantcastFileSystem2.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed under the Apache License, Version 2.0 4 | * (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | * implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | * 15 | * Unit tests for testing the KosmosFileSystem API implementation. 16 | */ 17 | 18 | package com.quantcast.qfs.hadoop; 19 | 20 | import java.io.IOException; 21 | import java.net.URI; 22 | import org.apache.hadoop.conf.Configuration; 23 | import org.apache.hadoop.fs.Path; 24 | import com.quantcast.qfs.hadoop.QuantcastFileSystem2; 25 | 26 | 27 | public class TestQuantcastFileSystem2 extends TestQuantcastFileSystem { 28 | 29 | @Override 30 | protected void setUp() throws IOException { 31 | Configuration conf = new Configuration(); 32 | 33 | qfsEmul = new QFSEmulationImpl(conf); 34 | quantcastFileSystem = new QuantcastFileSystem2(qfsEmul, null); 35 | // a dummy URI; we are not connecting to any setup here 36 | quantcastFileSystem.initialize(URI.create("qfs:///"), conf); 37 | baseDir = new Path(System.getProperty("test.build.data", "/tmp" ) + 38 | "/qfs-test"); 39 | } 40 | 41 | // @Test 42 | // Enasure HDFS compatibility 43 | public void testHDFSCompatibility() throws Exception { 44 | assertEquals(quantcastFileSystem.getScheme(), "qfs"); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /ext/jerasure/src/timing.c: -------------------------------------------------------------------------------- 1 | // Timing measurement utilities implementation. 2 | 3 | #include "timing.h" 4 | #include 5 | 6 | void 7 | timing_set( 8 | struct timing * t) 9 | { 10 | #ifdef USE_CLOCK 11 | t->clock = clock(); 12 | #else 13 | gettimeofday(&t->tv, NULL); 14 | #endif 15 | } 16 | 17 | double 18 | timing_get( 19 | struct timing * t) 20 | { 21 | #ifdef USE_CLOCK 22 | // The clock_t type is an "arithmetic type", which could be 23 | // integral, double, long double, or others. 24 | // 25 | // Add 0.0 to make it a double or long double, then divide (in 26 | // double or long double), then convert to double for our purposes. 27 | return (double) ((t->clock + 0.0) / CLOCKS_PER_SEC); 28 | #else 29 | return (double) t->tv.tv_sec + ((double) t->tv.tv_usec) / 1000000.0; 30 | #endif 31 | } 32 | 33 | double 34 | timing_now() 35 | { 36 | #ifdef USE_CLOCK 37 | return (double) ((clock() + 0.0) / CLOCKS_PER_SEC); 38 | #else 39 | struct timeval tv; 40 | gettimeofday(&tv, NULL); 41 | return (double) tv.tv_sec + ((double) tv.tv_usec) / 1000000.0; 42 | #endif 43 | } 44 | 45 | double 46 | timing_delta( 47 | struct timing * t1, 48 | struct timing * t2) 49 | { 50 | #ifdef USE_CLOCK 51 | // The clock_t type is an "arithmetic type", which could be 52 | // integral, double, long double, or others. 53 | // 54 | // Subtract first, resulting in another clock_t, then add 0.0 to 55 | // make it a double or long double, then divide (in double or long 56 | // double), then convert to double for our purposes. 57 | return (double) (((t2->clock - t1->clock) + 0.0) / CLOCKS_PER_SEC); 58 | #else 59 | double const d2 = (double) t2->tv.tv_sec + ((double) t2->tv.tv_usec) / 1000000.0; 60 | double const d1 = (double) t1->tv.tv_sec + ((double) t1->tv.tv_usec) / 1000000.0; 61 | return d2 - d1; 62 | #endif 63 | } 64 | -------------------------------------------------------------------------------- /ext/jerasure/COPYING: -------------------------------------------------------------------------------- 1 | 2 | Copyright (c) 2013, James S. Plank and Kevin Greenan 3 | All rights reserved. 4 | 5 | Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure Coding Techniques 6 | 7 | Revision 2.0: Galois Field backend now links to GF-Complete 8 | 9 | Redistribution and use in source and binary forms, with or without 10 | modification, are permitted provided that the following conditions 11 | are met: 12 | 13 | - Redistributions of source code must retain the above copyright 14 | notice, this list of conditions and the following disclaimer. 15 | 16 | - Redistributions in binary form must reproduce the above copyright 17 | notice, this list of conditions and the following disclaimer in 18 | the documentation and/or other materials provided with the 19 | distribution. 20 | 21 | - Neither the name of the University of Tennessee nor the names of its 22 | contributors may be used to endorse or promote products derived 23 | from this software without specific prior written permission. 24 | 25 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 26 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 27 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 28 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 29 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 30 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 31 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 32 | OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 33 | AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY 35 | WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 | POSSIBILITY OF SUCH DAMAGE. 37 | 38 | -------------------------------------------------------------------------------- /ext/jerasure/License.txt: -------------------------------------------------------------------------------- 1 | 2 | Copyright (c) 2013, James S. Plank and Kevin Greenan 3 | All rights reserved. 4 | 5 | Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure Coding Techniques 6 | 7 | Revision 2.0: Galois Field backend now links to GF-Complete 8 | 9 | Redistribution and use in source and binary forms, with or without 10 | modification, are permitted provided that the following conditions 11 | are met: 12 | 13 | - Redistributions of source code must retain the above copyright 14 | notice, this list of conditions and the following disclaimer. 15 | 16 | - Redistributions in binary form must reproduce the above copyright 17 | notice, this list of conditions and the following disclaimer in 18 | the documentation and/or other materials provided with the 19 | distribution. 20 | 21 | - Neither the name of the University of Tennessee nor the names of its 22 | contributors may be used to endorse or promote products derived 23 | from this software without specific prior written permission. 24 | 25 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 26 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 27 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 28 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 29 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 30 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 31 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 32 | OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 33 | AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY 35 | WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 | POSSIBILITY OF SUCH DAMAGE. 37 | 38 | -------------------------------------------------------------------------------- /src/test-scripts/debugchunkcore.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # $Id$ 4 | # 5 | # Created 2009 6 | # Author: Mike Ovsiannikov 7 | # 8 | # Copyright 2009,2016 Quantcast Corporation. All rights reserved. 9 | # 10 | # This file is part of Kosmos File System (KFS). 11 | # 12 | # Licensed under the Apache License, Version 2.0 13 | # (the "License"); you may not use this file except in compliance with 14 | # the License. You may obtain a copy of the License at 15 | # 16 | # http://www.apache.org/licenses/LICENSE-2.0 17 | # 18 | # Unless required by applicable law or agreed to in writing, software 19 | # distributed under the License is distributed on an "AS IS" BASIS, 20 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | # implied. See the License for the specific language governing 22 | # permissions and limitations under the License. 23 | # 24 | 25 | outd="./`basename "$0" .sh`.$$.out" 26 | mkdir "$outd" || exit 27 | awk '{ 28 | print $1 29 | }' ${0+"$@"} \ 30 | | while read h; do 31 | { 32 | echo "========================== $h ================================="; 33 | ssh -o StrictHostKeyChecking=no -l root "$h" sh -c \'\ 34 | 'hostname && ' \ 35 | 'cd /opt/kfs-sort && ls -ltr core.* bin/chunkserver && ' \ 36 | 'md5sum bin/chunkserver && ' \ 37 | 'gdbtmp=/tmp/$$tmp.gdb && ' \ 38 | '{ echo bt && echo set print pretty && echo quit ; } > "$gdbtmp" && ' \ 39 | 'ls -tr core.* | tr " " "\n" | tail -n 2 ' \ 40 | '| while read n; do ' \ 41 | 'echo ===================== "$n" =======================; ' \ 42 | 'gunzip "$n"; ' \ 43 | 'gdb bin/chunkserver `basename "$n" .gz` < "$gdbtmp"; ' \ 44 | 'echo ""; ' \ 45 | 'done; ' \ 46 | 'rm -rf "$gdbtmp"; ' \ 47 | \'; 48 | } > "$outd/$h.trace.gdb" 2>&1 & 49 | done 50 | wait 51 | -------------------------------------------------------------------------------- /ext/gf-complete/tools/Makefile.am: -------------------------------------------------------------------------------- 1 | # GF-Complete 'tools' AM file 2 | 3 | AM_CPPFLAGS = -I$(top_builddir)/include -I$(top_srcdir)/include 4 | AM_CFLAGS = -O3 -fPIC 5 | 6 | bin_PROGRAMS = gf_mult gf_div gf_add gf_time gf_methods gf_poly gf_inline_time 7 | 8 | gf_mult_SOURCES = gf_mult.c 9 | #gf_mult_LDFLAGS = -lgf_complete 10 | gf_mult_LDADD = ../src/libgf_complete.la 11 | 12 | gf_div_SOURCES = gf_div.c 13 | #gf_div_LDFLAGS = -lgf_complete 14 | gf_div_LDADD = ../src/libgf_complete.la 15 | 16 | gf_add_SOURCES = gf_add.c 17 | #gf_add_LDFLAGS = -lgf_complete 18 | gf_add_LDADD = ../src/libgf_complete.la 19 | 20 | gf_time_SOURCES = gf_time.c 21 | #gf_time_LDFLAGS = -lgf_complete 22 | gf_time_LDADD = ../src/libgf_complete.la 23 | 24 | gf_methods_SOURCES = gf_methods.c 25 | #gf_methods_LDFLAGS = -lgf_complete 26 | gf_methods_LDADD = ../src/libgf_complete.la 27 | 28 | gf_poly_SOURCES = gf_poly.c 29 | #gf_poly_LDFLAGS = -lgf_complete 30 | gf_poly_LDADD = ../src/libgf_complete.la 31 | 32 | gf_inline_time_SOURCES = gf_inline_time.c 33 | #gf_inline_time_LDFLAGS = -lgf_complete 34 | gf_inline_time_LDADD = ../src/libgf_complete.la 35 | 36 | # gf_unit 8 A -1 -m LOG_ZERO_EXT is excluded until http://lab.jerasure.org/jerasure/gf-complete/issues/13 is resolved 37 | if ENABLE_VALGRIND 38 | VALGRIND = | perl -p -e 's|^|../libtool --mode=execute valgrind --quiet --error-exitcode=1 --tool=memcheck | if(!/gf_unit 8 A -1 -m LOG_ZERO_EXT/)' 39 | endif 40 | 41 | # gf_unit tests as generated by gf_methods 42 | gf_unit_w%.sh: gf_methods 43 | ./$^ $(@:gf_unit_w%.sh=%) -A -U ${VALGRIND} > $@ || rm $@ 44 | 45 | TESTS = gf_unit_w128.sh \ 46 | gf_unit_w64.sh \ 47 | gf_unit_w32.sh \ 48 | gf_unit_w16.sh \ 49 | gf_unit_w8.sh \ 50 | gf_unit_w4.sh 51 | 52 | TEST_EXTENSIONS = .sh 53 | SH_LOG_COMPILER = $(SHELL) 54 | AM_SH_LOG_FLAGS = -e 55 | 56 | CLEANFILES = $(TESTS) 57 | -------------------------------------------------------------------------------- /src/cc/devtools/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # $Id$ 3 | # 4 | # Created 2006/10/20 5 | # Author: Sriram Rao (Kosmix Corp) 6 | # 7 | # Copyright 2008-2012,2016 Quantcast Corporation. All rights reserved. 8 | # Copyright 2006 Kosmix Corp. 9 | # 10 | # This file is part of Kosmos File System (KFS). 11 | # 12 | # Licensed under the Apache License, Version 2.0 13 | # (the "License"); you may not use this file except in compliance with 14 | # the License. You may obtain a copy of the License at 15 | # 16 | # http://www.apache.org/licenses/LICENSE-2.0 17 | # 18 | # Unless required by applicable law or agreed to in writing, software 19 | # distributed under the License is distributed on an "AS IS" BASIS, 20 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | # implied. See the License for the specific language governing 22 | # permissions and limitations under the License. 23 | # 24 | # 25 | 26 | set (exe_files 27 | checksum 28 | dirtree_creator 29 | logger 30 | rand-sfmt 31 | requestparser 32 | sortedhash 33 | stlset 34 | sslfiltertest 35 | dtokentest 36 | httpstest 37 | xmlscannertest 38 | net_forwarder_test 39 | ) 40 | 41 | # 42 | # Every executable depends on its namesake source with _main.cc 43 | # 44 | foreach (exe_file ${exe_files}) 45 | add_executable (${exe_file} ${exe_file}_main.cc) 46 | if (USE_STATIC_LIB_LINKAGE) 47 | add_dependencies (${exe_file} kfsClient) 48 | target_link_libraries (${exe_file} 49 | kfsClient 50 | ) 51 | else (USE_STATIC_LIB_LINKAGE) 52 | add_dependencies (${exe_file} kfsClient-shared) 53 | target_link_libraries (${exe_file} 54 | kfsClient-shared 55 | ) 56 | endif (USE_STATIC_LIB_LINKAGE) 57 | endforeach (exe_file) 58 | 59 | # 60 | install (TARGETS ${exe_files} 61 | RUNTIME DESTINATION bin/devtools) 62 | 63 | 64 | -------------------------------------------------------------------------------- /src/cc/kfsio/event.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2006/03/22 5 | // Author: Sriram Rao 6 | // 7 | // Copyright 2008-2010,2016 Quantcast Corporation. All rights reserved. 8 | // Copyright 2006-2008 Kosmix Corp. 9 | // 10 | // This file is part of Kosmos File System (KFS). 11 | // 12 | // Licensed under the Apache License, Version 2.0 13 | // (the "License"); you may not use this file except in compliance with 14 | // the License. You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | // implied. See the License for the specific language governing 22 | // permissions and limitations under the License. 23 | // 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef _LIBKFSIO_EVENT_H 28 | #define _LIBKFSIO_EVENT_H 29 | 30 | #include "KfsCallbackObj.h" 31 | 32 | namespace KFS 33 | { 34 | /// 35 | /// \enum EventCode_t 36 | /// Various event codes that a KfsCallbackObj is notified with when 37 | /// events occur. 38 | /// 39 | enum EventCode_t { 40 | EVENT_NEW_CONNECTION, 41 | EVENT_NET_READ, 42 | EVENT_NET_WROTE, 43 | EVENT_NET_ERROR, 44 | EVENT_DISK_READ, 45 | EVENT_DISK_WROTE, 46 | EVENT_DISK_ERROR, 47 | EVENT_CHECK_OPEN_STATUS_DONE, 48 | EVENT_CMD_DONE, 49 | EVENT_INACTIVITY_TIMEOUT, 50 | EVENT_TIMEOUT, 51 | EVENT_DISK_DELETE_DONE, 52 | EVENT_DISK_RENAME_DONE, 53 | EVENT_DISK_GET_FS_SPACE_AVAIL_DONE, 54 | EVENT_DISK_CHECK_DIR_READABLE_DONE, 55 | EVENT_DISK_CHECK_DIR_WRITABLE_DONE 56 | }; 57 | 58 | } 59 | 60 | #endif // _LIBKFSIO_EVENT_H 61 | -------------------------------------------------------------------------------- /ext/gf-complete/include/gf_w16.h: -------------------------------------------------------------------------------- 1 | /* 2 | * GF-Complete: A Comprehensive Open Source Library for Galois Field Arithmetic 3 | * James S. Plank, Ethan L. Miller, Kevin M. Greenan, 4 | * Benjamin A. Arnold, John A. Burnum, Adam W. Disney, Allen C. McBride. 5 | * 6 | * gf_w16.h 7 | * 8 | * Defines and data structures for 16-bit Galois fields 9 | */ 10 | 11 | #ifndef GF_COMPLETE_GF_W16_H 12 | #define GF_COMPLETE_GF_W16_H 13 | 14 | #include 15 | 16 | #define GF_FIELD_WIDTH (16) 17 | #define GF_FIELD_SIZE (1 << GF_FIELD_WIDTH) 18 | #define GF_MULT_GROUP_SIZE GF_FIELD_SIZE-1 19 | 20 | #define GF_BASE_FIELD_WIDTH (8) 21 | #define GF_BASE_FIELD_SIZE (1 << GF_BASE_FIELD_WIDTH) 22 | 23 | struct gf_w16_logtable_data { 24 | uint16_t log_tbl[GF_FIELD_SIZE]; 25 | uint16_t antilog_tbl[GF_FIELD_SIZE * 2]; 26 | uint16_t inv_tbl[GF_FIELD_SIZE]; 27 | uint16_t *d_antilog; 28 | }; 29 | 30 | struct gf_w16_zero_logtable_data { 31 | int log_tbl[GF_FIELD_SIZE]; 32 | uint16_t _antilog_tbl[GF_FIELD_SIZE * 4]; 33 | uint16_t *antilog_tbl; 34 | uint16_t inv_tbl[GF_FIELD_SIZE]; 35 | }; 36 | 37 | struct gf_w16_lazytable_data { 38 | uint16_t log_tbl[GF_FIELD_SIZE]; 39 | uint16_t antilog_tbl[GF_FIELD_SIZE * 2]; 40 | uint16_t inv_tbl[GF_FIELD_SIZE]; 41 | uint16_t *d_antilog; 42 | uint16_t lazytable[GF_FIELD_SIZE]; 43 | }; 44 | 45 | struct gf_w16_bytwo_data { 46 | uint64_t prim_poly; 47 | uint64_t mask1; 48 | uint64_t mask2; 49 | }; 50 | 51 | struct gf_w16_split_8_8_data { 52 | uint16_t tables[3][256][256]; 53 | }; 54 | 55 | struct gf_w16_group_4_4_data { 56 | uint16_t reduce[16]; 57 | uint16_t shift[16]; 58 | }; 59 | 60 | struct gf_w16_composite_data { 61 | uint8_t *mult_table; 62 | }; 63 | 64 | void gf_w16_neon_split_init(gf_t *gf); 65 | 66 | #endif /* GF_COMPLETE_GF_W16_H */ 67 | -------------------------------------------------------------------------------- /ext/gf-complete/include/gf_w32.h: -------------------------------------------------------------------------------- 1 | /* 2 | * GF-Complete: A Comprehensive Open Source Library for Galois Field Arithmetic 3 | * James S. Plank, Ethan L. Miller, Kevin M. Greenan, 4 | * Benjamin A. Arnold, John A. Burnum, Adam W. Disney, Allen C. McBride. 5 | * 6 | * gf_w32.h 7 | * 8 | * Defines and data structures for 32-bit Galois fields 9 | */ 10 | 11 | #ifndef GF_COMPLETE_GF_W32_H 12 | #define GF_COMPLETE_GF_W32_H 13 | 14 | #include 15 | 16 | #define GF_FIELD_WIDTH (32) 17 | #define GF_FIRST_BIT (1 << 31) 18 | 19 | #define GF_BASE_FIELD_WIDTH (16) 20 | #define GF_BASE_FIELD_SIZE (1 << GF_BASE_FIELD_WIDTH) 21 | #define GF_BASE_FIELD_GROUP_SIZE GF_BASE_FIELD_SIZE-1 22 | #define GF_MULTBY_TWO(p) (((p) & GF_FIRST_BIT) ? (((p) << 1) ^ h->prim_poly) : (p) << 1) 23 | 24 | struct gf_split_2_32_lazy_data { 25 | uint32_t tables[16][4]; 26 | uint32_t last_value; 27 | }; 28 | 29 | struct gf_w32_split_8_8_data { 30 | uint32_t tables[7][256][256]; 31 | uint32_t region_tables[4][256]; 32 | uint32_t last_value; 33 | }; 34 | 35 | struct gf_w32_group_data { 36 | uint32_t *reduce; 37 | uint32_t *shift; 38 | int tshift; 39 | uint64_t rmask; 40 | uint32_t *memory; 41 | }; 42 | 43 | struct gf_split_16_32_lazy_data { 44 | uint32_t tables[2][(1<<16)]; 45 | uint32_t last_value; 46 | }; 47 | 48 | struct gf_split_8_32_lazy_data { 49 | uint32_t tables[4][256]; 50 | uint32_t last_value; 51 | }; 52 | 53 | struct gf_split_4_32_lazy_data { 54 | uint32_t tables[8][16]; 55 | uint32_t last_value; 56 | }; 57 | 58 | struct gf_w32_bytwo_data { 59 | uint64_t prim_poly; 60 | uint64_t mask1; 61 | uint64_t mask2; 62 | }; 63 | 64 | struct gf_w32_composite_data { 65 | uint16_t *log; 66 | uint16_t *alog; 67 | }; 68 | 69 | void gf_w32_neon_split_init(gf_t *gf); 70 | 71 | #endif /* GF_COMPLETE_GF_W32_H */ 72 | -------------------------------------------------------------------------------- /src/cc/tools/kfsappend.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2007/09/20 5 | // Author: Sriram Rao 6 | // 7 | // Copyright 2008,2016 Quantcast Corporation. All rights reserved. 8 | // Copyright 2007-2008 Kosmix Corp. 9 | // 10 | // This file is part of Kosmos File System (KFS). 11 | // 12 | // Licensed under the Apache License, Version 2.0 13 | // (the "License"); you may not use this file except in compliance with 14 | // the License. You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | // implied. See the License for the specific language governing 22 | // permissions and limitations under the License. 23 | // 24 | // \brief Move all the blocks (chunks) from one file after the last block of 25 | // another file. 26 | // 27 | //---------------------------------------------------------------------------- 28 | 29 | #include "kfsshell.h" 30 | #include "libclient/KfsClient.h" 31 | 32 | #include 33 | 34 | #include 35 | 36 | namespace KFS { 37 | namespace tools { 38 | 39 | using std::cout; 40 | using std::vector; 41 | using std::string; 42 | 43 | int 44 | handleAppend(KfsClient* kfsClient, const vector& args) 45 | { 46 | if (args.size() != 2 || args[0] == "--help") { 47 | cout << "Usage: append src dst " << "\n"; 48 | return -EINVAL; 49 | } 50 | chunkOff_t offset = -1; 51 | const int ret = kfsClient->CoalesceBlocks( 52 | args[0].c_str(), args[1].c_str(), &offset); 53 | cout << "append status: " << ErrorCodeToStr(ret) << 54 | " offset: " << offset << "\n"; 55 | return ret; 56 | } 57 | 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/java/qfs-access/src/main/java/com/quantcast/qfs/access/BufferPool.java: -------------------------------------------------------------------------------- 1 | /** 2 | * $Id$ 3 | * 4 | * Created 2007/08/24 5 | * @author: Sriram Rao (Kosmix Corp.) 6 | * 7 | * Copyright 2007 Kosmix Corp. 8 | * 9 | * This file is part of Kosmos File System (KFS). 10 | * 11 | * Licensed under the Apache License, Version 2.0 12 | * (the "License"); you may not use this file except in compliance with 13 | * the License. You may obtain a copy of the License at 14 | * 15 | * http://www.apache.org/licenses/LICENSE-2.0 16 | * 17 | * Unless required by applicable law or agreed to in writing, software 18 | * distributed under the License is distributed on an "AS IS" BASIS, 19 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | * implied. See the License for the specific language governing 21 | * permissions and limitations under the License. 22 | * 23 | * \brief Java buffer pool for KFS client. 24 | */ 25 | 26 | package com.quantcast.qfs.access; 27 | 28 | import java.nio.ByteBuffer; 29 | import java.util.AbstractQueue; 30 | import java.util.concurrent.ConcurrentLinkedQueue; 31 | 32 | public final class BufferPool { 33 | public static final int BUFFER_SIZE = Integer.getInteger("qfs.access.buffer.size", 1 << 20); 34 | 35 | private final AbstractQueue buffers = new ConcurrentLinkedQueue(); 36 | 37 | private static final BufferPool INSTANCE = new BufferPool(); 38 | 39 | private BufferPool() { 40 | }; 41 | 42 | public static BufferPool getInstance() { 43 | return INSTANCE; 44 | } 45 | 46 | public ByteBuffer getBuffer() { 47 | ByteBuffer ret = buffers.poll(); 48 | if (ret == null) { 49 | ret = ByteBuffer.allocateDirect(BUFFER_SIZE); 50 | } 51 | return ret; 52 | } 53 | 54 | public void releaseBuffer(ByteBuffer b) { 55 | b.clear(); 56 | buffers.add(b); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/cc/kfsio/ProcessRestarter.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2016/09/14 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2016 Quantcast Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // Restart process by issuing exec with saved args, environment, and working 24 | // directory. 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #include 29 | 30 | namespace KFS 31 | { 32 | using std::string; 33 | 34 | class Properties; 35 | 36 | class ProcessRestarter 37 | { 38 | public: 39 | ProcessRestarter( 40 | bool inCloseFdsAtInitFlag = false, 41 | bool inSaveRestoreEnvFlag = false, 42 | bool inExitOnRestartFlag = false, 43 | bool inCloseFdsBeforeExecFlag = false, 44 | int inMaxGracefulRestartSeconds = 60 * 6); 45 | ~ProcessRestarter(); 46 | int Init( 47 | int inArgCnt, 48 | char** inArgsPtr); 49 | void SetParameters( 50 | const char* inPrefixPtr, 51 | const Properties& inProps); 52 | string Restart(); 53 | static void CloseFds( 54 | int inFrstFd); 55 | private: 56 | class Impl; 57 | Impl& mImpl; 58 | }; 59 | 60 | } // namespace KFS 61 | -------------------------------------------------------------------------------- /src/cc/chunk/utils.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2006/09/27 5 | // Author: Sriram Rao 6 | // 7 | // Copyright 2008-2012,2016 Quantcast Corporation. All rights reserved. 8 | // Copyright 2006-2008 Kosmix Corp. 9 | // 10 | // This file is part of Kosmos File System (KFS). 11 | // 12 | // Licensed under the Apache License, Version 2.0 13 | // (the "License"); you may not use this file except in compliance with 14 | // the License. You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | // implied. See the License for the specific language governing 22 | // permissions and limitations under the License. 23 | // 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef CHUNKSERVER_UTILS_H 28 | #define CHUNKSERVER_UTILS_H 29 | 30 | #include "common/kfstypes.h" 31 | 32 | #include 33 | 34 | namespace KFS 35 | { 36 | using std::string; 37 | 38 | class IOBuffer; 39 | /// 40 | /// Given some data in a buffer, determine if we have a received a 41 | /// valid op---one that ends with "\r\n\r\n". 42 | /// @param[in] iobuf : buffer containing data 43 | /// @param[out] msgLen : if we do have a valid command, return the length of 44 | /// the command 45 | /// @retval True if we have a valid command; False otherwise. 46 | /// 47 | bool IsMsgAvail(IOBuffer* iobuf, int* msgLen); 48 | 49 | /// 50 | /// \brief bomb out on "impossible" error 51 | /// \param[in] msg panic text 52 | /// 53 | void die(const string &msg); 54 | 55 | /// 56 | /// \brief random initial seq. number 57 | /// 58 | kfsSeq_t GetRandomSeq(); 59 | } 60 | 61 | #endif // CHUNKSERVER_UTILS_H 62 | -------------------------------------------------------------------------------- /src/cc/meta/kfstypes.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * $Id$ 3 | * 4 | * \file kfstypes.h 5 | * \brief simple typedefs and enums for the KFS metadata server 6 | * \author Blake Lewis (Kosmix Corp.) 7 | * 8 | * Copyright 2008-2012,2016 Quantcast Corporation. All rights reserved. 9 | * Copyright 2006-2008 Kosmix Corp. 10 | * 11 | * This file is part of Kosmos File System (KFS). 12 | * 13 | * Licensed under the Apache License, Version 2.0 14 | * (the "License"); you may not use this file except in compliance with 15 | * the License. You may obtain a copy of the License at 16 | * 17 | * http://www.apache.org/licenses/LICENSE-2.0 18 | * 19 | * Unless required by applicable law or agreed to in writing, software 20 | * distributed under the License is distributed on an "AS IS" BASIS, 21 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 22 | * implied. See the License for the specific language governing 23 | * permissions and limitations under the License. 24 | * 25 | */ 26 | #if !defined(KFS_TYPES_H) 27 | #define KFS_TYPES_H 28 | 29 | #include "common/kfstypes.h" 30 | 31 | namespace KFS { 32 | 33 | /*! 34 | * \brief KFS metadata types. 35 | */ 36 | enum MetaType { 37 | //!< Key class and meta tree ordering / logic depends on these values. 38 | KFS_UNINIT = 0x00, //!< uninitialized 39 | KFS_INTERNAL = 0x10, //!< internal node 40 | KFS_FATTR = 0x20, //!< file attributes 41 | KFS_CHUNKINFO = 0x21, //!< chunk information 42 | KFS_DENTRY = 0x22, //!< directory entry 43 | KFS_SENTINEL = 0x30 //!< internal use, must be largest 44 | }; 45 | 46 | /*! 47 | * \brief KFS file types 48 | */ 49 | enum FileType { 50 | KFS_NONE, //!< uninitialized 51 | KFS_FILE, //!< plain file 52 | KFS_DIR //!< directory 53 | }; 54 | 55 | /*! 56 | * \brief KFS lease types 57 | */ 58 | enum LeaseType { 59 | READ_LEASE, 60 | WRITE_LEASE 61 | }; 62 | 63 | } 64 | #endif // !defined(KFS_TYPES_H) 65 | -------------------------------------------------------------------------------- /ext/gf-complete/examples/gf_example_4.c: -------------------------------------------------------------------------------- 1 | /* 2 | * GF-Complete: A Comprehensive Open Source Library for Galois Field Arithmetic 3 | * James S. Plank, Ethan L. Miller, Kevin M. Greenan, 4 | * Benjamin A. Arnold, John A. Burnum, Adam W. Disney, Allen C. McBride. 5 | * 6 | * gf_example_4.c 7 | * 8 | * Identical to example_3 except it works in GF(2^128) 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "gf_complete.h" 19 | #include "gf_rand.h" 20 | 21 | #define LLUI (long long unsigned int) 22 | 23 | void usage(char *s) 24 | { 25 | fprintf(stderr, "usage: gf_example_3\n"); 26 | exit(1); 27 | } 28 | 29 | int main(int argc, char **argv) 30 | { 31 | uint64_t a[2], b[2], c[2]; 32 | uint64_t *r1, *r2; 33 | int i; 34 | gf_t gf; 35 | 36 | if (argc != 1) usage(NULL); 37 | 38 | /* Get two random numbers in a and b */ 39 | 40 | MOA_Seed(time(0)); 41 | MOA_Random_128(a); 42 | MOA_Random_128(b); 43 | 44 | /* Create the proper instance of the gf_t object using defaults: */ 45 | 46 | gf_init_easy(&gf, 128); 47 | 48 | /* And multiply a and b using the galois field: */ 49 | 50 | gf.multiply.w128(&gf, a, b, c); 51 | printf("%016llx%016llx * %016llx%016llx =\n%016llx%016llx\n", 52 | LLUI a[0], LLUI a[1], LLUI b[0], LLUI b[1], LLUI c[0], LLUI c[1]); 53 | 54 | r1 = (uint64_t *) malloc(32); 55 | r2 = (uint64_t *) malloc(32); 56 | 57 | for (i = 0; i < 4; i++) r1[i] = MOA_Random_64(); 58 | 59 | gf.multiply_region.w128(&gf, r1, r2, a, 32, 0); 60 | 61 | printf("\nmultiply_region by %016llx%016llx\n\n", LLUI a[0], LLUI a[1]); 62 | printf("R1 (the source): "); 63 | for (i = 0; i < 4; i += 2) printf(" %016llx%016llx", LLUI r1[i], LLUI r1[i+1]); 64 | 65 | printf("\nR2 (the product): "); 66 | for (i = 0; i < 4; i += 2) printf(" %016llx%016llx", LLUI r2[i], LLUI r2[i+1]); 67 | printf("\n"); 68 | exit(0); 69 | } 70 | -------------------------------------------------------------------------------- /ext/jerasure/Examples/Makefile.am: -------------------------------------------------------------------------------- 1 | # Jerasure AM file 2 | 3 | AM_CPPFLAGS = -I$(top_srcdir)/include 4 | AM_CFLAGS = $(SIMD_FLAGS) 5 | 6 | bin_PROGRAMS = jerasure_01 \ 7 | jerasure_02 \ 8 | jerasure_03 \ 9 | jerasure_04 \ 10 | jerasure_05 \ 11 | jerasure_06 \ 12 | jerasure_07 \ 13 | jerasure_08 \ 14 | reed_sol_01 \ 15 | reed_sol_02 \ 16 | reed_sol_03 \ 17 | reed_sol_04 \ 18 | reed_sol_test_gf \ 19 | reed_sol_time_gf \ 20 | cauchy_01 \ 21 | cauchy_02 \ 22 | cauchy_03 \ 23 | cauchy_04 \ 24 | liberation_01 \ 25 | encoder \ 26 | decoder 27 | 28 | TESTS=test_all_gfs.sh 29 | 30 | dist_noinst_SCRIPTS = test_all_gfs.sh time_all_gfs_argv_init.sh 31 | 32 | jerasure_01_SOURCES = jerasure_01.c 33 | jerasure_02_SOURCES = jerasure_02.c 34 | jerasure_03_SOURCES = jerasure_03.c 35 | jerasure_04_SOURCES = jerasure_04.c 36 | jerasure_05_SOURCES = jerasure_05.c 37 | jerasure_06_SOURCES = jerasure_06.c 38 | jerasure_07_SOURCES = jerasure_07.c 39 | jerasure_08_SOURCES = jerasure_08.c 40 | 41 | reed_sol_01_SOURCES = reed_sol_01.c 42 | reed_sol_02_SOURCES = reed_sol_02.c 43 | reed_sol_03_SOURCES = reed_sol_03.c 44 | reed_sol_04_SOURCES = reed_sol_04.c 45 | 46 | reed_sol_test_gf_SOURCES = reed_sol_test_gf.c 47 | reed_sol_time_gf_SOURCES = reed_sol_time_gf.c 48 | 49 | cauchy_01_SOURCES = cauchy_01.c 50 | cauchy_02_SOURCES = cauchy_02.c 51 | cauchy_03_SOURCES = cauchy_03.c 52 | cauchy_04_SOURCES = cauchy_04.c 53 | 54 | liberation_01_SOURCES = liberation_01.c 55 | 56 | decoder_SOURCES = decoder.c 57 | encoder_SOURCES = encoder.c 58 | 59 | LDADD = ../src/libJerasure.la 60 | decoder_LDADD = $(LDADD) ../src/libtiming.a 61 | encoder_LDADD = $(LDADD) ../src/libtiming.a 62 | reed_sol_time_gf_LDADD = $(LDADD) ../src/libtiming.a 63 | -------------------------------------------------------------------------------- /src/cc/chunk/IOMethodDef.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2015/09/11 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2014,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // Storage IO method definitions. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef KFS_CHUNK_IO_METHOD_DEF_H 28 | #define KFS_CHUNK_IO_METHOD_DEF_H 29 | 30 | #include "IOMethod.h" 31 | 32 | #define KFS_MAKE_REGISTERED_IO_METHOD_NAME(inType) \ 33 | IOMethodRegistry_KFS_IO_METHOD_##inType 34 | #define KFS_DECLARE_IO_METHOD(inType) \ 35 | IOMethod* KFS_MAKE_REGISTERED_IO_METHOD_NAME(inType)( \ 36 | const char* inUrlPtr, \ 37 | const char* inLogPrefixPtr, \ 38 | const char* inParamsPrefixPtr, \ 39 | const Properties& inParameters \ 40 | ) 41 | #define KFS_REGISTER_IO_METHOD(inType, inMethod) \ 42 | extern KFS_DECLARE_IO_METHOD(inType); \ 43 | KFS_DECLARE_IO_METHOD(inType) \ 44 | { \ 45 | return inMethod( \ 46 | inUrlPtr, \ 47 | inLogPrefixPtr, \ 48 | inParamsPrefixPtr, \ 49 | inParameters \ 50 | ); \ 51 | } 52 | 53 | #endif /* KFS_CHUNK_IO_METHOD_DEF_H */ 54 | -------------------------------------------------------------------------------- /src/cc/kfsio/Base64.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2013/9/9 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2013-2025 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // 24 | //---------------------------------------------------------------------------- 25 | 26 | #ifndef KFSIO_BASE64_H 27 | #define KFSIO_BASE64_H 28 | 29 | #include "common/kfstypes.h" 30 | 31 | namespace KFS 32 | { 33 | 34 | class Base64 35 | { 36 | public: 37 | KFS_CONSTEXPR static int EncodedLength( 38 | int inLength) 39 | { return ((inLength + 2) / 3 * 4); } 40 | KFS_CONSTEXPR static int GetEncodedMaxBufSize( 41 | int inLength) 42 | { return EncodedLength(inLength) + 1; } 43 | KFS_CONSTEXPR static int GetMaxDecodedLength( 44 | int inLength) 45 | { return ((inLength + 3) / 4 * 3); } 46 | static int Encode( 47 | const char* inBufPtr, 48 | int inBufLength, 49 | char* inEncodedBufPtr, 50 | bool inUrlSafeFmtFlag = false); 51 | static int Decode( 52 | const char* inBufPtr, 53 | int inBufLength, 54 | char* inDecodedBufPtr, 55 | bool inUrlSafeFmtFlag = false); 56 | }; 57 | 58 | } // namespace KFS 59 | 60 | #endif /* KFSIO_BASE64_H */ 61 | -------------------------------------------------------------------------------- /src/cc/kfsio/NetManagerWatcher.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2019/02/09 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2019 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Quantcast File System (QFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // Net manager watchdog interface implementation. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef KFS_KFSIO_NET_MANAGER_WATCHER_H 28 | #define KFS_KFSIO_NET_MANAGER_WATCHER_H 29 | 30 | #include "common/Watchdog.h" 31 | #include "NetManager.h" 32 | 33 | namespace KFS 34 | { 35 | 36 | class NetManagerWatcher : public Watchdog::Watched 37 | { 38 | public: 39 | NetManagerWatcher( 40 | const char* inNamePtr, 41 | const NetManager& inNetManager) 42 | : Watchdog::Watched(inNamePtr), 43 | mNetManager(inNetManager) 44 | {} 45 | virtual ~NetManagerWatcher() 46 | {} 47 | virtual uint64_t Poll() const 48 | { return (uint64_t)mNetManager.NowUsec(); } 49 | private: 50 | const NetManager& mNetManager; 51 | private: 52 | NetManagerWatcher( 53 | const NetManagerWatcher& inNetManagerWatcher); 54 | NetManagerWatcher& operator=( 55 | const NetManagerWatcher& inNetManagerWatcher); 56 | }; 57 | 58 | } 59 | #endif /* KFS_KFSIO_NET_MANAGER_WATCHER_H */ 60 | -------------------------------------------------------------------------------- /ext/gf-complete/examples/gf_example_7.c: -------------------------------------------------------------------------------- 1 | /* 2 | * GF-Complete: A Comprehensive Open Source Library for Galois Field Arithmetic 3 | * James S. Plank, Ethan L. Miller, Kevin M. Greenan, 4 | * Benjamin A. Arnold, John A. Burnum, Adam W. Disney, Allen C. McBride. 5 | * 6 | * gf_example_7.c 7 | * 8 | * Demonstrating extract_word and Cauchy 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "gf_complete.h" 19 | #include "gf_rand.h" 20 | 21 | void usage(char *s) 22 | { 23 | fprintf(stderr, "usage: gf_example_7\n"); 24 | exit(1); 25 | } 26 | 27 | int main(int argc, char **argv) 28 | { 29 | uint8_t *a, *b; 30 | int i, j; 31 | gf_t gf; 32 | 33 | if (gf_init_hard(&gf, 3, GF_MULT_TABLE, GF_REGION_CAUCHY, GF_DIVIDE_DEFAULT, 0, 0, 0, NULL, NULL) == 0) { 34 | fprintf(stderr, "gf_init_hard failed\n"); 35 | exit(1); 36 | } 37 | 38 | a = (uint8_t *) malloc(3); 39 | b = (uint8_t *) malloc(3); 40 | 41 | MOA_Seed(0); 42 | 43 | for (i = 0; i < 3; i++) a[i] = MOA_Random_W(8, 1); 44 | 45 | gf.multiply_region.w32(&gf, a, b, 5, 3, 0); 46 | 47 | printf("a: 0x%lx b: 0x%lx\n", (unsigned long) a, (unsigned long) b); 48 | 49 | printf("\n"); 50 | printf("a: 0x%02x 0x%02x 0x%02x\n", a[0], a[1], a[2]); 51 | printf("b: 0x%02x 0x%02x 0x%02x\n", b[0], b[1], b[2]); 52 | printf("\n"); 53 | 54 | printf("a bits:"); 55 | for (i = 0; i < 3; i++) { 56 | printf(" "); 57 | for (j = 7; j >= 0; j--) printf("%c", (a[i] & (1 << j)) ? '1' : '0'); 58 | } 59 | printf("\n"); 60 | 61 | printf("b bits:"); 62 | for (i = 0; i < 3; i++) { 63 | printf(" "); 64 | for (j = 7; j >= 0; j--) printf("%c", (b[i] & (1 << j)) ? '1' : '0'); 65 | } 66 | printf("\n"); 67 | 68 | printf("\n"); 69 | for (i = 0; i < 8; i++) { 70 | printf("Word %2d: %d * 5 = %d\n", i, 71 | gf.extract_word.w32(&gf, a, 3, i), 72 | gf.extract_word.w32(&gf, b, 3, i)); 73 | } 74 | return 0; 75 | } 76 | -------------------------------------------------------------------------------- /src/cc/kfsio/HttpChunkedDecoder.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2015/10/1 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2015,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief Http chunked encoding parser. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef KFSIO_HTTP_CHUNKED_DECODER_H 28 | #define KFSIO_HTTP_CHUNKED_DECODER_H 29 | 30 | namespace KFS 31 | { 32 | 33 | class IOBuffer; 34 | 35 | class HttpChunkedDecoder 36 | { 37 | public: 38 | HttpChunkedDecoder( 39 | IOBuffer& inIOBuffer, 40 | int inMaxReadAhead = 2 << 10) 41 | : mIOBuffer(inIOBuffer), 42 | mMaxReadAhead(inMaxReadAhead), 43 | mLength(-1), 44 | mChunkRem(0), 45 | mAlignedFlag(false) 46 | {} 47 | ~HttpChunkedDecoder() 48 | {} 49 | void Reset() 50 | { 51 | mLength = -1; 52 | mChunkRem = 0; 53 | mAlignedFlag = false; 54 | } 55 | int Parse( 56 | IOBuffer& inBuffer); 57 | private: 58 | IOBuffer& mIOBuffer; 59 | int mMaxReadAhead; 60 | int mLength; 61 | int mChunkRem; 62 | bool mAlignedFlag; 63 | }; 64 | 65 | } // namespace KFS 66 | 67 | #endif /* KFSIO_HTTP_CHUNKED_DECODER_H */ 68 | -------------------------------------------------------------------------------- /src/cc/common/kfsatomic.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2011/05/15 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2011-2012,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // "Atomic" variables of different sizes (well up to 64 bits on 64 bit 24 | // platforms). 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #ifndef KFS_ATOMIC_H 29 | #define KFS_ATOMIC_H 30 | 31 | namespace KFS 32 | { 33 | #if ! defined(_KFS_ATOMIC_USE_MUTEX) && (\ 34 | ! defined(__GNUC__) || (__GNUC__ < 4 || \ 35 | (__GNUC__ == 4 && (__GNUC_MINOR__ < 1 || \ 36 | (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ < 2))))) 37 | # define _KFS_ATOMIC_USE_MUTEX 38 | #endif 39 | 40 | #ifdef _KFS_ATOMIC_USE_MUTEX 41 | 42 | namespace atomicmpl 43 | { 44 | void AtomicLock(); 45 | void AtomicUnlock(); 46 | } 47 | 48 | template T SyncAddAndFetch(volatile T& val, T inc) 49 | { 50 | atomicmpl::AtomicLock(); 51 | val += inc; 52 | const T ret = val; 53 | atomicmpl::AtomicUnlock(); 54 | return ret; 55 | } 56 | 57 | #else 58 | 59 | template T SyncAddAndFetch(volatile T& val, T inc) 60 | { 61 | return __sync_add_and_fetch(&val, inc); 62 | } 63 | 64 | #endif /* _KFS_ATOMIC_USE_MUTEX */ 65 | } 66 | 67 | #endif /* KFS_ATOMIC_H */ 68 | -------------------------------------------------------------------------------- /src/cc/qcrs/encode.c: -------------------------------------------------------------------------------- 1 | /*---------------------------------------------------------- -*- Mode: C -*----- 2 | * $Id$ 3 | * 4 | * Created 2010/07/24 5 | * Author: Dan Adkins 6 | * 7 | * Copyright 2010-2012,2016 Quantcast Corporation. All rights reserved. 8 | * 9 | * This file is part of Kosmos File System (KFS). 10 | * 11 | * Licensed under the Apache License, Version 2.0 12 | * (the "License"); you may not use this file except in compliance with 13 | * the License. You may obtain a copy of the License at 14 | * 15 | * http://www.apache.org/licenses/LICENSE-2.0 16 | * 17 | * Unless required by applicable law or agreed to in writing, software 18 | * distributed under the License is distributed on an "AS IS" BASIS, 19 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | * implied. See the License for the specific language governing 21 | * permissions and limitations under the License. 22 | * 23 | * \file encode.c 24 | * \brief Reed Solomon encoder. 25 | * 26 | *------------------------------------------------------------------------------ 27 | */ 28 | 29 | #include 30 | #include "rs.h" 31 | #include "prim.h" 32 | 33 | /* 34 | * Reed-Solomon n+3 encoder. 35 | * nblocks is `n' data blocks plus 3 syndrome blocks. blocksize _must_ 36 | * be a multiple of 16. data contains pointers to blocks. The first 37 | * n are input data blocks. The last 3 are the P, Q, and R syndromes. 38 | */ 39 | void 40 | rs_encode(int nblocks, int blocksize, void **idata) 41 | { 42 | int i, j, n; 43 | v16 *p, *q, *r, **data = (v16**)idata; 44 | 45 | assert(nblocks > 3); 46 | assert(blocksize % 16 == 0); 47 | n = nblocks - 3; // # data blocks 48 | p = data[n]; 49 | q = data[n+1]; 50 | r = data[n+2]; 51 | for (i = 0; i < blocksize/sizeof(v16); i++) { 52 | p[i] = q[i] = r[i] = data[n-1][i]; 53 | for (j = n-2; j >= 0; j--) { 54 | p[i] ^= data[j][i]; 55 | q[i] = mul2(q[i]) ^ data[j][i]; 56 | r[i] = mul2(mul2(r[i])) ^ data[j][i]; 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /ext/gf-complete/tools/gf_div.c: -------------------------------------------------------------------------------- 1 | /* 2 | * GF-Complete: A Comprehensive Open Source Library for Galois Field Arithmetic 3 | * James S. Plank, Ethan L. Miller, Kevin M. Greenan, 4 | * Benjamin A. Arnold, John A. Burnum, Adam W. Disney, Allen C. McBride. 5 | * 6 | * gf_div.c 7 | * 8 | * Multiplies two numbers in gf_2^w 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "gf_complete.h" 18 | #include "gf_method.h" 19 | #include "gf_general.h" 20 | 21 | void usage(int why) 22 | { 23 | fprintf(stderr, "usage: gf_div a b w [method] - does division of a and b in GF(2^w)\n"); 24 | if (why == 'W') { 25 | fprintf(stderr, "Bad w.\n"); 26 | fprintf(stderr, "Legal w are: 1 - 32, 64 and 128.\n"); 27 | fprintf(stderr, "Append 'h' to w to treat a, b and the quotient as hexadecimal.\n"); 28 | fprintf(stderr, "w=128 is hex only (i.e. '128' will be an error - do '128h')\n"); 29 | } 30 | if (why == 'A') fprintf(stderr, "Bad a\n"); 31 | if (why == 'B') fprintf(stderr, "Bad b\n"); 32 | if (why == 'M') { 33 | fprintf(stderr, "Bad Method Specification: "); 34 | gf_error(); 35 | } 36 | exit(1); 37 | } 38 | 39 | int main(int argc, char **argv) 40 | { 41 | int hex, w; 42 | gf_t gf; 43 | gf_general_t a, b, c; 44 | char output[50]; 45 | 46 | if (argc < 4) usage(' '); 47 | 48 | if (sscanf(argv[3], "%d", &w) == 0) usage('W'); 49 | if (w <= 0 || (w > 32 && w != 64 && w != 128)) usage('W'); 50 | 51 | hex = (strchr(argv[3], 'h') != NULL); 52 | if (!hex && w == 128) usage('W'); 53 | 54 | if (argc == 4) { 55 | if (gf_init_easy(&gf, w) == 0) usage('M'); 56 | } else { 57 | if (create_gf_from_argv(&gf, w, argc, argv, 4) == 0) usage('M'); 58 | } 59 | 60 | if (!gf_general_s_to_val(&a, w, argv[1], hex)) usage('A'); 61 | if (!gf_general_s_to_val(&b, w, argv[2], hex)) usage('B'); 62 | 63 | gf_general_divide(&gf, &a, &b, &c); 64 | gf_general_val_to_s(&c, w, output, hex); 65 | 66 | printf("%s\n", output); 67 | exit(0); 68 | } 69 | -------------------------------------------------------------------------------- /src/cc/tools/utils.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2007/09/20 5 | // Author: Sriram Rao 6 | // 7 | // Copyright 2008,2016 Quantcast Corporation. All rights reserved. 8 | // Copyright 2007-2008 Kosmix Corp. 9 | // 10 | // This file is part of Kosmos File System (KFS). 11 | // 12 | // Licensed under the Apache License, Version 2.0 13 | // (the "License"); you may not use this file except in compliance with 14 | // the License. You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | // implied. See the License for the specific language governing 22 | // permissions and limitations under the License. 23 | // 24 | // \brief Common utility functions for kfsshell 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #include "kfsshell.h" 29 | #include "libclient/KfsClient.h" 30 | 31 | #include 32 | #include 33 | 34 | namespace KFS { 35 | namespace tools { 36 | 37 | using std::cout; 38 | using std::vector; 39 | using std::string; 40 | 41 | 42 | // Make the directory hierarchy in KFS defined by path. 43 | 44 | int 45 | doMkdirs(KfsClient* kfsClient, const char* path) 46 | { 47 | const int res = kfsClient->Mkdirs(path); 48 | if (res < 0 && res != -EEXIST) { 49 | cout << path << ": mkdirs failure: " << ErrorCodeToStr(res) << "\n"; 50 | return res; 51 | } 52 | return 0; 53 | } 54 | 55 | // remove a single directory in kfs 56 | 57 | int 58 | doRmdir(KfsClient* kfsClient, const char* dirname) 59 | { 60 | const int res = kfsClient->Rmdir(dirname); 61 | if (res < 0) { 62 | cout << dirname << ": rmdir failure: " << ErrorCodeToStr(res) << "\n"; 63 | return res; 64 | } 65 | return 0; 66 | } 67 | 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/test-scripts/allocaterecv.pl: -------------------------------------------------------------------------------- 1 | eval 'exec perl -wS $0 ${1+"$@"}' 2 | if 0; 3 | # 4 | # $Id$ 5 | # 6 | # Created 2010 7 | # Author: Mike Ovsiannikov 8 | # 9 | # Copyright 2010,2016 Quantcast Corporation. All rights reserved. 10 | # 11 | # This file is part of Kosmos File System (KFS). 12 | # 13 | # Licensed under the Apache License, Version 2.0 14 | # (the "License"); you may not use this file except in compliance with 15 | # the License. You may obtain a copy of the License at 16 | # 17 | # http://www.apache.org/licenses/LICENSE-2.0 18 | # 19 | # Unless required by applicable law or agreed to in writing, software 20 | # distributed under the License is distributed on an "AS IS" BASIS, 21 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 22 | # implied. See the License for the specific language governing 23 | # permissions and limitations under the License. 24 | # 25 | # 26 | 27 | my $numcmd = shift || 10; 28 | 29 | my $start = time(); 30 | my $seqf=999000000; 31 | my $seq=$seqf; 32 | my $end=$seq + $numcmd; 33 | my $step=1e4; 34 | my $next=$seq + $step - 1; 35 | if ($next > $end) { 36 | $next = $end - 1; 37 | } 38 | for (; $seq < $end; $seq++) { 39 | my $cs = -5; 40 | my $st = -1; 41 | $resp = ""; 42 | while (defined($line = <>)) { 43 | $resp = $resp . $line; 44 | if ($cs < 0 && $line =~ /Cseq: (\d+)/) { 45 | $cs = $1; 46 | } 47 | if ($st != 0 && $line =~ /Status: (\d+)/) { 48 | $st = $1; 49 | } 50 | last if ($line eq "\r\n"); 51 | } 52 | my $err = $cs != $seq || $st != 0; 53 | if ($seq >= $next || $err) { 54 | $next += $step; 55 | if ($next >= $end) { 56 | $next = $end - 1; 57 | } 58 | my $stop = time(); 59 | print $resp; 60 | my $n = $seq - $seqf + 1; 61 | printf("elapsed: %d %d %10.3f op/sec\n", 62 | $stop - $start, 63 | $n, 64 | $stop > $start ? ($n) / ($stop - $start) : 0 65 | ); 66 | } 67 | last if ($err); 68 | } 69 | printf("done\n"); 70 | -------------------------------------------------------------------------------- /ext/gf-complete/tools/gf_mult.c: -------------------------------------------------------------------------------- 1 | /* 2 | * GF-Complete: A Comprehensive Open Source Library for Galois Field Arithmetic 3 | * James S. Plank, Ethan L. Miller, Kevin M. Greenan, 4 | * Benjamin A. Arnold, John A. Burnum, Adam W. Disney, Allen C. McBride. 5 | * 6 | * gf_mult.c 7 | * 8 | * Multiplies two numbers in gf_2^w 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "gf_complete.h" 18 | #include "gf_method.h" 19 | #include "gf_general.h" 20 | 21 | void usage(int why) 22 | { 23 | fprintf(stderr, "usage: gf_mult a b w [method] - does multiplication of a and b in GF(2^w)\n"); 24 | if (why == 'W') { 25 | fprintf(stderr, "Bad w.\n"); 26 | fprintf(stderr, "Legal w are: 1 - 32, 64 and 128.\n"); 27 | fprintf(stderr, "Append 'h' to w to treat a, b and the product as hexadecimal.\n"); 28 | fprintf(stderr, "w=128 is hex only (i.e. '128' will be an error - do '128h')\n"); 29 | } 30 | if (why == 'A') fprintf(stderr, "Bad a\n"); 31 | if (why == 'B') fprintf(stderr, "Bad b\n"); 32 | if (why == 'M') { 33 | fprintf(stderr, "Bad Method Specification: "); 34 | gf_error(); 35 | } 36 | exit(1); 37 | } 38 | 39 | int main(int argc, char **argv) 40 | { 41 | int hex, w; 42 | gf_t gf; 43 | gf_general_t a, b, c; 44 | char output[50]; 45 | 46 | if (argc < 4) usage(' '); 47 | 48 | if (sscanf(argv[3], "%d", &w) == 0) usage('W'); 49 | if (w <= 0 || (w > 32 && w != 64 && w != 128)) usage('W'); 50 | 51 | hex = (strchr(argv[3], 'h') != NULL); 52 | if (!hex && w == 128) usage('W'); 53 | 54 | if (argc == 4) { 55 | if (gf_init_easy(&gf, w) == 0) usage('M'); 56 | } else { 57 | if (create_gf_from_argv(&gf, w, argc, argv, 4) == 0) usage('M'); 58 | } 59 | 60 | if (!gf_general_s_to_val(&a, w, argv[1], hex)) usage('A'); 61 | if (!gf_general_s_to_val(&b, w, argv[2], hex)) usage('B'); 62 | 63 | gf_general_multiply(&gf, &a, &b, &c); 64 | gf_general_val_to_s(&c, w, output, hex); 65 | 66 | printf("%s\n", output); 67 | exit(0); 68 | } 69 | -------------------------------------------------------------------------------- /src/cc/emulator/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # $Id$ 3 | # 4 | # Created 2006 5 | # Author: Sriram Rao (Kosmix Corp) 6 | # 7 | # Copyright 2008-2012,2016 Quantcast Corporation. All rights reserved. 8 | # Copyright 2006 Kosmix Corp. 9 | # 10 | # This file is part of Kosmos File System (KFS). 11 | # 12 | # Licensed under the Apache License, Version 2.0 13 | # (the "License"); you may not use this file except in compliance with 14 | # the License. You may obtain a copy of the License at 15 | # 16 | # http://www.apache.org/licenses/LICENSE-2.0 17 | # 18 | # Unless required by applicable law or agreed to in writing, software 19 | # distributed under the License is distributed on an "AS IS" BASIS, 20 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | # implied. See the License for the specific language governing 22 | # permissions and limitations under the License. 23 | # 24 | # 25 | 26 | # 27 | # For the library take everything except the *_main.cc files 28 | # 29 | set (lib_srcs 30 | ChunkServerEmulator.cc 31 | LayoutEmulator.cc 32 | emulator_setup.cc 33 | ) 34 | 35 | add_library (kfsEmulator STATIC ${lib_srcs}) 36 | set_target_properties (kfsEmulator PROPERTIES OUTPUT_NAME "qfs_emulator") 37 | set_target_properties (kfsEmulator PROPERTIES CLEAN_DIRECT_OUTPUT 1) 38 | add_dependencies (kfsEmulator 39 | kfsMeta 40 | ) 41 | target_link_libraries(kfsEmulator 42 | kfsMeta 43 | ) 44 | 45 | set (exe_files rebalanceplanner rebalanceexecutor replicachecker) 46 | foreach (exe_file ${exe_files}) 47 | add_executable (${exe_file} ${exe_file}_main.cc) 48 | target_link_libraries (${exe_file} 49 | kfsEmulator 50 | ) 51 | add_dependencies (${exe_file} 52 | kfsEmulator 53 | ) 54 | endforeach (exe_file) 55 | 56 | if (CMAKE_SYSTEM_NAME STREQUAL "SunOS") 57 | target_link_libraries(kfsEmulator mtmalloc) 58 | endif (CMAKE_SYSTEM_NAME STREQUAL "SunOS") 59 | 60 | # 61 | # Install them 62 | # 63 | install (TARGETS ${exe_files} kfsEmulator 64 | RUNTIME DESTINATION bin/emulator 65 | LIBRARY DESTINATION lib 66 | ARCHIVE DESTINATION lib/static) 67 | 68 | -------------------------------------------------------------------------------- /ext/gf-complete/include/gf_w4.h: -------------------------------------------------------------------------------- 1 | /* 2 | * GF-Complete: A Comprehensive Open Source Library for Galois Field Arithmetic 3 | * James S. Plank, Ethan L. Miller, Kevin M. Greenan, 4 | * Benjamin A. Arnold, John A. Burnum, Adam W. Disney, Allen C. McBride. 5 | * 6 | * gf_w4.h 7 | * 8 | * Defines and data structures for 4-bit Galois fields 9 | */ 10 | 11 | #ifndef GF_COMPLETE_GF_W4_H 12 | #define GF_COMPLETE_GF_W4_H 13 | 14 | #include 15 | 16 | #define GF_FIELD_WIDTH 4 17 | #define GF_DOUBLE_WIDTH (GF_FIELD_WIDTH*2) 18 | #define GF_FIELD_SIZE (1 << GF_FIELD_WIDTH) 19 | #define GF_MULT_GROUP_SIZE (GF_FIELD_SIZE-1) 20 | 21 | /* ------------------------------------------------------------ 22 | JSP: Each implementation has its own data, which is allocated 23 | at one time as part of the handle. For that reason, it 24 | shouldn't be hierarchical -- i.e. one should be able to 25 | allocate it with one call to malloc. */ 26 | 27 | struct gf_logtable_data { 28 | uint8_t log_tbl[GF_FIELD_SIZE]; 29 | uint8_t antilog_tbl[GF_FIELD_SIZE * 2]; 30 | uint8_t *antilog_tbl_div; 31 | }; 32 | 33 | struct gf_single_table_data { 34 | uint8_t mult[GF_FIELD_SIZE][GF_FIELD_SIZE]; 35 | uint8_t div[GF_FIELD_SIZE][GF_FIELD_SIZE]; 36 | }; 37 | 38 | struct gf_double_table_data { 39 | uint8_t div[GF_FIELD_SIZE][GF_FIELD_SIZE]; 40 | uint8_t mult[GF_FIELD_SIZE][GF_FIELD_SIZE*GF_FIELD_SIZE]; 41 | }; 42 | struct gf_quad_table_data { 43 | uint8_t div[GF_FIELD_SIZE][GF_FIELD_SIZE]; 44 | uint16_t mult[GF_FIELD_SIZE][(1<<16)]; 45 | }; 46 | 47 | struct gf_quad_table_lazy_data { 48 | uint8_t div[GF_FIELD_SIZE][GF_FIELD_SIZE]; 49 | uint8_t smult[GF_FIELD_SIZE][GF_FIELD_SIZE]; 50 | uint16_t mult[(1 << 16)]; 51 | }; 52 | 53 | struct gf_bytwo_data { 54 | uint64_t prim_poly; 55 | uint64_t mask1; 56 | uint64_t mask2; 57 | }; 58 | 59 | // ARM NEON init functions 60 | int gf_w4_neon_cfm_init(gf_t *gf); 61 | void gf_w4_neon_single_table_init(gf_t *gf); 62 | 63 | #endif /* GF_COMPLETE_GF_W4_H */ 64 | -------------------------------------------------------------------------------- /ext/gf-complete/src/gf_rand.c: -------------------------------------------------------------------------------- 1 | /* 2 | * GF-Complete: A Comprehensive Open Source Library for Galois Field Arithmetic 3 | * James S. Plank, Ethan L. Miller, Kevin M. Greenan, 4 | * Benjamin A. Arnold, John A. Burnum, Adam W. Disney, Allen C. McBride. 5 | * 6 | * gf_rand.c -- Random number generator. 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include "gf_rand.h" 13 | 14 | /* Lifted the "Mother of All" random number generator from http://www.agner.org/random/ */ 15 | 16 | static uint32_t MOA_X[5]; 17 | 18 | uint32_t MOA_Random_32() { 19 | uint64_t sum; 20 | sum = (uint64_t)2111111111UL * (uint64_t)MOA_X[3] + 21 | (uint64_t)1492 * (uint64_t)(MOA_X[2]) + 22 | (uint64_t)1776 * (uint64_t)(MOA_X[1]) + 23 | (uint64_t)5115 * (uint64_t)(MOA_X[0]) + 24 | (uint64_t)MOA_X[4]; 25 | MOA_X[3] = MOA_X[2]; MOA_X[2] = MOA_X[1]; MOA_X[1] = MOA_X[0]; 26 | MOA_X[4] = (uint32_t)(sum >> 32); 27 | MOA_X[0] = (uint32_t)sum; 28 | return MOA_X[0]; 29 | } 30 | 31 | uint64_t MOA_Random_64() { 32 | uint64_t sum; 33 | 34 | sum = MOA_Random_32(); 35 | sum <<= 32; 36 | sum |= MOA_Random_32(); 37 | return sum; 38 | } 39 | 40 | void MOA_Random_128(uint64_t *x) { 41 | x[0] = MOA_Random_64(); 42 | x[1] = MOA_Random_64(); 43 | return; 44 | } 45 | 46 | uint32_t MOA_Random_W(int w, int zero_ok) 47 | { 48 | uint32_t b; 49 | 50 | do { 51 | b = MOA_Random_32(); 52 | if (w == 31) b &= 0x7fffffff; 53 | if (w < 31) b %= (1 << w); 54 | } while (!zero_ok && b == 0); 55 | return b; 56 | } 57 | 58 | void MOA_Seed(uint32_t seed) { 59 | int i; 60 | uint32_t s = seed; 61 | for (i = 0; i < 5; i++) { 62 | s = s * 29943829 - 1; 63 | MOA_X[i] = s; 64 | } 65 | for (i=0; i<19; i++) MOA_Random_32(); 66 | } 67 | 68 | 69 | void MOA_Fill_Random_Region (void *reg, int size) 70 | { 71 | uint32_t *r32; 72 | uint8_t *r8; 73 | int i; 74 | 75 | r32 = (uint32_t *) reg; 76 | r8 = (uint8_t *) reg; 77 | for (i = 0; i < size/4; i++) r32[i] = MOA_Random_32(); 78 | for (i *= 4; i < size; i++) r8[i] = MOA_Random_W(8, 1); 79 | } 80 | 81 | -------------------------------------------------------------------------------- /src/cc/qfsc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # $Id$ 3 | # 4 | # Created 2013/06/17 5 | # Author: Stephen J Day 6 | # 7 | # Copyright 2013-2014,2016 Quantcast Corporation. All rights reserved. 8 | # 9 | # This file is part of Kosmos File System (KFS). 10 | # 11 | # Licensed under the Apache License, Version 2.0 12 | # (the "License"); you may not use this file except in compliance with 13 | # the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, software 18 | # distributed under the License is distributed on an "AS IS" BASIS, 19 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | # implied. See the License for the specific language governing 21 | # permissions and limitations under the License. 22 | # 23 | # 24 | 25 | add_library (qfsc STATIC qfsc.cc) 26 | add_library (qfsc-shared SHARED qfsc.cc) 27 | set_target_properties (qfsc PROPERTIES OUTPUT_NAME "qfsc") 28 | set_target_properties (qfsc-shared PROPERTIES OUTPUT_NAME "qfsc") 29 | set_target_properties (qfsc PROPERTIES CLEAN_DIRECT_OUTPUT 1) 30 | set_target_properties (qfsc-shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) 31 | 32 | add_dependencies (qfsc 33 | kfsClient 34 | ) 35 | add_dependencies (qfsc-shared 36 | kfsClient-shared 37 | ) 38 | target_link_libraries (qfsc 39 | kfsClient 40 | ) 41 | target_link_libraries (qfsc-shared 42 | kfsClient-shared 43 | ) 44 | 45 | add_executable (test-qfsc test-qfsc.c) 46 | set_target_properties (test-qfsc PROPERTIES LINKER_LANGUAGE CXX) 47 | 48 | if (USE_STATIC_LIB_LINKAGE) 49 | add_dependencies (test-qfsc qfsc) 50 | target_link_libraries (test-qfsc qfsc) 51 | else (USE_STATIC_LIB_LINKAGE) 52 | add_dependencies (test-qfsc qfsc-shared) 53 | target_link_libraries (test-qfsc qfsc-shared) 54 | endif (USE_STATIC_LIB_LINKAGE) 55 | 56 | install (TARGETS qfsc qfsc-shared 57 | LIBRARY DESTINATION lib 58 | ARCHIVE DESTINATION lib/static) 59 | install (FILES qfs.h DESTINATION include/kfs/c) 60 | install (TARGETS test-qfsc 61 | RUNTIME DESTINATION bin/devtools) 62 | 63 | -------------------------------------------------------------------------------- /ext/dns/src/cache.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================== 2 | * cache.h - Simple Query Cache for dns.c 3 | * -------------------------------------------------------------------------- 4 | * Copyright (c) 2010 William Ahern 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a 7 | * copy of this software and associated documentation files (the 8 | * "Software"), to deal in the Software without restriction, including 9 | * without limitation the rights to use, copy, modify, merge, publish, 10 | * distribute, sublicense, and/or sell copies of the Software, and to permit 11 | * persons to whom the Software is furnished to do so, subject to the 12 | * following conditions: 13 | * 14 | * The above copyright notice and this permission notice shall be included 15 | * in all copies or substantial portions of the Software. 16 | * 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 18 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN 20 | * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 21 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 22 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 23 | * USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | * ========================================================================== 25 | */ 26 | #ifndef CACHE_H 27 | #define CACHE_H 28 | 29 | #include /* FILE */ 30 | 31 | #include "dns.h" 32 | 33 | 34 | struct cache; 35 | 36 | struct cache *cache_open(int *); 37 | 38 | void cache_close(struct cache *); 39 | 40 | int cache_loadfile(struct cache *, FILE *, const char *, unsigned); 41 | 42 | int cache_loadpath(struct cache *, const char *, const char *, unsigned); 43 | 44 | struct dns_cache *cache_resi(struct cache *); 45 | 46 | int cache_insert(struct cache *, const char *, enum dns_type, unsigned, const void *); 47 | 48 | int cache_dumpfile(struct cache *, FILE *); 49 | 50 | 51 | #endif /* CACHE_H */ 52 | -------------------------------------------------------------------------------- /src/cc/common/time.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2010/10/27 5 | // Author: Dan Adkins 6 | // 7 | // Copyright 2010,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \file time.cc 24 | // \brief time related functions 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #include 29 | #include 30 | #include "time.h" 31 | 32 | namespace KFS { 33 | 34 | bool 35 | getcurrenttime(int64_t* sec, int64_t* usec) 36 | { 37 | struct timeval tv; 38 | 39 | if (gettimeofday(&tv, 0) < 0) { 40 | return false; 41 | } 42 | if (sec) { 43 | *sec = tv.tv_sec; 44 | } 45 | if (usec) { 46 | *usec = tv.tv_usec; 47 | } 48 | return true; 49 | } 50 | 51 | int64_t 52 | microseconds() 53 | { 54 | struct timeval tv; 55 | 56 | if (gettimeofday(&tv, 0) < 0) { 57 | return -1; 58 | } 59 | return (int64_t)tv.tv_sec*1000*1000 + tv.tv_usec; 60 | } 61 | 62 | int64_t 63 | cputime(int64_t* user, int64_t* sys) 64 | { 65 | struct rusage ru; 66 | 67 | if (getrusage(RUSAGE_SELF, &ru) < 0) { 68 | return -1; 69 | } 70 | *user = (int64_t)ru.ru_utime.tv_sec*1000*1000 + ru.ru_utime.tv_usec; 71 | *sys = (int64_t)ru.ru_stime.tv_sec*1000*1000 + ru.ru_stime.tv_usec; 72 | 73 | return *user + *sys; 74 | } 75 | 76 | } // namespace KFS 77 | -------------------------------------------------------------------------------- /src/cc/common/juliantime.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2016/1/5 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // Julian day number, and unix time calculations. 24 | // 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #ifndef KFS_COMMON_JULIAN_TIME_H 29 | #define KFS_COMMON_JULIAN_TIME_H 30 | 31 | #include 32 | 33 | namespace KFS 34 | { 35 | 36 | inline static int64_t 37 | ToJulianDay( 38 | int inYear, 39 | int inMonth, 40 | int inDay) 41 | { 42 | return ((int64_t(1461) * 43 | (inYear + 4800 + (inMonth - 14) / 12)) / 4 + 44 | (367 * (inMonth - 2 - 12 * ((inMonth - 14) / 12))) / 12 - 45 | (3 * ((inYear + 4900 + (inMonth - 14) / 12) / 100)) / 4 + 46 | inDay - 32075 47 | ); 48 | } 49 | 50 | const int64_t kUnixJulianTimeStartSec = 51 | ToJulianDay(1970, 1, 1) * 24 * 60 * 60; 52 | 53 | inline static int64_t 54 | ToUnixTime( 55 | int inYear, 56 | int inMonth, 57 | int inDay, 58 | int inHour, 59 | int inMinute, 60 | int inSecond) 61 | { 62 | return ((( 63 | ToJulianDay(inYear, inMonth, inDay) * 24 + 64 | inHour) * 60 + inMinute) * 60 + inSecond - kUnixJulianTimeStartSec 65 | ); 66 | } 67 | 68 | } // namespace KFS 69 | 70 | #endif /* KFS_COMMON_JULIAN_TIME_H */ 71 | -------------------------------------------------------------------------------- /src/cc/meta/IdempotentRequestTracker.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2015/01/27 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2015 Quantcast Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // Class to keep track of idempotent requests. 24 | // 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #ifndef META_IDEMPOTENT_REQUEST_H 29 | #define META_IDEMPOTENT_REQUEST_H 30 | 31 | #include "common/kfstypes.h" 32 | 33 | #include 34 | 35 | namespace KFS 36 | { 37 | 38 | using std::ostream; 39 | 40 | struct MetaIdempotentRequest; 41 | struct MetaAck; 42 | class Properties; 43 | 44 | class IdempotentRequestTracker 45 | { 46 | public: 47 | IdempotentRequestTracker(); 48 | ~IdempotentRequestTracker(); 49 | 50 | void SetParameters( 51 | const char* inPrefixPtr, 52 | const Properties& inProps); 53 | bool Handle( 54 | MetaIdempotentRequest& inRequest); 55 | bool Remove( 56 | MetaIdempotentRequest& inRequest); 57 | void Handle( 58 | MetaAck& inAck); 59 | int Write( 60 | ostream& inStream) const; 61 | int Read( 62 | const char* inPtr, 63 | size_t inLen); 64 | void Clear(); 65 | void SetDisableTimerFlag( 66 | bool inFlag); 67 | private: 68 | class Impl; 69 | Impl& mImpl; 70 | }; 71 | 72 | } // namespace KFS 73 | 74 | #endif /* META_IDEMPOTENT_REQUEST_H */ 75 | -------------------------------------------------------------------------------- /src/cc/tools/Trash.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2012/12/11 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2012,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief Trash a la hadoop. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | 28 | #include 29 | 30 | #include "FileSystem.h" 31 | 32 | namespace KFS 33 | { 34 | class Properties; 35 | 36 | namespace tools 37 | { 38 | using std::string; 39 | 40 | class Trash 41 | { 42 | public: 43 | typedef FileSystem::ErrorHandler ErrorHandler; 44 | Trash( 45 | FileSystem& inFs, 46 | const Properties& inProps, 47 | const string& inPrefix); 48 | ~Trash(); 49 | int SetParameters( 50 | const Properties& inProperties, 51 | const string& inPrefix); 52 | int MoveTo( 53 | const string& inPath, 54 | bool& outMovedFlag, 55 | string* inErrMsgPtr = 0); 56 | int Expunge( 57 | ErrorHandler* inErrorHandlerPtr = 0); 58 | bool IsEnabled() const; 59 | int RunEmptier( 60 | ErrorHandler* inErrorHandlerPtr = 0); 61 | int GetEmptierIntervalSec() const; 62 | private: 63 | class Impl; 64 | Impl& mImpl; 65 | 66 | private: 67 | Trash( 68 | const Trash& inTrash); 69 | Trash& operator=( 70 | const Trash& inTrash); 71 | }; 72 | 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/cc/common/AverageFilter.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2017/05/09 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2017 Quantcast Corp. 8 | // 9 | // This file is part of Quantcast File System (QFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // "Value" average IIR filter.. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef KFS_COMMON_AVERAGE_FILTER_H 28 | #define KFS_COMMON_AVERAGE_FILTER_H 29 | 30 | #include 31 | 32 | namespace KFS 33 | { 34 | 35 | class AverageFilter 36 | { 37 | public: 38 | enum { kAvgFracBits = 12 }; 39 | // DecayExponent = (1 << kAvgFracBits) / exp(1. / AvgIntervalSec) 40 | enum { kAvg5SecondsDecayExponent = 3353 }; 41 | enum { kAvg10SecondsDecayExponent = 3706 }; 42 | enum { kAvg15SecondsDecayExponent = 3832 }; 43 | enum { kAvg300SecondsDecayExponent = 4082 }; 44 | enum { kAvg600SecondsDecayExponent = 4089 }; 45 | enum { kAvg900SecondsDecayExponent = 4091 }; 46 | 47 | static int64_t Calculate( 48 | int64_t inAvg, 49 | int64_t inSample, 50 | int64_t inExponent) 51 | { 52 | // IIR filter 53 | const int64_t kAvgFixed_1 = int64_t(1) << kAvgFracBits; 54 | return (( 55 | inAvg * inExponent + 56 | (inSample << kAvgFracBits) * (kAvgFixed_1 - inExponent) 57 | ) >> kAvgFracBits); 58 | } 59 | }; 60 | 61 | } // namespace KFS 62 | 63 | #endif /* KFS_COMMON_AVERAGE_FILTER_H */ 64 | -------------------------------------------------------------------------------- /ext/gf-complete/examples/gf_example_5.c: -------------------------------------------------------------------------------- 1 | /* 2 | * GF-Complete: A Comprehensive Open Source Library for Galois Field Arithmetic 3 | * James S. Plank, Ethan L. Miller, Kevin M. Greenan, 4 | * Benjamin A. Arnold, John A. Burnum, Adam W. Disney, Allen C. McBride. 5 | * 6 | * gf_example_5.c 7 | * 8 | * Demonstrating altmap and extract_word 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "gf_complete.h" 19 | #include "gf_rand.h" 20 | 21 | void usage(char *s) 22 | { 23 | fprintf(stderr, "usage: gf_example_5\n"); 24 | exit(1); 25 | } 26 | 27 | int main(int argc, char **argv) 28 | { 29 | uint16_t *a, *b; 30 | int i, j; 31 | gf_t gf; 32 | 33 | if (gf_init_hard(&gf, 16, GF_MULT_SPLIT_TABLE, GF_REGION_ALTMAP, GF_DIVIDE_DEFAULT, 34 | 0, 16, 4, NULL, NULL) == 0) { 35 | fprintf(stderr, "gf_init_hard failed\n"); 36 | exit(1); 37 | } 38 | 39 | a = (uint16_t *) malloc(200); 40 | b = (uint16_t *) malloc(200); 41 | 42 | a += 6; 43 | b += 6; 44 | 45 | MOA_Seed(0); 46 | 47 | for (i = 0; i < 30; i++) a[i] = MOA_Random_W(16, 1); 48 | 49 | gf.multiply_region.w32(&gf, a, b, 0x1234, 30*2, 0); 50 | 51 | printf("a: 0x%lx b: 0x%lx\n", (unsigned long) a, (unsigned long) b); 52 | 53 | for (i = 0; i < 30; i += 10) { 54 | printf("\n"); 55 | printf(" "); 56 | for (j = 0; j < 10; j++) printf(" %4d", i+j); 57 | printf("\n"); 58 | 59 | printf("a:"); 60 | for (j = 0; j < 10; j++) printf(" %04x", a[i+j]); 61 | printf("\n"); 62 | 63 | printf("b:"); 64 | for (j = 0; j < 10; j++) printf(" %04x", b[i+j]); 65 | printf("\n"); 66 | printf("\n"); 67 | } 68 | 69 | for (i = 0; i < 15; i ++) { 70 | printf("Word %2d: 0x%04x * 0x1234 = 0x%04x ", i, 71 | gf.extract_word.w32(&gf, a, 30*2, i), 72 | gf.extract_word.w32(&gf, b, 30*2, i)); 73 | printf("Word %2d: 0x%04x * 0x1234 = 0x%04x\n", i+15, 74 | gf.extract_word.w32(&gf, a, 30*2, i+15), 75 | gf.extract_word.w32(&gf, b, 30*2, i+15)); 76 | } 77 | return 0; 78 | } 79 | -------------------------------------------------------------------------------- /scripts/rat.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # $Id$ 4 | # 5 | # Copyright 2016-2017 Quantcast Corporation. All rights reserved. 6 | # 7 | # This file is part of Quantcast File System. 8 | # 9 | # Licensed under the Apache License, Version 2.0 10 | # (the "License"); you may not use this file except in compliance with 11 | # the License. You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | # implied. See the License for the specific language governing 19 | # permissions and limitations under the License. 20 | 21 | if [ $# -ne 1 ]; then 22 | echo "usage: $0 source_dir" 23 | exit 1 24 | fi 25 | 26 | SRC="$(cd "$1" >/dev/null && pwd)" 27 | 28 | MYRAT_VERS=0.17 29 | MYURL="https://downloads.apache.org/creadur/apache-rat-${MYRAT_VERS}/apache-rat-${MYRAT_VERS}-bin.tar.gz" 30 | MYSHAURL="https://dlcdn.apache.org/creadur/apache-rat-${MYRAT_VERS}/apache-rat-${MYRAT_VERS}-bin.tar.gz.sha512" 31 | MYTAR=$(basename "$MYURL") 32 | MYNAME=$(basename "$MYTAR" -bin.tar.gz) 33 | MYJAR="$MYNAME/$MYNAME.jar" 34 | 35 | if [ -f "$MYJAR" ]; then 36 | true 37 | else 38 | rm -f "$MYTAR" 39 | if curl --retry 3 -Ss -o "$MYTAR" "$MYURL"; then 40 | MYTARSHA=$(curl --retry 3 -Ss "$MYSHAURL" | 41 | sed -e 's/^.*://' | tr -d ' \n' | tr ABCDEF abcdef) 42 | MYACTSHA=$(openssl sha512 <"$MYTAR" | sed -e 's/^.*)= *//') 43 | if [ x"$MYACTSHA" = x"$MYTARSHA" ]; then 44 | true 45 | else 46 | echo "$MYTAR: sha512 mismatch:" \ 47 | "downloaded: $MYACTSHA, expected: $MYTARSHA" 48 | rm "$MYTAR" 49 | exit 1 50 | fi 51 | else 52 | rm -f "$MYTAR" 53 | exit 1 54 | fi 55 | tar -xf "$MYTAR" 56 | status=$? 57 | rm "$MYTAR" 58 | if [ $status -ne 0 ]; then 59 | exit 60 | fi 61 | fi 62 | 63 | java -jar "$MYJAR" --output-style unapproved-licenses \ 64 | --input-exclude-file "$SRC/.ratignore" -- "$SRC" 65 | -------------------------------------------------------------------------------- /src/cc/kfsio/ZlibInflate.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2012/12/22 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2012,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief Zlib inflate. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef KFSIO_ZLIBINFLATE_H 28 | #define KFSIO_ZLIBINFLATE_H 29 | 30 | #include 31 | 32 | namespace KFS 33 | { 34 | 35 | class ZlibInflate 36 | { 37 | public: 38 | class Output 39 | { 40 | public: 41 | virtual int GetBuffer( 42 | char*& outBufferPtr, 43 | size_t& outBufferSize) = 0; 44 | virtual int Write( 45 | const char* inBufferPtr, 46 | size_t inBufferSize) = 0; 47 | protected: 48 | Output() 49 | {} 50 | virtual ~Output() 51 | {} 52 | }; 53 | ZlibInflate(); 54 | ~ZlibInflate(); 55 | int Run( 56 | const char* inBufferPtr, 57 | size_t inBufferSize, 58 | Output& inOutput, 59 | bool& outDoneFlag); 60 | void Reset(); 61 | const char* StrError( 62 | int inStatus); 63 | private: 64 | class Impl; 65 | 66 | Impl& mImpl; 67 | private: 68 | ZlibInflate( 69 | const ZlibInflate& inInflate); 70 | ZlibInflate& operator=( 71 | const ZlibInflate& inInflate); 72 | }; 73 | 74 | } 75 | #endif /* KFSIO_ZLIBINFLATE_H */ 76 | -------------------------------------------------------------------------------- /src/cc/tools/kfschgrp.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2012/06/14 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2012,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief chgrp. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #include "kfsshell.h" 28 | 29 | #include "libclient/KfsClient.h" 30 | 31 | #include 32 | #include 33 | #include 34 | 35 | namespace KFS { 36 | namespace tools { 37 | 38 | using std::cout; 39 | using std::vector; 40 | using std::string; 41 | 42 | int 43 | handleChgrp(KfsClient* kfsClient, const vector& args) 44 | { 45 | bool recursFlag = false; 46 | if (args.size() < 2 || args[0] == "--help" || 47 | ((recursFlag = args[0] == "-R") && args.size() < 3)) { 48 | cout << "Usage: chgrp [-R] " << "\n"; 49 | return 0; 50 | } 51 | 52 | size_t i = recursFlag ? 1 : 0; 53 | const char* const gname = args[i].c_str(); 54 | for (++i ; i < args.size(); i++) { 55 | const char* const name = args[i].c_str(); 56 | const int res = recursFlag ? 57 | kfsClient->ChownR(name, 0, gname) : 58 | kfsClient->Chown(name, 0, gname); 59 | if (res != 0) { 60 | cout << name << ": " << ErrorCodeToStr(res) << "\n"; 61 | return res; 62 | } 63 | } 64 | return 0; 65 | } 66 | 67 | } 68 | } 69 | 70 | -------------------------------------------------------------------------------- /src/cc/common/Show.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id$ 3 | // 4 | // Created 2013/9/8 5 | // Author: Mike Ovsiannikov 6 | // 7 | // Copyright 2013,2016 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // Invoke object's Show() method with ostream. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef COMMON_SHOW_H 28 | #define COMMON_SHOW_H 29 | 30 | #include 31 | 32 | namespace KFS 33 | { 34 | using std::ostream; 35 | 36 | template 37 | class ShowObjectT 38 | { 39 | public: 40 | ShowObjectT( 41 | const ObjT& inTarget) 42 | : mTarget(inTarget) 43 | {} 44 | StreamT& Show( 45 | StreamT& inStream) const 46 | { return mTarget.Show(inStream); } 47 | private: 48 | const ObjT& mTarget; 49 | }; 50 | 51 | template 52 | static inline StreamT& 53 | operator << ( 54 | StreamT& inStream, 55 | const ShowObjectT& inShow) 56 | { return inShow.Show(inStream); } 57 | 58 | template 59 | static inline ShowObjectT 60 | Show(const ObjT& inObj, StreamT* /* inNullArgPtr */) 61 | { return ShowObjectT(inObj); } 62 | 63 | template 64 | static inline ShowObjectT 65 | Show(const ObjT& inObj) 66 | { return ShowObjectT(inObj); } 67 | 68 | } // namespace KFS 69 | 70 | #endif /* COMMON_SHOW_H */ 71 | -------------------------------------------------------------------------------- /ext/dns/src/zone.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================== 2 | * zone.h - RFC 1035 Master File Parser 3 | * -------------------------------------------------------------------------- 4 | * Copyright (c) 2010 William Ahern 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a 7 | * copy of this software and associated documentation files (the 8 | * "Software"), to deal in the Software without restriction, including 9 | * without limitation the rights to use, copy, modify, merge, publish, 10 | * distribute, sublicense, and/or sell copies of the Software, and to permit 11 | * persons to whom the Software is furnished to do so, subject to the 12 | * following conditions: 13 | * 14 | * The above copyright notice and this permission notice shall be included 15 | * in all copies or substantial portions of the Software. 16 | * 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 18 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN 20 | * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 21 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 22 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 23 | * USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | * ========================================================================== 25 | */ 26 | #ifndef ZONE_H 27 | #define ZONE_H 28 | 29 | #include /* size_t */ 30 | #include /* FILE */ 31 | 32 | #include "dns.h" 33 | 34 | 35 | struct zonerr { 36 | char name[DNS_D_MAXNAME + 1]; 37 | enum dns_class class; 38 | enum dns_type type; 39 | unsigned ttl; 40 | union dns_any data; 41 | }; /* struct zonerr */ 42 | 43 | 44 | struct zonefile; 45 | 46 | struct zonefile *zone_open(const char *, unsigned, int *); 47 | 48 | void zone_close(struct zonefile *); 49 | 50 | size_t zone_parsesome(struct zonefile *, const void *, size_t); 51 | 52 | size_t zone_parsefile(struct zonefile *, FILE *); 53 | 54 | struct zonerr *zone_getrr(struct zonerr *, struct dns_soa **, struct zonefile *); 55 | 56 | 57 | #endif /* ZONE_H */ 58 | -------------------------------------------------------------------------------- /benchmarks/mstress/mstress_cleanup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # 4 | # $Id$ 5 | # 6 | # Copyright 2012,2016 Quantcast Corporation. All rights reserved. 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | # use this file except in compliance with the License. You may obtain a copy 10 | # of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | # License for the specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | # This is a helper script to cleanup the planfile and the logs from all 21 | # participating hosts. 22 | # 23 | 24 | from __future__ import print_function 25 | 26 | import sys 27 | 28 | if sys.version_info < (3, 0): 29 | from commands import getoutput 30 | else: 31 | from subprocess import getoutput 32 | 33 | if len(sys.argv) < 2 or sys.argv[1].startswith("-"): 34 | print( 35 | ( 36 | "Usage: %s \n" 37 | "This will cleanup the planfile and the logs from all" 38 | " participating hosts." 39 | ) 40 | % sys.argv[0] 41 | ) 42 | sys.exit(0) 43 | 44 | if not sys.argv[1].startswith("/tmp"): 45 | print("Planfile is typically in the /tmp directory. Are you sure?") 46 | sys.exit(1) 47 | 48 | planFile = sys.argv[1] 49 | hostsList = None 50 | f = None 51 | 52 | try: 53 | f = open(planFile, "r") 54 | except IOError as e: 55 | print("Planfile %s: %s" % (planFile, str(e))) 56 | sys.exit(1) 57 | 58 | for line in f: 59 | if line.startswith("#"): 60 | continue 61 | if line.startswith("hostslist="): 62 | hostsList = line[len("hostslist=") :].strip().split(",") 63 | break 64 | f.close() 65 | 66 | if len(hostsList) == 0: 67 | print("No hosts list found in plan file. Exiting.") 68 | sys.exit(1) 69 | 70 | for host in hostsList: 71 | cmd = 'ssh %s "rm -f %s*"' % (host, planFile) 72 | print('Executing "%s"' % cmd) 73 | print(getoutput(cmd)) 74 | 75 | print("Done") 76 | -------------------------------------------------------------------------------- /ext/gf-complete/examples/gf_example_3.c: -------------------------------------------------------------------------------- 1 | /* 2 | * GF-Complete: A Comprehensive Open Source Library for Galois Field Arithmetic 3 | * James S. Plank, Ethan L. Miller, Kevin M. Greenan, 4 | * Benjamin A. Arnold, John A. Burnum, Adam W. Disney, Allen C. McBride. 5 | * 6 | * gf_example_3.c 7 | * 8 | * Identical to example_2 except it works in GF(2^64) 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "gf_complete.h" 19 | #include "gf_rand.h" 20 | 21 | void usage(char *s) 22 | { 23 | fprintf(stderr, "usage: gf_example_3\n"); 24 | exit(1); 25 | } 26 | 27 | int main(int argc, char **argv) 28 | { 29 | uint64_t a, b, c; 30 | uint64_t *r1, *r2; 31 | int i; 32 | gf_t gf; 33 | 34 | if (argc != 1) usage(NULL); 35 | 36 | /* Get two random numbers in a and b */ 37 | 38 | MOA_Seed(time(0)); 39 | a = MOA_Random_64(); 40 | b = MOA_Random_64(); 41 | 42 | /* Create the proper instance of the gf_t object using defaults: */ 43 | 44 | gf_init_easy(&gf, 64); 45 | 46 | /* And multiply a and b using the galois field: */ 47 | 48 | c = gf.multiply.w64(&gf, a, b); 49 | printf("%llx * %llx = %llx\n", (long long unsigned int) a, (long long unsigned int) b, (long long unsigned int) c); 50 | 51 | /* Divide the product by a and b */ 52 | 53 | printf("%llx / %llx = %llx\n", (long long unsigned int) c, (long long unsigned int) a, (long long unsigned int) gf.divide.w64(&gf, c, a)); 54 | printf("%llx / %llx = %llx\n", (long long unsigned int) c, (long long unsigned int) b, (long long unsigned int) gf.divide.w64(&gf, c, b)); 55 | 56 | r1 = (uint64_t *) malloc(32); 57 | r2 = (uint64_t *) malloc(32); 58 | 59 | r1[0] = b; 60 | 61 | for (i = 1; i < 4; i++) r1[i] = MOA_Random_64(); 62 | 63 | gf.multiply_region.w64(&gf, r1, r2, a, 32, 0); 64 | 65 | printf("\nmultiply_region by %llx\n\n", (long long unsigned int) a); 66 | printf("R1 (the source): "); 67 | for (i = 0; i < 4; i++) printf(" %016llx", (long long unsigned int) r1[i]); 68 | 69 | printf("\nR2 (the product): "); 70 | for (i = 0; i < 4; i++) printf(" %016llx", (long long unsigned int) r2[i]); 71 | printf("\n"); 72 | 73 | exit(0); 74 | } 75 | --------------------------------------------------------------------------------