├── .travis.yml
├── CMakeLists.txt
├── EC2
    ├── create_instance.py
    ├── init_docker.sh
    ├── run_graph.txt
    └── run_matrix.txt
├── LICENSE
├── Makefile
├── Makefile.SAFS
├── Makefile.common
├── README.md
├── cmake
    ├── FindLibAio.cmake
    ├── FindLibNuma.cmake
    ├── FindLibRt.cmake
    ├── Findhwloc.cmake
    └── Findstxxl.cmake
├── conf
    ├── set_affinity.sh
    ├── set_amazon_ssds.pl
    ├── set_scheduler.pl
    ├── set_ssd_params.pl
    └── set_ssds.pl
├── config.h.in
├── docker
    └── Dockerfile
├── docs
    ├── Doxyfile
    └── html
    │   └── architecture.png
├── flash-graph
    ├── CMakeLists.txt
    ├── FG_basic_types.h
    ├── FG_vector.h
    ├── FGlib.cpp
    ├── FGlib.h
    ├── Makefile
    ├── bitmap.h
    ├── conf
    │   ├── data_files.txt
    │   ├── run_bfs.txt
    │   ├── run_bfs_mem.txt
    │   ├── run_graph.txt
    │   ├── run_lp.txt
    │   ├── run_lp_mem.txt
    │   ├── run_test.txt
    │   ├── run_triangle.txt
    │   └── run_triangle_mem.txt
    ├── fg_sparse_matrix.cpp
    ├── fg_utils.cpp
    ├── fg_utils.h
    ├── graph_config.cpp
    ├── graph_config.h
    ├── graph_engine.cpp
    ├── graph_engine.h
    ├── graph_exception.h
    ├── graph_file_header.h
    ├── graph_index.h
    ├── graphlab
    │   └── cuckoo_set_pow2.hpp
    ├── in_mem_storage.cpp
    ├── in_mem_storage.h
    ├── libgraph-algs
    │   ├── CMakeLists.txt
    │   ├── Makefile
    │   ├── betweenness_centrality.cpp
    │   ├── bfs_graph.cpp
    │   ├── develop
    │   │   ├── hash_louvain.cxx
    │   │   ├── louvain.cpp
    │   │   └── vect_async_louvain.cxx
    │   ├── diameter_graph.cpp
    │   ├── directed_triangle_graph.cpp
    │   ├── fast_triangle_graph.cpp
    │   ├── k_core.cpp
    │   ├── local_scan2_graph.cpp
    │   ├── local_scan_graph.cpp
    │   ├── overlap.cpp
    │   ├── page_rank.cpp
    │   ├── save_result.h
    │   ├── scan_graph.cpp
    │   ├── scan_graph.h
    │   ├── scc.cpp
    │   ├── sem_kmeans.cpp
    │   ├── sem_kmeans.h
    │   ├── sstsg.cpp
    │   ├── topK_scan_graph.cpp
    │   ├── triangle_shared.h
    │   ├── undirected_triangle_graph.cpp
    │   └── wcc.cpp
    ├── load_balancer.cpp
    ├── load_balancer.h
    ├── matrix
    │   ├── CMakeLists.txt
    │   ├── Makefile
    │   ├── kmeans.cpp
    │   └── kmeans.h
    ├── message_processor.cpp
    ├── message_processor.h
    ├── messaging.cpp
    ├── messaging.h
    ├── partitioner.cpp
    ├── partitioner.h
    ├── release-test
    │   ├── add_rand_weight.pl
    │   ├── data_files.txt
    │   ├── large_data_files.txt
    │   ├── run_release_test.sh
    │   ├── run_test.txt
    │   ├── run_test_large.txt
    │   ├── verify.kmeans.R
    │   └── verify_graphs.sh
    ├── scan_pointer.h
    ├── steal_state.h
    ├── test-algs
    │   ├── CMakeLists.txt
    │   ├── Makefile
    │   └── test_algs.cpp
    ├── tools
    │   ├── CMakeLists.txt
    │   ├── Makefile
    │   ├── print_graph.cpp
    │   ├── print_ts_graph.cpp
    │   └── rmat-gen.cpp
    ├── trace_logger.h
    ├── ts_graph.cpp
    ├── ts_graph.h
    ├── unit-test
    │   ├── Makefile
    │   ├── test-bitmap.cpp
    │   ├── test-partitioner.cpp
    │   ├── test-sparse_matrix.cpp
    │   └── test-vertex_index.cpp
    ├── utils.cpp
    ├── utils.h
    ├── utils
    │   ├── CMakeLists.txt
    │   ├── Makefile
    │   ├── crs_header.h
    │   ├── csr2fg.cpp
    │   ├── el2fg.cpp
    │   ├── fg2crs.cpp
    │   ├── fg2fm.cpp
    │   ├── fg_lcc.cpp
    │   └── sbm.cpp
    ├── vertex.cpp
    ├── vertex.h
    ├── vertex_compute.cpp
    ├── vertex_compute.h
    ├── vertex_index.cpp
    ├── vertex_index.h
    ├── vertex_index_constructor.cpp
    ├── vertex_index_constructor.h
    ├── vertex_index_reader.cpp
    ├── vertex_index_reader.h
    ├── vertex_pointer.h
    ├── vertex_program.cpp
    ├── vertex_program.h
    ├── vertex_request.h
    ├── worker_thread.cpp
    └── worker_thread.h
├── gen_doc.sh
├── get_num_nodes.sh
├── install_FlashGraphR.sh
├── install_FlashR.sh
├── install_FlashRLearn.sh
├── libsafs
    ├── CMakeLists.txt
    ├── LRU2Q.h
    ├── Makefile
    ├── NUMA_cache.h
    ├── NUMA_mapper.cpp
    ├── NUMA_mapper.h
    ├── RAID_config.cpp
    ├── RAID_config.h
    ├── aio_private.cpp
    ├── aio_private.h
    ├── aligned_allocator.h
    ├── associative_cache.cpp
    ├── associative_cache.h
    ├── cache.cpp
    ├── cache.h
    ├── cache_config.cpp
    ├── cache_config.h
    ├── comm_exception.h
    ├── common.cpp
    ├── common.h
    ├── common_c.h
    ├── comp_io_scheduler.cpp
    ├── comp_io_scheduler.h
    ├── compute_stat.h
    ├── concurrency.h
    ├── config_map.cpp
    ├── config_map.h
    ├── container.h
    ├── debugger.cpp
    ├── debugger.h
    ├── direct_comp_access.cpp
    ├── direct_comp_access.h
    ├── direct_private.cpp
    ├── direct_private.h
    ├── dirty_page_flusher.h
    ├── disk_read_thread.cpp
    ├── disk_read_thread.h
    ├── file_mapper.cpp
    ├── file_mapper.h
    ├── file_partition.h
    ├── global_cached_private.cpp
    ├── global_cached_private.h
    ├── in_mem_io.cpp
    ├── in_mem_io.h
    ├── io_interface.cpp
    ├── io_interface.h
    ├── io_request.cpp
    ├── io_request.h
    ├── log.cpp
    ├── log.h
    ├── mem_tracker.cpp
    ├── mem_tracker.h
    ├── memory_manager.cpp
    ├── memory_manager.h
    ├── messaging.cpp
    ├── messaging.h
    ├── native_file.cpp
    ├── native_file.h
    ├── parameters.cpp
    ├── parameters.h
    ├── part_global_cached_private.cpp
    ├── part_global_cached_private.h
    ├── read_private.cpp
    ├── read_private.h
    ├── remote_access.cpp
    ├── remote_access.h
    ├── safs_exception.h
    ├── safs_file.cpp
    ├── safs_file.h
    ├── safs_header.h
    ├── shadow_cell.cpp
    ├── shadow_cell.h
    ├── simple_KV_store.h
    ├── slab_allocator.cpp
    ├── slab_allocator.h
    ├── test
    │   ├── Makefile
    │   ├── conf
    │   │   ├── run_cache.txt
    │   │   ├── run_cache_real.txt
    │   │   ├── run_cache_tpcc.txt
    │   │   ├── run_cache_virt.txt
    │   │   ├── run_parted_cache_tpcc.txt
    │   │   ├── run_parted_cache_virt.txt
    │   │   ├── run_remote.txt
    │   │   ├── run_remote_real.txt
    │   │   └── run_remote_virt.txt
    │   ├── config.h
    │   ├── init_test.pl
    │   ├── test_rand_io.cpp
    │   ├── test_set.sh
    │   ├── thread_private.cpp
    │   ├── thread_private.h
    │   ├── workload-gen.cpp
    │   ├── workload-stat.cpp
    │   ├── workload.cpp
    │   └── workload.h
    ├── thread.cpp
    ├── thread.h
    ├── unit-test
    │   ├── GClock_unit_test.cpp
    │   ├── KV_store_unit_test.cpp
    │   ├── Makefile
    │   ├── SAHT_unit_test.cpp
    │   ├── SA_expand_shrink_test.cpp
    │   ├── conf
    │   │   ├── data_files.txt
    │   │   └── run_remote.txt
    │   ├── data_files.txt
    │   ├── file_mapper_unit_test.cpp
    │   ├── native_file_unit_test.cpp
    │   ├── run_test.txt
    │   ├── safs_file_unit_test.cpp
    │   ├── slab_allocator_test.cpp
    │   ├── test-NUMA_buffer.cpp
    │   ├── test-io.cpp
    │   ├── test_mem_tracker.cpp
    │   └── test_open_close.cpp
    ├── wpaio.cpp
    └── wpaio.h
├── mainpage.dox
├── matrix
    ├── CMakeLists.txt
    ├── EM_dense_matrix.cpp
    ├── EM_dense_matrix.h
    ├── EM_object.cpp
    ├── EM_object.h
    ├── EM_vector.cpp
    ├── EM_vector.h
    ├── EM_vv_store.cpp
    ├── EM_vv_store.h
    ├── IPW_matrix_store.cpp
    ├── IPW_matrix_store.h
    ├── Makefile
    ├── NUMA_dense_matrix.cpp
    ├── NUMA_dense_matrix.h
    ├── NUMA_vector.cpp
    ├── NUMA_vector.h
    ├── agg_matrix_store.cpp
    ├── agg_matrix_store.h
    ├── block_matrix.cpp
    ├── block_matrix.h
    ├── bulk_operate.cpp
    ├── bulk_operate.h
    ├── bulk_operate_ext.h
    ├── bulk_operate_impl.h
    ├── cached_matrix_store.cpp
    ├── cached_matrix_store.h
    ├── col_vec.cpp
    ├── col_vec.h
    ├── combined_matrix_store.cpp
    ├── combined_matrix_store.h
    ├── conf
    │   └── run_test.txt
    ├── cum_matrix.cpp
    ├── cum_matrix.h
    ├── data_frame.cpp
    ├── data_frame.h
    ├── data_io.cpp
    ├── data_io.h
    ├── dense_matrix.cpp
    ├── dense_matrix.h
    ├── eigensolver
    │   ├── CMakeLists.txt
    │   ├── FM_MultiVector.h
    │   ├── Makefile
    │   ├── block_dense_matrix.cpp
    │   ├── block_dense_matrix.h
    │   ├── collected_col_matrix_store.cpp
    │   ├── collected_col_matrix_store.h
    │   ├── dotp_matrix_store.h
    │   ├── eigensolver.cpp
    │   └── eigensolver.h
    ├── factor.cpp
    ├── factor.h
    ├── fm_utils.cpp
    ├── fm_utils.h
    ├── generic_hashtable.h
    ├── generic_type.cpp
    ├── generic_type.h
    ├── groupby_matrix_store.cpp
    ├── groupby_matrix_store.h
    ├── hilbert_curve.cpp
    ├── hilbert_curve.h
    ├── libmatrix-algs
    │   ├── Makefile
    │   ├── NMF.cpp
    │   ├── PageRank.cpp
    │   └── matrix_algs.h
    ├── local_matrix_store.cpp
    ├── local_matrix_store.h
    ├── local_mem_buffer.cpp
    ├── local_mem_buffer.h
    ├── local_vec_store.cpp
    ├── local_vec_store.h
    ├── local_vv_store.cpp
    ├── local_vv_store.h
    ├── mapply_matrix_store.cpp
    ├── mapply_matrix_store.h
    ├── materialize.cpp
    ├── materialize.h
    ├── matrix_config.cpp
    ├── matrix_config.h
    ├── matrix_exception.h
    ├── matrix_header.cpp
    ├── matrix_header.h
    ├── matrix_io.cpp
    ├── matrix_io.h
    ├── matrix_stats.cpp
    ├── matrix_stats.h
    ├── matrix_store.cpp
    ├── matrix_store.h
    ├── mem_matrix_store.cpp
    ├── mem_matrix_store.h
    ├── mem_vec_store.cpp
    ├── mem_vec_store.h
    ├── mem_vv_store.cpp
    ├── mem_vv_store.h
    ├── mem_worker_thread.cpp
    ├── mem_worker_thread.h
    ├── one_val_matrix_store.cpp
    ├── one_val_matrix_store.h
    ├── project_matrix_store.cpp
    ├── project_matrix_store.h
    ├── rand_gen.cpp
    ├── rand_gen.h
    ├── raw_data_array.cpp
    ├── raw_data_array.h
    ├── set_data_matrix_store.cpp
    ├── set_data_matrix_store.h
    ├── set_rc_matrix_store.cpp
    ├── set_rc_matrix_store.h
    ├── sink_matrix.cpp
    ├── sink_matrix.h
    ├── sorter.h
    ├── sparse_matrix.cpp
    ├── sparse_matrix.h
    ├── sparse_matrix_format.cpp
    ├── sparse_matrix_format.h
    ├── stl_algs.h
    ├── sub_matrix_store.cpp
    ├── sub_matrix_store.h
    ├── test
    │   ├── FG2Tpetra.cpp
    │   ├── FG2Tpetra.h
    │   ├── Makefile
    │   ├── benchmark_utils
    │   │   ├── Makefile
    │   │   ├── convert_matrix.cpp
    │   │   ├── convert_util.h
    │   │   └── create_matrix.cpp
    │   ├── crs_header.h
    │   ├── rand_mat_gen.cpp
    │   ├── test-2d_multiply.cpp
    │   ├── test-algs.cpp
    │   ├── test-anasazi_eigen.cpp
    │   ├── test-block_mv.cpp
    │   ├── test-dense_matrix.cpp
    │   ├── test-dgemm.cpp
    │   ├── test-eigen.cpp
    │   ├── test-mem_vector.cpp
    │   ├── test-mkl_eigen.cpp
    │   ├── test-mkl_multiply.cpp
    │   ├── test-sort.cpp
    │   ├── test-tpetra_MV_multiply.cpp
    │   └── test-tpetra_multiply.cpp
    ├── unit-test
    │   ├── Makefile
    │   ├── data_files.txt
    │   ├── run_test.txt
    │   ├── test-EM_matrix.cpp
    │   ├── test-EM_vector.cpp
    │   ├── test-EM_vector_vector.cpp
    │   ├── test-NUMA_dense_matrix.cpp
    │   ├── test-NUMA_vector.cpp
    │   ├── test-block_matrix.cpp
    │   ├── test-bulk_operate.cpp
    │   ├── test-data_frame.cpp
    │   ├── test-data_io.cpp
    │   ├── test-dense_matrix.cpp
    │   ├── test-hashtable.cpp
    │   ├── test-hilbert.cpp
    │   ├── test-local_matrix_store.cpp
    │   ├── test-mem_matrix_store.cpp
    │   ├── test-mem_vector.cpp
    │   ├── test-mem_vector_vector.cpp
    │   ├── test-projection.cpp
    │   ├── test-rand_gen.cpp
    │   ├── test-rounderror.cpp
    │   ├── test-sink_matrix.cpp
    │   ├── test-sorter.cpp
    │   ├── test-sparse_matrix.cpp
    │   ├── test-special_matrix_store.cpp
    │   └── test_io_gen.cpp
    ├── vec_store.cpp
    ├── vec_store.h
    ├── vector.cpp
    ├── vector.h
    ├── vector_vector.cpp
    ├── vector_vector.h
    ├── virtual_matrix_store.h
    ├── vv_store.cpp
    └── vv_store.h
├── scripts
    ├── addr2line.pl
    ├── check_threads.pl
    ├── init_ssds.pl
    ├── monitor_proc.pl
    ├── proc_stat.pl
    ├── set_ramdisk.sh
    ├── show_cpu.pl
    ├── show_multi_cpu.pl
    └── systemtap_reconstruct.pl
└── utils
    ├── CMakeLists.txt
    ├── Makefile
    └── SAFS-util.cpp


/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: cpp
 2 | dist: trusty
 3 | notifications:
 4 |   email: false
 5 | before_install:
 6 | - sudo apt-get update
 7 | - sudo apt-get install -y cmake libboost-dev libboost-system-dev libboost-filesystem-dev libnuma-dev libaio-dev libhwloc-dev libatlas-base-dev zlib1g-dev numactl
 8 | compiler:
 9 | - g++
10 | script:
11 | - mkdir -p build && cd build && cmake .. && make && cd ..
12 | - make clean; make
13 | - cd libsafs/unit-test && make test && cd ../..
14 | - cd flash-graph/unit-test && make test
15 | - cd ../release-test/ && ./run_release_test.sh
16 | - cd ../..
17 | - cd matrix/unit-test && make test && cd ../..
18 | 


--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required (VERSION 2.6)
 2 | 
 3 | project (FlashGraph)
 4 | 
 5 | include(CheckCCompilerFlag)
 6 | 
 7 | # The version number.
 8 | set (FlashGraph_VERSION_MAJOR 0)
 9 | set (FlashGraph_VERSION_MINOR 3)
10 | 
11 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/")
12 | 
13 | if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
14 | 	if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.6.3")
15 | 		message(FATAL_ERROR "Insufficient gcc version")
16 | 	endif()
17 | else()
18 | 	message(FATAL_ERROR "We require GCC")
19 | endif()
20 | 
21 | find_package(Boost REQUIRED)
22 | find_package(Threads REQUIRED)
23 | find_package(LibRt REQUIRED)
24 | 
25 | # configure a header file to pass some of the CMake settings
26 | # to the source code
27 | configure_file (
28 | 	"${PROJECT_SOURCE_DIR}/config.h.in"
29 | 	"${PROJECT_BINARY_DIR}/config.h"
30 | )
31 | 
32 | SET(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -std=c++0x -O3 -fPIC" )
33 | 
34 | execute_process(COMMAND bash ${PROJECT_SOURCE_DIR}/get_num_nodes.sh OUTPUT_VARIABLE num_nodes)
35 | message("#node: ${num_nodes}")
36 | 
37 | find_package(OpenMP)
38 | if (OPENMP_FOUND)
39 | 	set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
40 | 	set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
41 | endif()
42 | 
43 | if (${num_nodes} GREATER 1)
44 | 	find_package(hwloc)
45 | 	if (hwloc_FOUND)
46 | 		set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DUSE_HWLOC")
47 | 		set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_HWLOC")
48 | 	endif()
49 | endif()
50 | 
51 | if (${num_nodes} GREATER 1)
52 | 	find_package(LibNuma)
53 | 	if (LIBNUMA_FOUND)
54 | 		message(STATUS "Find libnuma.")
55 | 		set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DUSE_NUMA")
56 | 		set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_NUMA")
57 | 	endif()
58 | endif()
59 | 
60 | find_package(LibAio)
61 | if (LIBAIO_FOUND)
62 | 	message(STATUS "Find libaio.")
63 | 	set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DUSE_LIBAIO")
64 | 	set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_LIBAIO")
65 | endif()
66 | 
67 | check_c_compiler_flag("-mavx" HAVE_FLAG_M_AVX)
68 | if(HAVE_FLAG_M_AVX)
69 | 	set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx")
70 | 	set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx")
71 | endif()
72 | 
73 | #set(CMAKE_BUILD_TYPE Release)
74 | 
75 | # add the binary tree to the search path for include files
76 | # so that we will find TutorialConfig.h
77 | include_directories("${PROJECT_BINARY_DIR}"  libsafs flash-graph matrix)
78 | 
79 | subdirs(libsafs utils flash-graph matrix)
80 | 


--------------------------------------------------------------------------------
/EC2/create_instance.py:
--------------------------------------------------------------------------------
 1 | import boto3
 2 | import botocore
 3 | import time
 4 | 
 5 | ec2 = boto3.resource('ec2', region_name='us-east-1')
 6 | client = boto3.client('ec2')
 7 | 
 8 | # Create a security group
 9 | try:
10 |     sg = ec2.create_security_group(GroupName='jupyter', Description='EC2 for Jupyter Notebook')
11 |     response = client.authorize_security_group_ingress(GroupName='jupyter', IpPermissions=[{'PrefixListIds': [], 'UserIdGroupPairs': [], 'IpRanges': [{'CidrIp': '0.0.0.0/0'}], 'IpProtocol': 'tcp', 'Ipv6Ranges': [{'CidrIpv6': '::/0'}], 'ToPort': 8888, 'FromPort': 8888}])
12 |     print("create a security group")
13 | except botocore.exceptions.ClientError as e:
14 |     sg = client.describe_security_groups(GroupNames=['jupyter'])
15 |     print("the security group exist")
16 | 
17 | o = ec2.create_instances(ImageId='ami-622a0119', MinCount=1, MaxCount=1, InstanceType='i3.8xlarge', SecurityGroups=['jupyter'])
18 | print_res = False
19 | while (not print_res):
20 |     time.sleep(1)
21 |     for i in ec2.instances.filter(InstanceIds=[o[0].id]):
22 |         if i.public_ip_address is not None:
23 |             print("The public IP address: " + str(i.public_ip_address))
24 |             print_res = True
25 | 


--------------------------------------------------------------------------------
/EC2/init_docker.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/sh
 2 | 
 3 | dev_files=`ls /dev/nvme*n1`
 4 | i=1
 5 | rm -f /FlashX/data_files.txt
 6 | for file in $dev_files
 7 | do
 8 | 	echo $file
 9 | 	mkfs.xfs -f $file
10 | 	mkdir -p /mnt/ssd$i
11 | 	mount $file /mnt/ssd$i
12 | 	echo "0:/mnt/ssd$i" >> /FlashX/data_files.txt
13 | 	i=$((i+1))
14 | done
15 | 
16 | jupyter notebook --ip=0.0.0.0 --no-browser --allow-root
17 | 


--------------------------------------------------------------------------------
/EC2/run_graph.txt:
--------------------------------------------------------------------------------
 1 | # Parameters for SAFS
 2 | 
 3 | # How data blocks of a SAFS file are mapped to Linux filesystem.
 4 | RAID_mapping=RAID0
 5 | 
 6 | # The number of parallel I/Os is allowed in a single thread.
 7 | io_depth=64
 8 | 
 9 | # The page cache size in SAFS
10 | cache_size=512M
11 | 
12 | # The number of NUMA nodes
13 | # num_nodes=1
14 | 
15 | # change FG_TOP to the path of the top directory of FlashGraph.
16 | root_conf=/FlashX/data_files.txt
17 | 
18 | # whether or not to use Linux huge pages
19 | # huge_page=
20 | 
21 | # The number of I/O threads per NUMA node.
22 | # num_io_threads=1
23 | 
24 | 
25 | # Parameters for FlashGraph
26 | 
27 | # The number of worker threads to process a graph.
28 | # threads=1
29 | 
30 | # The max number of vertices being processed in parallel in a worker thread
31 | # max_processing_vertices=1000
32 | 
33 | # This defines the range size of a partition.
34 | # part_range_size_log=10
35 | 
36 | # Keep adjacency lists of vertices in memory
37 | # in_mem_graph=
38 | 
39 | # Run the vertex program on a vertx serially regardless of load balancing.
40 | serial_run=
41 | 
42 | # The number of vertical partitions on the graph.
43 | # num_vertical_parts=1
44 | 
45 | # The minimal degree that a vertex is partitioned.
46 | # min_vpart_degree=1
47 | 


--------------------------------------------------------------------------------
/EC2/run_matrix.txt:
--------------------------------------------------------------------------------
 1 | # Parameters for SAFS
 2 | 
 3 | # How data blocks of a SAFS file are mapped to Linux filesystem.
 4 | RAID_mapping=RAID0
 5 | 
 6 | # Allow users to write data to SAFS.
 7 | writable=
 8 | 
 9 | # The RAID block size in SAFS is 8MB. FlashMatrix requires a large RAID block
10 | # size.
11 | RAID_block_size=8388608
12 | 
13 | # The number of parallel I/Os is allowed in a single thread.
14 | io_depth=4
15 | 
16 | # The page cache size in SAFS
17 | cache_size=512M
18 | 
19 | # The number of NUMA nodes
20 | # num_nodes=1
21 | 
22 | # change FG_TOP to the path of the top directory of FlashGraph.
23 | root_conf=/FlashX/data_files.txt
24 | 
25 | # whether or not to use Linux huge pages
26 | # huge_page=
27 | 
28 | # The number of I/O threads per NUMA node.
29 | # num_io_threads=1
30 | 
31 | 
32 | # Parameters for FlashMatrix
33 | 
34 | # The number of worker threads to process a matrix.
35 | # threads=1
36 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 Open Connectome Project (http;//openconnecto.me)
 2 | # Written by Da Zheng (zhengda1936@gmail.com)
 3 | #
 4 | # This file is part of SAFSlib.
 5 | #
 6 | # Licensed under the Apache License, Version 2.0 (the "License");
 7 | # you may not use this file except in compliance with the License.
 8 | # You may obtain a copy of the License at
 9 | #
10 | #     http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | 
18 | include Makefile.common
19 | 
20 | all: build_lib utils
21 | 
22 | build_lib:
23 | 	$(MAKE) -C libsafs
24 | 	$(MAKE) -C matrix
25 | 	$(MAKE) -C flash-graph
26 | 
27 | utils: build_lib
28 | 	$(MAKE) -C utils
29 | 
30 | clean:
31 | 	rm -f *.d
32 | 	rm -f *.o
33 | 	rm -f *~
34 | 	rm -f include/*~
35 | 	find -name core -delete
36 | 	make --ignore-errors -C libsafs clean
37 | 	make --ignore-errors -C utils clean
38 | 	make --ignore-errors -C flash-graph clean
39 | 	make --ignore-errors -C matrix clean
40 | 
41 | -include $(DEPS) 
42 | 


--------------------------------------------------------------------------------
/Makefile.SAFS:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 Open Connectome Project (http;//openconnecto.me)
 2 | # Written by Da Zheng (zhengda1936@gmail.com)
 3 | #
 4 | # This file is part of SAFSlib.
 5 | #
 6 | # Licensed under the Apache License, Version 2.0 (the "License");
 7 | # you may not use this file except in compliance with the License.
 8 | # You may obtain a copy of the License at
 9 | #
10 | #     http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | 
18 | CC = gcc
19 | CXX = g++
20 | 
21 | all: build_lib test utils
22 | 
23 | build_lib:
24 | 	$(MAKE) -C libsafs
25 | 
26 | test: build_lib
27 | 	$(MAKE) -C test
28 | 
29 | utils: build_lib
30 | 	$(MAKE) -C utils
31 | 
32 | clean:
33 | 	rm -f *.d
34 | 	rm -f *.o
35 | 	rm -f *~
36 | 	rm -f include/*~
37 | 	find -name core -delete
38 | 	make --ignore-errors -C test clean
39 | 	make --ignore-errors -C libsafs clean
40 | 	make --ignore-errors -C utils clean
41 | 
42 | -include $(DEPS) 
43 | 


--------------------------------------------------------------------------------
/Makefile.common:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 Open Connectome Project (http;//openconnecto.me)
 2 | # Written by Da Zheng (zhengda1936@gmail.com)
 3 | #
 4 | # This file is part of SAFSlib.
 5 | #
 6 | # Licensed under the Apache License, Version 2.0 (the "License");
 7 | # you may not use this file except in compliance with the License.
 8 | # You may obtain a copy of the License at
 9 | #
10 | #     http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | 
18 | CC = gcc
19 | CXX = g++
20 | #MEMCHECK=1
21 | #PROF=1
22 | #MEMTRACE=1
23 | #BOOST_LOG=1
24 | #RELEASE=1
25 | USE_NUMA=1
26 | USE_LIBAIO=1
27 | #USE_OPENBLAS=1
28 | HWLOC=1
29 | CFLAGS = -g -O3 -DSTATISTICS -DPROFILER
30 | ifdef MEMCHECK
31 | TRACE_FLAGS = -fsanitize=address
32 | TRACE_FLAGS += -fno-omit-frame-pointer # for better stack traces in error messages
33 | TRACE_FLAGS += -fno-optimize-sibling-calls # disable tail call elimination
34 | endif
35 | ifeq ($(HWLOC), 1)
36 | CFLAGS += -DUSE_HWLOC
37 | CXXFLAGS += -DUSE_HWLOC
38 | LDFLAGS += -lhwloc
39 | endif
40 | ifeq ($(USE_LIBAIO), 1)
41 | 	LDFLAGS += -laio
42 | 	CFLAGS += -DUSE_LIBAIO
43 | 	CXXFLAGS += -DUSE_LIBAIO
44 | endif
45 | ifeq ($(USE_NUMA), 1)
46 | 	LDFLAGS += -lnuma
47 | 	CFLAGS += -DUSE_NUMA
48 | 	CXXFLAGS += -DUSE_NUMA
49 | endif
50 | ifeq ($(USE_OPENBLAS), 1)
51 | 	LDFLAGS += -lopenblas
52 | 	CFLAGS += -DUSE_OPENBLAS
53 | 	CXXFLAGS += -DUSE_OPENBLAS
54 | else
55 | 	LDFLAGS += -lcblas
56 | endif
57 | 
58 | CLANG_FLAGS = -Wno-attributes
59 | LDFLAGS += -lpthread $(TRACE_FLAGS) -rdynamic -lrt -fopenmp
60 | CXXFLAGS += -mavx -g -O3 -I. -Wall -fPIC -std=c++0x $(TRACE_FLAGS) $(CLANG_FLAGS) -DSTATISTICS -DBOOST_LOG_DYN_LINK -fopenmp
61 | ifdef PROF
62 | 	LDFLAGS +=-lprofiler
63 | 	CXXFLAGS += -DPROFILER
64 | endif
65 | ifdef BOOST_LOG
66 | 	LDFLAGS += -lboost_log
67 | 	CXXFLAGS += USE_BOOST_LOG
68 | endif
69 | ifdef MEMTRACE
70 | 	CXXFLAGS += -DENABLE_MEM_TRACE
71 | endif
72 | ifdef RELEASE
73 | 	CXXFLAGS += -DNDEBUG
74 | endif
75 | CPPFLAGS := -MD
76 | 
77 | ifdef MEMCHECK
78 | CXXFLAGS += -DMEMCHECK
79 | CC = clang
80 | CXX = clang++
81 | endif
82 | 
83 | SOURCE := $(wildcard *.c) $(wildcard *.cpp)
84 | OBJS := $(patsubst %.c,%.o,$(patsubst %.cpp,%.o,$(SOURCE)))
85 | DEPS := $(patsubst %.o,%.d,$(OBJS))
86 | MISSING_DEPS := $(filter-out $(wildcard $(DEPS)),$(DEPS))
87 | MISSING_DEPS_SOURCES := $(wildcard $(patsubst %.d,%.c,$(MISSING_DEPS)) $(patsubst %.d,%.cc,$(MISSING_DEPS)))
88 | 


--------------------------------------------------------------------------------
/cmake/FindLibAio.cmake:
--------------------------------------------------------------------------------
 1 | find_path(LIBAIO_INCLUDE_DIRS NAMES libaio.h)
 2 | find_library(LIBAIO_LIBRARIES NAMES aio)
 3 | 
 4 | include (FindPackageHandleStandardArgs)
 5 | find_package_handle_standard_args(LibAio DEFAULT_MSG
 6 |   LIBAIO_LIBRARIES
 7 |   LIBAIO_INCLUDE_DIRS)
 8 | 
 9 | mark_as_advanced(LIBAIO_INCLUDE_DIRS LIBAIO_LIBRARIES)
10 | 


--------------------------------------------------------------------------------
/cmake/FindLibNuma.cmake:
--------------------------------------------------------------------------------
 1 | #if (LIBNUMA_LIBRARIES AND LIBNUMA_INCLUDE_DIRS)
 2 | #  set (LibNuma_FIND_QUIETLY TRUE)
 3 | #endif (LIBNUMA_LIBRARIES AND LIBNUMA_INCLUDE_DIRS)
 4 | 
 5 | find_path(LIBNUMA_INCLUDE_DIRS NAMES numa.h)
 6 | find_library(LIBNUMA_LIBRARIES NAMES numa)
 7 | 
 8 | include (FindPackageHandleStandardArgs)
 9 | find_package_handle_standard_args(LibNuma DEFAULT_MSG
10 |   LIBNUMA_LIBRARIES
11 |   LIBNUMA_INCLUDE_DIRS)
12 | 
13 | mark_as_advanced(LIBNUMA_INCLUDE_DIRS LIBNUMA_LIBRARIES)
14 | 


--------------------------------------------------------------------------------
/cmake/FindLibRt.cmake:
--------------------------------------------------------------------------------
 1 | # Locate librt library
 2 | # This module defines
 3 | #  LIBRT_FOUND - System has librt
 4 | #  LIBRT_LIBRARIES - The librt library
 5 | 
 6 | find_library(LIBRT_LIBRARIES rt)
 7 | 
 8 | include(FindPackageHandleStandardArgs)
 9 | # Sets LIBRT_FOUND
10 | find_package_handle_standard_args(LibRt DEFAULT_MSG LIBRT_LIBRARIES)
11 | 
12 | mark_as_advanced(LIBRT_LIBRARIES)
13 | 


--------------------------------------------------------------------------------
/cmake/Findhwloc.cmake:
--------------------------------------------------------------------------------
 1 | # Copyright (c) 2012 Marwan Abdellah <marwan.abdellah@epfl.ch>
 2 | #                    Daniel Nachbaur <daniel.nachbaur@epfl.ch>
 3 | #               2013 Stefan.Eilemann@epfl.ch
 4 | 
 5 | # Use pkg-config to fetch the contents of the .pc file
 6 | # After that, use the directories refer to the libraries and
 7 | # also the headers
 8 | 
 9 | if(NOT PKGCONFIG_FOUND)
10 |   find_package(PkgConfig QUIET)
11 | endif()
12 | 
13 | if(HWLOC_ROOT)
14 |   set(ENV{PKG_CONFIG_PATH} "${HWLOC_ROOT}/lib/pkgconfig")
15 | else()
16 |   foreach(PREFIX ${CMAKE_PREFIX_PATH})
17 |     set(PKG_CONFIG_PATH "${PKG_CONFIG_PATH}:${PREFIX}/lib/pkgconfig")
18 |   endforeach()
19 |   set(ENV{PKG_CONFIG_PATH} "${PKG_CONFIG_PATH}:$ENV{PKG_CONFIG_PATH}")
20 | endif()
21 | 
22 | if(hwloc_FIND_REQUIRED)
23 |   set(_hwloc_OPTS "REQUIRED")
24 | endif()
25 | if(hwloc_FIND_QUIETLY)
26 |   set(_hwloc_OPTS "QUIET")
27 | endif()
28 | if(hwloc_FIND_REQUIRED AND hwloc_FIND_QUIETLY)
29 |   set(_hwloc_OPTS "REQUIRED QUIET")
30 | endif()
31 | if(NOT hwloc_FIND_QUIETLY)
32 |   set(_hwloc_output 1)
33 | endif()
34 | 
35 | find_path(hwloc_INCLUDE_DIR NAMES hwloc.h)
36 | find_library(hwloc_LIBRARIES NAMES hwloc)
37 | 
38 |   include(FindPackageHandleStandardArgs)
39 |   find_package_handle_standard_args(hwloc DEFAULT_MSG hwloc_LIBRARIES
40 |     hwloc_INCLUDE_DIR)
41 | 
42 |   if(NOT ${hwloc_VERSION} VERSION_LESS 1.7.0)
43 |     set(hwloc_GL_FOUND 1)
44 |   endif()
45 | 
46 |   if(_hwloc_output)
47 |     message(STATUS
48 |       "Found hwloc ${hwloc_VERSION} in ${hwloc_INCLUDE_DIR}:${hwloc_LIBRARIES}")
49 |   endif()
50 | 


--------------------------------------------------------------------------------
/cmake/Findstxxl.cmake:
--------------------------------------------------------------------------------
 1 | find_path(LIBSTXXL_INCLUDE_DIRS NAMES stxxl.h)
 2 | find_library(LIBSTXXL_LIBRARIES NAMES stxxl)
 3 | 
 4 | include (FindPackageHandleStandardArgs)
 5 | find_package_handle_standard_args(Libstxxl DEFAULT_MSG
 6 | 	LIBSTXXL_LIBRARIES
 7 | 	LIBSTXXL_INCLUDE_DIRS)
 8 | 
 9 | if (LIBSTXXL_INCLUDE_DIRS)
10 | 	set(STXXL_FOUND TRUE)
11 | endif()
12 | 
13 | mark_as_advanced(LIBSTXXL_INCLUDE_DIRS LIBSTXXL_LIBRARIES)
14 | 


--------------------------------------------------------------------------------
/conf/set_affinity.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | ctl_pattern="mpt.sas"
 4 | num_cpus=`grep processor /proc/cpuinfo | wc -l`
 5 | num_nodes=`grep "physical id" /proc/cpuinfo | awk 'BEGIN{max=0} {if (max < $4) max=$4} END{print max}'`
 6 | num_nodes=$((num_nodes + 1))
 7 | echo "There are ${num_cpus} CPUs in ${num_nodes} NUMA nodes"
 8 | 
 9 | set_affinity_node ()
10 | {
11 | 	start_cpu=$1
12 | 	ctl_num=$2
13 | 	irqs=`grep ${ctl_pattern}${ctl_num}-msix /proc/interrupts | awk '{print $1}' | awk -F : '{print $1}'`
14 | 	i=0
15 | 	for irq in $irqs
16 | 	do
17 | 		cpu_num=$(($start_cpu + $i * $num_nodes))
18 | 		irq_num=$irq
19 | 		i=$(($i + 1))
20 | 		if [ $cpu_num -ge $num_cpus ] ; then
21 | 			break
22 | 		fi
23 | 		echo $cpu_num > /proc/irq/$irq_num/smp_affinity_list
24 | 		echo "set $irq_num to $cpu_num"
25 | 		cat /proc/irq/$irq_num/smp_affinity_list
26 | 	done
27 | }
28 | 
29 | if [ -f /etc/init.d/irq_balancer ] ; then
30 |     /etc/init.d/irq_balancer stop
31 | elif [ -f /etc/init.d/irqbalance ]; then
32 |     /etc/init.d/irqbalance stop
33 | else
34 |     exit 1;
35 | fi;
36 | 
37 | ctl_num=0
38 | node_id=0
39 | while [ 0 ]
40 | do
41 | 	irq=`grep ${ctl_pattern}${ctl_num}-msix0 /proc/interrupts`
42 | 	if [ $? == 1 ]
43 | 	then
44 | 		break
45 | 	fi
46 | 
47 | 	echo "set for controller ${ctl_num}"
48 | 	set_affinity_node $node_id $ctl_num
49 | 
50 | 	ctl_num=$((ctl_num+1))
51 | 	node_id=$ctl_num
52 | done
53 | 


--------------------------------------------------------------------------------
/conf/set_amazon_ssds.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl
 2 | 
 3 | use strict;
 4 | 
 5 | # this file mounts SSDs to their corresponding directories and sets
 6 | # up data files correctly.
 7 | 
 8 | my $num_args = @ARGV;
 9 | print "$num_args\n";
10 | if ($num_args < 1) {
11 | 	print STDERR "set_ssds dev_list\n";
12 | 	exit 1;
13 | }
14 | 
15 | open(my $dev_fh, "<", $ARGV[0]) or die "open $ARGV[0]: $!";
16 | my @dev_names;
17 | while (<$dev_fh>) {
18 | 	chomp($_);
19 | 	if ($_ eq "") {
20 | 		next;
21 | 	}
22 | 	push(@dev_names, $_);
23 | 	print "ssd: $_\n";
24 | }
25 | my $num_devs = @dev_names;
26 | print "There are $num_devs SSDs\n";
27 | 
28 | sub get_main_devname {
29 | 	my $dev = $_;
30 | 	if ($dev =~ /(xvd[a-z]+)[0-9]+/) {
31 | 		return $1;
32 | 	}
33 | 	else {
34 | 		return $dev;
35 | 	}
36 | }
37 | 
38 | my $dev_idx = 1;
39 | open(my $fh, ">", "conf/data_files.txt");
40 | for (@dev_names) {
41 | 	my $dev = $_;
42 | 	my $main_name = get_main_devname($dev);
43 | 	my $data_file = "0:/mnt/ssd${dev_idx}/";
44 | 	print $fh "$data_file\n";
45 | 
46 | 	my $dev_file = "/dev/${dev}";
47 | 	my $mount_dir = "/mnt/ssd${dev_idx}";
48 | 	print "dev: $dev_file, dir: $mount_dir\n" ;
49 | 	system("mkdir -p $mount_dir");
50 | 	system("mkfs.xfs /dev/$dev");
51 | 	system("mount $dev_file $mount_dir");
52 | 	system("chown -R ubuntu $mount_dir");
53 | 	system("echo noop > /sys/block/${main_name}/queue/scheduler");
54 | 	system("cat /sys/block/${main_name}/queue/scheduler");
55 | 	system("echo 2 > /sys/block/${main_name}/queue/rq_affinity");
56 | 	system("echo 0 > /sys/block/${main_name}/queue/add_random");
57 | 	$dev_idx++;
58 | }
59 | 
60 | close $fh;
61 | 
62 | $dev_idx--;
63 | print "There are $dev_idx devices\n";
64 | 


--------------------------------------------------------------------------------
/conf/set_scheduler.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl
 2 | 
 3 | use strict;
 4 | 
 5 | my @dev_details = `ls -l /sys/block/ | grep sd | grep -v sda`;
 6 | 
 7 | my @dev_names;
 8 | while (@dev_details) {
 9 | 	my $item = shift(@dev_details);
10 | 	if ($item =~ /sd([b-z])$/) {
11 | 		my $dev = $1;
12 | 		# get all devices
13 | 		push(@dev_names, $dev);
14 | 		print "dev: $dev\n";
15 | 	}
16 | }
17 | 
18 | my $scheduler = "noop";
19 | 
20 | my $num_devs = @dev_names;
21 | for (my $i = 1; $i <= $num_devs; $i++) {
22 | 	my $dev_idx = $i - 1;
23 | 	my $dev_file = "/dev/sd$dev_names[$dev_idx]";
24 | 	system("echo $scheduler > /sys/block/sd$dev_names[$dev_idx]/queue/scheduler");
25 | 	system("cat /sys/block/sd$dev_names[$dev_idx]/queue/scheduler");
26 | 	system("echo 2 > /sys/block/sd$dev_names[$dev_idx]/queue/rq_affinity");
27 | 	system("cat /sys/block/sd$dev_names[$dev_idx]/queue/rq_affinity");
28 | }
29 | 
30 | 


--------------------------------------------------------------------------------
/conf/set_ssd_params.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl
 2 | 
 3 | use strict;
 4 | 
 5 | # This is optimized for sequential I/O on SSDs.
 6 | # This block size is for OCZ SSDs. We need to set the size differently
 7 | # for different SSDs.
 8 | my $max_block_size = 8192;
 9 | 
10 | # this file mounts SSDs to their corresponding directories and sets
11 | # up data files correctly.
12 | 
13 | my $num_args = @ARGV;
14 | print "$num_args\n";
15 | if ($num_args < 1) {
16 | 	print STDERR "set_ssds dev_list\n";
17 | 	exit 1;
18 | }
19 | 
20 | open(my $dev_fh, "<", $ARGV[0]) or die "open $ARGV[0]: $!";
21 | my @dev_names;
22 | while (<$dev_fh>) {
23 | 	chomp($_);
24 | 	if ($_ eq "") {
25 | 		next;
26 | 	}
27 | 	push(@dev_names, $_);
28 | 	print "ssd: $_\n";
29 | }
30 | my $num_devs = @dev_names;
31 | print "There are $num_devs SSDs\n";
32 | 
33 | sub get_main_devname {
34 | 	my $dev = $_;
35 | 	if ($dev =~ /(sd[a-z]+)[0-9]+/) {
36 | 		return $1;
37 | 	}
38 | 	else {
39 | 		return $dev;
40 | 	}
41 | }
42 | 
43 | foreach (@dev_names) {
44 | 	my $dev_name = $_;
45 | 	my $main_name = get_main_devname($_);
46 | 	# sda is the root disk.
47 | 	if ($main_name eq "sda") {
48 | 		next;
49 | 	}
50 | 
51 | 	system("echo noop > /sys/block/${main_name}/queue/scheduler");
52 | 	system("echo 2 > /sys/block/${main_name}/queue/rq_affinity");
53 | 	system("echo 0 > /sys/block/${main_name}/queue/add_random");
54 | 	system("echo $max_block_size > /sys/block/${main_name}/queue/max_sectors_kb");
55 | }
56 | 


--------------------------------------------------------------------------------
/conf/set_ssds.pl:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/perl
  2 | 
  3 | use strict;
  4 | 
  5 | # This is optimized for sequential I/O on SSDs.
  6 | my $max_block_size = 4096;
  7 | 
  8 | # this file mounts SSDs to their corresponding directories and sets
  9 | # up data files correctly.
 10 | 
 11 | my $num_args = @ARGV;
 12 | print "$num_args\n";
 13 | if ($num_args < 1) {
 14 | 	print STDERR "set_ssds dev_list\n";
 15 | 	exit 1;
 16 | }
 17 | 
 18 | open(my $dev_fh, "<", $ARGV[0]) or die "open $ARGV[0]: $!";
 19 | my @dev_names;
 20 | while (<$dev_fh>) {
 21 | 	chomp($_);
 22 | 	if ($_ eq "") {
 23 | 		next;
 24 | 	}
 25 | 	push(@dev_names, $_);
 26 | 	print "ssd: $_\n";
 27 | }
 28 | my $num_devs = @dev_names;
 29 | print "There are $num_devs SSDs\n";
 30 | 
 31 | my %devices;
 32 | my %host_ids;
 33 | 
 34 | sub get_main_devname {
 35 | 	my $dev = $_;
 36 | 	if ($dev =~ /(sd[a-z]+)[0-9]+/) {
 37 | 		return $1;
 38 | 	}
 39 | 	else {
 40 | 		return $dev;
 41 | 	}
 42 | }
 43 | 
 44 | foreach (@dev_names) {
 45 | 	my $dev_name = $_;
 46 | 	my $main_name = get_main_devname($_);
 47 | 	my $item = `ls -l /sys/block/ | grep $main_name`;
 48 | 	if ($item =~ /host([0-9]+).*sd([a-z]+)$/) {
 49 | 		# sda is the root disk.
 50 | 		if ($2 eq "a") {
 51 | 			next;
 52 | 		}
 53 | 		my $host = $1;
 54 | 		my $dev = $dev_name;
 55 | 
 56 | 		# get all devices
 57 | 		$devices{$dev} = $host;
 58 | 		# get all host Ids.
 59 | 		$host_ids{$host} = 1;
 60 | 	}
 61 | }
 62 | 
 63 | # We assign a PCI bus to a NUMA node.
 64 | # I hope there are few PCI buses than NUMA nodes.
 65 | my $node_id = 0;
 66 | for (sort {$a <=> $b} keys %host_ids) {
 67 | 	$host_ids{$_} = $node_id;
 68 | 	print "host $_ is assigned to node $node_id\n";
 69 | 	$node_id++;
 70 | }
 71 | 
 72 | my $dev_idx = 1;
 73 | open(my $fh, ">", "conf/data_files.txt") or die "can't open conf/data_files.txt";
 74 | for (sort keys %devices) {
 75 | 	my $dev = $_;
 76 | 	my $main_name = get_main_devname($dev);
 77 | 	my $host = $devices{$dev};
 78 | 	my $node_id = $host_ids{$host};
 79 | 
 80 | 	my $data_file = "$node_id:/mnt/ssd${dev_idx}/";
 81 | 	print $fh "$data_file\n";
 82 | 
 83 | 	my $dev_file = "/dev/${dev}";
 84 | 	my $mount_dir = "/mnt/ssd${dev_idx}";
 85 | 	print "dev: $dev_file, dir: $mount_dir, on node $node_id\n" ;
 86 | 	system("mkdir -p $mount_dir");
 87 | 	system("mount $dev_file $mount_dir");
 88 | #	system("chown -R zhengda.zhengda $mount_dir");
 89 | 	system("echo noop > /sys/block/${main_name}/queue/scheduler");
 90 | 	system("cat /sys/block/${main_name}/queue/scheduler");
 91 | 	system("echo 2 > /sys/block/${main_name}/queue/rq_affinity");
 92 | 	system("echo 0 > /sys/block/${main_name}/queue/add_random");
 93 | 	system("echo $max_block_size > /sys/block/${main_name}/queue/max_sectors_kb");
 94 | 	$dev_idx++;
 95 | }
 96 | 
 97 | close $fh;
 98 | 
 99 | $dev_idx--;
100 | print "There are $dev_idx devices\n";
101 | 
102 | 


--------------------------------------------------------------------------------
/config.h.in:
--------------------------------------------------------------------------------
1 | // the configured options and settings for Tutorial
2 | #define FlashGraph_VERSION_MAJOR @FlashGraph_VERSION_MAJOR@
3 | #define FlashGraph_VERSION_MINOR @FlashGraph_VERSION_MINOR@
4 | 


--------------------------------------------------------------------------------
/docs/html/architecture.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/flashxio/FlashX/2a649ffad07238344942ca7f7ae3d6c1d784b70d/docs/html/architecture.png


--------------------------------------------------------------------------------
/flash-graph/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | project (FlashGraph)
 2 | 
 3 | include_directories("${PROJECT_SOURCE_DIR}")
 4 | 
 5 | add_library(graph STATIC
 6 | 	FGlib.cpp
 7 | 	graph_engine.cpp
 8 | 	in_mem_storage.cpp
 9 | 	load_balancer.cpp
10 | 	message_processor.cpp
11 | 	messaging.cpp
12 | 	partitioner.cpp
13 | 	ts_graph.cpp
14 | 	vertex_compute.cpp
15 | 	vertex.cpp
16 | 	vertex_index.cpp
17 | 	vertex_index_reader.cpp
18 | 	worker_thread.cpp
19 | 	vertex_program.cpp
20 | 	utils.cpp
21 | 	vertex_index_constructor.cpp
22 | 	graph_config.cpp
23 | 	fg_utils.cpp
24 | 	fg_sparse_matrix.cpp
25 | )
26 | 
27 | find_package(ZLIB)
28 | subdirs(libgraph-algs
29 | 	matrix
30 | 	test-algs
31 | 	tools
32 | )
33 | subdirs(utils)
34 | 


--------------------------------------------------------------------------------
/flash-graph/FG_basic_types.h:
--------------------------------------------------------------------------------
 1 | #ifndef __FG_BASIC_TYPES_H__
 2 | #define __FG_BASIC_TYPES_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashGraph.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include <limits.h>
24 | #include <stdlib.h>
25 | 
26 | namespace fg
27 | {
28 | 
29 | /**
30 |   * \brief Basic data types used in FlashGraph
31 | */
32 | 
33 | typedef unsigned int vsize_t; 
34 | typedef unsigned int vertex_id_t; /** Used to represent vertex IDs in graph */
35 | const vertex_id_t MAX_VERTEX_ID = UINT_MAX;
36 | const vertex_id_t INVALID_VERTEX_ID = -1;
37 | 
38 | }
39 | 
40 | #endif
41 | 


--------------------------------------------------------------------------------
/flash-graph/Makefile:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 Open Connectome Project (http://openconnecto.me)
 2 | # Written by Da Zheng (zhengda1936@gmail.com)
 3 | #
 4 | # This file is part of FlashGraph.
 5 | #
 6 | # Licensed under the Apache License, Version 2.0 (the "License");
 7 | # you may not use this file except in compliance with the License.
 8 | # You may obtain a copy of the License at
 9 | #
10 | #     http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | 
18 | include ../Makefile.common
19 | 
20 | CXXFLAGS += -I../libsafs -I../matrix
21 | 
22 | all: libgraph unit-test tools matrix libgraph-algs test-algs utils
23 | 
24 | libgraph-algs: libgraph
25 | 	$(MAKE) -C libgraph-algs
26 | 
27 | libgraph: $(OBJS)
28 | 	rm -f libgraph.a
29 | 	ar -cvq libgraph.a $(OBJS)
30 | 
31 | tools: libgraph
32 | 	$(MAKE) -C tools
33 | 
34 | unit-test: libgraph
35 | 	$(MAKE) -C unit-test
36 | 
37 | test: libgraph
38 | 	$(MAKE) -C test
39 | 
40 | matrix: libgraph
41 | 	$(MAKE) -C matrix
42 | 
43 | test-algs: libgraph libgraph-algs matrix
44 | 	$(MAKE) -C test-algs
45 | 
46 | test-matrix: libgraph matrix
47 | 	$(MAKE) -C test-matrix
48 | 
49 | utils: libgraph libgraph-algs
50 | 	$(MAKE) -C utils
51 | 
52 | clean:
53 | 	rm -f *.d
54 | 	rm -f *.o
55 | 	rm -f *~
56 | 	rm -f libgraph.a
57 | 	find -name core -delete
58 | 	$(MAKE) clean -C tools
59 | 	$(MAKE) clean -C unit-test
60 | 	$(MAKE) clean -C test
61 | 	$(MAKE) clean -C matrix
62 | 	$(MAKE) clean -C libgraph-algs
63 | 	$(MAKE) clean -C test-algs
64 | 	$(MAKE) clean -C utils
65 | 	$(MAKE) clean -C test-matrix
66 | 
67 | -include $(DEPS) 
68 | 


--------------------------------------------------------------------------------
/flash-graph/conf/data_files.txt:
--------------------------------------------------------------------------------
1 | # change FG_TOP to the location of the top directory of FlashGraph.
2 | 0:FG_TOP/flash-graph/data
3 | 


--------------------------------------------------------------------------------
/flash-graph/conf/run_bfs.txt:
--------------------------------------------------------------------------------
 1 | RAID_mapping=RAID0
 2 | io_depth=64
 3 | cache_size=1G
 4 | num_nodes=4
 5 | root_conf=conf/data_files.txt
 6 | max_num_pending_ios=5000
 7 | #huge_page=
 8 | part_range_size_log=15
 9 | max_processing_vertices=8000
10 | 
11 | threads=32
12 | prof_file=graph.prof
13 | merge_reqs=
14 | index_file_weight=50
15 | SA_cell_size=16
16 | #in_mem_graph=
17 | enable_elevator=
18 | 


--------------------------------------------------------------------------------
/flash-graph/conf/run_bfs_mem.txt:
--------------------------------------------------------------------------------
 1 | RAID_mapping=RAID0
 2 | io_depth=64
 3 | cache_size=1G
 4 | num_nodes=4
 5 | root_conf=conf/data_files.txt
 6 | max_num_pending_ios=5000
 7 | #huge_page=
 8 | part_range_size_log=15
 9 | max_processing_vertices=1000
10 | 
11 | threads=32
12 | prof_file=graph.prof
13 | merge_reqs=
14 | index_file_weight=50
15 | SA_cell_size=16
16 | in_mem_graph=
17 | #enable_elevator=
18 | 


--------------------------------------------------------------------------------
/flash-graph/conf/run_graph.txt:
--------------------------------------------------------------------------------
 1 | RAID_mapping=RAID0
 2 | io_depth=64
 3 | cache_size=4G
 4 | num_nodes=4
 5 | root_conf=conf/data_files.txt
 6 | 
 7 | threads=32
 8 | print_io_stat=
 9 | prof_file=graph.prof
10 | merge_reqs=
11 | 


--------------------------------------------------------------------------------
/flash-graph/conf/run_lp.txt:
--------------------------------------------------------------------------------
 1 | RAID_mapping=RAID0
 2 | io_depth=1024
 3 | cache_size=1G
 4 | num_nodes=4
 5 | root_conf=conf/data_files.txt
 6 | max_num_pending_ios=5000
 7 | #huge_page=
 8 | part_range_size_log=15
 9 | max_processing_vertices=8000
10 | 
11 | threads=32
12 | prof_file=graph.prof
13 | merge_reqs=
14 | index_file_weight=50
15 | SA_cell_size=16
16 | #in_mem_graph=
17 | 


--------------------------------------------------------------------------------
/flash-graph/conf/run_lp_mem.txt:
--------------------------------------------------------------------------------
 1 | RAID_mapping=RAID0
 2 | io_depth=1024
 3 | cache_size=1G
 4 | num_nodes=4
 5 | root_conf=conf/data_files.txt
 6 | max_num_pending_ios=5000
 7 | #huge_page=
 8 | part_range_size_log=15
 9 | max_processing_vertices=1000
10 | 
11 | threads=32
12 | prof_file=graph.prof
13 | merge_reqs=
14 | index_file_weight=50
15 | SA_cell_size=16
16 | in_mem_graph=
17 | 


--------------------------------------------------------------------------------
/flash-graph/conf/run_test.txt:
--------------------------------------------------------------------------------
 1 | # Parameters for SAFS
 2 | 
 3 | # How data blocks of a SAFS file are mapped to Linux filesystem.
 4 | RAID_mapping=RAID0
 5 | 
 6 | # The number of parallel I/Os is allowed in a single thread.
 7 | io_depth=64
 8 | 
 9 | # The page cache size in SAFS
10 | cache_size=512M
11 | 
12 | # The number of NUMA nodes
13 | # num_nodes=1
14 | 
15 | # change FG_TOP to the path of the top directory of FlashGraph.
16 | root_conf=FG_TOP/flash-graph/conf/data_files.txt
17 | 
18 | # whether or not to use Linux huge pages
19 | # huge_page=
20 | 
21 | # The number of I/O threads per NUMA node.
22 | # num_io_threads=1
23 | 
24 | 
25 | # Parameters for FlashGraph
26 | 
27 | # The number of worker threads to process a graph.
28 | # threads=1
29 | 
30 | # The max number of vertices being processed in parallel in a worker thread
31 | # max_processing_vertices=1000
32 | 
33 | # This defines the range size of a partition.
34 | # part_range_size_log=10
35 | 
36 | # Keep adjacency lists of vertices in memory
37 | # in_mem_graph=
38 | 
39 | # Run the vertex program on a vertx serially regardless of load balancing.
40 | serial_run=
41 | 
42 | # The number of vertical partitions on the graph.
43 | # num_vertical_parts=1
44 | 
45 | # The minimal degree that a vertex is partitioned.
46 | # min_vpart_degree=1
47 | 


--------------------------------------------------------------------------------
/flash-graph/conf/run_triangle.txt:
--------------------------------------------------------------------------------
 1 | RAID_mapping=RAID0
 2 | io_depth=64
 3 | cache_size=1G
 4 | num_nodes=4
 5 | root_conf=conf/data_files.txt
 6 | max_num_pending_ios=5000
 7 | #huge_page=
 8 | part_range_size_log=10
 9 | max_processing_vertices=1000
10 | num_vparts=4
11 | #min_vpart_degree=30
12 | min_vpart_degree=1000
13 | 
14 | threads=32
15 | prof_file=graph.prof
16 | merge_reqs=
17 | index_file_weight=50
18 | SA_cell_size=16
19 | #in_mem_graph=
20 | serial_run=
21 | 


--------------------------------------------------------------------------------
/flash-graph/conf/run_triangle_mem.txt:
--------------------------------------------------------------------------------
 1 | RAID_mapping=RAID0
 2 | io_depth=64
 3 | cache_size=1G
 4 | num_nodes=4
 5 | root_conf=conf/data_files.txt
 6 | max_num_pending_ios=5000
 7 | #huge_page=
 8 | part_range_size_log=10
 9 | max_processing_vertices=1000
10 | num_vparts=4
11 | #min_vpart_degree=30
12 | min_vpart_degree=1000
13 | 
14 | threads=32
15 | prof_file=graph.prof
16 | merge_reqs=
17 | index_file_weight=50
18 | SA_cell_size=16
19 | in_mem_graph=
20 | serial_run=
21 | 


--------------------------------------------------------------------------------
/flash-graph/graph_exception.h:
--------------------------------------------------------------------------------
 1 | #ifndef __GRAPH_EXCEPTION_H__
 2 | #define __GRAPH_EXCEPTION_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashGraph.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include <string>
24 | #include <exception>
25 | 
26 | namespace fg
27 | {
28 | 
29 | class conf_exception: public std::exception
30 | {
31 | 	std::string msg;
32 | public:
33 | 	conf_exception(const std::string &msg) {
34 | 		this->msg = msg;
35 | 	}
36 | 
37 | 	~conf_exception() throw() {
38 | 	}
39 | 
40 | 	const char* what() const throw() {
41 | 		return msg.c_str();
42 | 	}
43 | };
44 | 
45 | class wrong_format: public std::exception
46 | {
47 | 	std::string msg;
48 | public:
49 | 	wrong_format(const std::string &msg) {
50 | 		this->msg = msg;
51 | 	}
52 | 
53 | 	~wrong_format() throw() {
54 | 	}
55 | 
56 | 	const char* what() const throw() {
57 | 		return msg.c_str();
58 | 	}
59 | };
60 | 
61 | }
62 | 
63 | #endif
64 | 


--------------------------------------------------------------------------------
/flash-graph/in_mem_storage.h:
--------------------------------------------------------------------------------
 1 | #ifndef __IN_MEM_STORAGE_H__
 2 | #define __IN_MEM_STORAGE_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashGraph.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | #include <stdlib.h>
23 | 
24 | #include <string>
25 | #include <memory>
26 | 
27 | #include "io_interface.h"
28 | #include "in_mem_io.h"
29 | 
30 | class in_mem_io;
31 | class thread_safe_page;
32 | 
33 | namespace fg
34 | {
35 | 
36 | class in_mem_graph
37 | {
38 | 	size_t graph_size;
39 | 	safs::NUMA_buffer::ptr graph_data;
40 | 	int graph_file_id;
41 | 	std::string graph_file_name;
42 | 
43 | 	in_mem_graph() {
44 | 		graph_size = 0;
45 | 		graph_file_id = -1;
46 | 	}
47 | public:
48 | 	typedef std::shared_ptr<in_mem_graph> ptr;
49 | 
50 | 	static ptr create(const std::string &graph_name, std::shared_ptr<char> buf,
51 | 			size_t size);
52 | 	/*
53 | 	 * in_mem_graph takes the memory buffer and is responsible to free it
54 | 	 * afterwards.
55 | 	 */
56 | 	static ptr create(const std::string &graph_name, safs::NUMA_buffer::ptr buf,
57 | 			size_t size) {
58 | 		in_mem_graph *g = new in_mem_graph();
59 | 		assert(buf->get_length() >= size);
60 | 		g->graph_size = size;
61 | 		g->graph_data = buf;
62 | 		// TODO we don't init graph_file_id here.
63 | 		g->graph_file_name = graph_name;
64 | 		return ptr(g);
65 | 	}
66 | 
67 | 	static ptr load_graph(const std::string &graph_file);
68 | 	static ptr load_safs_graph(const std::string &graph_file);
69 | 
70 | 	void dump(const std::string &file) const;
71 | 
72 | 	const safs::NUMA_buffer &get_data() const {
73 | 		return *graph_data;
74 | 	}
75 | 	std::shared_ptr<safs::file_io_factory> create_io_factory() const;
76 | };
77 | 
78 | }
79 | 
80 | #endif
81 | 


--------------------------------------------------------------------------------
/flash-graph/libgraph-algs/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | project (FlashGraph)
 2 | 
 3 | add_library(graph-algs STATIC
 4 | 	diameter_graph.cpp
 5 | 	directed_triangle_graph.cpp
 6 | 	fast_triangle_graph.cpp
 7 | 	k_core.cpp
 8 | 	local_scan_graph.cpp
 9 | 	local_scan2_graph.cpp
10 | 	overlap.cpp
11 | 	page_rank.cpp
12 | 	scan_graph.cpp
13 | 	scc.cpp
14 | 	sstsg.cpp
15 | 	topK_scan_graph.cpp
16 | 	undirected_triangle_graph.cpp
17 | 	wcc.cpp
18 | 	bfs_graph.cpp
19 | 	betweenness_centrality.cpp
20 |     sem_kmeans.cpp
21 | )
22 | 


--------------------------------------------------------------------------------
/flash-graph/libgraph-algs/Makefile:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 Open Connectome Project (http://openconnecto.me)
 2 | # Written by Da Zheng (zhengda1936@gmail.com)
 3 | #
 4 | # This file is part of FlashGraph.
 5 | #
 6 | # Licensed under the Apache License, Version 2.0 (the "License");
 7 | # you may not use this file except in compliance with the License.
 8 | # You may obtain a copy of the License at
 9 | #
10 | #     http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | 
18 | include ../../Makefile.common
19 | 
20 | LDFLAGS := -L.. -lgraph -L../../libsafs -lsafs -lrt $(LDFLAGS)
21 | CXXFLAGS += -I../../libsafs -I.. -I. -I../../matrix
22 | 
23 | all: libgraph-algs
24 | 
25 | libgraph-algs: $(OBJS) ../../libsafs/libsafs.a ../../matrix/libFMatrix.a
26 | 	rm -f libgraph-algs.a
27 | 	ar -cvq libgraph-algs.a $(OBJS)
28 | 
29 | clean:
30 | 	rm -f *.d
31 | 	rm -f *.o
32 | 	rm -f *~
33 | 	rm -f libgraph-algs.a
34 | 	rm -f libgraph-algs.so.*
35 | 
36 | -include $(DEPS) 
37 | 


--------------------------------------------------------------------------------
/flash-graph/libgraph-algs/save_result.h:
--------------------------------------------------------------------------------
 1 | #ifndef __SAVE_RESULT_H__
 2 | #define __SAVE_RESULT_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashGraph.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include "graph_engine.h"
24 | #include "mem_vec_store.h"
25 | 
26 | namespace {
27 | 
28 | template<class T, class VertexType>
29 | class save_query: public fg::vertex_query
30 | {
31 | 	fm::detail::mem_vec_store::ptr vec;
32 | public:
33 | 	save_query(fm::detail::mem_vec_store::ptr vec) {
34 | 		this->vec = vec;
35 | 	}
36 | 
37 | 	virtual void run(fg::graph_engine &graph, fg::compute_vertex &v1) {
38 | 		VertexType &v = (VertexType &) v1;
39 | 		vec->set(graph.get_graph_index().get_vertex_id(v), v.get_result());
40 | 	}
41 | 
42 | 	virtual void merge(fg::graph_engine &graph, fg::vertex_query::ptr q) {
43 | 	}
44 | 
45 | 	virtual ptr clone() {
46 | 		return fg::vertex_query::ptr(new save_query(vec));
47 | 	}
48 | };
49 | 
50 | }
51 | 
52 | #endif
53 | 


--------------------------------------------------------------------------------
/flash-graph/load_balancer.h:
--------------------------------------------------------------------------------
 1 | #ifndef __LOAD_BALANCER_H__
 2 | #define __LOAD_BALANCER_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashGraph.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include <unordered_map>
24 | 
25 | #include "container.h"
26 | #include "vertex.h"
27 | 
28 | namespace fg
29 | {
30 | 
31 | class worker_thread;
32 | class graph_engine;
33 | class compute_vertex;
34 | class compute_vertex_pointer;
35 | 
36 | /*
37 |  * This class is to help balance the load.
38 |  * If the owner thread has finished the work originally assigned to it,
39 |  * it can steal work from other threads through this class.
40 |  */
41 | class load_balancer
42 | {
43 | 	typedef std::unordered_map<const compute_vertex *, int> vertex_map_t;
44 | 
45 | 	worker_thread &owner;
46 | 	graph_engine &graph;
47 | 
48 | 	// This map records the owner threads of vertices stolen from another
49 | 	// partition.
50 | 	vertex_map_t stolen_vertex_map;
51 | 
52 | 	// This is a local buffer that contains the completed stolen vertices.
53 | 	// All vertices here need to be returned to their owner threads.
54 | 	fifo_queue<vertex_id_t> *completed_stolen_vertices;
55 | 	int num_completed_stolen_vertices;
56 | 	// The thread where we should steal activated vertices from.
57 | 	int steal_thread_id;
58 | public:
59 | 	load_balancer(graph_engine &_graph, worker_thread &_owner);
60 | 
61 | 	~load_balancer();
62 | 
63 | 	int get_stolen_vertex_part(const compute_vertex &v) const;
64 | 
65 | 	int steal_activated_vertices(compute_vertex_pointer vertices[], int num);
66 | 	/**
67 | 	 * After the thread finishes processing the stolen vertices, it needs to
68 | 	 * return all the vertices to their owner threads.
69 | 	 */
70 | 	void return_vertices(const compute_vertex_pointer vs[], int num);
71 | 
72 | 	// This method is to return all completed stolen vertices to their owner
73 | 	// threads.
74 | 	void process_completed_stolen_vertices();
75 | 
76 | 	void reset();
77 | };
78 | 
79 | }
80 | 
81 | #endif
82 | 


--------------------------------------------------------------------------------
/flash-graph/matrix/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | project (FlashGraph)
2 | 
3 | add_library(matrix STATIC
4 | 	kmeans.cpp
5 | )
6 | 


--------------------------------------------------------------------------------
/flash-graph/matrix/Makefile:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 Open Connectome Project (http://openconnecto.me)
 2 | # Written by Da Zheng (zhengda1936@gmail.com)
 3 | #
 4 | # This file is part of FlashGraph.
 5 | #
 6 | # Licensed under the Apache License, Version 2.0 (the "License");
 7 | # you may not use this file except in compliance with the License.
 8 | # You may obtain a copy of the License at
 9 | #
10 | #     http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | 
18 | include ../../Makefile.common
19 | 
20 | OMP_FLAG = -fopenmp
21 | LDFLAGS := -L.. -lgraph -L../../libsafs -lsafs -lrt $(OMP_FLAG) $(LDFLAGS)
22 | CXXFLAGS += -I../../libsafs -I.. -I. $(OMP_FLAG)
23 | 
24 | all: libmatrix
25 | 
26 | libmatrix: $(OBJS)
27 | 	rm -f libmatrix.a
28 | 	ar -cvq libmatrix.a $(OBJS)
29 | 
30 | clean:
31 | 	rm -f *.d
32 | 	rm -f *.o
33 | 	rm -f *~
34 | 	rm -f libmatrix.a
35 | 
36 | -include $(DEPS) 
37 | 


--------------------------------------------------------------------------------
/flash-graph/matrix/kmeans.h:
--------------------------------------------------------------------------------
 1 | #ifndef __FG_KMEANS_H__
 2 | #define __FG_KMEANS_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Disa Mhembere (disa@jhu.edu)
 7 |  *
 8 |  * This file is part of FlashMatrix.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY CURRENT_KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include <stdlib.h>
24 | #include <stdio.h>
25 | #include <math.h>
26 | #include <omp.h>
27 | #include <string.h>
28 | #include <sys/time.h>
29 | #ifdef PROFILER
30 | #include <gperftools/profiler.h>
31 | #endif
32 | #include <vector>
33 | #include <limits> 
34 | #include <iostream>
35 | #include <algorithm>
36 | 
37 | #include "log.h"
38 | #include "common.h"
39 | 
40 | /**
41 |  * \brief Compute kmeans on matrix of features
42 |  * \param matrix The matrix who's row IDs are being clustered.
43 |  * \param clusters The cluster centers (means).
44 |  * \param cluster_assignments Which cluster each sample falls into.
45 |  * \param cluster_assignment_counts How many members each cluster has.
46 |  * \param num_rows The number of rows in `matrix`.
47 |  * \param nev The number of eigenvalues / number of columns in `matrix`.
48 |  * \param k The number of clusters required.
49 |  * \param max_iters The maximum number of iterations of K-means to perform.
50 |  * \param init The type of initilization ["random", "forgy", "kmeanspp"]
51 |  **/
52 | namespace fg
53 | {
54 | unsigned compute_kmeans(const double* matrix, double* clusters, 
55 | 		unsigned* cluster_assignments, unsigned* cluster_assignment_counts,
56 | 		const unsigned num_rows, const unsigned nev, const size_t k, 
57 | 		const unsigned MAX_ITERS, const int max_threads, const std::string init="kmeanspp",
58 | 		const double tolerance=-1, const std::string dist_type="eucl");
59 | }
60 | #endif
61 | 


--------------------------------------------------------------------------------
/flash-graph/message_processor.h:
--------------------------------------------------------------------------------
 1 | #ifndef __MESSAGE_PROCESSOR_H__
 2 | #define __MESSAGE_PROCESSOR_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashGraph.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include <memory>
24 | 
25 | #include "container.h"
26 | 
27 | #include "messaging.h"
28 | #include "vertex.h"
29 | 
30 | namespace fg
31 | {
32 | 
33 | class graph_engine;
34 | class worker_thread;
35 | class steal_state_t;
36 | class compute_vertex;
37 | class compute_vertex_pointer;
38 | 
39 | /*
40 |  * This class is to process the messages sent to the owner thread.
41 |  * The complexity here is that some messages can't be processed when
42 |  * we try to process them. Therefore, we need an extra buffer to keep them
43 |  * and process them later.
44 |  */
45 | class message_processor
46 | {
47 | 	graph_engine &graph;
48 | 	worker_thread &owner;
49 | 
50 | 	std::shared_ptr<slab_allocator> msg_alloc;
51 | 
52 | 	// The queue of messages sent from other threads.
53 | 	msg_queue msg_q;
54 | 
55 | 	// The thread state of handling stealing vertices.
56 | 	std::unique_ptr<steal_state_t> steal_state;
57 | 
58 | 	// This is a message buffer to keep all messages whose destination vertices
59 | 	// have been stolen by other threads.
60 | 	fifo_queue<message> stolenv_msgs;
61 | 
62 | 	void buf_msg(vertex_message &msg);
63 | 	void buf_mmsg(local_vid_t id, multicast_message &mmsg);
64 | 
65 | 	void process_msg(message &msg, bool check_steal);
66 | 	void process_multicast_msg(multicast_message &mmsg, bool check_steal);
67 | 
68 | public:
69 | 	message_processor(graph_engine &_graph, worker_thread &_owner,
70 | 			std::shared_ptr<slab_allocator> msg_alloc);
71 | 
72 | 	void process_msgs();
73 | 
74 | 	void steal_vertices(compute_vertex_pointer vertices[], int num);
75 | 	void return_vertices(vertex_id_t ids[], int num);
76 | 
77 | 	msg_queue &get_msg_queue() {
78 | 		return msg_q;
79 | 	}
80 | 
81 | 	void reset();
82 | };
83 | 
84 | }
85 | 
86 | #endif
87 | 


--------------------------------------------------------------------------------
/flash-graph/release-test/add_rand_weight.pl:
--------------------------------------------------------------------------------
1 | #!/usr/bin/perl
2 | 
3 | while(<STDIN>) {
4 | 	/([0-9]+)\s([0-9]+)/;
5 | 	$v = rand(100);
6 | 	print "$1 $2 $v\n";
7 | }
8 | 


--------------------------------------------------------------------------------
/flash-graph/release-test/data_files.txt:
--------------------------------------------------------------------------------
1 | # change FG_TOP to the location of the top directory of FlashGraph.
2 | 0:data
3 | 


--------------------------------------------------------------------------------
/flash-graph/release-test/large_data_files.txt:
--------------------------------------------------------------------------------
1 | # change FG_TOP to the location of the top directory of FlashGraph.
2 | 0:/mnt/ram0/safs_store/
3 | 1:/mnt/ram1/safs_store/
4 | 2:/mnt/ram2/safs_store/
5 | 3:/mnt/ram3/safs_store/
6 | 


--------------------------------------------------------------------------------
/flash-graph/release-test/run_release_test.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/sh
 2 | 
 3 | mkdir -p data
 4 | rm -fR data/*
 5 | 
 6 | wget http://snap.stanford.edu/data/wiki-Vote.txt.gz
 7 | gunzip wiki-Vote.txt.gz
 8 | rm -f wiki-Vote.adj*
 9 | rm -f wiki-Vote.index*
10 | ../utils/el2fg ../conf/run_test.txt wiki-Vote.txt wiki-Vote
11 | sed '/^#/d' wiki-Vote.txt > wiki-Vote1.txt
12 | rm wiki-Vote.txt
13 | 
14 | wget http://snap.stanford.edu/data/facebook_combined.txt.gz
15 | gunzip facebook_combined.txt.gz
16 | rm -f facebook.adj*
17 | rm -f facebook.index*
18 | ../utils/el2fg -u ../conf/run_test.txt facebook_combined.txt facebook
19 | sed '/^#/d' facebook_combined.txt > facebook_combined1.txt
20 | 
21 | perl add_rand_weight.pl < facebook_combined.txt > fb-weighted.txt
22 | ../utils/el2fg -u -t D ../conf/run_test.txt fb-weighted.txt fb-weighted
23 | 
24 | rm -f facebook_combined.txt
25 | rm wiki-Vote.adj*
26 | rm wiki-Vote.index*
27 | rm facebook.adj*
28 | rm facebook.index*
29 | rm wiki-Vote1.txt
30 | rm facebook_combined1.txt
31 | rm fb-weighted*
32 | 


--------------------------------------------------------------------------------
/flash-graph/release-test/run_test.txt:
--------------------------------------------------------------------------------
 1 | RAID_mapping=RAID0
 2 | io_depth=64
 3 | cache_size=512M
 4 | num_nodes=1
 5 | # change FG_TOP to the path of the top directory of FlashGraph.
 6 | root_conf=data_files.txt
 7 | #preload=
 8 | num_vparts=1
 9 | min_vpart_degree=100
10 | part_range_size_log=10
11 | max_processing_vertices=1000
12 | serial_run=
13 | 
14 | threads=4
15 | print_io_stat=
16 | prof_file=graph.prof
17 | merge_reqs=
18 | #in_mem_graph=
19 | 


--------------------------------------------------------------------------------
/flash-graph/release-test/run_test_large.txt:
--------------------------------------------------------------------------------
 1 | RAID_mapping=RAID0
 2 | io_depth=64
 3 | cache_size=1G
 4 | num_nodes=4
 5 | # change FG_TOP to the path of the top directory of FlashGraph.
 6 | root_conf=/home/zhengda/parallel-cache/user-space-cache/conf/data_files.txt
 7 | max_num_pending_ios=5000
 8 | #preload=
 9 | num_vparts=1
10 | min_vpart_degree=100
11 | part_range_size_log=15
12 | max_processing_vertices=8000
13 | 
14 | threads=32
15 | #print_io_stat=
16 | prof_file=graph.prof
17 | merge_reqs=
18 | #in_mem_graph=
19 | serial_run=
20 | 


--------------------------------------------------------------------------------
/flash-graph/release-test/verify_graphs.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | 
3 | # verify generating large graphs.
4 | 
5 | ../tools/el2al -v -w -T 32 -d /mnt/ram4/twitter.adj /mnt/ram4/twitter.index /mnt/nfs1/graph-data/twitter_rv.net.gz
6 | rm /mnt/ram4/*
7 | ../tools/el2al -u -v -w -T 32 -d /mnt/ram4/friendster.adj /mnt/ram4/friendster.index /mnt/nfs1/graph-data/com-friendster.ungraph.txt.gz
8 | rm /mnt/ram4/*
9 | 


--------------------------------------------------------------------------------
/flash-graph/scan_pointer.h:
--------------------------------------------------------------------------------
  1 | #ifndef __SCAN_POINTER_H__
  2 | #define __SCAN_POINTER_H__
  3 | 
  4 | /*
  5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
  6 |  * Written by Da Zheng (zhengda1936@gmail.com)
  7 |  *
  8 |  * This file is part of FlashGraph.
  9 |  *
 10 |  * Licensed under the Apache License, Version 2.0 (the "License");
 11 |  * you may not use this file except in compliance with the License.
 12 |  * You may obtain a copy of the License at
 13 |  *
 14 |  *     http://www.apache.org/licenses/LICENSE-2.0
 15 |  *
 16 |  * Unless required by applicable law or agreed to in writing, software
 17 |  * distributed under the License is distributed on an "AS IS" BASIS,
 18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 19 |  * See the License for the specific language governing permissions and
 20 |  * limitations under the License.
 21 |  */
 22 | 
 23 | #include <stdlib.h>
 24 | 
 25 | namespace fg
 26 | {
 27 | 
 28 | /*
 29 |  * This class helps to scan a list from the beginning to the end
 30 |  * or from the end to the beginning.
 31 |  */
 32 | struct scan_pointer
 33 | {
 34 | 	size_t range_start;
 35 | 	size_t size;
 36 | 	size_t idx;
 37 | 	bool forward;
 38 | public:
 39 | 	scan_pointer(size_t size, bool forward) {
 40 | 		this->forward = forward;
 41 | 		this->range_start = 0;
 42 | 		this->size = size;
 43 | 		if (forward)
 44 | 			idx = range_start;
 45 | 		else
 46 | 			idx = size + range_start;
 47 | 	}
 48 | 
 49 | 	size_t get_num_remaining() const {
 50 | 		if (forward)
 51 | 			return size + range_start - idx;
 52 | 		else
 53 | 			return idx - range_start;
 54 | 	}
 55 | 
 56 | 	size_t get_curr_loc() const {
 57 | 		return idx;
 58 | 	}
 59 | 
 60 | 	size_t move(size_t dist) {
 61 | 		if (forward) {
 62 | 			idx += dist;
 63 | 			idx = min(idx, size);
 64 | 		}
 65 | 		else if (idx >= dist)
 66 | 			idx -= dist;
 67 | 		else
 68 | 			idx = 0;
 69 | 		return idx;
 70 | 	}
 71 | 
 72 | 	size_t get_range_start() const {
 73 | 		return range_start;
 74 | 	}
 75 | 
 76 | 	size_t get_range_end() const {
 77 | 		return range_start + size;
 78 | 	}
 79 | 
 80 | 	size_t get_range_size() const {
 81 | 		return size;
 82 | 	}
 83 | 
 84 | 	void set_scan_dir(bool forward) {
 85 | 		if (this->forward == forward)
 86 | 			return;
 87 | 
 88 | 		size_t new_range_start;
 89 | 		size_t new_range_size;
 90 | 		if (forward) {
 91 | 			// So the original direction is backward.
 92 | 			new_range_start = get_range_start();
 93 | 			new_range_size
 94 | 				= get_curr_loc() - get_range_start();
 95 | 			idx = new_range_start;
 96 | 		}
 97 | 		else {
 98 | 			// So the original direction is forward.
 99 | 			new_range_start = get_curr_loc();
100 | 			new_range_size
101 | 				= get_range_end() - get_curr_loc();
102 | 			idx = new_range_start + new_range_size;
103 | 
104 | 		}
105 | 		range_start = new_range_start;
106 | 		size = new_range_size;
107 | 		this->forward = forward;
108 | 	}
109 | };
110 | 
111 | }
112 | 
113 | #endif
114 | 


--------------------------------------------------------------------------------
/flash-graph/test-algs/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | project (FlashGraph)
 2 | 
 3 | add_executable(test_algs test_algs.cpp)
 4 | target_link_libraries(test_algs graph-algs graph FMatrix safs cblas pthread)
 5 | 
 6 | find_package(ZLIB)
 7 | if (ZLIB_FOUND)
 8 | 	target_link_libraries(test_algs z)
 9 | endif()
10 | 
11 | if (LIBNUMA_FOUND)
12 |     target_link_libraries(test_algs numa)
13 | endif()
14 | 
15 | if (LIBAIO_FOUND)
16 |     target_link_libraries(test_algs aio)
17 | endif()
18 | 
19 | if (hwloc_FOUND)
20 |     target_link_libraries(test_algs hwloc)
21 | endif()
22 | 


--------------------------------------------------------------------------------
/flash-graph/test-algs/Makefile:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 Open Connectome Project (http://openconnecto.me)
 2 | # Written by Da Zheng (zhengda1936@gmail.com)
 3 | #
 4 | # This file is part of FlashGraph.
 5 | #
 6 | # Licensed under the Apache License, Version 2.0 (the "License");
 7 | # you may not use this file except in compliance with the License.
 8 | # You may obtain a copy of the License at
 9 | #
10 | #     http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | 
18 | include ../../Makefile.common
19 | 
20 | LDFLAGS := -L../matrix -lmatrix -L../libgraph-algs -lgraph-algs -L.. -lgraph -L../../matrix -lFMatrix -lcblas -L../../libsafs -lsafs -lrt -lz $(LDFLAGS)
21 | CXXFLAGS += -I../../libsafs -I.. -I. -I../../matrix
22 | 
23 | all: test_algs
24 | 
25 | test_algs: test_algs.o ../libgraph.a ../libgraph-algs/libgraph-algs.a
26 | 	$(CXX) -o test_algs test_algs.o $(LDFLAGS)
27 | 
28 | clean:
29 | 	rm -f *.d
30 | 	rm -f *.o
31 | 	rm -f *~
32 | 	rm -f test_algs
33 | 
34 | -include $(DEPS) 
35 | 


--------------------------------------------------------------------------------
/flash-graph/tools/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | project (FlashGraph)
2 | 
3 | add_executable(rmat-gen rmat-gen.cpp)
4 | # ext_mem_vertex_iterator.cpp
5 | 


--------------------------------------------------------------------------------
/flash-graph/tools/Makefile:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 Open Connectome Project (http://openconnecto.me)
 2 | # Written by Da Zheng (zhengda1936@gmail.com)
 3 | #
 4 | # This file is part of FlashGraph.
 5 | #
 6 | # Licensed under the Apache License, Version 2.0 (the "License");
 7 | # you may not use this file except in compliance with the License.
 8 | # You may obtain a copy of the License at
 9 | #
10 | #     http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | 
18 | include ../../Makefile.common
19 | 
20 | LDFLAGS := -L.. -lgraph -L../../matrix -lFMatrix -lcblas -L../../libsafs -lsafs -lrt $(LDFLAGS) -lz
21 | CXXFLAGS += -I../../libsafs -I.. -I. -I../../matrix
22 | 
23 | all: rmat-gen print_graph
24 | 
25 | print_ts_graph: print_ts_graph.o ../libgraph.a
26 | 	$(CXX) -o print_ts_graph print_ts_graph.o $(LDFLAGS)
27 | 
28 | rmat-gen: rmat-gen.o
29 | 	$(CXX) -o rmat-gen rmat-gen.o $(LDFLAGS)
30 | 
31 | graph-stat: graph-stat.o ../libgraph.a
32 | 	$(CXX) -o graph-stat graph-stat.o $(LDFLAGS)
33 | 
34 | print_graph: print_graph.o ../libgraph.a
35 | 	$(CXX) -o print_graph print_graph.o $(LDFLAGS)
36 | 
37 | clean:
38 | 	rm -f *.d
39 | 	rm -f *.o
40 | 	rm -f *~
41 | 	rm -f print_ts_graph
42 | 	rm -f rmat-gen
43 | 	rm -f graph-stat
44 | 	rm -f print_graph
45 | 
46 | -include $(DEPS) 
47 | 


--------------------------------------------------------------------------------
/flash-graph/tools/print_graph.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 3 |  * Written by Da Zheng (zhengda1936@gmail.com)
 4 |  *
 5 |  * This file is part of FlashGraph.
 6 |  *
 7 |  * Licensed under the Apache License, Version 2.0 (the "License");
 8 |  * you may not use this file except in compliance with the License.
 9 |  * You may obtain a copy of the License at
10 |  *
11 |  *     http://www.apache.org/licenses/LICENSE-2.0
12 |  *
13 |  * Unless required by applicable law or agreed to in writing, software
14 |  * distributed under the License is distributed on an "AS IS" BASIS,
15 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 |  * See the License for the specific language governing permissions and
17 |  * limitations under the License.
18 |  */
19 | 
20 | /*
21 |  * This is a test file to print a small graph.
22 |  */
23 | 
24 | #include <stdio.h>
25 | 
26 | #include <string>
27 | 
28 | #include "vertex.h"
29 | #include "native_file.h"
30 | #include "vertex_index.h"
31 | #include "FGlib.h"
32 | #include "fg_utils.h"
33 | 
34 | using namespace fg;
35 | 
36 | int main(int argc, char *argv[])
37 | {
38 | 	if (argc < 3) {
39 | 		fprintf(stderr, "print_graph adj_list_file index_file [attr_type]\n");
40 | 		return -1;
41 | 	}
42 | 
43 | 	const std::string adj_file_name = argv[1];
44 | 	const std::string index_file_name = argv[2];
45 | 	std::string type;
46 | 	if (argc == 4)
47 | 		type = argv[3];
48 | 
49 | 	FG_graph::ptr graph = FG_graph::create(adj_file_name, index_file_name,
50 | 			config_map::ptr());
51 | 	print_graph_el(graph, "\t", type, stdout);
52 | }
53 | 


--------------------------------------------------------------------------------
/flash-graph/tools/print_ts_graph.cpp:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 3 |  * Written by Da Zheng (zhengda1936@gmail.com)
 4 |  *
 5 |  * This file is part of FlashGraph.
 6 |  *
 7 |  * Licensed under the Apache License, Version 2.0 (the "License");
 8 |  * you may not use this file except in compliance with the License.
 9 |  * You may obtain a copy of the License at
10 |  *
11 |  *     http://www.apache.org/licenses/LICENSE-2.0
12 |  *
13 |  * Unless required by applicable law or agreed to in writing, software
14 |  * distributed under the License is distributed on an "AS IS" BASIS,
15 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 |  * See the License for the specific language governing permissions and
17 |  * limitations under the License.
18 |  */
19 | 
20 | /**
21 |  * This is a test file to print a small time-series graph.
22 |  */
23 | 
24 | #include <stdio.h>
25 | 
26 | #include <string>
27 | 
28 | #include "native_file.h"
29 | #include "vertex_index.h"
30 | 
31 | int main(int argc, char *argv[])
32 | {
33 | 	if (argc < 3) {
34 | 		fprintf(stderr, "print_ts_graph adj_list_file index_file\n");
35 | 		return -1;
36 | 	}
37 | 
38 | 	const std::string adj_file_name = argv[1];
39 | 	const std::string index_file_name = argv[2];
40 | 
41 | 	native_file adj_file(adj_file_name);
42 | 	ssize_t adj_file_size = adj_file.get_size();
43 | 	char *adj_list = new char[adj_file_size];
44 | 	FILE *f = fopen(adj_file_name.c_str(), "r");
45 | 	if (f == NULL) {
46 | 		perror("fopen");
47 | 		return -1;
48 | 	}
49 | 	size_t ret = fread(adj_list, adj_file_size, 1, f);
50 | 	assert(ret == 1);
51 | 
52 | 	default_vertex_index::ptr index = default_vertex_index::load(index_file_name);
53 | 	int num_edges = 0;
54 | 	int num_vertices = 0;
55 | 	for (size_t i = 0; i < index->get_num_vertices(); i++) {
56 | 		off_t off = index->get_vertex_off(i);
57 | 		int size = index->get_vertex_size(i);
58 | 		ts_ext_mem_directed_vertex *v
59 | 			= (ts_ext_mem_directed_vertex *) (adj_list + off);
60 | 		if (v->get_num_edges() > 0) {
61 | 			v->print();
62 | 			num_vertices++;
63 | 			num_edges += v->get_num_edges();
64 | 		}
65 | 		assert(size == (int) v->get_size());
66 | 		assert(i == v->get_id());
67 | 	}
68 | 	printf("There are %d vertices and %d edges\n", num_vertices, num_edges);
69 | }
70 | 


--------------------------------------------------------------------------------
/flash-graph/tools/rmat-gen.cpp:
--------------------------------------------------------------------------------
 1 | #include <boost/graph/adjacency_list.hpp>
 2 | #include <boost/graph/rmat_graph_generator.hpp>
 3 | #include <boost/random/linear_congruential.hpp>
 4 | #include <boost/graph/graph_traits.hpp>
 5 | #include <iostream>
 6 | #include <ctime>
 7 | #include <stdio.h>
 8 | 
 9 | using namespace boost;
10 | typedef adjacency_list<> Graph;
11 | // The length of cycle of rand48 is 2^48-1, which is much longer than
12 | // minstd_rand.
13 | // http://www.boost.org/doc/libs/1_55_0/doc/html/boost_random/reference.html
14 | typedef rand48 rand_gen_t;
15 | typedef rmat_iterator<rand_gen_t, Graph> RMATGen;
16 | typedef graph_traits<Graph>::vertex_iterator vertex_iter;
17 | typedef property_map<Graph, vertex_index_t>::type IndexMap;
18 | 
19 | int main(int argc, char* argv[])
20 | {
21 | 	if (argc < 3) {
22 | 		fprintf(stderr, "usage: make_graph #vertices, #numedges [output]\n");
23 | 		return EXIT_FAILURE;
24 | 	}
25 | 	typedef boost::graph_traits<Graph>::vertices_size_type vertices_size_type;
26 | 	typedef boost::graph_traits<Graph>::edges_size_type edges_size_type;
27 | 	vertices_size_type n = atol(argv[1]);
28 | 	edges_size_type m = atol(argv[2]);
29 | 
30 | 	FILE *f = stdout;
31 | 	std::string out_file;
32 | 	if (argc >= 4) {
33 | 		out_file = argv[3];
34 | 		f = fopen(argv[3], "w");
35 | 		assert(f);
36 | 	}
37 | 
38 | 	fprintf(stderr, "Vertices = %ld\n", n);
39 | 	fprintf(stderr, "Edges = %ld\n", m);
40 | 
41 | 	std::clock_t start;
42 | 	start = std::clock();
43 | 	rand_gen_t gen(time(NULL));
44 | 	RMATGen gen_it(gen, n, m, 0.57, 0.19, 0.19, 0.05, true);
45 | 	RMATGen gen_end;
46 | 	for (; gen_it != gen_end; ++gen_it) {
47 | 		fprintf(f, "%ld %ld\n", gen_it->first, gen_it->second);
48 | 	}
49 | #if 0
50 | 	// Create graph with 100 nodes and 400 edges
51 | 	Graph g(RMATGen(gen, n, m, 0.57, 0.19, 0.19, 0.05, true), RMATGen(), n);
52 | 
53 | 	IndexMap index = get(vertex_index, g);
54 | 
55 | 	// Get vertex set
56 | #if 0
57 | 	std::pair<vertex_iter, vertex_iter> vp;
58 | 	for (vp = vertices(g); vp.first != vp.second; ++vp.first)
59 | 		std::cout << index[*vp.first] <<  " ";
60 | 	std::cout << std::endl;
61 | #endif
62 | 
63 | 	// Get edge set
64 | 	graph_traits<Graph>::edge_iterator ei, ei_end;
65 | 	for (tie(ei, ei_end) = edges(g); ei != ei_end; ++ei)
66 | 		std::cout << index[source(*ei, g)]<< " " << index[target(*ei, g)] << "\n";
67 | #endif
68 | 	if (f != stdout) {
69 | 		printf("close %s\n", out_file.c_str());
70 | 		fclose(f);
71 | 	}
72 | 
73 | 	std::cerr << "The time to build the graph = " << (std::clock()-start)/(double)CLOCKS_PER_SEC << std::endl;
74 | 	return 0;
75 | }
76 | 


--------------------------------------------------------------------------------
/flash-graph/ts_graph.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 3 |  * Written by Da Zheng (zhengda1936@gmail.com)
 4 |  *
 5 |  * This file is part of FlashGraph.
 6 |  *
 7 |  * Licensed under the Apache License, Version 2.0 (the "License");
 8 |  * you may not use this file except in compliance with the License.
 9 |  * You may obtain a copy of the License at
10 |  *
11 |  *     http://www.apache.org/licenses/LICENSE-2.0
12 |  *
13 |  * Unless required by applicable law or agreed to in writing, software
14 |  * distributed under the License is distributed on an "AS IS" BASIS,
15 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 |  * See the License for the specific language governing permissions and
17 |  * limitations under the License.
18 |  */
19 | 
20 | #include "ts_graph.h"
21 | 
22 | namespace fg
23 | {
24 | 
25 | edge_seq_iterator get_ts_iterator(const page_directed_vertex &v,
26 | 		edge_type type, time_t time_start, time_t time_interval)
27 | {
28 | 	safs::page_byte_array::const_iterator<ts_edge_data> begin_it
29 | 		= v.get_data_begin<ts_edge_data>(type);
30 | 	safs::page_byte_array::const_iterator<ts_edge_data> end_it
31 | 		= v.get_data_end<ts_edge_data>(type);
32 | 	safs::page_byte_array::const_iterator<ts_edge_data> ts_it = std::lower_bound(
33 | 			begin_it, end_it, ts_edge_data(time_start));
34 | 	// All timestamps are smaller than time_start
35 | 	if (ts_it == end_it)
36 | 		return v.get_neigh_seq_it(type, 0, 0);
37 | 	size_t start = ts_it - begin_it;
38 | 
39 | 	safs::page_byte_array::const_iterator<ts_edge_data> ts_end_it = std::lower_bound(
40 | 			begin_it, end_it, time_start + time_interval);
41 | 	size_t end = ts_end_it - begin_it;
42 | 
43 | 	return v.get_neigh_seq_it(type, start, end);
44 | }
45 | 
46 | }
47 | 


--------------------------------------------------------------------------------
/flash-graph/ts_graph.h:
--------------------------------------------------------------------------------
 1 | #ifndef __TS_ALGS_H__
 2 | #define __TS_ALGS_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashGraph.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include "vertex.h"
24 | 
25 | namespace fg
26 | {
27 | 
28 | const int HOUR_SECS = 3600;
29 | const int DAY_SECS = HOUR_SECS * 24;
30 | const int MONTH_SECS = DAY_SECS * 30;
31 | 
32 | edge_seq_iterator get_ts_iterator(const page_directed_vertex &v,
33 | 		edge_type type, time_t time_start, time_t time_interval);
34 | 
35 | static inline bool is_time_str(const std::string &str)
36 | {
37 | 	struct tm tm;
38 | 	memset(&tm, 0, sizeof(tm));
39 | 	char *ret = strptime(str.c_str(), "%Y-%m-%d", &tm);
40 | 	return ret != NULL;
41 | }
42 | 
43 | static inline time_t conv_str_to_time(const std::string &str)
44 | {
45 | 	struct tm tm;
46 | 	memset(&tm, 0, sizeof(tm));
47 | 	BOOST_VERIFY(strptime(str.c_str(), "%Y-%m-%d", &tm));
48 | 	tm.tm_isdst = 1;
49 | 	return mktime(&tm);
50 | }
51 | 
52 | }
53 | 
54 | #endif
55 | 


--------------------------------------------------------------------------------
/flash-graph/unit-test/Makefile:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 Open Connectome Project (http;//openconnecto.me)
 2 | # Written by Da Zheng (zhengda1936@gmail.com)
 3 | #
 4 | # This file is part of FlashGraph.
 5 | #
 6 | # Licensed under the Apache License, Version 2.0 (the "License");
 7 | # you may not use this file except in compliance with the License.
 8 | # You may obtain a copy of the License at
 9 | #
10 | #     http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | 
18 | include ../../Makefile.common
19 | 
20 | LDFLAGS := -L.. -lgraph -L../../matrix -lFMatrix -lcblas -L../../libsafs -lsafs -lrt -lz $(LDFLAGS)
21 | CXXFLAGS = -I.. -I../../libsafs -I../../matrix -g -std=c++0x
22 | 
23 | SOURCE := $(wildcard *.c) $(wildcard *.cpp)
24 | OBJS := $(patsubst %.c,%.o,$(patsubst %.cpp,%.o,$(SOURCE)))
25 | DEPS := $(patsubst %.o,%.d,$(OBJS))
26 | 
27 | UNITTEST = test-bitmap test-partitioner test-vertex_index test-sparse_matrix
28 | 
29 | all: $(UNITTEST)
30 | 
31 | test-bitmap: test-bitmap.o ../libgraph.a
32 | 	$(CXX) -o test-bitmap test-bitmap.o $(LDFLAGS)
33 | 
34 | test-sparse_matrix: test-sparse_matrix.o ../libgraph.a
35 | 	$(CXX) -o test-sparse_matrix test-sparse_matrix.o $(LDFLAGS)
36 | 
37 | test-partitioner: test-partitioner.o ../libgraph.a
38 | 	$(CXX) -o test-partitioner test-partitioner.o $(LDFLAGS)
39 | 
40 | test-vertex_index: test-vertex_index.o ../libgraph.a
41 | 	$(CXX) -o test-vertex_index test-vertex_index.o $(LDFLAGS)
42 | 
43 | test:
44 | 	./test-bitmap
45 | 	./test-partitioner
46 | 	./test-sparse_matrix
47 | 	./test-vertex_index
48 | 
49 | clean:
50 | 	rm -f *.o
51 | 	rm -f *.d
52 | 	rm -f *~
53 | 	rm -f $(UNITTEST)
54 | 
55 | -include $(DEPS) 
56 | 


--------------------------------------------------------------------------------
/flash-graph/unit-test/test-bitmap.cpp:
--------------------------------------------------------------------------------
 1 | #include <algorithm>
 2 | #include <set>
 3 | 
 4 | #define BOOST_TEST_MODULE bitmap
 5 | #include <boost/test/included/unit_test.hpp>
 6 | 
 7 | #include "bitmap.h"
 8 | 
 9 | const int max_bits = 1024 * 1024 * 128;
10 | 
11 | BOOST_AUTO_TEST_SUITE (bitmaptest) // name of the test suite
12 | 
13 | BOOST_AUTO_TEST_CASE (test1)
14 | {
15 | 	int num = 100000;
16 | 	bitmap map1(max_bits, 0);
17 | 	std::set<size_t> elements;
18 | 
19 | 	for (int test = 0; test < 10; test++) {
20 | 		for (int i = 0; i < num; i++) {
21 | 			int v = random() % max_bits;
22 | 			elements.insert(v);
23 | 			map1.set(v);
24 | 		}
25 | 		printf("There are %ld elements\n", elements.size());
26 | 
27 | 		printf("There are %ld set bits\n", map1.get_num_set_bits());
28 | 		std::vector<size_t> bitmap_results;
29 | 		map1.get_set_bits(bitmap_results);
30 | 
31 | 		std::vector<size_t> bitmap_results1;
32 | 		size_t num_bits = map1.get_num_bits();
33 | 		const int NUM_CHECK_BITS = 8192;
34 | 		for (size_t idx = 0; idx < num_bits; idx += NUM_CHECK_BITS)
35 | 			map1.get_set_bits(idx, idx + NUM_CHECK_BITS, bitmap_results1);
36 | 
37 | 		BOOST_CHECK(bitmap_results1.size() == elements.size());
38 | 		BOOST_CHECK(bitmap_results.size() == elements.size());
39 | 		std::vector<size_t>::iterator v_it = bitmap_results.begin();
40 | 		std::vector<size_t>::iterator v_it1 = bitmap_results1.begin();
41 | 		std::set<size_t>::iterator s_it = elements.begin();
42 | 		for (; v_it != bitmap_results.end() && v_it1 != bitmap_results1.end()
43 | 				&& s_it != elements.end(); v_it++, v_it1++, s_it++) {
44 | 			BOOST_CHECK(*v_it == *s_it);
45 | 			BOOST_CHECK(*v_it1 == *s_it);
46 | 		}
47 | 		BOOST_CHECK(v_it == bitmap_results.end());
48 | 		BOOST_CHECK(v_it1 == bitmap_results1.end());
49 | 		BOOST_CHECK(s_it == elements.end());
50 | 		map1.clear();
51 | 		elements.clear();
52 | 
53 | 		bitmap tmp(random() % max_bits, 0);
54 | 		tmp.set_all();
55 | 		std::vector<size_t> result2;
56 | 		tmp.get_set_bits(result2);
57 | 	}
58 | }
59 | 
60 | BOOST_AUTO_TEST_SUITE_END( )
61 | 


--------------------------------------------------------------------------------
/flash-graph/unit-test/test-partitioner.cpp:
--------------------------------------------------------------------------------
 1 | #include <stdlib.h>
 2 | 
 3 | #include "partitioner.h"
 4 | 
 5 | using namespace fg;
 6 | 
 7 | const int num_parts = 16;
 8 | const int M = 1024 * 1024;
 9 | 
10 | void test_partitioner(graph_partitioner &partitioner)
11 | {
12 | 	for (int k = 0; k < 100; k++) {
13 | 		std::vector<vertex_id_t> parts[num_parts];
14 | 		size_t num_vertices = random() % M + M;
15 | 		printf("there are %ld vertices\n", num_vertices);
16 | 		for (int i = 0; i < num_parts; i++) {
17 | 			partitioner.get_all_vertices_in_part(i, num_vertices, parts[i]);
18 | 			size_t computed_part_size = partitioner.get_part_size(i,
19 | 					num_vertices);
20 | 			assert(computed_part_size == parts[i].size());
21 | 		}
22 | 		for (vertex_id_t id = 0; id < num_vertices; id++) {
23 | 			int part_id;
24 | 			off_t off;
25 | 			partitioner.map2loc(id, part_id, off);
26 | 			assert(part_id == partitioner.map(id));
27 | 			assert(parts[part_id][off] == id);
28 | 		}
29 | 		for (int part_id = 0; part_id < num_parts; part_id++) {
30 | 			for (off_t off = 0; off < parts[part_id].size(); off++) {
31 | 				vertex_id_t id;
32 | 				partitioner.loc2map(part_id, off, id);
33 | 				assert(id == parts[part_id][off]);
34 | 			}
35 | 		}
36 | 		size_t tot = 0;
37 | 		for (int i = 0; i < num_parts; i++)
38 | 			tot += parts[i].size();
39 | 		printf("There are %ld vertices in all partitions\n", tot);
40 | 		assert(num_vertices == tot);
41 | 	}
42 | }
43 | 
44 | int main()
45 | {
46 | 	printf("test range_graph_partitioner\n");
47 | 	range_graph_partitioner r_partitioner(num_parts);
48 | 	test_partitioner(r_partitioner);
49 | 
50 | 	printf("test modulo_graph_partitioner\n");
51 | 	modulo_graph_partitioner m_partitioner(num_parts);
52 | 	test_partitioner(m_partitioner);
53 | 
54 | }
55 | 


--------------------------------------------------------------------------------
/flash-graph/utils/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | project (FlashGraph)
 2 | 
 3 | include_directories("${PROJECT_SOURCE_DIR}")
 4 | 
 5 | add_executable(el2fg el2fg.cpp)
 6 | target_link_libraries(el2fg graph FMatrix safs pthread cblas)
 7 | 
 8 | add_executable(fg2fm fg2fm.cpp)
 9 | target_link_libraries(fg2fm graph FMatrix safs pthread cblas)
10 | 
11 | if (LIBNUMA_FOUND)
12 |     target_link_libraries(el2fg numa)
13 |     target_link_libraries(fg2fm numa)
14 | endif()
15 | 
16 | if (LIBAIO_FOUND)
17 |     target_link_libraries(el2fg aio)
18 |     target_link_libraries(fg2fm aio)
19 | endif()
20 | 
21 | find_package(hwloc)
22 | if (hwloc_FOUND)
23 | 	target_link_libraries(el2fg hwloc)
24 | 	target_link_libraries(fg2fm hwloc)
25 | endif()
26 | 
27 | if (ZLIB_FOUND)
28 | 	target_link_libraries(el2fg z)
29 | 	target_link_libraries(fg2fm z)
30 | endif()
31 | 


--------------------------------------------------------------------------------
/flash-graph/utils/Makefile:
--------------------------------------------------------------------------------
 1 | # Copyright 2016 Open Connectome Project (http://openconnecto.me)
 2 | # Written by Da Zheng (zhengda1936@gmail.com)
 3 | #
 4 | # This file is part of FlashMatrix.
 5 | #
 6 | # Licensed under the Apache License, Version 2.0 (the "License");
 7 | # you may not use this file except in compliance with the License.
 8 | # You may obtain a copy of the License at
 9 | #
10 | #     http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | 
18 | include ../../Makefile.common
19 | 
20 | CXXFLAGS += -I../../libsafs -I.. -I../../matrix
21 | LDFLAGS := -L../ -lgraph -L../../matrix -lFMatrix -L../../libsafs -lsafs $(LDFLAGS)
22 | LDFLAGS += -lz -lcblas #-lprofiler
23 | 
24 | all: el2fg fg2fm fg2crs fg_lcc csr2fg sbm
25 | 
26 | el2fg: el2fg.o ../libgraph.a
27 | 	$(CXX) -o el2fg el2fg.o $(LDFLAGS)
28 | 
29 | csr2fg: csr2fg.o ../libgraph.a
30 | 	$(CXX) -o csr2fg csr2fg.o $(LDFLAGS)
31 | 
32 | fg2fm: fg2fm.o ../libgraph.a
33 | 	$(CXX) -o fg2fm fg2fm.o $(LDFLAGS)
34 | 
35 | fg2crs: fg2crs.o ../libgraph.a
36 | 	$(CXX) -o fg2crs fg2crs.o $(LDFLAGS)
37 | 
38 | fg_lcc: fg_lcc.o ../libgraph.a
39 | 	$(CXX) -o fg_lcc fg_lcc.o -L../../flash-graph/libgraph-algs -lgraph-algs $(LDFLAGS)
40 | 
41 | sbm: sbm.o ../libgraph.a
42 | 	$(CXX) -o sbm sbm.o $(LDFLAGS)
43 | 
44 | clean:
45 | 	rm -f *.d
46 | 	rm -f *.o
47 | 	rm -f *~
48 | 	rm -f el2fg fg2fm fg2crs fg_lcc csr2fg sbm
49 | 


--------------------------------------------------------------------------------
/flash-graph/utils/crs_header.h:
--------------------------------------------------------------------------------
 1 | #ifndef __CRS_HEADER_H__
 2 | #define __CRS_HEADER_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashMatrix.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include <stdlib.h>
24 | 
25 | class crs_header
26 | {
27 | 	union {
28 | 		char page[4096];
29 | 		struct {
30 | 			size_t nrows;
31 | 			size_t ncols;
32 | 			size_t nnz;
33 | 		} s;
34 | 	} data;
35 | public:
36 | 	crs_header() {
37 | 		memset(this, 0, sizeof(*this));
38 | 	}
39 | 
40 | 	crs_header(size_t nrows, size_t ncols, size_t nnz) {
41 | 		data.s.nrows = nrows;
42 | 		data.s.ncols = ncols;
43 | 		data.s.nnz = nnz;
44 | 	}
45 | 
46 | 	size_t get_num_rows() const {
47 | 		return data.s.nrows;
48 | 	}
49 | 
50 | 	size_t get_num_cols() const {
51 | 		return data.s.ncols;
52 | 	}
53 | 
54 | 	size_t get_num_non_zeros() const {
55 | 		return data.s.nnz;
56 | 	}
57 | };
58 | 
59 | typedef size_t crs_idx_t;
60 | 
61 | #endif
62 | 


--------------------------------------------------------------------------------
/flash-graph/vertex.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 3 |  * Written by Da Zheng (zhengda1936@gmail.com)
 4 |  *
 5 |  * This file is part of FlashGraph.
 6 |  *
 7 |  * Licensed under the Apache License, Version 2.0 (the "License");
 8 |  * you may not use this file except in compliance with the License.
 9 |  * You may obtain a copy of the License at
10 |  *
11 |  *     http://www.apache.org/licenses/LICENSE-2.0
12 |  *
13 |  * Unless required by applicable law or agreed to in writing, software
14 |  * distributed under the License is distributed on an "AS IS" BASIS,
15 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 |  * See the License for the specific language governing permissions and
17 |  * limitations under the License.
18 |  */
19 | 
20 | #include "vertex.h"
21 | #include "vertex_index.h"
22 | 
23 | namespace fg
24 | {
25 | 
26 | empty_data edge<empty_data>::data;
27 | 
28 | size_t ext_mem_undirected_vertex::serialize(const in_mem_vertex &v, char *buf,
29 | 		size_t size, edge_type type)
30 | {
31 | 	assert(ext_mem_undirected_vertex::get_header_size() <= size);
32 | 	ext_mem_undirected_vertex *ext_v = (ext_mem_undirected_vertex *) buf;
33 | 	ext_v->set_id(v.get_id());
34 | 	ext_v->num_edges = v.get_num_edges(type);
35 | 	if (v.has_edge_data())
36 | 		ext_v->edge_data_size = v.get_edge_data_size();
37 | 	else
38 | 		ext_v->edge_data_size = 0;
39 | 	size_t mem_size = ext_v->get_size();
40 | 	assert(size >= mem_size);
41 | 	v.serialize_edges(ext_v->neighbors, type);
42 | 	// serialize edge data
43 | 	if (v.has_edge_data()) {
44 | 		v.serialize_edge_data(ext_v->get_edge_data_addr(), type);
45 | 	}
46 | 
47 | 	return mem_size;
48 | }
49 | 
50 | }
51 | 


--------------------------------------------------------------------------------
/flash-graph/vertex_index_constructor.h:
--------------------------------------------------------------------------------
 1 | #ifndef __VERTEX_INDEX_CONSTRUCTOR_H__
 2 | #define __VERTEX_INDEX_CONSTRUCTOR_H__
 3 | 
 4 | /**
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashGraph.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | #include "vertex_index.h"
23 | 
24 | namespace fg
25 | {
26 | 
27 | /*
28 |  * These are the in-mem counterparts of vertex index above.
29 |  * These in-memory data structures are used to construct vertex indices.
30 |  */
31 | 
32 | class vertex_index_construct
33 | {
34 | public:
35 | 	typedef std::shared_ptr<vertex_index_construct> ptr;
36 | 
37 | 	static ptr create(bool directed);
38 | 	static ptr create_compressed(bool directed, size_t edge_data_size);
39 | 
40 | 	virtual ~vertex_index_construct() {
41 | 	}
42 | 
43 | 	virtual void add_vertex(const in_mem_vertex &) = 0;
44 | 	virtual void dump(const std::string &file, const graph_header &header,
45 | 			bool compressed) = 0;
46 | 	virtual vertex_index::ptr dump(const graph_header &header, bool compressed) = 0;
47 | };
48 | 
49 | }
50 | 
51 | #endif
52 | 


--------------------------------------------------------------------------------
/flash-graph/vertex_pointer.h:
--------------------------------------------------------------------------------
 1 | #ifndef __VERTEX_POINTER_H__
 2 | #define __VERTEX_POINTER_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashGraph.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | #include <stdlib.h>
23 | 
24 | namespace fg
25 | {
26 | 
27 | class compute_vertex;
28 | 
29 | class compute_vertex_pointer
30 | {
31 | 	static const int PART_BIT_LOC = 48;
32 | 	// TODO This is Linux specific.
33 | 	static const uint64_t PART_MASK = 0xFFFF000000000000UL;
34 | 	static const uint64_t ADDR_MASK = 0x0000FFFFFFFFFFFFUL;
35 | 	uint64_t addr;
36 | public:
37 | 	static compute_vertex **conv(compute_vertex_pointer *arr) {
38 | 		assert(sizeof(compute_vertex_pointer) == sizeof(arr[0]));
39 | 		return (compute_vertex **) arr;
40 | 	}
41 | 
42 | 	compute_vertex_pointer() {
43 | 		addr = 0;
44 | 	}
45 | 
46 | 	explicit compute_vertex_pointer(compute_vertex *v) {
47 | 		assert(v);
48 | 		addr = (uint64_t) v;
49 | 		assert(addr > 100000);
50 | 	}
51 | 
52 | 	compute_vertex_pointer(compute_vertex *v, bool part) {
53 | 		assert(v);
54 | 		assert(((uint64_t) v) > 100000);
55 | 		addr = ((uint64_t) v) + (((uint64_t) part) << PART_BIT_LOC);
56 | 	}
57 | 
58 | 	bool is_part() const {
59 | 		return addr & PART_MASK;
60 | 	}
61 | 
62 | 	bool is_valid() const {
63 | 		return (addr & ADDR_MASK) != 0;
64 | 	}
65 | 
66 | 	compute_vertex *get() const {
67 | 		assert((addr & ADDR_MASK) != 0);
68 | 		return (compute_vertex *) (uintptr_t) (addr & ADDR_MASK);
69 | 	}
70 | 
71 | 	compute_vertex *operator->() const {
72 | 		return get();
73 | 	}
74 | 
75 | 	compute_vertex &operator*() const {
76 | 		return *get();
77 | 	}
78 | };
79 | 
80 | }
81 | 
82 | #endif
83 | 


--------------------------------------------------------------------------------
/gen_doc.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | echo 'roxygen2::roxygenize("Rpkg/")' | R --no-save
4 | echo 'roxygen2::roxygenize("FlashGraphR/")' | R --no-save
5 | 


--------------------------------------------------------------------------------
/get_num_nodes.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/sh
 2 | 
 3 | num_nodes=`numactl -H | grep "nodes" | awk '{print $2}'`
 4 | if [ "$num_nodes" == 0 ]; then
 5 | 	echo 1
 6 | elif [ -z $num_nodes ]; then
 7 | 	echo 1
 8 | else
 9 | 	echo $num_nodes
10 | fi
11 | 


--------------------------------------------------------------------------------
/install_FlashGraphR.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/sh
 2 | 
 3 | if [ `numactl -H | grep "nodes" | awk '{print $2}'` -gt 1 ]; then
 4 | 	export ENABLE_NUMA=1
 5 | fi
 6 | 
 7 | mkdir -p build
 8 | cd build
 9 | cmake ..;
10 | if [ $? != 0 ]; then
11 | 	exit
12 | fi
13 | make
14 | cd ..
15 | 
16 | R CMD build FlashGraphR
17 | version=`R --version | grep "R version" | awk '{print $3}' | awk -F. '{print $1 "." $2}'`
18 | if [ -d ~/R ]; then
19 | 	path=`find ~/R -name Rcpp.h | grep ${version}`
20 | fi
21 | if [ -z "$path" ]; then
22 | 	path=`find /usr/local -name Rcpp.h`
23 | fi
24 | 
25 | echo "search for FlashR"
26 | libpaths=`R -e 'cat(.libPaths(), "\n")' --no-save --slave`
27 | FlashR_dir=""
28 | for x in $libpaths
29 | do
30 | 	if [ -f "$x/FlashR/libs/FlashR.so" ]; then
31 | 		FlashR_dir="$x/FlashR/libs/"
32 | 	fi
33 | done
34 | 
35 | if [ "$FlashR_dir" = "" ]; then
36 | 	echo "cannot find FlashR"
37 | 	exit 1
38 | fi
39 | 
40 | export FlashR_dir
41 | fg_lib=`pwd`/build
42 | if [ -n "$path" ]; then
43 | 	path=`dirname $path`
44 | 	echo $path
45 | 	RCPP_INCLUDE=$path FG_DIR=`pwd` FG_LIB=$fg_lib R CMD INSTALL --no-configure FlashGraphR_0.1-0.tar.gz
46 | else
47 | 	FG_DIR=`pwd` FG_LIB=$fg_lib R CMD INSTALL --no-configure FlashGraphR_0.1-0.tar.gz
48 | fi
49 | rm FlashGraphR_0.1-0.tar.gz
50 | 


--------------------------------------------------------------------------------
/install_FlashR.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/sh
 2 | 
 3 | if [ `numactl -H | grep "nodes" | awk '{print $2}'` -gt 1 ]; then
 4 | 	export ENABLE_NUMA=1
 5 | fi
 6 | 
 7 | mkdir -p build
 8 | cd build
 9 | cmake ..;
10 | if [ $? != 0 ]; then
11 | 	exit
12 | fi
13 | make
14 | cd ..
15 | 
16 | R CMD build FlashR
17 | version=`R --version | grep "R version" | awk '{print $3}' | awk -F. '{print $1 "." $2}'`
18 | if [ -d ~/R ]; then
19 | 	path=`find ~/R -name Rcpp.h | grep ${version}`
20 | fi
21 | if [ -z "$path" ]; then
22 | 	path=`find /usr/local -name Rcpp.h`
23 | fi
24 | 
25 | fg_lib=`pwd`/build
26 | eigen_path=`find $fg_lib/matrix -name libeigen.a`
27 | if [ -n "$eigen_path" ]; then
28 | 	echo "find libeigen.a"
29 | 	export ENABLE_TRILINOS=1
30 | fi
31 | if [ -n "$path" ]; then
32 | 	path=`dirname $path`
33 | 	echo $path
34 | 	RCPP_INCLUDE=$path FG_DIR=`pwd` FG_LIB=$fg_lib R CMD INSTALL --no-configure FlashR_0.1-0.tar.gz
35 | else
36 | 	FG_DIR=`pwd` FG_LIB=$fg_lib R CMD INSTALL --no-configure FlashR_0.1-0.tar.gz
37 | fi
38 | ret=$?
39 | rm FlashR_0.1-0.tar.gz
40 | exit $ret
41 | 


--------------------------------------------------------------------------------
/install_FlashRLearn.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | 
3 | R CMD build FlashR-learn
4 | version=`R --version | grep "R version" | awk '{print $3}' | awk -F. '{print $1 "." $2}'`
5 | 
6 | R CMD INSTALL --no-configure FlashRLearn_0.1-0.tar.gz
7 | rm FlashRLearn_0.1-0.tar.gz
8 | 


--------------------------------------------------------------------------------
/libsafs/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | project (FlashGraph)
 2 | 
 3 | add_library(safs STATIC
 4 | 	aio_private.cpp
 5 | 	debugger.cpp
 6 | 	messaging.cpp
 7 | 	read_private.cpp
 8 | 	associative_cache.cpp
 9 | 	direct_private.cpp
10 | 	io_interface.cpp
11 | 	native_file.cpp
12 | 	remote_access.cpp
13 | 	cache_config.cpp
14 | 	disk_read_thread.cpp
15 | 	io_request.cpp
16 | 	parameters.cpp
17 | 	safs_file.cpp
18 | 	cache.cpp
19 | 	file_mapper.cpp
20 | 	memory_manager.cpp
21 | 	part_global_cached_private.cpp
22 | 	shadow_cell.cpp
23 | 	global_cached_private.cpp
24 | 	RAID_config.cpp
25 | 	wpaio.cpp
26 | 	direct_comp_access.cpp
27 | 	comp_io_scheduler.cpp
28 | 	in_mem_io.cpp
29 | 	NUMA_mapper.cpp
30 | 	common.cpp
31 | 	config_map.cpp
32 | 	log.cpp
33 | 	mem_tracker.cpp
34 | 	slab_allocator.cpp
35 | 	thread.cpp
36 | )
37 | 


--------------------------------------------------------------------------------
/libsafs/Makefile:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 Open Connectome Project (http://openconnecto.me)
 2 | # Written by Da Zheng (zhengda1936@gmail.com)
 3 | #
 4 | # This file is part of SAFSlib.
 5 | #
 6 | # Licensed under the Apache License, Version 2.0 (the "License");
 7 | # you may not use this file except in compliance with the License.
 8 | # You may obtain a copy of the License at
 9 | #
10 | #     http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | 
18 | include ../Makefile.common
19 | 
20 | all: libsafs unit_test test
21 | 
22 | libsafs: $(OBJS)
23 | 	rm -f libsafs.a
24 | 	ar -cvq libsafs.a $(OBJS)
25 | 
26 | unit_test: libsafs
27 | ifndef MEMCHECK
28 | 	$(MAKE) -C unit-test
29 | endif
30 | 
31 | test: libsafs
32 | 	$(MAKE) -C test
33 | 
34 | clean:
35 | 	rm -f *.d
36 | 	rm -f *.o
37 | 	rm -f *~
38 | 	rm -f libsafs.a
39 | 	make --ignore-errors -C unit-test clean
40 | 	make --ignore-errors -C test clean
41 | 
42 | -include $(DEPS) 
43 | 


--------------------------------------------------------------------------------
/libsafs/NUMA_mapper.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 3 |  * Written by Da Zheng (zhengda1936@gmail.com)
 4 |  *
 5 |  * This file is part of FlashMatrix.
 6 |  *
 7 |  * Licensed under the Apache License, Version 2.0 (the "License");
 8 |  * you may not use this file except in compliance with the License.
 9 |  * You may obtain a copy of the License at
10 |  *
11 |  *     http://www.apache.org/licenses/LICENSE-2.0
12 |  *
13 |  * Unless required by applicable law or agreed to in writing, software
14 |  * distributed under the License is distributed on an "AS IS" BASIS,
15 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 |  * See the License for the specific language governing permissions and
17 |  * limitations under the License.
18 |  */
19 | 
20 | #include "NUMA_mapper.h"
21 | 
22 | std::vector<size_t> NUMA_mapper::cal_local_lengths(size_t len) const
23 | {
24 | 	std::vector<size_t> ret(get_num_nodes());
25 | 	size_t last_range_id = len >> range_size_log;
26 | 	size_t last_range_size = len & range_mask;
27 | 	auto phy_loc = map2physical(last_range_id << range_size_log);
28 | 	ret[phy_loc.first] = phy_loc.second + last_range_size;
29 | 	size_t range_off = last_range_id << range_size_log;
30 | 	for (size_t i = 0; i < get_num_nodes() - 1 && range_off >= range_size; i++) {
31 | 		range_off -= range_size;
32 | 		phy_loc = map2physical(range_off);
33 | 		assert(ret[phy_loc.first] == 0);
34 | 		ret[phy_loc.first] = phy_loc.second + range_size;
35 | 	}
36 | 	return ret;
37 | }
38 | 
39 | NUMA_mapper::NUMA_mapper(size_t num_nodes, size_t rs_log): range_size_log(
40 | 		rs_log), range_size(1 << range_size_log), range_mask(range_size - 1),
41 | 	numa_log(log2(num_nodes)), numa_mask((1 << numa_log) - 1)
42 | {
43 | 	assert(num_nodes == 1UL << numa_log);
44 | }
45 | 


--------------------------------------------------------------------------------
/libsafs/RAID_config.h:
--------------------------------------------------------------------------------
 1 | #ifndef __RAID_CONFIG_H__
 2 | #define __RAID_CONFIG_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of SAFSlib.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include <vector>
24 | #include <set>
25 | #include <memory>
26 | 
27 | #include "safs_file.h"
28 | 
29 | namespace safs
30 | {
31 | 
32 | class file_mapper;
33 | 
34 | enum {
35 | 	RAID0,
36 | 	RAID5,
37 | 	HASH,
38 | };
39 | 
40 | class RAID_config
41 | {
42 | 	/*
43 | 	 * These are default values for the RAID config.
44 | 	 * A per-file config can overwrite these parameters.
45 | 	 */
46 | 	int RAID_mapping_option;
47 | 	int RAID_block_size;
48 | 
49 | 	std::vector<part_file_info> root_paths;
50 | public:
51 | 	typedef std::shared_ptr<RAID_config> ptr;
52 | 
53 | 	static ptr create(const std::string &conf_file, int mapping_option,
54 | 			int block_size);
55 | 
56 | 	/**
57 | 	 * Create a file mapper for the RAID directories.
58 | 	 */
59 | 	std::shared_ptr<file_mapper> create_file_mapper() const;
60 | 	/**
61 | 	 * Create a file mapper for a file in the RAID.
62 | 	 */
63 | 	std::shared_ptr<file_mapper> create_file_mapper(
64 | 			const std::string &file_name) const;
65 | 
66 | 	/**
67 | 	 * This returns the nodes where the RAID attaches to.
68 | 	 */
69 | 	std::set<int> get_node_ids() const;
70 | 
71 | 	const part_file_info &get_disk(int idx) const {
72 | 		return root_paths[idx];
73 | 	}
74 | 
75 | 	std::vector<part_file_info> get_disks() const {
76 | 		return root_paths;
77 | 	}
78 | 
79 | 	int get_num_disks() const {
80 | 		return root_paths.size();
81 | 	}
82 | 
83 | 	int get_mapping_option() const {
84 | 		return RAID_mapping_option;
85 | 	}
86 | 
87 | 	int get_block_size() const {
88 | 		return RAID_block_size;
89 | 	}
90 | };
91 | 
92 | }
93 | 
94 | #endif
95 | 


--------------------------------------------------------------------------------
/libsafs/aligned_allocator.h:
--------------------------------------------------------------------------------
 1 | #ifndef __ALIGNED_ALLOCATOR_H__
 2 | #define __ALIGNED_ALLOCATOR_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of SAFSlib.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include <malloc.h>
24 | 
25 | #include "common.h"
26 | 
27 | /*
28 |  * The class can allocate aligned memory if the alignment provided
29 |  * by the user is a power of two.
30 |  */
31 | class aligned_allocator
32 | {
33 | 	size_t alignment;
34 | public:
35 | 	aligned_allocator(size_t alignment) {
36 | 		this->alignment = alignment;
37 | 		bool aligned = align_check(alignment);
38 | 		printf("aligned allocator: alignment: %ld, aligned: %d\n",
39 | 				this->alignment, aligned);
40 | 		// If we find it's not a power of 2, indicate it's not aligned.
41 | 		if (!aligned)
42 | 			this->alignment = 0;
43 | 	}
44 | 
45 | 	void *alloc(size_t size) {
46 | 		if (alignment) {
47 | 			void *buf = NULL;
48 | 			int ret = posix_memalign(&buf, alignment, size);
49 | 			if (ret == 0)
50 | 				return buf;
51 | 			else
52 | 				return NULL;
53 | 		}
54 | 		else
55 | 			return malloc(size);
56 | 	}
57 | 
58 | 	void dealloc(void *p) {
59 | 		free(p);
60 | 	}
61 | };
62 | 
63 | #endif
64 | 


--------------------------------------------------------------------------------
/libsafs/cache.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 3 |  * Written by Da Zheng (zhengda1936@gmail.com)
 4 |  *
 5 |  * This file is part of SAFSlib.
 6 |  *
 7 |  * Licensed under the Apache License, Version 2.0 (the "License");
 8 |  * you may not use this file except in compliance with the License.
 9 |  * You may obtain a copy of the License at
10 |  *
11 |  *     http://www.apache.org/licenses/LICENSE-2.0
12 |  *
13 |  * Unless required by applicable law or agreed to in writing, software
14 |  * distributed under the License is distributed on an "AS IS" BASIS,
15 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 |  * See the License for the specific language governing permissions and
17 |  * limitations under the License.
18 |  */
19 | 
20 | #include "cache.h"
21 | #include "io_interface.h"
22 | 
23 | namespace safs
24 | {
25 | 
26 | void page_byte_array::memcpy(off_t rel_off, char buf[], size_t size) const
27 | {
28 | 	if (size == 0)
29 | 		return;
30 | 
31 | 	// The offset relative to the beginning of the page array.
32 | 	off_t off = get_offset_in_first_page() + rel_off;
33 | 	off_t end = off + size;
34 | 	assert((size_t) end <= get_size() + get_offset_in_first_page());
35 | 
36 | 	off_t page_begin = ROUND(off, PAGE_SIZE);
37 | 	// If the element crosses the page boundary.
38 | 	if (end - page_begin > PAGE_SIZE) {
39 | 		off_t pg_idx = off / PAGE_SIZE;
40 | 		off_t last_pg_idx = end / PAGE_SIZE;
41 | 		off_t off_in_pg = off % PAGE_SIZE;
42 | 		size_t part1_size = PAGE_SIZE - off_in_pg;
43 | 		size_t copied = 0;
44 | 		::memcpy(buf, get_page(pg_idx) + off_in_pg, part1_size);
45 | 		copied += part1_size;
46 | 		pg_idx++;
47 | 		while (pg_idx < last_pg_idx) {
48 | 			::memcpy(buf + copied, get_page(pg_idx), PAGE_SIZE);
49 | 			copied += PAGE_SIZE;
50 | 			pg_idx++;
51 | 		}
52 | 		size_t part2_size = size - copied;
53 | 		if (part2_size > 0)
54 | 			::memcpy(buf + copied, get_page(pg_idx), part2_size);
55 | 	}
56 | 	else {
57 | 		off_t pg_idx = off / PAGE_SIZE;
58 | 		off_t off_in_pg = off % PAGE_SIZE;
59 | 		::memcpy(buf, get_page(pg_idx) + off_in_pg, size);
60 | 	}
61 | }
62 | 
63 | void page::hit()
64 | {
65 | 	hits++;
66 | }
67 | 
68 | }
69 | 


--------------------------------------------------------------------------------
/libsafs/comm_exception.h:
--------------------------------------------------------------------------------
 1 | #ifndef __COMM_EXCEPTION_H__
 2 | #define __COMM_EXCEPTION_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of SAFSlib.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | #include <string>
23 | #include <exception>
24 | 
25 | /* out of memory exception */
26 | class oom_exception: public std::exception
27 | {
28 | 	std::string msg;
29 | public:
30 | 	oom_exception() {
31 | 	}
32 | 
33 | 	oom_exception(const std::string &msg) {
34 | 		this->msg = msg;
35 | 	}
36 | 
37 | 	~oom_exception() throw() {
38 | 	}
39 | 
40 | 	const char* what() const throw() {
41 | 		return msg.c_str();
42 | 	}
43 | };
44 | 
45 | class unsupported_exception: public std::exception
46 | {
47 | 	std::string msg;
48 | public:
49 | 	unsupported_exception() {
50 | 	}
51 | 
52 | 	unsupported_exception(const std::string &msg) {
53 | 		this->msg = msg;
54 | 	}
55 | 
56 | 	~unsupported_exception() throw() {
57 | 	}
58 | 
59 | 	const char* what() const throw() {
60 | 		return msg.c_str();
61 | 	}
62 | };
63 | 
64 | class invalid_arg_exception: public std::exception
65 | {
66 | 	std::string msg;
67 | public:
68 | 	invalid_arg_exception(const std::string &msg) {
69 | 		this->msg = msg;
70 | 	}
71 | 
72 | 	~invalid_arg_exception() throw() {
73 | 	}
74 | 
75 | 	const char* what() const throw() {
76 | 		return msg.c_str();
77 | 	}
78 | };
79 | 
80 | #endif
81 | 


--------------------------------------------------------------------------------
/libsafs/compute_stat.h:
--------------------------------------------------------------------------------
 1 | #ifndef __COMPUTE_STAT_H__
 2 | #define __COMPUTE_STAT_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of SAFSlib.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include "concurrency.h"
24 | 
25 | /*
26 |  * These provides a thread-safe way to compute some statistics:
27 |  *	mean, max, min, etc on streaming data.
28 |  */
29 | 
30 | /*
31 |  * Compute the average value of elements in a vector.
32 |  */
33 | template<class T>
34 | class stat_mean
35 | {
36 | 	atomic_number<T> tot;
37 | 	atomic_number<T> num;
38 | public:
39 | 	void add(T v) {
40 | 		tot.inc(v);
41 | 		num.inc(1);
42 | 	}
43 | 
44 | 	double get() const {
45 | 		return ((double) tot.get()) / num.get();
46 | 	}
47 | };
48 | 
49 | template<class T>
50 | class stat_max
51 | {
52 | 	atomic_number<T> max;
53 | public:
54 | 	void add(T v) {
55 | 		while (true) {
56 | 			T orig_max = max.get();
57 | 			// The original max is smaller, try to switch to the new max.
58 | 			if (orig_max < v) {
59 | 				bool ret = max.CAS(orig_max, v);
60 | 				// If we succeed, we can stop now.
61 | 				if (ret)
62 | 					break;
63 | 				// Otherwise, we need to try again.
64 | 			}
65 | 			else
66 | 				// The original max is larger, we just stop the loop and
67 | 				// do nothing.
68 | 				break;
69 | 		}
70 | 	}
71 | 
72 | 	T get() const {
73 | 		return max.get();
74 | 	}
75 | };
76 | 
77 | #endif
78 | 


--------------------------------------------------------------------------------
/libsafs/debugger.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 3 |  * Written by Da Zheng (zhengda1936@gmail.com)
 4 |  *
 5 |  * This file is part of SAFSlib.
 6 |  *
 7 |  * Licensed under the Apache License, Version 2.0 (the "License");
 8 |  * you may not use this file except in compliance with the License.
 9 |  * You may obtain a copy of the License at
10 |  *
11 |  *     http://www.apache.org/licenses/LICENSE-2.0
12 |  *
13 |  * Unless required by applicable law or agreed to in writing, software
14 |  * distributed under the License is distributed on an "AS IS" BASIS,
15 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 |  * See the License for the specific language governing permissions and
17 |  * limitations under the License.
18 |  */
19 | 
20 | #include <signal.h>
21 | #include <stdio.h>
22 | 
23 | #include <boost/foreach.hpp>
24 | #include <system_error>
25 | 
26 | #include "debugger.h"
27 | 
28 | namespace safs
29 | {
30 | 
31 | static bool enable_debug = false;
32 | 
33 | bool is_debug_enabled()
34 | {
35 | 	return enable_debug;
36 | }
37 | 
38 | static void enable_debug_handler(int sig, siginfo_t *si, void *uc)
39 | {
40 | #if 0
41 | 	printf("debug mode is enabled\n");
42 | 	debug.run();
43 | #endif
44 | }
45 | 
46 | static void set_enable_debug_signal()
47 | {
48 | 	struct sigaction sa;
49 | 
50 | 	/* Establish handler for timer signal */
51 | 
52 | 	sa.sa_flags = SA_SIGINFO;
53 | 	sa.sa_sigaction = enable_debug_handler;
54 | 	sigemptyset(&sa.sa_mask);
55 | 	if (sigaction(SIGUSR1, &sa, NULL) == -1)
56 | 		throw std::system_error(std::make_error_code((std::errc) errno),
57 | 				"sigaction error");
58 | }
59 | 
60 | debugger::debugger()
61 | {
62 | 	set_enable_debug_signal();
63 | }
64 | 
65 | debugger::~debugger()
66 | {
67 | 	typedef std::map<int, debug_task *> task_map_t;
68 | 	BOOST_FOREACH(task_map_t::value_type &p, tasks) {
69 | 		delete p.second;
70 | 	}
71 | }
72 | 
73 | void debugger::run()
74 | {
75 | 	lock.lock();
76 | 	std::map<int, debug_task *> task_copies = tasks;
77 | 	lock.unlock();
78 | 	for (std::map<int, debug_task *>::const_iterator it
79 | 			= task_copies.begin(); it != task_copies.end(); it++) {
80 | 		it->second->run();
81 | 	}
82 | }
83 | 
84 | }
85 | 


--------------------------------------------------------------------------------
/libsafs/debugger.h:
--------------------------------------------------------------------------------
 1 | #ifndef __DEBUGGER_H__
 2 | #define __DEBUGGER_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of SAFSlib.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include <pthread.h>
24 | 
25 | #include <map>
26 | 
27 | #include "concurrency.h"
28 | 
29 | namespace safs
30 | {
31 | 
32 | class debug_task
33 | {
34 | public:
35 | 	virtual ~debug_task() {
36 | 	}
37 | 
38 | 	virtual void run() = 0;
39 | };
40 | 
41 | class debugger
42 | {
43 | 	// We have to use map instead of unordered_map here because
44 | 	// we want to preserve the order of the tasks.
45 | 	std::map<int, debug_task *> tasks;
46 | 	spin_lock lock;
47 | 	atomic_integer task_id_gen;
48 | public:
49 | 	debugger();
50 | 
51 | 	~debugger();
52 | 
53 | 	int register_task(debug_task *task) {
54 | 		int id = task_id_gen.inc(1);
55 | 		lock.lock();
56 | 		tasks.insert(std::pair<int, debug_task *>(id, task));
57 | 		lock.unlock();
58 | 		return id;
59 | 	}
60 | 
61 | 	void remove_task(int task_id) {
62 | 		lock.lock();
63 | 		tasks.erase(task_id);
64 | 		lock.unlock();
65 | 	}
66 | 
67 | 	void run();
68 | };
69 | 
70 | bool is_debug_enabled();
71 | 
72 | }
73 | 
74 | #endif
75 | 


--------------------------------------------------------------------------------
/libsafs/direct_comp_access.h:
--------------------------------------------------------------------------------
 1 | #ifndef __COMP_ACCESS_H__
 2 | #define __COMP_ACCESS_H__
 3 | 
 4 | /**
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of SAFSlib.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include "container.h"
24 | 
25 | #include "io_interface.h"
26 | 
27 | namespace safs
28 | {
29 | 
30 | class remote_io;
31 | class comp_io_scheduler;
32 | class direct_byte_array_allocator;
33 | 
34 | /**
35 |  * This class supports asynchronous user-task I/O requests.
36 |  * Unlike global_cached_io, it doesn't support page cache.
37 |  */
38 | class direct_comp_io: public io_interface
39 | {
40 | 	size_t num_disk_bytes;
41 | 	size_t num_req_bytes;
42 | 	size_t num_issued_areqs;
43 | 	size_t num_completed_areqs;
44 | 
45 | 	// The memory that have been allocated for the I/O requests that have
46 | 	// been issued to the underlying I/O.
47 | 	size_t alloc_mem_size;
48 | 	fifo_queue<io_request> req_buf;
49 | 	std::shared_ptr<remote_io> underlying;
50 | 	std::shared_ptr<comp_io_scheduler> comp_sched;
51 | 	std::unique_ptr<direct_byte_array_allocator> arr_alloc;
52 | 
53 | 	void process_buf_reqs();
54 | 	void process_incomplete_computes();
55 | public:
56 | 	direct_comp_io(std::shared_ptr<remote_io> io);
57 | 	~direct_comp_io();
58 | 
59 | 	/*
60 | 	 * This method is called when an I/O request is completed by
61 | 	 * the underlying I/O.
62 | 	 */
63 | 	void complete_req(const io_request &req);
64 | 
65 | 	virtual int get_file_id() const;
66 | 	virtual void cleanup();
67 | 	virtual bool support_aio() {
68 | 		return true;
69 | 	}
70 | 
71 | 	virtual void access(io_request *requests, int num,
72 | 			io_status *status = NULL);
73 | 	virtual void flush_requests();
74 | 	virtual int wait4complete(int num);
75 | 	virtual int num_pending_ios() const;
76 | 
77 | 	size_t get_num_disk_bytes() {
78 | 		return num_disk_bytes;
79 | 	}
80 | 
81 | 	size_t get_num_req_bytes() {
82 | 		return num_req_bytes;
83 | 	}
84 | 
85 | 	size_t get_num_reqs() {
86 | 		return num_issued_areqs;
87 | 	}
88 | 
89 | 	size_t get_num_completed_reqs() {
90 | 		return num_completed_areqs;
91 | 	}
92 | };
93 | 
94 | }
95 | 
96 | #endif
97 | 


--------------------------------------------------------------------------------
/libsafs/direct_private.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 3 |  * Written by Da Zheng (zhengda1936@gmail.com)
 4 |  *
 5 |  * This file is part of SAFSlib.
 6 |  *
 7 |  * Licensed under the Apache License, Version 2.0 (the "License");
 8 |  * you may not use this file except in compliance with the License.
 9 |  * You may obtain a copy of the License at
10 |  *
11 |  *     http://www.apache.org/licenses/LICENSE-2.0
12 |  *
13 |  * Unless required by applicable law or agreed to in writing, software
14 |  * distributed under the License is distributed on an "AS IS" BASIS,
15 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 |  * See the License for the specific language governing permissions and
17 |  * limitations under the License.
18 |  */
19 | 
20 | #include "direct_private.h"
21 | 
22 | namespace safs
23 | {
24 | 
25 | io_status direct_io::access(char *buf, off_t offset,
26 | 		ssize_t size, int access_method) {
27 | 	assert(size >= MIN_BLOCK_SIZE);
28 | 	assert(size % MIN_BLOCK_SIZE == 0);
29 | 	assert(offset % MIN_BLOCK_SIZE == 0);
30 | 	assert((long) buf % MIN_BLOCK_SIZE == 0);
31 | 	return buffered_io::access(buf, offset, size, access_method);
32 | }
33 | 
34 | }
35 | 


--------------------------------------------------------------------------------
/libsafs/direct_private.h:
--------------------------------------------------------------------------------
 1 | #ifndef __DIRECT_PRIVATE_H__
 2 | #define __DIRECT_PRIVATE_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of SAFSlib.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include "read_private.h"
24 | 
25 | namespace safs
26 | {
27 | 
28 | class direct_io: public buffered_io
29 | {
30 | public:
31 | 	direct_io(const logical_file_partition &partition, thread *t,
32 | 			const safs_header &header): buffered_io(partition, t, header,
33 | 				O_DIRECT | O_RDWR) {
34 | 	}
35 | 
36 | 	io_status access(char *buf, off_t offset, ssize_t size, int access_method);
37 | };
38 | 
39 | }
40 | 
41 | #endif
42 | 


--------------------------------------------------------------------------------
/libsafs/dirty_page_flusher.h:
--------------------------------------------------------------------------------
 1 | #ifndef __DIRTY_PAGE_FLUSHER_H__
 2 | #define __DIRTY_PAGE_FLUSHER_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of SAFSlib.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include "common.h"
24 | 
25 | namespace safs
26 | {
27 | 
28 | class thread_safe_page;
29 | class io_interface;
30 | 
31 | class dirty_page_flusher
32 | {
33 | public:
34 | 	virtual ~dirty_page_flusher() {
35 | 	}
36 | 
37 | 	/**
38 | 	 * This method flushes dirty pages with the specified io instance.
39 | 	 * The method has to be thread-safe, as it will be invoked in different
40 | 	 * thread contexts. It also shouldn't block threads.
41 | 	 */
42 | 	virtual void flush_dirty_pages(thread_safe_page *pages[], int num,
43 | 			io_interface &io) = 0;
44 | 
45 | 	virtual int flush_dirty_pages(page_filter *filter, int max_num) = 0;
46 | };
47 | 
48 | }
49 | 
50 | #endif
51 | 


--------------------------------------------------------------------------------
/libsafs/log.cpp:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 3 |  * Written by Da Zheng (zhengda1936@gmail.com)
 4 |  *
 5 |  * This file is part of SAFSlib.
 6 |  *
 7 |  * Licensed under the Apache License, Version 2.0 (the "License");
 8 |  * you may not use this file except in compliance with the License.
 9 |  * You may obtain a copy of the License at
10 |  *
11 |  *     http://www.apache.org/licenses/LICENSE-2.0
12 |  *
13 |  * Unless required by applicable law or agreed to in writing, software
14 |  * distributed under the License is distributed on an "AS IS" BASIS,
15 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 |  * See the License for the specific language governing permissions and
17 |  * limitations under the License.
18 |  */
19 | 
20 | #include "log.h"
21 | 
22 | c_log_level simple_log_stream::curr_log_level;
23 | 
24 | simple_log_stream log_debug(c_log_level::debug);
25 | simple_log_stream log_info(c_log_level::info);
26 | simple_log_stream log_warning(c_log_level::warning);
27 | simple_log_stream log_error(c_log_level::error);
28 | simple_log_stream log_fatal(c_log_level::fatal);
29 | 


--------------------------------------------------------------------------------
/libsafs/log.h:
--------------------------------------------------------------------------------
 1 | #ifndef __MY_LOG_H__
 2 | #define __MY_LOG_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of libcommon.
 9 |  * This library is used by SAFS and FlashGraph, and potentially by other
10 |  * systems.
11 |  *
12 |  * Licensed under the Apache License, Version 2.0 (the "License");
13 |  * you may not use this file except in compliance with the License.
14 |  * You may obtain a copy of the License at
15 |  *
16 |  *     http://www.apache.org/licenses/LICENSE-2.0
17 |  *
18 |  * Unless required by applicable law or agreed to in writing, software
19 |  * distributed under the License is distributed on an "AS IS" BASIS,
20 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 |  * See the License for the specific language governing permissions and
22 |  * limitations under the License.
23 |  */
24 | 
25 | #include <iostream>
26 | 
27 | enum c_log_level
28 | {
29 | 	debug,
30 | 	info,
31 | 	warning,
32 | 	error,
33 | 	fatal,
34 | };
35 | 
36 | #ifdef USE_BOOST_LOG
37 | 
38 | #include <boost/log/trivial.hpp>
39 | 
40 | static inline void set_log_level(enum c_log_level level)
41 | {
42 | 	boost::log::core::get()->set_filter(
43 | 			boost::log::trivial::severity > boost::log::trivial::info);
44 | }
45 | 
46 | #else
47 | 
48 | class simple_log_stream
49 | {
50 | 	static c_log_level curr_log_level;
51 | 	c_log_level level;
52 | 	bool log_data;
53 | public:
54 | 	simple_log_stream(c_log_level level) {
55 | 		this->level = level;
56 | 		log_data = false;
57 | 	}
58 | 
59 | 	~simple_log_stream() {
60 | 		if (!log_data && level >= curr_log_level)
61 | 			std::cout << "\n" << std::flush;
62 | 	}
63 | 
64 | 	static void set_global_log_level(c_log_level level) {
65 | 		curr_log_level = level;
66 | 	}
67 | 
68 | 	template<class T>
69 | 	simple_log_stream operator<<(const T &v) {
70 | 		log_data = true;
71 | 		if (level >= curr_log_level)
72 | 			std::cout << v;
73 | 		return simple_log_stream(level);
74 | 	}
75 | };
76 | 
77 | extern simple_log_stream log_debug;
78 | extern simple_log_stream log_info;
79 | extern simple_log_stream log_warning;
80 | extern simple_log_stream log_error;
81 | extern simple_log_stream log_fatal;
82 | 
83 | #define BOOST_LOG_TRIVIAL(x)		\
84 | 	log_##x
85 | 
86 | static inline void set_log_level(enum c_log_level level)
87 | {
88 | 	simple_log_stream::set_global_log_level(level);
89 | }
90 | 
91 | #endif
92 | 
93 | #endif
94 | 


--------------------------------------------------------------------------------
/libsafs/mem_tracker.h:
--------------------------------------------------------------------------------
 1 | #ifndef __MEM_TRACKER_H__
 2 | #define __MEM_TRACKER_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of SAFSlib.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include <stdlib.h>
24 | 
25 | #ifdef ENABLE_MEM_TRACE
26 | 
27 | #include <new>
28 | 
29 | void *operator new(size_t n) throw (std::bad_alloc);
30 | void operator delete(void *p) throw ();
31 | void *operator new[](size_t n) throw (std::bad_alloc);
32 | void operator delete[](void *p) throw ();
33 | 
34 | #endif
35 | 
36 | void mem_trace_start();
37 | void mem_trace_stop();
38 | 
39 | size_t get_alloc_objs();
40 | size_t get_alloc_bytes();
41 | size_t get_max_alloc_objs();
42 | size_t get_max_alloc_bytes();
43 | size_t get_max_alloc();
44 | 
45 | #endif
46 | 


--------------------------------------------------------------------------------
/libsafs/memory_manager.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 3 |  * Written by Da Zheng (zhengda1936@gmail.com)
 4 |  *
 5 |  * This file is part of SAFSlib.
 6 |  *
 7 |  * Licensed under the Apache License, Version 2.0 (the "License");
 8 |  * you may not use this file except in compliance with the License.
 9 |  * You may obtain a copy of the License at
10 |  *
11 |  *     http://www.apache.org/licenses/LICENSE-2.0
12 |  *
13 |  * Unless required by applicable law or agreed to in writing, software
14 |  * distributed under the License is distributed on an "AS IS" BASIS,
15 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 |  * See the License for the specific language governing permissions and
17 |  * limitations under the License.
18 |  */
19 | 
20 | #include "memory_manager.h"
21 | 
22 | namespace safs
23 | {
24 | 
25 | const long SHRINK_NPAGES = 1024;
26 | const long INCREASE_SIZE = 1024 * 1024 * 128;
27 | 
28 | memory_manager::memory_manager(
29 | 		long max_size, int node_id): slab_allocator(
30 | 			std::string("mem_manager-") + itoa(node_id), PAGE_SIZE,
31 | 			INCREASE_SIZE <= max_size ? INCREASE_SIZE : max_size,
32 | 			// We don't initialize pages but we pin pages.
33 | 			max_size, node_id, false, true) {
34 | }
35 | 
36 | /**
37 |  * get `npages' pages for `request_cache'.
38 |  * In the case of shrinking caches, it makes no sense
39 |  * to shrink the cache that is requesting free pages.
40 |  */
41 | bool memory_manager::get_free_pages(int npages,
42 | 		char **pages, page_cache *request_cache) {
43 | 	int ret = slab_allocator::alloc(pages, npages);
44 | 	/* 
45 | 	 * slab_allocator allocates either all required number of 
46 | 	 * pages or none of them.
47 | 	 */
48 | 	if (ret == 0) {
49 | 		long size = 0;
50 | 		page_cache *cache = NULL;
51 | 		/* shrink the cache of the largest size */
52 | 		for (unsigned int i = 0; i < caches.size(); i++) {
53 | 			if (size < caches[i]->size()) {
54 | 				size = caches[i]->size();
55 | 				cache = caches[i];
56 | 			}
57 | 		}
58 | 		if (cache == NULL)
59 | 			return false;
60 | 		/* 
61 | 		 * if we are going to shrink the cache that requests
62 | 		 * free pages, it just fails.
63 | 		 */
64 | 		if (request_cache == cache) {
65 | 			return false;
66 | 		}
67 | 		int num_shrink = SHRINK_NPAGES;
68 | 		if (num_shrink < npages)
69 | 			num_shrink = npages;
70 | 		char *buf[num_shrink];
71 | 		if (!cache->shrink(num_shrink, buf)) {
72 | 			return false;
73 | 		}
74 | 		slab_allocator::free(buf, num_shrink);
75 | 		/* now it's guaranteed that we have enough free pages. */
76 | 		ret = slab_allocator::alloc(pages, npages);
77 | 	}
78 | 	return true;
79 | }
80 | 
81 | void memory_manager::free_pages(int npages, char **pages) {
82 | 	slab_allocator::free(pages, npages);
83 | }
84 | 
85 | }
86 | 


--------------------------------------------------------------------------------
/libsafs/memory_manager.h:
--------------------------------------------------------------------------------
 1 | #ifndef __MEMORY_MANAGER__
 2 | #define __MEMORY_MANAGER__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of SAFSlib.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include <vector>
24 | 
25 | #include "cache.h"
26 | #include "slab_allocator.h"
27 | 
28 | namespace safs
29 | {
30 | 
31 | /**
32 |  * manage free pages in the cache.
33 |  * It also allocates pages from the operating system.
34 |  */
35 | class memory_manager: public slab_allocator
36 | {
37 | 	std::vector<page_cache *> caches;
38 | 
39 | 	memory_manager(long max_size, int node_id);
40 | 
41 | 	~memory_manager() {
42 | 		// TODO
43 | 	}
44 | public:
45 | 	static memory_manager *create(long max_size, int node_id) {
46 | 		assert(node_id >= 0);
47 | 		return new memory_manager(max_size, node_id);
48 | 	}
49 | 
50 | 	static void destroy(memory_manager *m) {
51 | 		delete m;
52 | 	}
53 | 
54 | 	void register_cache(page_cache *cache) {
55 | 		caches.push_back(cache);
56 | 	}
57 | 
58 | 	void unregister_cache(page_cache *cache) {
59 | 		// TODO
60 | 	}
61 | 
62 | 	bool get_free_pages(int npages, char **pages, page_cache *cache);
63 | 	void free_pages(int npages, char **pages);
64 | 
65 | 	long average_cache_size() {
66 | 		return get_max_size() / caches.size();
67 | 	}
68 | };
69 | 
70 | }
71 | 
72 | #endif
73 | 


--------------------------------------------------------------------------------
/libsafs/read_private.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 3 |  * Written by Da Zheng (zhengda1936@gmail.com)
 4 |  *
 5 |  * This file is part of SAFSlib.
 6 |  *
 7 |  * Licensed under the Apache License, Version 2.0 (the "License");
 8 |  * you may not use this file except in compliance with the License.
 9 |  * You may obtain a copy of the License at
10 |  *
11 |  *     http://www.apache.org/licenses/LICENSE-2.0
12 |  *
13 |  * Unless required by applicable law or agreed to in writing, software
14 |  * distributed under the License is distributed on an "AS IS" BASIS,
15 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 |  * See the License for the specific language governing permissions and
17 |  * limitations under the License.
18 |  */
19 | 
20 | #include <system_error>
21 | 
22 | #include "read_private.h"
23 | #include "file_mapper.h"
24 | 
25 | namespace safs
26 | {
27 | 
28 | buffered_io::buffered_io(const logical_file_partition &partition_, thread *t,
29 | 		const safs_header &header, int flags): io_interface(t, header), partition(
30 | 			partition_), fds(partition.get_num_files())
31 | {
32 | 	this->flags = flags;
33 | 
34 | 	for (int i = 0; i < partition.get_num_files(); i++) {
35 | 		int ret;
36 | 		fds[i] = open(partition.get_file_name(i).c_str(), flags);
37 | 		if (fds[i] < 0) {
38 | 			char err_msg[128];
39 | 			snprintf(err_msg, sizeof(err_msg),
40 | 					"open %s", partition.get_file_name(i).c_str());
41 | 			perror(err_msg);
42 | 			throw std::system_error(std::make_error_code((std::errc) errno),
43 | 					"open");
44 | 		}
45 | 		ret = posix_fadvise(fds[i], 0, 0, POSIX_FADV_RANDOM);
46 | 		if (ret < 0)
47 | 			perror("posix_fadvise");
48 | 	}
49 | }
50 | 
51 | io_status buffered_io::access(char *buf, off_t offset, ssize_t size, int access_method) {
52 | 	ASSERT_EQ(get_thread(), thread::get_curr_thread());
53 | 	int fd;
54 | 	if (fds.size() == 1)
55 | 		fd = fds[0];
56 | 	else {
57 | 		struct block_identifier bid;
58 | 		partition.map(offset / PAGE_SIZE, bid);
59 | 		fd = fds[bid.idx];
60 | 		offset = bid.off * PAGE_SIZE;
61 | 	}
62 | 	ssize_t ret;
63 | 	// TODO I need to make sure all data is read or written to the file.
64 | 	if (access_method == WRITE)
65 | 		ret = pwrite(fd, buf, size, offset);
66 | 	else
67 | 		ret = pread(fd, buf, size, offset);
68 | 	io_status status;
69 | 	if (ret < 0)
70 | 		status = IO_FAIL;
71 | 	else
72 | 		status = IO_OK;
73 | 	status.set_priv_data(ret);
74 | 	return status;
75 | }
76 | 
77 | }
78 | 


--------------------------------------------------------------------------------
/libsafs/read_private.h:
--------------------------------------------------------------------------------
 1 | #ifndef __READ_PRIVATE_H__
 2 | #define __READ_PRIVATE_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of SAFSlib.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include <sys/time.h>
24 | #include <sys/types.h>
25 | #include <sys/stat.h>
26 | #include <fcntl.h>
27 | 
28 | #include "io_interface.h"
29 | #include "file_partition.h"
30 | #include "parameters.h"
31 | 
32 | namespace safs
33 | {
34 | 
35 | class buffered_io: public io_interface
36 | {
37 | 	logical_file_partition partition;
38 | 	/* the array of files that it's going to access */
39 | 	std::vector<int> fds;
40 | 
41 | 	int flags;
42 | public:
43 | 	buffered_io(const logical_file_partition &partition_,
44 | 			thread *t, const safs_header &header, int flags = O_RDWR);
45 | 
46 | 	virtual ~buffered_io() {
47 | 		for (unsigned i = 0; i < fds.size(); i++)
48 | 			BOOST_VERIFY(close(fds[i]) == 0);
49 | 	}
50 | 
51 | 	/* get the file descriptor corresponding to the offset. */
52 | 	int get_fd(long offset) {
53 | 		if (fds.size() == 1)
54 | 			return fds[0];
55 | 
56 | 		int idx = partition.map2file(offset / PAGE_SIZE);
57 | 		return fds[idx];
58 | 	}
59 | 
60 | 	const std::vector<int> &get_fds() const {
61 | 		return fds;
62 | 	}
63 | 
64 | 	int num_open_files() {
65 | 		return partition.get_num_files();
66 | 	}
67 | 
68 | 	const logical_file_partition &get_partition() const {
69 | 		return partition;
70 | 	}
71 | 
72 | 	int get_file_id() const {
73 | 		return partition.get_file_id();
74 | 	}
75 | 
76 | 	void cleanup() {
77 | 		for (size_t i = 0; i < fds.size(); i++)
78 | 			fsync(fds[i]);
79 | 	}
80 | 
81 | 	io_status access(char *buf, off_t offset, ssize_t size, int access_method);
82 | };
83 | 
84 | }
85 | 
86 | #endif
87 | 


--------------------------------------------------------------------------------
/libsafs/safs_exception.h:
--------------------------------------------------------------------------------
 1 | #ifndef __SAFS_EXCEPTION_H__
 2 | #define __SAFS_EXCEPTION_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of SAFSlib.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | #include <string>
23 | #include <exception>
24 | 
25 | namespace safs
26 | {
27 | 
28 | class init_error: public std::exception
29 | {
30 | 	std::string msg;
31 | public:
32 | 	init_error(const std::string &msg) {
33 | 		this->msg = msg;
34 | 	}
35 | 
36 | 	~init_error() throw() {
37 | 	}
38 | 
39 | 	const char* what() const throw() {
40 | 		return msg.c_str();
41 | 	}
42 | };
43 | 
44 | class io_exception: public std::exception
45 | {
46 | 	std::string msg;
47 | public:
48 | 	io_exception(const std::string &msg) {
49 | 		this->msg = msg;
50 | 	}
51 | 
52 | 	~io_exception() throw() {
53 | 	}
54 | 
55 | 	const char* what() const throw() {
56 | 		return msg.c_str();
57 | 	}
58 | };
59 | 
60 | }
61 | 
62 | #endif
63 | 


--------------------------------------------------------------------------------
/libsafs/safs_header.h:
--------------------------------------------------------------------------------
  1 | #ifndef __SAFS_HEADER_H__
  2 | #define __SAFS_HEADER_H__
  3 | 
  4 | /*
  5 |  * Copyright 2015 Open Connectome Project (http://openconnecto.me)
  6 |  * Written by Da Zheng (zhengda1936@gmail.com)
  7 |  *
  8 |  * This file is part of SAFSlib.
  9 |  *
 10 |  * Licensed under the Apache License, Version 2.0 (the "License");
 11 |  * you may not use this file except in compliance with the License.
 12 |  * You may obtain a copy of the License at
 13 |  *
 14 |  *     http://www.apache.org/licenses/LICENSE-2.0
 15 |  *
 16 |  * Unless required by applicable law or agreed to in writing, software
 17 |  * distributed under the License is distributed on an "AS IS" BASIS,
 18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 19 |  * See the License for the specific language governing permissions and
 20 |  * limitations under the License.
 21 |  */
 22 | 
 23 | #include "io_request.h"
 24 | 
 25 | namespace safs
 26 | {
 27 | 
 28 | class safs_header
 29 | {
 30 | 	static const int64_t MAGIC_NUMBER = 0x123456789FFFFFEL;
 31 | 	static const int CURR_VERSION = 1;
 32 | 
 33 | 	int64_t magic_number;
 34 | 	int version_number;
 35 | 	// In the number of pages.
 36 | 	uint32_t block_size;
 37 | 	uint32_t mapping_option;
 38 | 	uint32_t writable;
 39 | 	uint64_t num_bytes;
 40 | public:
 41 | 	static size_t get_header_size() {
 42 | 		return PAGE_SIZE;
 43 | 	}
 44 | 
 45 | 	safs_header() {
 46 | 		this->magic_number = MAGIC_NUMBER;
 47 | 		this->version_number = CURR_VERSION;
 48 | 		this->block_size = 0;
 49 | 		this->mapping_option = 0;
 50 | 		this->writable = false;
 51 | 		this->num_bytes = 0;
 52 | 	}
 53 | 
 54 | 	safs_header(int block_size, int mapping_option, bool writable,
 55 | 			size_t file_size) {
 56 | 		this->magic_number = MAGIC_NUMBER;
 57 | 		this->version_number = CURR_VERSION;
 58 | 		this->block_size = block_size;
 59 | 		this->mapping_option = mapping_option;
 60 | 		this->writable = writable;
 61 | 		this->num_bytes = file_size;
 62 | 	}
 63 | 
 64 | 	int get_block_size() const {
 65 | 		return block_size;
 66 | 	}
 67 | 
 68 | 	int get_mapping_option() const {
 69 | 		return mapping_option;
 70 | 	}
 71 | 
 72 | 	bool is_writable() const {
 73 | 		return writable;
 74 | 	}
 75 | 
 76 | 	bool is_safs_file() const {
 77 | 		return magic_number == MAGIC_NUMBER;
 78 | 	}
 79 | 
 80 | 	bool is_right_version() const {
 81 | 		return version_number == CURR_VERSION;
 82 | 	}
 83 | 
 84 | 	bool is_valid() const {
 85 | 		return block_size != 0;
 86 | 	}
 87 | 
 88 | 	void resize(size_t new_size) {
 89 | 		this->num_bytes = new_size;
 90 | 	}
 91 | 
 92 | 	size_t get_size() const {
 93 | 		return num_bytes;
 94 | 	}
 95 | };
 96 | 
 97 | }
 98 | 
 99 | #endif
100 | 


--------------------------------------------------------------------------------
/libsafs/test/Makefile:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 Open Connectome Project (http://openconnecto.me)
 2 | # Written by Da Zheng (zhengda1936@gmail.com)
 3 | #
 4 | # This file is part of SAFSlib.
 5 | #
 6 | # Licensed under the Apache License, Version 2.0 (the "License");
 7 | # you may not use this file except in compliance with the License.
 8 | # You may obtain a copy of the License at
 9 | #
10 | #     http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | 
18 | include ../../Makefile.common
19 | 
20 | CFLAGS = -g -I../ $(TRACE_FLAGS)
21 | 
22 | LDFLAGS := -L.. -lsafs $(LDFLAGS)
23 | CXXFLAGS += -I.. -I../
24 | 
25 | all: test_rand_io workload-gen workload-stat
26 | 
27 | test_rand_io: test_rand_io.o thread_private.o workload.o ../libsafs.a
28 | 	$(CXX) -o test_rand_io test_rand_io.o thread_private.o workload.o $(LDFLAGS)
29 | 
30 | workload-gen: workload-gen.o workload.o ../libsafs.a
31 | 	$(CXX) -o workload-gen workload-gen.o workload.o $(LDFLAGS)
32 | 
33 | workload-stat: workload-stat.o workload.o ../libsafs.a
34 | 	$(CXX) -o workload-stat workload-stat.o workload.o $(LDFLAGS)
35 | 
36 | clean:
37 | 	rm -f *.o
38 | 	rm -f *.d
39 | 	rm -f *~
40 | 	rm -f test_rand_io
41 | 	rm -f workload-gen
42 | 	rm -f workload-stat
43 | 
44 | -include $(DEPS) 
45 | 


--------------------------------------------------------------------------------
/libsafs/test/conf/run_cache.txt:
--------------------------------------------------------------------------------
 1 | cache_type=associative
 2 | cache_size=4G
 3 | io_depth=64
 4 | 
 5 | option=global_cache
 6 | num_reqs=10485760
 7 | threads=32
 8 | workload=RAND
 9 | entry_size=4096
10 | num_nodes=4
11 | access=read
12 | buf_size=16777216
13 | read_percent=100
14 | root_conf=conf/data_files.txt
15 | use_flusher=
16 | cache_large_write=
17 | 


--------------------------------------------------------------------------------
/libsafs/test/conf/run_cache_real.txt:
--------------------------------------------------------------------------------
 1 | cache_type=associative
 2 | io_depth=64
 3 | cache_size=4G
 4 | 
 5 | option=global_cache
 6 | threads=32
 7 | num_nodes=4
 8 | buf_size=16777216
 9 | root_conf=conf/data_files.txt
10 | #use_flusher=
11 | cache_large_write=
12 | merge_reqs=
13 | 


--------------------------------------------------------------------------------
/libsafs/test/conf/run_cache_tpcc.txt:
--------------------------------------------------------------------------------
 1 | cache_type=associative
 2 | virt_aio=
 3 | verify_content=
 4 | 
 5 | option=global_cache
 6 | threads=32
 7 | workload=test/workload/mysqld_tpcc_datafile.data
 8 | num_nodes=4
 9 | buf_size=16777216
10 | root_conf=conf/TEST_ROOTS.txt
11 | #use_flusher=
12 | cache_large_write=
13 | merge_reqs=
14 | 


--------------------------------------------------------------------------------
/libsafs/test/conf/run_cache_virt.txt:
--------------------------------------------------------------------------------
 1 | cache_type=associative
 2 | virt_aio=
 3 | verify_content=
 4 | 
 5 | option=global_cache
 6 | num_reqs=10485760
 7 | threads=16
 8 | workload=RAND
 9 | entry_size=4096
10 | num_nodes=4
11 | access=read
12 | buf_size=16777216
13 | read_percent=100
14 | root_conf=conf/TEST_ROOTS.txt
15 | use_flusher=
16 | cache_large_write=
17 | merge_reqs=
18 | 


--------------------------------------------------------------------------------
/libsafs/test/conf/run_parted_cache_tpcc.txt:
--------------------------------------------------------------------------------
 1 | cache_type=associative
 2 | virt_aio=
 3 | verify_content=
 4 | io_depth=1024
 5 | 
 6 | option=parted_global
 7 | threads=16
 8 | workload=test/workload/mysqld_tpcc_datafile.data
 9 | num_nodes=4
10 | buf_size=16777216
11 | root_conf=conf/TEST_ROOTS.txt
12 | #use_flusher=
13 | cache_large_write=
14 | numa_num_process_threads=8
15 | buf_type=SINGLE_SMALL
16 | 


--------------------------------------------------------------------------------
/libsafs/test/conf/run_parted_cache_virt.txt:
--------------------------------------------------------------------------------
 1 | cache_type=associative
 2 | virt_aio=
 3 | verify_content=
 4 | io_depth=1024
 5 | 
 6 | option=parted_global
 7 | num_reqs=10485760
 8 | threads=16
 9 | workload=RAND
10 | entry_size=4096
11 | num_nodes=4
12 | access=read
13 | buf_size=16777216
14 | read_percent=100
15 | root_conf=conf/TEST_ROOTS.txt
16 | use_flusher=
17 | cache_large_write=
18 | numa_num_process_threads=4
19 | 


--------------------------------------------------------------------------------
/libsafs/test/conf/run_remote.txt:
--------------------------------------------------------------------------------
 1 | RAID_mapping=RAID0
 2 | io_depth=64
 3 | 
 4 | option=remote
 5 | num_reqs=10485760
 6 | threads=16
 7 | workload=RAND
 8 | entry_size=4096
 9 | num_nodes=4
10 | access=read
11 | buf_size=16777216
12 | read_percent=100
13 | root_conf=conf/data_files.txt
14 | 


--------------------------------------------------------------------------------
/libsafs/test/conf/run_remote_real.txt:
--------------------------------------------------------------------------------
1 | RAID_mapping=RAID0
2 | 
3 | option=remote
4 | threads=32
5 | num_nodes=4
6 | buf_size=16777216
7 | root_conf=conf/data_files.txt
8 | 


--------------------------------------------------------------------------------
/libsafs/test/conf/run_remote_virt.txt:
--------------------------------------------------------------------------------
 1 | virt_aio=
 2 | verify_content=
 3 | 
 4 | option=remote
 5 | num_reqs=10485760
 6 | threads=16
 7 | workload=RAND
 8 | entry_size=4096
 9 | num_nodes=4
10 | access=read
11 | buf_size=16777216
12 | read_percent=100
13 | root_conf=conf/TEST_ROOTS.txt
14 | 


--------------------------------------------------------------------------------
/libsafs/test/init_test.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl
 2 | 
 3 | use strict;
 4 | 
 5 | my $num_args = @ARGV;
 6 | if ($num_args < 2) {
 7 | 	print STDERR "init_test root_conf data_file1 [data_files ...]\n";
 8 | 	exit 1;
 9 | }
10 | my $test_roots = $ARGV[0];
11 | my @data_files;
12 | for (my $i = 1; $i < $num_args; $i++) {
13 | 	push(@data_files, $ARGV[$i]);
14 | }
15 | my $num_files = @data_files;
16 | print "There are $num_files data files\n";
17 | open(FILE, $test_roots) or die "can't open $test_roots: $!";
18 | 
19 | my $id = 0;
20 | while (<FILE>) {
21 | 	if (/[0-9]+:(.+)/) {
22 | 		for (my $i = 0; $i < $num_files; $i++) {
23 | 			my $data_file = $data_files[$i];
24 | 			system("mkdir -p $1/$data_file");
25 | 			system("touch $1/$data_file/$id");
26 | 		}
27 | 		$id++;
28 | 	}
29 | }
30 | 


--------------------------------------------------------------------------------
/libsafs/test/thread_private.h:
--------------------------------------------------------------------------------
 1 | #ifndef __THREAD_PRIVATE_H__
 2 | #define __THREAD_PRIVATE_H__
 3 | 
 4 | /**
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of SAFSlib.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include "workload.h"
24 | #include "io_interface.h"
25 | #include "thread.h"
26 | #include "config.h"
27 | 
28 | using namespace safs;
29 | 
30 | class cleanup_callback;
31 | class sum_compute_allocator;
32 | class write_compute_allocator;
33 | 
34 | /* this data structure stores the thread-private info. */
35 | class thread_private: public thread
36 | {
37 | 	/* the location in the thread descriptor array. */
38 | 	int idx;
39 | 	int node_id;
40 | 	workload_gen *gen;
41 | 	io_interface::ptr io;
42 | 	file_io_factory::shared_ptr factory;
43 | 
44 | 	ssize_t read_bytes;
45 | 	long num_accesses;
46 | 
47 | 	/* compute average number of pending IOs. */
48 | 	long tot_num_pending;
49 | 	long num_sampling;
50 | 	int max_num_pending;
51 | 
52 | 	std::shared_ptr<cleanup_callback> cb;
53 | 	
54 | 	struct timeval start_time, end_time;
55 | 
56 | 	sum_compute_allocator *sum_alloc;
57 | 	write_compute_allocator *write_alloc;
58 | 
59 | #ifdef STATISTICS
60 | public:
61 | 	atomic_integer num_completes;
62 | 	atomic_integer num_pending;
63 | #endif
64 | 
65 | public:
66 | 	~thread_private() {
67 | 	}
68 | 
69 | 	void init();
70 | 
71 | 	int get_node_id() {
72 | 		return node_id;
73 | 	}
74 | 
75 | 	thread_private(int node_id, int idx, int entry_size,
76 | 			file_io_factory::shared_ptr factory, workload_gen *gen);
77 | 
78 | 	int attach2cpu();
79 | 
80 | 	void run();
81 | 
82 | 	ssize_t get_read_bytes();
83 | 
84 | 	void print_stat();
85 | };
86 | 
87 | #endif
88 | 


--------------------------------------------------------------------------------
/libsafs/unit-test/KV_store_unit_test.cpp:
--------------------------------------------------------------------------------
 1 | #include "simple_KV_store.h"
 2 | 
 3 | size_t arr_len;
 4 | std::unique_ptr<int32_t[]> int_arr;
 5 | 
 6 | size_t num_runs;
 7 | 
 8 | class data_check
 9 | {
10 | 	size_t idx;
11 | public:
12 | 	data_check() {
13 | 		idx = 0;
14 | 	}
15 | 
16 | 	data_check(size_t idx) {
17 | 		this->idx = idx;
18 | 	}
19 | 
20 | 	size_t get_idx() const {
21 | 		return idx;
22 | 	}
23 | 
24 | 	size_t get_num_entries() const {
25 | 		return 2;
26 | 	}
27 | 
28 | 	bool merge(const data_check &task) {
29 | 		return false;
30 | 	}
31 | 
32 | 	void run(page_byte_array::seq_const_iterator<int32_t> &it) {
33 | 		assert(it.get_num_tot_entries() == 2);
34 | 		assert(int_arr[idx] == it.next());
35 | 		assert(int_arr[idx + 1] == it.next());
36 | 		num_runs++;
37 | 	}
38 | 
39 | 	bool operator<(const data_check &check) const {
40 | 		return this->idx < check.idx;
41 | 	}
42 | };
43 | 
44 | int main(int argc, char *argv[])
45 | {
46 | 	if (argc < 3) {
47 | 		fprintf(stderr, "test conf_file file_name\n");
48 | 		exit(1);
49 | 	}
50 | 
51 | 	std::string conf_file = argv[1];
52 | 	std::string file_name = argv[2];
53 | 
54 | 	config_map::ptr configs = config_map::create(conf_file);
55 | 	init_io_system(configs);
56 | 	file_io_factory::shared_ptr factory = create_io_factory(file_name,
57 | 			GLOBAL_CACHE_ACCESS);
58 | 	size_t file_size = factory->get_file_size();
59 | 	thread *curr = thread::get_curr_thread();
60 | 	io_interface::ptr io = factory->create_io(curr);
61 | 
62 | 	arr_len = file_size / sizeof(int32_t);
63 | 	int_arr = std::unique_ptr<int32_t[]>(new int32_t[arr_len]);
64 | 	io->access((char *) int_arr.get(), 0, arr_len * sizeof(int32_t), READ);
65 | 
66 | 	simple_KV_store<int32_t, data_check>::ptr store
67 | 		= simple_KV_store<int32_t, data_check>::create(io);
68 | 	int num_checks = 1000;
69 | 	for (int test = 0; test < 5; test++) {
70 | 		for (int i = 0; i < num_checks; i++) {
71 | 			size_t idx = random() % arr_len;
72 | 			data_check check(idx);
73 | 			store->async_request(check);
74 | 		}
75 | 		store->flush_requests();
76 | 		printf("There are %d pending IOs\n", io->num_pending_ios());
77 | 		while (io->num_pending_ios() > 0) {
78 | 			io->wait4complete(1);
79 | 			printf("There are %d pending IOs\n", io->num_pending_ios());
80 | 		}
81 | 		printf("There are %ld runs\n", num_runs);
82 | 	}
83 | }
84 | 


--------------------------------------------------------------------------------
/libsafs/unit-test/SA_expand_shrink_test.cpp:
--------------------------------------------------------------------------------
 1 | #include <vector>
 2 | #include <set>
 3 | 
 4 | #include "associative_cache.h"
 5 | 
 6 | int main()
 7 | {
 8 | 	associative_cache *cache = new associative_cache(16 * 1024 * 1024, 512 * 1024 * 1024, 0, 1);
 9 | 	std::vector<off_t> added_offs;
10 | 	std::set<off_t> evicted_offs;
11 | 	for (int i = 0; i < 4096; i++) {
12 | 		off_t off = (random() & 0xfffffff) * 4096;
13 | 		added_offs.push_back(off);
14 | 		off_t old_off = -1;
15 | 		thread_safe_page *pg = (thread_safe_page *) cache->search(off, old_off);
16 | 		if (old_off != -1)
17 | 			evicted_offs.insert(old_off);
18 | 		pg->dec_ref();
19 | 	}
20 | 
21 | 	int ret;
22 | 	ret = cache->expand(8 * 1024 * 1024 / 4096);
23 | 	printf("expand %d pages\n", ret);
24 | 	cache->sanity_check();
25 | 	for (size_t i = 0; i < added_offs.size(); i++) {
26 | 		page *pg = cache->search(added_offs[i]);
27 | 		if (pg == NULL)
28 | 			assert(evicted_offs.find(added_offs[i]) != evicted_offs.end());
29 | 		else
30 | 			pg->dec_ref();
31 | 	}
32 | 
33 | 	for (int i = 0; i < 4096; i++) {
34 | 		off_t off = (random() & 0xfffffff) * 4096;
35 | 		added_offs.push_back(off);
36 | 		off_t old_off = -1;
37 | 		thread_safe_page *pg = (thread_safe_page *) cache->search(off, old_off);
38 | 		if (old_off != -1)
39 | 			evicted_offs.insert(old_off);
40 | 		pg->dec_ref();
41 | 	}
42 | 
43 | 	ret = cache->expand(16 * 1024 * 1024 / 4096);
44 | 	printf("expand %d pages\n", ret);
45 | 	cache->sanity_check();
46 | 	for (size_t i = 0; i < added_offs.size(); i++) {
47 | 		page *pg = cache->search(added_offs[i]);
48 | 		if (pg == NULL) {
49 | 			if (evicted_offs.find(added_offs[i]) == evicted_offs.end())
50 | 				fprintf(stderr, "can't find off %ld\n", added_offs[i]);
51 | 			assert(evicted_offs.find(added_offs[i]) != evicted_offs.end());
52 | 		}
53 | 		else
54 | 			pg->dec_ref();
55 | 	}
56 | 
57 | 	int num_shrink = 8 * 1024 * 1024 / 4096;
58 | 	char *pages[num_shrink];
59 | 	cache->shrink(num_shrink, pages);
60 | 	cache->sanity_check();
61 | }
62 | 


--------------------------------------------------------------------------------
/libsafs/unit-test/conf/data_files.txt:
--------------------------------------------------------------------------------
1 | 0:data
2 | 


--------------------------------------------------------------------------------
/libsafs/unit-test/conf/run_remote.txt:
--------------------------------------------------------------------------------
1 | RAID_mapping=RAID0
2 | io_depth=64
3 | root_conf=conf/data_files.txt
4 | writable=
5 | 


--------------------------------------------------------------------------------
/libsafs/unit-test/data_files.txt:
--------------------------------------------------------------------------------
1 | 0:/tmp/safs_data
2 | 


--------------------------------------------------------------------------------
/libsafs/unit-test/native_file_unit_test.cpp:
--------------------------------------------------------------------------------
 1 | #include <stdio.h>
 2 | #include <assert.h>
 3 | 
 4 | #include "native_file.h"
 5 | 
 6 | using namespace safs;
 7 | 
 8 | int main()
 9 | {
10 | 	std::string dir_name = "/tmp/1/1/1";
11 | 	native_dir dir(dir_name);
12 | 	assert(!dir.exist());
13 | 	assert(dir.create_dir(true));
14 | 
15 | 	native_dir dir2("/tmp/1");
16 | 	assert(dir2.delete_dir(true));
17 | 
18 | 	dir_name = "1/1/1";
19 | 	native_dir dir3(dir_name);
20 | 	assert(!dir3.exist());
21 | 	assert(dir3.create_dir(true));
22 | 
23 | 	native_dir dir4("1");
24 | 	assert(dir4.delete_dir(true));
25 | }
26 | 


--------------------------------------------------------------------------------
/libsafs/unit-test/run_test.txt:
--------------------------------------------------------------------------------
 1 | # Parameters for SAFS
 2 | 
 3 | # How data blocks of a SAFS file are mapped to Linux filesystem.
 4 | RAID_mapping=RAID0
 5 | 
 6 | # Allow users to write data to SAFS.
 7 | writable=
 8 | 
 9 | # The RAID block size in SAFS is 8MB. FlashMatrix requires a large RAID block
10 | # size.
11 | RAID_block_size=8388608
12 | 
13 | # The number of parallel I/Os is allowed in a single thread.
14 | io_depth=4
15 | 
16 | # The page cache size in SAFS
17 | cache_size=512M
18 | 
19 | # The number of NUMA nodes
20 | # num_nodes=1
21 | 
22 | # change FG_TOP to the path of the top directory of FlashGraph.
23 | root_conf=data_files.txt
24 | 
25 | # whether or not to use Linux huge pages
26 | # huge_page=
27 | 
28 | # The number of I/O threads per NUMA node.
29 | # num_io_threads=1
30 | 
31 | 
32 | # Parameters for FlashMatrix
33 | 
34 | # The number of worker threads to process a matrix.
35 | # threads=1
36 | 


--------------------------------------------------------------------------------
/libsafs/unit-test/safs_file_unit_test.cpp:
--------------------------------------------------------------------------------
 1 | #include "safs_file.h"
 2 | #include "RAID_config.h"
 3 | 
 4 | using namespace safs;
 5 | 
 6 | int main(int argc, char *argv[])
 7 | {
 8 | 	RAID_config::ptr raid = RAID_config::create(argv[1], 0, 16);
 9 | 	safs_file f(*raid, "test1");
10 | 
11 | 	printf("Check the existence of %s\n", f.get_name().c_str());
12 | 	assert(!f.exist());
13 | 
14 | 	printf("Create file %s\n", f.get_name().c_str());
15 | 	size_t file_size = 32 * 1024 * 1024;
16 | 	f.create_file(file_size);
17 | 	printf("%s has %ld bytes\n", f.get_name().c_str(), f.get_size());
18 | 	f.delete_file();
19 | }
20 | 


--------------------------------------------------------------------------------
/libsafs/unit-test/slab_allocator_test.cpp:
--------------------------------------------------------------------------------
 1 | #include "slab_allocator.h"
 2 | 
 3 | int main()
 4 | {
 5 | 	const int num_objs = 1000;
 6 | 	slab_allocator::linked_obj_list list1;
 7 | 	slab_allocator::linked_obj_list list2;
 8 | 	slab_allocator::linked_obj objs[num_objs];
 9 | 	for (int i = 0; i < num_objs / 2; i++)
10 | 		list1.add(&objs[i]);
11 | 	for (int i = num_objs / 2; i < num_objs; i++)
12 | 		list2.add(&objs[i]);
13 | 	printf("There are %d objs in list1\n", list1.get_size());
14 | 	printf("There are %d objs in list2\n", list2.get_size());
15 | 
16 | 	slab_allocator::linked_obj_list list3;
17 | 	list3.add_list(&list1);
18 | 	printf("After moving objs from list 1 to list 3, there are %d objs in list1\n",
19 | 			list1.get_size());
20 | 	printf("There are %d objs in list3\n", list3.get_size());
21 | 
22 | 	list3.add_list(&list2);
23 | 	printf("After moving objs from list 2 to list 3, there are %d objs in list2\n",
24 | 			list2.get_size());
25 | 	printf("There are %d objs in list3\n", list3.get_size());
26 | 
27 | 	slab_allocator::linked_obj *obj = list3.pop(500);
28 | 	int num = 0;
29 | 	while (obj != NULL) {
30 | 		obj = obj->get_next();
31 | 		num++;
32 | 	}
33 | 	printf("pop 500 objects, there are %d objs in the retuend list\n", num);
34 | 	printf("There are %d objs in list 3\n", list3.get_size());
35 | 
36 | 	obj = list3.pop(600);
37 | 	num = 0;
38 | 	while (obj != NULL) {
39 | 		obj = obj->get_next();
40 | 		num++;
41 | 	}
42 | 	printf("pop 600 objects, there are %d objs in the retuend list\n", num);
43 | 	printf("There are %d objs in list 3\n", list3.get_size());
44 | }
45 | 


--------------------------------------------------------------------------------
/libsafs/unit-test/test_mem_tracker.cpp:
--------------------------------------------------------------------------------
 1 | #include <stdio.h>
 2 | 
 3 | #include "mem_tracker.h"
 4 | 
 5 | extern size_t get_alloc_objs();
 6 | extern size_t get_alloc_bytes();
 7 | 
 8 | struct test
 9 | {
10 | 	char data[10];
11 | };
12 | 
13 | int main()
14 | {
15 | 	test *t = new test;
16 | 	printf("alloc %p\n", t);
17 | 	for (int i = 0; i < sizeof(t->data); i++)
18 | 		printf("data %d:%p: %d\n", i, &t->data[i], t->data[i]);
19 | 	printf("alloc %ld objs and %ld bytes\n",
20 | 			get_alloc_objs(), get_alloc_bytes());
21 | 	delete t;
22 | 	printf("alloc %ld objs and %ld bytes\n",
23 | 			get_alloc_objs(), get_alloc_bytes());
24 | 
25 | 	t = new test[3];
26 | 	printf("alloc %p\n", t);
27 | 	for (int j = 0; j < 3; j++)
28 | 		for (int i = 0; i < sizeof(t->data); i++)
29 | 			printf("data %d:%d:%p: %d\n", j, i, &t[j].data[i], t[j].data[i]);
30 | 	printf("alloc %ld objs and %ld bytes\n",
31 | 			get_alloc_objs(), get_alloc_bytes());
32 | 	delete [] t;
33 | 	printf("alloc %ld objs and %ld bytes\n",
34 | 			get_alloc_objs(), get_alloc_bytes());
35 | }
36 | 


--------------------------------------------------------------------------------
/libsafs/unit-test/test_open_close.cpp:
--------------------------------------------------------------------------------
 1 | #include "io_interface.h"
 2 | #include "thread.h"
 3 | #include "safs_file.h"
 4 | 
 5 | using namespace safs;
 6 | 
 7 | class test_thread: public thread
 8 | {
 9 | 	std::vector<std::string> data_files;
10 | public:
11 | 	test_thread(const std::vector<std::string> &files, int start,
12 | 			int end): thread("test", 0) {
13 | 		data_files.assign(files.begin() + start, files.begin() + end);
14 | 	}
15 | 
16 | 	void run() {
17 | 		for (unsigned i = 0; i < data_files.size(); i++) {
18 | 			create_io_factory(data_files[i], REMOTE_ACCESS);
19 | 		}
20 | 		this->stop();
21 | 	}
22 | };
23 | 
24 | int main(int argc, char *argv[])
25 | {
26 | 	const int NUM_FILES = 1000;
27 | 	const int NUM_THREADS = 10;
28 | 
29 | 	const char *opts[1];
30 | 	std::string opt = std::string("root_conf=") + argv[1];
31 | 	opts[0] = opt.c_str();
32 | 	config_map::ptr configs = config_map::create();
33 | 	configs->add_options(opts, 1);
34 | 
35 | 	init_io_system(configs);
36 | 	const RAID_config &raid = get_sys_RAID_conf();
37 | 	std::vector<std::string> data_files;
38 | 	for (int i = 0; i < NUM_FILES; i++) {
39 | 		std::string file_name = std::string("test-") + itoa(i);
40 | 		safs_file file(raid, file_name);
41 | 		assert(file.create_file(0));
42 | 		data_files.push_back(file_name);
43 | 	}
44 | 
45 | 	std::vector<thread *> threads;
46 | 	int num_files_per_thread = NUM_FILES / NUM_THREADS;
47 | 	for (int i = 0; i < NUM_THREADS; i++) {
48 | 		thread *t = new test_thread(data_files, i * num_files_per_thread,
49 | 				(i + 1) * num_files_per_thread);
50 | 		t->start();
51 | 		threads.push_back(t);
52 | 	}
53 | 	for (int i = 0; i < NUM_THREADS; i++)
54 | 		threads[i]->join();
55 | 
56 | 	for (int i = 0; i < NUM_FILES; i++) {
57 | 		std::string file_name = data_files[i];
58 | 		safs_file file(raid, file_name);
59 | 		assert(file.delete_file());
60 | 	}
61 | }
62 | 


--------------------------------------------------------------------------------
/matrix/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | project (FlashGraph)
 2 | 
 3 | include_directories("${PROJECT_SOURCE_DIR}")
 4 | 
 5 | find_package(BLAS REQUIRED)
 6 | message( STATUS BLAS found: ${BLAS_LIBRARIES} )
 7 | 
 8 | find_package(ZLIB)
 9 | if (ZLIB_FOUND)
10 | 	set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DUSE_GZIP")
11 | 	set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_GZIP")
12 | endif()
13 | 
14 | 
15 | add_library(FMatrix STATIC
16 | 	dense_matrix.cpp
17 | 	EM_vector.cpp
18 | 	sparse_matrix.cpp
19 | 	EM_dense_matrix.cpp
20 | 	matrix_io.cpp
21 | 	data_frame.cpp
22 | 	generic_type.cpp
23 | 	matrix_header.cpp
24 | 	sparse_matrix_format.cpp
25 | 	hilbert_curve.cpp
26 | 	NUMA_dense_matrix.cpp
27 | 	NUMA_vector.cpp
28 | 	raw_data_array.cpp
29 | 	mem_worker_thread.cpp
30 | 	local_matrix_store.cpp
31 | 	data_io.cpp
32 | 	local_vec_store.cpp
33 | 	mem_matrix_store.cpp
34 | 	mapply_matrix_store.cpp
35 | 	mem_vec_store.cpp
36 | 	one_val_matrix_store.cpp
37 | 	rand_gen.cpp
38 | 	matrix_store.cpp
39 | 	mem_vv_store.cpp
40 | 	local_vv_store.cpp
41 | 	vec_store.cpp
42 | 	vector.cpp
43 | 	vector_vector.cpp
44 | 	vv_store.cpp
45 | 	EM_object.cpp
46 | 	local_mem_buffer.cpp
47 | 	bulk_operate.cpp
48 | 	factor.cpp
49 | 	col_vec.cpp
50 | 	combined_matrix_store.cpp
51 | 	cached_matrix_store.cpp
52 | 	matrix_config.cpp
53 | 	agg_matrix_store.cpp
54 | 	matrix_stats.cpp
55 | 	block_matrix.cpp
56 | 	IPW_matrix_store.cpp
57 | 	sink_matrix.cpp
58 | 	project_matrix_store.cpp
59 | 	EM_vv_store.cpp
60 | 	groupby_matrix_store.cpp
61 | 	materialize.cpp
62 | 	sub_matrix_store.cpp
63 | 	set_data_matrix_store.cpp
64 | 	fm_utils.cpp
65 | 	set_rc_matrix_store.cpp
66 | 	cum_matrix.cpp
67 | )
68 | 
69 | if(ENABLE_TRILINOS)
70 | 	set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DENABLE_TRILINOS")
71 | 	set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_TRILINOS")
72 | 	subdirs(eigensolver)
73 | endif()
74 | 


--------------------------------------------------------------------------------
/matrix/EM_vv_store.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2016 Open Connectome Project (http://openconnecto.me)
 3 |  * Written by Da Zheng (zhengda1936@gmail.com)
 4 |  *
 5 |  * This file is part of FlashMatrix.
 6 |  *
 7 |  * Licensed under the Apache License, Version 2.0 (the "License");
 8 |  * you may not use this file except in compliance with the License.
 9 |  * You may obtain a copy of the License at
10 |  *
11 |  *     http://www.apache.org/licenses/LICENSE-2.0
12 |  *
13 |  * Unless required by applicable law or agreed to in writing, software
14 |  * distributed under the License is distributed on an "AS IS" BASIS,
15 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 |  * See the License for the specific language governing permissions and
17 |  * limitations under the License.
18 |  */
19 | 
20 | #include <sys/time.h>
21 | 
22 | #include <boost/format.hpp>
23 | 
24 | #include "EM_vv_store.h"
25 | #include "local_vv_store.h"
26 | 
27 | namespace fm
28 | {
29 | 
30 | namespace detail
31 | {
32 | 
33 | local_vec_store::ptr EM_vv_store::get_portion_async(off_t start,
34 | 		size_t len, portion_compute::ptr compute) const
35 | {
36 | 	if (start + len > get_num_vecs()) {
37 | 		BOOST_LOG_TRIVIAL(error) << boost::format(
38 | 				"can't get the portion [%1%, %2%)") % start % (start + len);
39 | 		return local_vec_store::ptr();
40 | 	}
41 | 
42 | 	off_t start_ele = get_vec_off(start) / get_type().get_size();
43 | 	local_vec_store::ptr data = get_EM_data().get_portion_async(start_ele,
44 | 			get_num_eles(start, len), compute);
45 | 	return local_vv_store::ptr(new local_vv_store(start, get_off_it(start),
46 | 				get_off_it(start + len + 1), data));
47 | }
48 | 
49 | }
50 | 
51 | }
52 | 


--------------------------------------------------------------------------------
/matrix/EM_vv_store.h:
--------------------------------------------------------------------------------
 1 | #ifndef __EM_VV_STORE_H__
 2 | #define __EM_VV_STORE_H__
 3 | 
 4 | /*
 5 |  * Copyright 2015 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashMatrix.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include "vv_store.h"
24 | #include "EM_object.h"
25 | #include "EM_vector.h"
26 | #include "local_vv_store.h"
27 | 
28 | namespace fm
29 | {
30 | 
31 | namespace detail
32 | {
33 | 
34 | class EM_vv_store: public vv_store, public EM_object
35 | {
36 | 	EM_vv_store(const scalar_type &type): vv_store(type, false) {
37 | 	}
38 | 	EM_vv_store(const std::vector<off_t> &offs,
39 | 			EM_vec_store::ptr store): vv_store(offs, store) {
40 | 	}
41 | 
42 | 	const EM_vec_store &get_EM_data() const {
43 | 		return static_cast<const EM_vec_store &>(get_data());
44 | 	}
45 | public:
46 | 	typedef std::shared_ptr<EM_vv_store> ptr;
47 | 	typedef std::shared_ptr<const EM_vv_store> const_ptr;
48 | 
49 | 	static ptr create(const scalar_type &type) {
50 | 		return ptr(new EM_vv_store(type));
51 | 	}
52 | 	static ptr create(const std::vector<off_t> &offs, EM_vec_store::ptr store) {
53 | 		return ptr(new EM_vv_store(offs, store));
54 | 	}
55 | 	virtual local_vec_store::ptr get_portion_async(off_t start,
56 | 			size_t len, portion_compute::ptr compute) const;
57 | 
58 | 	virtual std::vector<safs::io_interface::ptr> create_ios() const {
59 | 		return dynamic_cast<const EM_object &>(get_data()).create_ios();
60 | 	}
61 | 
62 | 	virtual vec_store::ptr shallow_copy() {
63 | 		return ptr(new EM_vv_store(*this));
64 | 	}
65 | 	virtual vec_store::const_ptr shallow_copy() const {
66 | 		return const_ptr(new EM_vv_store(*this));
67 | 	}
68 | };
69 | 
70 | }
71 | 
72 | }
73 | 
74 | #endif
75 | 


--------------------------------------------------------------------------------
/matrix/Makefile:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 Open Connectome Project (http://openconnecto.me)
 2 | # Written by Da Zheng (zhengda1936@gmail.com)
 3 | #
 4 | # This file is part of FlashMatrix.
 5 | #
 6 | # Licensed under the Apache License, Version 2.0 (the "License");
 7 | # you may not use this file except in compliance with the License.
 8 | # You may obtain a copy of the License at
 9 | #
10 | #     http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | 
18 | include ../Makefile.common
19 | 
20 | CXXFLAGS += -DUSE_GZIP -I../libsafs
21 | 
22 | all: libmatrix eigensolver test unit-test libmatrix-algs
23 | 
24 | libmatrix: $(OBJS)
25 | 	rm -f libFMatrix.a
26 | 	ar -cvq libFMatrix.a $(OBJS)
27 | 
28 | libmatrix-algs: libmatrix
29 | 	$(MAKE) -C libmatrix-algs
30 | 
31 | test: libmatrix eigensolver libmatrix-algs
32 | 	$(MAKE) -C test
33 | 
34 | unit-test: libmatrix eigensolver
35 | 	$(MAKE) -C unit-test
36 | 
37 | eigensolver: libmatrix
38 | 	$(MAKE) -C eigensolver
39 | 
40 | clean:
41 | 	rm -f *.d
42 | 	rm -f *.o
43 | 	rm -f *~
44 | 	rm -f libFMatrix.a
45 | 	find -name core -delete
46 | 	$(MAKE) -C test clean
47 | 	$(MAKE) -C unit-test clean
48 | 	$(MAKE) -C eigensolver clean
49 | 	$(MAKE) -C libmatrix-algs clean
50 | 	$(MAKE) -C libmatrix-algs clean
51 | 
52 | -include $(DEPS) 
53 | 


--------------------------------------------------------------------------------
/matrix/col_vec.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2016 Open Connectome Project (http://openconnecto.me)
 3 |  * Written by Da Zheng (zhengda1936@gmail.com)
 4 |  *
 5 |  * This file is part of FlashMatrix.
 6 |  *
 7 |  * Licensed under the Apache License, Version 2.0 (the "License");
 8 |  * you may not use this file except in compliance with the License.
 9 |  * You may obtain a copy of the License at
10 |  *
11 |  *     http://www.apache.org/licenses/LICENSE-2.0
12 |  *
13 |  * Unless required by applicable law or agreed to in writing, software
14 |  * distributed under the License is distributed on an "AS IS" BASIS,
15 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 |  * See the License for the specific language governing permissions and
17 |  * limitations under the License.
18 |  */
19 | 
20 | #include "col_vec.h"
21 | #include "vector.h"
22 | #include "mem_worker_thread.h"
23 | #include "generic_hashtable.h"
24 | 
25 | namespace fm
26 | {
27 | 
28 | col_vec::ptr col_vec::create(detail::matrix_store::const_ptr store)
29 | {
30 | 	if (store->get_num_cols() > 1 && store->get_num_rows() > 1) {
31 | 		BOOST_LOG_TRIVIAL(error)
32 | 			<< "can't convert a matrix store with multiple cols&rows to a vector";
33 | 		assert(0);
34 | 		return ptr();
35 | 	}
36 | 	dense_matrix::ptr mat = dense_matrix::create(store);
37 | 	if (mat->get_num_cols() > 1)
38 | 		mat = mat->transpose();
39 | 	if (mat->get_data().store_layout() == matrix_layout_t::L_ROW)
40 | 		mat = mat->conv2(matrix_layout_t::L_COL);
41 | 	return ptr(new col_vec(mat->get_raw_store()));
42 | }
43 | 
44 | col_vec::ptr col_vec::create(detail::vec_store::const_ptr store)
45 | {
46 | 	auto mat = store->conv2mat(store->get_length(), 1, false);
47 | 	return col_vec::create(mat);
48 | }
49 | 
50 | col_vec::ptr col_vec::create(dense_matrix::ptr mat)
51 | {
52 | 	if (mat->get_num_cols() > 1 && mat->get_num_rows() > 1) {
53 | 		BOOST_LOG_TRIVIAL(error)
54 | 			<< "can't convert a matrix with multiple cols&rows to a vector";
55 | 		return ptr();
56 | 	}
57 | 	if (mat->get_num_cols() > 1)
58 | 		mat = mat->transpose();
59 | 
60 | 	if (mat->get_data().store_layout() == matrix_layout_t::L_ROW)
61 | 		mat = mat->conv2(matrix_layout_t::L_COL);
62 | 	return ptr(new col_vec(mat->get_raw_store()));
63 | }
64 | 
65 | col_vec::ptr col_vec::create(vector::const_ptr vec)
66 | {
67 | 	dense_matrix::ptr mat = vec->conv2mat(vec->get_length(), 1, false);
68 | 	assert(mat->store_layout() == matrix_layout_t::L_COL);
69 | 	if (mat == NULL)
70 | 		return col_vec::ptr();
71 | 	else
72 | 		return col_vec::create(mat);
73 | }
74 | 
75 | bool col_vec::is_seq() const
76 | {
77 | 	// TODO
78 | 	return true;
79 | }
80 | 
81 | }
82 | 


--------------------------------------------------------------------------------
/matrix/conf/run_test.txt:
--------------------------------------------------------------------------------
 1 | # Parameters for SAFS
 2 | 
 3 | # How data blocks of a SAFS file are mapped to Linux filesystem.
 4 | RAID_mapping=RAID0
 5 | 
 6 | # Allow users to write data to SAFS.
 7 | writable=
 8 | 
 9 | # The RAID block size in SAFS is 8MB. FlashMatrix requires a large RAID block
10 | # size.
11 | RAID_block_size=8388608
12 | 
13 | # The number of parallel I/Os is allowed in a single thread.
14 | io_depth=4
15 | 
16 | # The page cache size in SAFS
17 | cache_size=512M
18 | 
19 | # The number of NUMA nodes
20 | # num_nodes=1
21 | 
22 | # change FG_TOP to the path of the top directory of FlashGraph.
23 | root_conf=FG_TOP/matrix/conf/data_files.txt
24 | 
25 | # whether or not to use Linux huge pages
26 | # huge_page=
27 | 
28 | # The number of I/O threads per NUMA node.
29 | # num_io_threads=1
30 | 
31 | 
32 | # Parameters for FlashMatrix
33 | 
34 | # The number of worker threads to process a matrix.
35 | # threads=1
36 | 


--------------------------------------------------------------------------------
/matrix/eigensolver/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | project (FlashGraph)
 2 | 
 3 | include_directories("${PROJECT_SOURCE_DIR}")
 4 | 
 5 | add_library(eigen STATIC
 6 | 	block_dense_matrix.cpp
 7 | 	eigensolver.cpp
 8 | 	collected_col_matrix_store.cpp
 9 | )
10 | 


--------------------------------------------------------------------------------
/matrix/eigensolver/Makefile:
--------------------------------------------------------------------------------
 1 | include ../../Makefile.common
 2 | 
 3 | all: libeigen
 4 | 
 5 | CXXFLAGS += -I.. -I../../libsafs -I../../flash-graph
 6 | 
 7 | libeigen: $(OBJS)
 8 | 	rm -f libeigen.a
 9 | 	ar -cvq libeigen.a $(OBJS)
10 | 
11 | clean:
12 | 	rm -f *~
13 | 	rm -f *.o
14 | 	rm -f *.a
15 | 	rm -f *.d
16 | 
17 | -include $(DEPS) 
18 | 


--------------------------------------------------------------------------------
/matrix/eigensolver/eigensolver.h:
--------------------------------------------------------------------------------
 1 | #ifndef __EIGENSOLVER_H__
 2 | #define __EIGENSOLVER_H__
 3 | 
 4 | /*
 5 |  * Copyright 2015 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashMatrix.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | #include <vector>
23 | #include <string>
24 | #include <memory>
25 | 
26 | #include "dense_matrix.h"
27 | #include "matrix_store.h"
28 | 
29 | namespace fm
30 | {
31 | 
32 | namespace eigen
33 | {
34 | 
35 | class spm_function
36 | {
37 | public:
38 | 	typedef std::shared_ptr<const spm_function> const_ptr;
39 | 
40 | 	/*
41 | 	 * We pass a reference of the shared pointer to the run method.
42 | 	 * So the user-defined run method can deallocate the input dense matrix.
43 | 	 */
44 | 	virtual fm::dense_matrix::ptr run(fm::dense_matrix::ptr &x) const = 0;
45 | 	virtual size_t get_num_cols() const = 0;
46 | 	virtual size_t get_num_rows() const = 0;
47 | };
48 | 
49 | struct eigen_res
50 | {
51 | 	std::vector<double> vals;
52 | 	fm::dense_matrix::ptr vecs;
53 | 	struct eigen_status {
54 | 		size_t num_iters;
55 | 		size_t num_ops;
56 | 	} status;
57 | };
58 | 
59 | struct eigen_options
60 | {
61 | 	double tol;
62 | 	int num_blocks;
63 | 	int max_restarts;
64 | 	int max_iters;
65 | 	int block_size;
66 | 	int nev;
67 | 	std::string solver;
68 | 	std::string which;
69 | 	bool in_mem;
70 | 
71 | 	eigen_options();
72 | 
73 | 	bool init(int nev, std::string solver = "KrylovSchur");
74 | };
75 | 
76 | /*
77 |  * `func' will be destroyed by this function.
78 |  */
79 | eigen_res compute_eigen(spm_function *func, bool sym,
80 | 		struct eigen_options &opts, bool verbose=false);
81 | 
82 | }
83 | 
84 | }
85 | 
86 | #endif
87 | 


--------------------------------------------------------------------------------
/matrix/fm_utils.h:
--------------------------------------------------------------------------------
 1 | #ifndef __FM_UTILS_H__
 2 | #define __FM_UTILS_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashMatrix.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include "sparse_matrix_format.h"
24 | #include "data_frame.h"
25 | 
26 | namespace fm
27 | {
28 | 
29 | class sparse_matrix;
30 | 
31 | typedef uint32_t ele_idx_t;
32 | const ele_idx_t INVALID_IDX_VAL = std::numeric_limits<ele_idx_t>::max();
33 | 
34 | void set_deduplicate(bool v);
35 | void set_remove_self_edge(bool v);
36 | 
37 | std::shared_ptr<sparse_matrix> create_2d_matrix(data_frame::ptr df,
38 | 		const block_2d_size &block_size, const fm::scalar_type *entry_type,
39 | 		bool is_sym, const std::string &name = "");
40 | 
41 | }
42 | 
43 | #endif
44 | 


--------------------------------------------------------------------------------
/matrix/hilbert_curve.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * The code in this file is copied from wikipedia.
 3 |  * http://en.wikipedia.org/wiki/Hilbert_curve
 4 |  */
 5 | #include "hilbert_curve.h"
 6 | 
 7 | //rotate/flip a quadrant appropriately
 8 | void hilbert_rot(int n, int *x, int *y, int rx, int ry)
 9 | {
10 | 	if (ry == 0) {
11 | 		if (rx == 1) {
12 | 			*x = n-1 - *x;
13 | 			*y = n-1 - *y;
14 | 		}
15 | 
16 | 		//Swap x and y
17 | 		int t  = *x;
18 | 		*x = *y;
19 | 		*y = t;
20 | 	}
21 | }
22 | 
23 | //convert (x,y) to d
24 | int hilbert_xy2d (int n, int x, int y)
25 | {
26 | 	int rx, ry, s, d=0;
27 | 	for (s=n/2; s>0; s/=2) {
28 | 		rx = (x & s) > 0;
29 | 		ry = (y & s) > 0;
30 | 		d += s * s * ((3 * rx) ^ ry);
31 | 		hilbert_rot(s, &x, &y, rx, ry);
32 | 	}
33 | 	return d;
34 | }
35 | 


--------------------------------------------------------------------------------
/matrix/hilbert_curve.h:
--------------------------------------------------------------------------------
 1 | #ifndef __HILBERT_CURVE_H__
 2 | #define __HILBERT_CURVE_H__
 3 | 
 4 | /*
 5 |  * This converts 2d coordinates x and y into the location in the hilbert curve.
 6 |  * n indicates the size of the square (2^n x 2^n).
 7 |  */
 8 | int hilbert_xy2d (int n, int x, int y);
 9 | 
10 | #endif
11 | 


--------------------------------------------------------------------------------
/matrix/libmatrix-algs/Makefile:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 Open Connectome Project (http://openconnecto.me)
 2 | # Written by Da Zheng (zhengda1936@gmail.com)
 3 | #
 4 | # This file is part of FlashMatrix.
 5 | #
 6 | # Licensed under the Apache License, Version 2.0 (the "License");
 7 | # you may not use this file except in compliance with the License.
 8 | # You may obtain a copy of the License at
 9 | #
10 | #     http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | 
18 | include ../../Makefile.common
19 | 
20 | CXXFLAGS += -DUSE_GZIP -I../../libsafs -I../../flash-graph -I..
21 | 
22 | all: libmatrix-algs
23 | 
24 | libmatrix-algs: $(OBJS)
25 | 	rm -f libFMatrix-algs.a
26 | 	ar -cvq libFMatrix-algs.a $(OBJS)
27 | 
28 | clean:
29 | 	rm -f *.d
30 | 	rm -f *.o
31 | 	rm -f *~
32 | 	rm -f libFMatrix-algs.a
33 | 	find -name core -delete
34 | 
35 | -include $(DEPS) 
36 | 


--------------------------------------------------------------------------------
/matrix/libmatrix-algs/matrix_algs.h:
--------------------------------------------------------------------------------
 1 | #ifndef __MATRIX_ALGS_H__
 2 | #define __MATRIX_ALGS_H__
 3 | 
 4 | /*
 5 |  * Copyright 2015 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashMatrix.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include "sparse_matrix.h"
24 | #include "dense_matrix.h"
25 | 
26 | namespace fm
27 | {
28 | 
29 | namespace alg
30 | {
31 | 
32 | /**
33 |   * \brief Compute PageRank.
34 |   *
35 |   * \param mat The adjacency matrix of a sparse graph.
36 |   * \param max_niters The maximal number of iterations.
37 |   * \param damping_factor The damping factor. Originally .85.
38 |   * \param num_in_mem The number of vectors that allow to be in memory.
39 |   * \return A one-column dense matrix with a PageRank value for each vertex
40 |   * in the graph.
41 |   */
42 | dense_matrix::ptr PageRank(sparse_matrix::ptr mat, size_t max_niters,
43 | 		float damping_factor, size_t num_in_mem);
44 | 
45 | std::pair<dense_matrix::ptr, dense_matrix::ptr> NMF(sparse_matrix::ptr mat,
46 | 		size_t k, size_t max_niters, size_t num_in_mem);
47 | }
48 | 
49 | }
50 | 
51 | #endif
52 | 


--------------------------------------------------------------------------------
/matrix/matrix_exception.h:
--------------------------------------------------------------------------------
 1 | #ifndef __MATRIX_EXCEPTION_H__
 2 | #define __MATRIX_EXCEPTION_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashMatrix.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | namespace fm
24 | {
25 | 
26 | class wrong_format: public std::exception
27 | {
28 | 	std::string msg;
29 | public:
30 | 	wrong_format(const std::string &msg) {
31 | 		this->msg = msg;
32 | 	}
33 | 
34 | 	~wrong_format() throw() {
35 | 	}
36 | 
37 | 	const char* what() const throw() {
38 | 		return msg.c_str();
39 | 	}
40 | };
41 | 
42 | class alloc_error: public std::exception
43 | {
44 | 	std::string msg;
45 | public:
46 | 	alloc_error(const std::string &msg) {
47 | 		this->msg = msg;
48 | 	}
49 | 
50 | 	~alloc_error() throw() {
51 | 	}
52 | 
53 | 	const char* what() const throw() {
54 | 		return msg.c_str();
55 | 	}
56 | };
57 | 
58 | }
59 | 
60 | #endif
61 | 


--------------------------------------------------------------------------------
/matrix/matrix_header.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 3 |  * Written by Da Zheng (zhengda1936@gmail.com)
 4 |  *
 5 |  * This file is part of FlashMatrix.
 6 |  *
 7 |  * Licensed under the Apache License, Version 2.0 (the "License");
 8 |  * you may not use this file except in compliance with the License.
 9 |  * You may obtain a copy of the License at
10 |  *
11 |  *     http://www.apache.org/licenses/LICENSE-2.0
12 |  *
13 |  * Unless required by applicable law or agreed to in writing, software
14 |  * distributed under the License is distributed on an "AS IS" BASIS,
15 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 |  * See the License for the specific language governing permissions and
17 |  * limitations under the License.
18 |  */
19 | 
20 | #include <boost/format.hpp>
21 | 
22 | #include "matrix_header.h"
23 | #include "matrix_exception.h"
24 | 
25 | namespace fm
26 | {
27 | 
28 | void matrix_header::init(matrix_type mat_type, size_t entry_size, size_t nrows,
29 | 		size_t ncols, matrix_layout_t layout, prim_type data_type)
30 | {
31 | 	memset(u.page, 0, sizeof(u.page));
32 | 
33 | 	u.d.magic_number = MAGIC_NUMBER;
34 | 	u.d.version_number = CURR_VERSION;
35 | 	u.d.type = mat_type;
36 | 	u.d.entry_size = entry_size;
37 | 	u.d.nrows = nrows;
38 | 	u.d.ncols = ncols;
39 | 	u.d.layout = layout;
40 | 	u.d.data_type = data_type;
41 | }
42 | 
43 | matrix_header::matrix_header(matrix_type mat_type, size_t entry_size,
44 | 		size_t nrows, size_t ncols, matrix_layout_t layout,
45 | 		prim_type data_type, const block_2d_size &block_size)
46 | {
47 | 	init(mat_type, entry_size, nrows, ncols, layout, data_type);
48 | 	u.d.block_2d_height = block_size.get_num_rows();
49 | 	u.d.block_2d_width = block_size.get_num_cols();
50 | }
51 | 
52 | void matrix_header::verify() const
53 | {
54 | 	if (!is_matrix_file())
55 | 		throw wrong_format(boost::str(boost::format("wrong magic number: %1%")
56 | 					% u.d.magic_number));
57 | 	if (!is_right_version())
58 | 		throw wrong_format(boost::str(boost::format("wrong version number: %1%")
59 | 					% u.d.version_number));
60 | }
61 | 
62 | }
63 | 


--------------------------------------------------------------------------------
/matrix/matrix_stats.h:
--------------------------------------------------------------------------------
 1 | #ifndef __MATRIX_STATS_H__
 2 | #define __MATRIX_STATS_H__
 3 | 
 4 | /*
 5 |  * Copyright 2015 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashMatrix.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include <stdlib.h>
24 | 
25 | #include <atomic>
26 | 
27 | namespace fm
28 | {
29 | 
30 | namespace detail
31 | {
32 | 
33 | /*
34 |  * This class maintains the statistics of I/O and computation that occurs
35 |  * to matrices.
36 |  */
37 | class matrix_stats_t
38 | {
39 | 	std::atomic<size_t> mem_read_bytes;
40 | 	std::atomic<size_t> mem_write_bytes;
41 | 	std::atomic<size_t> EM_read_bytes;
42 | 	std::atomic<size_t> EM_write_bytes;
43 | 	std::atomic<size_t> double_multiplies;
44 | public:
45 | 	matrix_stats_t() {
46 | 		mem_read_bytes = 0;
47 | 		mem_write_bytes = 0;
48 | 		EM_read_bytes = 0;
49 | 		EM_write_bytes = 0;
50 | 		double_multiplies = 0;
51 | 	}
52 | 
53 | 	matrix_stats_t(const matrix_stats_t &stats) {
54 | 		mem_read_bytes = stats.mem_read_bytes.load();
55 | 		mem_write_bytes = stats.mem_write_bytes.load();
56 | 		EM_read_bytes = stats.EM_read_bytes.load();
57 | 		EM_write_bytes = stats.EM_write_bytes.load();
58 | 		double_multiplies = stats.double_multiplies.load();
59 | 	}
60 | 
61 | 	size_t inc_read_bytes(size_t bytes, bool in_mem);
62 | 	size_t get_read_bytes(bool in_mem) const;
63 | 	size_t inc_write_bytes(size_t bytes, bool in_mem);
64 | 	size_t get_write_bytes(bool in_mem) const;
65 | 	size_t inc_multiplies(size_t multiplies);
66 | 	size_t get_multiplies() const;
67 | 
68 | 	void print_diff(const matrix_stats_t &orig) const;
69 | };
70 | 
71 | extern matrix_stats_t matrix_stats;
72 | 
73 | }
74 | 
75 | }
76 | 
77 | #endif
78 | 


--------------------------------------------------------------------------------
/matrix/mem_vv_store.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2015 Open Connectome Project (http://openconnecto.me)
 3 |  * Written by Da Zheng (zhengda1936@gmail.com)
 4 |  *
 5 |  * This file is part of FlashMatrix.
 6 |  *
 7 |  * Licensed under the Apache License, Version 2.0 (the "License");
 8 |  * you may not use this file except in compliance with the License.
 9 |  * You may obtain a copy of the License at
10 |  *
11 |  *     http://www.apache.org/licenses/LICENSE-2.0
12 |  *
13 |  * Unless required by applicable law or agreed to in writing, software
14 |  * distributed under the License is distributed on an "AS IS" BASIS,
15 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 |  * See the License for the specific language governing permissions and
17 |  * limitations under the License.
18 |  */
19 | 
20 | #include <boost/format.hpp>
21 | 
22 | #include "log.h"
23 | 
24 | #include "vector_vector.h"
25 | #include "mem_vv_store.h"
26 | #include "local_vv_store.h"
27 | #include "local_vec_store.h"
28 | 
29 | namespace fm
30 | {
31 | 
32 | namespace detail
33 | {
34 | 
35 | mem_vv_store::ptr mem_vv_store::cast(vec_store::ptr store)
36 | {
37 | 	if (!store->is_in_mem() || store->get_entry_size() != 0) {
38 | 		BOOST_LOG_TRIVIAL(error) << "This isn't mem vv store\n";
39 | 		return mem_vv_store::ptr();
40 | 	}
41 | 	return std::static_pointer_cast<mem_vv_store>(store);
42 | }
43 | 
44 | void mem_vv_store::set_data(const set_vv_operate &op)
45 | {
46 | #pragma omp parallel for
47 | 	for (size_t i = 0; i < get_num_vecs(); i++)
48 | 		op.set(i, get_raw_arr(i), get_length(i));
49 | }
50 | 
51 | }
52 | 
53 | }
54 | 


--------------------------------------------------------------------------------
/matrix/rand_gen.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 3 |  * Written by Da Zheng (zhengda1936@gmail.com)
 4 |  *
 5 |  * This file is part of FlashMatrix.
 6 |  *
 7 |  * Licensed under the Apache License, Version 2.0 (the "License");
 8 |  * you may not use this file except in compliance with the License.
 9 |  * You may obtain a copy of the License at
10 |  *
11 |  *     http://www.apache.org/licenses/LICENSE-2.0
12 |  *
13 |  * Unless required by applicable law or agreed to in writing, software
14 |  * distributed under the License is distributed on an "AS IS" BASIS,
15 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 |  * See the License for the specific language governing permissions and
17 |  * limitations under the License.
18 |  */
19 | 
20 | #include "rand_gen.h"
21 | 
22 | namespace fm
23 | {
24 | 
25 | unsigned int rand_gen::gen_rand_seed()
26 | {
27 | 	static std::random_device rd;
28 | 	return rd();
29 | }
30 | 
31 | ////////////////////////// Random distribution /////////////////////////////
32 | 
33 | void urand_gen_impl<int64_t>::gen(void *buf, size_t len)
34 | {
35 | 	int64_t *t_buf = (int64_t *) buf;
36 | 	for (size_t i = 0; i < len; i++)
37 | 		t_buf[i] = dist(generator);
38 | }
39 | 
40 | void urand_gen_impl<float>::gen(void *buf, size_t len)
41 | {
42 | 	float *t_buf = (float *) buf;
43 | 	for (size_t i = 0; i < len; i++)
44 | 		t_buf[i] = dist(generator);
45 | }
46 | 
47 | void urand_gen_impl<double>::gen(void *buf, size_t len)
48 | {
49 | 	double *t_buf = (double *) buf;
50 | 	for (size_t i = 0; i < len; i++)
51 | 		t_buf[i] = dist(generator);
52 | }
53 | 
54 | void urand_gen_impl<long double>::gen(void *buf, size_t len)
55 | {
56 | 	long double *t_buf = (long double *) buf;
57 | 	for (size_t i = 0; i < len; i++)
58 | 		t_buf[i] = dist(generator);
59 | }
60 | 
61 | void urand_gen_impl<bool>::gen(void *buf, size_t len)
62 | {
63 | 	bool *t_buf = (bool *) buf;
64 | 	for (size_t i = 0; i < len; i++)
65 | 		t_buf[i] = dist(generator);
66 | }
67 | 
68 | ////////////////////////// Normal distribution /////////////////////////////
69 | 
70 | void nrand_gen_impl<double>::gen(void *buf, size_t len)
71 | {
72 | 	double *t_buf = (double *) buf;
73 | 	for (size_t i = 0; i < len; i++)
74 | 		t_buf[i] = dist(generator);
75 | }
76 | 
77 | void nrand_gen_impl<long double>::gen(void *buf, size_t len)
78 | {
79 | 	long double *t_buf = (long double *) buf;
80 | 	for (size_t i = 0; i < len; i++)
81 | 		t_buf[i] = dist(generator);
82 | }
83 | 
84 | }
85 | 


--------------------------------------------------------------------------------
/matrix/stl_algs.h:
--------------------------------------------------------------------------------
 1 | #ifndef __STL_ALGS_H__
 2 | #define __STL_ALGS_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashMatrix.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include <algorithm>
24 | 
25 | namespace fm
26 | {
27 | 
28 | class stl_algs
29 | {
30 | public:
31 | 	virtual off_t lower_bound(const char *start, const char *end,
32 | 			const char *val) const = 0;
33 | };
34 | 
35 | template<class T>
36 | class stl_algs_impl: public stl_algs
37 | {
38 | public:
39 | 	virtual off_t lower_bound(const char *start, const char *end,
40 | 			const char *val) const {
41 | 		const T *t_start = (const T *) start;
42 | 		const T *t_end = (const T *) end;
43 | 		T t_val = *(const T *) val;
44 | 		const T *ret = std::lower_bound(t_start, t_end, t_val);
45 | 		return ret - t_start;
46 | 	}
47 | };
48 | 
49 | }
50 | 
51 | #endif
52 | 


--------------------------------------------------------------------------------
/matrix/test/FG2Tpetra.h:
--------------------------------------------------------------------------------
 1 | #ifndef __FG2TPETRA_H__
 2 | #define __FG2TPETRA_H__
 3 | 
 4 | #include "sparse_matrix.h"
 5 | #include "vertex.h"
 6 | #include "vertex_index.h"
 7 | #include "in_mem_storage.h"
 8 | #include "io_interface.h"
 9 | 
10 | #include "Tpetra_Map.hpp"
11 | #include "Tpetra_MultiVector.hpp"
12 | #include "Tpetra_DefaultPlatform.hpp"
13 | #include "Tpetra_CrsMatrix_decl.hpp"
14 | #include "Tpetra_CrsMatrix_def.hpp"
15 | 
16 | using Teuchos::RCP;
17 | using Teuchos::rcp;
18 | using Teuchos::ArrayRCP;
19 | using std::cerr;
20 | using std::cout;
21 | using std::endl;
22 | 
23 | typedef size_t local_ordinal_type;
24 | typedef size_t global_ordinal_type;
25 | 
26 | typedef Tpetra::MultiVector<double, global_ordinal_type, global_ordinal_type> MV;
27 | 
28 | typedef Tpetra::Map<local_ordinal_type, global_ordinal_type> map_type;
29 | typedef Tpetra::CrsMatrix<double, local_ordinal_type, global_ordinal_type> crs_matrix_type;
30 | 
31 | RCP<crs_matrix_type> create_crs(const std::string &graph_file,
32 | 		fg::vertex_index::ptr index, fg::edge_type type,
33 | 		RCP<map_type> map, int my_rank, size_t edge_data_size);
34 | 
35 | #endif
36 | 


--------------------------------------------------------------------------------
/matrix/test/benchmark_utils/Makefile:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 Open Connectome Project (http://openconnecto.me)
 2 | # Written by Disa Mhembere (disa@jh.edu)
 3 | #
 4 | # This file is part of FlashMatrix.
 5 | #
 6 | # Licensed under the Apache License, Version 2.0 (the "License");
 7 | # you may not use this file except in compliance with the License.
 8 | # You may obtain a copy of the License at
 9 | #
10 | #     http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | 
18 | include ../../../Makefile.common
19 | 
20 | OMP_FLAG = -fopenmp
21 | CXXFLAGS := -g -std=c++0x -I../../../libsafs -I.. $(OMP_FLAG)
22 | LDFLAGS := $(OMP_FLAG) -L../../eigensolver -leigen -L../../ -lFMatrix -L../../../libsafs -lsafs $(LDFLAGS)
23 | LDFLAGS += -lnuma -lz -laio -lcblas
24 | 
25 | all: convert_matrix create_matrix
26 | 
27 | 
28 | convert_matrix: convert_matrix.o ../../libFMatrix.a
29 | 	$(CXX) -o convert_matrix convert_matrix.o $(LDFLAGS)
30 | 
31 | create_matrix: create_matrix.o ../../libFMatrix.a
32 | 	$(CXX) -o create_matrix create_matrix.o $(LDFLAGS)
33 | 
34 | clean:
35 | 	rm -f *.d
36 | 	rm -f *.o
37 | 	rm -f *~
38 | 	rm -f convert_matrix
39 | 	rm -f create_matrix
40 | 
41 | -include $(DEPS) 
42 | 


--------------------------------------------------------------------------------
/matrix/test/crs_header.h:
--------------------------------------------------------------------------------
 1 | #ifndef __CRS_HEADER_H__
 2 | #define __CRS_HEADER_H__
 3 | 
 4 | /*
 5 |  * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 6 |  * Written by Da Zheng (zhengda1936@gmail.com)
 7 |  *
 8 |  * This file is part of FlashMatrix.
 9 |  *
10 |  * Licensed under the Apache License, Version 2.0 (the "License");
11 |  * you may not use this file except in compliance with the License.
12 |  * You may obtain a copy of the License at
13 |  *
14 |  *     http://www.apache.org/licenses/LICENSE-2.0
15 |  *
16 |  * Unless required by applicable law or agreed to in writing, software
17 |  * distributed under the License is distributed on an "AS IS" BASIS,
18 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 |  * See the License for the specific language governing permissions and
20 |  * limitations under the License.
21 |  */
22 | 
23 | #include <stdlib.h>
24 | 
25 | class crs_header
26 | {
27 | 	union {
28 | 		char page[4096];
29 | 		struct {
30 | 			size_t nrows;
31 | 			size_t ncols;
32 | 			size_t nnz;
33 | 		} s;
34 | 	} data;
35 | public:
36 | 	crs_header() {
37 | 		memset(this, 0, sizeof(*this));
38 | 	}
39 | 
40 | 	crs_header(size_t nrows, size_t ncols, size_t nnz) {
41 | 		data.s.nrows = nrows;
42 | 		data.s.ncols = ncols;
43 | 		data.s.nnz = nnz;
44 | 	}
45 | 
46 | 	size_t get_num_rows() const {
47 | 		return data.s.nrows;
48 | 	}
49 | 
50 | 	size_t get_num_cols() const {
51 | 		return data.s.ncols;
52 | 	}
53 | 
54 | 	size_t get_num_non_zeros() const {
55 | 		return data.s.nnz;
56 | 	}
57 | };
58 | 
59 | typedef size_t crs_idx_t;
60 | 
61 | #endif
62 | 


--------------------------------------------------------------------------------
/matrix/test/test-dgemm.cpp:
--------------------------------------------------------------------------------
 1 | #include <stdio.h>
 2 | #include <stdlib.h>
 3 | #include <sys/time.h>
 4 | #include <malloc.h>
 5 | 
 6 | #ifdef USE_MKL
 7 | #define MKL_ILP64
 8 | #define FLEXCOMPLEX
 9 | #define MKL_INT size_t
10 | #include <mkl.h>
11 | #else
12 | #include <cblas.h>
13 | #endif
14 | 
15 | size_t num_repeats = 1;
16 | 
17 | inline static float time_diff(struct timeval time1, struct timeval time2)
18 | {
19 | 	return time2.tv_sec - time1.tv_sec
20 | 			+ ((float)(time2.tv_usec - time1.tv_usec))/1000000;
21 | }
22 | 
23 | void test_dgemm(size_t m, size_t n, size_t k)
24 | {
25 | 	struct timeval start, end;
26 | 	double *A, *B, *C;
27 | 
28 | 	printf("test col-wise matrix in MKL BLAS: M(%ld x %ld) * M(%ld %ld)\n",
29 | 			m, k, k, n);
30 | 	A = (double *) memalign(64, m*k*sizeof( double ));
31 | 	B = (double *) memalign(64, k*n*sizeof( double ));
32 | 	if (A == NULL || B == NULL) {
33 | 		printf( "\n ERROR: Can't allocate memory for matrices. Aborting... \n\n");
34 | 		free(A);
35 | 		free(B);
36 | 		return;
37 | 	}
38 | #pragma omp parallel for
39 | 	for (size_t i = 0; i < (m*k); i++) {
40 | 		A[i] = (double)(i+1);
41 | 	}
42 | 
43 | #pragma omp parallel for
44 | 	for (size_t i = 0; i < (k*n); i++) {
45 | 		B[i] = (double)(-i-1);
46 | 	}
47 | 
48 | //#pragma omp parallel for
49 | //	for (size_t i = 0; i < (m*n); i++) {
50 | //		C[i] = 0.0;
51 | //	}
52 | 
53 | 	for (size_t i = 0; i < num_repeats; i++) {
54 | 		gettimeofday(&start, NULL);
55 | 		C = (double *) memalign(64, m*n*sizeof( double ));
56 | 		cblas_dgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, m, n, k, 1, A,
57 | 				m, B, k, 0, C, m);
58 | 		free(C);
59 | 		gettimeofday(&end, NULL);
60 | 		printf("It takes %.3f seconds\n", time_diff(start, end));
61 | 	}
62 | 
63 | 	free(A);
64 | 	free(B);
65 | }
66 | 
67 | void test_dgemms(size_t block_size, size_t max_num_blocks)
68 | {
69 | 	size_t long_dim = 60 * 1024 * 1024;
70 | 	for (size_t num_blocks = 1; num_blocks <= max_num_blocks; num_blocks *= 2)
71 | 		test_dgemm(long_dim, block_size, num_blocks * block_size);
72 | 
73 | 	for (size_t num_blocks = 1; num_blocks <= max_num_blocks; num_blocks *= 2)
74 | 		test_dgemm(num_blocks * block_size, block_size, long_dim);
75 | }
76 | 
77 | int main()
78 | {
79 | 	test_dgemms(4, 128);
80 | 	test_dgemms(64, 8);
81 | }
82 | 


--------------------------------------------------------------------------------
/matrix/test/test-mem_vector.cpp:
--------------------------------------------------------------------------------
 1 | #include <boost/format.hpp>
 2 | 
 3 | #include "log.h"
 4 | #include "common.h"
 5 | 
 6 | #include "mem_vec_store.h"
 7 | #include "vector.h"
 8 | 
 9 | using namespace fm;
10 | 
11 | detail::smp_vec_store::ptr get(const detail::smp_vec_store &vec,
12 | 		detail::smp_vec_store &idxs)
13 | {
14 | 	detail::smp_vec_store::ptr ret = detail::smp_vec_store::create(idxs.get_length(),
15 | 			vec.get_type());
16 | #pragma omp parallel for
17 | 	for (size_t i = 0; i < idxs.get_length(); i++) {
18 | 		off_t idx = idxs.get<off_t>(i);
19 | 		// Check if it's out of the range.
20 | 		if (idx < 0 && (size_t) idx >= vec.get_length()) {
21 | 			BOOST_LOG_TRIVIAL(error)
22 | 				<< boost::format("%1% is out of range") % idx;
23 | 			continue;
24 | 		}
25 | 
26 | 		ret->set(i, vec.get<int>(idx));
27 | 	}
28 | 	return ret;
29 | }
30 | 
31 | /*
32 |  * This is to measure the performance difference of random permutation
33 |  * with and without compile-time type.
34 |  */
35 | void test_permute()
36 | {
37 | 	printf("test permutation\n");
38 | 	detail::smp_vec_store::ptr vec = detail::smp_vec_store::create(1000000000,
39 | 			get_scalar_type<int>());
40 | 	for (size_t i = 0; i < vec->get_length(); i++)
41 | 		vec->set(i, random());
42 | 	detail::smp_vec_store::ptr clone = detail::smp_vec_store::cast(vec->deep_copy());
43 | 	vector::ptr vec1 = vector::create(clone);
44 | 	vector::ptr vec2 = vector::create(vec);
45 | 	assert(vec1->equals(*vec2));
46 | 
47 | 	struct timeval start, end;
48 | 	gettimeofday(&start, NULL);
49 | 	detail::smp_vec_store::ptr idxs = detail::smp_vec_store::cast(
50 | 			vec->sort_with_index());
51 | 	gettimeofday(&end, NULL);
52 | 	printf("sort takes %fseconds\n", time_diff(start, end));
53 | 
54 | 	gettimeofday(&start, NULL);
55 | 	// This has compile-time type.
56 | 	detail::smp_vec_store::ptr sorted1 = get(*clone, *idxs);
57 | 	gettimeofday(&end, NULL);
58 | 	printf("permute with type takes %fseconds\n", time_diff(start, end));
59 | 
60 | 	gettimeofday(&start, NULL);
61 | 	// This doesn't have compile-time type.
62 | 	detail::smp_vec_store::ptr sorted2 = clone->detail::smp_vec_store::get(*idxs);
63 | 	gettimeofday(&end, NULL);
64 | 	printf("permute without type takes %fseconds\n", time_diff(start, end));
65 | 	vec1 = vector::create(sorted1);
66 | 	vec2 = vector::create(sorted2);
67 | 	assert(vec1->equals(*vec2));
68 | }
69 | 
70 | int main()
71 | {
72 | 	test_permute();
73 | }
74 | 


--------------------------------------------------------------------------------
/matrix/test/test-sort.cpp:
--------------------------------------------------------------------------------
 1 | #include "NUMA_vector.h"
 2 | #include "matrix_config.h"
 3 | #include "vector.h"
 4 | #include "mem_worker_thread.h"
 5 | 
 6 | const size_t num_eles = 1024 * 1024 * 1024;
 7 | 
 8 | using namespace fm;
 9 | 
10 | int num_nodes = 1;
11 | int nthreads = 8;
12 | 
13 | class rand_set: public type_set_vec_operate<long>
14 | {
15 | public:
16 | 	void set(long *arr, size_t num_eles, off_t start_idx) const {
17 | 		for (size_t i = 0; i < num_eles; i++)
18 | 			arr[i] = random();
19 | 	}
20 | };
21 | 
22 | void test_sort()
23 | {
24 | 	size_t len = (random() % num_eles) + num_eles;
25 | 	vector::ptr mem_vec = vector::create(len, get_scalar_type<long>(), -1, true,
26 | 			rand_set());
27 | 	detail::NUMA_vec_store::ptr numa_vec = detail::NUMA_vec_store::create(len,
28 | 			num_nodes, get_scalar_type<long>());
29 | 	numa_vec->copy_from(
30 | 			dynamic_cast<const detail::mem_vec_store &>(mem_vec->get_data()).get_raw_arr(),
31 | 			mem_vec->get_length() * sizeof(long));
32 | 
33 | 	struct timeval start, end;
34 | 	gettimeofday(&start, NULL);
35 | 	numa_vec->sort();
36 | 	gettimeofday(&end, NULL);
37 | 	printf("sort %ld elements in NUMA vector takes %.3f\n", numa_vec->get_length(),
38 | 			time_diff(start, end));
39 | 
40 | 	gettimeofday(&start, NULL);
41 | 	mem_vec->sort();
42 | 	gettimeofday(&end, NULL);
43 | 	printf("sort %ld elements in mem vector takes %.3f\n", numa_vec->get_length(),
44 | 			time_diff(start, end));
45 | }
46 | 
47 | int main(int argc, char *argv[])
48 | {
49 | 	if (argc >= 3) {
50 | 		num_nodes = atoi(argv[1]);
51 | 		nthreads = atoi(argv[2]);
52 | 	}
53 | 
54 | 	matrix_conf.set_num_nodes(num_nodes);
55 | 	matrix_conf.set_num_DM_threads(nthreads);
56 | 	detail::mem_thread_pool::init_global_mem_threads(num_nodes,
57 | 			nthreads / num_nodes);
58 | 	test_sort();
59 | }
60 | 


--------------------------------------------------------------------------------
/matrix/test/test-tpetra_multiply.cpp:
--------------------------------------------------------------------------------
 1 | #include "FG2Tpetra.h"
 2 | 
 3 | using namespace fm;
 4 | 
 5 | void test_tpetra(const std::string graph_file, fg::vertex_index::ptr index)
 6 | {
 7 | 	typedef Tpetra::DefaultPlatform::DefaultPlatformType Platform;
 8 | 
 9 | 	Platform &platform = Tpetra::DefaultPlatform::getDefaultPlatform();
10 | 	RCP<const Teuchos::Comm<int> > comm = platform.getComm();
11 | 	const int myRank = comm->getRank ();
12 | 	RCP<map_type> Map = rcp (new map_type(
13 | 				index->get_graph_header().get_num_vertices(), 0, comm));
14 | 
15 | 	struct timeval start, end;
16 | 	gettimeofday(&start, NULL);
17 | 	RCP<crs_matrix_type> A = create_crs(graph_file, index,
18 | 			fg::edge_type::OUT_EDGE, Map, myRank,
19 | 			index->get_graph_header().get_edge_data_size());
20 | 	gettimeofday(&end, NULL);
21 | 	printf("conversion takes %.3f seconds\n", time_diff(start, end));
22 | 
23 | 	for (size_t num_vecs = 1; num_vecs <= 16; num_vecs *= 2) {
24 | 		if (myRank == 0)
25 | 			printf("SpMM: #vecs: %ld\n", num_vecs);
26 | 		RCP<MV> evecs = rcp(new MV(Map, num_vecs));
27 | 		RCP<MV> res = rcp(new MV(Map, num_vecs));
28 | 		evecs->randomize ();
29 | 
30 | 		printf("start SpMM, A on proc %d has %ld rows and %ld cols\n",
31 | 				myRank, A->getNodeNumRows(), A->getNodeNumCols());
32 | 		struct timeval start, end;
33 | 		gettimeofday(&start, NULL);
34 | 		A->apply (*evecs, *res, Teuchos::NO_TRANS);
35 | 		gettimeofday(&end, NULL);
36 | 		printf("SpMM(%ld) takes %.3fs\n", num_vecs, time_diff(start, end));
37 | 		comm->barrier();
38 | 	}
39 | }
40 | 
41 | int main(int argc, char *argv[])
42 | {
43 | 	if (argc < 3) {
44 | 		fprintf(stderr, "multiply graph_file index_file\n");
45 | 		exit(1);
46 | 	}
47 | 
48 | 	std::string graph_file = argv[1];
49 | 	std::string index_file = argv[2];
50 | 	fg::vertex_index::ptr index = fg::vertex_index::load(index_file);
51 | 
52 | 	Teuchos::GlobalMPISession mpiSession (&argc, &argv);
53 | 	printf("MPI initialized: %d, #procs: %d, rank: %d\n",
54 | 			Teuchos::GlobalMPISession::mpiIsInitialized(),
55 | 			Teuchos::GlobalMPISession::getNProc(),
56 | 			Teuchos::GlobalMPISession::getRank());
57 | 
58 | 	test_tpetra(graph_file, index);
59 | }
60 | 


--------------------------------------------------------------------------------
/matrix/unit-test/data_files.txt:
--------------------------------------------------------------------------------
1 | # change FG_TOP to the location of the top directory of FlashGraph.
2 | 0:data
3 | 


--------------------------------------------------------------------------------
/matrix/unit-test/run_test.txt:
--------------------------------------------------------------------------------
 1 | # Parameters for SAFS
 2 | 
 3 | # How data blocks of a SAFS file are mapped to Linux filesystem.
 4 | RAID_mapping=RAID0
 5 | 
 6 | # Allow users to write data to SAFS.
 7 | writable=
 8 | 
 9 | # The RAID block size in SAFS is 8MB. FlashMatrix requires a large RAID block
10 | # size.
11 | RAID_block_size=8388608
12 | 
13 | # The number of parallel I/Os is allowed in a single thread.
14 | io_depth=4
15 | 
16 | # The page cache size in SAFS
17 | cache_size=512M
18 | 
19 | # The number of NUMA nodes
20 | # num_nodes=1
21 | 
22 | # change FG_TOP to the path of the top directory of FlashGraph.
23 | root_conf=safs_data
24 | 
25 | # whether or not to use Linux huge pages
26 | # huge_page=
27 | 
28 | # The number of I/O threads per NUMA node.
29 | # num_io_threads=1
30 | 
31 | 
32 | # Parameters for FlashMatrix
33 | 
34 | # The number of worker threads to process a matrix.
35 | # threads=1
36 | 


--------------------------------------------------------------------------------
/matrix/unit-test/test-bulk_operate.cpp:
--------------------------------------------------------------------------------
 1 | #include "bulk_operate.h"
 2 | 
 3 | using namespace fm;
 4 | 
 5 | template<class T>
 6 | void test_add(size_t num_eles)
 7 | {
 8 | 	std::vector<T> vec(num_eles);
 9 | 	for (size_t i = 0; i < vec.size(); i++)
10 | 		vec[i] = i;
11 | 
12 | 	const scalar_type &type = get_scalar_type<T>();
13 | 	const bulk_operate &op = type.get_basic_ops().get_add();
14 | 
15 | 	// Test agg
16 | 	T res;
17 | 	op.runAgg(vec.size(), vec.data(), &res);
18 | 	assert(vec.size() * (vec.size() - 1) / 2 == res);
19 | 
20 | 	// Test runAA
21 | 	std::vector<T> res_vec(num_eles);
22 | 	op.runAA(vec.size(), vec.data(), vec.data(), res_vec.data());
23 | 	for (size_t i = 0; i < res_vec.size(); i++)
24 | 		assert(vec[i] * 2 == res_vec[i]);
25 | 
26 | 	// Test runAE
27 | 	T val = 2;
28 | 	op.runAE(vec.size(), vec.data(), &val, res_vec.data());
29 | 	for (size_t i = 0; i < res_vec.size(); i++)
30 | 		assert(vec[i] + val == res_vec[i]);
31 | 
32 | 	// Test runEA
33 | 	op.runEA(vec.size(), &val, vec.data(), res_vec.data());
34 | 	for (size_t i = 0; i < res_vec.size(); i++)
35 | 		assert(vec[i] + val == res_vec[i]);
36 | }
37 | 
38 | template<class T>
39 | void test_sub(size_t num_eles)
40 | {
41 | 	std::vector<T> vec(num_eles);
42 | 	for (size_t i = 0; i < vec.size(); i++)
43 | 		vec[i] = i;
44 | 
45 | 	const scalar_type &type = get_scalar_type<T>();
46 | 	const bulk_operate &op = type.get_basic_ops().get_sub();
47 | 
48 | 	// Test runAA
49 | 	std::vector<T> res_vec(num_eles);
50 | 	op.runAA(vec.size(), vec.data(), vec.data(), res_vec.data());
51 | 	for (size_t i = 0; i < res_vec.size(); i++)
52 | 		assert(0 == res_vec[i]);
53 | 
54 | 	// Test runAE
55 | 	T val = 2;
56 | 	op.runAE(vec.size(), vec.data(), &val, res_vec.data());
57 | 	for (size_t i = 0; i < res_vec.size(); i++)
58 | 		assert(vec[i] - val == res_vec[i]);
59 | 
60 | 	// Test runEA
61 | 	op.runEA(vec.size(), &val, vec.data(), res_vec.data());
62 | 	for (size_t i = 0; i < res_vec.size(); i++)
63 | 		assert(val - vec[i] == res_vec[i]);
64 | }
65 | 
66 | int main()
67 | {
68 | 	test_add<int>(100);
69 | 	test_add<int>(10000);
70 | 	test_sub<int>(100);
71 | 	test_sub<int>(10000);
72 | 
73 | 	test_add<double>(100);
74 | 	test_add<double>(10000);
75 | 	test_sub<double>(100);
76 | 	test_sub<double>(10000);
77 | }
78 | 


--------------------------------------------------------------------------------
/matrix/unit-test/test-hashtable.cpp:
--------------------------------------------------------------------------------
 1 | #include "generic_hashtable.h"
 2 | 
 3 | using namespace fm;
 4 | 
 5 | int main()
 6 | {
 7 | 	std::vector<int> keys(1000);
 8 | 	for (size_t i = 0; i < keys.size(); i++)
 9 | 		keys[i] = random() % 100;
10 | 	std::vector<int> vals(1000);
11 | 	for (size_t i = 0; i < vals.size(); i++)
12 | 		vals[i] = 1;
13 | 
14 | 	std::unordered_map<int, int> counts;
15 | 	for (size_t i = 0; i < keys.size(); i++)
16 | 		counts.insert(std::pair<int, int>(keys[i], 0));
17 | 	for (size_t i = 0; i < keys.size(); i++)
18 | 		counts[keys[i]]++;
19 | 
20 | 	bulk_operate::const_ptr add
21 | 		= bulk_operate::conv2ptr(get_scalar_type<int>().get_basic_ops().get_add());
22 | 	agg_operate::const_ptr sum = agg_operate::create(add);
23 | 
24 | 	generic_hashtable_impl<int, sizeof(int)> ht(get_scalar_type<int>());
25 | 	generic_hashtable_impl<int, sizeof(int)> ht2(get_scalar_type<int>());
26 | 	ht.insert(keys.size(), keys.data(), vals.data(), *sum);
27 | 	data_frame::ptr df = ht.conv2df();
28 | 	detail::smp_vec_store::const_ptr key_vec = detail::smp_vec_store::cast(
29 | 			df->get_vec(0));
30 | 	detail::smp_vec_store::const_ptr val_vec = detail::smp_vec_store::cast(
31 | 			df->get_vec(1));
32 | 	for (size_t i = 0; i < key_vec->get_length(); i++) {
33 | 		int key = key_vec->get<int>(i);
34 | 		int val = val_vec->get<int>(i);
35 | 		auto ret = counts.find(key);
36 | 		assert(ret != counts.end());
37 | 		assert(ret->second == val);
38 | 	}
39 | 
40 | 	ht2.insert(keys.size(), keys.data(), vals.data(), *sum);
41 | 	ht.merge(ht2, *sum);
42 | 	df = ht.conv2df();
43 | 	key_vec = detail::smp_vec_store::cast(df->get_vec(0));
44 | 	val_vec = detail::smp_vec_store::cast(df->get_vec(1));
45 | 	for (size_t i = 0; i < key_vec->get_length(); i++) {
46 | 		int key = key_vec->get<int>(i);
47 | 		int val = val_vec->get<int>(i);
48 | 		auto ret = counts.find(key);
49 | 		assert(ret != counts.end());
50 | 		assert(ret->second * 2 == val);
51 | 	}
52 | 
53 | 	return 0;
54 | }
55 | 


--------------------------------------------------------------------------------
/matrix/unit-test/test-hilbert.cpp:
--------------------------------------------------------------------------------
 1 | #include <stdlib.h>
 2 | #include <assert.h>
 3 | 
 4 | #include <utility>
 5 | #include <vector>
 6 | #include <algorithm>
 7 | 
 8 | #include "hilbert_curve.h"
 9 | 
10 | typedef std::pair<off_t, off_t> coordinate_t;
11 | 
12 | struct coordinate_order
13 | {
14 | 	coordinate_t coo;
15 | 	size_t order;
16 | 
17 | 	bool operator<(const coordinate_order &o) const {
18 | 		return this->order < o.order;
19 | 	}
20 | };
21 | 
22 | std::vector<coordinate_t> gen_coos(size_t n)
23 | {
24 | 	std::vector<coordinate_order> orders;
25 | 	for (size_t i = 0; i < n; i++) {
26 | 		for (size_t j = 0; j < n; j++) {
27 | 			coordinate_order o;
28 | 			o.coo.first = i;
29 | 			o.coo.second = j;
30 | 			o.order = hilbert_xy2d(n, i, j);
31 | 			orders.push_back(o);
32 | 		}
33 | 	}
34 | 	std::sort(orders.begin(), orders.end());
35 | 
36 | 	std::vector<coordinate_t> coos(orders.size());
37 | 	for (size_t i = 0; i < coos.size(); i++)
38 | 		coos[i] = orders[i].coo;
39 | 	return coos;
40 | }
41 | 
42 | void verify_coos(std::vector<coordinate_t>::iterator begin,
43 | 		std::vector<coordinate_t>::iterator end, size_t n)
44 | {
45 | 	if (n == 1)
46 | 		return;
47 | 	assert(end - begin == n * n);
48 | 	assert(n % 2 == 0);
49 | 	off_t min_x = std::numeric_limits<off_t>::max();
50 | 	off_t min_y = std::numeric_limits<off_t>::max();
51 | 	for (std::vector<coordinate_t>::iterator it = begin; it != end; it++) {
52 | 		coordinate_t coo = *it;
53 | 		min_x = std::min(min_x, coo.first);
54 | 		min_y = std::min(min_y, coo.second);
55 | 	}
56 | 	for (std::vector<coordinate_t>::iterator it = begin; it != end; it++) {
57 | 		coordinate_t coo = *it;
58 | 		assert(coo.first <= min_x + n);
59 | 		assert(coo.second <= min_y + n);
60 | 	}
61 | 	size_t half_len = n * n / 4;
62 | 	verify_coos(begin, begin + half_len, n / 2);
63 | 	verify_coos(begin + half_len, begin + half_len * 2, n / 2);
64 | 	verify_coos(begin + half_len * 2, begin + half_len * 3, n / 2);
65 | 	verify_coos(begin + half_len * 3, begin + half_len * 4, n / 2);
66 | 	assert(begin + half_len * 4 == end);
67 | }
68 | 
69 | void test_size(size_t n)
70 | {
71 | 	std::vector<coordinate_t> coos = gen_coos(n);
72 | 	assert(coos.size() == n * n);
73 | 	verify_coos(coos.begin(), coos.end(), n);
74 | }
75 | 
76 | int main()
77 | {
78 | 	for (size_t i = 1; i <= 64; i *= 2)
79 | 		test_size(i);
80 | }
81 | 


--------------------------------------------------------------------------------
/matrix/unit-test/test-rand_gen.cpp:
--------------------------------------------------------------------------------
 1 | #include <assert.h>
 2 | 
 3 | #include <iostream>
 4 | #include <memory>
 5 | 
 6 | #include "rand_gen.h"
 7 | 
 8 | using namespace fm;
 9 | 
10 | int num_tries = 1000000;
11 | 
12 | template<class T>
13 | void test_rand(T min, T max)
14 | {
15 | 	rand_gen::ptr gen = rand_gen::create_randu<T>(min, max);
16 | 	assert(gen->get_type() == get_scalar_type<T>());
17 | 	std::vector<T> randv(num_tries);
18 | 	gen->gen((char *) randv.data(), randv.size());
19 | 	for (size_t i = 0; i < randv.size(); i++)
20 | 		assert(randv[i] >= min && randv[i] <= max);
21 | }
22 | 
23 | void test_rand_bool()
24 | {
25 | 	rand_gen::ptr gen = rand_gen::create_randu<bool>(0, 1);
26 | 	assert(gen->get_type() == get_scalar_type<bool>());
27 | 	std::unique_ptr<bool[]> randv(new bool[num_tries]);
28 | 	gen->gen((char *) randv.get(), num_tries);
29 | 	size_t num_trues = 0;
30 | 	for (int i = 0; i < num_tries; i++)
31 | 		num_trues += randv[i];
32 | 	printf("There are %ld true values out of %d values\n", num_trues, num_tries);
33 | }
34 | 
35 | int main()
36 | {
37 | 	test_rand<int>(1, 10);
38 | 	test_rand<long>(1L << 40, 10L << 40);
39 | 	test_rand<float>(1, 10);
40 | 	test_rand<double>(1, 10);
41 | 	test_rand_bool();
42 | }
43 | 


--------------------------------------------------------------------------------
/matrix/unit-test/test-sorter.cpp:
--------------------------------------------------------------------------------
 1 | #include <assert.h>
 2 | 
 3 | #include "sorter.h"
 4 | #include "generic_type.h"
 5 | 
 6 | using namespace fm;
 7 | 
 8 | const int vec_len = 1000000;
 9 | 
10 | void test_sort()
11 | {
12 | 	printf("test sort\n");
13 | 	std::vector<std::vector<long> > vals(50);
14 | 	for (size_t i = 0; i < vals.size(); i++) {
15 | 		vals[i].resize(vec_len + random() % vec_len);
16 | 		for (size_t j = 0; j < vals[i].size(); j++)
17 | 			vals[i][j] = random();
18 | 		get_scalar_type<long>().get_sorter().sort((char *) vals[i].data(),
19 | 				vals[i].size(), false);
20 | 		assert(std::is_sorted(vals[i].begin(), vals[i].end()));
21 | 	}
22 | 	std::vector<std::pair<const char *, const char *> > pairs(vals.size());
23 | 	size_t tot_vals = 0;
24 | 	for (size_t i = 0; i < vals.size(); i++) {
25 | 		tot_vals += vals[i].size();
26 | 		pairs[i] = std::pair<const char *, const char *>((char *) vals[i].data(),
27 | 				(char *) (vals[i].data() + vals[i].size()));
28 | 	}
29 | 	std::vector<long> res(tot_vals);
30 | 	get_scalar_type<long>().get_sorter().merge(pairs, (char *) res.data(),
31 | 			tot_vals);
32 | 	assert(std::is_sorted(res.begin(), res.end()));
33 | }
34 | 
35 | void test_merge_with_index()
36 | {
37 | 	printf("test merge with index\n");
38 | 	std::vector<std::vector<long> > vals(10);
39 | 	for (size_t i = 0; i < vals.size(); i++) {
40 | 		size_t len = random() % vec_len;
41 | 		vals[i].resize(len);
42 | #pragma omp parallel for
43 | 		for (size_t j = 0; j < vals[i].size(); j++)
44 | 			vals[i][j] = random();
45 | 		std::sort(vals[i].begin(), vals[i].end());
46 | 	}
47 | 
48 | 	type_sorter<long> sort;
49 | 	std::vector<std::pair<const char *, const char *> > arrs(vals.size());
50 | 	size_t num_eles = 0;
51 | 	for (size_t i = 0; i < arrs.size(); i++) {
52 | 		num_eles += vals[i].size();
53 | 		arrs[i] = std::pair<const char *, const char *>(
54 | 				(const char *) vals[i].data(),
55 | 				(const char *) (vals[i].data() + vals[i].size()));
56 | 	}
57 | 	std::vector<std::pair<int, off_t> > merge_index(num_eles);
58 | 	std::vector<long> merge_res(num_eles);
59 | 	sort.merge_with_index(arrs, (char *) merge_res.data(), num_eles,
60 | 			merge_index);
61 | 
62 | 	assert(std::is_sorted(merge_res.begin(), merge_res.end()));
63 | 
64 | 	std::vector<long> merge_res1(num_eles);
65 | 	sort.merge(arrs, merge_index, (char *) merge_res1.data(), merge_res1.size());
66 | 	for (size_t i = 0; i < merge_res1.size(); i++)
67 | 		assert(merge_res[i] == merge_res1[i]);
68 | }
69 | 
70 | int main()
71 | {
72 | 	test_merge_with_index();
73 | 	test_sort();
74 | }
75 | 


--------------------------------------------------------------------------------
/matrix/unit-test/test_io_gen.cpp:
--------------------------------------------------------------------------------
 1 | #include "matrix_io.h"
 2 | #include "matrix_config.h"
 3 | 
 4 | using namespace fm;
 5 | 
 6 | struct comp_io
 7 | {
 8 | 	bool operator()(const matrix_io &io1, const matrix_io &io2) const {
 9 | 		return io1.get_top_left().get_row_idx()
10 | 			< io2.get_top_left().get_row_idx();
11 | 	}
12 | };
13 | 
14 | void run_test()
15 | {
16 | 	size_t row_block_size = 1024;
17 | 	size_t num_rows = random() + 1000000000L;
18 | 	size_t num_blocks = ceil(((double) num_rows) / row_block_size);
19 | 	std::vector<row_block> blocks;
20 | 	off_t off = 0;
21 | 	for (size_t i = 0; i < num_blocks; i++) {
22 | 		blocks.emplace_back(off);
23 | 		off += random() % (1024 * 1024);
24 | 	}
25 | 	blocks.emplace_back(off);
26 | 
27 | 	int num_gens = 32;
28 | 	std::vector<matrix_io> ios;
29 | 	row_block_mapper mapper(blocks, matrix_conf.get_rb_io_size());
30 | 	matrix_io_generator::ptr io_gen = matrix_io_generator::create(blocks,
31 | 			num_rows, 1000, 0, mapper);
32 | 	while (io_gen->has_next_io())
33 | 		ios.push_back(io_gen->get_next_io());
34 | 	std::sort(ios.begin(), ios.end(), comp_io());
35 | 	printf("There are %ld ios\n", ios.size());
36 | 	off_t prev_off = ios[0].get_loc().get_offset();
37 | 	size_t prev_size = ios[0].get_size();
38 | 	off_t prev_row = ios[0].get_top_left().get_row_idx();
39 | 	size_t prev_num_rows = ios[0].get_num_rows();
40 | 	assert(prev_off == 0);
41 | 	assert(prev_row == 0);
42 | 	for (size_t i = 1; i < ios.size() - 1; i++) {
43 | 		assert(ios[i].get_loc().get_offset() == prev_off + prev_size);
44 | 		assert(ios[i].get_top_left().get_row_idx() == prev_row + prev_num_rows);
45 | 		prev_off = ios[i].get_loc().get_offset();
46 | 		prev_size = ios[i].get_size();
47 | 		prev_row = ios[i].get_top_left().get_row_idx();
48 | 		prev_num_rows = ios[i].get_num_rows();
49 | 	}
50 | 	assert(ios.back().get_loc().get_offset() + ios.back().get_size()
51 | 			== blocks.back().get_offset());
52 | 	assert(ios.back().get_top_left().get_row_idx()
53 | 			+ ios.back().get_num_rows() == num_rows);
54 | }
55 | 
56 | int main()
57 | {
58 | 	for (int test = 0; test < 100; test++)
59 | 		run_test();
60 | }
61 | 


--------------------------------------------------------------------------------
/scripts/addr2line.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl
 2 | 
 3 | use strict;
 4 | 
 5 | my $addresses = "";
 6 | while (<STDIN>) {
 7 | 	if (/#[0-9]+\s(0x[a-z0-9]+)\s/) {
 8 | 		$addresses = $addresses . $1 . " ";
 9 | 	}
10 | }
11 | print "$addresses\n";
12 | system("addr2line -e $ARGV[0] $addresses");
13 | 


--------------------------------------------------------------------------------
/scripts/check_threads.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl
 2 | 
 3 | # This perl script prints out the backtrace of all threads of a process
 4 | # with gdb.
 5 | 
 6 | use strict;
 7 | 
 8 | my $num_args = @ARGV;
 9 | if ($num_args < 2) {
10 | 	print STDERR "check_threads.pl program pid\n";
11 | 	exit;
12 | }
13 | 
14 | my $prog = $ARGV[0];
15 | my $pid = $ARGV[1];
16 | 
17 | system("echo \"info threads\" > /tmp/gdb_cmds.$$.txt");
18 | my @lines = `gdb $prog $pid -x /tmp/gdb_cmds.$$.txt -batch`;
19 | for my $line (@lines) {
20 | 	if ($line =~ /^\*?\s+([0-9]+)/) {
21 | 		system("echo \"thread $1\" >> /tmp/gdb_cmds_each_thread.$$.txt");
22 | 		system("echo backtrace >> /tmp/gdb_cmds_each_thread.$$.txt");
23 | 	}
24 | }
25 | system("gdb $prog $pid -x /tmp/gdb_cmds_each_thread.$$.txt -batch");
26 | 


--------------------------------------------------------------------------------
/scripts/init_ssds.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl
 2 | 
 3 | use strict;
 4 | 
 5 | my @dev_names=("b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q");
 6 | my $file_size = "40G";
 7 | my $num_devs = @dev_names;
 8 | 
 9 | for (my $i = 1; $i <= $num_devs; $i++) {
10 | 	my $dev_idx = $i - 1;
11 | 	my $dev_file = "/dev/sd$dev_names[$dev_idx]1";
12 | 	my $dir = "/mnt/ssd${i}";
13 | 	print "dev: $dev_file, dir: $dir\n" ;
14 | 	system("mkfs.xfs $dev_file");
15 | 	system("mkdir -p $dir");
16 | 	system("mount $dev_file $dir");
17 | }
18 | 
19 | my $size = 40 * $num_devs;
20 | system("tools/create_files data_files_RAID0.txt ${size}G RAID0 16");
21 | 
22 | for (my $i = 1; $i <= $num_devs; $i++) {
23 | 	my $dev_idx = $i - 1;
24 | 	my $dev_file = "/dev/sd$dev_names[$dev_idx]1";
25 | 	my $dir = "/mnt/ssd${i}";
26 | 	system("chown -R zhengda.zhengda $dir");
27 | 	system("echo noop > /sys/block/sd$dev_names[$dev_idx]/queue/scheduler");
28 | 	system("cat /sys/block/sd$dev_names[$dev_idx]/queue/scheduler");
29 | }
30 | 


--------------------------------------------------------------------------------
/scripts/monitor_proc.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl
 2 | 
 3 | use strict;
 4 | use POSIX;
 5 | use Time::HiRes qw(usleep nanosleep);
 6 | 
 7 | my $enable_log = 1;
 8 | my $num_args = @ARGV;
 9 | if ($num_args < 1) {
10 | 	print STDERR "monitor_proc cmd\n";
11 | 	exit;
12 | }
13 | my $log_file = "monitor.log";
14 | if ($enable_log) {
15 | 	open LOG, ">", $log_file or die "can't open $log_file: $!";
16 | }
17 | my $cmd = $ARGV[0];
18 | my $pid = run($cmd);
19 | while (1) {
20 | 	print "========================================\n";
21 | 	my @tids = collect_info($pid);
22 | 	# the process probably has exit.
23 | 	if (@tids == 0) {
24 | 		last;
25 | 	}
26 | 	waitpid $pid, WNOHANG;
27 | 	usleep(300000);
28 | }
29 | if ($enable_log) {
30 | 	close LOG;
31 | }
32 | 
33 | sub parse_numa_maps {
34 | 	my @lines = @{$_[0]};
35 | 	my @mem_sizes = (0, 0, 0, 0);
36 | 	for (my $i = 0; $i < @lines; $i++) {
37 | 		my @fields = split / /, $lines[$i];
38 | 		for (my $j = 0; $j < @fields; $j++) {
39 | 			if ($fields[$j] =~ /N([0-9]+)=([0-9]+)/) {
40 | 				$mem_sizes[$1] += $2;
41 | 			}
42 | 		}
43 | 	}
44 | 	for (my $i = 0; $i < @mem_sizes; $i++) {
45 | 		print "node$i: $mem_sizes[$i]\n"
46 | 	}
47 | }
48 | 
49 | sub collect_info {
50 | 	my $pid = $_[0];
51 | 	my $numa_maps = `cat /proc/$pid/numa_maps`;
52 | 	if ($enable_log) {
53 | 		print LOG $numa_maps;
54 | 		print LOG `cat /proc/$pid/maps`;
55 | 	}
56 | 	my @lines = split /\n/, $numa_maps;
57 | 	parse_numa_maps(\@lines);
58 | 
59 | 	my @tids = collect_tids($pid);
60 | 	for (my $i = 0; $i < @tids; $i++) {
61 | 		my @fields = split(/ /, `cat /proc/$pid/task/$tids[$i]/stat`);
62 | 		print "thread $tids[$i] runs on core $fields[38]\n";
63 | 	}
64 | 	return @tids;
65 | }
66 | 
67 | sub collect_tids {
68 | 	my $pid = $_[0];
69 | 	my @tids;
70 | 	opendir(D, "/proc/$pid/task") || return ();
71 | 	while (my $f = readdir(D)) {
72 | 		if (isdigit($f)) {
73 | 			push(@tids, $f);
74 | 		}
75 | 	}
76 | 	closedir(D);
77 | 	return @tids;
78 | }
79 | 
80 | sub run {
81 | 	my $cmd = $_[0];
82 | 	my $pid = fork();
83 | 	if ($pid == 0) {		# child
84 | 		exec($cmd) or die "can't execute: $!";
85 | 		exit;
86 | 	}
87 | 	elsif ($pid > 0) {
88 | 		return $pid;
89 | 	}
90 | 	else {
91 | 		die "can't create a process: $!";
92 | 	}
93 | }
94 | 


--------------------------------------------------------------------------------
/scripts/set_ramdisk.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/sh
 2 | 
 3 | size="40G"
 4 | 
 5 | if [ $# > 1 ]; then
 6 | 	size=$1
 7 | fi
 8 | echo "file size is $size"
 9 | 
10 | mkdir -p /mnt/ram0
11 | mkdir -p /mnt/ram1
12 | mkdir -p /mnt/ram2
13 | mkdir -p /mnt/ram3
14 | 
15 | mkfs.xfs /dev/ram0
16 | mkfs.xfs /dev/ram1
17 | mkfs.xfs /dev/ram2
18 | mkfs.xfs /dev/ram3
19 | 
20 | mount /dev/ram0 /mnt/ram0
21 | mount /dev/ram1 /mnt/ram1
22 | mount /dev/ram2 /mnt/ram2
23 | mount /dev/ram3 /mnt/ram3
24 | 
25 | numactl -m 0 -N 0 tools/create_file /mnt/ram0/test $size
26 | numactl -m 1 -N 1 tools/create_file /mnt/ram1/test $size
27 | numactl -m 2 -N 2 tools/create_file /mnt/ram2/test $size
28 | numactl -m 3 -N 3 tools/create_file /mnt/ram3/test $size
29 | 
30 | chown -R zhengda.zhengda /mnt/ram0
31 | chown -R zhengda.zhengda /mnt/ram1
32 | chown -R zhengda.zhengda /mnt/ram2
33 | chown -R zhengda.zhengda /mnt/ram3
34 | 


--------------------------------------------------------------------------------
/scripts/show_cpu.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl
 2 | 
 3 | use strict;
 4 | 
 5 | my $num_args = @ARGV;
 6 | if ($num_args < 1) {
 7 | 	print STDERR "show_cpu.pl prog_name\n";
 8 | 	exit;
 9 | }
10 | my $prog_name = $ARGV[0];
11 | my @pids = `ps -aux 2>/dev/null | awk '{print \$2, \$11}' | grep $prog_name | awk '{print \$1}'`;
12 | my $num_pids = @pids;
13 | if ($num_pids != 1) {
14 | 	print STDERR "There are $num_pids processes named after $prog_name\n";
15 | 	foreach(@pids) {
16 | 		print $_, "\n";
17 | 	}
18 | 	exit;
19 | }
20 | 
21 | system("top -b -p $pids[0] -d 1");
22 | 


--------------------------------------------------------------------------------
/scripts/show_multi_cpu.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl
 2 | 
 3 | use strict;
 4 | 
 5 | my $num_args = @ARGV;
 6 | 
 7 | my $iterations = 100000000;
 8 | my $interval = 1;	# 1 second
 9 | if ($num_args < 0) {
10 | 	print "show_cpu.pl node_id\n";
11 | 	exit;
12 | }
13 | 
14 | my $node_id = $ARGV[0];
15 | 
16 | my @cores;
17 | for (my $i = 0; $i < 8; $i++) {
18 | 	push(@cores, $node_id + 4 * $i);
19 | }
20 | 
21 | my $str = join(',', @cores);
22 | print $str, "\n";
23 | 
24 | system("mpstat -P $str $interval");
25 | 


--------------------------------------------------------------------------------
/utils/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | project (FlashGraph)
 2 | 
 3 | link_directories (${PROJECT_BINARY_DIR}/libsafs) 
 4 | 
 5 | add_executable(SAFS-util SAFS-util.cpp)
 6 | 
 7 | target_link_libraries (SAFS-util safs pthread)
 8 | 
 9 | if (LIBNUMA_FOUND)
10 |     target_link_libraries(SAFS-util numa)
11 | endif()
12 | 
13 | if (LIBAIO_FOUND)
14 |     target_link_libraries(SAFS-util aio)
15 | endif()
16 | 
17 | if (hwloc_FOUND)
18 | 	target_link_libraries(SAFS-util hwloc)
19 | endif()
20 | 


--------------------------------------------------------------------------------
/utils/Makefile:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 Open Connectome Project (http;//openconnecto.me)
 2 | # Written by Da Zheng (zhengda1936@gmail.com)
 3 | #
 4 | # This file is part of SAFSlib.
 5 | #
 6 | # Licensed under the Apache License, Version 2.0 (the "License");
 7 | # you may not use this file except in compliance with the License.
 8 | # You may obtain a copy of the License at
 9 | #
10 | #     http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | 
18 | include ../Makefile.common
19 | 
20 | CXXFLAGS += -I.. -I../libsafs
21 | CFLAGS = -g -O3 -I.. -I../libsafs $(TRACE_FLAGS)
22 | LDFLAGS := -L../libsafs -lsafs $(LDFLAGS)
23 | LIBFILE = ../libsafs/libsafs.a
24 | 
25 | TARGETS = SAFS-util
26 | 
27 | all: $(TARGETS)
28 | 
29 | SAFS-util: SAFS-util.o $(LIBFILE)
30 | 	$(CXX) -o SAFS-util SAFS-util.o $(LDFLAGS)
31 | 
32 | clean:
33 | 	rm -f *.o
34 | 	rm -f *~
35 | 	rm -f *.d
36 | 	rm -f $(TARGETS)
37 | 


--------------------------------------------------------------------------------