├── .gitignore ├── riffle-server ├── src │ ├── grpc │ │ ├── mod.rs │ │ ├── layer │ │ │ ├── mod.rs │ │ │ └── tracing.rs │ │ └── protobuf │ │ │ └── mod.rs │ ├── mem_allocator │ │ ├── mimalloc.rs │ │ ├── system_std.rs │ │ ├── jemalloc.rs │ │ ├── default.rs │ │ ├── profiling.rs │ │ └── mod.rs │ ├── urpc │ │ ├── mod.rs │ │ └── shutdown.rs │ ├── store │ │ ├── alignment │ │ │ ├── mod.rs │ │ │ └── allocator.rs │ │ ├── local │ │ │ ├── layers.rs │ │ │ ├── options.rs │ │ │ └── io_layer_read_ahead │ │ │ │ └── sequential_tasks.rs │ │ ├── mem │ │ │ ├── capacity.rs │ │ │ └── mod.rs │ │ ├── spill │ │ │ ├── storage_select_handler.rs │ │ │ └── storage_flush_handler.rs │ │ └── hadoop │ │ │ ├── delegator.rs │ │ │ └── mod.rs │ ├── raw_pipe.rs │ ├── common.rs │ ├── raw_io.rs │ ├── storage.rs │ ├── constant.rs │ ├── tracing.rs │ ├── http │ │ └── historical_apps.rs │ ├── app_manager │ │ ├── partition_identifier.rs │ │ └── partition_meta.rs │ ├── logforth_service.rs │ ├── lazy_initializer.rs │ ├── runtime │ │ └── metrics.rs │ ├── histogram │ │ └── mod.rs │ ├── lib.rs │ ├── composed_bytes.rs │ ├── log_service.rs │ ├── signal.rs │ ├── await_tree.rs │ ├── panic_hook.rs │ └── deadlock.rs ├── benches │ └── conf_bytestring.rs └── tests │ ├── graceful_shutdown.rs │ └── write_read.rs ├── dev ├── integration │ ├── coordinator.conf │ ├── sql_set │ │ ├── basic.scala │ │ ├── q55.sql │ │ ├── q3.sql │ │ ├── q96.sql │ │ ├── q52.sql │ │ ├── q22.sql │ │ ├── q42.sql │ │ ├── ss_max.sql │ │ ├── ss_maxb.sql │ │ ├── q37.sql │ │ ├── q82.sql │ │ ├── q15.sql │ │ ├── q19.sql │ │ ├── q7.sql │ │ ├── q93.sql │ │ ├── q84.sql │ │ ├── q98.sql │ │ ├── q26.sql │ │ ├── q32.sql │ │ ├── q6.sql │ │ ├── q86.sql │ │ ├── q92.sql │ │ ├── q12.sql │ │ ├── q1.sql │ │ ├── q20.sql │ │ ├── q27.sql │ │ ├── q45.sql │ │ ├── q36.sql │ │ ├── q65.sql │ │ ├── q24a.sql │ │ ├── q67.sql │ │ ├── q24b.sql │ │ ├── q40.sql │ │ ├── q94.sql │ │ ├── q38.sql │ │ ├── q87.sql │ │ ├── q79.sql │ │ ├── q97.sql │ │ ├── q16.sql │ │ ├── q90.sql │ │ ├── q43.sql │ │ ├── q89.sql │ │ ├── q21.sql │ │ ├── q91.sql │ │ ├── q25.sql │ │ ├── q39a.sql │ │ ├── q70.sql │ │ ├── q39b.sql │ │ ├── q73.sql │ │ ├── q95.sql │ │ ├── q76.sql │ │ ├── q62.sql │ │ ├── q99.sql │ │ ├── q30.sql │ │ ├── q29.sql │ │ ├── q46.sql │ │ ├── q18.sql │ │ ├── q72.sql │ │ ├── q81.sql │ │ ├── q53.sql │ │ ├── q61.sql │ │ ├── q68.sql │ │ ├── q71.sql │ │ ├── q34.sql │ │ ├── q63.sql │ │ ├── q69.sql │ │ ├── q48.sql │ │ ├── q50.sql │ │ ├── q13.sql │ │ └── q35.sql │ ├── grafana │ │ └── provisioning │ │ │ ├── datasources │ │ │ └── prometheus.yml │ │ │ └── dashboards │ │ │ └── dashboard.yml │ ├── prometheus.yml │ ├── riffle.conf.1 │ ├── riffle.conf.2 │ └── spark-defaults.conf ├── anolisos8 │ ├── arm64 │ │ ├── docker-compose.yml │ │ └── Dockerfile │ └── amd64 │ │ ├── docker-compose.yml │ │ └── Dockerfile └── centos7 │ └── amd64 │ ├── docker-compose.yml │ └── Dockerfile ├── .github └── workflows │ ├── license_check.yml.disable │ └── spark_sql_test.yml ├── rust-toolchain.toml ├── rustfmt.toml └── riffle-ctl ├── Cargo.toml └── src └── actions └── hdfs_append.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | .DS_Store 3 | .idea 4 | riffle-server/src/grpc/protobuf/uniffle.rs 5 | -------------------------------------------------------------------------------- /riffle-server/src/grpc/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod layer; 2 | pub mod protobuf; 3 | pub mod service; 4 | -------------------------------------------------------------------------------- /riffle-server/src/grpc/layer/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod awaittree; 2 | pub mod metric; 3 | pub mod tracing; 4 | -------------------------------------------------------------------------------- /riffle-server/src/mem_allocator/mimalloc.rs: -------------------------------------------------------------------------------- 1 | pub type Allocator = mimalloc::MiMalloc; 2 | pub const fn allocator() -> Allocator { 3 | mimalloc::MiMalloc 4 | } 5 | -------------------------------------------------------------------------------- /riffle-server/src/urpc/mod.rs: -------------------------------------------------------------------------------- 1 | // this is the customize urpc definition to implement in rust 2 | pub mod client; 3 | pub mod command; 4 | pub mod connection; 5 | pub mod frame; 6 | pub mod server; 7 | pub mod shutdown; 8 | -------------------------------------------------------------------------------- /dev/integration/coordinator.conf: -------------------------------------------------------------------------------- 1 | rss.coordinator.server.heartbeat.timeout=30000 2 | rss.coordinator.app.expired=60000 3 | rss.coordinator.server.heartbeat.interval=10000 4 | rss.rpc.server.port=21000 5 | rss.jetty.http.port=19995 6 | 7 | -------------------------------------------------------------------------------- /dev/integration/sql_set/basic.scala: -------------------------------------------------------------------------------- 1 | val data = sc.parallelize(1 to 100, 4) 2 | val pairs = data.map(x => (x % 5, x)) 3 | val grouped = pairs.groupByKey() 4 | val result = grouped.mapValues(_.sum).collect().sortBy(_._1) 5 | result.foreach(println) -------------------------------------------------------------------------------- /dev/integration/grafana/provisioning/datasources/prometheus.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: Prometheus 5 | type: prometheus 6 | access: proxy 7 | url: http://prometheus:9090 8 | isDefault: true 9 | editable: true 10 | -------------------------------------------------------------------------------- /riffle-server/src/store/alignment/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::store::alignment::allocator::AlignedAllocator; 2 | 3 | pub mod allocator; 4 | pub mod io_buffer_pool; 5 | pub mod io_bytes; 6 | 7 | pub const ALIGN: usize = 4096; 8 | pub const IO_BUFFER_ALLOCATOR: AlignedAllocator = AlignedAllocator::new(); 9 | -------------------------------------------------------------------------------- /dev/integration/grafana/provisioning/dashboards/dashboard.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: 'Riffle Dashboards' 5 | orgId: 1 6 | folder: '' 7 | type: file 8 | disableDeletion: false 9 | updateIntervalSeconds: 10 10 | allowUiUpdates: true 11 | options: 12 | path: /etc/grafana/dashboards 13 | foldersFromFilesStructure: true 14 | -------------------------------------------------------------------------------- /riffle-server/src/raw_pipe.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::os::fd::{AsRawFd, RawFd}; 3 | 4 | #[derive(Debug)] 5 | pub struct RawPipe { 6 | pub pipe_in_fd: File, 7 | pub pipe_out_fd: File, 8 | pub length: usize, 9 | } 10 | 11 | impl RawPipe { 12 | pub fn from(pipe_in_fd: File, pipe_out_fd: File, length: usize) -> Self { 13 | Self { 14 | pipe_in_fd, 15 | pipe_out_fd, 16 | length, 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /riffle-server/src/common.rs: -------------------------------------------------------------------------------- 1 | use crate::app_manager::{SHUFFLE_SERVER_ID, SHUFFLE_SERVER_IP}; 2 | use crate::config::Config; 3 | use crate::util::{generate_worker_uid, get_local_ip}; 4 | 5 | pub fn init_global_variable(config: &Config) { 6 | let worker_uid = generate_worker_uid(&config); 7 | SHUFFLE_SERVER_ID.get_or_init(|| worker_uid.clone()); 8 | 9 | let worker_ip = get_local_ip().unwrap().to_string(); 10 | SHUFFLE_SERVER_IP.get_or_init(|| worker_ip); 11 | } 12 | -------------------------------------------------------------------------------- /dev/integration/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | evaluation_interval: 15s 4 | external_labels: 5 | cluster: 'riffle-cluster' 6 | environment: 'dev' 7 | 8 | scrape_configs: 9 | - job_name: 'pushgateway' 10 | honor_labels: true 11 | static_configs: 12 | - targets: ['pushgateway:9091'] 13 | labels: 14 | service: 'pushgateway' 15 | 16 | - job_name: 'prometheus' 17 | static_configs: 18 | - targets: ['localhost:9090'] 19 | -------------------------------------------------------------------------------- /dev/anolisos8/arm64/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | build-release: 5 | build: ../arm64 6 | volumes: 7 | - ~/.m2:/root/.m2:rw 8 | - ~/.sbt:/root/.sbt:rw 9 | - ~/.cargo/git:/root/.cargo/git:rw 10 | - ~/.cargo/registry:/root/.cargo/registry:rw 11 | - ./../../../:/R1:rw 12 | - ./../../../target-docker:/R1/target:rw 13 | command: "bash -c 'source ~/.bashrc && cd /R1 && cargo build --features hdrs,logforth,memory-prof,io-uring --release'" 14 | -------------------------------------------------------------------------------- /riffle-server/src/raw_io.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::os::fd::{AsRawFd, RawFd}; 3 | 4 | #[derive(Debug)] 5 | pub struct RawIO { 6 | pub file: File, 7 | pub offset: u64, 8 | pub length: u64, 9 | pub raw_fd: RawFd, 10 | } 11 | 12 | impl RawIO { 13 | pub fn new(f: File, offset: u64, length: u64) -> RawIO { 14 | let raw_fd = f.as_raw_fd(); 15 | Self { 16 | file: f, 17 | offset, 18 | length, 19 | raw_fd, 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /dev/centos7/amd64/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | build-release: 5 | build: ../amd64 6 | volumes: 7 | - ~/.m2:/root/.m2:rw 8 | - ~/.sbt:/root/.sbt:rw 9 | - ~/.cargo/git:/root/.cargo/git:rw 10 | - ~/.cargo/registry:/root/.cargo/registry:rw 11 | - ./../../../:/R1:rw 12 | - ./../../../target-docker:/R1/target:rw 13 | # environment: 14 | # RUSTFLAGS: "-C target-cpu=skylake" 15 | command: "bash -c 'source ~/.bashrc && cd /R1 && cargo build --features hdrs,logforth,memory-prof --release'" 16 | -------------------------------------------------------------------------------- /dev/anolisos8/amd64/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | build-release: 5 | build: ../amd64 6 | volumes: 7 | - ~/.m2:/root/.m2:rw 8 | - ~/.sbt:/root/.sbt:rw 9 | - ~/.cargo/git:/root/.cargo/git:rw 10 | - ~/.cargo/registry:/root/.cargo/registry:rw 11 | - ./../../../:/R1:rw 12 | - ./../../../target-docker:/R1/target:rw 13 | # environment: 14 | # RUSTFLAGS: "-C target-cpu=skylake" 15 | command: "bash -c 'source ~/.bashrc && cd /R1 && cargo build --features hdrs,logforth,memory-prof,io-uring --release'" 16 | -------------------------------------------------------------------------------- /.github/workflows/license_check.yml.disable: -------------------------------------------------------------------------------- 1 | name: License Checker 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - release-*.* 8 | pull_request: 9 | branches: 10 | - master 11 | - "v*.*.*-rc" 12 | - "v*.*.*" 13 | - release-*.* 14 | jobs: 15 | license-header-check: 16 | runs-on: ubuntu-latest 17 | name: license-header-check 18 | steps: 19 | - uses: actions/checkout@v4 20 | with: 21 | persist-credentials: false 22 | - name: Check License Header 23 | uses: apache/skywalking-eyes/header@v0.6.0 -------------------------------------------------------------------------------- /riffle-server/benches/conf_bytestring.rs: -------------------------------------------------------------------------------- 1 | use criterion::{black_box, criterion_group, criterion_main, Criterion}; 2 | use riffle_server::config_ref::{ByteString, ConfRef, DynamicConfRef}; 3 | 4 | #[inline] 5 | fn reconf_bytestring(batch: usize) { 6 | let reconf_ref = DynamicConfRef::new("key", ByteString::new("19M")); 7 | for _ in 0..batch { 8 | let _ = reconf_ref.get().as_u64(); 9 | } 10 | } 11 | 12 | pub fn criterion_benchmark(c: &mut Criterion) { 13 | c.bench_function("conf_bytestring", |b| { 14 | b.iter(|| reconf_bytestring(black_box(2000))) 15 | }); 16 | } 17 | 18 | criterion_group!(benches, criterion_benchmark); 19 | criterion_main!(benches); 20 | -------------------------------------------------------------------------------- /riffle-server/src/store/local/layers.rs: -------------------------------------------------------------------------------- 1 | use crate::store::local::LocalIO; 2 | use std::sync::Arc; 3 | 4 | pub type Handler = Arc>; 5 | 6 | pub struct OperatorBuilder { 7 | handler: Handler, 8 | } 9 | 10 | impl OperatorBuilder { 11 | pub fn new(handler: Handler) -> Self { 12 | OperatorBuilder { handler } 13 | } 14 | 15 | pub fn layer(self, layer: impl Layer) -> OperatorBuilder { 16 | OperatorBuilder { 17 | handler: layer.wrap(self.handler), 18 | } 19 | } 20 | 21 | pub fn build(self) -> Handler { 22 | self.handler 23 | } 24 | } 25 | 26 | pub trait Layer { 27 | fn wrap(&self, handler: Handler) -> Handler; 28 | } 29 | -------------------------------------------------------------------------------- /riffle-server/src/storage.rs: -------------------------------------------------------------------------------- 1 | use crate::config::Config; 2 | use crate::config_reconfigure::ReconfigurableConfManager; 3 | use crate::runtime::manager::RuntimeManager; 4 | use crate::store::hybrid::HybridStore; 5 | use crate::store::{Store, StoreProvider}; 6 | use std::sync::Arc; 7 | 8 | pub type HybridStorage = Arc; 9 | 10 | pub struct StorageService; 11 | 12 | impl StorageService { 13 | pub fn init( 14 | runtime_manager: &RuntimeManager, 15 | config: &Config, 16 | reconfig_manager: &ReconfigurableConfManager, 17 | ) -> HybridStorage { 18 | let store = Arc::new(StoreProvider::get( 19 | runtime_manager.clone(), 20 | config.clone(), 21 | reconfig_manager, 22 | )); 23 | store.clone().start(); 24 | store.clone() 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /riffle-server/src/constant.rs: -------------------------------------------------------------------------------- 1 | pub const CPU_ARCH: &str = std::env::consts::ARCH; 2 | 3 | #[allow(non_camel_case_types)] 4 | pub enum StatusCode { 5 | SUCCESS = 0, 6 | DOUBLE_REGISTER = 1, 7 | NO_BUFFER = 2, 8 | INVALID_STORAGE = 3, 9 | NO_REGISTER = 4, 10 | NO_PARTITION = 5, 11 | INTERNAL_ERROR = 6, 12 | TIMEOUT = 7, 13 | ACCESS_DENIED = 8, 14 | INVALID_REQUEST = 9, 15 | NO_BUFFER_FOR_HUGE_PARTITION = 10, 16 | // to indicate shuffle-writing not retry! 17 | // todo: we should introduce the dedicated status code to indicate this 18 | EXCEED_HUGE_PARTITION_HARD_LIMIT = 12, 19 | HARD_SPLIT_FROM_SERVER = 15, 20 | } 21 | 22 | impl Into for StatusCode { 23 | fn into(self) -> i32 { 24 | self as i32 25 | } 26 | } 27 | 28 | pub const ALL_LABEL: &str = "ALL"; 29 | 30 | pub const INVALID_BLOCK_ID: i64 = -1; 31 | -------------------------------------------------------------------------------- /riffle-server/src/store/mem/capacity.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug)] 2 | pub struct CapacitySnapshot { 3 | capacity: i64, 4 | allocated: i64, 5 | used: i64, 6 | } 7 | 8 | unsafe impl Send for CapacitySnapshot {} 9 | unsafe impl Sync for CapacitySnapshot {} 10 | 11 | impl From<(i64, i64, i64)> for CapacitySnapshot { 12 | fn from(value: (i64, i64, i64)) -> Self { 13 | CapacitySnapshot { 14 | capacity: value.0, 15 | allocated: value.1, 16 | used: value.2, 17 | } 18 | } 19 | } 20 | 21 | impl CapacitySnapshot { 22 | pub fn capacity(&self) -> i64 { 23 | self.capacity 24 | } 25 | pub fn allocated(&self) -> i64 { 26 | self.allocated 27 | } 28 | pub fn used(&self) -> i64 { 29 | self.used 30 | } 31 | 32 | pub fn available(&self) -> i64 { 33 | self.capacity - self.allocated - self.used 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /riffle-server/src/grpc/protobuf/mod.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #[rustfmt::skip] 19 | pub mod uniffle; 20 | -------------------------------------------------------------------------------- /.github/workflows/spark_sql_test.yml: -------------------------------------------------------------------------------- 1 | name: Spark SQL Integration Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | workflow_dispatch: 9 | 10 | concurrency: 11 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 12 | cancel-in-progress: ${{ github.event_name == 'pull_request' }} 13 | 14 | jobs: 15 | spark-sql-integration-test: 16 | runs-on: ubuntu-22.04 17 | timeout-minutes: 60 18 | 19 | steps: 20 | - name: Remove unnecessary files 21 | run: | 22 | sudo rm -rf /usr/share/dotnet 23 | sudo rm -rf "$AGENT_TOOLSDIRECTORY" 24 | 25 | - name: Checkout code 26 | uses: actions/checkout@v4 27 | 28 | # - name: ssh to debug 29 | # uses: valeriangalliat/action-sshd-cloudflared@v4 30 | 31 | - name: Run Spark SQL Test 32 | run: docker compose -f dev/integration/docker-compose.yml up -d riffle-test -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | [toolchain] 19 | channel = "nightly-2025-06-01" 20 | components = ["rustfmt", "clippy", "rust-analyzer"] -------------------------------------------------------------------------------- /riffle-server/src/store/mem/mod.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | pub mod budget; 19 | pub mod buffer; 20 | pub mod capacity; 21 | pub mod ticket; 22 | 23 | pub use await_tree::InstrumentAwait; 24 | -------------------------------------------------------------------------------- /riffle-server/src/mem_allocator/system_std.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | pub type Allocator = std::alloc::System; 19 | pub const fn allocator() -> Allocator { 20 | std::alloc::System 21 | } 22 | -------------------------------------------------------------------------------- /dev/integration/riffle.conf.1: -------------------------------------------------------------------------------- 1 | store_type = "MEMORY_LOCALFILE" 2 | grpc_port = 21100 3 | urpc_port = 22200 4 | http_port = 19998 5 | coordinator_quorum = ["uniffle-coordinator:21000"] 6 | tags = ["riffle2", "datanode", "GRPC", "ss_v5", "GRPC_NETTY"] 7 | 8 | [urpc_config] 9 | get_index_rpc_version = 'V2' 10 | 11 | [memory_store] 12 | capacity = "2G" 13 | dashmap_shard_amount = 128 14 | 15 | [localfile_store] 16 | data_paths = ["/tmp/riffle-server-1/data"] 17 | healthy_check_min_disks = 0 18 | disk_max_concurrency = 2000 19 | disk_high_watermark = 1.0 20 | 21 | [hybrid_store] 22 | memory_spill_high_watermark = 0.5 23 | memory_spill_low_watermark = 0.2 24 | memory_spill_max_concurrency = 1000 25 | 26 | [runtime_config] 27 | read_thread_num = 20 28 | write_thread_num = 50 29 | grpc_thread_num = 50 30 | http_thread_num = 5 31 | default_thread_num = 10 32 | dispatch_thread_num = 5 33 | 34 | [metrics] 35 | push_gateway_endpoint = "http://pushgateway:9091/" 36 | push_interval_sec = 5 37 | labels = { mode = "test" } 38 | 39 | [log] 40 | path = "/tmp/riffle-server-1/log" 41 | rotation = "Never" 42 | -------------------------------------------------------------------------------- /dev/integration/riffle.conf.2: -------------------------------------------------------------------------------- 1 | store_type = "MEMORY_LOCALFILE" 2 | grpc_port = 21101 3 | urpc_port = 22201 4 | http_port = 19999 5 | coordinator_quorum = ["uniffle-coordinator:21000"] 6 | tags = ["riffle2", "datanode", "GRPC", "ss_v5", "GRPC_NETTY"] 7 | 8 | [urpc_config] 9 | get_index_rpc_version = 'V2' 10 | 11 | [memory_store] 12 | capacity = "2G" 13 | dashmap_shard_amount = 128 14 | 15 | [localfile_store] 16 | data_paths = ["/tmp/riffle-server-2/data"] 17 | healthy_check_min_disks = 0 18 | disk_max_concurrency = 2000 19 | disk_high_watermark = 1.0 20 | 21 | [hybrid_store] 22 | memory_spill_high_watermark = 0.5 23 | memory_spill_low_watermark = 0.2 24 | memory_spill_max_concurrency = 1000 25 | 26 | [runtime_config] 27 | read_thread_num = 20 28 | write_thread_num = 50 29 | grpc_thread_num = 50 30 | http_thread_num = 5 31 | default_thread_num = 10 32 | dispatch_thread_num = 5 33 | 34 | [metrics] 35 | push_gateway_endpoint = "http://pushgateway:9091/" 36 | push_interval_sec = 5 37 | labels = { mode = "test" } 38 | 39 | [log] 40 | path = "/tmp/riffle-server-2/log" 41 | rotation = "Never" 42 | -------------------------------------------------------------------------------- /riffle-server/src/tracing.rs: -------------------------------------------------------------------------------- 1 | use crate::app_manager::SHUFFLE_SERVER_ID; 2 | use crate::config::Config; 3 | use log::warn; 4 | 5 | pub struct FastraceWrapper; 6 | 7 | impl FastraceWrapper { 8 | pub fn init(config: Config) { 9 | if config.tracing.is_none() { 10 | warn!("No any tracing config. Ignore initializing..."); 11 | return; 12 | } 13 | let config = config.tracing.unwrap(); 14 | let jaeger_endpoint = config.jaeger_reporter_endpoint.as_str(); 15 | let jaeger_service_name = config.jaeger_service_name.as_str(); 16 | let reporter = fastrace_jaeger::JaegerReporter::new( 17 | jaeger_endpoint.parse().unwrap(), 18 | format!( 19 | "{}-{}", 20 | jaeger_service_name, 21 | SHUFFLE_SERVER_ID.get().unwrap() 22 | ), 23 | ) 24 | .unwrap(); 25 | fastrace::set_reporter(reporter, fastrace::collector::Config::default()); 26 | } 27 | } 28 | 29 | impl Drop for FastraceWrapper { 30 | fn drop(&mut self) { 31 | fastrace::flush(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | edition = "2021" 19 | reorder_imports = true 20 | 21 | # format_code_in_doc_comments = true 22 | # group_imports = "StdExternalCrate" 23 | # imports_granularity = "Item" 24 | # overflow_delimited_expr = true 25 | # trailing_comma = "Vertical" 26 | # where_single_line = true 27 | -------------------------------------------------------------------------------- /riffle-server/src/http/historical_apps.rs: -------------------------------------------------------------------------------- 1 | use crate::app_manager::APP_MANAGER_REF; 2 | use crate::historical_apps::HistoricalAppInfo; 3 | use crate::http::Handler; 4 | use log::info; 5 | use poem::web::Json; 6 | use poem::{handler, RouteMethod}; 7 | use tokio::time::Instant; 8 | 9 | #[derive(Default)] 10 | pub struct HistoricalAppsHandler; 11 | 12 | impl Handler for HistoricalAppsHandler { 13 | fn get_route_method(&self) -> RouteMethod { 14 | RouteMethod::new().get(json) 15 | } 16 | 17 | fn get_route_path(&self) -> String { 18 | "/apps/history".to_string() 19 | } 20 | } 21 | 22 | #[handler] 23 | async fn json() -> Json> { 24 | let timer = Instant::now(); 25 | let manager_ref = APP_MANAGER_REF.get().unwrap(); 26 | let mut apps = vec![]; 27 | if let Some(historical_manager) = manager_ref.get_historical_app_manager() { 28 | apps = historical_manager.load().await.unwrap(); 29 | } 30 | let num = apps.len(); 31 | let apps = Json(apps); 32 | info!( 33 | "Gotten {} historical apps from http that costs {} ms", 34 | num, 35 | timer.elapsed().as_millis() 36 | ); 37 | apps 38 | } 39 | -------------------------------------------------------------------------------- /riffle-server/src/mem_allocator/jemalloc.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #[allow(non_upper_case_globals)] 19 | #[export_name = "malloc_conf"] 20 | pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\0"; 21 | 22 | pub type Allocator = tikv_jemallocator::Jemalloc; 23 | pub const fn allocator() -> Allocator { 24 | tikv_jemallocator::Jemalloc 25 | } 26 | -------------------------------------------------------------------------------- /riffle-server/src/app_manager/partition_identifier.rs: -------------------------------------------------------------------------------- 1 | use crate::app_manager::application_identifier::ApplicationId; 2 | use std::fmt::{Display, Formatter}; 3 | use std::hash::{DefaultHasher, Hash, Hasher}; 4 | 5 | #[derive(Ord, PartialOrd, Default, Debug, Hash, Clone, PartialEq, Eq)] 6 | pub struct PartitionUId { 7 | pub app_id: ApplicationId, 8 | pub shuffle_id: i32, 9 | pub partition_id: i32, 10 | } 11 | 12 | impl PartitionUId { 13 | pub fn new(app_id: &ApplicationId, shuffle_id: i32, partition_id: i32) -> PartitionUId { 14 | PartitionUId { 15 | app_id: app_id.clone(), 16 | shuffle_id, 17 | partition_id, 18 | } 19 | } 20 | 21 | pub fn get_hash(uid: &PartitionUId) -> u64 { 22 | let mut hasher = DefaultHasher::new(); 23 | 24 | uid.hash(&mut hasher); 25 | let hash_value = hasher.finish(); 26 | 27 | hash_value 28 | } 29 | } 30 | 31 | impl Display for PartitionUId { 32 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 33 | write!( 34 | f, 35 | "app_id: {}, shuffle_id: {}, partition_id: {}", 36 | self.app_id, self.shuffle_id, self.partition_id 37 | ) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q55.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q55.sql-- 19 | 20 | select i_brand_id brand_id, i_brand brand, 21 | sum(ss_ext_sales_price) ext_price 22 | from date_dim, store_sales, item 23 | where d_date_sk = ss_sold_date_sk 24 | and ss_item_sk = i_item_sk 25 | and i_manager_id=28 26 | and d_moy=11 27 | and d_year=1999 28 | group by i_brand, i_brand_id 29 | order by ext_price desc, brand_id 30 | limit 100 31 | 32 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q3.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q3.sql-- 19 | 20 | SELECT dt.d_year, item.i_brand_id brand_id, item.i_brand brand,SUM(ss_ext_sales_price) sum_agg 21 | FROM date_dim dt, store_sales, item 22 | WHERE dt.d_date_sk = store_sales.ss_sold_date_sk 23 | AND store_sales.ss_item_sk = item.i_item_sk 24 | AND item.i_manufact_id = 128 25 | AND dt.d_moy=11 26 | GROUP BY dt.d_year, item.i_brand, item.i_brand_id 27 | ORDER BY dt.d_year, sum_agg desc, brand_id 28 | LIMIT 100 29 | 30 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q96.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q96.sql-- 19 | 20 | select count(*) 21 | from store_sales, household_demographics, time_dim, store 22 | where ss_sold_time_sk = time_dim.t_time_sk 23 | and ss_hdemo_sk = household_demographics.hd_demo_sk 24 | and ss_store_sk = s_store_sk 25 | and time_dim.t_hour = 20 26 | and time_dim.t_minute >= 30 27 | and household_demographics.hd_dep_count = 7 28 | and store.s_store_name = 'ese' 29 | order by count(*) 30 | limit 100 31 | 32 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q52.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q52.sql-- 19 | 20 | select dt.d_year 21 | ,item.i_brand_id brand_id 22 | ,item.i_brand brand 23 | ,sum(ss_ext_sales_price) ext_price 24 | from date_dim dt, store_sales, item 25 | where dt.d_date_sk = store_sales.ss_sold_date_sk 26 | and store_sales.ss_item_sk = item.i_item_sk 27 | and item.i_manager_id = 1 28 | and dt.d_moy=11 29 | and dt.d_year=2000 30 | group by dt.d_year, item.i_brand, item.i_brand_id 31 | order by dt.d_year, ext_price desc, brand_id 32 | limit 100 33 | 34 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q22.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q22.sql-- 19 | 20 | select i_product_name, i_brand, i_class, i_category, avg(inv_quantity_on_hand) qoh 21 | from inventory, date_dim, item, warehouse 22 | where inv_date_sk=d_date_sk 23 | and inv_item_sk=i_item_sk 24 | and inv_warehouse_sk = w_warehouse_sk 25 | and d_month_seq between 1200 and 1200 + 11 26 | group by rollup(i_product_name, i_brand, i_class, i_category) 27 | order by qoh, i_product_name, i_brand, i_class, i_category 28 | limit 100 29 | 30 | -------------------------------------------------------------------------------- /dev/anolisos8/amd64/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.io/openanolis/anolisos:8 2 | 3 | RUN yum install -y libzip unzip wget cmake openssl-devel llvm clang-devel clang krb5-workstation git gcc gcc-c++ 4 | 5 | # install rust nightly toolchain 6 | RUN curl https://sh.rustup.rs > /rustup-init 7 | RUN chmod +x /rustup-init 8 | RUN /rustup-init -y --default-toolchain nightly-2025-06-01-x86_64-unknown-linux-gnu 9 | 10 | # install java 11 | RUN yum install -y java-1.8.0-openjdk java-1.8.0-openjdk-devel 12 | RUN echo 'export JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk"' >> ~/.bashrc 13 | 14 | # install protoc 15 | RUN wget -O /protobuf-21.7-linux-x86_64.zip https://github.com/protocolbuffers/protobuf/releases/download/v21.7/protoc-21.7-linux-x86_64.zip 16 | RUN mkdir /protobuf-bin && (cd /protobuf-bin && unzip /protobuf-21.7-linux-x86_64.zip) 17 | RUN echo 'export PATH="$PATH:/protobuf-bin/bin"' >> ~/.bashrc 18 | 19 | # attach libjvm.so 20 | RUN echo 'export LD_LIBRARY_PATH=${JAVA_HOME}/jre/lib/amd64/server:${LD_LIBRARY_PATH}' >> ~/.bashrc 21 | 22 | # setup hadoop env 23 | RUN curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /root 24 | RUN echo "export HADOOP_HOME=/root/hadoop-3.3.5" >> ~/.bashrc 25 | RUN echo "export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> ~/.bashrc 26 | RUN echo "export HDRS_NAMENODE=default" >> ~/.bashrc 27 | RUN echo "export HDRS_WORKDIR=/tmp/hdrs/" >> ~/.bashrc -------------------------------------------------------------------------------- /dev/integration/sql_set/q42.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q42.sql-- 19 | 20 | select dt.d_year, item.i_category_id, item.i_category, sum(ss_ext_sales_price) 21 | from date_dim dt, store_sales, item 22 | where dt.d_date_sk = store_sales.ss_sold_date_sk 23 | and store_sales.ss_item_sk = item.i_item_sk 24 | and item.i_manager_id = 1 25 | and dt.d_moy=11 26 | and dt.d_year=2000 27 | group by dt.d_year 28 | ,item.i_category_id 29 | ,item.i_category 30 | order by sum(ss_ext_sales_price) desc,dt.d_year 31 | ,item.i_category_id 32 | ,item.i_category 33 | limit 100 34 | 35 | -------------------------------------------------------------------------------- /riffle-ctl/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "riffle-ctl" 3 | version = "0.19.0" 4 | edition = "2021" 5 | 6 | [[bin]] 7 | name = "riffle-ctl" 8 | path = "src/main.rs" 9 | 10 | [features] 11 | hdrs_append = ["riffle-server/hdrs"] 12 | 13 | [dependencies] 14 | riffle-server = { workspace = true } 15 | tokio = { workspace = true, features = ["full"] } 16 | clap = { workspace = true, features = ["derive"] } 17 | serde_json = { workspace = true } 18 | datafusion = { workspace = true } 19 | csv = { workspace = true } 20 | reqwest = { workspace = true, features = ["json"] } 21 | indicatif = { workspace = true } 22 | anyhow = { workspace = true } 23 | serde = { workspace = true, features = ["derive"] } 24 | url = { workspace = true } 25 | async-trait = { workspace = true } 26 | tempfile = { workspace = true } 27 | futures = { workspace = true } 28 | bytesize = { workspace = true, features = ["serde"] } 29 | 30 | bytes = { workspace = true } 31 | dashmap = { workspace = true } 32 | 33 | strum = { workspace = true } 34 | strum_macros = { workspace = true } 35 | 36 | pgwire = { workspace = true } 37 | datafusion-postgres = { workspace = true } 38 | 39 | sqlparser = { workspace = true } 40 | log = { workspace = true } 41 | env_logger = { workspace = true } 42 | logforth = { workspace = true } 43 | toml = { workspace = true } 44 | dirs = { workspace = true } 45 | 46 | [dev-dependencies] 47 | poem = { workspace = true, features = ["rustls", "test", "anyhow"] } 48 | -------------------------------------------------------------------------------- /dev/integration/sql_set/ss_max.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | select 19 | count(*) as total, 20 | count(ss_sold_date_sk) as not_null_total, 21 | count(distinct ss_sold_date_sk) as unique_days, 22 | max(ss_sold_date_sk) as max_ss_sold_date_sk, 23 | max(ss_sold_time_sk) as max_ss_sold_time_sk, 24 | max(ss_item_sk) as max_ss_item_sk, 25 | max(ss_customer_sk) as max_ss_customer_sk, 26 | max(ss_cdemo_sk) as max_ss_cdemo_sk, 27 | max(ss_hdemo_sk) as max_ss_hdemo_sk, 28 | max(ss_addr_sk) as max_ss_addr_sk, 29 | max(ss_store_sk) as max_ss_store_sk, 30 | max(ss_promo_sk) as max_ss_promo_sk 31 | from store_sales 32 | -------------------------------------------------------------------------------- /dev/integration/sql_set/ss_maxb.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | select 19 | count(*) as total, 20 | count(ss_sold_date_sk) as not_null_total, 21 | --count(distinct ss_sold_date_sk) as unique_days, 22 | max(ss_sold_date_sk) as max_ss_sold_date_sk, 23 | max(ss_sold_time_sk) as max_ss_sold_time_sk, 24 | max(ss_item_sk) as max_ss_item_sk, 25 | max(ss_customer_sk) as max_ss_customer_sk, 26 | max(ss_cdemo_sk) as max_ss_cdemo_sk, 27 | max(ss_hdemo_sk) as max_ss_hdemo_sk, 28 | max(ss_addr_sk) as max_ss_addr_sk, 29 | max(ss_store_sk) as max_ss_store_sk, 30 | max(ss_promo_sk) as max_ss_promo_sk 31 | from store_sales 32 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q37.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q37.sql-- 19 | 20 | select i_item_id, i_item_desc, i_current_price 21 | from item, inventory, date_dim, catalog_sales 22 | where i_current_price between 68 and 68 + 30 23 | and inv_item_sk = i_item_sk 24 | and d_date_sk=inv_date_sk 25 | and d_date between cast('2000-02-01' as date) and (cast('2000-02-01' as date) + interval '60' day) 26 | and i_manufact_id in (677,940,694,808) 27 | and inv_quantity_on_hand between 100 and 500 28 | and cs_item_sk = i_item_sk 29 | group by i_item_id,i_item_desc,i_current_price 30 | order by i_item_id 31 | limit 100 32 | 33 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q82.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q82.sql-- 19 | 20 | select i_item_id, i_item_desc, i_current_price 21 | from item, inventory, date_dim, store_sales 22 | where i_current_price between 62 and 62+30 23 | and inv_item_sk = i_item_sk 24 | and d_date_sk=inv_date_sk 25 | and d_date between cast('2000-05-25' as date) and (cast('2000-05-25' as date) + interval '60' day) 26 | and i_manufact_id in (129, 270, 821, 423) 27 | and inv_quantity_on_hand between 100 and 500 28 | and ss_item_sk = i_item_sk 29 | group by i_item_id,i_item_desc,i_current_price 30 | order by i_item_id 31 | limit 100 32 | 33 | -------------------------------------------------------------------------------- /riffle-server/src/mem_allocator/default.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | use super::error::{ProfError, ProfResult}; 19 | 20 | pub fn is_prof_enabled() -> bool { 21 | false 22 | } 23 | 24 | pub async fn dump_prof(_path: &str) -> ProfResult> { 25 | Err(ProfError::MemProfilingNotEnabled) 26 | } 27 | 28 | pub async fn dump_heap_flamegraph() -> ProfResult> { 29 | Err(ProfError::MemProfilingNotEnabled) 30 | } 31 | 32 | pub fn activate_prof() -> ProfResult<()> { 33 | Err(ProfError::MemProfilingNotEnabled) 34 | } 35 | 36 | pub fn deactivate_prof() -> ProfResult<()> { 37 | Err(ProfError::MemProfilingNotEnabled) 38 | } 39 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q15.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q15.sql-- 19 | 20 | select ca_zip, sum(cs_sales_price) 21 | from catalog_sales, customer, customer_address, date_dim 22 | where cs_bill_customer_sk = c_customer_sk 23 | and c_current_addr_sk = ca_address_sk 24 | and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', 25 | '85392', '85460', '80348', '81792') 26 | or ca_state in ('CA','WA','GA') 27 | or cs_sales_price > 500) 28 | and cs_sold_date_sk = d_date_sk 29 | and d_qoy = 2 and d_year = 2001 30 | group by ca_zip 31 | order by ca_zip 32 | limit 100 33 | 34 | -------------------------------------------------------------------------------- /riffle-server/src/logforth_service.rs: -------------------------------------------------------------------------------- 1 | use crate::config::{LogConfig, LogLevel, RotationConfig}; 2 | use crate::util; 3 | use logforth::append::rolling_file::{RollingFileBuilder, Rotation}; 4 | use logforth::append::{rolling_file, RollingFile}; 5 | use logforth::DropGuard; 6 | 7 | const LOG_FILE_NAME_PREFIX: &str = "riffle-server"; 8 | 9 | pub struct LogService; 10 | impl LogService { 11 | pub fn init(log: &LogConfig) -> DropGuard { 12 | let rotation = match log.rotation { 13 | RotationConfig::Hourly => Rotation::Hourly, 14 | RotationConfig::Daily => Rotation::Daily, 15 | RotationConfig::Never => Rotation::Never, 16 | }; 17 | let max_file_size = util::to_bytes(&log.max_file_size); 18 | let (rolling_writer, _guard) = RollingFileBuilder::new(&log.path) 19 | .rotation(rotation) 20 | .filename_prefix(LOG_FILE_NAME_PREFIX) 21 | .max_file_size(max_file_size as usize) 22 | .max_log_files(log.max_log_files) 23 | .build() 24 | .unwrap(); 25 | 26 | let log_level = match log.log_level { 27 | LogLevel::DEBUG => log::LevelFilter::Debug, 28 | LogLevel::INFO => log::LevelFilter::Info, 29 | LogLevel::WARN => log::LevelFilter::Warn, 30 | }; 31 | logforth::builder() 32 | .dispatch(|d| d.filter(log_level).append(rolling_writer)) 33 | .apply(); 34 | 35 | _guard 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /dev/integration/spark-defaults.conf: -------------------------------------------------------------------------------- 1 | spark.shuffle.manager org.apache.spark.shuffle.RssShuffleManager 2 | spark.rss.coordinator.quorum uniffle-coordinator:21000 3 | spark.rss.storage.type MEMORY_LOCALFILE 4 | spark.executor.instances 2 5 | spark.executor.cores 2 6 | spark.executor.memory 2g 7 | spark.sql.shuffle.partitions 4 8 | 9 | spark.serializer = org.apache.spark.serializer.KryoSerializer 10 | 11 | spark.rss.dynamicClientConf.enabled = false 12 | spark.rss.client.type = GRPC_NETTY 13 | 14 | spark.rss.client.assignment.shuffle.nodes.max = 1 15 | 16 | spark.sql.catalog.tpcds=org.apache.kyuubi.spark.connector.tpcds.TPCDSCatalog 17 | 18 | spark.rss.client.io.compression.codec zstd 19 | spark.rss.client.io.compression.zstd.level 3 20 | 21 | spark.plugins org.apache.spark.UnifflePlugin 22 | 23 | spark.rss.client.reassign.enabled true 24 | 25 | spark.rss.client.write.overlappingCompressionEnable true 26 | spark.rss.client.write.overlappingCompressionThreads 1 27 | 28 | spark.rss.riffle.hdfsClientEagerLoadingEnabled=true 29 | 30 | spark.rss.riffle.readAheadEnabled=true 31 | 32 | spark.rss.client.read.nextReadSegmentsReportEnabled true 33 | 34 | spark.rss.client.read.overlappingDecompressionEnable true 35 | spark.rss.client.read.prefetch.enabled true 36 | 37 | spark.rss.client.integrityValidation.enabled true 38 | spark.rss.client.integrityValidation.failureAnalysisEnabled true 39 | spark.rss.client.integrityValidation.serverManagementEnabled true 40 | 41 | spark.rss.riffle.getMemoryDataUrpcVersion V2 -------------------------------------------------------------------------------- /dev/integration/sql_set/q19.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q19.sql-- 19 | 20 | select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact, 21 | sum(ss_ext_sales_price) ext_price 22 | from date_dim, store_sales, item,customer,customer_address,store 23 | where d_date_sk = ss_sold_date_sk 24 | and ss_item_sk = i_item_sk 25 | and i_manager_id = 8 26 | and d_moy = 11 27 | and d_year = 1998 28 | and ss_customer_sk = c_customer_sk 29 | and c_current_addr_sk = ca_address_sk 30 | and substr(ca_zip,1,5) <> substr(s_zip,1,5) 31 | and ss_store_sk = s_store_sk 32 | group by i_brand, i_brand_id, i_manufact_id, i_manufact 33 | order by ext_price desc, brand, brand_id, i_manufact_id, i_manufact 34 | limit 100 35 | 36 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q7.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q7.sql-- 19 | 20 | SELECT i_item_id, 21 | avg(ss_quantity) agg1, 22 | avg(ss_list_price) agg2, 23 | avg(ss_coupon_amt) agg3, 24 | avg(ss_sales_price) agg4 25 | FROM store_sales, customer_demographics, date_dim, item, promotion 26 | WHERE ss_sold_date_sk = d_date_sk AND 27 | ss_item_sk = i_item_sk AND 28 | ss_cdemo_sk = cd_demo_sk AND 29 | ss_promo_sk = p_promo_sk AND 30 | cd_gender = 'M' AND 31 | cd_marital_status = 'S' AND 32 | cd_education_status = 'College' AND 33 | (p_channel_email = 'N' or p_channel_event = 'N') AND 34 | d_year = 2000 35 | GROUP BY i_item_id 36 | ORDER BY i_item_id LIMIT 100 37 | 38 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q93.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q93.sql-- 19 | 20 | select ss_customer_sk, sum(act_sales) sumsales 21 | from (select 22 | ss_item_sk, ss_ticket_number, ss_customer_sk, 23 | case when sr_return_quantity is not null then (ss_quantity-sr_return_quantity)*ss_sales_price 24 | else (ss_quantity*ss_sales_price) end act_sales 25 | from store_sales 26 | left outer join store_returns 27 | on (sr_item_sk = ss_item_sk and sr_ticket_number = ss_ticket_number), 28 | reason 29 | where sr_reason_sk = r_reason_sk and r_reason_desc = 'reason 28') t 30 | group by ss_customer_sk 31 | order by sumsales, ss_customer_sk 32 | limit 100 33 | 34 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q84.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q84.sql-- 19 | 20 | select c_customer_id as customer_id 21 | ,coalesce(c_last_name,'') + ', ' + coalesce(c_first_name,'') as customername 22 | from customer 23 | ,customer_address 24 | ,customer_demographics 25 | ,household_demographics 26 | ,income_band 27 | ,store_returns 28 | where ca_city = 'Edgewood' 29 | and c_current_addr_sk = ca_address_sk 30 | and ib_lower_bound >= 38128 31 | and ib_upper_bound <= 38128 + 50000 32 | and ib_income_band_sk = hd_income_band_sk 33 | and cd_demo_sk = c_current_cdemo_sk 34 | and hd_demo_sk = c_current_hdemo_sk 35 | and sr_cdemo_sk = cd_demo_sk 36 | order by c_customer_id 37 | limit 100 38 | 39 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q98.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q98.sql-- 19 | 20 | select i_item_desc, i_category, i_class, i_current_price 21 | ,sum(ss_ext_sales_price) as itemrevenue 22 | ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over 23 | (partition by i_class) as revenueratio 24 | from 25 | store_sales, item, date_dim 26 | where 27 | ss_item_sk = i_item_sk 28 | and i_category in ('Sports', 'Books', 'Home') 29 | and ss_sold_date_sk = d_date_sk 30 | and d_date between cast('1999-02-22' as date) 31 | and (cast('1999-02-22' as date) + interval '30' day) 32 | group by 33 | i_item_id, i_item_desc, i_category, i_class, i_current_price 34 | order by 35 | i_category, i_class, i_item_id, i_item_desc, revenueratio 36 | 37 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q26.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q26.sql-- 19 | 20 | select i_item_id, 21 | avg(cs_quantity) agg1, 22 | avg(cs_list_price) agg2, 23 | avg(cs_coupon_amt) agg3, 24 | avg(cs_sales_price) agg4 25 | from catalog_sales, customer_demographics, date_dim, item, promotion 26 | where cs_sold_date_sk = d_date_sk and 27 | cs_item_sk = i_item_sk and 28 | cs_bill_cdemo_sk = cd_demo_sk and 29 | cs_promo_sk = p_promo_sk and 30 | cd_gender = 'M' and 31 | cd_marital_status = 'S' and 32 | cd_education_status = 'College' and 33 | (p_channel_email = 'N' or p_channel_event = 'N') and 34 | d_year = 2000 35 | group by i_item_id 36 | order by i_item_id 37 | limit 100 38 | 39 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q32.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q32.sql-- 19 | 20 | select sum(cs_ext_discount_amt) as `excess discount amount` 21 | from 22 | catalog_sales, item, date_dim 23 | where 24 | i_manufact_id = 977 25 | and i_item_sk = cs_item_sk 26 | and d_date between cast ('2000-01-27' as date) and (cast('2000-01-27' as date) + interval '90' day) 27 | and d_date_sk = cs_sold_date_sk 28 | and cs_ext_discount_amt > ( 29 | select 1.3 * avg(cs_ext_discount_amt) 30 | from catalog_sales, date_dim 31 | where cs_item_sk = i_item_sk 32 | and d_date between cast ('2000-01-27' as date) and (cast('2000-01-27' as date) + interval '90' day) 33 | and d_date_sk = cs_sold_date_sk) 34 | limit 100 35 | 36 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q6.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q6.sql-- 19 | 20 | SELECT state, cnt FROM ( 21 | SELECT a.ca_state state, count(*) cnt 22 | FROM 23 | customer_address a, customer c, store_sales s, date_dim d, item i 24 | WHERE a.ca_address_sk = c.c_current_addr_sk 25 | AND c.c_customer_sk = s.ss_customer_sk 26 | AND s.ss_sold_date_sk = d.d_date_sk 27 | AND s.ss_item_sk = i.i_item_sk 28 | AND d.d_month_seq = 29 | (SELECT distinct (d_month_seq) FROM date_dim 30 | WHERE d_year = 2001 AND d_moy = 1) 31 | AND i.i_current_price > 1.2 * 32 | (SELECT avg(j.i_current_price) FROM item j 33 | WHERE j.i_category = i.i_category) 34 | GROUP BY a.ca_state 35 | ) x 36 | WHERE cnt >= 10 37 | ORDER BY cnt LIMIT 100 38 | 39 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q86.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q86.sql-- 19 | 20 | select sum(ws_net_paid) as total_sum, i_category, i_class, 21 | grouping(i_category)+grouping(i_class) as lochierarchy, 22 | rank() over ( 23 | partition by grouping(i_category)+grouping(i_class), 24 | case when grouping(i_class) = 0 then i_category end 25 | order by sum(ws_net_paid) desc) as rank_within_parent 26 | from 27 | web_sales, date_dim d1, item 28 | where 29 | d1.d_month_seq between 1200 and 1200+11 30 | and d1.d_date_sk = ws_sold_date_sk 31 | and i_item_sk = ws_item_sk 32 | group by rollup(i_category,i_class) 33 | order by 34 | lochierarchy desc, 35 | case when lochierarchy = 0 then i_category end, 36 | rank_within_parent 37 | limit 100 38 | 39 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q92.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q92.sql-- 19 | 20 | select sum(ws_ext_discount_amt) as `Excess Discount Amount` 21 | from web_sales, item, date_dim 22 | where i_manufact_id = 350 23 | and i_item_sk = ws_item_sk 24 | and d_date between cast ('2000-01-27' as date) and (cast('2000-01-27' as date) + interval '90' day) 25 | and d_date_sk = ws_sold_date_sk 26 | and ws_ext_discount_amt > 27 | ( 28 | SELECT 1.3 * avg(ws_ext_discount_amt) 29 | FROM web_sales, date_dim 30 | WHERE ws_item_sk = i_item_sk 31 | and d_date between cast ('2000-01-27' as date) and (cast('2000-01-27' as date) + interval '90' day) 32 | and d_date_sk = ws_sold_date_sk 33 | ) 34 | order by sum(ws_ext_discount_amt) 35 | limit 100 36 | 37 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q12.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q12.sql-- 19 | 20 | select i_item_id, 21 | i_item_desc, i_category, i_class, i_current_price, 22 | sum(ws_ext_sales_price) as itemrevenue, 23 | sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over 24 | (partition by i_class) as revenueratio 25 | from 26 | web_sales, item, date_dim 27 | where 28 | ws_item_sk = i_item_sk 29 | and i_category in ('Sports', 'Books', 'Home') 30 | and ws_sold_date_sk = d_date_sk 31 | and d_date between cast('1999-02-22' as date) 32 | and (cast('1999-02-22' as date) + interval '30' day) 33 | group by 34 | i_item_id, i_item_desc, i_category, i_class, i_current_price 35 | order by 36 | i_category, i_class, i_item_id, i_item_desc, revenueratio 37 | LIMIT 100 38 | 39 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q1.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q1.sql-- 19 | 20 | WITH customer_total_return AS 21 | (SELECT sr_customer_sk AS ctr_customer_sk, sr_store_sk AS ctr_store_sk, 22 | sum(sr_return_amt) AS ctr_total_return 23 | FROM store_returns, date_dim 24 | WHERE sr_returned_date_sk = d_date_sk AND d_year = 2000 25 | GROUP BY sr_customer_sk, sr_store_sk) 26 | SELECT c_customer_id 27 | FROM customer_total_return ctr1, store, customer 28 | WHERE ctr1.ctr_total_return > 29 | (SELECT avg(ctr_total_return)*1.2 30 | FROM customer_total_return ctr2 31 | WHERE ctr1.ctr_store_sk = ctr2.ctr_store_sk) 32 | AND s_store_sk = ctr1.ctr_store_sk 33 | AND s_state = 'TN' 34 | AND ctr1.ctr_customer_sk = c_customer_sk 35 | ORDER BY c_customer_id LIMIT 100 36 | 37 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q20.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q20.sql-- 19 | 20 | select i_item_id, i_item_desc 21 | ,i_category 22 | ,i_class 23 | ,i_current_price 24 | ,sum(cs_ext_sales_price) as itemrevenue 25 | ,sum(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over 26 | (partition by i_class) as revenueratio 27 | from catalog_sales, item, date_dim 28 | where cs_item_sk = i_item_sk 29 | and i_category in ('Sports', 'Books', 'Home') 30 | and cs_sold_date_sk = d_date_sk 31 | and d_date between cast('1999-02-22' as date) 32 | and (cast('1999-02-22' as date) + interval '30' day) 33 | group by i_item_id, i_item_desc, i_category, i_class, i_current_price 34 | order by i_category, i_class, i_item_id, i_item_desc, revenueratio 35 | limit 100 36 | 37 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q27.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q27.sql-- 19 | 20 | select i_item_id, 21 | s_state, grouping(s_state) g_state, 22 | avg(ss_quantity) agg1, 23 | avg(ss_list_price) agg2, 24 | avg(ss_coupon_amt) agg3, 25 | avg(ss_sales_price) agg4 26 | from store_sales, customer_demographics, date_dim, store, item 27 | where ss_sold_date_sk = d_date_sk and 28 | ss_item_sk = i_item_sk and 29 | ss_store_sk = s_store_sk and 30 | ss_cdemo_sk = cd_demo_sk and 31 | cd_gender = 'M' and 32 | cd_marital_status = 'S' and 33 | cd_education_status = 'College' and 34 | d_year = 2002 and 35 | s_state in ('TN','TN', 'TN', 'TN', 'TN', 'TN') 36 | group by rollup (i_item_id, s_state) 37 | order by i_item_id, s_state 38 | limit 100 39 | 40 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q45.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q45.sql-- 19 | 20 | select ca_zip, ca_city, sum(ws_sales_price) 21 | from web_sales, customer, customer_address, date_dim, item 22 | where ws_bill_customer_sk = c_customer_sk 23 | and c_current_addr_sk = ca_address_sk 24 | and ws_item_sk = i_item_sk 25 | and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') 26 | or 27 | i_item_id in (select i_item_id 28 | from item 29 | where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) 30 | ) 31 | ) 32 | and ws_sold_date_sk = d_date_sk 33 | and d_qoy = 2 and d_year = 2001 34 | group by ca_zip, ca_city 35 | order by ca_zip, ca_city 36 | limit 100 37 | 38 | -------------------------------------------------------------------------------- /dev/anolisos8/arm64/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.io/openanolis/anolisos:8 2 | 3 | RUN yum install -y libzip unzip wget cmake openssl-devel llvm clang-devel clang krb5-workstation git gcc gcc-c++ 4 | 5 | # install rust nightly toolchain 6 | RUN curl https://sh.rustup.rs > /rustup-init 7 | RUN chmod +x /rustup-init 8 | # Note: Specify the correct target for aarch64 9 | RUN /rustup-init -y --default-toolchain nightly-2025-06-01-aarch64-unknown-linux-gnu 10 | 11 | # install java 12 | RUN yum install -y java-1.8.0-openjdk java-1.8.0-openjdk-devel 13 | # Note: Update JAVA_HOME path for aarch64 14 | RUN echo 'export JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk"' >> ~/.bashrc 15 | 16 | # install protoc 17 | # Note: Download the aarch64 version of protoc 18 | RUN wget -O /protobuf-21.7-linux-aarch64.zip https://github.com/protocolbuffers/protobuf/releases/download/v21.7/protoc-21.7-linux-aarch_64.zip 19 | RUN mkdir /protobuf-bin && (cd /protobuf-bin && unzip /protobuf-21.7-linux-aarch64.zip) 20 | RUN echo 'export PATH="$PATH:/protobuf-bin/bin"' >> ~/.bashrc 21 | 22 | # attach libjvm.so 23 | # Note: Update the library path for aarch64 24 | RUN echo 'export LD_LIBRARY_PATH=${JAVA_HOME}/jre/lib/aarch64/server:${LD_LIBRARY_PATH}' >> ~/.bashrc 25 | 26 | # setup hadoop env 27 | RUN curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5-aarch64.tar.gz | tar zxf - -C /root 28 | RUN echo "export HADOOP_HOME=/root/hadoop-3.3.5" >> ~/.bashrc 29 | RUN echo "export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> ~/.bashrc 30 | RUN echo "export HDRS_NAMENODE=default" >> ~/.bashrc 31 | RUN echo "export HDRS_WORKDIR=/tmp/hdrs/" >> ~/.bashrc 32 | 33 | RUN echo "export RUST_BACKTRACE=1" >> ~/.bashrc -------------------------------------------------------------------------------- /riffle-server/src/store/spill/storage_select_handler.rs: -------------------------------------------------------------------------------- 1 | use crate::event_bus::{Event, Subscriber}; 2 | use crate::store::hybrid::HybridStore; 3 | use crate::store::spill::{ 4 | handle_spill_failure, handle_spill_failure_whatever_error, SpillMessage, 5 | }; 6 | use async_trait::async_trait; 7 | use log::error; 8 | use std::sync::Arc; 9 | 10 | #[derive(Clone)] 11 | pub struct StorageSelectHandler { 12 | pub store: Arc, 13 | } 14 | 15 | impl StorageSelectHandler { 16 | pub fn new(store: &Arc) -> Self { 17 | Self { 18 | store: store.clone(), 19 | } 20 | } 21 | } 22 | unsafe impl Send for StorageSelectHandler {} 23 | unsafe impl Sync for StorageSelectHandler {} 24 | 25 | #[async_trait] 26 | impl Subscriber for StorageSelectHandler { 27 | type Input = SpillMessage; 28 | 29 | async fn on_event(&self, event: Event) -> bool { 30 | let msg = event.get_data(); 31 | let select_result = self.store.select_storage_for_buffer(msg).await; 32 | let upstream_event_bus = &self.store.event_bus; 33 | match select_result { 34 | Ok(storage) => { 35 | if let Some(event_bus) = upstream_event_bus.children.get(&storage) { 36 | msg.set_candidate_storage_type(storage); 37 | let _ = event_bus.publish(event).await; 38 | } 39 | true 40 | } 41 | Err(e) => { 42 | error!("Errors on the selecting storage for app: {:?} and then drop this event. error: {:?}", &msg.ctx.uid, &e); 43 | handle_spill_failure_whatever_error(msg, self.store.clone(), e).await; 44 | false 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /riffle-server/src/urpc/shutdown.rs: -------------------------------------------------------------------------------- 1 | use tokio::sync::broadcast; 2 | 3 | /// Listens for the server shutdown signal. 4 | /// 5 | /// Shutdown is signalled using a `broadcast::Receiver`. Only a single value is 6 | /// ever sent. Once a value has been sent via the broadcast channel, the server 7 | /// should shutdown. 8 | /// 9 | /// The `Shutdown` struct listens for the signal and tracks that the signal has 10 | /// been received. Callers may query for whether the shutdown signal has been 11 | /// received or not. 12 | #[derive(Debug)] 13 | pub struct Shutdown { 14 | /// `true` if the shutdown signal has been received 15 | is_shutdown: bool, 16 | 17 | /// The receive half of the channel used to listen for shutdown. 18 | notify: broadcast::Receiver<()>, 19 | } 20 | 21 | impl Shutdown { 22 | /// Create a new `Shutdown` backed by the given `broadcast::Receiver`. 23 | pub(crate) fn new(notify: broadcast::Receiver<()>) -> Shutdown { 24 | Shutdown { 25 | is_shutdown: false, 26 | notify, 27 | } 28 | } 29 | 30 | /// Returns `true` if the shutdown signal has been received. 31 | pub(crate) fn is_shutdown(&self) -> bool { 32 | self.is_shutdown 33 | } 34 | 35 | /// Receive the shutdown notice, waiting if necessary. 36 | pub(crate) async fn recv(&mut self) { 37 | // If the shutdown signal has already been received, then return 38 | // immediately. 39 | if self.is_shutdown { 40 | return; 41 | } 42 | 43 | // Cannot receive a "lag error" as only one value is ever sent. 44 | let _ = self.notify.recv().await; 45 | 46 | // Remember that the signal has been received. 47 | self.is_shutdown = true; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /riffle-server/src/store/local/options.rs: -------------------------------------------------------------------------------- 1 | use crate::app_manager::request_context::PurgeDataContext; 2 | use crate::store::DataBytes; 3 | use std::fmt::{Display, Formatter}; 4 | 5 | /// If the append is false, the offset should be None. 6 | /// Otherwise, if the offset is None, it indicates using buffer io. 7 | /// Otherwise, it uses the direct IO 8 | pub struct WriteOptions { 9 | pub append: bool, 10 | pub data: DataBytes, 11 | // if the offset is empty, using the buffer io otherwise direct io. 12 | pub offset: Option, 13 | } 14 | 15 | impl WriteOptions { 16 | pub fn with_write_all(data: DataBytes) -> Self { 17 | Self { 18 | append: false, 19 | data, 20 | offset: None, 21 | } 22 | } 23 | 24 | pub fn with_append_of_direct_io(data: DataBytes, offset: u64) -> Self { 25 | Self { 26 | append: true, 27 | data, 28 | offset: Some(offset), 29 | } 30 | } 31 | 32 | pub fn with_append_of_buffer_io(data: DataBytes) -> Self { 33 | Self { 34 | append: true, 35 | data, 36 | offset: None, 37 | } 38 | } 39 | 40 | pub fn is_append(&self) -> bool { 41 | self.append 42 | } 43 | 44 | pub fn is_direct_io(&self) -> bool { 45 | self.offset.is_some() 46 | } 47 | } 48 | 49 | impl Display for WriteOptions { 50 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 51 | write!( 52 | f, 53 | "append: {}, data_len: {}, offset: {:?}", 54 | self.append, 55 | self.data.len(), 56 | self.offset 57 | ) 58 | } 59 | } 60 | 61 | pub enum CreateOptions { 62 | FILE, 63 | DIR, 64 | } 65 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q36.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q36.sql-- 19 | 20 | select 21 | sum(ss_net_profit)/sum(ss_ext_sales_price) as gross_margin 22 | ,i_category 23 | ,i_class 24 | ,grouping(i_category)+grouping(i_class) as lochierarchy 25 | ,rank() over ( 26 | partition by grouping(i_category)+grouping(i_class), 27 | case when grouping(i_class) = 0 then i_category end 28 | order by sum(ss_net_profit)/sum(ss_ext_sales_price) asc) as rank_within_parent 29 | from 30 | store_sales, date_dim d1, item, store 31 | where 32 | d1.d_year = 2001 33 | and d1.d_date_sk = ss_sold_date_sk 34 | and i_item_sk = ss_item_sk 35 | and s_store_sk = ss_store_sk 36 | and s_state in ('TN','TN','TN','TN','TN','TN','TN','TN') 37 | group by rollup(i_category,i_class) 38 | order by 39 | lochierarchy desc 40 | ,case when lochierarchy = 0 then i_category end 41 | ,rank_within_parent 42 | limit 100 43 | 44 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q65.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q65.sql-- 19 | 20 | select 21 | s_store_name, i_item_desc, sc.revenue, i_current_price, i_wholesale_cost, i_brand 22 | from store, item, 23 | (select ss_store_sk, avg(revenue) as ave 24 | from 25 | (select ss_store_sk, ss_item_sk, 26 | sum(ss_sales_price) as revenue 27 | from store_sales, date_dim 28 | where ss_sold_date_sk = d_date_sk and d_month_seq between 1176 and 1176+11 29 | group by ss_store_sk, ss_item_sk) sa 30 | group by ss_store_sk) sb, 31 | (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue 32 | from store_sales, date_dim 33 | where ss_sold_date_sk = d_date_sk and d_month_seq between 1176 and 1176+11 34 | group by ss_store_sk, ss_item_sk) sc 35 | where sb.ss_store_sk = sc.ss_store_sk and 36 | sc.revenue <= 0.1 * sb.ave and 37 | s_store_sk = sc.ss_store_sk and 38 | i_item_sk = sc.ss_item_sk 39 | order by s_store_name, i_item_desc 40 | limit 100 41 | 42 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q24a.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q24a.sql-- 19 | 20 | with ssales as 21 | (select c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, 22 | i_current_price, i_manager_id, i_units, i_size, sum(ss_net_paid) netpaid 23 | from store_sales, store_returns, store, item, customer, customer_address 24 | where ss_ticket_number = sr_ticket_number 25 | and ss_item_sk = sr_item_sk 26 | and ss_customer_sk = c_customer_sk 27 | and ss_item_sk = i_item_sk 28 | and ss_store_sk = s_store_sk 29 | and c_birth_country = upper(ca_country) 30 | and s_zip = ca_zip 31 | and s_market_id = 8 32 | group by c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, 33 | i_current_price, i_manager_id, i_units, i_size) 34 | select c_last_name, c_first_name, s_store_name, sum(netpaid) paid 35 | from ssales 36 | where i_color = 'pale' 37 | group by c_last_name, c_first_name, s_store_name 38 | having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) 39 | 40 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q67.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q67.sql-- 19 | 20 | select * from 21 | (select i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy, s_store_id, 22 | sumsales, rank() over (partition by i_category order by sumsales desc) rk 23 | from 24 | (select i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy, 25 | s_store_id, sum(coalesce(ss_sales_price*ss_quantity,0)) sumsales 26 | from store_sales, date_dim, store, item 27 | where ss_sold_date_sk=d_date_sk 28 | and ss_item_sk=i_item_sk 29 | and ss_store_sk = s_store_sk 30 | and d_month_seq between 1200 and 1200+11 31 | group by rollup(i_category, i_class, i_brand, i_product_name, d_year, d_qoy, 32 | d_moy,s_store_id))dw1) dw2 33 | where rk <= 100 34 | order by 35 | i_category, i_class, i_brand, i_product_name, d_year, 36 | d_qoy, d_moy, s_store_id, sumsales, rk 37 | limit 100 38 | 39 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q24b.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q24b.sql-- 19 | 20 | with ssales as 21 | (select c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, 22 | i_current_price, i_manager_id, i_units, i_size, sum(ss_net_paid) netpaid 23 | from store_sales, store_returns, store, item, customer, customer_address 24 | where ss_ticket_number = sr_ticket_number 25 | and ss_item_sk = sr_item_sk 26 | and ss_customer_sk = c_customer_sk 27 | and ss_item_sk = i_item_sk 28 | and ss_store_sk = s_store_sk 29 | and c_birth_country = upper(ca_country) 30 | and s_zip = ca_zip 31 | and s_market_id = 8 32 | group by c_last_name, c_first_name, s_store_name, ca_state, s_state, 33 | i_color, i_current_price, i_manager_id, i_units, i_size) 34 | select c_last_name, c_first_name, s_store_name, sum(netpaid) paid 35 | from ssales 36 | where i_color = 'chiffon' 37 | group by c_last_name, c_first_name, s_store_name 38 | having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) 39 | 40 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q40.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q40.sql-- 19 | 20 | select 21 | w_state 22 | ,i_item_id 23 | ,sum(case when (cast(d_date as date) < cast('2000-03-11' as date)) 24 | then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_before 25 | ,sum(case when (cast(d_date as date) >= cast('2000-03-11' as date)) 26 | then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_after 27 | from 28 | catalog_sales left outer join catalog_returns on 29 | (cs_order_number = cr_order_number 30 | and cs_item_sk = cr_item_sk) 31 | ,warehouse, item, date_dim 32 | where 33 | i_current_price between 0.99 and 1.49 34 | and i_item_sk = cs_item_sk 35 | and cs_warehouse_sk = w_warehouse_sk 36 | and cs_sold_date_sk = d_date_sk 37 | and d_date between (cast('2000-03-11' as date) - interval '30' day) 38 | and (cast('2000-03-11' as date) + interval '30' day) 39 | group by w_state,i_item_id 40 | order by w_state,i_item_id 41 | limit 100 42 | 43 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q94.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q94.sql-- 19 | 20 | select 21 | count(distinct ws_order_number) as `order count` 22 | ,sum(ws_ext_ship_cost) as `total shipping cost` 23 | ,sum(ws_net_profit) as `total net profit` 24 | from 25 | web_sales ws1, date_dim, customer_address, web_site 26 | where 27 | d_date between cast('1999-02-01' as date) and 28 | (cast('1999-02-01' as date) + interval '60' day) 29 | and ws1.ws_ship_date_sk = d_date_sk 30 | and ws1.ws_ship_addr_sk = ca_address_sk 31 | and ca_state = 'IL' 32 | and ws1.ws_web_site_sk = web_site_sk 33 | and web_company_name = 'pri' 34 | and exists (select * 35 | from web_sales ws2 36 | where ws1.ws_order_number = ws2.ws_order_number 37 | and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) 38 | and not exists(select * 39 | from web_returns wr1 40 | where ws1.ws_order_number = wr1.wr_order_number) 41 | order by count(distinct ws_order_number) 42 | limit 100 43 | 44 | -------------------------------------------------------------------------------- /riffle-server/src/lazy_initializer.rs: -------------------------------------------------------------------------------- 1 | use once_cell::sync::OnceCell; 2 | use parking_lot::Mutex; 3 | use std::sync::atomic::AtomicBool; 4 | use std::sync::atomic::Ordering::SeqCst; 5 | 6 | pub struct LazyInit { 7 | initializer: Mutex Option + Send>>>, 8 | value: OnceCell>, 9 | initialized: AtomicBool, 10 | } 11 | 12 | impl LazyInit { 13 | pub fn new(initializer: F) -> Self 14 | where 15 | F: FnOnce() -> Option + Send + 'static, 16 | { 17 | LazyInit { 18 | initializer: Mutex::new(Some(Box::new(initializer))), 19 | value: OnceCell::new(), 20 | initialized: AtomicBool::new(false), 21 | } 22 | } 23 | 24 | pub fn get_or_init(&self) -> Option<&T> { 25 | let v = self.value.get_or_init(|| { 26 | let initializer = self.initializer.lock().take().unwrap(); 27 | let v = initializer(); 28 | self.initialized.store(true, SeqCst); 29 | v 30 | }); 31 | v.as_ref() 32 | } 33 | 34 | pub fn is_initialized(&self) -> bool { 35 | self.initialized.load(SeqCst) 36 | } 37 | } 38 | 39 | #[cfg(test)] 40 | mod tests { 41 | use crate::lazy_initializer::LazyInit; 42 | use std::sync::atomic::AtomicU64; 43 | use std::sync::atomic::Ordering::SeqCst; 44 | use std::sync::Arc; 45 | 46 | #[test] 47 | fn test() { 48 | let tag = Arc::new(AtomicU64::new(0)); 49 | let tag_fork = tag.clone(); 50 | let lazy_value = LazyInit::new(move || { 51 | println!("Initializing..."); 52 | tag_fork.fetch_add(1, SeqCst); 53 | Some(()) 54 | }); 55 | 56 | let value = lazy_value.get_or_init(); 57 | let value = lazy_value.get_or_init(); 58 | 59 | assert_eq!(tag.load(SeqCst), 1); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q38.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q38.sql-- 19 | 20 | select count(*) from ( 21 | select distinct c_last_name, c_first_name, d_date 22 | from store_sales, date_dim, customer 23 | where store_sales.ss_sold_date_sk = date_dim.d_date_sk 24 | and store_sales.ss_customer_sk = customer.c_customer_sk 25 | and d_month_seq between 1200 and 1200 + 11 26 | intersect 27 | select distinct c_last_name, c_first_name, d_date 28 | from catalog_sales, date_dim, customer 29 | where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk 30 | and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk 31 | and d_month_seq between 1200 and 1200 + 11 32 | intersect 33 | select distinct c_last_name, c_first_name, d_date 34 | from web_sales, date_dim, customer 35 | where web_sales.ws_sold_date_sk = date_dim.d_date_sk 36 | and web_sales.ws_bill_customer_sk = customer.c_customer_sk 37 | and d_month_seq between 1200 and 1200 + 11 38 | ) hot_cust 39 | limit 100 40 | 41 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q87.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q87.sql-- 19 | 20 | select count(*) 21 | from ((select distinct c_last_name, c_first_name, d_date 22 | from store_sales, date_dim, customer 23 | where store_sales.ss_sold_date_sk = date_dim.d_date_sk 24 | and store_sales.ss_customer_sk = customer.c_customer_sk 25 | and d_month_seq between 1200 and 1200+11) 26 | except 27 | (select distinct c_last_name, c_first_name, d_date 28 | from catalog_sales, date_dim, customer 29 | where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk 30 | and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk 31 | and d_month_seq between 1200 and 1200+11) 32 | except 33 | (select distinct c_last_name, c_first_name, d_date 34 | from web_sales, date_dim, customer 35 | where web_sales.ws_sold_date_sk = date_dim.d_date_sk 36 | and web_sales.ws_bill_customer_sk = customer.c_customer_sk 37 | and d_month_seq between 1200 and 1200+11) 38 | ) cool_cust 39 | 40 | -------------------------------------------------------------------------------- /riffle-server/src/mem_allocator/profiling.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | use hyper::StatusCode; 19 | use log::error; 20 | 21 | use super::error::{ProfError, ProfResult}; 22 | 23 | /// Dump the profile to the `path`. path usually is a temp file generated by `tempfile` crate. 24 | pub async fn dump_prof(path: &str) -> ProfResult> { 25 | let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; 26 | let pprof = prof_ctl.dump_pprof().map_err(|err| { 27 | let msg = format!("Errors on jemalloc profile. err: {:?}", &err); 28 | error!("{}", &msg); 29 | ProfError::JemallocError(msg) 30 | })?; 31 | Ok(pprof) 32 | } 33 | 34 | pub async fn dump_heap_flamegraph() -> ProfResult> { 35 | let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; 36 | let svg = prof_ctl.dump_flamegraph().map_err(|err| { 37 | let msg = format!("Errors on jemalloc prof flamegraph. err: {:?}", &err); 38 | error!("{}", &msg); 39 | ProfError::JemallocError(msg) 40 | })?; 41 | Ok(svg) 42 | } 43 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q79.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q79.sql-- 19 | 20 | select 21 | c_last_name,c_first_name,substr(s_city,1,30),ss_ticket_number,amt,profit 22 | from 23 | (select ss_ticket_number 24 | ,ss_customer_sk 25 | ,store.s_city 26 | ,sum(ss_coupon_amt) amt 27 | ,sum(ss_net_profit) profit 28 | from store_sales,date_dim,store,household_demographics 29 | where store_sales.ss_sold_date_sk = date_dim.d_date_sk 30 | and store_sales.ss_store_sk = store.s_store_sk 31 | and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk 32 | and (household_demographics.hd_dep_count = 6 or 33 | household_demographics.hd_vehicle_count > 2) 34 | and date_dim.d_dow = 1 35 | and date_dim.d_year in (1999,1999+1,1999+2) 36 | and store.s_number_employees between 200 and 295 37 | group by ss_ticket_number,ss_customer_sk,ss_addr_sk,store.s_city) ms,customer 38 | where ss_customer_sk = c_customer_sk 39 | order by c_last_name,c_first_name,substr(s_city,1,30), profit 40 | limit 100 41 | 42 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q97.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q97.sql-- 19 | 20 | with ssci as ( 21 | select ss_customer_sk customer_sk, ss_item_sk item_sk 22 | from store_sales,date_dim 23 | where ss_sold_date_sk = d_date_sk 24 | and d_month_seq between 1200 and 1200 + 11 25 | group by ss_customer_sk, ss_item_sk), 26 | csci as( 27 | select cs_bill_customer_sk customer_sk, cs_item_sk item_sk 28 | from catalog_sales,date_dim 29 | where cs_sold_date_sk = d_date_sk 30 | and d_month_seq between 1200 and 1200 + 11 31 | group by cs_bill_customer_sk, cs_item_sk) 32 | select sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only 33 | ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only 34 | ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog 35 | from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk 36 | and ssci.item_sk = csci.item_sk) 37 | limit 100 38 | 39 | -------------------------------------------------------------------------------- /riffle-server/src/runtime/metrics.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | use crate::metric::{GAUGE_RUNTIME_ALIVE_THREAD_NUM, GAUGE_RUNTIME_IDLE_THREAD_NUM}; 19 | use prometheus::IntGauge; 20 | 21 | #[derive(Debug)] 22 | pub struct Metrics { 23 | pub thread_alive_gauge: IntGauge, 24 | pub thread_idle_gauge: IntGauge, 25 | } 26 | 27 | impl Metrics { 28 | pub fn new(name: &str) -> Self { 29 | Self { 30 | thread_alive_gauge: GAUGE_RUNTIME_ALIVE_THREAD_NUM.with_label_values(&[name]), 31 | thread_idle_gauge: GAUGE_RUNTIME_IDLE_THREAD_NUM.with_label_values(&[name]), 32 | } 33 | } 34 | 35 | #[inline] 36 | pub fn on_thread_start(&self) { 37 | self.thread_alive_gauge.inc(); 38 | } 39 | 40 | #[inline] 41 | pub fn on_thread_stop(&self) { 42 | self.thread_alive_gauge.dec(); 43 | } 44 | 45 | #[inline] 46 | pub fn on_thread_park(&self) { 47 | self.thread_idle_gauge.inc(); 48 | } 49 | 50 | #[inline] 51 | pub fn on_thread_unpark(&self) { 52 | self.thread_idle_gauge.dec(); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q16.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q16.sql-- 19 | 20 | select 21 | count(distinct cs_order_number) as `order count`, 22 | sum(cs_ext_ship_cost) as `total shipping cost`, 23 | sum(cs_net_profit) as `total net profit` 24 | from 25 | catalog_sales cs1, date_dim, customer_address, call_center 26 | where 27 | d_date between cast ('2002-02-01' as date) and (cast('2002-02-01' as date) + interval '60' day) 28 | and cs1.cs_ship_date_sk = d_date_sk 29 | and cs1.cs_ship_addr_sk = ca_address_sk 30 | and ca_state = 'GA' 31 | and cs1.cs_call_center_sk = cc_call_center_sk 32 | and cc_county in ('Williamson County','Williamson County','Williamson County','Williamson County', 'Williamson County') 33 | and exists (select * 34 | from catalog_sales cs2 35 | where cs1.cs_order_number = cs2.cs_order_number 36 | and cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) 37 | and not exists(select * 38 | from catalog_returns cr1 39 | where cs1.cs_order_number = cr1.cr_order_number) 40 | order by count(distinct cs_order_number) 41 | limit 100 42 | 43 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q90.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q90.sql-- 19 | 20 | select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio 21 | from ( select count(*) amc 22 | from web_sales, household_demographics , time_dim, web_page 23 | where ws_sold_time_sk = time_dim.t_time_sk 24 | and ws_ship_hdemo_sk = household_demographics.hd_demo_sk 25 | and ws_web_page_sk = web_page.wp_web_page_sk 26 | and time_dim.t_hour between 8 and 8+1 27 | and household_demographics.hd_dep_count = 6 28 | and web_page.wp_char_count between 5000 and 5200) at cross join 29 | ( select count(*) pmc 30 | from web_sales, household_demographics , time_dim, web_page 31 | where ws_sold_time_sk = time_dim.t_time_sk 32 | and ws_ship_hdemo_sk = household_demographics.hd_demo_sk 33 | and ws_web_page_sk = web_page.wp_web_page_sk 34 | and time_dim.t_hour between 19 and 19+1 35 | and household_demographics.hd_dep_count = 6 36 | and web_page.wp_char_count between 5000 and 5200) pt 37 | order by am_pm_ratio 38 | limit 100 39 | 40 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q43.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q43.sql-- 19 | 20 | select s_store_name, s_store_id, 21 | sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, 22 | sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, 23 | sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, 24 | sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, 25 | sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, 26 | sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, 27 | sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales 28 | from date_dim, store_sales, store 29 | where d_date_sk = ss_sold_date_sk and 30 | s_store_sk = ss_store_sk and 31 | s_gmt_offset = -5 and 32 | d_year = 2000 33 | group by s_store_name, s_store_id 34 | order by s_store_name, s_store_id,sun_sales,mon_sales,tue_sales,wed_sales, 35 | thu_sales,fri_sales,sat_sales 36 | limit 100 37 | 38 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q89.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q89.sql-- 19 | 20 | select * 21 | from( 22 | select i_category, i_class, i_brand, 23 | s_store_name, s_company_name, 24 | d_moy, 25 | sum(ss_sales_price) sum_sales, 26 | avg(sum(ss_sales_price)) over 27 | (partition by i_category, i_brand, s_store_name, s_company_name) 28 | avg_monthly_sales 29 | from item, store_sales, date_dim, store 30 | where ss_item_sk = i_item_sk and 31 | ss_sold_date_sk = d_date_sk and 32 | ss_store_sk = s_store_sk and 33 | d_year in (1999) and 34 | ((i_category in ('Books','Electronics','Sports') and 35 | i_class in ('computers','stereo','football')) 36 | or (i_category in ('Men','Jewelry','Women') and 37 | i_class in ('shirts','birdal','dresses'))) 38 | group by i_category, i_class, i_brand, 39 | s_store_name, s_company_name, d_moy) tmp1 40 | where case when (avg_monthly_sales <> 0) then (abs(sum_sales - avg_monthly_sales) / avg_monthly_sales) else null end > 0.1 41 | order by sum_sales - avg_monthly_sales, s_store_name 42 | limit 100 43 | 44 | -------------------------------------------------------------------------------- /riffle-server/src/histogram/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::metric::REGISTRY; 2 | use log::{error, info}; 3 | use once_cell::sync::Lazy; 4 | use parking_lot::Mutex; 5 | use prometheus::{register_int_gauge_vec, IntGaugeVec}; 6 | 7 | pub struct Histogram { 8 | name: String, 9 | recorder: Mutex>, 10 | gauge: IntGaugeVec, 11 | } 12 | 13 | impl Histogram { 14 | pub fn new(name: &str) -> Histogram { 15 | info!("Registering metrics for {}", name); 16 | let gauge: IntGaugeVec = { register_int_gauge_vec!(name, name, &["quantile"]).unwrap() }; 17 | 18 | REGISTRY.register(Box::new(gauge.clone())).expect(""); 19 | 20 | Self { 21 | name: name.to_owned(), 22 | recorder: Mutex::new(hdrhistogram::Histogram::new(4).unwrap()), 23 | gauge, 24 | } 25 | } 26 | 27 | pub fn record(&self, value: u64) { 28 | let mut recorder = self.recorder.lock(); 29 | if let Err(e) = recorder.record(value) { 30 | error!("failed to record `{}`: {}", self.name, e); 31 | } 32 | } 33 | 34 | fn clear(&self) { 35 | let mut recorder = self.recorder.lock(); 36 | recorder.clear() 37 | } 38 | 39 | pub fn observe(&self) { 40 | let mut recorder = self.recorder.lock(); 41 | let p99 = recorder.value_at_quantile(0.99); 42 | let p95 = recorder.value_at_quantile(0.95); 43 | let p90 = recorder.value_at_quantile(0.90); 44 | let p80 = recorder.value_at_quantile(0.80); 45 | let p50 = recorder.value_at_quantile(0.50); 46 | 47 | self.gauge.with_label_values(&["p99"]).set(p99 as i64); 48 | self.gauge.with_label_values(&["p95"]).set(p95 as i64); 49 | self.gauge.with_label_values(&["p90"]).set(p90 as i64); 50 | self.gauge.with_label_values(&["p80"]).set(p80 as i64); 51 | self.gauge.with_label_values(&["p50"]).set(p50 as i64); 52 | 53 | recorder.clear(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q21.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q21.sql-- 19 | 20 | select * from( 21 | select w_warehouse_name, i_item_id, 22 | sum(case when (cast(d_date as date) < cast ('2000-03-11' as date)) 23 | then inv_quantity_on_hand 24 | else 0 end) as inv_before, 25 | sum(case when (cast(d_date as date) >= cast ('2000-03-11' as date)) 26 | then inv_quantity_on_hand 27 | else 0 end) as inv_after 28 | from inventory, warehouse, item, date_dim 29 | where i_current_price between 0.99 and 1.49 30 | and i_item_sk = inv_item_sk 31 | and inv_warehouse_sk = w_warehouse_sk 32 | and inv_date_sk = d_date_sk 33 | and d_date between (cast('2000-03-11' as date) - interval '30' day) 34 | and (cast('2000-03-11' as date) + interval '30' day) 35 | group by w_warehouse_name, i_item_id) x 36 | where (case when inv_before > 0 37 | then inv_after / inv_before 38 | else null 39 | end) between 2.0/3.0 and 3.0/2.0 40 | order by w_warehouse_name, i_item_id 41 | limit 100 42 | -------------------------------------------------------------------------------- /riffle-server/src/store/spill/storage_flush_handler.rs: -------------------------------------------------------------------------------- 1 | use crate::event_bus::{Event, Subscriber}; 2 | use crate::store::hybrid::HybridStore; 3 | use crate::store::spill::metrics::FlushingMetricsMonitor; 4 | use crate::store::spill::{handle_spill_failure, handle_spill_success, SpillMessage}; 5 | use async_trait::async_trait; 6 | use log::{error, warn}; 7 | use std::sync::Arc; 8 | 9 | #[derive(Clone)] 10 | pub struct StorageFlushHandler { 11 | pub store: Arc, 12 | } 13 | impl StorageFlushHandler { 14 | pub fn new(store: &Arc) -> Self { 15 | Self { 16 | store: store.clone(), 17 | } 18 | } 19 | } 20 | unsafe impl Send for StorageFlushHandler {} 21 | unsafe impl Sync for StorageFlushHandler {} 22 | 23 | #[async_trait] 24 | impl Subscriber for StorageFlushHandler { 25 | type Input = SpillMessage; 26 | 27 | async fn on_event(&self, event: Event) -> bool { 28 | let message = event.get_data(); 29 | let app_id = &message.ctx.uid.app_id; 30 | 31 | let _ = 32 | FlushingMetricsMonitor::new(app_id, message.size, message.get_candidate_storage_type()); 33 | 34 | let result = self.store.flush_storage_for_buffer(message).await; 35 | match result { 36 | Ok(_) => { 37 | handle_spill_success(message, self.store.clone()).await; 38 | } 39 | Err(err) => { 40 | message.inc_retry_counter(); 41 | let could_be_retried = handle_spill_failure(err, message, self.store.clone()).await; 42 | if could_be_retried { 43 | if let Err(e) = &self.store.event_bus.publish(event).await { 44 | error!( 45 | "Errors on resending the event into parent event bus. err: {:#?}", 46 | e 47 | ); 48 | } 49 | } 50 | } 51 | } 52 | true 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q91.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q91.sql-- 19 | 20 | select 21 | cc_call_center_id Call_Center, cc_name Call_Center_Name, cc_manager Manager, 22 | sum(cr_net_loss) Returns_Loss 23 | from 24 | call_center, catalog_returns, date_dim, customer, customer_address, 25 | customer_demographics, household_demographics 26 | where 27 | cr_call_center_sk = cc_call_center_sk 28 | and cr_returned_date_sk = d_date_sk 29 | and cr_returning_customer_sk = c_customer_sk 30 | and cd_demo_sk = c_current_cdemo_sk 31 | and hd_demo_sk = c_current_hdemo_sk 32 | and ca_address_sk = c_current_addr_sk 33 | and d_year = 1998 34 | and d_moy = 11 35 | and ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') 36 | or(cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) 37 | and hd_buy_potential like 'Unknown%' 38 | and ca_gmt_offset = -7 39 | group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status 40 | order by sum(cr_net_loss) desc 41 | 42 | -------------------------------------------------------------------------------- /riffle-server/src/mem_allocator/mod.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | // Allocators 19 | #[cfg(feature = "jemalloc")] 20 | #[path = "jemalloc.rs"] 21 | mod imp; 22 | 23 | #[cfg(not(any(feature = "jemalloc", feature = "mimalloc")))] 24 | #[path = "system_std.rs"] 25 | mod imp; 26 | 27 | #[cfg(all(unix, feature = "allocator-analysis"))] 28 | use cap::Cap; 29 | 30 | #[global_allocator] 31 | #[cfg(all(unix, feature = "allocator-analysis"))] 32 | pub static ALLOCATOR: Cap = Cap::new(imp::allocator(), usize::max_value()); 33 | #[cfg(not(all(unix, feature = "allocator-analysis")))] 34 | pub static ALLOCATOR: imp::Allocator = imp::allocator(); 35 | 36 | pub mod error; 37 | pub type AllocStats = Vec<(&'static str, usize)>; 38 | 39 | // when memory-prof feature is enabled, provide empty profiling functions 40 | #[cfg(not(all(unix, feature = "memory-prof")))] 41 | mod default; 42 | 43 | #[cfg(not(all(unix, feature = "memory-prof")))] 44 | pub use default::*; 45 | 46 | // when memory-prof feature is enabled, provide jemalloc profiling functions 47 | #[cfg(all(unix, feature = "memory-prof"))] 48 | mod profiling; 49 | 50 | #[cfg(all(unix, feature = "memory-prof"))] 51 | pub use profiling::*; 52 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q25.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q25.sql-- 19 | 20 | select i_item_id, i_item_desc, s_store_id, s_store_name, 21 | sum(ss_net_profit) as store_sales_profit, 22 | sum(sr_net_loss) as store_returns_loss, 23 | sum(cs_net_profit) as catalog_sales_profit 24 | from 25 | store_sales, store_returns, catalog_sales, date_dim d1, date_dim d2, date_dim d3, 26 | store, item 27 | where 28 | d1.d_moy = 4 29 | and d1.d_year = 2001 30 | and d1.d_date_sk = ss_sold_date_sk 31 | and i_item_sk = ss_item_sk 32 | and s_store_sk = ss_store_sk 33 | and ss_customer_sk = sr_customer_sk 34 | and ss_item_sk = sr_item_sk 35 | and ss_ticket_number = sr_ticket_number 36 | and sr_returned_date_sk = d2.d_date_sk 37 | and d2.d_moy between 4 and 10 38 | and d2.d_year = 2001 39 | and sr_customer_sk = cs_bill_customer_sk 40 | and sr_item_sk = cs_item_sk 41 | and cs_sold_date_sk = d3.d_date_sk 42 | and d3.d_moy between 4 and 10 43 | and d3.d_year = 2001 44 | group by 45 | i_item_id, i_item_desc, s_store_id, s_store_name 46 | order by 47 | i_item_id, i_item_desc, s_store_id, s_store_name 48 | limit 100 49 | 50 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q39a.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q39a.sql-- 19 | 20 | with inv as 21 | (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy 22 | ,stdev,mean, case mean when 0 then null else stdev/mean end cov 23 | from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy 24 | ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean 25 | from inventory, item, warehouse, date_dim 26 | where inv_item_sk = i_item_sk 27 | and inv_warehouse_sk = w_warehouse_sk 28 | and inv_date_sk = d_date_sk 29 | and d_year = 2001 30 | group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo 31 | where case mean when 0 then 0 else stdev/mean end > 1) 32 | select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov 33 | ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov 34 | from inv inv1,inv inv2 35 | where inv1.i_item_sk = inv2.i_item_sk 36 | and inv1.w_warehouse_sk = inv2.w_warehouse_sk 37 | and inv1.d_moy=1 38 | and inv2.d_moy=1+1 39 | order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov 40 | ,inv2.d_moy,inv2.mean, inv2.cov 41 | 42 | -------------------------------------------------------------------------------- /riffle-server/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #![allow(dead_code, unused)] 19 | #![feature(impl_trait_in_assoc_type)] 20 | extern crate core; 21 | pub mod app_manager; 22 | pub mod await_tree; 23 | pub mod client_configs; 24 | pub mod common; 25 | mod composed_bytes; 26 | pub mod config; 27 | pub mod constant; 28 | pub mod error; 29 | pub mod grpc; 30 | mod heartbeat; 31 | pub mod http; 32 | pub mod log_service; 33 | pub mod mem_allocator; 34 | pub mod metric; 35 | pub mod rpc; 36 | pub mod runtime; 37 | pub mod signal; 38 | pub mod store; 39 | pub mod tracing; 40 | pub mod urpc; 41 | pub mod util; 42 | 43 | pub mod event_bus; 44 | mod health_service; 45 | mod kerberos; 46 | pub mod semaphore_with_index; 47 | pub mod storage; 48 | 49 | pub mod bits; 50 | pub mod block_id_manager; 51 | pub mod histogram; 52 | pub mod id_layout; 53 | pub mod lazy_initializer; 54 | 55 | mod config_reconfigure; 56 | #[cfg(feature = "deadlock-detection")] 57 | pub mod deadlock; 58 | pub mod historical_apps; 59 | pub mod panic_hook; 60 | pub mod server_state_manager; 61 | 62 | pub mod config_ref; 63 | mod ddashmap; 64 | pub mod mini_riffle; 65 | mod partition_stats; 66 | mod raw_io; 67 | mod system_libc; 68 | 69 | pub mod raw_pipe; 70 | -------------------------------------------------------------------------------- /riffle-server/src/composed_bytes.rs: -------------------------------------------------------------------------------- 1 | use bytes::{Bytes, BytesMut}; 2 | 3 | /// To compose multi Bytes into one for zero copy. 4 | #[derive(Clone, Debug)] 5 | pub struct ComposedBytes { 6 | composed: Vec, 7 | total_len: usize, 8 | } 9 | 10 | impl ComposedBytes { 11 | pub fn new() -> ComposedBytes { 12 | Self { 13 | composed: vec![], 14 | total_len: 0, 15 | } 16 | } 17 | 18 | pub fn from(all: Vec, total_size: usize) -> ComposedBytes { 19 | Self { 20 | composed: all, 21 | total_len: total_size, 22 | } 23 | } 24 | 25 | pub fn put(&mut self, bytes: Bytes) { 26 | self.total_len += bytes.len(); 27 | self.composed.push(bytes); 28 | } 29 | 30 | /// this is expensive to consume like the Bytes 31 | pub fn freeze(&self) -> Bytes { 32 | let mut bytes_mut = BytesMut::with_capacity(self.total_len); 33 | for x in self.composed.iter() { 34 | bytes_mut.extend_from_slice(x); 35 | } 36 | bytes_mut.freeze() 37 | } 38 | 39 | pub fn iter(&self) -> impl Iterator + '_ { 40 | self.composed.iter() 41 | } 42 | 43 | pub fn to_vec(self) -> Vec { 44 | self.composed 45 | } 46 | 47 | pub fn len(&self) -> usize { 48 | self.total_len 49 | } 50 | } 51 | 52 | #[cfg(test)] 53 | mod test { 54 | use crate::composed_bytes::ComposedBytes; 55 | use bytes::Bytes; 56 | 57 | #[test] 58 | fn test_bytes() { 59 | let mut composed = ComposedBytes::new(); 60 | composed.put(Bytes::copy_from_slice(b"hello")); 61 | composed.put(Bytes::copy_from_slice(b"world")); 62 | assert_eq!(10, composed.len()); 63 | 64 | let mut iter = composed.iter(); 65 | assert_eq!(b"hello", iter.next().unwrap().as_ref()); 66 | assert_eq!(b"world", iter.next().unwrap().as_ref()); 67 | 68 | let data = composed.freeze(); 69 | assert_eq!(b"helloworld", data.as_ref()); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q70.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q70.sql-- 19 | 20 | select 21 | sum(ss_net_profit) as total_sum, s_state, s_county 22 | ,grouping(s_state)+grouping(s_county) as lochierarchy 23 | ,rank() over ( 24 | partition by grouping(s_state)+grouping(s_county), 25 | case when grouping(s_county) = 0 then s_state end 26 | order by sum(ss_net_profit) desc) as rank_within_parent 27 | from 28 | store_sales, date_dim d1, store 29 | where 30 | d1.d_month_seq between 1200 and 1200+11 31 | and d1.d_date_sk = ss_sold_date_sk 32 | and s_store_sk = ss_store_sk 33 | and s_state in 34 | (select s_state from 35 | (select s_state as s_state, 36 | rank() over ( partition by s_state order by sum(ss_net_profit) desc) as ranking 37 | from store_sales, store, date_dim 38 | where d_month_seq between 1200 and 1200+11 39 | and d_date_sk = ss_sold_date_sk 40 | and s_store_sk = ss_store_sk 41 | group by s_state) tmp1 42 | where ranking <= 5) 43 | group by rollup(s_state,s_county) 44 | order by 45 | lochierarchy desc 46 | ,case when lochierarchy = 0 then s_state end 47 | ,rank_within_parent 48 | limit 100 49 | 50 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q39b.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q39b.sql-- 19 | 20 | with inv as 21 | (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy 22 | ,stdev,mean, case mean when 0 then null else stdev/mean end cov 23 | from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy 24 | ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean 25 | from inventory, item, warehouse, date_dim 26 | where inv_item_sk = i_item_sk 27 | and inv_warehouse_sk = w_warehouse_sk 28 | and inv_date_sk = d_date_sk 29 | and d_year = 2001 30 | group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo 31 | where case mean when 0 then 0 else stdev/mean end > 1) 32 | select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov 33 | ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov 34 | from inv inv1,inv inv2 35 | where inv1.i_item_sk = inv2.i_item_sk 36 | and inv1.w_warehouse_sk = inv2.w_warehouse_sk 37 | and inv1.d_moy=1 38 | and inv2.d_moy=1+1 39 | and inv1.cov > 1.5 40 | order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov 41 | ,inv2.d_moy,inv2.mean, inv2.cov 42 | 43 | -------------------------------------------------------------------------------- /riffle-ctl/src/actions/hdfs_append.rs: -------------------------------------------------------------------------------- 1 | use crate::actions::Action; 2 | use bytes::Bytes; 3 | use log::info; 4 | use riffle_server::store::hadoop::get_hdfs_client; 5 | use riffle_server::store::DataBytes; 6 | use riffle_server::util; 7 | use std::path::Path; 8 | use url::Url; 9 | 10 | pub struct HdfsAppendAction { 11 | root: String, 12 | file_name: String, 13 | total_size: u64, 14 | batch_size: u64, 15 | } 16 | 17 | impl HdfsAppendAction { 18 | pub fn new(absolute_path: &str, total_size: &str, batch_size: &str) -> Self { 19 | let total_size = util::to_bytes(total_size); 20 | let batch_size = util::to_bytes(batch_size); 21 | 22 | let path = Path::new(absolute_path); 23 | let file_name = path.file_name().unwrap().to_str().unwrap().to_string(); 24 | let root = path.parent().unwrap().to_str().unwrap().to_string(); 25 | 26 | info!( 27 | "root: {}. file_name: {}. total_size: {}. batch_size: {}", 28 | &root, &file_name, total_size, batch_size 29 | ); 30 | HdfsAppendAction { 31 | root, 32 | file_name, 33 | total_size, 34 | batch_size, 35 | } 36 | } 37 | } 38 | 39 | #[async_trait::async_trait] 40 | impl Action for HdfsAppendAction { 41 | async fn act(&self) -> anyhow::Result<()> { 42 | let client = get_hdfs_client(&self.root, Default::default())?; 43 | 44 | info!("Creating file: {}", &self.file_name); 45 | client.touch(&self.file_name).await?; 46 | 47 | let loop_cnt = self.total_size / self.batch_size; 48 | info!("Will append {} loop cnt", loop_cnt); 49 | 50 | let test_data = vec![0u8; self.batch_size as usize]; 51 | let test_data = Bytes::from(test_data); 52 | 53 | for idx in 0..loop_cnt { 54 | info!("Appending with index:{}", idx); 55 | client 56 | .append(&self.file_name, test_data.clone().into()) 57 | .await?; 58 | } 59 | 60 | info!("Finished."); 61 | Ok(()) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q73.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q73.sql-- 19 | 20 | select 21 | c_last_name, c_first_name, c_salutation, c_preferred_cust_flag, 22 | ss_ticket_number, cnt from 23 | (select ss_ticket_number, ss_customer_sk, count(*) cnt 24 | from store_sales,date_dim,store,household_demographics 25 | where store_sales.ss_sold_date_sk = date_dim.d_date_sk 26 | and store_sales.ss_store_sk = store.s_store_sk 27 | and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk 28 | and date_dim.d_dom between 1 and 2 29 | and (household_demographics.hd_buy_potential = '>10000' or 30 | household_demographics.hd_buy_potential = 'unknown') 31 | and household_demographics.hd_vehicle_count > 0 32 | and case when household_demographics.hd_vehicle_count > 0 then 33 | household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end > 1 34 | and date_dim.d_year in (1999,1999+1,1999+2) 35 | and store.s_county in ('Williamson County','Franklin Parish','Bronx County','Orange County') 36 | group by ss_ticket_number,ss_customer_sk) dj,customer 37 | where ss_customer_sk = c_customer_sk 38 | and cnt between 1 and 5 39 | order by cnt desc, c_last_name asc 40 | 41 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q95.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q95.sql-- 19 | 20 | with ws_wh as 21 | (select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 22 | from web_sales ws1,web_sales ws2 23 | where ws1.ws_order_number = ws2.ws_order_number 24 | and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) 25 | select 26 | count(distinct ws_order_number) as `order count` 27 | ,sum(ws_ext_ship_cost) as `total shipping cost` 28 | ,sum(ws_net_profit) as `total net profit` 29 | from 30 | web_sales ws1, date_dim, customer_address, web_site 31 | where 32 | d_date between cast ('1999-02-01' as date) and 33 | (cast('1999-02-01' as date) + interval '60' day) 34 | and ws1.ws_ship_date_sk = d_date_sk 35 | and ws1.ws_ship_addr_sk = ca_address_sk 36 | and ca_state = 'IL' 37 | and ws1.ws_web_site_sk = web_site_sk 38 | and web_company_name = 'pri' 39 | and ws1.ws_order_number in (select ws_order_number 40 | from ws_wh) 41 | and ws1.ws_order_number in (select wr_order_number 42 | from web_returns,ws_wh 43 | where wr_order_number = ws_wh.ws_order_number) 44 | order by count(distinct ws_order_number) 45 | limit 100 46 | 47 | -------------------------------------------------------------------------------- /riffle-server/src/grpc/layer/tracing.rs: -------------------------------------------------------------------------------- 1 | use fastrace::collector::SpanContext; 2 | use fastrace::future::FutureExt; 3 | use fastrace::Span; 4 | use hyper::Body; 5 | use std::future::Future; 6 | use std::task::{Context, Poll}; 7 | use tower::layer::util::Identity; 8 | use tower::util::Either; 9 | use tower::{Layer, Service}; 10 | 11 | #[derive(Clone)] 12 | pub struct TracingMiddleWareLayer {} 13 | 14 | #[derive(Clone)] 15 | pub struct TracingMiddleWare { 16 | inner: S, 17 | } 18 | 19 | pub type OptionalAwaitTreeMiddlewareLayer = Either; 20 | 21 | impl TracingMiddleWareLayer { 22 | pub fn new() -> OptionalAwaitTreeMiddlewareLayer { 23 | Either::A(Self {}) 24 | } 25 | } 26 | 27 | impl Layer for TracingMiddleWareLayer { 28 | type Service = TracingMiddleWare; 29 | 30 | fn layer(&self, service: S) -> Self::Service { 31 | TracingMiddleWare { inner: service } 32 | } 33 | } 34 | 35 | impl Service> for TracingMiddleWare 36 | where 37 | S: Service> + Clone + Send + 'static, 38 | S::Future: Send + 'static, 39 | { 40 | type Response = S::Response; 41 | type Error = S::Error; 42 | 43 | type Future = impl Future>; 44 | 45 | fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { 46 | self.inner.poll_ready(cx) 47 | } 48 | 49 | fn call(&mut self, req: hyper::Request) -> Self::Future { 50 | // This is necessary because tonic internally uses `tower::buffer::Buffer`. 51 | // See https://github.com/tower-rs/tower/issues/547#issuecomment-767629149 52 | // for details on why this is necessary 53 | let clone = self.inner.clone(); 54 | let mut inner = std::mem::replace(&mut self.inner, clone); 55 | 56 | let uri: String = req.uri().path().into(); 57 | let parent = SpanContext::random(); 58 | let span = Span::root(uri, parent); 59 | 60 | async move { inner.call(req).await }.in_span(span) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /riffle-server/src/app_manager/partition_meta.rs: -------------------------------------------------------------------------------- 1 | use crate::app_manager::partition_identifier::PartitionUId; 2 | use anyhow::Result; 3 | use futures::AsyncWriteExt; 4 | use log::warn; 5 | use parking_lot::RwLock; 6 | use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; 7 | use std::sync::Arc; 8 | 9 | #[derive(Clone)] 10 | pub struct PartitionMeta { 11 | inner: Arc, 12 | } 13 | 14 | struct Inner { 15 | total_size: AtomicU64, 16 | is_huge_partition: AtomicBool, 17 | is_split: AtomicBool, 18 | 19 | // this option is for the read ahead mechanism in localfile 20 | is_sequential_read: AtomicBool, 21 | } 22 | 23 | impl PartitionMeta { 24 | pub fn new() -> Self { 25 | PartitionMeta { 26 | inner: Arc::new(Inner { 27 | total_size: Default::default(), 28 | is_huge_partition: Default::default(), 29 | is_split: Default::default(), 30 | is_sequential_read: AtomicBool::new(false), 31 | }), 32 | } 33 | } 34 | 35 | pub fn get_size(&self) -> u64 { 36 | self.inner.total_size.load(Ordering::SeqCst) 37 | } 38 | 39 | pub fn inc_size(&self, data_size: u64) -> u64 { 40 | self.inner.total_size.fetch_add(data_size, Ordering::SeqCst) 41 | } 42 | 43 | // about huge partition 44 | pub fn is_huge_partition(&self) -> bool { 45 | self.inner.is_huge_partition.load(Ordering::SeqCst) 46 | } 47 | 48 | pub fn mark_as_huge_partition(&self) { 49 | self.inner.is_huge_partition.store(true, Ordering::SeqCst); 50 | } 51 | 52 | // about partition split tag 53 | pub fn is_split(&self) -> bool { 54 | self.inner.is_split.load(Ordering::SeqCst) 55 | } 56 | 57 | pub fn mark_as_split(&self) { 58 | self.inner.is_split.store(true, Ordering::SeqCst); 59 | } 60 | 61 | pub fn mark_as_sequential_read(&self) { 62 | self.inner.is_sequential_read.store(true, Ordering::SeqCst); 63 | } 64 | 65 | pub fn is_sequential_read(&self) -> bool { 66 | self.inner.is_sequential_read.load(Ordering::SeqCst) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q76.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q76.sql-- 19 | 20 | SELECT 21 | channel, col_name, d_year, d_qoy, i_category, COUNT(*) sales_cnt, 22 | SUM(ext_sales_price) sales_amt 23 | FROM( 24 | SELECT 25 | 'store' as channel, ss_store_sk col_name, d_year, d_qoy, i_category, 26 | ss_ext_sales_price ext_sales_price 27 | FROM store_sales, item, date_dim 28 | WHERE ss_store_sk IS NULL 29 | AND ss_sold_date_sk=d_date_sk 30 | AND ss_item_sk=i_item_sk 31 | UNION ALL 32 | SELECT 33 | 'web' as channel, ws_ship_customer_sk col_name, d_year, d_qoy, i_category, 34 | ws_ext_sales_price ext_sales_price 35 | FROM web_sales, item, date_dim 36 | WHERE ws_ship_customer_sk IS NULL 37 | AND ws_sold_date_sk=d_date_sk 38 | AND ws_item_sk=i_item_sk 39 | UNION ALL 40 | SELECT 41 | 'catalog' as channel, cs_ship_addr_sk col_name, d_year, d_qoy, i_category, 42 | cs_ext_sales_price ext_sales_price 43 | FROM catalog_sales, item, date_dim 44 | WHERE cs_ship_addr_sk IS NULL 45 | AND cs_sold_date_sk=d_date_sk 46 | AND cs_item_sk=i_item_sk) foo 47 | GROUP BY channel, col_name, d_year, d_qoy, i_category 48 | ORDER BY channel, col_name, d_year, d_qoy, i_category 49 | limit 100 50 | 51 | -------------------------------------------------------------------------------- /riffle-server/src/store/hadoop/delegator.rs: -------------------------------------------------------------------------------- 1 | use crate::error::WorkerError; 2 | use crate::store::hadoop::{FileStatus, HdfsClient}; 3 | use crate::store::DataBytes; 4 | use anyhow::Context; 5 | use async_trait::async_trait; 6 | use await_tree::InstrumentAwait; 7 | use std::time::Duration; 8 | use tokio::time::timeout; 9 | 10 | pub struct HdfsClientDelegator { 11 | root: String, 12 | inner: Box, 13 | operation_duration_secs: u64, 14 | } 15 | 16 | impl HdfsClientDelegator { 17 | pub fn new(root: &str, operation_duration_secs: u64, client: Box) -> Self { 18 | Self { 19 | root: root.to_owned(), 20 | inner: client, 21 | operation_duration_secs, 22 | } 23 | } 24 | } 25 | 26 | #[async_trait] 27 | impl HdfsClient for HdfsClientDelegator { 28 | async fn touch(&self, file_path: &str) -> anyhow::Result<()> { 29 | self.inner.touch(file_path).await 30 | } 31 | 32 | async fn append(&self, file_path: &str, data: DataBytes) -> anyhow::Result<(), WorkerError> { 33 | let f = self.inner.append(file_path, data); 34 | timeout(Duration::from_secs(self.operation_duration_secs), f) 35 | .await 36 | .with_context(|| format!("Errors on appending on hdfs: {}", &self.root))??; 37 | Ok(()) 38 | } 39 | 40 | async fn len(&self, file_path: &str) -> anyhow::Result { 41 | self.inner.len(file_path).await 42 | } 43 | 44 | async fn create_dir(&self, dir: &str) -> anyhow::Result<()> { 45 | self.inner.create_dir(dir).await 46 | } 47 | 48 | async fn delete_dir(&self, dir: &str) -> anyhow::Result<(), WorkerError> { 49 | self.inner.delete_dir(dir).await 50 | } 51 | 52 | async fn delete_file(&self, file_path: &str) -> anyhow::Result<(), WorkerError> { 53 | self.inner.delete_file(file_path).await 54 | } 55 | 56 | async fn list_status(&self, dir: &str) -> anyhow::Result, WorkerError> { 57 | self.inner.list_status(dir).await 58 | } 59 | 60 | fn root(&self) -> String { 61 | self.inner.root() 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q62.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q62.sql-- 19 | 20 | select 21 | substr(w_warehouse_name,1,20) 22 | ,sm_type 23 | ,web_name 24 | ,sum(case when (ws_ship_date_sk - ws_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` 25 | ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 30) and 26 | (ws_ship_date_sk - ws_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` 27 | ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 60) and 28 | (ws_ship_date_sk - ws_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` 29 | ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 90) and 30 | (ws_ship_date_sk - ws_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` 31 | ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 120) then 1 else 0 end) as `>120 days` 32 | from 33 | web_sales, warehouse, ship_mode, web_site, date_dim 34 | where 35 | d_month_seq between 1200 and 1200 + 11 36 | and ws_ship_date_sk = d_date_sk 37 | and ws_warehouse_sk = w_warehouse_sk 38 | and ws_ship_mode_sk = sm_ship_mode_sk 39 | and ws_web_site_sk = web_site_sk 40 | group by 41 | substr(w_warehouse_name,1,20), sm_type, web_name 42 | order by 43 | substr(w_warehouse_name,1,20), sm_type, web_name 44 | limit 100 45 | 46 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q99.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q99.sql-- 19 | 20 | select 21 | substr(w_warehouse_name,1,20), sm_type, cc_name 22 | ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` 23 | ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and 24 | (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` 25 | ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and 26 | (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` 27 | ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and 28 | (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` 29 | ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 120) then 1 else 0 end) as `>120 days` 30 | from 31 | catalog_sales, warehouse, ship_mode, call_center, date_dim 32 | where 33 | d_month_seq between 1200 and 1200 + 11 34 | and cs_ship_date_sk = d_date_sk 35 | and cs_warehouse_sk = w_warehouse_sk 36 | and cs_ship_mode_sk = sm_ship_mode_sk 37 | and cs_call_center_sk = cc_call_center_sk 38 | group by 39 | substr(w_warehouse_name,1,20), sm_type, cc_name 40 | order by substr(w_warehouse_name,1,20), sm_type, cc_name 41 | limit 100 42 | 43 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q30.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q30.sql-- 19 | 20 | with customer_total_return as 21 | (select wr_returning_customer_sk as ctr_customer_sk 22 | ,ca_state as ctr_state, 23 | sum(wr_return_amt) as ctr_total_return 24 | from web_returns, date_dim, customer_address 25 | where wr_returned_date_sk = d_date_sk 26 | and d_year = 2002 27 | and wr_returning_addr_sk = ca_address_sk 28 | group by wr_returning_customer_sk,ca_state) 29 | select c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag 30 | ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address 31 | ,c_last_review_date_sk,ctr_total_return 32 | from customer_total_return ctr1, customer_address, customer 33 | where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 34 | from customer_total_return ctr2 35 | where ctr1.ctr_state = ctr2.ctr_state) 36 | and ca_address_sk = c_current_addr_sk 37 | and ca_state = 'GA' 38 | and ctr1.ctr_customer_sk = c_customer_sk 39 | order by c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag 40 | ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address 41 | ,c_last_review_date_sk,ctr_total_return 42 | limit 100 43 | 44 | -------------------------------------------------------------------------------- /riffle-server/src/log_service.rs: -------------------------------------------------------------------------------- 1 | use tracing_appender::non_blocking::WorkerGuard; 2 | use tracing_subscriber::layer::SubscriberExt; 3 | use tracing_subscriber::util::SubscriberInitExt; 4 | use tracing_subscriber::{fmt, EnvFilter, Registry}; 5 | 6 | use crate::config::{LogConfig, RotationConfig}; 7 | 8 | const LOG_FILE_NAME: &str = "riffle-server.log"; 9 | 10 | pub struct LogService; 11 | impl LogService { 12 | pub fn init_for_test() { 13 | let env_filter = 14 | EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("debug")); 15 | let formatting_layer = fmt::layer().pretty().with_writer(std::io::stderr); 16 | 17 | Registry::default() 18 | .with(env_filter) 19 | .with(formatting_layer) 20 | .init(); 21 | } 22 | 23 | pub fn init(log: &LogConfig) -> WorkerGuard { 24 | let file_appender = match log.rotation { 25 | RotationConfig::Hourly => tracing_appender::rolling::hourly(&log.path, LOG_FILE_NAME), 26 | RotationConfig::Daily => tracing_appender::rolling::daily(&log.path, LOG_FILE_NAME), 27 | RotationConfig::Never => tracing_appender::rolling::never(&log.path, LOG_FILE_NAME), 28 | }; 29 | 30 | let env_filter = 31 | EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); 32 | let formatting_layer = fmt::layer().pretty().with_writer(std::io::stderr); 33 | 34 | let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender); 35 | let file_layer = fmt::layer() 36 | .with_ansi(false) 37 | .with_line_number(true) 38 | .with_writer(non_blocking); 39 | 40 | Registry::default() 41 | .with(env_filter) 42 | .with(formatting_layer) 43 | .with(file_layer) 44 | .init(); 45 | 46 | // Note: _guard is a WorkerGuard which is returned by tracing_appender::non_blocking to 47 | // ensure buffered logs are flushed to their output in the case of abrupt terminations of a process. 48 | // See WorkerGuard module for more details. 49 | _guard 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q29.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q29.sql-- 19 | 20 | select 21 | i_item_id 22 | ,i_item_desc 23 | ,s_store_id 24 | ,s_store_name 25 | ,sum(ss_quantity) as store_sales_quantity 26 | ,sum(sr_return_quantity) as store_returns_quantity 27 | ,sum(cs_quantity) as catalog_sales_quantity 28 | from 29 | store_sales, store_returns, catalog_sales, date_dim d1, date_dim d2, 30 | date_dim d3, store, item 31 | where 32 | d1.d_moy = 9 33 | and d1.d_year = 1999 34 | and d1.d_date_sk = ss_sold_date_sk 35 | and i_item_sk = ss_item_sk 36 | and s_store_sk = ss_store_sk 37 | and ss_customer_sk = sr_customer_sk 38 | and ss_item_sk = sr_item_sk 39 | and ss_ticket_number = sr_ticket_number 40 | and sr_returned_date_sk = d2.d_date_sk 41 | and d2.d_moy between 9 and 9 + 3 42 | and d2.d_year = 1999 43 | and sr_customer_sk = cs_bill_customer_sk 44 | and sr_item_sk = cs_item_sk 45 | and cs_sold_date_sk = d3.d_date_sk 46 | and d3.d_year in (1999,1999+1,1999+2) 47 | group by 48 | i_item_id, i_item_desc, s_store_id, s_store_name 49 | order by 50 | i_item_id, i_item_desc, s_store_id, s_store_name 51 | limit 100 52 | 53 | -------------------------------------------------------------------------------- /riffle-server/src/signal.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #[cfg(unix)] 19 | pub mod details { 20 | use log::info; 21 | use signal_hook::{consts::TERM_SIGNALS, iterator::Signals}; 22 | 23 | pub fn wait_for_signal() { 24 | let mut sigs = Signals::new(TERM_SIGNALS).expect("Failed to register signal handlers"); 25 | 26 | for signal in &mut sigs { 27 | if TERM_SIGNALS.contains(&signal) { 28 | info!("Received signal {}, stopping server...", signal); 29 | break; 30 | } 31 | } 32 | } 33 | 34 | pub fn graceful_wait_for_signal(tx: tokio::sync::broadcast::Sender<()>) { 35 | let mut sigs = Signals::new(TERM_SIGNALS).expect("Failed to register signal handlers"); 36 | 37 | for signal in &mut sigs { 38 | if TERM_SIGNALS.contains(&signal) { 39 | info!("Received signal {}, stopping server...", signal); 40 | let _ = tx.send(()); 41 | break; 42 | } 43 | } 44 | } 45 | } 46 | 47 | #[cfg(not(unix))] 48 | pub mod details { 49 | use std::thread; 50 | use std::time::Duration; 51 | 52 | // todo: this should be handled elegantly. 53 | pub fn wait_for_signal() { 54 | while 1 { 55 | thread::sleep(Duration::from_millis(20)); 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q46.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q46.sql-- 19 | 20 | select c_last_name, c_first_name, ca_city, bought_city, ss_ticket_number, amt,profit 21 | from 22 | (select ss_ticket_number 23 | ,ss_customer_sk 24 | ,ca_city bought_city 25 | ,sum(ss_coupon_amt) amt 26 | ,sum(ss_net_profit) profit 27 | from store_sales, date_dim, store, household_demographics, customer_address 28 | where store_sales.ss_sold_date_sk = date_dim.d_date_sk 29 | and store_sales.ss_store_sk = store.s_store_sk 30 | and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk 31 | and store_sales.ss_addr_sk = customer_address.ca_address_sk 32 | and (household_demographics.hd_dep_count = 4 or 33 | household_demographics.hd_vehicle_count= 3) 34 | and date_dim.d_dow in (6,0) 35 | and date_dim.d_year in (1999,1999+1,1999+2) 36 | and store.s_city in ('Fairview','Midway','Fairview','Fairview','Fairview') 37 | group by ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city) dn,customer,customer_address current_addr 38 | where ss_customer_sk = c_customer_sk 39 | and customer.c_current_addr_sk = current_addr.ca_address_sk 40 | and current_addr.ca_city <> bought_city 41 | order by c_last_name, c_first_name, ca_city, bought_city, ss_ticket_number 42 | limit 100 43 | 44 | -------------------------------------------------------------------------------- /riffle-server/src/await_tree.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | use await_tree::{init_global_registry, span, AnyKey, Config, Registry, Span, Tree, TreeRoot}; 19 | use once_cell::sync::Lazy; 20 | use parking_lot::Mutex; 21 | use std::sync::atomic::{AtomicU64, Ordering}; 22 | use std::sync::Arc; 23 | 24 | pub static AWAIT_TREE_REGISTRY: Lazy = Lazy::new(|| AwaitTreeDelegator::new()); 25 | 26 | #[derive(Clone)] 27 | pub struct AwaitTreeDelegator { 28 | registry: Registry, 29 | next_id: Arc, 30 | } 31 | 32 | impl AwaitTreeDelegator { 33 | fn new() -> Self { 34 | init_global_registry(Config::default()); 35 | let registry = Registry::current(); 36 | Self { 37 | registry, 38 | next_id: Arc::new(Default::default()), 39 | } 40 | } 41 | 42 | pub async fn register(&self, msg: impl Into) -> TreeRoot { 43 | let id = self.next_id.fetch_add(1, Ordering::SeqCst); 44 | self.registry.register(id, msg) 45 | } 46 | 47 | pub fn collect_all(&self) -> Vec<(u64, Tree)> { 48 | self.registry 49 | .collect_all() 50 | .iter() 51 | .map(|(key, tree)| { 52 | let key: u64 = *key.downcast_ref().unwrap(); 53 | (key, tree.clone()) 54 | }) 55 | .collect() 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q18.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q18.sql-- 19 | 20 | select i_item_id, 21 | ca_country, 22 | ca_state, 23 | ca_county, 24 | avg( cast(cs_quantity as decimal(12,2))) agg1, 25 | avg( cast(cs_list_price as decimal(12,2))) agg2, 26 | avg( cast(cs_coupon_amt as decimal(12,2))) agg3, 27 | avg( cast(cs_sales_price as decimal(12,2))) agg4, 28 | avg( cast(cs_net_profit as decimal(12,2))) agg5, 29 | avg( cast(c_birth_year as decimal(12,2))) agg6, 30 | avg( cast(cd1.cd_dep_count as decimal(12,2))) agg7 31 | from catalog_sales, customer_demographics cd1, 32 | customer_demographics cd2, customer, customer_address, date_dim, item 33 | where cs_sold_date_sk = d_date_sk and 34 | cs_item_sk = i_item_sk and 35 | cs_bill_cdemo_sk = cd1.cd_demo_sk and 36 | cs_bill_customer_sk = c_customer_sk and 37 | cd1.cd_gender = 'F' and 38 | cd1.cd_education_status = 'Unknown' and 39 | c_current_cdemo_sk = cd2.cd_demo_sk and 40 | c_current_addr_sk = ca_address_sk and 41 | c_birth_month in (1,6,8,9,12,2) and 42 | d_year = 1998 and 43 | ca_state in ('MS','IN','ND','OK','NM','VA','MS') 44 | group by rollup (i_item_id, ca_country, ca_state, ca_county) 45 | order by ca_country, ca_state, ca_county, i_item_id 46 | LIMIT 100 47 | 48 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q72.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q72.sql-- 19 | 20 | select i_item_desc 21 | ,w_warehouse_name 22 | ,d1.d_week_seq 23 | ,sum(case when p_promo_sk is null then 1 else 0 end) no_promo 24 | ,sum(case when p_promo_sk is not null then 1 else 0 end) promo 25 | ,count(*) total_cnt 26 | from catalog_sales 27 | join inventory on (cs_item_sk = inv_item_sk) 28 | join warehouse on (w_warehouse_sk=inv_warehouse_sk) 29 | join item on (i_item_sk = cs_item_sk) 30 | join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) 31 | join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) 32 | join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) 33 | join date_dim d2 on (inv_date_sk = d2.d_date_sk) 34 | join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) 35 | left outer join promotion on (cs_promo_sk=p_promo_sk) 36 | left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) 37 | where d1.d_week_seq = d2.d_week_seq 38 | and inv_quantity_on_hand < cs_quantity 39 | and d3.d_date > (cast(d1.d_date AS DATE) + interval '5' day) 40 | and hd_buy_potential = '>10000' 41 | and d1.d_year = 1999 42 | and cd_marital_status = 'D' 43 | group by i_item_desc,w_warehouse_name,d1.d_week_seq 44 | order by total_cnt desc, i_item_desc, w_warehouse_name, d1.d_week_seq 45 | limit 100 46 | 47 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q81.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q81.sql-- 19 | 20 | with customer_total_return as 21 | (select 22 | cr_returning_customer_sk as ctr_customer_sk, ca_state as ctr_state, 23 | sum(cr_return_amt_inc_tax) as ctr_total_return 24 | from catalog_returns, date_dim, customer_address 25 | where cr_returned_date_sk = d_date_sk 26 | and d_year = 2000 27 | and cr_returning_addr_sk = ca_address_sk 28 | group by cr_returning_customer_sk, ca_state ) 29 | select 30 | c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name, 31 | ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country, 32 | ca_gmt_offset,ca_location_type,ctr_total_return 33 | from customer_total_return ctr1, customer_address, customer 34 | where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 35 | from customer_total_return ctr2 36 | where ctr1.ctr_state = ctr2.ctr_state) 37 | and ca_address_sk = c_current_addr_sk 38 | and ca_state = 'GA' 39 | and ctr1.ctr_customer_sk = c_customer_sk 40 | order by c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name 41 | ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset 42 | ,ca_location_type,ctr_total_return 43 | limit 100 44 | 45 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q53.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q53.sql-- 19 | 20 | select * from 21 | (select i_manufact_id, 22 | sum(ss_sales_price) sum_sales, 23 | avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales 24 | from item, store_sales, date_dim, store 25 | where ss_item_sk = i_item_sk and 26 | ss_sold_date_sk = d_date_sk and 27 | ss_store_sk = s_store_sk and 28 | d_month_seq in (1200,1200+1,1200+2,1200+3,1200+4,1200+5,1200+6, 29 | 1200+7,1200+8,1200+9,1200+10,1200+11) and 30 | ((i_category in ('Books','Children','Electronics') and 31 | i_class in ('personal','portable','reference','self-help') and 32 | i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 33 | 'exportiunivamalg #9','scholaramalgamalg #9')) 34 | or 35 | (i_category in ('Women','Music','Men') and 36 | i_class in ('accessories','classical','fragrances','pants') and 37 | i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 38 | 'importoamalg #1'))) 39 | group by i_manufact_id, d_qoy ) tmp1 40 | where case when avg_quarterly_sales > 0 41 | then abs (sum_sales - avg_quarterly_sales)/ avg_quarterly_sales 42 | else null end > 0.1 43 | order by avg_quarterly_sales, 44 | sum_sales, 45 | i_manufact_id 46 | limit 100 47 | 48 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q61.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q61.sql-- 19 | 20 | select promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100 21 | from 22 | (select sum(ss_ext_sales_price) promotions 23 | from store_sales, store, promotion, date_dim, customer, customer_address, item 24 | where ss_sold_date_sk = d_date_sk 25 | and ss_store_sk = s_store_sk 26 | and ss_promo_sk = p_promo_sk 27 | and ss_customer_sk= c_customer_sk 28 | and ca_address_sk = c_current_addr_sk 29 | and ss_item_sk = i_item_sk 30 | and ca_gmt_offset = -5 31 | and i_category = 'Jewelry' 32 | and (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y') 33 | and s_gmt_offset = -5 34 | and d_year = 1998 35 | and d_moy = 11) promotional_sales cross join 36 | (select sum(ss_ext_sales_price) total 37 | from store_sales, store, date_dim, customer, customer_address, item 38 | where ss_sold_date_sk = d_date_sk 39 | and ss_store_sk = s_store_sk 40 | and ss_customer_sk= c_customer_sk 41 | and ca_address_sk = c_current_addr_sk 42 | and ss_item_sk = i_item_sk 43 | and ca_gmt_offset = -5 44 | and i_category = 'Jewelry' 45 | and s_gmt_offset = -5 46 | and d_year = 1998 47 | and d_moy = 11) all_sales 48 | order by promotions, total 49 | limit 100 50 | 51 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q68.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q68.sql-- 19 | 20 | select 21 | c_last_name, c_first_name, ca_city, bought_city, ss_ticket_number, extended_price, 22 | extended_tax, list_price 23 | from (select 24 | ss_ticket_number, ss_customer_sk, ca_city bought_city, 25 | sum(ss_ext_sales_price) extended_price, 26 | sum(ss_ext_list_price) list_price, 27 | sum(ss_ext_tax) extended_tax 28 | from store_sales, date_dim, store, household_demographics, customer_address 29 | where store_sales.ss_sold_date_sk = date_dim.d_date_sk 30 | and store_sales.ss_store_sk = store.s_store_sk 31 | and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk 32 | and store_sales.ss_addr_sk = customer_address.ca_address_sk 33 | and date_dim.d_dom between 1 and 2 34 | and (household_demographics.hd_dep_count = 4 or 35 | household_demographics.hd_vehicle_count = 3) 36 | and date_dim.d_year in (1999,1999+1,1999+2) 37 | and store.s_city in ('Midway','Fairview') 38 | group by ss_ticket_number, ss_customer_sk, ss_addr_sk,ca_city) dn, 39 | customer, 40 | customer_address current_addr 41 | where ss_customer_sk = c_customer_sk 42 | and customer.c_current_addr_sk = current_addr.ca_address_sk 43 | and current_addr.ca_city <> bought_city 44 | order by c_last_name, ss_ticket_number 45 | limit 100 46 | 47 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q71.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q71.sql-- 19 | 20 | select i_brand_id brand_id, i_brand brand,t_hour,t_minute, 21 | sum(ext_price) ext_price 22 | from item, 23 | (select 24 | ws_ext_sales_price as ext_price, 25 | ws_sold_date_sk as sold_date_sk, 26 | ws_item_sk as sold_item_sk, 27 | ws_sold_time_sk as time_sk 28 | from web_sales, date_dim 29 | where d_date_sk = ws_sold_date_sk 30 | and d_moy=11 31 | and d_year=1999 32 | union all 33 | select 34 | cs_ext_sales_price as ext_price, 35 | cs_sold_date_sk as sold_date_sk, 36 | cs_item_sk as sold_item_sk, 37 | cs_sold_time_sk as time_sk 38 | from catalog_sales, date_dim 39 | where d_date_sk = cs_sold_date_sk 40 | and d_moy=11 41 | and d_year=1999 42 | union all 43 | select 44 | ss_ext_sales_price as ext_price, 45 | ss_sold_date_sk as sold_date_sk, 46 | ss_item_sk as sold_item_sk, 47 | ss_sold_time_sk as time_sk 48 | from store_sales,date_dim 49 | where d_date_sk = ss_sold_date_sk 50 | and d_moy=11 51 | and d_year=1999 52 | ) tmp, time_dim 53 | where 54 | sold_item_sk = i_item_sk 55 | and i_manager_id=1 56 | and time_sk = t_time_sk 57 | and (t_meal_time = 'breakfast' or t_meal_time = 'dinner') 58 | group by i_brand, i_brand_id,t_hour,t_minute 59 | order by ext_price desc, i_brand_id 60 | 61 | -------------------------------------------------------------------------------- /riffle-server/src/panic_hook.rs: -------------------------------------------------------------------------------- 1 | use crate::await_tree::AWAIT_TREE_REGISTRY; 2 | use crate::metric::PANIC_SIGNAL; 3 | use backtrace::Backtrace; 4 | use log::error; 5 | use once_cell::sync::Lazy; 6 | use std::panic; 7 | use std::sync::atomic::AtomicBool; 8 | use std::sync::atomic::Ordering::SeqCst; 9 | 10 | pub static PANIC_TAG: Lazy = Lazy::new(|| AtomicBool::new(false)); 11 | 12 | // Refer from the greptimedb: 13 | // https://github.com/GreptimeTeam/greptimedb/blob/main/src/common/telemetry/src/panic_hook.rs 14 | pub fn set_panic_hook() { 15 | let default_hook = panic::take_hook(); 16 | panic::set_hook(Box::new(move |panic| { 17 | let backtrace = Backtrace::new(); 18 | let backtrace = format!("{backtrace:?}"); 19 | if let Some(location) = panic.location() { 20 | error!( 21 | "[Panic] ============================================================\ 22 | \nmessage: {}\n backtrace: {}\n file: {}. line: {}. column: {}\n\ 23 | ====================================================================", 24 | panic, 25 | backtrace, 26 | location.file(), 27 | location.line(), 28 | location.column() 29 | ); 30 | } else { 31 | error!( 32 | "[Panic] ============================================================\ 33 | \nmessage: {}\n backtrace: {}\n\ 34 | ====================================================================", 35 | panic, backtrace, 36 | ); 37 | } 38 | 39 | PANIC_TAG.store(true, SeqCst); 40 | PANIC_SIGNAL.set(1); 41 | 42 | default_hook(panic); 43 | })); 44 | } 45 | 46 | #[cfg(test)] 47 | mod test { 48 | use crate::panic_hook::{set_panic_hook, PANIC_TAG}; 49 | use std::sync::atomic::Ordering::SeqCst; 50 | use std::thread::sleep; 51 | use std::time::Duration; 52 | 53 | #[test] 54 | fn test_thread_internal_panic() { 55 | set_panic_hook(); 56 | 57 | std::thread::spawn(|| { 58 | sleep(Duration::from_millis(100)); 59 | panic!(); 60 | }); 61 | 62 | awaitility::at_most(Duration::from_secs(1)).until(|| PANIC_TAG.load(SeqCst)); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q34.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q34.sql-- 19 | 20 | select c_last_name, c_first_name, c_salutation, c_preferred_cust_flag, ss_ticket_number, 21 | cnt 22 | FROM 23 | (select ss_ticket_number, ss_customer_sk, count(*) cnt 24 | from store_sales,date_dim,store,household_demographics 25 | where store_sales.ss_sold_date_sk = date_dim.d_date_sk 26 | and store_sales.ss_store_sk = store.s_store_sk 27 | and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk 28 | and (date_dim.d_dom between 1 and 3 or date_dim.d_dom between 25 and 28) 29 | and (household_demographics.hd_buy_potential = '>10000' or 30 | household_demographics.hd_buy_potential = 'unknown') 31 | and household_demographics.hd_vehicle_count > 0 32 | and (case when household_demographics.hd_vehicle_count > 0 33 | then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count 34 | else null 35 | end) > 1.2 36 | and date_dim.d_year in (1999, 1999+1, 1999+2) 37 | and store.s_county in ('Williamson County','Williamson County','Williamson County','Williamson County', 38 | 'Williamson County','Williamson County','Williamson County','Williamson County') 39 | group by ss_ticket_number,ss_customer_sk) dn,customer 40 | where ss_customer_sk = c_customer_sk 41 | and cnt between 15 and 20 42 | order by c_last_name,c_first_name,c_salutation,c_preferred_cust_flag desc, ss_ticket_number 43 | 44 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q63.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q63.sql-- 19 | 20 | select * 21 | from (select i_manager_id 22 | ,sum(ss_sales_price) sum_sales 23 | ,avg(sum(ss_sales_price)) over (partition by i_manager_id) avg_monthly_sales 24 | from item 25 | ,store_sales 26 | ,date_dim 27 | ,store 28 | where ss_item_sk = i_item_sk 29 | and ss_sold_date_sk = d_date_sk 30 | and ss_store_sk = s_store_sk 31 | and d_month_seq in (1200,1200+1,1200+2,1200+3,1200+4,1200+5,1200+6,1200+7, 32 | 1200+8,1200+9,1200+10,1200+11) 33 | and (( i_category in ('Books','Children','Electronics') 34 | and i_class in ('personal','portable','reference','self-help') 35 | and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 36 | 'exportiunivamalg #9','scholaramalgamalg #9')) 37 | or( i_category in ('Women','Music','Men') 38 | and i_class in ('accessories','classical','fragrances','pants') 39 | and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 40 | 'importoamalg #1'))) 41 | group by i_manager_id, d_moy) tmp1 42 | where case when avg_monthly_sales > 0 then abs (sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 43 | order by i_manager_id 44 | ,avg_monthly_sales 45 | ,sum_sales 46 | limit 100 47 | 48 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q69.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q69.sql-- 19 | 20 | select 21 | cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, 22 | cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3 23 | from 24 | customer c,customer_address ca,customer_demographics 25 | where 26 | c.c_current_addr_sk = ca.ca_address_sk and 27 | ca_state in ('KY', 'GA', 'NM') and 28 | cd_demo_sk = c.c_current_cdemo_sk and 29 | exists (select * from store_sales, date_dim 30 | where c.c_customer_sk = ss_customer_sk and 31 | ss_sold_date_sk = d_date_sk and 32 | d_year = 2001 and 33 | d_moy between 4 and 4+2) and 34 | (not exists (select * from web_sales, date_dim 35 | where c.c_customer_sk = ws_bill_customer_sk and 36 | ws_sold_date_sk = d_date_sk and 37 | d_year = 2001 and 38 | d_moy between 4 and 4+2) and 39 | not exists (select * from catalog_sales, date_dim 40 | where c.c_customer_sk = cs_ship_customer_sk and 41 | cs_sold_date_sk = d_date_sk and 42 | d_year = 2001 and 43 | d_moy between 4 and 4+2)) 44 | group by cd_gender, cd_marital_status, cd_education_status, 45 | cd_purchase_estimate, cd_credit_rating 46 | order by cd_gender, cd_marital_status, cd_education_status, 47 | cd_purchase_estimate, cd_credit_rating 48 | limit 100 49 | 50 | -------------------------------------------------------------------------------- /riffle-server/src/deadlock.rs: -------------------------------------------------------------------------------- 1 | use crate::metric::DEADLOCK_SIGNAL; 2 | use log::{error, info, warn}; 3 | use once_cell::sync::Lazy; 4 | use std::sync::atomic::AtomicBool; 5 | use std::sync::atomic::Ordering::SeqCst; 6 | use std::time::Duration; 7 | 8 | pub static DEADLOCK_TAG: Lazy = Lazy::new(|| AtomicBool::new(false)); 9 | 10 | pub fn detect_deadlock() { 11 | let _ = std::thread::spawn(move || loop { 12 | std::thread::sleep(Duration::from_secs(2)); 13 | let deadlocks = parking_lot::deadlock::check_deadlock(); 14 | if deadlocks.is_empty() { 15 | continue; 16 | } 17 | 18 | error!("{} deadlocks detected", deadlocks.len()); 19 | for (i, threads) in deadlocks.iter().enumerate() { 20 | error!("Deadlock #{}", i); 21 | for t in threads { 22 | error!("Thread Id {:#?}", t.thread_id()); 23 | error!("{:#?}", t.backtrace()); 24 | } 25 | } 26 | DEADLOCK_SIGNAL.set(1); 27 | DEADLOCK_TAG.store(true, SeqCst); 28 | }); 29 | } 30 | 31 | #[cfg(test)] 32 | mod tests { 33 | use crate::deadlock::{detect_deadlock, DEADLOCK_TAG}; 34 | use parking_lot::Mutex; 35 | use std::sync::atomic::Ordering; 36 | use std::sync::atomic::Ordering::SeqCst; 37 | use std::sync::Arc; 38 | use std::thread; 39 | use std::time::Duration; 40 | 41 | #[test] 42 | fn test_detect_deadlock() { 43 | let mutex1 = Arc::new(Mutex::new(0)); 44 | let mutex2 = Arc::new(Mutex::new(0)); 45 | 46 | let m1_clone = Arc::clone(&mutex1); 47 | let m2_clone = Arc::clone(&mutex2); 48 | 49 | let handle1 = thread::spawn(move || { 50 | let _lock1 = mutex1.lock(); 51 | thread::sleep(Duration::from_millis(100)); // Sleep to ensure the deadlock 52 | let _lock2 = mutex2.lock(); 53 | }); 54 | 55 | let handle2 = thread::spawn(move || { 56 | let _lock2 = m2_clone.lock(); 57 | thread::sleep(Duration::from_millis(100)); // Sleep to ensure the deadlock 58 | let _lock1 = m1_clone.lock(); 59 | }); 60 | 61 | // Start the deadlock detection thread 62 | detect_deadlock(); 63 | 64 | awaitility::at_most(Duration::from_secs(5)).until(|| DEADLOCK_TAG.load(SeqCst)); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /riffle-server/tests/graceful_shutdown.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #[cfg(test)] 19 | mod test { 20 | use log::info; 21 | use riffle_server::config::Config; 22 | use riffle_server::mini_riffle; 23 | use riffle_server::mini_riffle::shuffle_testing; 24 | use signal_hook::consts::SIGTERM; 25 | use signal_hook::low_level::raise; 26 | use std::time::Duration; 27 | 28 | fn init_logger() { 29 | let _ = env_logger::builder().is_test(true).try_init(); 30 | } 31 | 32 | #[tokio::test] 33 | async fn graceful_shutdown_test() -> anyhow::Result<()> { 34 | init_logger(); 35 | let temp_dir = tempdir::TempDir::new("test_write_read").unwrap(); 36 | let temp_path = temp_dir.path().to_str().unwrap().to_string(); 37 | info!("temp file path: {} created", &temp_path); 38 | 39 | let grpc_port = 21101; 40 | let config = Config::create_mem_localfile_config(grpc_port, "1G".to_string(), temp_path); 41 | let app_ref = mini_riffle::start(&config).await?; 42 | tokio::time::sleep(Duration::from_secs(1)).await; 43 | 44 | let jh = tokio::spawn(async move { shuffle_testing(&config, app_ref).await }); 45 | 46 | // raise shutdown signal 47 | tokio::spawn(async { 48 | raise(SIGTERM).expect("failed to raise shutdown signal"); 49 | eprintln!("successfully raised shutdown signal"); 50 | }); 51 | 52 | let _ = jh.await.expect("Task panicked or failed."); 53 | 54 | Ok(()) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /riffle-server/src/store/alignment/allocator.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 foyer Project Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use allocator_api2::alloc::{AllocError, Allocator, Global}; 16 | 17 | #[derive(Debug, Clone, Copy)] 18 | pub struct AlignedAllocator; 19 | 20 | impl Default for AlignedAllocator { 21 | fn default() -> Self { 22 | Self::new() 23 | } 24 | } 25 | 26 | impl AlignedAllocator { 27 | pub const fn new() -> Self { 28 | assert!(N.is_power_of_two()); 29 | Self 30 | } 31 | } 32 | 33 | unsafe impl Allocator for AlignedAllocator { 34 | fn allocate(&self, layout: std::alloc::Layout) -> Result, AllocError> { 35 | Global.allocate(layout.align_to(N).unwrap()) 36 | } 37 | 38 | unsafe fn deallocate(&self, ptr: std::ptr::NonNull, layout: std::alloc::Layout) { 39 | Global.deallocate(ptr, layout.align_to(N).unwrap()) 40 | } 41 | } 42 | 43 | #[cfg(test)] 44 | mod tests { 45 | use super::*; 46 | use crate::bits; 47 | use allocator_api2::vec::Vec as VecA; 48 | 49 | #[test] 50 | fn test_aligned_buffer() { 51 | const ALIGN: usize = 512; 52 | let allocator = AlignedAllocator::::new(); 53 | 54 | let mut buf: VecA = VecA::with_capacity_in(ALIGN * 8, &allocator); 55 | bits::assert_aligned(ALIGN, buf.as_ptr() as _); 56 | 57 | buf.extend_from_slice(&[b'x'; ALIGN * 8]); 58 | bits::assert_aligned(ALIGN, buf.as_ptr() as _); 59 | assert_eq!(buf, [b'x'; ALIGN * 8]); 60 | 61 | buf.extend_from_slice(&[b'x'; ALIGN * 8]); 62 | bits::assert_aligned(ALIGN, buf.as_ptr() as _); 63 | assert_eq!(buf, [b'x'; ALIGN * 16]) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q48.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q48.sql-- 19 | 20 | select sum (ss_quantity) 21 | from store_sales, store, customer_demographics, customer_address, date_dim 22 | where s_store_sk = ss_store_sk 23 | and ss_sold_date_sk = d_date_sk and d_year = 2000 24 | and 25 | ( 26 | ( 27 | cd_demo_sk = ss_cdemo_sk 28 | and 29 | cd_marital_status = 'M' 30 | and 31 | cd_education_status = '4 yr Degree' 32 | and 33 | ss_sales_price between 100.00 and 150.00 34 | ) 35 | or 36 | ( 37 | cd_demo_sk = ss_cdemo_sk 38 | and 39 | cd_marital_status = 'D' 40 | and 41 | cd_education_status = '2 yr Degree' 42 | and 43 | ss_sales_price between 50.00 and 100.00 44 | ) 45 | or 46 | ( 47 | cd_demo_sk = ss_cdemo_sk 48 | and 49 | cd_marital_status = 'S' 50 | and 51 | cd_education_status = 'College' 52 | and 53 | ss_sales_price between 150.00 and 200.00 54 | ) 55 | ) 56 | and 57 | ( 58 | ( 59 | ss_addr_sk = ca_address_sk 60 | and 61 | ca_country = 'United States' 62 | and 63 | ca_state in ('CO', 'OH', 'TX') 64 | and ss_net_profit between 0 and 2000 65 | ) 66 | or 67 | (ss_addr_sk = ca_address_sk 68 | and 69 | ca_country = 'United States' 70 | and 71 | ca_state in ('OR', 'MN', 'KY') 72 | and ss_net_profit between 150 and 3000 73 | ) 74 | or 75 | (ss_addr_sk = ca_address_sk 76 | and 77 | ca_country = 'United States' 78 | and 79 | ca_state in ('VA', 'CA', 'MS') 80 | and ss_net_profit between 50 and 25000 81 | ) 82 | ) 83 | 84 | -------------------------------------------------------------------------------- /riffle-server/src/store/hadoop/mod.rs: -------------------------------------------------------------------------------- 1 | mod delegator; 2 | #[cfg(feature = "hdfs")] 3 | mod hdfs_native; 4 | #[cfg(feature = "hdrs")] 5 | mod hdrs; 6 | 7 | #[cfg(feature = "hdfs")] 8 | use crate::store::hadoop::hdfs_native::HdfsNativeClient; 9 | #[cfg(feature = "hdrs")] 10 | use crate::store::hadoop::hdrs::HdrsClient; 11 | 12 | use crate::error::WorkerError; 13 | use crate::store::hadoop::delegator::HdfsClientDelegator; 14 | use crate::store::DataBytes; 15 | use anyhow::Result; 16 | use async_trait::async_trait; 17 | use std::collections::HashMap; 18 | use std::path::PathBuf; 19 | 20 | #[async_trait] 21 | pub trait HdfsClient: Send + Sync { 22 | async fn touch(&self, file_path: &str) -> Result<()>; 23 | async fn append(&self, file_path: &str, data: DataBytes) -> Result<(), WorkerError>; 24 | async fn len(&self, file_path: &str) -> Result; 25 | 26 | async fn create_dir(&self, dir: &str) -> Result<()>; 27 | async fn delete_dir(&self, dir: &str) -> Result<(), WorkerError>; 28 | 29 | async fn delete_file(&self, file_path: &str) -> Result<(), WorkerError>; 30 | 31 | async fn list_status(&self, dir: &str) -> Result, WorkerError>; 32 | 33 | fn root(&self) -> String; 34 | 35 | fn without_root(&self, path: &str) -> Result { 36 | let root = self.root(); 37 | let root = root.as_str(); 38 | let path = if path.starts_with(root) { 39 | path.strip_prefix(root).unwrap() 40 | } else { 41 | path 42 | }; 43 | let path = if path.starts_with("/") { 44 | path.strip_prefix("/").unwrap() 45 | } else { 46 | path 47 | }; 48 | Ok(path.to_string()) 49 | } 50 | 51 | fn with_root(&self, path: &str) -> Result { 52 | Ok(format!("{}/{}", &self.root(), path)) 53 | } 54 | } 55 | 56 | #[cfg(feature = "hdfs")] 57 | pub fn get_hdfs_client( 58 | root: &str, 59 | configs: HashMap, 60 | ) -> Result> { 61 | #[cfg(not(feature = "hdrs"))] 62 | let client = Box::new(HdfsNativeClient::new(root.to_owned(), configs)?); 63 | 64 | #[cfg(feature = "hdrs")] 65 | let client = Box::new(HdrsClient::new(root.to_owned(), configs)?); 66 | 67 | const DURATION: u64 = 10 * 60; 68 | 69 | let client = Box::new(HdfsClientDelegator::new(root, DURATION, client)); 70 | Ok(client) 71 | } 72 | 73 | pub struct FileStatus { 74 | pub path: String, 75 | pub is_dir: bool, 76 | } 77 | -------------------------------------------------------------------------------- /riffle-server/tests/write_read.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #[cfg(test)] 19 | mod tests { 20 | use anyhow::Result; 21 | use log::info; 22 | use riffle_server::config::Config; 23 | use riffle_server::metric::GAUGE_MEMORY_ALLOCATED; 24 | use riffle_server::mini_riffle; 25 | use riffle_server::mini_riffle::shuffle_testing; 26 | use std::time::Duration; 27 | 28 | fn init_logger() { 29 | let _ = env_logger::builder().is_test(true).try_init(); 30 | } 31 | 32 | #[tokio::test(flavor = "multi_thread", worker_threads = 4)] 33 | async fn shuffle_write_read_testing() -> Result<()> { 34 | init_logger(); 35 | let temp_dir = tempdir::TempDir::new("test_write_read").unwrap(); 36 | let temp_path = temp_dir.path().to_str().unwrap().to_string(); 37 | info!("temp file path: {} created", &temp_path); 38 | 39 | let grpc_port = 21101; 40 | let urpc_port = 21102; 41 | let mut config = 42 | Config::create_mem_localfile_config(grpc_port, "1G".to_string(), temp_path); 43 | config.urpc_port = Some(urpc_port); 44 | config.hybrid_store.memory_single_buffer_max_spill_size = Some("1B".to_string()); 45 | config.localfile_store.as_mut().unwrap().disk_high_watermark = 1.0; 46 | 47 | let _app_ref = mini_riffle::start(&config).await?; 48 | // wait all setup 49 | tokio::time::sleep(Duration::from_secs(1)).await; 50 | 51 | // after one batch write/read process, the allocated memory size should be 0 52 | assert_eq!(0, GAUGE_MEMORY_ALLOCATED.get()); 53 | 54 | shuffle_testing(&config, _app_ref).await 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q50.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q50.sql-- 19 | 20 | select 21 | s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, 22 | s_suite_number, s_city, s_county, s_state, s_zip 23 | ,sum(case when (sr_returned_date_sk - ss_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` 24 | ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 30) and 25 | (sr_returned_date_sk - ss_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` 26 | ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 60) and 27 | (sr_returned_date_sk - ss_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` 28 | ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 90) and 29 | (sr_returned_date_sk - ss_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` 30 | ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 120) then 1 else 0 end) as `>120 days` 31 | from 32 | store_sales, store_returns, store, date_dim d1, date_dim d2 33 | where 34 | d2.d_year = 2001 35 | and d2.d_moy = 8 36 | and ss_ticket_number = sr_ticket_number 37 | and ss_item_sk = sr_item_sk 38 | and ss_sold_date_sk = d1.d_date_sk 39 | and sr_returned_date_sk = d2.d_date_sk 40 | and ss_customer_sk = sr_customer_sk 41 | and ss_store_sk = s_store_sk 42 | group by 43 | s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, 44 | s_suite_number, s_city, s_county, s_state, s_zip 45 | order by 46 | s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, 47 | s_suite_number, s_city, s_county, s_state, s_zip 48 | limit 100 49 | 50 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q13.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q13.sql-- 19 | 20 | select avg(ss_quantity) 21 | ,avg(ss_ext_sales_price) 22 | ,avg(ss_ext_wholesale_cost) 23 | ,sum(ss_ext_wholesale_cost) 24 | from store_sales 25 | ,store 26 | ,customer_demographics 27 | ,household_demographics 28 | ,customer_address 29 | ,date_dim 30 | where s_store_sk = ss_store_sk 31 | and ss_sold_date_sk = d_date_sk and d_year = 2001 32 | and((ss_hdemo_sk=hd_demo_sk 33 | and cd_demo_sk = ss_cdemo_sk 34 | and cd_marital_status = 'M' 35 | and cd_education_status = 'Advanced Degree' 36 | and ss_sales_price between 100.00 and 150.00 37 | and hd_dep_count = 3 38 | )or 39 | (ss_hdemo_sk=hd_demo_sk 40 | and cd_demo_sk = ss_cdemo_sk 41 | and cd_marital_status = 'S' 42 | and cd_education_status = 'College' 43 | and ss_sales_price between 50.00 and 100.00 44 | and hd_dep_count = 1 45 | ) or 46 | (ss_hdemo_sk=hd_demo_sk 47 | and cd_demo_sk = ss_cdemo_sk 48 | and cd_marital_status = 'W' 49 | and cd_education_status = '2 yr Degree' 50 | and ss_sales_price between 150.00 and 200.00 51 | and hd_dep_count = 1 52 | )) 53 | and((ss_addr_sk = ca_address_sk 54 | and ca_country = 'United States' 55 | and ca_state in ('TX', 'OH', 'TX') 56 | and ss_net_profit between 100 and 200 57 | ) or 58 | (ss_addr_sk = ca_address_sk 59 | and ca_country = 'United States' 60 | and ca_state in ('OR', 'NM', 'KY') 61 | and ss_net_profit between 150 and 300 62 | ) or 63 | (ss_addr_sk = ca_address_sk 64 | and ca_country = 'United States' 65 | and ca_state in ('VA', 'TX', 'MS') 66 | and ss_net_profit between 50 and 250 67 | )) 68 | 69 | -------------------------------------------------------------------------------- /dev/centos7/amd64/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7 2 | 3 | # install common tools 4 | RUN echo "sslverify=false" >> /etc/yum.conf 5 | RUN sed -i "s/mirror.centos.org/vault.centos.org/g" /etc/yum.repos.d/*.repo 6 | RUN sed -i "s/^#.*baseurl=http/baseurl=https/g" /etc/yum.repos.d/*.repo 7 | RUN sed -i "s/^mirrorlist/#mirrorlist/g" /etc/yum.repos.d/*.repo 8 | RUN yum update -y 9 | RUN yum install -y centos-release-scl epel-release 10 | RUN sed -i "s/mirror.centos.org/vault.centos.org/g" /etc/yum.repos.d/*.repo 11 | RUN sed -i "s/^#.*baseurl=http/baseurl=https/g" /etc/yum.repos.d/*.repo 12 | RUN sed -i "s/^mirrorlist/#mirrorlist/g" /etc/yum.repos.d/*.repo 13 | RUN yum install -y libzip unzip wget cmake3 openssl-devel llvm clang-devel clang krb5-workstation clang-devel git gcc 14 | 15 | # install gcc-8 16 | #RUN yum install -y devtoolset-8-gcc devtoolset-8-gcc-c++ 17 | #RUN echo '. /opt/rh/devtoolset-8/enable' >> ~/.bashrc 18 | 19 | # install gcc-11 20 | RUN yum install -y devtoolset-11-gcc devtoolset-11-gcc-c++ 21 | RUN echo '. /opt/rh/devtoolset-11/enable' >> ~/.bashrc 22 | 23 | RUN yum install -y llvm-toolset-7 24 | RUN echo '. /opt/rh/llvm-toolset-7/enable' >> ~/.bashrc 25 | 26 | # install rust nightly toolchain 27 | RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain nightly-2025-06-01 28 | ENV PATH="/root/.cargo/bin:${PATH}" 29 | RUN rustc --version 30 | 31 | # install java 32 | RUN yum install -y java-1.8.0-openjdk java-1.8.0-openjdk-devel 33 | RUN echo 'export JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk"' >> ~/.bashrc 34 | 35 | # install maven 36 | # RUN yum install -y rh-maven35 37 | # RUN echo 'source /opt/rh/rh-maven35/enable' >> ~/.bashrc 38 | 39 | # install protoc 40 | RUN wget -O /protobuf-21.7-linux-x86_64.zip https://github.com/protocolbuffers/protobuf/releases/download/v21.7/protoc-21.7-linux-x86_64.zip 41 | RUN mkdir /protobuf-bin && (cd /protobuf-bin && unzip /protobuf-21.7-linux-x86_64.zip) 42 | RUN echo 'export PATH="$PATH:/protobuf-bin/bin"' >> ~/.bashrc 43 | 44 | # attach libjvm.so 45 | RUN echo 'export LD_LIBRARY_PATH=${JAVA_HOME}/jre/lib/amd64/server:${LD_LIBRARY_PATH}' >> ~/.bashrc 46 | 47 | # setup hadoop env 48 | RUN curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /root 49 | RUN echo "export HADOOP_HOME=/root/hadoop-3.3.5" >> ~/.bashrc 50 | RUN echo "export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> ~/.bashrc 51 | RUN echo "export HDRS_NAMENODE=default" >> ~/.bashrc 52 | RUN echo "export HDRS_WORKDIR=/tmp/hdrs/" >> ~/.bashrc 53 | 54 | RUN echo "export RUST_BACKTRACE=1" >> ~/.bashrc -------------------------------------------------------------------------------- /riffle-server/src/store/local/io_layer_read_ahead/sequential_tasks.rs: -------------------------------------------------------------------------------- 1 | use crate::metric::READ_AHEAD_OPERATION_DURATION_OF_SEQUENTIAL; 2 | use crate::store::local::io_layer_read_ahead::do_read_ahead; 3 | use std::fs::File; 4 | use std::sync::Arc; 5 | 6 | #[derive(Clone)] 7 | pub struct SequentialReadAheadTask { 8 | inner: Arc>, 9 | } 10 | 11 | struct Inner { 12 | absolute_path: String, 13 | file: File, 14 | 15 | is_initialized: bool, 16 | load_start_offset: u64, 17 | load_length: u64, 18 | 19 | batch_size: usize, 20 | batch_number: usize, 21 | } 22 | 23 | impl SequentialReadAheadTask { 24 | pub fn new(abs_path: &str, batch_size: usize, batch_number: usize) -> anyhow::Result { 25 | let file = File::options() 26 | .read(true) 27 | .write(true) 28 | .create(true) 29 | .append(true) 30 | .open(abs_path)?; 31 | Ok(Self { 32 | inner: Arc::new(tokio::sync::Mutex::new(Inner { 33 | absolute_path: abs_path.to_string(), 34 | file, 35 | is_initialized: false, 36 | load_start_offset: 0, 37 | load_length: 0, 38 | batch_size, 39 | batch_number, 40 | })), 41 | }) 42 | } 43 | 44 | // the return value shows whether the load operation happens 45 | pub async fn load(&self, off: u64, len: u64) -> anyhow::Result { 46 | let mut inner = self.inner.lock().await; 47 | if !inner.is_initialized && off == 0 { 48 | let load_len = (inner.batch_number * inner.batch_size) as u64; 49 | do_read_ahead(&inner.file, inner.absolute_path.as_str(), 0, load_len); 50 | inner.is_initialized = true; 51 | inner.load_length = load_len; 52 | return Ok(true); 53 | } 54 | 55 | let diff = inner.load_length - off; 56 | let next_load_bytes = 2 * inner.batch_size as u64; 57 | if diff > 0 && diff < next_load_bytes { 58 | let _timer = READ_AHEAD_OPERATION_DURATION_OF_SEQUENTIAL.start_timer(); 59 | let load_len = next_load_bytes; 60 | do_read_ahead( 61 | &inner.file, 62 | inner.absolute_path.as_str(), 63 | inner.load_start_offset + inner.load_length, 64 | load_len, 65 | ); 66 | inner.load_length += load_len; 67 | return Ok(true); 68 | } 69 | 70 | Ok(false) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /dev/integration/sql_set/q35.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Licensed to the Apache Software Foundation (ASF) under one or more 3 | -- contributor license agreements. See the NOTICE file distributed with 4 | -- this work for additional information regarding copyright ownership. 5 | -- The ASF licenses this file to You under the Apache License, Version 2.0 6 | -- (the "License"); you may not use this file except in compliance with 7 | -- the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | -- 17 | 18 | --q35.sql-- 19 | 20 | select 21 | ca_state, 22 | cd_gender, 23 | cd_marital_status, 24 | cd_dep_count, 25 | count(*) cnt1, 26 | min(cd_dep_count), 27 | max(cd_dep_count), 28 | avg(cd_dep_count), 29 | cd_dep_employed_count, 30 | count(*) cnt2, 31 | min(cd_dep_employed_count), 32 | max(cd_dep_employed_count), 33 | avg(cd_dep_employed_count), 34 | cd_dep_college_count, 35 | count(*) cnt3, 36 | min(cd_dep_college_count), 37 | max(cd_dep_college_count), 38 | avg(cd_dep_college_count) 39 | from 40 | customer c,customer_address ca,customer_demographics 41 | where 42 | c.c_current_addr_sk = ca.ca_address_sk and 43 | cd_demo_sk = c.c_current_cdemo_sk and 44 | exists (select * from store_sales, date_dim 45 | where c.c_customer_sk = ss_customer_sk and 46 | ss_sold_date_sk = d_date_sk and 47 | d_year = 2002 and 48 | d_qoy < 4) and 49 | (exists (select * from web_sales, date_dim 50 | where c.c_customer_sk = ws_bill_customer_sk and 51 | ws_sold_date_sk = d_date_sk and 52 | d_year = 2002 and 53 | d_qoy < 4) or 54 | exists (select * from catalog_sales, date_dim 55 | where c.c_customer_sk = cs_ship_customer_sk and 56 | cs_sold_date_sk = d_date_sk and 57 | d_year = 2002 and 58 | d_qoy < 4)) 59 | group by ca_state, cd_gender, cd_marital_status, cd_dep_count, 60 | cd_dep_employed_count, cd_dep_college_count 61 | order by ca_state, cd_gender, cd_marital_status, cd_dep_count, 62 | cd_dep_employed_count, cd_dep_college_count 63 | limit 100 64 | 65 | --------------------------------------------------------------------------------