>,
37 | shutdown_handle: Shutdown,
38 | ) -> Self {
39 | Self {
40 | rebalance_rx,
41 | stream_state,
42 | shutdown_handle,
43 | }
44 | }
45 |
46 | pub fn start(self) {
47 | let mut rebalance_receiver = self.rebalance_rx;
48 | let stream_state = self.stream_state.clone();
49 | let shutdown_handle = self.shutdown_handle.clone();
50 | let tokio_runtime_handle = Handle::current();
51 |
52 | std::thread::Builder::new().name("rebalance-listener-thread".to_string()).spawn(move || {
53 | tokio_runtime_handle.block_on(async move {
54 | loop {
55 | tokio::select! {
56 | rebalance = rebalance_receiver.recv() => {
57 | match rebalance {
58 | Some(RebalanceEvent::Assign(tpl)) => info!("RebalanceEvent Assign: {:?}", tpl),
59 | Some(RebalanceEvent::Revoke(tpl, callback)) => {
60 | info!("RebalanceEvent Revoke: {:?}", tpl);
61 | if let Ok(mut stream_state) = stream_state.try_write() {
62 | stream_state.terminate_partition_streams(tpl).await;
63 | drop(stream_state);
64 | } else {
65 | warn!("Stream state lock is busy, skipping rebalance revoke for {:?}", tpl);
66 | }
67 | if let Err(err) = callback.send(()) {
68 | warn!("Error during sending response to context. Cause: {:?}", err);
69 | }
70 | info!("Finished Rebalance Revoke");
71 | }
72 | None => {
73 | info!("Rebalance event sender is closed!");
74 | break
75 | }
76 | }
77 | },
78 | _ = shutdown_handle.recv() => {
79 | info!("Gracefully stopping rebalance listener!");
80 | break;
81 | },
82 | }
83 | }
84 | })
85 | }).expect("Failed to start rebalance listener thread");
86 | }
87 | }
88 |
--------------------------------------------------------------------------------
/src/connectors/kafka/sink.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 | use crate::connectors::common::build_runtime;
19 | use crate::connectors::common::processor::Processor;
20 | use crate::connectors::kafka::consumer::KafkaStreams;
21 | use crate::connectors::kafka::processor::StreamWorker;
22 | use crate::connectors::kafka::ConsumerRecord;
23 | use anyhow::Result;
24 | use futures_util::StreamExt;
25 | use rdkafka::consumer::Consumer;
26 | use std::sync::Arc;
27 | use tokio::runtime::Runtime;
28 | use tracing::{error, info};
29 |
30 | pub struct KafkaSinkConnector
31 | where
32 | P: Processor, ()>,
33 | {
34 | streams: KafkaStreams,
35 | stream_processor: Arc>,
36 | runtime: Runtime,
37 | }
38 |
39 | impl KafkaSinkConnector
40 | where
41 | P: Processor, ()> + Send + Sync + 'static,
42 | {
43 | pub fn new(kafka_streams: KafkaStreams, processor: P) -> Self {
44 | let consumer = kafka_streams.consumer();
45 | let stream_processor = Arc::new(StreamWorker::new(
46 | Arc::new(processor),
47 | Arc::clone(&consumer),
48 | ));
49 |
50 | let runtime = build_runtime(
51 | consumer.context().config.partition_listener_concurrency,
52 | "kafka-sink-worker",
53 | )
54 | .expect("Failed to build runtime");
55 | let _ = runtime.enter();
56 |
57 | Self {
58 | streams: kafka_streams,
59 | stream_processor,
60 | runtime,
61 | }
62 | }
63 |
64 | pub async fn run(self) -> Result<()> {
65 | self.streams
66 | .partitioned()
67 | .map(|partition_stream| {
68 | let worker = Arc::clone(&self.stream_processor);
69 | let tp = partition_stream.topic_partition().clone();
70 | self.runtime.spawn(async move {
71 | partition_stream
72 | .run_drain(|partition_records| async {
73 | info!("Starting task for partition: {:?}", tp);
74 |
75 | worker
76 | .process_partition(tp.clone(), partition_records)
77 | .await
78 | .unwrap();
79 | })
80 | .await;
81 |
82 | info!("Task completed for partition: {:?}", tp);
83 | })
84 | })
85 | .for_each_concurrent(None, |task| async {
86 | if let Err(e) = task.await {
87 | error!("Task failed: {:?}", e);
88 | }
89 | })
90 | .await;
91 |
92 | Ok(())
93 | }
94 | }
95 |
--------------------------------------------------------------------------------
/src/connectors/kafka/state.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use crate::connectors::kafka::partition_stream::PartitionStreamSender;
20 | use crate::connectors::kafka::{TopicPartition, TopicPartitionList};
21 | use std::collections::HashMap;
22 | use tracing::info;
23 |
24 | pub struct StreamState {
25 | partition_senders: HashMap,
26 | }
27 |
28 | impl StreamState {
29 | pub fn new(capacity: usize) -> Self {
30 | Self {
31 | partition_senders: HashMap::with_capacity(capacity),
32 | }
33 | }
34 |
35 | pub fn insert_partition_sender(
36 | &mut self,
37 | tp: TopicPartition,
38 | sender: PartitionStreamSender,
39 | ) -> Option {
40 | self.partition_senders.insert(tp, sender)
41 | }
42 |
43 | pub fn get_partition_sender(&self, tp: &TopicPartition) -> Option<&PartitionStreamSender> {
44 | self.partition_senders.get(tp)
45 | }
46 |
47 | pub async fn terminate_partition_streams(&mut self, tpl: TopicPartitionList) {
48 | info!("Terminating streams: {:?}", tpl);
49 |
50 | for tp in tpl.tpl {
51 | if let Some(sender) = self.partition_senders.remove(&tp) {
52 | info!("Terminating stream for {:?}", tp);
53 | sender.terminate();
54 | drop(sender);
55 | info!("Stream terminated for {:?}", tp);
56 | } else {
57 | info!("Stream already completed for {:?}", tp);
58 | }
59 | }
60 |
61 | info!("All streams terminated!");
62 | }
63 |
64 | pub fn clear(&mut self) {
65 | info!("Clearing all stream states...");
66 | self.partition_senders.clear();
67 | }
68 | }
69 |
--------------------------------------------------------------------------------
/src/connectors/mod.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use std::sync::Arc;
20 |
21 | use actix_web_prometheus::PrometheusMetrics;
22 | use common::{processor::Processor, shutdown::Shutdown};
23 | use kafka::{
24 | config::KafkaConfig, consumer::KafkaStreams, metrics::KafkaMetricsCollector,
25 | processor::ParseableSinkProcessor, rebalance_listener::RebalanceListener,
26 | sink::KafkaSinkConnector, state::StreamState, ConsumerRecord, KafkaContext,
27 | };
28 | use prometheus::Registry;
29 | use tokio::sync::RwLock;
30 | use tracing::{info, warn};
31 |
32 | use crate::{option::Mode, parseable::PARSEABLE};
33 |
34 | pub mod common;
35 | pub mod kafka;
36 |
37 | pub async fn init(prometheus: &PrometheusMetrics) -> anyhow::Result<()> {
38 | if matches!(PARSEABLE.options.mode, Mode::Ingest | Mode::All) {
39 | match PARSEABLE.kafka_config.validate() {
40 | Err(e) => {
41 | warn!("Kafka connector configuration invalid. {}", e);
42 | }
43 | Ok(_) => {
44 | let config = PARSEABLE.kafka_config.clone();
45 | let shutdown_handle = Shutdown::default();
46 | let registry = prometheus.registry.clone();
47 | let processor = ParseableSinkProcessor;
48 |
49 | tokio::spawn({
50 | let shutdown_handle = shutdown_handle.clone();
51 | async move {
52 | shutdown_handle.signal_listener().await;
53 | info!("Connector received shutdown signal!");
54 | }
55 | });
56 |
57 | run_kafka2parseable(config, registry, processor, shutdown_handle).await?;
58 | }
59 | }
60 | }
61 |
62 | Ok(())
63 | }
64 |
65 | async fn run_kafka2parseable(
66 | config: KafkaConfig,
67 | registry: Registry,
68 | processor: P,
69 | shutdown_handle: Shutdown,
70 | ) -> anyhow::Result<()>
71 | where
72 | P: Processor, ()> + Send + Sync + 'static,
73 | {
74 | info!("Initializing KafkaSink connector...");
75 |
76 | let kafka_config = Arc::new(config.clone());
77 | let (kafka_context, rebalance_rx) = KafkaContext::new(kafka_config);
78 |
79 | //TODO: fetch topics metadata from kafka then give dynamic value to StreamState
80 | let stream_state = Arc::new(RwLock::new(StreamState::new(60)));
81 | let rebalance_listener = RebalanceListener::new(
82 | rebalance_rx,
83 | Arc::clone(&stream_state),
84 | shutdown_handle.clone(),
85 | );
86 |
87 | let kafka_streams = KafkaStreams::init(kafka_context, stream_state, shutdown_handle.clone())?;
88 |
89 | let stats = kafka_streams.statistics();
90 | registry.register(Box::new(KafkaMetricsCollector::new(stats)?))?;
91 |
92 | let kafka_parseable_sink_connector = KafkaSinkConnector::new(kafka_streams, processor);
93 |
94 | rebalance_listener.start();
95 | kafka_parseable_sink_connector.run().await?;
96 |
97 | Ok(())
98 | }
99 |
--------------------------------------------------------------------------------
/src/enterprise/mod.rs:
--------------------------------------------------------------------------------
1 | pub mod utils;
2 |
--------------------------------------------------------------------------------
/src/handlers/http/about.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use actix_web::web::Json;
20 | use serde_json::{json, Value};
21 |
22 | use crate::{
23 | about::{self, get_latest_release},
24 | parseable::PARSEABLE,
25 | storage::StorageMetadata,
26 | };
27 | use std::path::PathBuf;
28 |
29 | /// {
30 | /// "version": current_version,
31 | /// "uiVersion": ui_version,
32 | /// "commit": commit,
33 | /// "deploymentId": deployment_id,
34 | /// "updateAvailable": update_available,
35 | /// "latestVersion": latest_release,
36 | /// "llmActive": is_llm_active,
37 | /// "llmProvider": llm_provider,
38 | /// "oidcActive": is_oidc_active,
39 | /// "license": "AGPL-3.0-only",
40 | /// "mode": mode,
41 | /// "staging": staging,
42 | /// "grpcPort": grpc_port,
43 | /// "store": {
44 | /// "type": PARSEABLE.get_storage_mode_string(),
45 | /// "path": store_endpoint
46 | /// }
47 | /// }
48 | pub async fn about() -> Json {
49 | let meta = StorageMetadata::global();
50 |
51 | let current_release = about::current();
52 | let latest_release = get_latest_release();
53 | let (update_available, latest_release) = match latest_release {
54 | Some(latest_release) => (
55 | latest_release.version > current_release.released_version,
56 | Some(format!("v{}", latest_release.version)),
57 | ),
58 | None => (false, None),
59 | };
60 |
61 | let current_version = format!("v{}", current_release.released_version);
62 | let commit = current_release.commit_hash;
63 | let deployment_id = meta.deployment_id.to_string();
64 | let mode = PARSEABLE.get_server_mode_string();
65 | let staging = PARSEABLE.options.staging_dir().display().to_string();
66 | let grpc_port = PARSEABLE.options.grpc_port;
67 |
68 | let store_endpoint = PARSEABLE.storage.get_endpoint();
69 | let is_llm_active = &PARSEABLE.options.open_ai_key.is_some();
70 | let llm_provider = is_llm_active.then_some("OpenAI");
71 | let is_oidc_active = PARSEABLE.options.openid().is_some();
72 | let ui_version = option_env!("UI_VERSION").unwrap_or("development");
73 |
74 | let hot_tier_details: String = if PARSEABLE.hot_tier_dir().is_none() {
75 | "Disabled".to_string()
76 | } else {
77 | let hot_tier_dir: &Option = PARSEABLE.hot_tier_dir();
78 | format!(
79 | "Enabled, Path: {}",
80 | hot_tier_dir.as_ref().unwrap().display(),
81 | )
82 | };
83 |
84 | let ms_clarity_tag = &PARSEABLE.options.ms_clarity_tag;
85 |
86 | Json(json!({
87 | "version": current_version,
88 | "uiVersion": ui_version,
89 | "commit": commit,
90 | "deploymentId": deployment_id,
91 | "updateAvailable": update_available,
92 | "latestVersion": latest_release,
93 | "llmActive": is_llm_active,
94 | "llmProvider": llm_provider,
95 | "oidcActive": is_oidc_active,
96 | "license": "AGPL-3.0-only",
97 | "mode": mode,
98 | "staging": staging,
99 | "hotTier": hot_tier_details,
100 | "grpcPort": grpc_port,
101 | "store": {
102 | "type": PARSEABLE.get_storage_mode_string(),
103 | "path": store_endpoint
104 | },
105 | "analytics": {
106 | "clarityTag": ms_clarity_tag
107 | },
108 | }))
109 | }
110 |
--------------------------------------------------------------------------------
/src/handlers/http/correlation.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use actix_web::web::{Json, Path};
20 | use actix_web::{web, HttpRequest, HttpResponse, Responder};
21 | use anyhow::Error;
22 | use itertools::Itertools;
23 |
24 | use crate::rbac::Users;
25 | use crate::utils::actix::extract_session_key_from_req;
26 | use crate::utils::{get_hash, get_user_from_request, user_auth_for_datasets};
27 |
28 | use crate::correlation::{CorrelationConfig, CorrelationError, CORRELATIONS};
29 |
30 | pub async fn list(req: HttpRequest) -> Result {
31 | let session_key = extract_session_key_from_req(&req)
32 | .map_err(|err| CorrelationError::AnyhowError(Error::msg(err.to_string())))?;
33 |
34 | let correlations = CORRELATIONS.list_correlations(&session_key).await?;
35 |
36 | Ok(web::Json(correlations))
37 | }
38 |
39 | pub async fn get(
40 | req: HttpRequest,
41 | correlation_id: Path,
42 | ) -> Result {
43 | let correlation_id = correlation_id.into_inner();
44 | let session_key = extract_session_key_from_req(&req)
45 | .map_err(|err| CorrelationError::AnyhowError(Error::msg(err.to_string())))?;
46 |
47 | let correlation = CORRELATIONS.get_correlation(&correlation_id).await?;
48 |
49 | let permissions = Users.get_permissions(&session_key);
50 |
51 | let tables = &correlation
52 | .table_configs
53 | .iter()
54 | .map(|t| t.table_name.clone())
55 | .collect_vec();
56 |
57 | user_auth_for_datasets(&permissions, tables)?;
58 |
59 | Ok(web::Json(correlation))
60 | }
61 |
62 | pub async fn post(
63 | req: HttpRequest,
64 | Json(mut correlation): Json,
65 | ) -> Result {
66 | let session_key = extract_session_key_from_req(&req)
67 | .map_err(|err| CorrelationError::AnyhowError(anyhow::Error::msg(err.to_string())))?;
68 | let user_id = get_user_from_request(&req)
69 | .map(|s| get_hash(&s.to_string()))
70 | .map_err(|err| CorrelationError::AnyhowError(Error::msg(err.to_string())))?;
71 | correlation.user_id = user_id;
72 |
73 | let correlation = CORRELATIONS.create(correlation, &session_key).await?;
74 |
75 | Ok(web::Json(correlation))
76 | }
77 |
78 | pub async fn modify(
79 | req: HttpRequest,
80 | correlation_id: Path,
81 | Json(mut correlation): Json,
82 | ) -> Result {
83 | correlation.id = correlation_id.into_inner();
84 | correlation.user_id = get_user_from_request(&req)
85 | .map(|s| get_hash(&s.to_string()))
86 | .map_err(|err| CorrelationError::AnyhowError(Error::msg(err.to_string())))?;
87 |
88 | let session_key = extract_session_key_from_req(&req)
89 | .map_err(|err| CorrelationError::AnyhowError(anyhow::Error::msg(err.to_string())))?;
90 |
91 | let correlation = CORRELATIONS.update(correlation, &session_key).await?;
92 |
93 | Ok(web::Json(correlation))
94 | }
95 |
96 | pub async fn delete(
97 | req: HttpRequest,
98 | correlation_id: Path,
99 | ) -> Result {
100 | let correlation_id = correlation_id.into_inner();
101 | let user_id = get_user_from_request(&req)
102 | .map(|s| get_hash(&s.to_string()))
103 | .map_err(|err| CorrelationError::AnyhowError(Error::msg(err.to_string())))?;
104 |
105 | CORRELATIONS.delete(&correlation_id, &user_id).await?;
106 |
107 | Ok(HttpResponse::Ok().finish())
108 | }
109 |
--------------------------------------------------------------------------------
/src/handlers/http/health_check.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use std::sync::Arc;
20 |
21 | use actix_web::{
22 | body::MessageBody,
23 | dev::{ServiceRequest, ServiceResponse},
24 | error::Error,
25 | error::ErrorServiceUnavailable,
26 | middleware::Next,
27 | HttpResponse,
28 | };
29 | use http::StatusCode;
30 | use once_cell::sync::Lazy;
31 | use tokio::{sync::Mutex, task::JoinSet};
32 | use tracing::{error, info, warn};
33 |
34 | use crate::parseable::PARSEABLE;
35 |
36 | // Create a global variable to store signal status
37 | static SIGNAL_RECEIVED: Lazy>> = Lazy::new(|| Arc::new(Mutex::new(false)));
38 |
39 | pub async fn liveness() -> HttpResponse {
40 | HttpResponse::new(StatusCode::OK)
41 | }
42 |
43 | pub async fn check_shutdown_middleware(
44 | req: ServiceRequest,
45 | next: Next,
46 | ) -> Result, Error> {
47 | // Acquire the shutdown flag to check if the server is shutting down.
48 | if *SIGNAL_RECEIVED.lock().await {
49 | // Return 503 Service Unavailable if the server is shutting down.
50 | Err(ErrorServiceUnavailable("Server is shutting down"))
51 | } else {
52 | // Continue processing the request if the server is not shutting down.
53 | next.call(req).await
54 | }
55 | }
56 |
57 | // This function is called when the server is shutting down
58 | pub async fn shutdown() {
59 | // Set the shutdown flag to true
60 | let mut shutdown_flag = SIGNAL_RECEIVED.lock().await;
61 | *shutdown_flag = true;
62 |
63 | let mut joinset = JoinSet::new();
64 |
65 | // Sync staging
66 | PARSEABLE.streams.flush_and_convert(&mut joinset, true);
67 |
68 | while let Some(res) = joinset.join_next().await {
69 | match res {
70 | Ok(Ok(_)) => info!("Successfully converted arrow files to parquet."),
71 | Ok(Err(err)) => warn!("Failed to convert arrow files to parquet. {err:?}"),
72 | Err(err) => error!("Failed to join async task: {err}"),
73 | }
74 | }
75 |
76 | if let Err(e) = PARSEABLE
77 | .storage
78 | .get_object_store()
79 | .upload_files_from_staging()
80 | .await
81 | {
82 | warn!("Failed to sync local data with object store. {:?}", e);
83 | } else {
84 | info!("Successfully synced all data to S3.");
85 | }
86 | }
87 |
88 | pub async fn readiness() -> HttpResponse {
89 | // Check the object store connection
90 | if PARSEABLE.storage.get_object_store().check().await.is_ok() {
91 | HttpResponse::new(StatusCode::OK)
92 | } else {
93 | HttpResponse::new(StatusCode::SERVICE_UNAVAILABLE)
94 | }
95 | }
96 |
--------------------------------------------------------------------------------
/src/handlers/http/kinesis.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use base64::{engine::general_purpose::STANDARD, Engine as _};
20 | use serde::{Deserialize, Serialize};
21 | use serde_json::{Map, Value};
22 | use std::str;
23 |
24 | use crate::utils::json::flatten::{generic_flattening, has_more_than_max_allowed_levels};
25 |
26 | #[derive(Serialize, Deserialize, Debug)]
27 | #[serde(rename_all = "camelCase")]
28 | pub struct Message {
29 | records: Vec,
30 | request_id: String,
31 | timestamp: u64,
32 | }
33 | #[derive(Serialize, Deserialize, Debug)]
34 | struct Data {
35 | data: String,
36 | }
37 |
38 | // Flatten Kinesis logs is used to flatten the Kinesis logs into a queryable JSON format.
39 | // Kinesis logs are in the format
40 | // {
41 | // "requestId": "9b848d8a-2d89-474b-b073-04b8e5232210",
42 | // "timestamp": 1705026780451,
43 | // "records": [
44 | // {
45 | // "data": "eyJDSEFOR0UiOi0wLjQ1LCJQUklDRSI6NS4zNiwiVElDS0VSX1NZTUJPTCI6IkRFRyIsIlNFQ1RPUiI6IkVORVJHWSJ9"
46 | // }
47 | // ]
48 | // }
49 | // The data field is base64 encoded JSON (there can be multiple data fields), and there is a requestId and timestamp field.
50 | // Kinesis logs are flattened to the following format:
51 | // {
52 | // "CHANGE": 3.16,
53 | // "PRICE": 73.76,
54 | // "SECTOR": "RETAIL",
55 | // "TICKER_SYMBOL": "WMT",
56 | // "p_metadata": "",
57 | // "p_tags": "",
58 | // "p_timestamp": "2024-01-11T09:08:34.290",
59 | // "requestId": "b858288a-f5d8-4181-a746-3f3dd716be8a",
60 | // "timestamp": "1704964113659"
61 | // }
62 | pub async fn flatten_kinesis_logs(message: Message) -> Result, anyhow::Error> {
63 | let mut vec_kinesis_json = Vec::new();
64 |
65 | for record in message.records.iter() {
66 | let bytes = STANDARD.decode(record.data.clone())?;
67 | if let Ok(json_string) = String::from_utf8(bytes) {
68 | let json: serde_json::Value = serde_json::from_str(&json_string)?;
69 | // Check if the JSON has more than the allowed levels of nesting
70 | // If it has less than or equal to the allowed levels, we flatten it.
71 | // If it has more than the allowed levels, we just push it as is
72 | // without flattening or modifying it.
73 | if !has_more_than_max_allowed_levels(&json, 1) {
74 | let flattened_json_arr = generic_flattening(&json)?;
75 | for flattened_json in flattened_json_arr {
76 | let mut kinesis_json: Map =
77 | serde_json::from_value(flattened_json)?;
78 | kinesis_json.insert(
79 | "requestId".to_owned(),
80 | Value::String(message.request_id.clone()),
81 | );
82 | kinesis_json.insert(
83 | "timestamp".to_owned(),
84 | Value::String(message.timestamp.to_string()),
85 | );
86 |
87 | vec_kinesis_json.push(Value::Object(kinesis_json));
88 | }
89 | } else {
90 | // If the JSON has more than the allowed levels, we just push it as is
91 | // without flattening or modifying it.
92 | // This is a fallback to ensure we don't lose data.
93 | tracing::warn!(
94 | "Kinesis log with requestId {} and timestamp {} has more than the allowed levels of nesting, skipping flattening for this record.",
95 | message.request_id, message.timestamp
96 | );
97 | vec_kinesis_json.push(json);
98 | }
99 | } else {
100 | tracing::error!(
101 | "Failed to decode base64 data for kinesis log with requestId {} and timestamp {}",
102 | message.request_id,
103 | message.timestamp
104 | );
105 | return Err(anyhow::anyhow!(
106 | "Failed to decode base64 data for record with requestId {} and timestamp {}",
107 | message.request_id,
108 | message.timestamp
109 | ));
110 | }
111 | }
112 |
113 | Ok(vec_kinesis_json)
114 | }
115 |
--------------------------------------------------------------------------------
/src/handlers/http/modal/ingest/ingestor_ingest.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use actix_web::{HttpRequest, HttpResponse};
20 | use bytes::Bytes;
21 |
22 | use crate::{handlers::http::{ingest::PostError, modal::utils::ingest_utils::flatten_and_push_logs}, metadata::PARSEABLE.streams};
23 |
24 |
25 | // Handler for POST /api/v1/logstream/{logstream}
26 | // only ingests events into the specified logstream
27 | // fails if the logstream does not exist
28 | pub async fn post_event(req: HttpRequest, body: Bytes) -> Result {
29 | let stream_name: String = req.match_info().get("logstream").unwrap().parse().unwrap();
30 | let internal_stream_names = PARSEABLE.streams.list_internal_streams();
31 | if internal_stream_names.contains(&stream_name) {
32 | return Err(PostError::Invalid(anyhow::anyhow!(
33 | "Stream {} is an internal stream and cannot be ingested into",
34 | stream_name
35 | )));
36 | }
37 | if !PARSEABLE.streams.stream_exists(&stream_name) {
38 | return Err(PostError::StreamNotFound(stream_name));
39 | }
40 |
41 | flatten_and_push_logs(req, body, stream_name).await?;
42 | Ok(HttpResponse::Ok().finish())
43 | }
--------------------------------------------------------------------------------
/src/handlers/http/modal/ingest/ingestor_logstream.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use std::fs;
20 |
21 | use actix_web::{
22 | web::{Json, Path},
23 | HttpRequest, Responder,
24 | };
25 | use bytes::Bytes;
26 | use http::StatusCode;
27 | use tracing::warn;
28 |
29 | use crate::{
30 | catalog::remove_manifest_from_snapshot,
31 | handlers::http::logstream::error::StreamError,
32 | parseable::{StreamNotFound, PARSEABLE},
33 | stats,
34 | };
35 |
36 | pub async fn retention_cleanup(
37 | stream_name: Path,
38 | Json(date_list): Json>,
39 | ) -> Result {
40 | let stream_name = stream_name.into_inner();
41 | let storage = PARSEABLE.storage.get_object_store();
42 | // if the stream not found in memory map,
43 | //check if it exists in the storage
44 | //create stream and schema from storage
45 | if !PARSEABLE.streams.contains(&stream_name)
46 | && !PARSEABLE
47 | .create_stream_and_schema_from_storage(&stream_name)
48 | .await
49 | .unwrap_or(false)
50 | {
51 | return Err(StreamNotFound(stream_name.clone()).into());
52 | }
53 |
54 | let res = remove_manifest_from_snapshot(storage.clone(), &stream_name, date_list).await;
55 | let first_event_at: Option = res.unwrap_or_default();
56 |
57 | Ok((first_event_at, StatusCode::OK))
58 | }
59 |
60 | pub async fn delete(stream_name: Path) -> Result {
61 | let stream_name = stream_name.into_inner();
62 |
63 | // Delete from staging
64 | let stream_dir = PARSEABLE.get_stream(&stream_name)?;
65 | if let Err(err) = fs::remove_dir_all(&stream_dir.data_path) {
66 | warn!(
67 | "failed to delete local data for stream {} with error {err}. Clean {} manually",
68 | stream_name,
69 | stream_dir.data_path.to_string_lossy()
70 | )
71 | }
72 |
73 | // Delete from memory
74 | PARSEABLE.streams.delete(&stream_name);
75 | stats::delete_stats(&stream_name, "json")
76 | .unwrap_or_else(|e| warn!("failed to delete stats for stream {}: {:?}", stream_name, e));
77 |
78 | Ok((format!("log stream {stream_name} deleted"), StatusCode::OK))
79 | }
80 |
81 | pub async fn put_stream(
82 | req: HttpRequest,
83 | stream_name: Path,
84 | body: Bytes,
85 | ) -> Result {
86 | let stream_name = stream_name.into_inner();
87 | PARSEABLE
88 | .create_update_stream(req.headers(), &body, &stream_name)
89 | .await?;
90 |
91 | Ok(("Log stream created", StatusCode::OK))
92 | }
93 |
--------------------------------------------------------------------------------
/src/handlers/http/modal/ingest/ingestor_rbac.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use std::collections::HashSet;
20 |
21 | use actix_web::{web, Responder};
22 | use tokio::sync::Mutex;
23 |
24 | use crate::{
25 | handlers::http::{modal::utils::rbac_utils::get_metadata, rbac::RBACError},
26 | rbac::{
27 | user::{self, User as ParseableUser},
28 | Users,
29 | },
30 | storage,
31 | };
32 |
33 | // async aware lock for updating storage metadata and user map atomicically
34 | static UPDATE_LOCK: Mutex<()> = Mutex::const_new(());
35 |
36 | // Handler for POST /api/v1/user/{username}
37 | // Creates a new user by username if it does not exists
38 | pub async fn post_user(
39 | username: web::Path,
40 | body: Option>,
41 | ) -> Result {
42 | let username = username.into_inner();
43 |
44 | let generated_password = String::default();
45 | let metadata = get_metadata().await?;
46 | if let Some(body) = body {
47 | let user: ParseableUser = serde_json::from_value(body.into_inner())?;
48 | let _ = storage::put_staging_metadata(&metadata);
49 | let created_role = user.roles.clone();
50 | Users.put_user(user.clone());
51 | Users.put_role(&username, created_role.clone());
52 | }
53 |
54 | Ok(generated_password)
55 | }
56 |
57 | // Handler for DELETE /api/v1/user/delete/{username}
58 | pub async fn delete_user(username: web::Path) -> Result {
59 | let username = username.into_inner();
60 | let _ = UPDATE_LOCK.lock().await;
61 | // fail this request if the user does not exists
62 | if !Users.contains(&username) {
63 | return Err(RBACError::UserDoesNotExist);
64 | };
65 | // delete from parseable.json first
66 | let mut metadata = get_metadata().await?;
67 | metadata.users.retain(|user| user.username() != username);
68 |
69 | let _ = storage::put_staging_metadata(&metadata);
70 |
71 | // update in mem table
72 | Users.delete_user(&username);
73 | Ok(format!("deleted user: {username}"))
74 | }
75 |
76 | // Handler PUT /user/{username}/roles => Put roles for user
77 | // Put roles for given user
78 | pub async fn put_role(
79 | username: web::Path,
80 | role: web::Json>,
81 | ) -> Result {
82 | let username = username.into_inner();
83 | let role = role.into_inner();
84 |
85 | if !Users.contains(&username) {
86 | return Err(RBACError::UserDoesNotExist);
87 | };
88 | // update parseable.json first
89 | let mut metadata = get_metadata().await?;
90 | if let Some(user) = metadata
91 | .users
92 | .iter_mut()
93 | .find(|user| user.username() == username)
94 | {
95 | user.roles.clone_from(&role);
96 | } else {
97 | // should be unreachable given state is always consistent
98 | return Err(RBACError::UserDoesNotExist);
99 | }
100 |
101 | let _ = storage::put_staging_metadata(&metadata);
102 | // update in mem table
103 | Users.put_role(&username.clone(), role.clone());
104 |
105 | Ok(format!("Roles updated successfully for {username}"))
106 | }
107 |
108 | // Handler for POST /api/v1/user/{username}/generate-new-password
109 | // Resets password for the user to a newly generated one and returns it
110 | pub async fn post_gen_password(username: web::Path) -> Result {
111 | let username = username.into_inner();
112 | let mut new_hash = String::default();
113 | let mut metadata = get_metadata().await?;
114 |
115 | let _ = storage::put_staging_metadata(&metadata);
116 | if let Some(user) = metadata
117 | .users
118 | .iter_mut()
119 | .filter_map(|user| match user.ty {
120 | user::UserType::Native(ref mut user) => Some(user),
121 | _ => None,
122 | })
123 | .find(|user| user.username == username)
124 | {
125 | new_hash.clone_from(&user.password_hash);
126 | } else {
127 | return Err(RBACError::UserDoesNotExist);
128 | }
129 | Users.change_password_hash(&username, &new_hash);
130 |
131 | Ok("Updated")
132 | }
133 |
--------------------------------------------------------------------------------
/src/handlers/http/modal/ingest/ingestor_role.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use actix_web::{
20 | web::{self, Json},
21 | HttpResponse, Responder,
22 | };
23 |
24 | use crate::{
25 | handlers::http::{modal::utils::rbac_utils::get_metadata, role::RoleError},
26 | rbac::{map::mut_roles, role::model::DefaultPrivilege},
27 | storage,
28 | };
29 |
30 | // Handler for PUT /api/v1/role/{name}
31 | // Creates a new role or update existing one
32 | pub async fn put(
33 | name: web::Path,
34 | Json(privileges): Json>,
35 | ) -> Result {
36 | let name = name.into_inner();
37 | let mut metadata = get_metadata().await?;
38 | metadata.roles.insert(name.clone(), privileges.clone());
39 |
40 | let _ = storage::put_staging_metadata(&metadata);
41 | mut_roles().insert(name.clone(), privileges);
42 |
43 | Ok(HttpResponse::Ok().finish())
44 | }
45 |
--------------------------------------------------------------------------------
/src/handlers/http/modal/ingest/mod.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | pub mod ingestor_logstream;
20 | pub mod ingestor_rbac;
21 | pub mod ingestor_role;
22 |
--------------------------------------------------------------------------------
/src/handlers/http/modal/query/mod.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | pub mod querier_ingest;
20 | pub mod querier_logstream;
21 | pub mod querier_rbac;
22 | pub mod querier_role;
23 |
--------------------------------------------------------------------------------
/src/handlers/http/modal/query/querier_ingest.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use actix_web::HttpResponse;
20 |
21 | use crate::handlers::http::ingest::PostError;
22 |
23 | // Handler for POST /api/v1/logstream/{logstream}
24 | // only ingests events into the specified logstream
25 | // fails if the logstream does not exist
26 | #[allow(unused)]
27 | pub async fn post_event() -> Result {
28 | Err(PostError::IngestionNotAllowed)
29 | }
30 |
--------------------------------------------------------------------------------
/src/handlers/http/modal/query/querier_role.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use actix_web::{
20 | web::{self, Json},
21 | HttpResponse, Responder,
22 | };
23 |
24 | use crate::{
25 | handlers::http::{
26 | cluster::sync_role_update_with_ingestors,
27 | modal::utils::rbac_utils::{get_metadata, put_metadata},
28 | role::RoleError,
29 | },
30 | rbac::{map::mut_roles, role::model::DefaultPrivilege},
31 | };
32 |
33 | // Handler for PUT /api/v1/role/{name}
34 | // Creates a new role or update existing one
35 | pub async fn put(
36 | name: web::Path,
37 | Json(privileges): Json>,
38 | ) -> Result {
39 | let name = name.into_inner();
40 | let mut metadata = get_metadata().await?;
41 | metadata.roles.insert(name.clone(), privileges.clone());
42 |
43 | put_metadata(&metadata).await?;
44 | mut_roles().insert(name.clone(), privileges.clone());
45 |
46 | sync_role_update_with_ingestors(name.clone(), privileges.clone()).await?;
47 |
48 | Ok(HttpResponse::Ok().finish())
49 | }
50 |
--------------------------------------------------------------------------------
/src/handlers/http/modal/ssl_acceptor.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use std::{
20 | fs::{self, File},
21 | io::BufReader,
22 | path::PathBuf,
23 | };
24 |
25 | use rustls::ServerConfig;
26 |
27 | pub fn get_ssl_acceptor(
28 | tls_cert: &Option,
29 | tls_key: &Option,
30 | other_certs: &Option,
31 | ) -> anyhow::Result> {
32 | match (tls_cert, tls_key) {
33 | (Some(cert), Some(key)) => {
34 | let server_config = ServerConfig::builder().with_no_client_auth();
35 |
36 | let cert_file = &mut BufReader::new(File::open(cert)?);
37 | let key_file = &mut BufReader::new(File::open(key)?);
38 |
39 | let mut certs = rustls_pemfile::certs(cert_file).collect::, _>>()?;
40 | // Load CA certificates from the directory
41 | if let Some(other_cert_dir) = other_certs {
42 | if other_cert_dir.is_dir() {
43 | for entry in fs::read_dir(other_cert_dir)? {
44 | let path = entry.unwrap().path();
45 |
46 | if path.is_file() {
47 | let other_cert_file = &mut BufReader::new(File::open(&path)?);
48 | let mut other_certs = rustls_pemfile::certs(other_cert_file)
49 | .collect::, _>>()?;
50 | certs.append(&mut other_certs);
51 | }
52 | }
53 | }
54 | }
55 | let private_key = rustls_pemfile::private_key(key_file)?
56 | .ok_or(anyhow::anyhow!("Could not parse private key."))?;
57 |
58 | Ok(Some(server_config.with_single_cert(certs, private_key)?))
59 | }
60 | (_, _) => Ok(None),
61 | }
62 | }
63 |
--------------------------------------------------------------------------------
/src/handlers/http/modal/utils/logstream_utils.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use actix_web::http::header::HeaderMap;
20 |
21 | use crate::{
22 | event::format::LogSource,
23 | handlers::{
24 | CUSTOM_PARTITION_KEY, LOG_SOURCE_KEY, STATIC_SCHEMA_FLAG, STREAM_TYPE_KEY,
25 | TIME_PARTITION_KEY, TIME_PARTITION_LIMIT_KEY, UPDATE_STREAM_KEY,
26 | },
27 | storage::StreamType,
28 | };
29 |
30 | #[derive(Debug, Default)]
31 | pub struct PutStreamHeaders {
32 | pub time_partition: String,
33 | pub time_partition_limit: String,
34 | pub custom_partition: Option,
35 | pub static_schema_flag: bool,
36 | pub update_stream_flag: bool,
37 | pub stream_type: StreamType,
38 | pub log_source: LogSource,
39 | }
40 |
41 | impl From<&HeaderMap> for PutStreamHeaders {
42 | fn from(headers: &HeaderMap) -> Self {
43 | PutStreamHeaders {
44 | time_partition: headers
45 | .get(TIME_PARTITION_KEY)
46 | .map_or("", |v| v.to_str().unwrap())
47 | .to_string(),
48 | time_partition_limit: headers
49 | .get(TIME_PARTITION_LIMIT_KEY)
50 | .map_or("", |v| v.to_str().unwrap())
51 | .to_string(),
52 | custom_partition: headers
53 | .get(CUSTOM_PARTITION_KEY)
54 | .map(|v| v.to_str().unwrap().to_string()),
55 | static_schema_flag: headers
56 | .get(STATIC_SCHEMA_FLAG)
57 | .is_some_and(|v| v.to_str().unwrap() == "true"),
58 | update_stream_flag: headers
59 | .get(UPDATE_STREAM_KEY)
60 | .is_some_and(|v| v.to_str().unwrap() == "true"),
61 | stream_type: headers
62 | .get(STREAM_TYPE_KEY)
63 | .map(|v| StreamType::from(v.to_str().unwrap()))
64 | .unwrap_or_default(),
65 | log_source: headers
66 | .get(LOG_SOURCE_KEY)
67 | .map_or(LogSource::default(), |v| v.to_str().unwrap().into()),
68 | }
69 | }
70 | }
71 |
--------------------------------------------------------------------------------
/src/handlers/http/modal/utils/mod.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | pub mod ingest_utils;
20 | pub mod logstream_utils;
21 | pub mod rbac_utils;
22 |
--------------------------------------------------------------------------------
/src/handlers/http/modal/utils/rbac_utils.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use crate::{
20 | parseable::PARSEABLE,
21 | storage::{self, ObjectStorageError, StorageMetadata},
22 | };
23 |
24 | pub async fn get_metadata() -> Result {
25 | let metadata = PARSEABLE
26 | .storage
27 | .get_object_store()
28 | .get_metadata()
29 | .await?
30 | .expect("metadata is initialized");
31 | Ok(metadata)
32 | }
33 |
34 | pub async fn put_metadata(metadata: &StorageMetadata) -> Result<(), ObjectStorageError> {
35 | storage::put_remote_metadata(metadata).await?;
36 | storage::put_staging_metadata(metadata)?;
37 | Ok(())
38 | }
39 |
--------------------------------------------------------------------------------
/src/handlers/http/prism_home.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use std::collections::HashMap;
20 |
21 | use actix_web::{web, HttpRequest, Responder};
22 |
23 | use crate::{
24 | prism::home::{generate_home_response, generate_home_search_response, PrismHomeError},
25 | utils::actix::extract_session_key_from_req,
26 | };
27 |
28 | const HOME_SEARCH_QUERY_PARAM: &str = "key";
29 |
30 | /// Fetches the data to populate Prism's home
31 | ///
32 | ///
33 | /// # Returns
34 | ///
35 | /// A JSONified version of the `HomeResponse` struct.
36 | pub async fn home_api(req: HttpRequest) -> Result {
37 | let key = extract_session_key_from_req(&req)
38 | .map_err(|err| PrismHomeError::Anyhow(anyhow::Error::msg(err.to_string())))?;
39 |
40 | let res = generate_home_response(&key).await?;
41 |
42 | Ok(web::Json(res))
43 | }
44 |
45 | pub async fn home_search(req: HttpRequest) -> Result {
46 | let key = extract_session_key_from_req(&req)
47 | .map_err(|err| PrismHomeError::Anyhow(anyhow::Error::msg(err.to_string())))?;
48 | let query_map = web::Query::>::from_query(req.query_string())
49 | .map_err(|_| PrismHomeError::InvalidQueryParameter(HOME_SEARCH_QUERY_PARAM.to_string()))?;
50 |
51 | if query_map.is_empty() {
52 | return Ok(web::Json(serde_json::json!({})));
53 | }
54 |
55 | let query_value = query_map
56 | .get(HOME_SEARCH_QUERY_PARAM)
57 | .ok_or_else(|| PrismHomeError::InvalidQueryParameter(HOME_SEARCH_QUERY_PARAM.to_string()))?
58 | .to_lowercase();
59 | let res = generate_home_search_response(&key, &query_value).await?;
60 | let json_res = serde_json::to_value(res)
61 | .map_err(|err| PrismHomeError::Anyhow(anyhow::Error::msg(err.to_string())))?;
62 |
63 | Ok(web::Json(json_res))
64 | }
65 |
--------------------------------------------------------------------------------
/src/handlers/http/prism_logstream.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use actix_web::{
20 | web::{self, Json, Path},
21 | HttpRequest, Responder,
22 | };
23 |
24 | use crate::{
25 | prism::logstream::{get_prism_logstream_info, PrismDatasetRequest, PrismLogstreamError},
26 | utils::actix::extract_session_key_from_req,
27 | };
28 |
29 | /// This API is essentially just combining the responses of /info and /schema together
30 | pub async fn get_info(stream_name: Path) -> Result {
31 | let prism_logstream_info = get_prism_logstream_info(&stream_name).await?;
32 |
33 | Ok(web::Json(prism_logstream_info))
34 | }
35 |
36 | /// A combination of /stats, /retention, /hottier, /info, /counts and /query
37 | pub async fn post_datasets(
38 | dataset_req: Option>,
39 | req: HttpRequest,
40 | ) -> Result {
41 | let session_key = extract_session_key_from_req(&req)?;
42 | let dataset = dataset_req
43 | .map(|Json(r)| r)
44 | .unwrap_or_default()
45 | .get_datasets(session_key)
46 | .await?;
47 |
48 | Ok(web::Json(dataset))
49 | }
50 |
--------------------------------------------------------------------------------
/src/handlers/http/users/mod.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | pub mod dashboards;
20 | pub mod filters;
21 |
22 | pub const USERS_ROOT_DIR: &str = ".users";
23 | pub const DASHBOARDS_DIR: &str = "dashboards";
24 | pub const FILTER_DIR: &str = "filters";
25 | pub const CORRELATION_DIR: &str = "correlations";
26 |
--------------------------------------------------------------------------------
/src/handlers/mod.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | pub mod airplane;
20 | pub mod http;
21 | pub mod livetail;
22 |
23 | pub const STREAM_NAME_HEADER_KEY: &str = "x-p-stream";
24 | const LOG_SOURCE_KEY: &str = "x-p-log-source";
25 | const EXTRACT_LOG_KEY: &str = "x-p-extract-log";
26 | const TIME_PARTITION_KEY: &str = "x-p-time-partition";
27 | const TIME_PARTITION_LIMIT_KEY: &str = "x-p-time-partition-limit";
28 | const CUSTOM_PARTITION_KEY: &str = "x-p-custom-partition";
29 | const STATIC_SCHEMA_FLAG: &str = "x-p-static-schema-flag";
30 | const AUTHORIZATION_KEY: &str = "authorization";
31 | const UPDATE_STREAM_KEY: &str = "x-p-update-stream";
32 | pub const STREAM_TYPE_KEY: &str = "x-p-stream-type";
33 | const OIDC_SCOPE: &str = "openid profile email";
34 | const COOKIE_AGE_DAYS: usize = 7;
35 | const SESSION_COOKIE_NAME: &str = "session";
36 | const USER_COOKIE_NAME: &str = "username";
37 |
38 | // constants for log Source values for known sources and formats
39 | const LOG_SOURCE_KINESIS: &str = "kinesis";
40 |
41 | // AWS Kinesis constants
42 | const KINESIS_COMMON_ATTRIBUTES_KEY: &str = "x-amz-firehose-common-attributes";
43 |
--------------------------------------------------------------------------------
/src/lib.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | pub mod about;
20 | pub mod alerts;
21 | pub mod analytics;
22 | pub mod audit;
23 | pub mod banner;
24 | pub mod catalog;
25 | mod cli;
26 | #[cfg(feature = "kafka")]
27 | pub mod connectors;
28 | pub mod correlation;
29 | pub mod enterprise;
30 | pub mod event;
31 | pub mod handlers;
32 | pub mod hottier;
33 | mod livetail;
34 | mod metadata;
35 | pub mod metrics;
36 | pub mod migration;
37 | mod oidc;
38 | pub mod option;
39 | pub mod otel;
40 | pub mod parseable;
41 | pub mod prism;
42 | pub mod query;
43 | pub mod rbac;
44 | mod response;
45 | mod static_schema;
46 | mod stats;
47 | pub mod storage;
48 | pub mod sync;
49 | pub mod users;
50 | pub mod utils;
51 | mod validator;
52 |
53 | use std::time::Duration;
54 |
55 | pub use handlers::http::modal::{
56 | ingest_server::IngestServer, query_server::QueryServer, server::Server, ParseableServer,
57 | };
58 | use once_cell::sync::Lazy;
59 | use parseable::PARSEABLE;
60 | use reqwest::{Client, ClientBuilder};
61 |
62 | // It is very unlikely that panic will occur when dealing with locks.
63 | pub const LOCK_EXPECT: &str = "Thread shouldn't panic while holding a lock";
64 |
65 | /// Describes the duration at the end of which in-memory buffers are flushed,
66 | /// arrows files are "finished" and compacted into parquet files.
67 | pub const LOCAL_SYNC_INTERVAL: Duration = Duration::from_secs(60);
68 |
69 | /// Duration used to configure prefix generation.
70 | pub const OBJECT_STORE_DATA_GRANULARITY: u32 = LOCAL_SYNC_INTERVAL.as_secs() as u32 / 60;
71 |
72 | /// Describes the duration at the end of which parquets are pushed into objectstore.
73 | pub const STORAGE_UPLOAD_INTERVAL: Duration = Duration::from_secs(30);
74 |
75 | // A single HTTP client for all outgoing HTTP requests from the parseable server
76 | pub static HTTP_CLIENT: Lazy = Lazy::new(|| {
77 | ClientBuilder::new()
78 | .connect_timeout(Duration::from_secs(3)) // set a timeout of 3s for each connection setup
79 | .timeout(Duration::from_secs(30)) // set a timeout of 30s for each request
80 | .pool_idle_timeout(Duration::from_secs(90)) // set a timeout of 90s for each idle connection
81 | .pool_max_idle_per_host(32) // max 32 idle connections per host
82 | .gzip(true) // gzip compress for all requests
83 | .brotli(true) // brotli compress for all requests
84 | .use_rustls_tls() // use only the rustls backend
85 | .http1_only() // use only http/1.1
86 | .build()
87 | .expect("Construction of client shouldn't fail")
88 | });
89 |
90 | //separate client is created for intra cluster communication
91 | //allow invalid certificates for connecting other nodes in the cluster
92 | //required when querier/prism server tries to connect to other nodes via IP address directly
93 | //but the certificate is valid for a specific domain name
94 | pub static INTRA_CLUSTER_CLIENT: Lazy = Lazy::new(|| {
95 | ClientBuilder::new()
96 | .connect_timeout(Duration::from_secs(3)) // set a timeout of 3s for each connection setup
97 | .timeout(Duration::from_secs(30)) // set a timeout of 30s for each request
98 | .pool_idle_timeout(Duration::from_secs(90)) // set a timeout of 90s for each idle connection
99 | .pool_max_idle_per_host(32) // max 32 idle connections per host
100 | .gzip(true) // gzip compress for all requests
101 | .brotli(true) // brotli compress for all requests
102 | .use_rustls_tls() // use only the rustls backend
103 | .http1_only() // use only http/1.1
104 | .danger_accept_invalid_certs(PARSEABLE.options.tls_skip_verify)
105 | .build()
106 | .expect("Construction of client shouldn't fail")
107 | });
108 |
--------------------------------------------------------------------------------
/src/logstream/mod.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use std::sync::Arc;
20 |
21 | use arrow_schema::Schema;
22 | use http::StatusCode;
23 |
24 | use crate::{handlers::http::{logstream::error::StreamError, query::update_schema_when_distributed}, parseable::{StreamNotFound, PARSEABLE}, storage::StreamInfo, LOCK_EXPECT};
25 |
26 |
27 |
28 | pub async fn get_stream_schema_helper(stream_name: &str) -> Result, StreamError> {
29 | // Ensure parseable is aware of stream in distributed mode
30 | if !PARSEABLE.check_or_load_stream(&stream_name).await {
31 | return Err(StreamNotFound(stream_name.to_owned()).into());
32 | }
33 |
34 | let stream = PARSEABLE.get_stream(&stream_name)?;
35 | match update_schema_when_distributed(&vec![stream_name.to_owned()]).await {
36 | Ok(_) => {
37 | let schema = stream.get_schema();
38 | Ok(schema)
39 | }
40 | Err(err) => Err(StreamError::Custom {
41 | msg: err.to_string(),
42 | status: StatusCode::EXPECTATION_FAILED,
43 | }),
44 | }
45 | }
46 |
47 | pub async fn get_stream_info_helper(stream_name: &str) -> Result {
48 | // For query mode, if the stream not found in memory map,
49 | //check if it exists in the storage
50 | //create stream and schema from storage
51 | if !PARSEABLE.check_or_load_stream(&stream_name).await {
52 | return Err(StreamNotFound(stream_name.to_owned()).into());
53 | }
54 |
55 | let storage = PARSEABLE.storage.get_object_store();
56 | // if first_event_at is not found in memory map, check if it exists in the storage
57 | // if it exists in the storage, update the first_event_at in memory map
58 | let stream_first_event_at =
59 | if let Some(first_event_at) = PARSEABLE.get_stream(&stream_name)?.get_first_event() {
60 | Some(first_event_at)
61 | } else if let Ok(Some(first_event_at)) =
62 | storage.get_first_event_from_storage(&stream_name).await
63 | {
64 | PARSEABLE
65 | .update_first_event_at(&stream_name, &first_event_at)
66 | .await
67 | } else {
68 | None
69 | };
70 |
71 | let stream_meta = PARSEABLE.get_stream(stream_name)?
72 | .metadata
73 | .read()
74 | .expect(LOCK_EXPECT);
75 |
76 | let stream_info = StreamInfo {
77 | stream_type: stream_meta.stream_type,
78 | created_at: stream_meta.created_at.clone(),
79 | first_event_at: stream_first_event_at,
80 | time_partition: stream_meta.time_partition.clone(),
81 | time_partition_limit: stream_meta
82 | .time_partition_limit
83 | .map(|limit| limit.to_string()),
84 | custom_partition: stream_meta.custom_partition.clone(),
85 | static_schema_flag: stream_meta.static_schema_flag,
86 | log_source: stream_meta.log_source.clone(),
87 | };
88 |
89 | Ok(stream_info)
90 | }
--------------------------------------------------------------------------------
/src/main.rs:
--------------------------------------------------------------------------------
1 | use std::process::exit;
2 |
3 | /*
4 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
5 | *
6 | * This program is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * This program is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with this program. If not, see .
18 | *
19 | */
20 | #[cfg(feature = "kafka")]
21 | use parseable::connectors;
22 | use parseable::{
23 | banner, metrics, option::Mode, parseable::PARSEABLE, rbac, storage, IngestServer,
24 | ParseableServer, QueryServer, Server,
25 | };
26 | use tokio::signal::ctrl_c;
27 | use tokio::sync::oneshot;
28 | use tracing::Level;
29 | use tracing::{info, warn};
30 | use tracing_subscriber::layer::SubscriberExt;
31 | use tracing_subscriber::util::SubscriberInitExt;
32 | use tracing_subscriber::{fmt, EnvFilter, Registry};
33 |
34 | #[actix_web::main]
35 | async fn main() -> anyhow::Result<()> {
36 | init_logger();
37 |
38 | // these are empty ptrs so mem footprint should be minimal
39 | let server: Box = match &PARSEABLE.options.mode {
40 | Mode::Query => Box::new(QueryServer),
41 | Mode::Ingest => Box::new(IngestServer),
42 | Mode::Index => {
43 | println!("Indexing is an enterprise feature. Check out https://www.parseable.com/pricing to know more!");
44 | exit(0)
45 | }
46 | Mode::Prism => {
47 | println!("Prism is an enterprise feature. Check out https://www.parseable.com/pricing to know more!");
48 | exit(0)
49 | }
50 | Mode::All => Box::new(Server),
51 | };
52 |
53 | // load metadata from persistence
54 | let parseable_json = server.load_metadata().await?;
55 | let metadata = storage::resolve_parseable_metadata(&parseable_json).await?;
56 | banner::print(&PARSEABLE, &metadata).await;
57 | // initialize the rbac map
58 | rbac::map::init(&metadata);
59 | // keep metadata info in mem
60 | metadata.set_global();
61 |
62 | // Spawn a task to trigger graceful shutdown on appropriate signal
63 | let (shutdown_trigger, shutdown_rx) = oneshot::channel::<()>();
64 | tokio::spawn(async move {
65 | block_until_shutdown_signal().await;
66 |
67 | // Trigger graceful shutdown
68 | warn!("Received shutdown signal, notifying server to shut down...");
69 | shutdown_trigger.send(()).unwrap();
70 | });
71 |
72 | let prometheus = metrics::build_metrics_handler();
73 | // Start servers
74 | #[cfg(feature = "kafka")]
75 | {
76 | let parseable_server = server.init(&prometheus, shutdown_rx);
77 | let connectors = connectors::init(&prometheus);
78 |
79 | tokio::try_join!(parseable_server, connectors)?;
80 | }
81 |
82 | #[cfg(not(feature = "kafka"))]
83 | {
84 | let parseable_server = server.init(&prometheus, shutdown_rx);
85 | parseable_server.await?;
86 | }
87 |
88 | Ok(())
89 | }
90 |
91 | pub fn init_logger() {
92 | let filter_layer = EnvFilter::try_from_default_env().unwrap_or_else(|_| {
93 | let default_level = if cfg!(debug_assertions) {
94 | Level::DEBUG
95 | } else {
96 | Level::WARN
97 | };
98 | EnvFilter::new(default_level.to_string())
99 | });
100 |
101 | let fmt_layer = fmt::layer()
102 | .with_thread_names(true)
103 | .with_thread_ids(true)
104 | .with_line_number(true)
105 | .with_timer(tracing_subscriber::fmt::time::UtcTime::rfc_3339())
106 | .with_target(true)
107 | .compact();
108 |
109 | Registry::default()
110 | .with(filter_layer)
111 | .with(fmt_layer)
112 | .init();
113 | }
114 |
115 | #[cfg(windows)]
116 | /// Asynchronously blocks until a shutdown signal is received
117 | pub async fn block_until_shutdown_signal() {
118 | _ = ctrl_c().await;
119 | info!("Received a CTRL+C event");
120 | }
121 |
122 | #[cfg(unix)]
123 | /// Asynchronously blocks until a shutdown signal is received
124 | pub async fn block_until_shutdown_signal() {
125 | use tokio::signal::unix::{signal, SignalKind};
126 | let mut sigterm =
127 | signal(SignalKind::terminate()).expect("Failed to create SIGTERM signal handler");
128 |
129 | tokio::select! {
130 | _ = ctrl_c() => info!("Received SIGINT signal"),
131 | _ = sigterm.recv() => info!("Received SIGTERM signal"),
132 | }
133 | }
134 |
--------------------------------------------------------------------------------
/src/metrics/storage.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use actix_web_prometheus::PrometheusMetrics;
20 |
21 | pub trait StorageMetrics {
22 | fn register_metrics(&self, handler: &PrometheusMetrics);
23 | }
24 |
25 | pub mod localfs {
26 | use crate::{metrics::METRICS_NAMESPACE, storage::FSConfig};
27 | use once_cell::sync::Lazy;
28 | use prometheus::{HistogramOpts, HistogramVec};
29 |
30 | use super::StorageMetrics;
31 |
32 | pub static REQUEST_RESPONSE_TIME: Lazy = Lazy::new(|| {
33 | HistogramVec::new(
34 | HistogramOpts::new("local_fs_response_time", "FileSystem Request Latency")
35 | .namespace(METRICS_NAMESPACE),
36 | &["method", "status"],
37 | )
38 | .expect("metric can be created")
39 | });
40 |
41 | impl StorageMetrics for FSConfig {
42 | fn register_metrics(&self, handler: &actix_web_prometheus::PrometheusMetrics) {
43 | handler
44 | .registry
45 | .register(Box::new(REQUEST_RESPONSE_TIME.clone()))
46 | .expect("metric can be registered");
47 | }
48 | }
49 | }
50 |
51 | pub mod s3 {
52 | use crate::{metrics::METRICS_NAMESPACE, storage::S3Config};
53 | use once_cell::sync::Lazy;
54 | use prometheus::{HistogramOpts, HistogramVec};
55 |
56 | use super::StorageMetrics;
57 |
58 | pub static REQUEST_RESPONSE_TIME: Lazy = Lazy::new(|| {
59 | HistogramVec::new(
60 | HistogramOpts::new("s3_response_time", "S3 Request Latency")
61 | .namespace(METRICS_NAMESPACE),
62 | &["method", "status"],
63 | )
64 | .expect("metric can be created")
65 | });
66 |
67 | pub static QUERY_LAYER_STORAGE_REQUEST_RESPONSE_TIME: Lazy = Lazy::new(|| {
68 | HistogramVec::new(
69 | HistogramOpts::new("query_s3_response_time", "S3 Request Latency")
70 | .namespace(METRICS_NAMESPACE),
71 | &["method", "status"],
72 | )
73 | .expect("metric can be created")
74 | });
75 |
76 | impl StorageMetrics for S3Config {
77 | fn register_metrics(&self, handler: &actix_web_prometheus::PrometheusMetrics) {
78 | handler
79 | .registry
80 | .register(Box::new(REQUEST_RESPONSE_TIME.clone()))
81 | .expect("metric can be registered");
82 | handler
83 | .registry
84 | .register(Box::new(QUERY_LAYER_STORAGE_REQUEST_RESPONSE_TIME.clone()))
85 | .expect("metric can be registered");
86 | }
87 | }
88 | }
89 |
90 | pub mod azureblob {
91 | use crate::{metrics::METRICS_NAMESPACE, storage::AzureBlobConfig};
92 | use once_cell::sync::Lazy;
93 | use prometheus::{HistogramOpts, HistogramVec};
94 |
95 | use super::StorageMetrics;
96 |
97 | pub static REQUEST_RESPONSE_TIME: Lazy = Lazy::new(|| {
98 | HistogramVec::new(
99 | HistogramOpts::new("azr_blob_response_time", "AzureBlob Request Latency")
100 | .namespace(METRICS_NAMESPACE),
101 | &["method", "status"],
102 | )
103 | .expect("metric can be created")
104 | });
105 |
106 | pub static QUERY_LAYER_STORAGE_REQUEST_RESPONSE_TIME: Lazy = Lazy::new(|| {
107 | HistogramVec::new(
108 | HistogramOpts::new("query_azr_blob_response_time", "AzureBlob Request Latency")
109 | .namespace(METRICS_NAMESPACE),
110 | &["method", "status"],
111 | )
112 | .expect("metric can be created")
113 | });
114 |
115 | impl StorageMetrics for AzureBlobConfig {
116 | fn register_metrics(&self, handler: &actix_web_prometheus::PrometheusMetrics) {
117 | handler
118 | .registry
119 | .register(Box::new(REQUEST_RESPONSE_TIME.clone()))
120 | .expect("metric can be registered");
121 | handler
122 | .registry
123 | .register(Box::new(QUERY_LAYER_STORAGE_REQUEST_RESPONSE_TIME.clone()))
124 | .expect("metric can be registered");
125 | }
126 | }
127 | }
128 |
--------------------------------------------------------------------------------
/src/migration/schema_migration.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | *
18 | */
19 |
20 | use std::collections::HashMap;
21 |
22 | use arrow_schema::{DataType, Field, Schema};
23 | use serde_json::Value;
24 |
25 | pub(super) fn v1_v4(schema: Option) -> anyhow::Result {
26 | if let Some(schema) = schema {
27 | value_to_schema(schema)
28 | } else {
29 | Ok(Schema::empty())
30 | }
31 | }
32 |
33 | pub(super) fn v2_v4(schemas: HashMap) -> anyhow::Result {
34 | let mut derived_schemas = Vec::new();
35 |
36 | for value in schemas.into_values() {
37 | let schema = value_to_schema(value)?;
38 | derived_schemas.push(schema);
39 | }
40 |
41 | let schema = Schema::try_merge(derived_schemas)?;
42 | let mut fields: Vec<_> = schema.fields.iter().cloned().collect();
43 | fields.sort_by(|a, b| a.name().cmp(b.name()));
44 | Ok(Schema::new(fields))
45 | }
46 |
47 | fn value_to_schema(schema: Value) -> Result {
48 | let fields = schema
49 | .as_object()
50 | .expect("schema is an object")
51 | .get("fields")
52 | .expect("fields exists")
53 | .as_array()
54 | .expect("fields is an array");
55 |
56 | let mut new_fields = Vec::new();
57 |
58 | for field in fields {
59 | let field = field.as_object().unwrap();
60 | let field_name: String =
61 | serde_json::from_value(field.get("name").unwrap().clone()).unwrap();
62 | let field_dt: DataType =
63 | serde_json::from_value(field.get("data_type").unwrap().clone()).unwrap();
64 | new_fields.push(Field::new(field_name, field_dt, true));
65 | }
66 | new_fields.sort_by(|a, b| a.name().cmp(b.name()));
67 |
68 | Ok(Schema::new(new_fields))
69 | }
70 |
--------------------------------------------------------------------------------
/src/oidc.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use std::collections::HashMap;
20 |
21 | use openid::{Client, CompactJson, CustomClaims, Discovered, StandardClaims};
22 | use url::Url;
23 |
24 | pub type DiscoveredClient = Client;
25 |
26 | // If domain is not configured then
27 | // we can assume running in a development mode or private environment
28 | #[derive(Debug, Clone)]
29 | pub enum Origin {
30 | // socket address
31 | Local { socket_addr: String, https: bool },
32 | // domain url
33 | Production(Url),
34 | }
35 |
36 | /// Configuration for OpenID Connect
37 | #[derive(Debug, Clone)]
38 | pub struct OpenidConfig {
39 | /// Client id
40 | pub id: String,
41 | /// Client Secret
42 | pub secret: String,
43 | /// OP host address over which discovery can be done
44 | pub issuer: Url,
45 | /// Current client host address which will be used for redirects
46 | pub origin: Origin,
47 | }
48 |
49 | impl OpenidConfig {
50 | /// Create a new oidc client from server configuration.
51 | /// redirect_suffix
52 | pub async fn connect(
53 | self,
54 | redirect_to: &str,
55 | ) -> Result {
56 | let redirect_uri = match self.origin {
57 | Origin::Local { socket_addr, https } => {
58 | let protocol = if https { "https" } else { "http" };
59 | url::Url::parse(&format!("{protocol}://{socket_addr}")).expect("valid url")
60 | }
61 | Origin::Production(url) => url,
62 | };
63 |
64 | let redirect_uri = redirect_uri.join(redirect_to).expect("valid suffix");
65 | DiscoveredClient::discover(self.id, self.secret, redirect_uri.to_string(), self.issuer)
66 | .await
67 | }
68 | }
69 |
70 | #[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
71 | pub struct Claims {
72 | #[serde(flatten)]
73 | pub standard: StandardClaims,
74 | #[serde(flatten)]
75 | pub other: HashMap,
76 | }
77 |
78 | impl CustomClaims for Claims {
79 | fn standard_claims(&self) -> &StandardClaims {
80 | &self.standard
81 | }
82 | }
83 |
84 | impl CompactJson for Claims {}
85 |
--------------------------------------------------------------------------------
/src/otel.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | pub mod logs;
20 | pub mod metrics;
21 | pub mod otel_utils;
22 | pub mod traces;
23 |
--------------------------------------------------------------------------------
/src/parseable/staging/mod.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | *
18 | */
19 |
20 | pub mod reader;
21 | pub mod writer;
22 |
23 | #[derive(Debug, thiserror::Error)]
24 | pub enum StagingError {
25 | #[error("Unable to create recordbatch stream")]
26 | Arrow(#[from] arrow_schema::ArrowError),
27 | #[error("Could not generate parquet file")]
28 | Parquet(#[from] parquet::errors::ParquetError),
29 | #[error("IO Error {0}")]
30 | ObjectStorage(#[from] std::io::Error),
31 | #[error("Could not generate parquet file")]
32 | Create,
33 | // #[error("Metadata Error: {0}")]
34 | // Metadata(#[from] MetadataError),
35 | }
36 |
--------------------------------------------------------------------------------
/src/prism/mod.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | pub mod home;
20 | pub mod logstream;
21 |
--------------------------------------------------------------------------------
/src/rbac/utils.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 | use std::collections::HashMap;
19 |
20 | use url::Url;
21 |
22 | use crate::parseable::PARSEABLE;
23 |
24 | use super::{
25 | map::roles,
26 | role::model::DefaultPrivilege,
27 | user::{User, UserType},
28 | Users, UsersPrism,
29 | };
30 |
31 | pub fn to_prism_user(user: &User) -> UsersPrism {
32 | let (id, method, email, picture) = match &user.ty {
33 | UserType::Native(_) => (user.username(), "native", None, None),
34 | UserType::OAuth(oauth) => (
35 | user.username(),
36 | "oauth",
37 | oauth.user_info.email.clone(),
38 | oauth.user_info.picture.clone(),
39 | ),
40 | };
41 | let roles: HashMap> = Users
42 | .get_role(id)
43 | .iter()
44 | .filter_map(|role_name| {
45 | roles()
46 | .get(role_name)
47 | .map(|role| (role_name.to_owned(), role.clone()))
48 | })
49 | .collect();
50 |
51 | UsersPrism {
52 | id: id.into(),
53 | method: method.into(),
54 | email: mask_pii_string(email),
55 | picture: mask_pii_url(picture),
56 | roles,
57 | }
58 | }
59 |
60 | //mask PII string if the P_MASK_PII is set
61 | fn mask_pii_string(input: Option) -> Option {
62 | if !PARSEABLE.options.mask_pii {
63 | return input;
64 | }
65 | mask_string(input)
66 | }
67 |
68 | //mask PII url if the P_MASK_PII is set
69 | fn mask_pii_url(input: Option) -> Option {
70 | if !PARSEABLE.options.mask_pii {
71 | return input;
72 | }
73 | None
74 | }
75 |
76 | fn mask_string(input: Option) -> Option {
77 | let input = input.as_ref()?;
78 | if input.contains('@') {
79 | // masking an email
80 | let parts: Vec<&str> = input.split('@').collect();
81 | //mask everything if not a proper email format
82 | if parts.len() != 2 {
83 | return Some("X".repeat(input.len()));
84 | }
85 |
86 | let username = parts[0];
87 |
88 | // Mask the username - first letter capitalized, rest are X
89 | let masked_username = if !username.is_empty() {
90 | let first = username.chars().next().unwrap().to_uppercase().to_string();
91 | format!("{}{}", first, "X".repeat(username.len() - 1))
92 | } else {
93 | return Some("X".repeat(input.len()));
94 | };
95 |
96 | // mask to XXX for everything after the @ symbol
97 | Some(format!("{}@XXX", masked_username))
98 | } else {
99 | // mask all other strings with X
100 | Some("X".repeat(input.len()))
101 | }
102 | }
103 |
104 | #[cfg(test)]
105 | mod tests {
106 | use super::*;
107 |
108 | #[test]
109 | fn test_mask_string_with_email() {
110 | // Test masking a valid email
111 | let email = Some("test@example.com".to_string());
112 | let masked_email = mask_string(email);
113 | assert_eq!(masked_email, Some("TXXX@XXX".to_string()));
114 | }
115 |
116 | #[test]
117 | fn test_mask_string_with_invalid_email() {
118 | // Test masking an invalid email
119 | let invalid_email = Some("invalid-email".to_string());
120 | let masked_email = mask_string(invalid_email);
121 | assert_eq!(masked_email, Some("XXXXXXXXXXXXX".to_string()));
122 | }
123 |
124 | #[test]
125 | fn test_mask_string_with_empty_string() {
126 | // Test masking an empty string
127 | let empty_string = Some("".to_string());
128 | let masked_string = mask_string(empty_string);
129 | assert_eq!(masked_string, Some("".to_string()));
130 | }
131 |
132 | #[test]
133 | fn test_mask_string_with_generic_string() {
134 | // Test masking a generic string
135 | let generic_string = Some("sensitive_data".to_string());
136 | let masked_string = mask_string(generic_string);
137 | assert_eq!(masked_string, Some("XXXXXXXXXXXXXX".to_string()));
138 | }
139 |
140 | #[test]
141 | fn test_mask_string_with_none() {
142 | // Test masking a None value
143 | let none_value: Option = None;
144 | let masked_value = mask_string(none_value);
145 | assert_eq!(masked_value, None);
146 | }
147 | }
148 |
--------------------------------------------------------------------------------
/src/response.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use crate::{handlers::http::query::QueryError, utils::arrow::record_batches_to_json};
20 | use datafusion::arrow::record_batch::RecordBatch;
21 | use itertools::Itertools;
22 | use serde_json::{json, Value};
23 | use tracing::info;
24 |
25 | pub struct QueryResponse {
26 | pub records: Vec,
27 | pub fields: Vec,
28 | pub fill_null: bool,
29 | pub with_fields: bool,
30 | }
31 |
32 | impl QueryResponse {
33 | pub fn to_json(&self) -> Result {
34 | info!("{}", "Returning query results");
35 | let mut json_records = record_batches_to_json(&self.records)?;
36 |
37 | if self.fill_null {
38 | for map in &mut json_records {
39 | for field in &self.fields {
40 | if !map.contains_key(field) {
41 | map.insert(field.clone(), Value::Null);
42 | }
43 | }
44 | }
45 | }
46 | let values = json_records.into_iter().map(Value::Object).collect_vec();
47 |
48 | let response = if self.with_fields {
49 | json!({
50 | "fields": self.fields,
51 | "records": values,
52 | })
53 | } else {
54 | Value::Array(values)
55 | };
56 |
57 | Ok(response)
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/src/users/mod.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | pub mod dashboards;
20 | pub mod filters;
21 |
22 | use serde::{Deserialize, Serialize};
23 |
24 | #[derive(Debug, Serialize, Deserialize, Default, Clone, PartialEq, Eq)]
25 | pub struct TimeFilter {
26 | to: String,
27 | from: String,
28 | }
29 |
--------------------------------------------------------------------------------
/src/utils/actix.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | *
18 | */
19 |
20 | use actix_web::{
21 | dev::ServiceRequest,
22 | error::{ErrorUnauthorized, ErrorUnprocessableEntity},
23 | Error, FromRequest, HttpRequest,
24 | };
25 | use actix_web_httpauth::extractors::basic::BasicAuth;
26 |
27 | use crate::rbac::map::SessionKey;
28 |
29 | pub fn extract_session_key(req: &mut ServiceRequest) -> Result {
30 | // Extract username and password from the request using basic auth extractor.
31 | let creds = req.extract::().into_inner();
32 | let basic = creds.map(|creds| {
33 | let username = creds.user_id().trim().to_owned();
34 | // password is not mandatory by basic auth standard.
35 | // If not provided then treat as empty string
36 | let password = creds.password().unwrap_or("").trim().to_owned();
37 | SessionKey::BasicAuth { username, password }
38 | });
39 |
40 | if let Ok(basic) = basic {
41 | Ok(basic)
42 | } else if let Some(cookie) = req.cookie("session") {
43 | let ulid = ulid::Ulid::from_string(cookie.value())
44 | .map_err(|_| ErrorUnprocessableEntity("Cookie is tampered with or invalid"))?;
45 | Ok(SessionKey::SessionId(ulid))
46 | } else {
47 | Err(ErrorUnauthorized("No authentication method supplied"))
48 | }
49 | }
50 |
51 | pub fn extract_session_key_from_req(req: &HttpRequest) -> Result {
52 | // Extract username and password from the request using basic auth extractor.
53 | let creds = BasicAuth::extract(req).into_inner();
54 | let basic = creds.map(|creds| {
55 | let username = creds.user_id().trim().to_owned();
56 | // password is not mandatory by basic auth standard.
57 | // If not provided then treat as empty string
58 | let password = creds.password().unwrap_or("").trim().to_owned();
59 | SessionKey::BasicAuth { username, password }
60 | });
61 |
62 | if let Ok(basic) = basic {
63 | Ok(basic)
64 | } else if let Some(cookie) = req.cookie("session") {
65 | let ulid = ulid::Ulid::from_string(cookie.value())
66 | .map_err(|_| ErrorUnprocessableEntity("Cookie is tampered with or invalid"))?;
67 | Ok(SessionKey::SessionId(ulid))
68 | } else {
69 | Err(ErrorUnauthorized("No authentication method supplied"))
70 | }
71 | }
72 |
--------------------------------------------------------------------------------
/src/utils/arrow/batch_adapter.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use datafusion::arrow::array::new_null_array;
20 | use datafusion::arrow::array::ArrayRef;
21 | use datafusion::arrow::datatypes::Schema;
22 | use datafusion::arrow::record_batch::RecordBatch;
23 |
24 | use std::sync::Arc;
25 |
26 | // This function takes a new event's record batch and the
27 | // current schema of the log stream. It returns a new record
28 | // with nulls added to the fields that don't exist
29 | // in the record batch (i.e. the event) but are present in the
30 | // log stream schema.
31 | // This is necessary because all the record batches in a log
32 | // stream need to have all the fields.
33 | pub fn adapt_batch(table_schema: &Schema, batch: &RecordBatch) -> RecordBatch {
34 | let batch_schema = &*batch.schema();
35 | let batch_cols = batch.columns().to_vec();
36 |
37 | let mut cols: Vec = Vec::with_capacity(table_schema.fields().len());
38 | for table_field in table_schema.fields() {
39 | if let Some((batch_idx, _)) = batch_schema.column_with_name(table_field.name().as_str()) {
40 | cols.push(Arc::clone(&batch_cols[batch_idx]));
41 | } else {
42 | cols.push(new_null_array(table_field.data_type(), batch.num_rows()))
43 | }
44 | }
45 |
46 | let merged_schema = Arc::new(table_schema.clone());
47 | RecordBatch::try_new(merged_schema, cols).unwrap()
48 | }
49 |
--------------------------------------------------------------------------------
/src/utils/header_parsing.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use actix_web::{HttpResponse, ResponseError};
20 |
21 | #[derive(Debug, thiserror::Error)]
22 | pub enum ParseHeaderError {
23 | #[error("Too many headers received. Limit is of 5 headers")]
24 | MaxHeadersLimitExceeded,
25 | #[error("A value passed in header can't be formatted to plain visible ASCII")]
26 | InvalidValue,
27 | #[error("Invalid Key was passed which terminated just after the end of prefix")]
28 | Emptykey,
29 | #[error("A key passed in header contains reserved char {0}")]
30 | SeperatorInKey(char),
31 | #[error("A value passed in header contains reserved char {0}")]
32 | SeperatorInValue(char),
33 | #[error("Stream name not found in header [x-p-stream]")]
34 | MissingStreamName,
35 | #[error("Log source not found in header [x-p-log-source]")]
36 | MissingLogSource,
37 | }
38 |
39 | impl ResponseError for ParseHeaderError {
40 | fn status_code(&self) -> http::StatusCode {
41 | http::StatusCode::BAD_REQUEST
42 | }
43 |
44 | fn error_response(&self) -> HttpResponse {
45 | HttpResponse::build(self.status_code()).body(self.to_string())
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/src/utils/human_size.rs:
--------------------------------------------------------------------------------
1 | use std::str::FromStr;
2 |
3 | use human_size::{Any, SpecificSize};
4 | use serde::{de, Deserialize, Deserializer, Serializer};
5 |
6 | #[derive(Debug, thiserror::Error)]
7 | enum ParsingError {
8 | #[error("Expected 'X' | 'X Bytes', but error: {0}")]
9 | Int(#[from] std::num::ParseIntError),
10 | #[error("Could not parse given string as human size, erro: {0}")]
11 | HumanSize(#[from] human_size::ParsingError),
12 | }
13 |
14 | // Function to convert human-readable size to bytes (already provided)
15 | // NOTE: consider number values as byte count, e.g. "1234" is 1234 bytes.
16 | fn human_size_to_bytes(s: &str) -> Result {
17 | let s = s.trim();
18 | if let Some(s) = s.strip_suffix("Bytes") {
19 | let size: u64 = s.trim().parse()?;
20 | return Ok(size);
21 | } else if let Ok(size) = s.parse() {
22 | return Ok(size);
23 | }
24 |
25 | fn parse_and_map(s: &str) -> Result {
26 | SpecificSize::::from_str(s).map(|x| x.to_bytes())
27 | }
28 | let size = parse_and_map::(s)?;
29 |
30 | Ok(size)
31 | }
32 |
33 | // Function to convert bytes to human-readable size (already provided)
34 | pub fn bytes_to_human_size(bytes: u64) -> String {
35 | const KIB: u64 = 1024;
36 | const MIB: u64 = KIB * 1024;
37 | const GIB: u64 = MIB * 1024;
38 | const TIB: u64 = GIB * 1024;
39 | const PIB: u64 = TIB * 1024;
40 |
41 | if bytes < KIB {
42 | format!("{} B", bytes)
43 | } else if bytes < MIB {
44 | format!("{:.2} KB", bytes as f64 / KIB as f64)
45 | } else if bytes < GIB {
46 | format!("{:.2} MiB", bytes as f64 / MIB as f64)
47 | } else if bytes < TIB {
48 | format!("{:.2} GiB", bytes as f64 / GIB as f64)
49 | } else if bytes < PIB {
50 | format!("{:.2} TiB", bytes as f64 / TIB as f64)
51 | } else {
52 | format!("{:.2} PiB", bytes as f64 / PIB as f64)
53 | }
54 | }
55 |
56 | pub fn serialize(bytes: &u64, serializer: S) -> Result
57 | where
58 | S: Serializer,
59 | {
60 | // let human_readable = bytes_to_human_size(*bytes);
61 | // NOTE: frontend expects the size in bytes
62 | let human_readable = format!("{bytes} Bytes");
63 | serializer.serialize_str(&human_readable)
64 | }
65 |
66 | pub fn deserialize<'de, D>(deserializer: D) -> Result
67 | where
68 | D: Deserializer<'de>,
69 | {
70 | let s = String::deserialize(deserializer)?;
71 | human_size_to_bytes(&s).map_err(de::Error::custom)
72 | }
73 |
74 | #[cfg(test)]
75 | mod tests {
76 | use super::*;
77 |
78 | #[test]
79 | fn parse_numeric_input_without_unit() {
80 | assert_eq!(human_size_to_bytes("1234").unwrap(), 1234);
81 | }
82 |
83 | #[test]
84 | fn parse_bytes_string_to_bytes() {
85 | assert_eq!(human_size_to_bytes("1234 Bytes").unwrap(), 1234);
86 | }
87 |
88 | #[test]
89 | fn handle_empty_string_input() {
90 | assert!(matches!(
91 | human_size_to_bytes(""),
92 | Err(ParsingError::HumanSize(_))
93 | ));
94 | }
95 |
96 | #[test]
97 | fn handle_byte_string_input_without_value() {
98 | assert!(matches!(
99 | human_size_to_bytes("Bytes"),
100 | Err(ParsingError::Int(_))
101 | ));
102 | }
103 |
104 | #[test]
105 | fn convert_mebibyte_string_to_bytes() {
106 | assert_eq!(human_size_to_bytes("1 MiB").unwrap(), 1048576);
107 | }
108 |
109 | #[test]
110 | fn parse_gigabyte_string_input() {
111 | assert_eq!(human_size_to_bytes("1 GB").unwrap(), 1_000_000_000);
112 | }
113 | }
114 |
--------------------------------------------------------------------------------
/src/utils/uid.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use ulid::Ulid;
20 |
21 | pub type Uid = Ulid;
22 |
23 | pub fn gen() -> Ulid {
24 | Ulid::new()
25 | }
26 |
--------------------------------------------------------------------------------
/src/utils/update.rs:
--------------------------------------------------------------------------------
1 | /*
2 | * Parseable Server (C) 2022 - 2024 Parseable, Inc.
3 | *
4 | * This program is free software: you can redistribute it and/or modify
5 | * it under the terms of the GNU Affero General Public License as
6 | * published by the Free Software Foundation, either version 3 of the
7 | * License, or (at your option) any later version.
8 | *
9 | * This program is distributed in the hope that it will be useful,
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | * GNU Affero General Public License for more details.
13 | *
14 | * You should have received a copy of the GNU Affero General Public License
15 | * along with this program. If not, see .
16 | *
17 | */
18 |
19 | use std::time::Duration;
20 |
21 | use anyhow::anyhow;
22 | use chrono::{DateTime, Utc};
23 |
24 | use crate::about;
25 |
26 | use super::uid;
27 |
28 | #[derive(Debug, Clone)]
29 | pub struct LatestRelease {
30 | pub version: semver::Version,
31 | pub date: DateTime,
32 | }
33 |
34 | pub async fn get_latest(deployment_id: &uid::Uid) -> Result {
35 | let agent = reqwest::ClientBuilder::new()
36 | .user_agent(about::user_agent(deployment_id))
37 | .timeout(Duration::from_secs(8))
38 | .build()
39 | .expect("client can be built on this system");
40 | let json: serde_json::Value = agent
41 | .get("https://download.parseable.io/latest-version")
42 | .send()
43 | .await?
44 | .json()
45 | .await?;
46 | let version = json["tag_name"]
47 | .as_str()
48 | .and_then(|ver| ver.strip_prefix('v'))
49 | .and_then(|ver| semver::Version::parse(ver).ok())
50 | .ok_or_else(|| anyhow!("Failed parsing version"))?;
51 | let date = json["published_at"]
52 | .as_str()
53 | .ok_or_else(|| anyhow!("Failed parsing published date"))?;
54 |
55 | let date = chrono::DateTime::parse_from_rfc3339(date)
56 | .expect("date-time from github is in rfc3339 format")
57 | .into();
58 |
59 | Ok(LatestRelease { version, date })
60 | }
61 |
--------------------------------------------------------------------------------
/systemd/README.md:
--------------------------------------------------------------------------------
1 | # Linux Systemd Service
2 |
3 | Refer the Systemd setup [documentation ↗︎](https://www.parseable.io/docs/installation/systemd)
4 |
--------------------------------------------------------------------------------
/systemd/parseable.local.service:
--------------------------------------------------------------------------------
1 | [Unit]
2 | Description=Parseable
3 | Wants=network-online.target
4 | After=network-online.target
5 | AssertFileIsExecutable=/usr/local/bin/parseable
6 |
7 | [Service]
8 | WorkingDirectory=/usr/local/
9 |
10 | User=parseable-user
11 | Group=parseable-user
12 | ProtectProc=invisible
13 |
14 | EnvironmentFile=/etc/default/parseable
15 | ExecStart=/usr/local/bin/parseable local-store
16 |
17 | # Let systemd restart this service always
18 | Restart=always
19 |
20 | # Specifies the maximum file descriptor number that can be opened by this process
21 | LimitNOFILE=1048576
22 |
23 | # Specifies the maximum number of threads this process can create
24 | TasksMax=infinity
25 |
26 | # Disable timeout logic and wait until process is stopped
27 | TimeoutStopSec=infinity
28 | SendSIGKILL=no
29 |
30 | [Install]
31 | WantedBy=multi-user.target
32 |
33 | # Built for ${project.name}-${project.version} (${project.name})
34 |
--------------------------------------------------------------------------------
/systemd/parseable.s3.service:
--------------------------------------------------------------------------------
1 | [Unit]
2 | Description=Parseable
3 | Wants=network-online.target
4 | After=network-online.target
5 | AssertFileIsExecutable=/usr/local/bin/parseable
6 |
7 | [Service]
8 | WorkingDirectory=/usr/local/
9 |
10 | User=parseable-user
11 | Group=parseable-user
12 | ProtectProc=invisible
13 |
14 | EnvironmentFile=/etc/default/parseable
15 | ExecStart=/usr/local/bin/parseable s3-store
16 |
17 | # Let systemd restart this service always
18 | Restart=always
19 |
20 | # Specifies the maximum file descriptor number that can be opened by this process
21 | LimitNOFILE=1048576
22 |
23 | # Specifies the maximum number of threads this process can create
24 | TasksMax=infinity
25 |
26 | # Disable timeout logic and wait until process is stopped
27 | TimeoutStopSec=infinity
28 | SendSIGKILL=no
29 |
30 | [Install]
31 | WantedBy=multi-user.target
32 |
33 | # Built for ${project.name}-${project.version} (${project.name})
34 |
--------------------------------------------------------------------------------