├── .nuget └── icon.png ├── .vscode ├── settings.json ├── tasks.json └── launch.json ├── src └── Knet.Kudu.Client │ ├── Util │ ├── CharUtil.cs │ ├── ISystemClock.cs │ ├── SystemClock.cs │ ├── Murmur2.cs │ ├── HybridTimeUtil.cs │ ├── FastHash.cs │ ├── EndpointParser.cs │ └── EpochTime.cs │ ├── ComparisonOp.cs │ ├── MasterLeaderInfo.cs │ ├── RangeSchema.cs │ ├── ReplicaRole.cs │ ├── Requests │ ├── KuduMasterRpc.cs │ ├── KuduTxnRpc.cs │ ├── DeleteTableRequest.cs │ ├── IsAlterTableDoneRequest.cs │ ├── AbortTransactionRequest.cs │ ├── GetTableLocationsRequest.cs │ ├── IsCreateTableDoneRequest.cs │ ├── BeginTransactionRequest.cs │ ├── CommitTransactionRequest.cs │ ├── GetTableStatisticsRequest.cs │ ├── ListTabletServersRequest.cs │ ├── GetTransactionStateRequest.cs │ ├── KeepTransactionAliveRequest.cs │ ├── CreateTableRequest.cs │ ├── ListTablesRequest.cs │ ├── KuduTabletRpc.cs │ ├── ConnectToMasterRequest.cs │ ├── GetTableSchemaRequest.cs │ ├── KeepAliveRequest.cs │ ├── AlterTableRequest.cs │ ├── WriteRequest.cs │ ├── SplitKeyRangeRequest.cs │ └── KuduRpc.cs │ ├── Scanner │ └── PartitionKeyRange.cs │ ├── AlterTableResponse.cs │ ├── WriteResponse.cs │ ├── Tablet │ ├── KeyEncoder.netstandard.cs │ ├── FindTabletResult.cs │ ├── TableLocationEntry.cs │ ├── KeyRange.cs │ ├── RemoteTabletExtensions.cs │ ├── KeyEncoder.sse.cs │ └── RemoteTablet.cs │ ├── Exceptions │ ├── RpcRemoteException.cs │ ├── FaultTolerantScannerExpiredException.cs │ ├── InvalidAuthzTokenException.cs │ ├── InvalidAuthnTokenException.cs │ ├── RecoverableException.cs │ ├── NonRecoverableException.cs │ ├── KuduException.cs │ ├── NoLeaderFoundException.cs │ ├── KuduWriteException.cs │ └── NonCoveredRangeException.cs │ ├── KuduTransactionSerializationOptions.cs │ ├── Protocol │ ├── SidecarOffset.cs │ ├── ParseStep.cs │ └── KuduMessage.cs │ ├── RangePartitionBound.cs │ ├── HashBucketSchema.cs │ ├── Assembly.cs │ ├── Internal │ ├── AvlTreeExtensions.cs │ ├── ThreadSafeRandom.cs │ ├── ArrayPoolBuffer.cs │ ├── KuduTypeFlags.cs │ ├── TaskCompletionSource.cs │ ├── PeriodicTimer.cs │ ├── SequenceReaderExtensions.cs │ ├── AuthzTokenCache.cs │ ├── SecurityUtil.cs │ ├── KuduTypeValidation.cs │ └── Netstandard2Extensions.cs │ ├── SessionExceptionContext.cs │ ├── CompressionType.cs │ ├── TableInfo.cs │ ├── EncodingType.cs │ ├── TabletServerState.cs │ ├── Connection │ ├── IKuduConnectionFactory.cs │ ├── HostAndPort.cs │ ├── ServerInfo.cs │ ├── RequestTracker.cs │ └── ISecurityContext.cs │ ├── EncryptionPolicy.cs │ ├── KuduReplica.cs │ ├── KuduTableStatistics.cs │ ├── KuduType.cs │ ├── Negotiate │ └── SaslPlain.cs │ ├── Mapper │ ├── ResultSetMapper.cs │ ├── DelegateCache.cs │ └── ColumnNameMatcher.cs │ ├── ReplicaSelection.cs │ ├── PredicateType.cs │ ├── KuduScannerBuilder.cs │ ├── PartialRowOperation.cs │ ├── Protos │ └── kudu │ │ ├── util │ │ ├── hash.proto │ │ ├── compression │ │ │ └── compression.proto │ │ ├── block_bloom_filter.proto │ │ └── pb_util.proto │ │ └── consensus │ │ ├── opid.proto │ │ └── replica_management.proto │ ├── MasterManager.cs │ ├── PartitionSchema.cs │ ├── KuduClientOptions.cs │ ├── HiveMetastoreConfig.cs │ ├── KuduScannerExtensions.cs │ ├── KuduSessionOptions.cs │ ├── KuduBloomFilterBuilder.cs │ ├── Logging │ └── LoggerHelperExtensions.cs │ ├── KuduOperation.cs │ ├── TabletServerInfo.cs │ ├── RowOperation.cs │ ├── ResourceMetrics.cs │ ├── IKuduSession.cs │ ├── ColumnTypeAttributes.cs │ ├── Knet.Kudu.Client.csproj │ ├── ReadMode.cs │ ├── KuduPartitioner.cs │ └── ExternalConsistencyMode.cs ├── examples └── InsertLoadgen │ ├── InsertLoadgen.csproj │ └── Program.cs ├── test ├── Knet.Kudu.Client.Tests │ ├── MurmurHashTests.cs │ ├── Knet.Kudu.Client.Tests.csproj │ ├── FastHashTests.cs │ ├── HostAndPortTests.cs │ ├── DecimalUtilTests.cs │ ├── EndpointParserTests.cs │ ├── EpochTimeTests.cs │ ├── KuduStatusTests.cs │ ├── TableBuilderTests.cs │ ├── PartitionTests.cs │ └── RequestTrackerTests.cs └── Knet.Kudu.Client.FunctionalTests │ ├── Knet.Kudu.Client.FunctionalTests.csproj │ ├── MiniCluster │ ├── MiniKuduClusterTestAttribute.cs │ └── MiniKuduClusterBuilder.cs │ ├── DeleteTableTests.cs │ ├── Util │ └── TestExtensions.cs │ ├── TimeoutTests.cs │ ├── LeaderFailoverTests.cs │ ├── HandleTooBusyTests.cs │ ├── SessionTests.cs │ ├── MasterFailoverTests.cs │ ├── ClientTests.cs │ └── MultipleLeaderFailoverTests.cs ├── .editorconfig ├── .github └── workflows │ └── ci.yml └── README.md /.nuget/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xqrzd/kudu-client-net/HEAD/.nuget/icon.png -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "dotnet.defaultSolution": "kudu-client-net.sln", 3 | "files.exclude": { 4 | "**/bin": true, 5 | "**/obj": true 6 | } 7 | } -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Util/CharUtil.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client.Util; 2 | 3 | public static class CharUtil 4 | { 5 | public const int MinVarcharLength = 1; 6 | public const int MaxVarcharLength = 65535; 7 | } 8 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/ComparisonOp.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client; 2 | 3 | /// 4 | /// The comparison operator of a predicate. 5 | /// 6 | public enum ComparisonOp 7 | { 8 | Greater, 9 | GreaterEqual, 10 | Equal, 11 | Less, 12 | LessEqual 13 | } 14 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/MasterLeaderInfo.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Connection; 2 | 3 | namespace Knet.Kudu.Client; 4 | 5 | internal sealed record MasterLeaderInfo( 6 | string Location, 7 | string ClusterId, 8 | ServerInfo ServerInfo, 9 | HiveMetastoreConfig? HiveMetastoreConfig); 10 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/RangeSchema.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace Knet.Kudu.Client; 4 | 5 | public class RangeSchema 6 | { 7 | public List ColumnIds { get; } 8 | 9 | public RangeSchema(List columnIds) 10 | { 11 | ColumnIds = columnIds; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/ReplicaRole.cs: -------------------------------------------------------------------------------- 1 | using static Knet.Kudu.Client.Protobuf.Consensus.RaftPeerPB.Types; 2 | 3 | namespace Knet.Kudu.Client; 4 | 5 | public enum ReplicaRole 6 | { 7 | Follower = Role.Follower, 8 | Leader = Role.Leader, 9 | Learner = Role.Learner, 10 | NonParticipant = Role.NonParticipant 11 | } 12 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/KuduMasterRpc.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Protobuf.Master; 2 | 3 | namespace Knet.Kudu.Client.Requests; 4 | 5 | internal abstract class KuduMasterRpc : KuduRpc 6 | { 7 | public MasterErrorPB? Error { get; protected set; } 8 | 9 | public KuduMasterRpc() 10 | { 11 | ServiceName = MasterServiceName; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Scanner/PartitionKeyRange.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client.Scanner; 2 | 3 | public readonly struct PartitionKeyRange 4 | { 5 | public byte[] Lower { get; } 6 | 7 | public byte[] Upper { get; } 8 | 9 | public PartitionKeyRange(byte[] lower, byte[] upper) 10 | { 11 | Lower = lower; 12 | Upper = upper; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/KuduTxnRpc.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Protobuf.Transactions; 2 | 3 | namespace Knet.Kudu.Client.Requests; 4 | 5 | internal abstract class KuduTxnRpc : KuduRpc 6 | { 7 | public TxnManagerErrorPB? Error { get; protected set; } 8 | 9 | public KuduTxnRpc() 10 | { 11 | ServiceName = TxnManagerServiceName; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/AlterTableResponse.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client; 2 | 3 | public class AlterTableResponse 4 | { 5 | public string TableId { get; } 6 | 7 | public uint SchemaVersion { get; } 8 | 9 | public AlterTableResponse(string tableId, uint schemaVersion) 10 | { 11 | TableId = tableId; 12 | SchemaVersion = schemaVersion; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/WriteResponse.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client; 2 | 3 | public class WriteResponse 4 | { 5 | /// 6 | /// The HybridTime-encoded write timestamp. 7 | /// 8 | public long WriteTimestampRaw { get; } 9 | 10 | public WriteResponse(long writeTimestampRaw) 11 | { 12 | WriteTimestampRaw = writeTimestampRaw; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Util/ISystemClock.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client.Util; 2 | 3 | public interface ISystemClock 4 | { 5 | /// 6 | /// Retrieve the current milliseconds. This value should only 7 | /// be used to measure how much time has passed relative to 8 | /// another call to this property. 9 | /// 10 | long CurrentMilliseconds { get; } 11 | } 12 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Tablet/KeyEncoder.netstandard.cs: -------------------------------------------------------------------------------- 1 | #if !NETCOREAPP3_1_OR_GREATER 2 | using System; 3 | 4 | namespace Knet.Kudu.Client.Tablet; 5 | 6 | public static partial class KeyEncoder 7 | { 8 | private static int EncodeBinary( 9 | ReadOnlySpan source, Span destination) 10 | { 11 | return EncodeBinaryStandard(source, destination); 12 | } 13 | } 14 | #endif 15 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Exceptions/RpcRemoteException.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Protobuf.Rpc; 2 | 3 | namespace Knet.Kudu.Client.Exceptions; 4 | 5 | public class RpcRemoteException : NonRecoverableException 6 | { 7 | public ErrorStatusPB ErrorPb { get; } 8 | 9 | public RpcRemoteException(KuduStatus status, ErrorStatusPB errorPb) 10 | : base(status) 11 | { 12 | ErrorPb = errorPb; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Exceptions/FaultTolerantScannerExpiredException.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client.Exceptions; 2 | 3 | /// 4 | /// A scanner expired exception only used for fault tolerant scanner. 5 | /// 6 | public class FaultTolerantScannerExpiredException : NonRecoverableException 7 | { 8 | public FaultTolerantScannerExpiredException(KuduStatus status) 9 | : base(status) 10 | { 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/KuduTransactionSerializationOptions.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client; 2 | 3 | public record KuduTransactionSerializationOptions 4 | { 5 | /// 6 | /// Whether the created from these 7 | /// options will send keepalive messages to avoid automatic rollback 8 | /// of the underlying transaction. 9 | /// 10 | public bool EnableKeepalive { get; init; } 11 | } 12 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Protocol/SidecarOffset.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client.Protocol; 2 | 3 | public readonly struct SidecarOffset 4 | { 5 | public readonly int Start; 6 | 7 | public readonly int Length; 8 | 9 | public SidecarOffset(int start, int length) 10 | { 11 | Start = start; 12 | Length = length; 13 | } 14 | 15 | public override string ToString() => 16 | $"Start: {Start}, Length: {Length}"; 17 | } 18 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/RangePartitionBound.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client; 2 | 3 | /// 4 | /// Specifies whether a range partition bound is inclusive or exclusive. 5 | /// 6 | public enum RangePartitionBound 7 | { 8 | /// 9 | /// An exclusive range partition bound. 10 | /// 11 | Exclusive, 12 | /// 13 | /// An inclusive range partition bound. 14 | /// 15 | Inclusive 16 | } 17 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Util/SystemClock.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | 4 | namespace Knet.Kudu.Client.Util; 5 | 6 | public sealed class SystemClock : ISystemClock 7 | { 8 | #if NETCOREAPP3_1_OR_GREATER 9 | public long CurrentMilliseconds => Environment.TickCount64; 10 | #else 11 | private readonly Stopwatch _stopwatch = Stopwatch.StartNew(); 12 | 13 | public long CurrentMilliseconds => _stopwatch.ElapsedMilliseconds; 14 | #endif 15 | } 16 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/HashBucketSchema.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace Knet.Kudu.Client; 4 | 5 | public class HashBucketSchema 6 | { 7 | public List ColumnIds { get; } 8 | 9 | public int NumBuckets { get; } 10 | 11 | public uint Seed { get; } 12 | 13 | public HashBucketSchema(List columnIds, int numBuckets, uint seed) 14 | { 15 | ColumnIds = columnIds; 16 | NumBuckets = numBuckets; 17 | Seed = seed; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /examples/InsertLoadgen/InsertLoadgen.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net7.0 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Assembly.cs: -------------------------------------------------------------------------------- 1 | using System.ComponentModel; 2 | using System.Runtime.CompilerServices; 3 | 4 | [assembly: InternalsVisibleTo("Knet.Kudu.Client.FunctionalTests")] 5 | [assembly: InternalsVisibleTo("Knet.Kudu.Client.Tests")] 6 | 7 | #if NET5_0_OR_GREATER 8 | 9 | [module: SkipLocalsInit] 10 | 11 | #else 12 | 13 | namespace System.Runtime.CompilerServices 14 | { 15 | [EditorBrowsable(EditorBrowsableState.Never)] 16 | internal static class IsExternalInit { } 17 | } 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Exceptions/InvalidAuthzTokenException.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client.Exceptions; 2 | 3 | /// 4 | /// Receiving this exception means the authorization token used to make a 5 | /// request is no longer valid and a new one is needed to make requests that 6 | /// access data. 7 | /// 8 | public class InvalidAuthzTokenException : RecoverableException 9 | { 10 | public InvalidAuthzTokenException(KuduStatus status) 11 | : base(status) 12 | { 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Internal/AvlTreeExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Knet.Kudu.Client.Tablet; 3 | 4 | namespace Knet.Kudu.Client.Internal; 5 | 6 | internal static class AvlTreeExtensions 7 | { 8 | public static TableLocationEntry? FloorEntry( 9 | this AvlTree avlTree, ReadOnlySpan partitionKey) 10 | { 11 | avlTree.SearchLeftRight( 12 | partitionKey, 13 | out TableLocationEntry left, 14 | out _); 15 | 16 | return left; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/SessionExceptionContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace Knet.Kudu.Client; 5 | 6 | public sealed class SessionExceptionContext 7 | { 8 | public Exception Exception { get; } 9 | 10 | public IReadOnlyList Rows { get; } 11 | 12 | public SessionExceptionContext( 13 | Exception exception, 14 | IReadOnlyList rows) 15 | { 16 | Exception = exception; 17 | Rows = rows; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Exceptions/InvalidAuthnTokenException.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client.Exceptions; 2 | 3 | /// 4 | /// Receiving this exception means the current authentication token is no 5 | /// longer valid and a new one is needed to establish connections to the 6 | /// Kudu servers for sending RPCs. 7 | /// 8 | public class InvalidAuthnTokenException : RecoverableException 9 | { 10 | public InvalidAuthnTokenException(KuduStatus status) 11 | : base(status) 12 | { 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Exceptions/RecoverableException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Knet.Kudu.Client.Exceptions; 4 | 5 | /// 6 | /// An exception that's possible to retry. 7 | /// 8 | public class RecoverableException : KuduException 9 | { 10 | public RecoverableException(KuduStatus status) 11 | : base(status) 12 | { 13 | } 14 | 15 | public RecoverableException(KuduStatus status, Exception? innerException) 16 | : base(status, innerException) 17 | { 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Exceptions/NonRecoverableException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Knet.Kudu.Client.Exceptions; 4 | 5 | /// 6 | /// An exception that cannot be retried. 7 | /// 8 | public class NonRecoverableException : KuduException 9 | { 10 | public NonRecoverableException(KuduStatus status) 11 | : base(status) 12 | { 13 | } 14 | 15 | public NonRecoverableException(KuduStatus status, Exception? innerException) 16 | : base(status, innerException) 17 | { 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Exceptions/KuduException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Knet.Kudu.Client.Exceptions; 4 | 5 | public abstract class KuduException : Exception 6 | { 7 | public KuduStatus Status { get; } 8 | 9 | public KuduException(KuduStatus status) 10 | : base(status.Message) 11 | { 12 | Status = status; 13 | } 14 | 15 | public KuduException(KuduStatus status, Exception? innerException) 16 | : base(status.Message, innerException) 17 | { 18 | Status = status; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/CompressionType.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client; 2 | 3 | /// 4 | /// Supported compression for Kudu columns. 5 | /// See https://kudu.apache.org/docs/schema_design.html#compression 6 | /// 7 | public enum CompressionType 8 | { 9 | DefaultCompression = Protobuf.CompressionType.DefaultCompression, 10 | NoCompression = Protobuf.CompressionType.NoCompression, 11 | Snappy = Protobuf.CompressionType.Snappy, 12 | Lz4 = Protobuf.CompressionType.Lz4, 13 | Zlib = Protobuf.CompressionType.Zlib 14 | } 15 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Exceptions/NoLeaderFoundException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Knet.Kudu.Client.Exceptions; 4 | 5 | /// 6 | /// Indicates that the request failed because we couldn't find a leader. 7 | /// It is retried as long as the original call hasn't timed out. 8 | /// 9 | public class NoLeaderFoundException : RecoverableException 10 | { 11 | public NoLeaderFoundException(string message, Exception? innerException) 12 | : base(KuduStatus.NetworkError(message), innerException) 13 | { 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/TableInfo.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client; 2 | 3 | public class TableInfo 4 | { 5 | /// 6 | /// The table name. 7 | /// 8 | public string TableName { get; } 9 | 10 | /// 11 | /// The table Id. 12 | /// 13 | public string TableId { get; } 14 | 15 | public TableInfo(string tableName, string tableId) 16 | { 17 | TableName = tableName; 18 | TableId = tableId; 19 | } 20 | 21 | public override string ToString() => TableName; 22 | } 23 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Internal/ThreadSafeRandom.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | 4 | namespace Knet.Kudu.Client.Internal; 5 | 6 | internal static class ThreadSafeRandom 7 | { 8 | #if NET6_0_OR_GREATER 9 | public static Random Instance => Random.Shared; 10 | #else 11 | private static int _seed = Environment.TickCount; 12 | 13 | private static readonly ThreadLocal _random = 14 | new(() => new Random(Interlocked.Increment(ref _seed))); 15 | 16 | public static Random Instance => _random.Value!; 17 | #endif 18 | } 19 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/EncodingType.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client; 2 | 3 | /// 4 | /// Supported encoding for Kudu columns. 5 | /// See https://kudu.apache.org/docs/schema_design.html#encoding 6 | /// 7 | public enum EncodingType 8 | { 9 | AutoEncoding = Protobuf.EncodingType.AutoEncoding, 10 | PlainEncoding = Protobuf.EncodingType.PlainEncoding, 11 | PrefixEncoding = Protobuf.EncodingType.PrefixEncoding, 12 | Rle = Protobuf.EncodingType.Rle, 13 | DictEncoding = Protobuf.EncodingType.DictEncoding, 14 | BitShuffle = Protobuf.EncodingType.BitShuffle 15 | } 16 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "label": "build", 6 | "command": "dotnet", 7 | "type": "process", 8 | "args": [ 9 | "build", 10 | "${workspaceFolder}/kudu-client-net.sln", 11 | "/property:GenerateFullPaths=true", 12 | "/consoleloggerparameters:NoSummary" 13 | ], 14 | "problemMatcher": "$msCompile", 15 | "group": { 16 | "kind": "build", 17 | "isDefault": true 18 | } 19 | } 20 | ] 21 | } -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Internal/ArrayPoolBuffer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Buffers; 3 | 4 | namespace Knet.Kudu.Client.Internal; 5 | 6 | internal sealed class ArrayPoolBuffer : IDisposable 7 | { 8 | public T[] Buffer { get; private set; } 9 | 10 | public ArrayPoolBuffer(int minimumLength) 11 | { 12 | Buffer = ArrayPool.Shared.Rent(minimumLength); 13 | } 14 | 15 | public void Dispose() 16 | { 17 | var buffer = Buffer; 18 | if (buffer is not null) 19 | { 20 | Buffer = null!; 21 | ArrayPool.Shared.Return(buffer); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.Tests/MurmurHashTests.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Internal; 2 | using Knet.Kudu.Client.Util; 3 | using Xunit; 4 | 5 | namespace Knet.Kudu.Client.Tests; 6 | 7 | public class MurmurHashTests 8 | { 9 | [Theory] 10 | [InlineData("ab", 0, 7115271465109541368UL)] 11 | [InlineData("abcdefg", 0, 2601573339036254301UL)] 12 | [InlineData("quick brown fox", 42, 3575930248840144026UL)] 13 | public void TestMurmur2Hash64(string data, ulong seed, ulong expectedHash) 14 | { 15 | ulong hash = Murmur2.Hash64(data.ToUtf8ByteArray(), seed); 16 | Assert.Equal(expectedHash, hash); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/TabletServerState.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Protobuf.Master; 2 | 3 | namespace Knet.Kudu.Client; 4 | 5 | public enum TabletServerState 6 | { 7 | /// 8 | /// Default value for backwards compatibility. 9 | /// 10 | Unknown = TServerStatePB.UnknownState, 11 | /// 12 | /// No state for the tserver. 13 | /// 14 | None = TServerStatePB.None, 15 | /// 16 | /// New replicas are not added to the tserver, and failed replicas on 17 | /// the tserver are not re-replicated. 18 | /// 19 | MaintenanceMode = TServerStatePB.MaintenanceMode, 20 | } 21 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Connection/IKuduConnectionFactory.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | 5 | namespace Knet.Kudu.Client.Connection; 6 | 7 | public interface IKuduConnectionFactory 8 | { 9 | Task ConnectAsync( 10 | ServerInfo serverInfo, CancellationToken cancellationToken = default); 11 | 12 | Task> GetMasterServerInfoAsync( 13 | HostAndPort hostPort, CancellationToken cancellationToken = default); 14 | 15 | Task GetTabletServerInfoAsync( 16 | HostAndPort hostPort, string uuid, string? location, CancellationToken cancellationToken = default); 17 | } 18 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.Tests/Knet.Kudu.Client.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net7.0 5 | latest 6 | false 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Examples: InsertLoadgen", 6 | "type": "coreclr", 7 | "request": "launch", 8 | "preLaunchTask": "build", 9 | "program": "${workspaceFolder}/examples/InsertLoadgen/bin/Debug/net7.0/InsertLoadgen.dll", 10 | "args": [], 11 | "cwd": "${workspaceFolder}/examples/InsertLoadgen", 12 | "console": "internalConsole", 13 | "stopAtEntry": false 14 | }, 15 | { 16 | "name": ".NET Core Attach", 17 | "type": "coreclr", 18 | "request": "attach" 19 | } 20 | ] 21 | } -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/EncryptionPolicy.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client; 2 | 3 | public enum EncryptionPolicy 4 | { 5 | /// 6 | /// Optional, it uses encrypted connection if the server supports it, 7 | /// but it can connect to insecure servers too. 8 | /// 9 | Optional, 10 | /// 11 | /// Only connects to remote servers that support encryption, fails 12 | /// otherwise. It can connect to insecure servers only locally. 13 | /// 14 | RequiredRemote, 15 | /// 16 | /// Only connects to any server, including on the loopback interface, 17 | /// that support encryption, fails otherwise. 18 | /// 19 | Required 20 | } 21 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome:http://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | [*] 7 | indent_style = space 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | # Dotnet code style settings: 13 | [*.{cs,vb}] 14 | indent_size = 4 15 | # Sort using and Import directives with System.* appearing first 16 | dotnet_sort_system_directives_first = true 17 | 18 | # Namespace settings 19 | csharp_style_namespace_declarations = file_scoped 20 | 21 | [*.{xml,config,*proj,nuspec,props,resx,targets,yml}] 22 | indent_size = 2 23 | 24 | [*.json] 25 | indent_size = 2 26 | 27 | [*.cs] 28 | # Use range operator 29 | dotnet_diagnostic.IDE0057.severity = silent 30 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/KuduReplica.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Connection; 2 | 3 | namespace Knet.Kudu.Client; 4 | 5 | /// 6 | /// One of the replicas of the tablet. 7 | /// 8 | public class KuduReplica 9 | { 10 | public HostAndPort HostPort { get; } 11 | 12 | public ReplicaRole Role { get; } 13 | 14 | public string? DimensionLabel { get; } 15 | 16 | public KuduReplica(HostAndPort hostPort, ReplicaRole role, string? dimensionLabel) 17 | { 18 | HostPort = hostPort; 19 | Role = role; 20 | DimensionLabel = dimensionLabel; 21 | } 22 | 23 | public override string ToString() => 24 | $"Replica(host={HostPort}, role={Role}, dimensionLabel={DimensionLabel})"; 25 | } 26 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/KuduTableStatistics.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client; 2 | 3 | /// 4 | /// Represent statistics belongs to a specific kudu table. 5 | /// 6 | public class KuduTableStatistics 7 | { 8 | /// 9 | /// The table's on disk size in bytes, this statistic is pre-replication. 10 | /// 11 | public long OnDiskSize { get; } 12 | 13 | /// 14 | /// The table's live row count, this statistic is pre-replication. 15 | /// 16 | public long LiveRowCount { get; } 17 | 18 | public KuduTableStatistics(long onDiskSize, long liveRowCount) 19 | { 20 | OnDiskSize = onDiskSize; 21 | LiveRowCount = liveRowCount; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Protocol/ParseStep.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client.Protocol; 2 | 3 | internal enum ParseStep 4 | { 5 | /// 6 | /// Total message length (4 bytes). 7 | /// 8 | TotalMessageLength, 9 | /// 10 | /// RPC Header protobuf length (variable encoding). 11 | /// 12 | HeaderLength, 13 | /// 14 | /// RPC Header protobuf. 15 | /// 16 | Header, 17 | /// 18 | /// Main message length (variable encoding), 19 | /// including sidecars (if any). 20 | /// 21 | MainMessageLength, 22 | /// 23 | /// Main message protobuf, including any sidecars. 24 | /// 25 | MainMessage 26 | } 27 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Internal/KuduTypeFlags.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Knet.Kudu.Client.Internal; 4 | 5 | [Flags] 6 | internal enum KuduTypeFlags 7 | { 8 | Int8 = 1 << KuduType.Int8, 9 | Int16 = 1 << KuduType.Int16, 10 | Int32 = 1 << KuduType.Int32, 11 | Int64 = 1 << KuduType.Int64, 12 | String = 1 << KuduType.String, 13 | Bool = 1 << KuduType.Bool, 14 | Float = 1 << KuduType.Float, 15 | Double = 1 << KuduType.Double, 16 | Binary = 1 << KuduType.Binary, 17 | UnixtimeMicros = 1 << KuduType.UnixtimeMicros, 18 | Decimal32 = 1 << KuduType.Decimal32, 19 | Decimal64 = 1 << KuduType.Decimal64, 20 | Decimal128 = 1 << KuduType.Decimal128, 21 | Varchar = 1 << KuduType.Varchar, 22 | Date = 1 << KuduType.Date 23 | } 24 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/KuduType.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Protobuf; 2 | 3 | namespace Knet.Kudu.Client; 4 | 5 | /// 6 | /// Supported Kudu data types. 7 | /// See https://kudu.apache.org/docs/schema_design.html#column-design 8 | /// 9 | public enum KuduType 10 | { 11 | Int8 = DataType.Int8, 12 | Int16 = DataType.Int16, 13 | Int32 = DataType.Int32, 14 | Int64 = DataType.Int64, 15 | String = DataType.String, 16 | Bool = DataType.Bool, 17 | Float = DataType.Float, 18 | Double = DataType.Double, 19 | Binary = DataType.Binary, 20 | UnixtimeMicros = DataType.UnixtimeMicros, 21 | Decimal32 = DataType.Decimal32, 22 | Decimal64 = DataType.Decimal64, 23 | Decimal128 = DataType.Decimal128, 24 | Varchar = DataType.Varchar, 25 | Date = DataType.Date 26 | } 27 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Internal/TaskCompletionSource.cs: -------------------------------------------------------------------------------- 1 | #if !NET5_0_OR_GREATER 2 | 3 | using System.Threading.Tasks; 4 | 5 | namespace Knet.Kudu.Client.Internal; 6 | 7 | internal class TaskCompletionSource : TaskCompletionSource 8 | { 9 | public TaskCompletionSource() : base() { } 10 | 11 | public TaskCompletionSource(object state) : base(state) { } 12 | 13 | public TaskCompletionSource(TaskCreationOptions creationOptions) : base(creationOptions) { } 14 | 15 | public TaskCompletionSource(object state, TaskCreationOptions creationOptions) 16 | : base(state, creationOptions) { } 17 | 18 | public void SetResult() 19 | { 20 | SetResult(null); 21 | } 22 | 23 | public bool TrySetResult() 24 | { 25 | return TrySetResult(null); 26 | } 27 | } 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Negotiate/SaslPlain.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Net; 3 | using System.Text; 4 | using Knet.Kudu.Client.Internal; 5 | 6 | namespace Knet.Kudu.Client.Negotiate; 7 | 8 | public static class SaslPlain 9 | { 10 | public static byte[] CreateToken(NetworkCredential credentials) 11 | { 12 | var usernameLength = Encoding.UTF8.GetByteCount(credentials.UserName); 13 | var passwordLength = Encoding.UTF8.GetByteCount(credentials.Password); 14 | 15 | var token = new byte[usernameLength + passwordLength + 2]; 16 | var span = token.AsSpan(1); // Skip authorization identity. 17 | 18 | Encoding.UTF8.GetBytes(credentials.UserName, span); 19 | Encoding.UTF8.GetBytes(credentials.Password, span.Slice(usernameLength + 1)); 20 | 21 | return token; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Mapper/ResultSetMapper.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Knet.Kudu.Client.Mapper; 4 | 5 | internal sealed class ResultSetMapper 6 | { 7 | private readonly DelegateCache _cache = new(); 8 | 9 | public Func CreateDelegate(KuduSchema projectionSchema) 10 | { 11 | if (_cache.TryGetDelegate(typeof(T), projectionSchema, out var func)) 12 | { 13 | return (Func)func; 14 | } 15 | 16 | return CreateNewDelegate(projectionSchema); 17 | } 18 | 19 | private Func CreateNewDelegate(KuduSchema projectionSchema) 20 | { 21 | var func = MappingProfileFactory.Create(projectionSchema); 22 | _cache.AddDelegate(typeof(T), projectionSchema, func); 23 | 24 | return func; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/ReplicaSelection.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client; 2 | 3 | /// 4 | /// Policy with which to choose amongst multiple replicas. 5 | /// 6 | public enum ReplicaSelection 7 | { 8 | /// 9 | /// Select the LEADER replica. 10 | /// 11 | LeaderOnly = Protobuf.ReplicaSelection.LeaderOnly, 12 | /// 13 | /// Select the closest replica to the client. Replicas are classified 14 | /// from closest to furthest as follows: 15 | /// 16 | /// Local replicas 17 | /// 18 | /// Replicas whose tablet server has the same location as the client 19 | /// 20 | /// All other replicas 21 | /// 22 | /// 23 | ClosestReplica = Protobuf.ReplicaSelection.ClosestReplica 24 | } 25 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/DeleteTableRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf; 3 | using Knet.Kudu.Client.Protobuf.Master; 4 | using Knet.Kudu.Client.Protocol; 5 | 6 | namespace Knet.Kudu.Client.Requests; 7 | 8 | internal sealed class DeleteTableRequest : KuduMasterRpc 9 | { 10 | private readonly DeleteTableRequestPB _request; 11 | 12 | public DeleteTableRequest(DeleteTableRequestPB request) 13 | { 14 | MethodName = "DeleteTable"; 15 | _request = request; 16 | } 17 | 18 | public override int CalculateSize() => _request.CalculateSize(); 19 | 20 | public override void WriteTo(IBufferWriter output) => _request.WriteTo(output); 21 | 22 | public override void ParseResponse(KuduMessage message) 23 | { 24 | Output = DeleteTableResponsePB.Parser.ParseFrom(message.MessageProtobuf); 25 | Error = Output.Error; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/IsAlterTableDoneRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf; 3 | using Knet.Kudu.Client.Protobuf.Master; 4 | using Knet.Kudu.Client.Protocol; 5 | 6 | namespace Knet.Kudu.Client.Requests; 7 | 8 | internal sealed class IsAlterTableDoneRequest : KuduMasterRpc 9 | { 10 | private readonly IsAlterTableDoneRequestPB _request; 11 | 12 | public IsAlterTableDoneRequest(IsAlterTableDoneRequestPB request) 13 | { 14 | MethodName = "IsAlterTableDone"; 15 | _request = request; 16 | } 17 | 18 | public override int CalculateSize() => _request.CalculateSize(); 19 | 20 | public override void WriteTo(IBufferWriter output) => _request.WriteTo(output); 21 | 22 | public override void ParseResponse(KuduMessage message) 23 | { 24 | Output = IsAlterTableDoneResponsePB.Parser.ParseFrom(message.MessageProtobuf); 25 | Error = Output.Error; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/AbortTransactionRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf; 3 | using Knet.Kudu.Client.Protobuf.Transactions; 4 | using Knet.Kudu.Client.Protocol; 5 | 6 | namespace Knet.Kudu.Client.Requests; 7 | 8 | internal sealed class AbortTransactionRequest : KuduTxnRpc 9 | { 10 | private readonly AbortTransactionRequestPB _request; 11 | 12 | public AbortTransactionRequest(AbortTransactionRequestPB request) 13 | { 14 | MethodName = "AbortTransaction"; 15 | _request = request; 16 | } 17 | 18 | public override int CalculateSize() => _request.CalculateSize(); 19 | 20 | public override void WriteTo(IBufferWriter output) => _request.WriteTo(output); 21 | 22 | public override void ParseResponse(KuduMessage message) 23 | { 24 | Output = AbortTransactionResponsePB.Parser.ParseFrom(message.MessageProtobuf); 25 | Error = Output.Error; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/GetTableLocationsRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf; 3 | using Knet.Kudu.Client.Protobuf.Master; 4 | using Knet.Kudu.Client.Protocol; 5 | 6 | namespace Knet.Kudu.Client.Requests; 7 | 8 | internal sealed class GetTableLocationsRequest : KuduMasterRpc 9 | { 10 | private readonly GetTableLocationsRequestPB _request; 11 | 12 | public GetTableLocationsRequest(GetTableLocationsRequestPB request) 13 | { 14 | MethodName = "GetTableLocations"; 15 | _request = request; 16 | } 17 | 18 | public override int CalculateSize() => _request.CalculateSize(); 19 | 20 | public override void WriteTo(IBufferWriter output) => _request.WriteTo(output); 21 | 22 | public override void ParseResponse(KuduMessage message) 23 | { 24 | Output = GetTableLocationsResponsePB.Parser.ParseFrom(message.MessageProtobuf); 25 | Error = Output.Error; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/IsCreateTableDoneRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf; 3 | using Knet.Kudu.Client.Protobuf.Master; 4 | using Knet.Kudu.Client.Protocol; 5 | 6 | namespace Knet.Kudu.Client.Requests; 7 | 8 | internal sealed class IsCreateTableDoneRequest : KuduMasterRpc 9 | { 10 | private readonly IsCreateTableDoneRequestPB _request; 11 | 12 | public IsCreateTableDoneRequest(IsCreateTableDoneRequestPB request) 13 | { 14 | MethodName = "IsCreateTableDone"; 15 | _request = request; 16 | } 17 | 18 | public override int CalculateSize() => _request.CalculateSize(); 19 | 20 | public override void WriteTo(IBufferWriter output) => _request.WriteTo(output); 21 | 22 | public override void ParseResponse(KuduMessage message) 23 | { 24 | Output = IsCreateTableDoneResponsePB.Parser.ParseFrom(message.MessageProtobuf); 25 | Error = Output.Error; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/BeginTransactionRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Knet.Kudu.Client.Internal; 3 | using Knet.Kudu.Client.Protobuf.Transactions; 4 | using Knet.Kudu.Client.Protocol; 5 | 6 | namespace Knet.Kudu.Client.Requests; 7 | 8 | internal sealed class BeginTransactionRequest : KuduTxnRpc 9 | { 10 | private static readonly byte[] _requestBytes = ProtobufHelper.ToByteArray( 11 | new BeginTransactionRequestPB()); 12 | 13 | public BeginTransactionRequest() 14 | { 15 | MethodName = "BeginTransaction"; 16 | } 17 | 18 | public override int CalculateSize() => _requestBytes.Length; 19 | 20 | public override void WriteTo(IBufferWriter output) => output.Write(_requestBytes); 21 | 22 | public override void ParseResponse(KuduMessage message) 23 | { 24 | Output = BeginTransactionResponsePB.Parser.ParseFrom(message.MessageProtobuf); 25 | Error = Output.Error; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/CommitTransactionRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf; 3 | using Knet.Kudu.Client.Protobuf.Transactions; 4 | using Knet.Kudu.Client.Protocol; 5 | 6 | namespace Knet.Kudu.Client.Requests; 7 | 8 | internal sealed class CommitTransactionRequest : KuduTxnRpc 9 | { 10 | private readonly CommitTransactionRequestPB _request; 11 | 12 | public CommitTransactionRequest(CommitTransactionRequestPB request) 13 | { 14 | MethodName = "CommitTransaction"; 15 | _request = request; 16 | } 17 | 18 | public override int CalculateSize() => _request.CalculateSize(); 19 | 20 | public override void WriteTo(IBufferWriter output) => _request.WriteTo(output); 21 | 22 | public override void ParseResponse(KuduMessage message) 23 | { 24 | Output = CommitTransactionResponsePB.Parser.ParseFrom(message.MessageProtobuf); 25 | Error = Output.Error; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/GetTableStatisticsRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf; 3 | using Knet.Kudu.Client.Protobuf.Master; 4 | using Knet.Kudu.Client.Protocol; 5 | 6 | namespace Knet.Kudu.Client.Requests; 7 | 8 | internal sealed class GetTableStatisticsRequest : KuduMasterRpc 9 | { 10 | private readonly GetTableStatisticsRequestPB _request; 11 | 12 | public GetTableStatisticsRequest(GetTableStatisticsRequestPB request) 13 | { 14 | MethodName = "GetTableStatistics"; 15 | _request = request; 16 | } 17 | 18 | public override int CalculateSize() => _request.CalculateSize(); 19 | 20 | public override void WriteTo(IBufferWriter output) => _request.WriteTo(output); 21 | 22 | public override void ParseResponse(KuduMessage message) 23 | { 24 | Output = GetTableStatisticsResponsePB.Parser.ParseFrom(message.MessageProtobuf); 25 | Error = Output.Error; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/ListTabletServersRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Knet.Kudu.Client.Internal; 3 | using Knet.Kudu.Client.Protobuf.Master; 4 | using Knet.Kudu.Client.Protocol; 5 | 6 | namespace Knet.Kudu.Client.Requests; 7 | 8 | internal sealed class ListTabletServersRequest : KuduMasterRpc 9 | { 10 | private static readonly byte[] _requestBytes = ProtobufHelper.ToByteArray( 11 | new ListTabletServersRequestPB()); 12 | 13 | public ListTabletServersRequest() 14 | { 15 | MethodName = "ListTabletServers"; 16 | } 17 | 18 | public override int CalculateSize() => _requestBytes.Length; 19 | 20 | public override void WriteTo(IBufferWriter output) => output.Write(_requestBytes); 21 | 22 | public override void ParseResponse(KuduMessage message) 23 | { 24 | Output = ListTabletServersResponsePB.Parser.ParseFrom(message.MessageProtobuf); 25 | Error = Output.Error; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/GetTransactionStateRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf; 3 | using Knet.Kudu.Client.Protobuf.Transactions; 4 | using Knet.Kudu.Client.Protocol; 5 | 6 | namespace Knet.Kudu.Client.Requests; 7 | 8 | internal sealed class GetTransactionStateRequest : KuduTxnRpc 9 | { 10 | private readonly GetTransactionStateRequestPB _request; 11 | 12 | public GetTransactionStateRequest(GetTransactionStateRequestPB request) 13 | { 14 | MethodName = "GetTransactionState"; 15 | _request = request; 16 | } 17 | 18 | public override int CalculateSize() => _request.CalculateSize(); 19 | 20 | public override void WriteTo(IBufferWriter output) => _request.WriteTo(output); 21 | 22 | public override void ParseResponse(KuduMessage message) 23 | { 24 | Output = GetTransactionStateResponsePB.Parser.ParseFrom(message.MessageProtobuf); 25 | Error = Output.Error; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/KeepTransactionAliveRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf; 3 | using Knet.Kudu.Client.Protobuf.Transactions; 4 | using Knet.Kudu.Client.Protocol; 5 | 6 | namespace Knet.Kudu.Client.Requests; 7 | 8 | internal sealed class KeepTransactionAliveRequest : KuduTxnRpc 9 | { 10 | private readonly KeepTransactionAliveRequestPB _request; 11 | 12 | public KeepTransactionAliveRequest(KeepTransactionAliveRequestPB request) 13 | { 14 | MethodName = "KeepTransactionAlive"; 15 | _request = request; 16 | } 17 | 18 | public override int CalculateSize() => _request.CalculateSize(); 19 | 20 | public override void WriteTo(IBufferWriter output) => _request.WriteTo(output); 21 | 22 | public override void ParseResponse(KuduMessage message) 23 | { 24 | Output = KeepTransactionAliveResponsePB.Parser.ParseFrom(message.MessageProtobuf); 25 | Error = Output.Error; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Connection/HostAndPort.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Knet.Kudu.Client.Connection; 4 | 5 | public class HostAndPort : IEquatable 6 | { 7 | public string Host { get; } 8 | 9 | public int Port { get; } 10 | 11 | public HostAndPort(string host, int port) 12 | { 13 | Host = host; 14 | Port = port; 15 | } 16 | 17 | public bool Equals(HostAndPort? other) 18 | { 19 | if (other is null) 20 | return false; 21 | 22 | if (ReferenceEquals(this, other)) 23 | return true; 24 | 25 | return StringComparer.OrdinalIgnoreCase.Equals(Host, other.Host) && 26 | Port == other.Port; 27 | } 28 | 29 | public override bool Equals(object? obj) => Equals(obj as HostAndPort); 30 | 31 | public override int GetHashCode() => 32 | HashCode.Combine(StringComparer.OrdinalIgnoreCase.GetHashCode(Host), Port); 33 | 34 | public override string ToString() => $"{Host}:{Port}"; 35 | } 36 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/PredicateType.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client; 2 | 3 | public enum PredicateType 4 | { 5 | /// 6 | /// A predicate which filters all rows. 7 | /// 8 | None, 9 | /// 10 | /// A predicate which filters all rows not equal to a value. 11 | /// 12 | Equality, 13 | /// 14 | /// A predicate which filters all rows not in a range. 15 | /// 16 | Range, 17 | /// 18 | /// A predicate which filters all null rows. 19 | /// 20 | IsNotNull, 21 | /// 22 | /// A predicate which filters all non-null rows. 23 | /// 24 | IsNull, 25 | /// 26 | /// A predicate which filters all rows not matching a list of values. 27 | /// 28 | InList, 29 | /// 30 | /// A predicate which evaluates to true if the column value is present in 31 | /// a bloom filter. 32 | /// 33 | InBloomFilter 34 | } 35 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Exceptions/KuduWriteException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace Knet.Kudu.Client.Exceptions; 6 | 7 | /// 8 | /// An exception that indicates the overall write operation succeeded, 9 | /// but individual rows failed, such as inserting a row that already 10 | /// exists, or updating or deleting a row that doesn't exist. 11 | /// 12 | public class KuduWriteException : KuduException 13 | { 14 | public IReadOnlyList PerRowErrors { get; } 15 | 16 | public KuduWriteException(List errors) 17 | : base(GetStatus(errors)) 18 | { 19 | PerRowErrors = errors; 20 | } 21 | 22 | private static KuduStatus GetStatus(List errors) 23 | { 24 | var stringBuilder = new StringBuilder("Per row errors:"); 25 | foreach (var error in errors) 26 | { 27 | stringBuilder.Append($"{Environment.NewLine}{error.Message}"); 28 | } 29 | return KuduStatus.InvalidArgument(stringBuilder.ToString()); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/KuduScannerBuilder.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Logging; 2 | 3 | namespace Knet.Kudu.Client; 4 | 5 | public class KuduScannerBuilder : AbstractKuduScannerBuilder 6 | { 7 | internal readonly ILogger Logger; 8 | 9 | public KuduScannerBuilder(KuduClient client, KuduTable table, ILogger logger) 10 | : base(client, table) 11 | { 12 | Logger = logger; 13 | } 14 | 15 | public KuduScanner Build() 16 | { 17 | return new KuduScanner( 18 | Logger, 19 | Client, 20 | Table, 21 | ProjectedColumnNames, 22 | ProjectedColumnIndexes, 23 | Predicates, 24 | ReadMode, 25 | ReplicaSelection, 26 | IsFaultTolerant, 27 | BatchSizeBytes, 28 | Limit, 29 | CacheBlocks, 30 | StartTimestamp, 31 | HtTimestamp, 32 | LowerBoundPrimaryKey, 33 | UpperBoundPrimaryKey, 34 | LowerBoundPartitionKey, 35 | UpperBoundPartitionKey); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/CreateTableRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf; 3 | using Knet.Kudu.Client.Protobuf.Master; 4 | using Knet.Kudu.Client.Protocol; 5 | 6 | namespace Knet.Kudu.Client.Requests; 7 | 8 | internal sealed class CreateTableRequest : KuduMasterRpc 9 | { 10 | private readonly CreateTableRequestPB _request; 11 | 12 | public CreateTableRequest(CreateTableRequestPB request) 13 | { 14 | MethodName = "CreateTable"; 15 | _request = request; 16 | 17 | // We don't need to set required feature ADD_DROP_RANGE_PARTITIONS here, 18 | // as it's supported in Kudu 1.3, the oldest version this client supports. 19 | } 20 | 21 | public override int CalculateSize() => _request.CalculateSize(); 22 | 23 | public override void WriteTo(IBufferWriter output) => _request.WriteTo(output); 24 | 25 | public override void ParseResponse(KuduMessage message) 26 | { 27 | Output = CreateTableResponsePB.Parser.ParseFrom(message.MessageProtobuf); 28 | Error = Output.Error; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/ListTablesRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf; 3 | using Knet.Kudu.Client.Protobuf.Master; 4 | using Knet.Kudu.Client.Protocol; 5 | 6 | namespace Knet.Kudu.Client.Requests; 7 | 8 | internal sealed class ListTablesRequest : KuduMasterRpc 9 | { 10 | private static readonly ListTablesRequestPB _noFilterRequest = new(); 11 | 12 | private readonly ListTablesRequestPB _request; 13 | 14 | public ListTablesRequest(string? nameFilter = null) 15 | { 16 | MethodName = "ListTables"; 17 | 18 | _request = nameFilter is null 19 | ? _noFilterRequest 20 | : new ListTablesRequestPB { NameFilter = nameFilter }; 21 | } 22 | 23 | public override int CalculateSize() => _request.CalculateSize(); 24 | 25 | public override void WriteTo(IBufferWriter output) => _request.WriteTo(output); 26 | 27 | public override void ParseResponse(KuduMessage message) 28 | { 29 | Output = ListTablesResponsePB.Parser.ParseFrom(message.MessageProtobuf); 30 | Error = Output.Error; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/KuduTabletRpc.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Protobuf.Security; 2 | using Knet.Kudu.Client.Protobuf.Tserver; 3 | using Knet.Kudu.Client.Tablet; 4 | 5 | namespace Knet.Kudu.Client.Requests; 6 | 7 | internal abstract class KuduTabletRpc : KuduRpc 8 | { 9 | public long PropagatedTimestamp { get; set; } = KuduClient.NoTimestamp; 10 | 11 | /// 12 | /// Returns the partition key this RPC is for. 13 | /// 14 | public byte[] PartitionKey { get; init; } = null!; 15 | 16 | public RemoteTablet? Tablet { get; internal set; } 17 | 18 | public ReplicaSelection ReplicaSelection { get; init; } = ReplicaSelection.LeaderOnly; 19 | 20 | public bool NeedsAuthzToken { get; init; } 21 | 22 | internal SignedTokenPB? AuthzToken { get; set; } 23 | 24 | /// 25 | /// The table this RPC is for. 26 | /// 27 | public string TableId { get; init; } = null!; 28 | 29 | public TabletServerErrorPB? Error { get; protected set; } 30 | 31 | public KuduTabletRpc() 32 | { 33 | ServiceName = TabletServerServiceName; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.FunctionalTests/Knet.Kudu.Client.FunctionalTests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net7.0 5 | latest 6 | false 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/PartialRowOperation.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Knet.Kudu.Client; 4 | 5 | public class PartialRowOperation : PartialRow 6 | { 7 | internal RowOperation Operation { get; } 8 | 9 | public PartialRowOperation(KuduSchema schema, RowOperation operation) 10 | : base(schema) 11 | { 12 | Operation = operation; 13 | } 14 | 15 | internal PartialRowOperation(PartialRowOperation row, RowOperation operation) 16 | : base(row) 17 | { 18 | Operation = operation; 19 | } 20 | 21 | public void WriteToWithOperation( 22 | Span rowDestination, 23 | Span indirectDestination, 24 | int indirectDataStart, 25 | out int rowBytesWritten, 26 | out int indirectBytesWritten) 27 | { 28 | rowDestination[0] = (byte)Operation; 29 | rowDestination = rowDestination.Slice(1); 30 | 31 | WriteTo( 32 | rowDestination, 33 | indirectDestination, 34 | indirectDataStart, 35 | out rowBytesWritten, 36 | out indirectBytesWritten); 37 | 38 | rowBytesWritten += 1; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.FunctionalTests/MiniCluster/MiniKuduClusterTestAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Runtime.InteropServices; 3 | using Knet.Kudu.Binary; 4 | using McMaster.Extensions.Xunit; 5 | 6 | namespace Knet.Kudu.Client.FunctionalTests.MiniCluster; 7 | 8 | [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)] 9 | public class MiniKuduClusterTestAttribute : Attribute, ITestCondition 10 | { 11 | private static readonly Lazy _hasKuduMiniCluster = new Lazy(HasKuduMiniCluster); 12 | 13 | public bool IsMet => _hasKuduMiniCluster.Value; 14 | 15 | public string SkipReason => "This test requires Linux with Kudu installed."; 16 | 17 | private static bool HasKuduMiniCluster() 18 | { 19 | if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) 20 | { 21 | try 22 | { 23 | // This method throws an exception if Kudu can't be found. 24 | KuduBinaryLocator.FindBinary("kudu"); 25 | return true; 26 | } 27 | catch { } 28 | } 29 | 30 | return false; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Internal/PeriodicTimer.cs: -------------------------------------------------------------------------------- 1 | #if !NET6_0_OR_GREATER 2 | 3 | using System; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | 7 | namespace Knet.Kudu.Client.Internal; 8 | 9 | internal sealed class PeriodicTimer : IDisposable 10 | { 11 | private readonly TimeSpan _period; 12 | private readonly CancellationTokenSource _stoppingCts; 13 | 14 | public PeriodicTimer(TimeSpan period) 15 | { 16 | _period = period; 17 | _stoppingCts = new CancellationTokenSource(); 18 | } 19 | 20 | public void Dispose() 21 | { 22 | _stoppingCts.Cancel(); 23 | } 24 | 25 | public async ValueTask WaitForNextTickAsync() 26 | { 27 | var token = _stoppingCts.Token; 28 | 29 | if (token.IsCancellationRequested) 30 | { 31 | return false; 32 | } 33 | 34 | try 35 | { 36 | await Task.Delay(_period, token).ConfigureAwait(false); 37 | return true; 38 | } 39 | catch (OperationCanceledException) when (token.IsCancellationRequested) 40 | { 41 | return false; 42 | } 43 | } 44 | } 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Internal/SequenceReaderExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Buffers; 3 | 4 | namespace Knet.Kudu.Client.Internal; 5 | 6 | internal static class SequenceReaderExtensions 7 | { 8 | public static bool TryReadVarint(this ref SequenceReader reader, out int value) 9 | { 10 | value = 0; 11 | 12 | for (int i = 0; i < 4; i++) 13 | { 14 | if (reader.TryRead(out byte chunk)) 15 | { 16 | value |= (chunk & 0x7F) << 7 * i; 17 | if ((chunk & 0x80) == 0) 18 | return true; 19 | } 20 | else 21 | { 22 | reader.Rewind(i); 23 | return false; 24 | } 25 | } 26 | 27 | if (reader.TryRead(out byte b1)) 28 | { 29 | value |= b1 << 28; // Can only use 4 bits from this chunk. 30 | if ((b1 & 0xF0) == 0) 31 | return true; 32 | } 33 | else 34 | { 35 | reader.Rewind(4); 36 | return false; 37 | } 38 | 39 | throw new OverflowException("Error decoding varint32"); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Exceptions/NonCoveredRangeException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Knet.Kudu.Client.Exceptions; 4 | 5 | /// 6 | /// Exception indicating that an operation attempted to access a 7 | /// non-covered range partition. 8 | /// 9 | public class NonCoveredRangeException : NonRecoverableException 10 | { 11 | public byte[] NonCoveredRangeStart; 12 | 13 | public byte[] NonCoveredRangeEnd; 14 | 15 | public NonCoveredRangeException( 16 | byte[] nonCoveredRangeStart, 17 | byte[] nonCoveredRangeEnd) 18 | : base(KuduStatus.NotFound(GetMessage( 19 | nonCoveredRangeStart, nonCoveredRangeEnd))) 20 | { 21 | NonCoveredRangeStart = nonCoveredRangeStart; 22 | NonCoveredRangeEnd = nonCoveredRangeEnd; 23 | } 24 | 25 | private static string GetMessage(byte[] start, byte[] end) 26 | { 27 | var startStr = start is null || start.Length == 0 ? 28 | "" : BitConverter.ToString(start); 29 | 30 | var endStr = end is null || end.Length == 0 ? 31 | "" : BitConverter.ToString(end); 32 | 33 | return $"[{startStr}, {endStr})"; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.Tests/FastHashTests.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Internal; 2 | using Knet.Kudu.Client.Util; 3 | using Xunit; 4 | 5 | namespace Knet.Kudu.Client.Tests; 6 | 7 | public class FastHashTests 8 | { 9 | [Theory] 10 | [InlineData("ab", 0, 17293172613997361769)] 11 | [InlineData("abcdefg", 0, 10206404559164245992)] 12 | [InlineData("quick brown fox", 42, 3757424404558187042)] 13 | [InlineData("", 0, 4144680785095980158)] 14 | [InlineData("", 1234, 3296774803014270295)] 15 | public void TestFastHash64(string data, ulong seed, ulong expectedHash) 16 | { 17 | ulong hash = FastHash.Hash64(data.ToUtf8ByteArray(), seed); 18 | Assert.Equal(expectedHash, hash); 19 | } 20 | 21 | [Theory] 22 | [InlineData("ab", 0, 2564147595)] 23 | [InlineData("abcdefg", 0, 1497700618)] 24 | [InlineData("quick brown fox", 42, 1676541068)] 25 | [InlineData("", 0, 3045300040)] 26 | [InlineData("", 1234, 811548192)] 27 | public void TestFastHash32(string data, uint seed, uint expectedHash) 28 | { 29 | uint hash = FastHash.Hash32(data.ToUtf8ByteArray(), seed); 30 | Assert.Equal(expectedHash, hash); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Protos/kudu/util/hash.proto: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | syntax = "proto2"; 18 | package kudu; 19 | 20 | option java_package = "org.apache.kudu"; 21 | option csharp_namespace = "Knet.Kudu.Client.Protobuf"; 22 | 23 | // Implemented hash algorithms. 24 | enum HashAlgorithm { 25 | UNKNOWN_HASH = 0; 26 | MURMUR_HASH_2 = 1; 27 | CITY_HASH = 2; 28 | FAST_HASH = 3; 29 | } 30 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/ConnectToMasterRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf.Collections; 3 | using Knet.Kudu.Client.Internal; 4 | using Knet.Kudu.Client.Protobuf.Master; 5 | using Knet.Kudu.Client.Protocol; 6 | 7 | namespace Knet.Kudu.Client.Requests; 8 | 9 | internal sealed class ConnectToMasterRequest : KuduMasterRpc 10 | { 11 | private static readonly RepeatedField _requiredFeatures = new() 12 | { 13 | (uint)MasterFeatures.ConnectToMaster 14 | }; 15 | 16 | private static readonly byte[] _requestBytes = ProtobufHelper.ToByteArray( 17 | new ConnectToMasterRequestPB()); 18 | 19 | public ConnectToMasterRequest() 20 | { 21 | MethodName = "ConnectToMaster"; 22 | RequiredFeatures = _requiredFeatures; 23 | } 24 | 25 | public override int CalculateSize() => _requestBytes.Length; 26 | 27 | public override void WriteTo(IBufferWriter output) => output.Write(_requestBytes); 28 | 29 | public override void ParseResponse(KuduMessage message) 30 | { 31 | Output = ConnectToMasterResponsePB.Parser.ParseFrom(message.MessageProtobuf); 32 | Error = Output.Error; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Protos/kudu/util/compression/compression.proto: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | syntax = "proto2"; 18 | package kudu; 19 | 20 | option java_package = "org.apache.kudu"; 21 | option csharp_namespace = "Knet.Kudu.Client.Protobuf"; 22 | 23 | enum CompressionType { 24 | UNKNOWN_COMPRESSION = 999; 25 | DEFAULT_COMPRESSION = 0; 26 | NO_COMPRESSION = 1; 27 | SNAPPY = 2; 28 | LZ4 = 3; 29 | ZLIB = 4; 30 | } 31 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.FunctionalTests/DeleteTableTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using Knet.Kudu.Client.Exceptions; 4 | using Knet.Kudu.Client.FunctionalTests.MiniCluster; 5 | using McMaster.Extensions.Xunit; 6 | using Xunit; 7 | 8 | namespace Knet.Kudu.Client.FunctionalTests; 9 | 10 | [MiniKuduClusterTest] 11 | public class DeleteTableTests 12 | { 13 | [SkippableFact] 14 | public async Task CreateAndDeleteTable() 15 | { 16 | await using var miniCluster = await new MiniKuduClusterBuilder() 17 | .NumMasters(3) 18 | .NumTservers(3) 19 | .BuildAsync(); 20 | 21 | await using var client = miniCluster.CreateClient(); 22 | 23 | var tableName = Guid.NewGuid().ToString(); 24 | var builder = new TableBuilder() 25 | .SetTableName(tableName) 26 | .SetNumReplicas(1) 27 | .AddColumn("pk", KuduType.Int32, opt => opt.Key(true)); 28 | 29 | var table = await client.CreateTableAsync(builder); 30 | Assert.Equal(tableName, table.TableName); 31 | 32 | await client.DeleteTableAsync(tableName); 33 | 34 | await Assert.ThrowsAsync( 35 | () => client.DeleteTableAsync(tableName)); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Tablet/FindTabletResult.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics.CodeAnalysis; 2 | 3 | namespace Knet.Kudu.Client.Tablet; 4 | 5 | internal readonly struct FindTabletResult 6 | { 7 | public RemoteTablet? Tablet { get; } 8 | 9 | public int Index { get; } 10 | 11 | public byte[]? NonCoveredRangeStart { get; } 12 | 13 | public byte[]? NonCoveredRangeEnd { get; } 14 | 15 | public FindTabletResult(RemoteTablet tablet, int index) 16 | { 17 | Tablet = tablet; 18 | Index = index; 19 | NonCoveredRangeStart = null; 20 | NonCoveredRangeEnd = null; 21 | } 22 | 23 | public FindTabletResult(byte[] nonCoveredRangeStart, byte[] nonCoveredRangeEnd) 24 | { 25 | Tablet = null; 26 | Index = -1; 27 | NonCoveredRangeStart = nonCoveredRangeStart; 28 | NonCoveredRangeEnd = nonCoveredRangeEnd; 29 | } 30 | 31 | [MemberNotNullWhen(true, nameof(Tablet))] 32 | [MemberNotNullWhen(false, nameof(NonCoveredRangeStart), nameof(NonCoveredRangeEnd))] 33 | public bool IsCoveredRange => Tablet is not null; 34 | 35 | [MemberNotNullWhen(true, nameof(NonCoveredRangeStart), nameof(NonCoveredRangeEnd))] 36 | [MemberNotNullWhen(false, nameof(Tablet))] 37 | public bool IsNonCoveredRange => Tablet is null; 38 | } 39 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/MasterManager.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Connection; 2 | 3 | namespace Knet.Kudu.Client; 4 | 5 | internal sealed class MasterManager 6 | { 7 | private readonly object _lock = new(); 8 | 9 | private MasterLeaderInfo? _currentLeader; 10 | private volatile MasterLeaderInfo? _lastKnownLeader; 11 | 12 | public MasterLeaderInfo? LeaderInfo 13 | { 14 | get 15 | { 16 | lock (_lock) 17 | { 18 | return _currentLeader; 19 | } 20 | } 21 | } 22 | 23 | public MasterLeaderInfo? LastKnownLeaderInfo => _lastKnownLeader; 24 | 25 | public void UpdateLeader(MasterLeaderInfo masterLeaderInfo) 26 | { 27 | lock (_lock) 28 | { 29 | _currentLeader = masterLeaderInfo; 30 | } 31 | 32 | _lastKnownLeader = masterLeaderInfo; 33 | } 34 | 35 | public void RemoveLeader(ServerInfo serverInfo) 36 | { 37 | lock (_lock) 38 | { 39 | var cachedServerInfo = _currentLeader?.ServerInfo; 40 | 41 | if (ReferenceEquals(serverInfo, cachedServerInfo)) 42 | { 43 | // The bad leader is still cached, remove it. 44 | _currentLeader = null; 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/GetTableSchemaRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf; 3 | using Google.Protobuf.Collections; 4 | using Knet.Kudu.Client.Protobuf.Master; 5 | using Knet.Kudu.Client.Protocol; 6 | 7 | namespace Knet.Kudu.Client.Requests; 8 | 9 | internal sealed class GetTableSchemaRequest : KuduMasterRpc 10 | { 11 | private static readonly RepeatedField _requiredFeatures = new() 12 | { 13 | (uint)MasterFeatures.GenerateAuthzToken 14 | }; 15 | 16 | private readonly GetTableSchemaRequestPB _request; 17 | 18 | public GetTableSchemaRequest( 19 | GetTableSchemaRequestPB request, 20 | bool requiresAuthzTokenSupport) 21 | { 22 | MethodName = "GetTableSchema"; 23 | _request = request; 24 | 25 | if (requiresAuthzTokenSupport) 26 | RequiredFeatures = _requiredFeatures; 27 | } 28 | 29 | public override int CalculateSize() => _request.CalculateSize(); 30 | 31 | public override void WriteTo(IBufferWriter output) => _request.WriteTo(output); 32 | 33 | public override void ParseResponse(KuduMessage message) 34 | { 35 | Output = GetTableSchemaResponsePB.Parser.ParseFrom(message.MessageProtobuf); 36 | Error = Output.Error; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/KeepAliveRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf; 3 | using Knet.Kudu.Client.Protobuf.Tserver; 4 | using Knet.Kudu.Client.Protocol; 5 | using Knet.Kudu.Client.Tablet; 6 | 7 | namespace Knet.Kudu.Client.Requests; 8 | 9 | internal sealed class KeepAliveRequest : KuduTabletRpc 10 | { 11 | private readonly ScannerKeepAliveRequestPB _request; 12 | 13 | public KeepAliveRequest( 14 | ByteString scannerId, 15 | ReplicaSelection replicaSelection, 16 | string tableId, 17 | RemoteTablet? tablet, 18 | byte[] partitionKey) 19 | { 20 | _request = new ScannerKeepAliveRequestPB { ScannerId = scannerId }; 21 | 22 | MethodName = "ScannerKeepAlive"; 23 | ReplicaSelection = replicaSelection; 24 | TableId = tableId; 25 | Tablet = tablet; 26 | PartitionKey = partitionKey; 27 | } 28 | 29 | public override int CalculateSize() => _request.CalculateSize(); 30 | 31 | public override void WriteTo(IBufferWriter output) => _request.WriteTo(output); 32 | 33 | public override void ParseResponse(KuduMessage message) 34 | { 35 | Output = ScannerKeepAliveResponsePB.Parser.ParseFrom(message.MessageProtobuf); 36 | Error = Output.Error; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push] 4 | 5 | env: 6 | DOTNET_NOLOGO: 1 7 | DOTNET_CLI_TELEMETRY_OPTOUT: 1 8 | 9 | jobs: 10 | linux_build: 11 | name: Linux build 12 | runs-on: ubuntu-22.04 13 | 14 | steps: 15 | - uses: actions/checkout@v3 16 | 17 | - name: Setup dotnet 18 | uses: actions/setup-dotnet@v3 19 | with: 20 | dotnet-version: '7.0.x' 21 | 22 | # Needed for Kudu test binary used in integration tests 23 | - name: Install libncurses5 24 | run: sudo apt install -y libncurses5 25 | 26 | - name: Build 27 | run: dotnet build --configuration Release 28 | 29 | - name: Test 30 | run: dotnet test --no-restore --no-build --configuration Release 31 | 32 | publish: 33 | name: Publish nuget package 34 | needs: linux_build 35 | runs-on: ubuntu-22.04 36 | 37 | if: github.ref == 'refs/heads/main' 38 | steps: 39 | - uses: actions/checkout@v3 40 | 41 | - name: Setup dotnet 42 | uses: actions/setup-dotnet@v3 43 | with: 44 | dotnet-version: '7.0.x' 45 | 46 | - name: Publish NuGet package on version change 47 | uses: alirezanet/publish-nuget@v3.0.4 48 | with: 49 | PROJECT_FILE_PATH: src/Knet.Kudu.Client/Knet.Kudu.Client.csproj 50 | NUGET_KEY: ${{secrets.KUDU_NUGET_API_KEY}} 51 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Connection/ServerInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Net; 2 | 3 | namespace Knet.Kudu.Client.Connection; 4 | 5 | /// 6 | /// Container class for server information that never changes, like UUID and hostname. 7 | /// 8 | public class ServerInfo 9 | { 10 | public string Uuid { get; } 11 | 12 | public HostAndPort HostPort { get; } 13 | 14 | public IPEndPoint Endpoint { get; } 15 | 16 | public string? Location { get; } 17 | 18 | public bool IsLocal { get; } 19 | 20 | public ServerInfo( 21 | string uuid, 22 | HostAndPort hostPort, 23 | IPEndPoint endpoint, 24 | string? location, 25 | bool isLocal) 26 | { 27 | Uuid = uuid; 28 | HostPort = hostPort; 29 | Endpoint = endpoint; 30 | Location = location; 31 | IsLocal = isLocal; 32 | } 33 | 34 | public bool HasLocation => !string.IsNullOrEmpty(Location); 35 | 36 | /// 37 | /// Returns true if the server is in the same location as the given location. 38 | /// 39 | /// The location to check. 40 | public bool InSameLocation(string? location) => 41 | !string.IsNullOrEmpty(location) && location == Location; 42 | 43 | public override string ToString() => $"{Uuid} ({HostPort})"; 44 | } 45 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/PartitionSchema.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace Knet.Kudu.Client; 4 | 5 | public class PartitionSchema 6 | { 7 | public RangeSchema RangeSchema { get; } 8 | 9 | public List HashBucketSchemas { get; } 10 | 11 | /// 12 | /// Returns true if the partition schema does not include any hash components, 13 | /// and the range columns match the table's primary key columns. 14 | /// 15 | public bool IsSimpleRangePartitioning { get; } 16 | 17 | public PartitionSchema( 18 | RangeSchema rangeSchema, 19 | List hashBucketSchemas, 20 | KuduSchema schema) 21 | { 22 | RangeSchema = rangeSchema; 23 | HashBucketSchemas = hashBucketSchemas; 24 | 25 | bool isSimple = hashBucketSchemas.Count == 0 && 26 | rangeSchema.ColumnIds.Count == schema.PrimaryKeyColumnCount; 27 | 28 | if (isSimple) 29 | { 30 | int i = 0; 31 | foreach (int id in rangeSchema.ColumnIds) 32 | { 33 | if (schema.GetColumnIndex(id) != i++) 34 | { 35 | isSimple = false; 36 | break; 37 | } 38 | } 39 | } 40 | 41 | IsSimpleRangePartitioning = isSimple; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/AlterTableRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf; 3 | using Knet.Kudu.Client.Protobuf.Master; 4 | using Knet.Kudu.Client.Protocol; 5 | 6 | namespace Knet.Kudu.Client.Requests; 7 | 8 | internal sealed class AlterTableRequest : KuduMasterRpc 9 | { 10 | private readonly AlterTableRequestPB _request; 11 | 12 | public AlterTableRequest(AlterTableRequestPB request) 13 | { 14 | MethodName = "AlterTable"; 15 | _request = request; 16 | 17 | // We don't need to set required feature ADD_DROP_RANGE_PARTITIONS here, 18 | // as it's supported in Kudu 1.3, the oldest version this client supports. 19 | } 20 | 21 | public override int CalculateSize() => _request.CalculateSize(); 22 | 23 | public override void WriteTo(IBufferWriter output) => _request.WriteTo(output); 24 | 25 | public override void ParseResponse(KuduMessage message) 26 | { 27 | var responsePb = AlterTableResponsePB.Parser.ParseFrom(message.MessageProtobuf); 28 | 29 | if (responsePb.Error is null) 30 | { 31 | Output = new AlterTableResponse( 32 | responsePb.TableId.ToStringUtf8(), 33 | responsePb.SchemaVersion); 34 | } 35 | else 36 | { 37 | Error = responsePb.Error; 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.Tests/HostAndPortTests.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Connection; 2 | using Xunit; 3 | 4 | namespace Knet.Kudu.Client.Tests; 5 | 6 | public class HostAndPortTests 7 | { 8 | [Fact] 9 | public void SameHostAndPort() 10 | { 11 | var hostPort1 = new HostAndPort("localhost", 7051); 12 | var hostPort2 = new HostAndPort("localhost", 7051); 13 | 14 | Assert.Equal(hostPort1, hostPort2); 15 | Assert.Equal(hostPort1.GetHashCode(), hostPort2.GetHashCode()); 16 | } 17 | 18 | [Fact] 19 | public void HostShouldBeCaseInsensitive() 20 | { 21 | var hostPort1 = new HostAndPort("localhost", 7051); 22 | var hostPort2 = new HostAndPort("LOCALHOST", 7051); 23 | 24 | Assert.Equal(hostPort1, hostPort2); 25 | Assert.Equal(hostPort1.GetHashCode(), hostPort2.GetHashCode()); 26 | } 27 | 28 | [Fact] 29 | public void DifferentHostSamePort() 30 | { 31 | var hostPort1 = new HostAndPort("1.2.3.4", 7051); 32 | var hostPort2 = new HostAndPort("1.2.3.5", 7051); 33 | 34 | Assert.NotEqual(hostPort1, hostPort2); 35 | } 36 | 37 | [Fact] 38 | public void DifferentPortSameHost() 39 | { 40 | var hostPort1 = new HostAndPort("127.0.0.1", 7051); 41 | var hostPort2 = new HostAndPort("127.0.0.1", 7050); 42 | 43 | Assert.NotEqual(hostPort1, hostPort2); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/KuduClientOptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO.Pipelines; 4 | using Knet.Kudu.Client.Connection; 5 | 6 | namespace Knet.Kudu.Client; 7 | 8 | public class KuduClientOptions 9 | { 10 | public IReadOnlyList MasterAddresses { get; } 11 | 12 | public TimeSpan DefaultOperationTimeout { get; } 13 | 14 | public string? SaslProtocolName { get; } 15 | 16 | public bool RequireAuthentication { get; } 17 | 18 | public EncryptionPolicy EncryptionPolicy { get; } 19 | 20 | public PipeOptions SendPipeOptions { get; } 21 | 22 | public PipeOptions ReceivePipeOptions { get; } 23 | 24 | public KuduClientOptions( 25 | IReadOnlyList masterAddresses, 26 | TimeSpan defaultOperationTimeout, 27 | string? saslProtocolName, 28 | bool requireAuthentication, 29 | EncryptionPolicy encryptionPolicy, 30 | PipeOptions sendPipeOptions, 31 | PipeOptions receivePipeOptions) 32 | { 33 | MasterAddresses = masterAddresses; 34 | DefaultOperationTimeout = defaultOperationTimeout; 35 | SaslProtocolName = saslProtocolName; 36 | RequireAuthentication = requireAuthentication; 37 | EncryptionPolicy = encryptionPolicy; 38 | SendPipeOptions = sendPipeOptions; 39 | ReceivePipeOptions = receivePipeOptions; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Protos/kudu/consensus/opid.proto: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | syntax = "proto2"; 18 | package kudu.consensus; 19 | 20 | option java_package = "org.apache.kudu.consensus"; 21 | option csharp_namespace = "Knet.Kudu.Client.Protobuf.Consensus"; 22 | 23 | // An id for a generic state machine operation. Composed of the leaders' term 24 | // plus the index of the operation in that term, e.g., the th operation 25 | // of the th leader. 26 | message OpId { 27 | // The term of an operation or the leader's sequence id. 28 | required int64 term = 1; 29 | required int64 index = 2; 30 | } 31 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Protocol/KuduMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Knet.Kudu.Client.Internal; 3 | 4 | namespace Knet.Kudu.Client.Protocol; 5 | 6 | public sealed class KuduMessage 7 | { 8 | private ArrayPoolBuffer? _messageBuffer; 9 | private int _messageProtobufLength; 10 | private SidecarOffset[]? _sidecarOffsets; 11 | 12 | internal void Init( 13 | ArrayPoolBuffer messageBuffer, 14 | int messageProtobufLength, 15 | SidecarOffset[] sidecarOffsets) 16 | { 17 | _messageBuffer = messageBuffer; 18 | _messageProtobufLength = messageProtobufLength; 19 | _sidecarOffsets = sidecarOffsets; 20 | } 21 | 22 | public byte[] Buffer => _messageBuffer!.Buffer; 23 | 24 | public ReadOnlySpan MessageProtobuf => 25 | Buffer.AsSpan(0, _messageProtobufLength); 26 | 27 | public SidecarOffset GetSidecarOffset(int sidecar) => _sidecarOffsets![sidecar]; 28 | 29 | internal ArrayPoolBuffer TakeMemory() 30 | { 31 | var messageBuffer = _messageBuffer; 32 | _messageBuffer = null; 33 | return messageBuffer!; 34 | } 35 | 36 | internal void Reset() 37 | { 38 | var buffer = _messageBuffer; 39 | if (buffer is not null) 40 | { 41 | _messageBuffer = null; 42 | buffer.Dispose(); 43 | } 44 | 45 | _messageProtobufLength = 0; 46 | _sidecarOffsets = null; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/HiveMetastoreConfig.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client; 2 | 3 | public class HiveMetastoreConfig 4 | { 5 | /// 6 | /// Address(es) of the Hive Metastore instance(s). 7 | /// 8 | /// For more info see the Kudu master --hive_metastore_uris flag for more info, 9 | /// or the Hive Metastore hive.metastore.uris configuration. 10 | /// 11 | public string? HiveMetastoreUris { get; } 12 | 13 | /// 14 | /// Whether the Hive Metastore instance uses SASL (Kerberos) security. 15 | /// 16 | /// For more info see the Kudu master --hive_metastore_sasl_enabled flag, or 17 | /// the Hive Metastore hive.metastore.sasl.enabled configuration. 18 | /// 19 | public bool HiveMetastoreSaslEnabled { get; } 20 | 21 | /// 22 | /// An ID which uniquely identifies the Hive Metastore instance. 23 | /// 24 | /// NOTE: this is provided on a best-effort basis, as not all Hive Metastore 25 | /// versions which Kudu is compatible with include the necessary APIs. See 26 | /// HIVE-16452 for more info. 27 | /// 28 | public string? HiveMetastoreUuid { get; } 29 | 30 | public HiveMetastoreConfig( 31 | string? hiveMetastoreUris, 32 | bool hiveMetastoreSaslEnabled, 33 | string? hiveMetastoreUuid) 34 | { 35 | HiveMetastoreUris = hiveMetastoreUris; 36 | HiveMetastoreSaslEnabled = hiveMetastoreSaslEnabled; 37 | HiveMetastoreUuid = hiveMetastoreUuid; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/KuduScannerExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | 5 | namespace Knet.Kudu.Client; 6 | 7 | public static class KuduScannerExtensions 8 | { 9 | /// 10 | /// Enumerates the scanner and maps the results to the generic type. 11 | /// 12 | /// The type to project a row to. 13 | public static async ValueTask> ScanToListAsync( 14 | this KuduScanner scanner, 15 | CancellationToken cancellationToken = default) 16 | { 17 | var list = new List(0); 18 | 19 | await foreach (var resultSet in scanner.WithCancellation(cancellationToken).ConfigureAwait(false)) 20 | { 21 | resultSet.MapTo(list); 22 | } 23 | 24 | return list; 25 | } 26 | 27 | /// 28 | /// Counts the number of rows returned by the scanner. Use 29 | /// 30 | /// when constructing the scanner to avoid transferring unnecessary data. 31 | /// 32 | public static async ValueTask CountAsync( 33 | this KuduScanner scanner, 34 | CancellationToken cancellationToken = default) 35 | { 36 | long count = 0; 37 | 38 | await foreach (var resultSet in scanner.WithCancellation(cancellationToken).ConfigureAwait(false)) 39 | { 40 | count += resultSet.Count; 41 | } 42 | 43 | return count; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/WriteRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf; 3 | using Knet.Kudu.Client.Protobuf.Tserver; 4 | using Knet.Kudu.Client.Protocol; 5 | 6 | namespace Knet.Kudu.Client.Requests; 7 | 8 | internal sealed class WriteRequest : KuduTabletRpc 9 | { 10 | private readonly WriteRequestPB _request; 11 | 12 | public WriteRequest( 13 | WriteRequestPB request, 14 | string tableId, 15 | byte[] partitionKey, 16 | ExternalConsistencyMode externalConsistencyMode) 17 | { 18 | _request = request; 19 | MethodName = "Write"; 20 | TableId = tableId; 21 | PartitionKey = partitionKey; 22 | NeedsAuthzToken = true; 23 | IsRequestTracked = true; 24 | ExternalConsistencyMode = externalConsistencyMode; 25 | } 26 | 27 | public override int CalculateSize() 28 | { 29 | if (AuthzToken != null) 30 | _request.AuthzToken = AuthzToken; 31 | 32 | if (PropagatedTimestamp != KuduClient.NoTimestamp) 33 | _request.PropagatedTimestamp = (ulong)PropagatedTimestamp; 34 | 35 | _request.TabletId = ByteString.CopyFromUtf8(Tablet!.TabletId); 36 | 37 | return _request.CalculateSize(); 38 | } 39 | 40 | public override void WriteTo(IBufferWriter output) => _request.WriteTo(output); 41 | 42 | public override void ParseResponse(KuduMessage message) 43 | { 44 | Output = WriteResponsePB.Parser.ParseFrom(message.MessageProtobuf); 45 | Error = Output.Error; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.FunctionalTests/Util/TestExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | 5 | namespace Knet.Kudu.Client.FunctionalTests.Util; 6 | 7 | public static class TestExtensions 8 | { 9 | /// 10 | /// Returns a view of the portion of this set whose elements are greater than 11 | /// (or equal to, if inclusive is true) start. 12 | /// 13 | public static SortedSet TailSet(this SortedSet set, T start, bool inclusive = true) 14 | { 15 | if (inclusive) 16 | { 17 | return set.GetViewBetween(start, set.Max); 18 | } 19 | else 20 | { 21 | // There's no overload of GetViewBetween that supports this. 22 | var comparer = set.Comparer; 23 | return new SortedSet(set.Where(v => comparer.Compare(v, start) > 0), comparer); 24 | } 25 | } 26 | 27 | /// 28 | /// Returns a view of the portion of this set whose elements are less than 29 | /// (or equal to, if inclusive is true) end. 30 | /// 31 | public static SortedSet HeadSet(this SortedSet set, T end, bool inclusive = false) 32 | { 33 | if (inclusive) 34 | { 35 | return set.GetViewBetween(set.Min, end); 36 | } 37 | else 38 | { 39 | // There's no overload of GetViewBetween that supports this. 40 | var comparer = set.Comparer; 41 | return new SortedSet(set.Where(v => comparer.Compare(v, end) < 0), comparer); 42 | } 43 | } 44 | 45 | public static bool NextBool(this Random random) 46 | { 47 | return random.Next(2) == 0; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/KuduSessionOptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | 5 | namespace Knet.Kudu.Client; 6 | 7 | public record KuduSessionOptions 8 | { 9 | /// 10 | /// The maximum number of items to send in a single write request. 11 | /// Hitting this number of items will trigger a flush immediately, 12 | /// regardless of the value of . 13 | /// Default: 2000. 14 | /// 15 | public int BatchSize { get; init; } = 2000; 16 | 17 | /// 18 | /// The maximum number of items the session may store before calls to 19 | /// 20 | /// will block until a batch completes. Default: 40000. 21 | /// 22 | public int Capacity { get; init; } = 40000; 23 | 24 | /// 25 | /// The maximum duration of time to wait for new items before flushing. 26 | /// Hitting this duration will trigger a flush immediately, regardless 27 | /// of the value of . Default: 1 second. 28 | /// 29 | public TimeSpan FlushInterval { get; init; } = TimeSpan.FromSeconds(1); 30 | 31 | /// 32 | /// The external consistency mode for this session. 33 | /// Default: . 34 | /// 35 | public ExternalConsistencyMode ExternalConsistencyMode { get; init; } = 36 | ExternalConsistencyMode.ClientPropagated; 37 | 38 | /// 39 | /// Optional callback to be notified of errors. 40 | /// 41 | public Func? ExceptionHandler { get; init; } 42 | } 43 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/KuduBloomFilterBuilder.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Util; 2 | 3 | namespace Knet.Kudu.Client; 4 | 5 | /// 6 | /// Builder class to help build to be used with 7 | /// IN Bloom filter predicates. 8 | /// 9 | public class KuduBloomFilterBuilder 10 | { 11 | private readonly ColumnSchema _column; 12 | private readonly ulong _numKeys; 13 | private uint _hashSeed = 0; 14 | private double _fpp = 0.01; 15 | 16 | /// The column schema. 17 | /// Expected number of unique elements to be inserted in the Bloom filter. 18 | public KuduBloomFilterBuilder(ColumnSchema column, ulong numKeys) 19 | { 20 | _column = column; 21 | _numKeys = numKeys; 22 | } 23 | 24 | /// 25 | /// Seed used with hash algorithm to hash the keys before inserting to 26 | /// the Bloom filter. If not provided, defaults to 0. 27 | /// 28 | public KuduBloomFilterBuilder SetHashSeed(uint seed) 29 | { 30 | _hashSeed = seed; 31 | return this; 32 | } 33 | 34 | /// 35 | /// Desired false positive probability between 0.0 and 1.0. 36 | /// If not provided, defaults to 0.01. 37 | /// 38 | public KuduBloomFilterBuilder SetFalsePositiveProbability(double fpp) 39 | { 40 | _fpp = fpp; 41 | return this; 42 | } 43 | 44 | public KuduBloomFilter Build() 45 | { 46 | int logSpaceBytes = BlockBloomFilter.MinLogSpace(_numKeys, _fpp); 47 | var blockBloomFilter = new BlockBloomFilter(logSpaceBytes); 48 | 49 | return new KuduBloomFilter(blockBloomFilter, _hashSeed, _column); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/SplitKeyRangeRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Google.Protobuf; 3 | using Knet.Kudu.Client.Protobuf.Tserver; 4 | using Knet.Kudu.Client.Protocol; 5 | 6 | namespace Knet.Kudu.Client.Requests; 7 | 8 | internal sealed class SplitKeyRangeRequest : KuduTabletRpc 9 | { 10 | private readonly SplitKeyRangeRequestPB _request; 11 | 12 | public SplitKeyRangeRequest( 13 | string tableId, 14 | byte[] startPrimaryKey, 15 | byte[] endPrimaryKey, 16 | byte[] partitionKey, 17 | long splitSizeBytes) 18 | { 19 | _request = new SplitKeyRangeRequestPB 20 | { 21 | TargetChunkSizeBytes = (ulong)splitSizeBytes 22 | }; 23 | 24 | if (startPrimaryKey != null && startPrimaryKey.Length > 0) 25 | _request.StartPrimaryKey = UnsafeByteOperations.UnsafeWrap(startPrimaryKey); 26 | 27 | if (endPrimaryKey != null && endPrimaryKey.Length > 0) 28 | _request.StopPrimaryKey = UnsafeByteOperations.UnsafeWrap(endPrimaryKey); 29 | 30 | MethodName = "SplitKeyRange"; 31 | TableId = tableId; 32 | PartitionKey = partitionKey; 33 | NeedsAuthzToken = true; 34 | } 35 | 36 | public override int CalculateSize() 37 | { 38 | _request.TabletId = ByteString.CopyFromUtf8(Tablet!.TabletId); 39 | _request.AuthzToken = AuthzToken; 40 | 41 | return _request.CalculateSize(); 42 | } 43 | 44 | public override void WriteTo(IBufferWriter output) => _request.WriteTo(output); 45 | 46 | public override void ParseResponse(KuduMessage message) 47 | { 48 | Output = SplitKeyRangeResponsePB.Parser.ParseFrom(message.MessageProtobuf); 49 | Error = Output.Error; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Protos/kudu/util/block_bloom_filter.proto: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | syntax = "proto2"; 18 | package kudu; 19 | 20 | option java_package = "org.apache.kudu"; 21 | option csharp_namespace = "Knet.Kudu.Client.Protobuf"; 22 | 23 | import "kudu/util/hash.proto"; 24 | import "kudu/util/pb_util.proto"; 25 | 26 | message BlockBloomFilterPB { 27 | // Log2 of the space required for the BlockBloomFilter. 28 | optional int32 log_space_bytes = 1; 29 | // The bloom filter bitmap. 30 | optional bytes bloom_data = 2 [(kudu.REDACT) = true]; 31 | // Whether the BlockBloomFilter is empty and hence always returns false for lookups. 32 | optional bool always_false = 3; 33 | // Hash algorithm to generate 32-bit unsigned integer hash values before inserting 34 | // in the BlockBloomFilter. 35 | optional HashAlgorithm hash_algorithm = 4 [default = FAST_HASH]; 36 | // Seed used to hash the input values in the hash algorithm. 37 | optional uint32 hash_seed = 5 [default = 0]; 38 | } 39 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Logging/LoggerHelperExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Google.Protobuf; 5 | using Knet.Kudu.Client.Connection; 6 | using Knet.Kudu.Client.Internal; 7 | using Knet.Kudu.Client.Protobuf; 8 | using Knet.Kudu.Client.Tablet; 9 | using Microsoft.Extensions.Logging; 10 | 11 | namespace Knet.Kudu.Client.Logging; 12 | 13 | internal static class LoggerHelperExtensions 14 | { 15 | public static void MisconfiguredMasterAddresses( 16 | this ILogger logger, 17 | IReadOnlyList clientMasters, 18 | IReadOnlyList clusterMasters) 19 | { 20 | var clientMastersStr = string.Join(",", clientMasters); 21 | var clusterMastersStr = string.Join(",", clusterMasters 22 | .Select(m => m.ToHostAndPort())); 23 | 24 | logger.MisconfiguredMasterAddresses( 25 | clientMasters.Count, 26 | clusterMasters.Count, 27 | clientMastersStr, 28 | clusterMastersStr); 29 | } 30 | 31 | public static void UnableToConnectToServer( 32 | this ILogger logger, 33 | Exception exception, 34 | ServerInfo serverInfo) 35 | { 36 | var hostPort = serverInfo.HostPort; 37 | var ip = serverInfo.Endpoint.Address; 38 | var uuid = serverInfo.Uuid; 39 | 40 | logger.UnableToConnectToServer(exception, hostPort, ip, uuid); 41 | } 42 | 43 | public static void ScannerExpired( 44 | this ILogger logger, 45 | ByteString scannerId, 46 | string tableName, 47 | RemoteTablet? tablet) 48 | { 49 | var scannerIdStr = scannerId.ToStringUtf8(); 50 | var tabletId = tablet?.TabletId; 51 | 52 | logger.ScannerExpired(scannerIdStr, tableName, tabletId); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.FunctionalTests/TimeoutTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | using Knet.Kudu.Client.FunctionalTests.MiniCluster; 5 | using Knet.Kudu.Client.FunctionalTests.Util; 6 | using McMaster.Extensions.Xunit; 7 | using Xunit; 8 | 9 | namespace Knet.Kudu.Client.FunctionalTests; 10 | 11 | [MiniKuduClusterTest] 12 | public class TimeoutTests 13 | { 14 | /// 15 | /// This test checks that, even if there is no event on the channel over which 16 | /// an RPC was sent (e.g., even if the server hangs and does not respond), RPCs 17 | /// will still time out. 18 | /// 19 | [SkippableFact] 20 | public async Task TestTimeoutEvenWhenServerHangs() 21 | { 22 | await using var harness = await new MiniKuduClusterBuilder() 23 | .AddTabletServerFlag("--scanner_inject_latency_on_each_batch_ms=200000") 24 | .BuildHarnessAsync(); 25 | 26 | await using var client = harness.CreateClient(); 27 | 28 | var tableBuilder = ClientTestUtil.GetBasicSchema() 29 | .SetTableName(nameof(TestTimeoutEvenWhenServerHangs)); 30 | 31 | var table = await client.CreateTableAsync(tableBuilder); 32 | 33 | var row = ClientTestUtil.CreateBasicSchemaInsert(table, 1); 34 | await client.WriteAsync(new[] { row }); 35 | 36 | var scanner = client.NewScanBuilder(table).Build(); 37 | 38 | // Scan with a short timeout. 39 | var timeout = TimeSpan.FromSeconds(1); 40 | using var cts = new CancellationTokenSource(timeout); 41 | 42 | // The server will not respond for the lifetime of the test, so we 43 | // expect the operation to time out. 44 | await Assert.ThrowsAsync( 45 | async () => await scanner.CountAsync(cts.Token)); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/KuduOperation.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace Knet.Kudu.Client; 5 | 6 | public class KuduOperation : PartialRowOperation 7 | { 8 | public KuduTable Table { get; } 9 | 10 | public KuduOperation(KuduTable table, RowOperation operation) 11 | : base(table.Schema, operation) 12 | { 13 | Table = table; 14 | } 15 | } 16 | 17 | public static class OperationsEncoder 18 | { 19 | public static void ComputeSize( 20 | List operations, 21 | out int rowSize, 22 | out int indirectSize) where T : PartialRowOperation 23 | { 24 | int localRowSize = 0; 25 | int localIndirectSize = 0; 26 | 27 | foreach (var row in operations) 28 | { 29 | row.CalculateSize(out int rSize, out int iSize); 30 | localRowSize += rSize + 1; // Add 1 for RowOperation. 31 | localIndirectSize += iSize; 32 | } 33 | 34 | rowSize = localRowSize; 35 | indirectSize = localIndirectSize; 36 | } 37 | 38 | public static void Encode( 39 | List operations, 40 | Span rowDestination, 41 | Span indirectDestination) where T : PartialRowOperation 42 | { 43 | int indirectDataOffset = 0; 44 | 45 | foreach (var row in operations) 46 | { 47 | row.WriteToWithOperation( 48 | rowDestination, 49 | indirectDestination, 50 | indirectDataOffset, 51 | out int rowBytesWritten, 52 | out int indirectBytesWritten); 53 | 54 | rowDestination = rowDestination.Slice(rowBytesWritten); 55 | indirectDestination = indirectDestination.Slice(indirectBytesWritten); 56 | 57 | indirectDataOffset += indirectBytesWritten; 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Protos/kudu/consensus/replica_management.proto: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | syntax = "proto2"; 19 | package kudu.consensus; 20 | 21 | option java_package = "org.apache.kudu.consensus"; 22 | option csharp_namespace = "Knet.Kudu.Client.Protobuf.Consensus"; 23 | 24 | // Communicates replica management information between servers. 25 | message ReplicaManagementInfoPB { 26 | // Replica replacement schemes. 27 | enum ReplacementScheme { 28 | UNKNOWN = 999; 29 | 30 | // The leader replica evicts the failed replica first, and then the new 31 | // voter replica is added (a.k.a. '3-2-3' replica management scheme). 32 | EVICT_FIRST = 0; 33 | 34 | // Add a new non-voter replica, promote the replica to voter once it 35 | // caught up with the leader, and only after that evict the failed replica 36 | // (a.k.a. '3-4-3' replica managment scheme). 37 | PREPARE_REPLACEMENT_BEFORE_EVICTION = 1; 38 | } 39 | 40 | // Using 'optional' instead of 'required' because at some point we may decide 41 | // to obsolete this field. 42 | optional ReplacementScheme replacement_scheme = 1; 43 | } 44 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/TabletServerInfo.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using Knet.Kudu.Client.Connection; 4 | 5 | namespace Knet.Kudu.Client; 6 | 7 | public class TabletServerInfo 8 | { 9 | /// 10 | /// Unique ID which is created when the server is first started 11 | /// up. This is stored persistently on disk. 12 | /// 13 | public string TsUuid { get; } 14 | 15 | public int MillisSinceHeartbeat { get; } 16 | 17 | public string Location { get; } 18 | 19 | public TabletServerState State { get; } 20 | 21 | public IReadOnlyList RpcAddresses { get; } 22 | 23 | public IReadOnlyList HttpAddresses { get; } 24 | 25 | public string SoftwareVersion { get; } 26 | 27 | /// 28 | /// True if HTTPS has been enabled for the web interface. 29 | /// In this case, https:// URLs should be generated for the above 30 | /// 'http_addresses' field. 31 | /// 32 | public bool HttpsEnabled { get; } 33 | 34 | /// 35 | /// The wall clock time when the server started. 36 | /// 37 | public DateTimeOffset StartTime { get; } 38 | 39 | public TabletServerInfo( 40 | string tsUuid, 41 | int millisSinceHeartbeat, 42 | string location, 43 | TabletServerState state, 44 | IReadOnlyList rpcAddresses, 45 | IReadOnlyList httpAddresses, 46 | string softwareVersion, 47 | bool httpsEnabled, 48 | DateTimeOffset startTime) 49 | { 50 | TsUuid = tsUuid; 51 | MillisSinceHeartbeat = millisSinceHeartbeat; 52 | Location = location; 53 | State = state; 54 | RpcAddresses = rpcAddresses; 55 | HttpAddresses = httpAddresses; 56 | SoftwareVersion = softwareVersion; 57 | HttpsEnabled = httpsEnabled; 58 | StartTime = startTime; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Util/Murmur2.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Runtime.InteropServices; 3 | 4 | namespace Knet.Kudu.Client.Util; 5 | 6 | /// 7 | /// A C# implementation of the Murmur 2 hashing algorithm as presented at 8 | /// https://sites.google.com/site/murmurhash. 9 | /// 10 | /// Hash64 is from 11 | /// https://sites.google.com/site/murmurhash/MurmurHash2_64.cpp 12 | /// 13 | public static class Murmur2 14 | { 15 | /// 16 | /// Compute the Murmur2 hash (64-bit version) as described in the original source code. 17 | /// 18 | /// The data that needs to be hashed. 19 | /// The seed to use to compute the hash. 20 | public static ulong Hash64(ReadOnlySpan key, ulong seed) 21 | { 22 | uint length = (uint)key.Length; 23 | const ulong m = 0xc6a4a7935bd1e995; 24 | const int r = 47; 25 | 26 | ulong h = seed ^ (length * m); 27 | 28 | var data = MemoryMarshal.Cast(key); 29 | 30 | foreach (ulong d in data) 31 | { 32 | ulong k = d; 33 | 34 | k *= m; 35 | k ^= k >> r; 36 | k *= m; 37 | 38 | h ^= k; 39 | h *= m; 40 | } 41 | 42 | var data2 = key.Slice(data.Length * sizeof(ulong)); 43 | 44 | switch (length & 7) 45 | { 46 | case 7: h ^= (ulong)data2[6] << 48; goto case 6; 47 | case 6: h ^= (ulong)data2[5] << 40; goto case 5; 48 | case 5: h ^= (ulong)data2[4] << 32; goto case 4; 49 | case 4: h ^= (ulong)data2[3] << 24; goto case 3; 50 | case 3: h ^= (ulong)data2[2] << 16; goto case 2; 51 | case 2: h ^= (ulong)data2[1] << 8; goto case 1; 52 | case 1: 53 | h ^= data2[0]; 54 | h *= m; 55 | break; 56 | } 57 | 58 | h ^= h >> r; 59 | h *= m; 60 | h ^= h >> r; 61 | 62 | return h; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.Tests/DecimalUtilTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Knet.Kudu.Client.Util; 3 | using Xunit; 4 | 5 | namespace Knet.Kudu.Client.Tests; 6 | 7 | public class DecimalUtilTests 8 | { 9 | [Theory] 10 | [InlineData(6023345402697246, 15782978151453050464, 10)] // 11111111111111111111111111.11111 11 | [InlineData(-6023345402697247, 2663765922256501152, 10)] //-11111111111111111111111111.11111 12 | [InlineData(667386670618854951, 11070687437558349184, 35)] // 123.1111111111111111111111111111 13 | [InlineData(-667386670618854952, 7376056636151202432, 35)] //-123.1111111111111111111111111111 14 | public void KuduDecimalTooLarge(long high, ulong low, int scale) 15 | { 16 | var value = new Int128((ulong)high, low); 17 | 18 | Assert.Throws( 19 | () => DecimalUtil.DecodeDecimal128(value, scale)); 20 | } 21 | 22 | [Theory] 23 | // Decimal values with precision of 9 or less are stored in 4 bytes. 24 | [InlineData(1, KuduType.Decimal32, 4)] 25 | [InlineData(9, KuduType.Decimal32, 4)] 26 | // Decimal values with precision of 10 through 18 are stored in 8 bytes. 27 | [InlineData(10, KuduType.Decimal64, 8)] 28 | [InlineData(18, KuduType.Decimal64, 8)] 29 | // Decimal values with precision of 19 through 38 are stored in 16 bytes. 30 | [InlineData(19, KuduType.Decimal128, 16)] 31 | [InlineData(38, KuduType.Decimal128, 16)] 32 | public void PrecisionToSize(int precision, KuduType expectedType, int expectedSize) 33 | { 34 | var type = DecimalUtil.PrecisionToKuduType(precision); 35 | Assert.Equal(expectedType, type); 36 | 37 | var size = DecimalUtil.PrecisionToSize(precision); 38 | Assert.Equal(expectedSize, size); 39 | } 40 | 41 | [Fact] 42 | public void PrecisionTooLarge() 43 | { 44 | Assert.Throws(() => DecimalUtil.PrecisionToKuduType(39)); 45 | Assert.Throws(() => DecimalUtil.PrecisionToSize(39)); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Internal/AuthzTokenCache.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Concurrent; 2 | using Knet.Kudu.Client.Protobuf.Security; 3 | 4 | namespace Knet.Kudu.Client.Internal; 5 | 6 | /// 7 | /// Cache for authz tokens received from the master of unbounded capacity. A 8 | /// client will receive an authz token upon opening a table and put it into the 9 | /// cache. A subsequent operation that requires an authz token (e.g. writes, 10 | /// scans) will fetch it from the cache and attach it to the operation request. 11 | /// 12 | internal sealed class AuthzTokenCache 13 | { 14 | // Map from a table ID to an authz token for that table. 15 | private readonly ConcurrentDictionary _cache; 16 | 17 | public AuthzTokenCache() 18 | { 19 | _cache = new ConcurrentDictionary(); 20 | } 21 | 22 | /// 23 | /// Puts the given token into the cache. No validation is done on the validity 24 | /// or expiration of the token -- that happens on the tablet servers. 25 | /// 26 | /// The table ID the authz token is for. 27 | /// An authz token to put into the cache. 28 | public void SetAuthzToken(string tableId, SignedTokenPB token) 29 | { 30 | _cache[tableId] = token; 31 | } 32 | 33 | /// 34 | /// Returns the cached token for the given 'tableId' if one exists. 35 | /// 36 | /// Table ID to get an authz token for. 37 | public SignedTokenPB? GetAuthzToken(string tableId) 38 | { 39 | _cache.TryGetValue(tableId, out var token); 40 | return token; 41 | } 42 | 43 | /// 44 | /// Removes the cached token for the given 'tableId' if one exists. 45 | /// 46 | /// Table ID to clear authz token for. 47 | public void RemoveAuthzToken(string tableId) 48 | { 49 | _cache.TryRemove(tableId, out _); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Protos/kudu/util/pb_util.proto: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | syntax = "proto2"; 18 | package kudu; 19 | 20 | option java_package = "org.apache.kudu"; 21 | option csharp_namespace = "Knet.Kudu.Client.Protobuf"; 22 | 23 | import "google/protobuf/descriptor.proto"; 24 | 25 | // ============================================================================ 26 | // Protobuf container metadata 27 | // ============================================================================ 28 | 29 | // Supplemental protobuf container header, after the main header (see 30 | // pb_util.h for details). 31 | message ContainerSupHeaderPB { 32 | // The protobuf schema for the messages expected in this container. 33 | // 34 | // This schema is complete, that is, it includes all of its dependencies 35 | // (i.e. other schemas defined in .proto files imported by this schema's 36 | // .proto file). 37 | required google.protobuf.FileDescriptorSet protos = 1; 38 | 39 | // The PB message type expected in each data entry in this container. Must 40 | // be fully qualified (i.e. kudu.tablet.TabletSuperBlockPB). 41 | required string pb_type = 2; 42 | } 43 | 44 | extend google.protobuf.FieldOptions { 45 | optional bool REDACT = 50001 [default=false]; 46 | } -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.FunctionalTests/LeaderFailoverTests.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using System.Threading.Tasks; 3 | using Knet.Kudu.Client.FunctionalTests.MiniCluster; 4 | using Knet.Kudu.Client.FunctionalTests.Util; 5 | using McMaster.Extensions.Xunit; 6 | using Xunit; 7 | 8 | namespace Knet.Kudu.Client.FunctionalTests; 9 | 10 | [MiniKuduClusterTest] 11 | public class LeaderFailoverTests 12 | { 13 | /// 14 | /// This test writes 3 rows, kills the leader, then tries to write another 3 rows. 15 | /// Finally it counts to make sure we have 6 of them. 16 | /// 17 | [SkippableTheory] 18 | [InlineData(true)] 19 | [InlineData(false)] 20 | public async Task TestFailover(bool restart) 21 | { 22 | await using var harness = await new MiniKuduClusterBuilder() 23 | .NumMasters(3) 24 | .NumTservers(3) 25 | .BuildHarnessAsync(); 26 | 27 | await using var client = harness.CreateClient(); 28 | 29 | var builder = ClientTestUtil.GetBasicSchema() 30 | .SetTableName("LeaderFailoverTest") 31 | .CreateBasicRangePartition(); 32 | var table = await client.CreateTableAsync(builder); 33 | 34 | var rows = Enumerable.Range(0, 3) 35 | .Select(i => ClientTestUtil.CreateBasicSchemaInsert(table, i)); 36 | 37 | await client.WriteAsync(rows); 38 | 39 | // Make sure the rows are in there before messing things up. 40 | long numRows = await ClientTestUtil.CountRowsAsync(client, table); 41 | Assert.Equal(3, numRows); 42 | 43 | if (restart) 44 | await harness.RestartLeaderMasterAsync(); 45 | else 46 | await harness.KillLeaderMasterServerAsync(); 47 | 48 | var rows2 = Enumerable.Range(3, 3) 49 | .Select(i => ClientTestUtil.CreateBasicSchemaInsert(table, i)); 50 | 51 | await client.WriteAsync(rows2); 52 | 53 | long numRows2 = await ClientTestUtil.CountRowsAsync(client, table); 54 | 55 | Assert.Equal(6, numRows2); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Internal/SecurityUtil.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Security.Cryptography; 3 | using System.Security.Cryptography.X509Certificates; 4 | 5 | namespace Knet.Kudu.Client.Internal; 6 | 7 | internal static class SecurityUtil 8 | { 9 | public static byte[] GetEndpointChannelBindings(this X509Certificate2 certificate) 10 | { 11 | using var hashAlgo = GetHashForChannelBinding(certificate); 12 | return hashAlgo.ComputeHash(certificate.RawData); 13 | } 14 | 15 | private static HashAlgorithm GetHashForChannelBinding(X509Certificate2 cert) 16 | { 17 | // https://github.com/dotnet/runtime/blob/master/src/libraries/System.Net.Security/src/System/Net/Security/Pal.Managed/EndpointChannelBindingToken.cs 18 | var signatureAlgorithm = cert.SignatureAlgorithm; 19 | switch (signatureAlgorithm.Value) 20 | { 21 | // RFC 5929 4.1 says that MD5 and SHA1 both upgrade to SHA256 for cbt calculation 22 | case "1.2.840.113549.2.5": // MD5 23 | case "1.2.840.113549.1.1.4": // MD5RSA 24 | case "1.3.14.3.2.26": // SHA1 25 | case "1.2.840.10040.4.3": // SHA1DSA 26 | case "1.2.840.10045.4.1": // SHA1ECDSA 27 | case "1.2.840.113549.1.1.5": // SHA1RSA 28 | case "2.16.840.1.101.3.4.2.1": // SHA256 29 | case "1.2.840.10045.4.3.2": // SHA256ECDSA 30 | case "1.2.840.113549.1.1.11": // SHA256RSA 31 | return SHA256.Create(); 32 | 33 | case "2.16.840.1.101.3.4.2.2": // SHA384 34 | case "1.2.840.10045.4.3.3": // SHA384ECDSA 35 | case "1.2.840.113549.1.1.12": // SHA384RSA 36 | return SHA384.Create(); 37 | 38 | case "2.16.840.1.101.3.4.2.3": // SHA512 39 | case "1.2.840.10045.4.3.4": // SHA512ECDSA 40 | case "1.2.840.113549.1.1.13": // SHA512RSA 41 | return SHA512.Create(); 42 | 43 | default: 44 | throw new ArgumentException(signatureAlgorithm.Value); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.Tests/EndpointParserTests.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Util; 2 | using Xunit; 3 | 4 | namespace Knet.Kudu.Client.Tests; 5 | 6 | /// 7 | /// Tests from https://github.com/StackExchange/StackExchange.Redis/blob/master/tests/StackExchange.Redis.Tests/FormatTests.cs 8 | /// 9 | public class EndpointParserTests 10 | { 11 | [Theory] 12 | [InlineData("localhost", "localhost", 0)] 13 | [InlineData("localhost:6390", "localhost", 6390)] 14 | [InlineData("bob.the.builder.com", "bob.the.builder.com", 0)] 15 | [InlineData("bob.the.builder.com:6390", "bob.the.builder.com", 6390)] 16 | // IPv4 17 | [InlineData("0.0.0.0", "0.0.0.0", 0)] 18 | [InlineData("127.0.0.1", "127.0.0.1", 0)] 19 | [InlineData("127.1", "127.1", 0)] 20 | [InlineData("127.1:6389", "127.1", 6389)] 21 | [InlineData("127.0.0.1:6389", "127.0.0.1", 6389)] 22 | [InlineData("127.0.0.1:1", "127.0.0.1", 1)] 23 | [InlineData("127.0.0.1:2", "127.0.0.1", 2)] 24 | [InlineData("10.10.9.18:2", "10.10.9.18", 2)] 25 | // IPv6 26 | [InlineData("::1", "::1", 0)] 27 | [InlineData("[::1]:6379", "::1", 6379)] 28 | [InlineData("[::1]", "::1", 0)] 29 | [InlineData("[::1]:1000", "::1", 1000)] 30 | [InlineData("[2001:db7:85a3:8d2:1319:8a2e:370:7348]", "2001:db7:85a3:8d2:1319:8a2e:370:7348", 0)] 31 | [InlineData("[2001:db7:85a3:8d2:1319:8a2e:370:7348]:1000", "2001:db7:85a3:8d2:1319:8a2e:370:7348", 1000)] 32 | public void CanParseEndpoint(string endpoint, string expectedHost, int expectedPort) 33 | { 34 | var success = EndpointParser.TryParse(endpoint, 0, out var hostPort); 35 | 36 | Assert.True(success); 37 | Assert.Equal(expectedHost, hostPort.Host); 38 | Assert.Equal(expectedPort, hostPort.Port); 39 | } 40 | 41 | [Fact] 42 | public void CanUseDefaultPort() 43 | { 44 | var success = EndpointParser.TryParse("127.0.0.1", 7051, out var hostPort); 45 | 46 | Assert.True(success); 47 | Assert.Equal("127.0.0.1", hostPort.Host); 48 | Assert.Equal(7051, hostPort.Port); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Util/HybridTimeUtil.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Knet.Kudu.Client.Util; 4 | 5 | /// 6 | /// Set of common utility methods to handle HybridTime and related timestamps. 7 | /// 8 | public class HybridTimeUtil 9 | { 10 | public const int HybridTimeNumBitsToShift = 12; 11 | public const int HybridTimeLogicalBitsMask = (1 << HybridTimeNumBitsToShift) - 1; 12 | 13 | /// 14 | /// Converts the provided timestamp in microseconds to the HybridTime 15 | /// timestamp format. Logical bits are set to 0. 16 | /// 17 | /// 18 | /// The value of the timestamp, must be greater than 0. 19 | /// 20 | public static long ClockTimestampToHtTimestamp(long timestampInMicros) 21 | { 22 | if (timestampInMicros < 0) 23 | { 24 | throw new ArgumentOutOfRangeException( 25 | nameof(timestampInMicros), "Timestamp cannot be less than 0"); 26 | } 27 | 28 | return timestampInMicros << HybridTimeNumBitsToShift; 29 | } 30 | 31 | /// 32 | /// Extracts the physical and logical values from an HT timestamp. 33 | /// 34 | /// The encoded HT timestamp. 35 | public static (long timestampMicros, long logicalValue) HtTimestampToPhysicalAndLogical( 36 | long htTimestamp) 37 | { 38 | long timestampInMicros = htTimestamp >> HybridTimeNumBitsToShift; 39 | long logicalValues = htTimestamp & HybridTimeLogicalBitsMask; 40 | return (timestampInMicros, logicalValues); 41 | } 42 | 43 | /// 44 | /// Encodes separate physical and logical components into a single HT timestamp. 45 | /// 46 | /// The physical component, in microseconds. 47 | /// The logical component. 48 | public static long PhysicalAndLogicalToHtTimestamp(long physical, long logical) 49 | { 50 | return (physical << HybridTimeNumBitsToShift) + logical; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Requests/KuduRpc.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Buffers; 3 | using Google.Protobuf.Collections; 4 | using Knet.Kudu.Client.Connection; 5 | using Knet.Kudu.Client.Protocol; 6 | 7 | namespace Knet.Kudu.Client.Requests; 8 | 9 | // TODO: These types need a lot of refactoring. 10 | public abstract class KuduRpc 11 | { 12 | // Service names. 13 | protected const string MasterServiceName = "kudu.master.MasterService"; 14 | protected const string TabletServerServiceName = "kudu.tserver.TabletServerService"; 15 | protected const string TxnManagerServiceName = "kudu.transactions.TxnManagerService"; 16 | 17 | public string ServiceName { get; init; } = null!; 18 | 19 | public string MethodName { get; init; } = null!; 20 | 21 | /// 22 | /// Returns the set of application-specific feature flags required to service the RPC. 23 | /// 24 | public RepeatedField? RequiredFeatures { get; init; } 25 | 26 | /// 27 | /// The external consistency mode for this RPC. 28 | /// 29 | public ExternalConsistencyMode ExternalConsistencyMode { get; init; } = 30 | ExternalConsistencyMode.ClientPropagated; 31 | 32 | /// 33 | /// The number of times this RPC has been retried. 34 | /// 35 | internal int Attempt { get; set; } 36 | 37 | /// 38 | /// The last exception when handling this RPC. 39 | /// 40 | internal Exception? Exception { get; set; } 41 | 42 | /// 43 | /// If this RPC needs to be tracked on the client and server-side. 44 | /// Some RPCs require exactly-once semantics which is enabled by tracking them. 45 | /// 46 | public bool IsRequestTracked { get; init; } 47 | 48 | internal long SequenceId { get; set; } = RequestTracker.NoSeqNo; 49 | 50 | public abstract int CalculateSize(); 51 | 52 | public abstract void WriteTo(IBufferWriter output); 53 | 54 | public abstract void ParseResponse(KuduMessage message); 55 | } 56 | 57 | public abstract class KuduRpc : KuduRpc 58 | { 59 | public T? Output { get; protected set; } 60 | } 61 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/RowOperation.cs: -------------------------------------------------------------------------------- 1 | using static Knet.Kudu.Client.Protobuf.RowOperationsPB.Types; 2 | 3 | namespace Knet.Kudu.Client; 4 | 5 | /// 6 | /// A set of operations (INSERT, UPDATE, UPSERT, or DELETE) to apply to a table, 7 | /// or the set of split rows and range bounds when creating or altering table. 8 | /// Range bounds determine the boundaries of range partitions during table 9 | /// creation, split rows further subdivide the ranges into more partitions. 10 | /// 11 | public enum RowOperation : byte 12 | { 13 | Insert = Type.Insert, 14 | Update = Type.Update, 15 | Delete = Type.Delete, 16 | Upsert = Type.Upsert, 17 | InsertIgnore = Type.InsertIgnore, 18 | UpdateIgnore = Type.UpdateIgnore, 19 | DeleteIgnore = Type.DeleteIgnore, 20 | /// 21 | /// Used when specifying split rows on table creation. 22 | /// 23 | SplitRow = Type.SplitRow, 24 | /// 25 | /// Used when specifying an inclusive lower bound range on table creation. 26 | /// Should be followed by the associated upper bound. If all values are 27 | /// missing, then signifies unbounded. 28 | /// 29 | RangeLowerBound = Type.RangeLowerBound, 30 | /// 31 | /// Used when specifying an exclusive upper bound range on table creation. 32 | /// Should be preceded by the associated lower bound. If all values are 33 | /// missing, then signifies unbounded. 34 | /// 35 | RangeUpperBound = Type.RangeUpperBound, 36 | /// 37 | /// Used when specifying an exclusive lower bound range on table creation. 38 | /// Should be followed by the associated upper bound. If all values are 39 | /// missing, then signifies unbounded. 40 | /// 41 | ExclusiveRangeLowerBound = Type.ExclusiveRangeLowerBound, 42 | /// 43 | /// Used when specifying an inclusive upper bound range on table creation. 44 | /// Should be preceded by the associated lower bound. If all values are 45 | /// missing, then signifies unbounded. 46 | /// 47 | InclusiveRangeUpperBound = Type.InclusiveRangeUpperBound, 48 | } 49 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/ResourceMetrics.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Protobuf.Tserver; 2 | 3 | namespace Knet.Kudu.Client; 4 | 5 | public class ResourceMetrics 6 | { 7 | /// 8 | /// Number of bytes that were read because of a block cache miss. 9 | /// 10 | public long CfileCacheMissBytes { get; private set; } 11 | 12 | /// 13 | /// Number of bytes that were read from the block cache because of a hit. 14 | /// 15 | public long CfileCacheHitBytes { get; private set; } 16 | 17 | /// 18 | /// Number of bytes read from disk (or cache) by the scanner. 19 | /// 20 | public long BytesRead { get; private set; } 21 | 22 | /// 23 | /// Total time taken between scan rpc requests being accepted and when they 24 | /// were handled in nanoseconds for this scanner. 25 | /// 26 | public long QueueDurationNanos { get; private set; } 27 | 28 | /// 29 | /// Total time taken for all scan rpc requests to complete in nanoseconds 30 | /// for this scanner. 31 | /// 32 | public long TotalDurationNanos { get; private set; } 33 | 34 | /// 35 | /// Total elapsed CPU user time in nanoseconds for all scan rpc requests 36 | /// for this scanner. 37 | /// 38 | public long CpuUserNanos { get; private set; } 39 | 40 | /// 41 | /// Total elapsed CPU system time in nanoseconds for all scan rpc requests 42 | /// for this scanner. 43 | /// 44 | public long CpuSystemNanos { get; private set; } 45 | 46 | internal void Update(ResourceMetricsPB resourceMetricsPb) 47 | { 48 | CfileCacheMissBytes += resourceMetricsPb.CfileCacheMissBytes; 49 | CfileCacheHitBytes += resourceMetricsPb.CfileCacheHitBytes; 50 | BytesRead += resourceMetricsPb.BytesRead; 51 | QueueDurationNanos += resourceMetricsPb.QueueDurationNanos; 52 | TotalDurationNanos += resourceMetricsPb.TotalDurationNanos; 53 | CpuUserNanos += resourceMetricsPb.CpuUserNanos; 54 | CpuSystemNanos += resourceMetricsPb.CpuSystemNanos; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Connection/RequestTracker.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Diagnostics; 3 | 4 | namespace Knet.Kudu.Client.Connection; 5 | 6 | public class RequestTracker 7 | { 8 | public const long NoSeqNo = -1; 9 | 10 | private readonly SortedSet _incompleteRpcs; 11 | private long _nextSeqNo; 12 | 13 | public string ClientId { get; } 14 | 15 | public RequestTracker(string clientId) 16 | { 17 | ClientId = clientId; 18 | _incompleteRpcs = new SortedSet(); 19 | _nextSeqNo = 1; 20 | } 21 | 22 | /// 23 | /// Returns the oldest sequence number that wasn't marked as completed. 24 | /// If there is no incomplete RPC then is returned. 25 | /// 26 | public long FirstIncomplete 27 | { 28 | get 29 | { 30 | lock (_incompleteRpcs) 31 | { 32 | if (_incompleteRpcs.Count == 0) 33 | return NoSeqNo; 34 | 35 | return _incompleteRpcs.Min; 36 | } 37 | } 38 | } 39 | 40 | /// 41 | /// Generates a new sequence number and tracks it. 42 | /// 43 | public long GetNewSeqNo() 44 | { 45 | lock (_incompleteRpcs) 46 | { 47 | long seq = _nextSeqNo++; 48 | _incompleteRpcs.Add(seq); 49 | return seq; 50 | } 51 | } 52 | 53 | /// 54 | /// Marks the given sequence number as complete. The provided sequence number must 55 | /// be a valid number that was previously returned by . 56 | /// It is illegal to call this method twice with the same sequence number. 57 | /// 58 | /// The sequence number to mark as complete. 59 | public void CompleteRpc(long sequenceNo) 60 | { 61 | Debug.Assert(sequenceNo != NoSeqNo); 62 | lock (_incompleteRpcs) 63 | { 64 | bool removed = _incompleteRpcs.Remove(sequenceNo); 65 | Debug.Assert(removed, $"Could not remove seqid {sequenceNo} from request tracker"); 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.FunctionalTests/HandleTooBusyTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading.Tasks; 4 | using Knet.Kudu.Client.FunctionalTests.MiniCluster; 5 | using Knet.Kudu.Client.FunctionalTests.Util; 6 | using McMaster.Extensions.Xunit; 7 | 8 | namespace Knet.Kudu.Client.FunctionalTests; 9 | 10 | [MiniKuduClusterTest] 11 | public class HandleTooBusyTests 12 | { 13 | /// 14 | /// Provoke overflows in the master RPC queue while connecting to the master 15 | /// and performing location lookups. 16 | /// 17 | [SkippableFact] 18 | public async Task TestMasterLookupOverflow() 19 | { 20 | var tableName = "TestHandleTooBusy"; 21 | 22 | await using var harness = await new MiniKuduClusterBuilder() 23 | // Short queue to provoke overflow. 24 | .AddMasterServerFlag("--rpc_service_queue_length=1") 25 | // Low number of service threads, so things stay in the queue. 26 | .AddMasterServerFlag("--rpc_num_service_threads=3") 27 | // Inject latency so lookups process slowly. 28 | .AddMasterServerFlag("--master_inject_latency_on_tablet_lookups_ms=100") 29 | .BuildHarnessAsync(); 30 | 31 | await using var client1 = harness.CreateClient(); 32 | 33 | await client1.CreateTableAsync(ClientTestUtil.GetBasicSchema() 34 | .SetTableName(tableName)); 35 | 36 | var tasks = new List(); 37 | 38 | for (int i = 0; i < 10; i++) 39 | { 40 | var task = Task.Run(async () => 41 | { 42 | for (int j = 0; j < 5; j++) 43 | { 44 | await using var client = harness.CreateClient(); 45 | var table = await client.OpenTableAsync(tableName); 46 | 47 | for (int k = 0; k < 5; k++) 48 | { 49 | await client.GetTableLocationsAsync( 50 | table.TableId, Array.Empty(), 1); 51 | } 52 | } 53 | }); 54 | 55 | tasks.Add(task); 56 | } 57 | 58 | await Task.WhenAll(tasks); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.Tests/EpochTimeTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Knet.Kudu.Client.Util; 3 | using Xunit; 4 | 5 | namespace Knet.Kudu.Client.Tests; 6 | 7 | public class EpochTimeTests 8 | { 9 | [Theory] 10 | [InlineData("1970-01-01T00:00:00.0000000Z", 0)] 11 | [InlineData("1970-01-01T00:00:00.1234560Z", 123456)] 12 | [InlineData("1923-12-01T00:44:36.8765440Z", -1454368523123456)] 13 | [InlineData("2018-12-25T15:44:30.5510000Z", 1545752670551000)] 14 | public void TestDateTimeConversion(string date, long micros) 15 | { 16 | var dateTime = DateTime.Parse(date).ToUniversalTime(); 17 | 18 | var toMicros = EpochTime.ToUnixTimeMicros(dateTime); 19 | Assert.Equal(micros, toMicros); 20 | 21 | var fromMicros = EpochTime.FromUnixTimeMicros(micros); 22 | Assert.Equal(dateTime, fromMicros); 23 | } 24 | 25 | [Fact] 26 | public void TestNonZuluDateTimeConversion() 27 | { 28 | var dateTime = DateTime.Parse("2016-08-19T12:12:12.1210000"); 29 | var micros = 1471626732121000; 30 | var timeZone = TimeZoneInfo.FindSystemTimeZoneById("America/Chicago"); 31 | 32 | var utcTime = TimeZoneInfo.ConvertTimeToUtc(dateTime, timeZone); 33 | var localTime = utcTime.ToLocalTime(); 34 | 35 | var toMicros = EpochTime.ToUnixTimeMicros(localTime); 36 | Assert.Equal(micros, toMicros); 37 | 38 | var fromMicros = EpochTime.FromUnixTimeMicros(micros); 39 | Assert.Equal(utcTime, fromMicros); 40 | } 41 | 42 | [Theory] 43 | [InlineData(int.MinValue)] 44 | [InlineData(int.MaxValue)] 45 | [InlineData(EpochTime.MinDateValue - 1)] 46 | [InlineData(EpochTime.MaxDateValue + 1)] 47 | public void TestDateOutOfRange(int days) 48 | { 49 | Assert.Throws( 50 | () => EpochTime.CheckDateWithinRange(days)); 51 | } 52 | 53 | [Theory] 54 | [InlineData("1/1/0001", EpochTime.MinDateValue)] 55 | [InlineData("12/31/9999", EpochTime.MaxDateValue)] 56 | public void TestDateConversion(DateTime date, int days) 57 | { 58 | var toDays = EpochTime.ToUnixTimeDays(date); 59 | Assert.Equal(days, toDays); 60 | 61 | var fromDays = EpochTime.FromUnixTimeDays(days); 62 | Assert.Equal(date, fromDays); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Mapper/DelegateCache.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Diagnostics.CodeAnalysis; 5 | 6 | namespace Knet.Kudu.Client.Mapper; 7 | 8 | internal sealed class DelegateCache 9 | { 10 | private readonly ConcurrentDictionary _cache = new(new CacheKeyComparer()); 11 | 12 | public bool TryGetDelegate( 13 | Type destinationType, 14 | KuduSchema projectionSchema, 15 | [NotNullWhen(true)] out Delegate? value) 16 | { 17 | var key = new CacheKey(destinationType, projectionSchema); 18 | return _cache.TryGetValue(key, out value); 19 | } 20 | 21 | public void AddDelegate( 22 | Type destinationType, 23 | KuduSchema projectionSchema, 24 | Delegate value) 25 | { 26 | var key = new CacheKey(destinationType, projectionSchema); 27 | _cache[key] = value; 28 | } 29 | 30 | private readonly record struct CacheKey(Type DestinationType, KuduSchema Schema); 31 | 32 | private sealed class CacheKeyComparer : IEqualityComparer 33 | { 34 | public bool Equals(CacheKey x, CacheKey y) 35 | { 36 | if (x.DestinationType != y.DestinationType) 37 | return false; 38 | 39 | var columnsX = x.Schema.Columns; 40 | var columnsY = y.Schema.Columns; 41 | 42 | if (columnsX.Count != columnsY.Count) 43 | return false; 44 | 45 | var numColumns = columnsX.Count; 46 | 47 | for (int i = 0; i < numColumns; i++) 48 | { 49 | var columnX = columnsX[i]; 50 | var columnY = columnsY[i]; 51 | 52 | if (columnX != columnY) 53 | { 54 | return false; 55 | } 56 | } 57 | 58 | return true; 59 | } 60 | 61 | public int GetHashCode([DisallowNull] CacheKey obj) 62 | { 63 | var hashcode = new HashCode(); 64 | hashcode.Add(obj.DestinationType); 65 | 66 | foreach (var column in obj.Schema.Columns) 67 | { 68 | hashcode.Add(column); 69 | } 70 | 71 | return hashcode.ToHashCode(); 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.FunctionalTests/SessionTests.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using Knet.Kudu.Client.Exceptions; 3 | using Knet.Kudu.Client.FunctionalTests.MiniCluster; 4 | using Knet.Kudu.Client.FunctionalTests.Util; 5 | using McMaster.Extensions.Xunit; 6 | using Xunit; 7 | 8 | namespace Knet.Kudu.Client.FunctionalTests; 9 | 10 | [MiniKuduClusterTest] 11 | public class SessionTests : IAsyncLifetime 12 | { 13 | private KuduTestHarness _harness; 14 | private KuduClient _client; 15 | 16 | public async Task InitializeAsync() 17 | { 18 | _harness = await new MiniKuduClusterBuilder().BuildHarnessAsync(); 19 | _client = _harness.CreateClient(); 20 | } 21 | 22 | public async Task DisposeAsync() 23 | { 24 | await _client.DisposeAsync(); 25 | await _harness.DisposeAsync(); 26 | } 27 | 28 | [SkippableFact] 29 | public async Task TestExceptionCallback() 30 | { 31 | int numCallbacks = 0; 32 | SessionExceptionContext sessionContext = null; 33 | 34 | var builder = ClientTestUtil.GetBasicSchema() 35 | .SetTableName(nameof(TestExceptionCallback)); 36 | 37 | var table = await _client.CreateTableAsync(builder); 38 | var row1 = ClientTestUtil.CreateBasicSchemaInsert(table, 1); 39 | var row2 = ClientTestUtil.CreateBasicSchemaInsert(table, 1); 40 | 41 | var sessionOptions = new KuduSessionOptions 42 | { 43 | ExceptionHandler = HandleSessionExceptionAsync 44 | }; 45 | 46 | await using var session = _client.NewSession(sessionOptions); 47 | 48 | await session.EnqueueAsync(row1); 49 | await session.FlushAsync(); 50 | 51 | await session.EnqueueAsync(row2); 52 | await session.FlushAsync(); 53 | 54 | ValueTask HandleSessionExceptionAsync(SessionExceptionContext context) 55 | { 56 | numCallbacks++; 57 | sessionContext = context; 58 | return new ValueTask(); 59 | } 60 | 61 | Assert.Equal(1, numCallbacks); 62 | 63 | var errorRow = Assert.Single(sessionContext.Rows); 64 | Assert.Same(row2, errorRow); 65 | 66 | var exception = Assert.IsType(sessionContext.Exception); 67 | var exceptionRow = Assert.Single(exception.PerRowErrors); 68 | Assert.True(exceptionRow.IsAlreadyPresent); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.FunctionalTests/MiniCluster/MiniKuduClusterBuilder.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using System.Threading.Tasks; 3 | using Knet.Kudu.Client.Protobuf.Tools; 4 | 5 | namespace Knet.Kudu.Client.FunctionalTests.MiniCluster; 6 | 7 | public class MiniKuduClusterBuilder 8 | { 9 | private readonly CreateClusterRequestPB _options; 10 | 11 | public MiniKuduClusterBuilder() 12 | { 13 | _options = new CreateClusterRequestPB 14 | { 15 | NumMasters = 3, 16 | NumTservers = 3 17 | }; 18 | } 19 | 20 | /// 21 | /// Builds and starts a new . 22 | /// 23 | public async Task BuildAsync() 24 | { 25 | if (string.IsNullOrWhiteSpace(_options.ClusterRoot)) 26 | { 27 | _options.ClusterRoot = Path.Combine( 28 | Path.GetTempPath(), 29 | $"mini-kudu-cluster-{Path.GetFileNameWithoutExtension(Path.GetRandomFileName())}"); 30 | } 31 | 32 | var miniCluster = new MiniKuduCluster(_options); 33 | await miniCluster.StartAsync(); 34 | return miniCluster; 35 | } 36 | 37 | public async Task BuildHarnessAsync() 38 | { 39 | var miniCluster = await BuildAsync(); 40 | return new KuduTestHarness(miniCluster, disposeMiniCluster: true); 41 | } 42 | 43 | public MiniKuduClusterBuilder NumMasters(int numMasters) 44 | { 45 | _options.NumMasters = numMasters; 46 | return this; 47 | } 48 | 49 | public MiniKuduClusterBuilder NumTservers(int numTservers) 50 | { 51 | _options.NumTservers = numTservers; 52 | return this; 53 | } 54 | 55 | /// 56 | /// Adds a new flag to be passed to the Master daemons on start. 57 | /// 58 | /// The flag to pass. 59 | public MiniKuduClusterBuilder AddMasterServerFlag(string flag) 60 | { 61 | _options.ExtraMasterFlags.Add(flag); 62 | return this; 63 | } 64 | 65 | /// 66 | /// Adds a new flag to be passed to the Tablet Server daemons on start. 67 | /// 68 | /// The flag to pass. 69 | public MiniKuduClusterBuilder AddTabletServerFlag(string flag) 70 | { 71 | _options.ExtraTserverFlags.Add(flag); 72 | return this; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Tablet/TableLocationEntry.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics.CodeAnalysis; 2 | using Knet.Kudu.Client.Util; 3 | 4 | namespace Knet.Kudu.Client.Tablet; 5 | 6 | public class TableLocationEntry 7 | { 8 | /// 9 | /// The lower bound partition key. 10 | /// 11 | public byte[] LowerBoundPartitionKey { get; } 12 | 13 | /// 14 | /// The upper bound partition key. 15 | /// 16 | public byte[] UpperBoundPartitionKey { get; } 17 | 18 | /// 19 | /// The remote tablet, only set if this entry represents a tablet. 20 | /// 21 | public RemoteTablet? Tablet { get; } 22 | 23 | /// 24 | /// When this entry will expire, based on . 25 | /// 26 | public long Expiration { get; } 27 | 28 | public TableLocationEntry( 29 | RemoteTablet? tablet, 30 | byte[] lowerBoundPartitionKey, 31 | byte[] upperBoundPartitionKey, 32 | long expiration) 33 | { 34 | Tablet = tablet; 35 | LowerBoundPartitionKey = lowerBoundPartitionKey; 36 | UpperBoundPartitionKey = upperBoundPartitionKey; 37 | Expiration = expiration; 38 | } 39 | 40 | /// 41 | /// If this entry is a non-covered range. 42 | /// 43 | [MemberNotNullWhen(false, nameof(Tablet))] 44 | public bool IsNonCoveredRange => Tablet is null; 45 | 46 | /// 47 | /// If this entry is a covered range. 48 | /// 49 | [MemberNotNullWhen(true, nameof(Tablet))] 50 | public bool IsCoveredRange => Tablet is not null; 51 | 52 | public static TableLocationEntry NewNonCoveredRange( 53 | byte[] lowerBoundPartitionKey, 54 | byte[] upperBoundPartitionKey, 55 | long expiration) 56 | { 57 | return new TableLocationEntry( 58 | null, 59 | lowerBoundPartitionKey, 60 | upperBoundPartitionKey, 61 | expiration); 62 | } 63 | 64 | public static TableLocationEntry NewCoveredRange(RemoteTablet tablet, long expiration) 65 | { 66 | var partition = tablet.Partition; 67 | var lowerBoundPartitionKey = partition.PartitionKeyStart; 68 | var upperBoundPartitionKey = partition.PartitionKeyEnd; 69 | 70 | return new TableLocationEntry( 71 | tablet, lowerBoundPartitionKey, upperBoundPartitionKey, expiration); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /examples/InsertLoadgen/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using Knet.Kudu.Client; 6 | using Microsoft.Extensions.Logging; 7 | 8 | namespace InsertLoadgen; 9 | 10 | class Program 11 | { 12 | static async Task Main() 13 | { 14 | var masterAddresses = "localhost:7051,localhost:7151,localhost:7251"; 15 | var tableName = $"test_table_{Guid.NewGuid():N}"; 16 | var numRows = 10000; 17 | 18 | var loggerFactory = LoggerFactory.Create(builder => builder 19 | .SetMinimumLevel(LogLevel.Trace) 20 | .AddConsole()); 21 | 22 | await using var client = KuduClient.NewBuilder(masterAddresses) 23 | .SetLoggerFactory(loggerFactory) 24 | .Build(); 25 | 26 | var tableBuilder = new TableBuilder(tableName) 27 | .AddColumn("host", KuduType.String, opt => opt.Key(true)) 28 | .AddColumn("metric", KuduType.String, opt => opt.Key(true)) 29 | .AddColumn("timestamp", KuduType.UnixtimeMicros, opt => opt.Key(true)) 30 | .AddColumn("value", KuduType.Double) 31 | .SetNumReplicas(1) 32 | .SetRangePartitionColumns("host", "metric", "timestamp"); 33 | 34 | var table = await client.CreateTableAsync(tableBuilder); 35 | Console.WriteLine($"Created table {tableName}"); 36 | 37 | var batches = CreateRows(table, numRows).Chunk(2000); 38 | var writtenRows = 0; 39 | 40 | foreach (var batch in batches) 41 | { 42 | await client.WriteAsync(batch); 43 | writtenRows += batch.Length; 44 | Console.WriteLine($"Wrote {writtenRows} rows"); 45 | } 46 | } 47 | 48 | private static IEnumerable CreateRows(KuduTable table, int numRows) 49 | { 50 | var hosts = new[] { "host1.example.com", "host2.example.com" }; 51 | var metrics = new[] { "cpuload.avg1", "cpuload.avg5", "cpuload.avg15" }; 52 | var now = DateTime.UtcNow; 53 | 54 | for (int i = 0; i < numRows; i++) 55 | { 56 | var row = table.NewInsert(); 57 | row.SetString("host", hosts[i % hosts.Length]); 58 | row.SetString("metric", metrics[i % metrics.Length]); 59 | row.SetDateTime("timestamp", now.AddSeconds(i)); 60 | row.SetDouble("value", Random.Shared.NextDouble()); 61 | 62 | yield return row; 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Mapper/ColumnNameMatcher.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics.CodeAnalysis; 4 | using System.Globalization; 5 | using System.Linq; 6 | 7 | namespace Knet.Kudu.Client.Mapper; 8 | 9 | internal sealed class ColumnNameMatcher where T : class 10 | { 11 | #if NETSTANDARD2_0 12 | private static readonly StringComparer _columnComparer = StringComparer.Create( 13 | CultureInfo.InvariantCulture, 14 | ignoreCase: true); 15 | #else 16 | private static readonly StringComparer _columnComparer = StringComparer.Create( 17 | CultureInfo.InvariantCulture, 18 | CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace | CompareOptions.IgnoreSymbols); 19 | #endif 20 | 21 | private readonly ILookup _projectedColumns; 22 | private readonly Func _nameSelector; 23 | 24 | public ColumnNameMatcher(IEnumerable projectedColumns, Func nameSelector) 25 | { 26 | _projectedColumns = projectedColumns.ToLookup(nameSelector, _columnComparer); 27 | _nameSelector = nameSelector; 28 | } 29 | 30 | public bool TryGetColumn(string destinationName, [NotNullWhen(true)] out T? columnInfo) 31 | { 32 | var columns = _projectedColumns[destinationName]; 33 | 34 | T? caseInsensitiveMatch = null; 35 | T? firstMatch = null; 36 | 37 | // We could get multiple matches here, take the best one. 38 | foreach (var column in columns) 39 | { 40 | var projectedName = _nameSelector(column); 41 | 42 | if (StringComparer.Ordinal.Equals(destinationName, projectedName)) 43 | { 44 | // Exact match. 45 | columnInfo = column; 46 | return true; 47 | } 48 | 49 | if (StringComparer.OrdinalIgnoreCase.Equals(destinationName, projectedName)) 50 | { 51 | caseInsensitiveMatch ??= column; 52 | } 53 | else 54 | { 55 | firstMatch ??= column; 56 | } 57 | } 58 | 59 | if (caseInsensitiveMatch is not null) 60 | { 61 | columnInfo = caseInsensitiveMatch; 62 | return true; 63 | } 64 | 65 | if (firstMatch is not null) 66 | { 67 | columnInfo = firstMatch; 68 | return true; 69 | } 70 | 71 | columnInfo = null; 72 | return false; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/IKuduSession.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | 5 | namespace Knet.Kudu.Client; 6 | 7 | /// 8 | /// 9 | /// Within a session, multiple operations may be accumulated and batched 10 | /// together for better efficiency. There is a guarantee that writes from 11 | /// different sessions do not get batched together into the same RPCs -- 12 | /// this means that latency-sensitive clients can run through the same 13 | /// object as throughput-oriented clients, perhaps 14 | /// by adjusting for different uses. 15 | /// 16 | /// 17 | /// 18 | /// Sessions are thread safe by default, however FlushAsync is only guaranteed 19 | /// to flush rows for which EnqueueAsync has completed. To guarantee ordering, 20 | /// prior calls to EnqueueAsync must be awaited before issuing another write. 21 | /// There are no ordering guarantees across concurrent calls to EnqueueAsync. 22 | /// Calling FlushAsync concurrently will wait for prior flushes to complete, 23 | /// and thus will preserve ordering guarantees. 24 | /// 25 | /// 26 | public interface IKuduSession : IAsyncDisposable 27 | { 28 | /// 29 | /// Asynchronously enqueues a row to the session. The row will be written 30 | /// to the server when one of the following: 31 | /// 32 | /// 33 | /// threshold is met 34 | /// 35 | /// 36 | /// threshold is met 37 | /// 38 | /// 39 | /// is called 40 | /// 41 | /// 42 | /// The session is disposed via 43 | /// 44 | /// 45 | /// 46 | /// The row to write. 47 | /// The cancellation token. 48 | /// 49 | ValueTask EnqueueAsync(KuduOperation operation, CancellationToken cancellationToken = default); 50 | 51 | /// 52 | /// Writes all buffered rows. 53 | /// 54 | /// The cancellation token. 55 | Task FlushAsync(CancellationToken cancellationToken = default); 56 | } 57 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/ColumnTypeAttributes.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Knet.Kudu.Client; 4 | 5 | public sealed class ColumnTypeAttributes : IEquatable 6 | { 7 | public int? Precision { get; } 8 | 9 | public int? Scale { get; } 10 | 11 | public int? Length { get; } 12 | 13 | public ColumnTypeAttributes(int? precision, int? scale, int? length) 14 | { 15 | Precision = precision; 16 | Scale = scale; 17 | Length = length; 18 | } 19 | 20 | /// 21 | /// Return a string representation appropriate for `type`. 22 | /// This is meant to be postfixed to the name of a primitive type to 23 | /// describe the full type, e.g. decimal(10, 4). 24 | /// 25 | /// The data type. 26 | public string ToStringForType(KuduType type) 27 | { 28 | switch (type) 29 | { 30 | case KuduType.Decimal32: 31 | case KuduType.Decimal64: 32 | case KuduType.Decimal128: 33 | return $"({Precision}, {Scale})"; 34 | case KuduType.Varchar: 35 | return $"({Length})"; 36 | default: 37 | return ""; 38 | } 39 | } 40 | 41 | public bool Equals(ColumnTypeAttributes? other) 42 | { 43 | if (other is null) 44 | return false; 45 | 46 | if (ReferenceEquals(this, other)) 47 | return true; 48 | 49 | return 50 | Precision == other.Precision && 51 | Scale == other.Scale && 52 | Length == other.Length; 53 | } 54 | 55 | public override bool Equals(object? obj) => Equals(obj as ColumnTypeAttributes); 56 | 57 | public override int GetHashCode() => HashCode.Combine(Precision, Scale, Length); 58 | 59 | public static bool operator ==(ColumnTypeAttributes? lhs, ColumnTypeAttributes? rhs) 60 | { 61 | if (lhs is null) 62 | { 63 | return rhs is null; 64 | } 65 | 66 | return lhs.Equals(rhs); 67 | } 68 | 69 | public static bool operator !=(ColumnTypeAttributes? lhs, ColumnTypeAttributes? rhs) => !(lhs == rhs); 70 | 71 | public static ColumnTypeAttributes NewDecimalAttributes( 72 | int precision, int scale) 73 | { 74 | return new ColumnTypeAttributes(precision, scale, null); 75 | } 76 | 77 | public static ColumnTypeAttributes NewVarcharAttributes(int length) 78 | { 79 | return new ColumnTypeAttributes(null, null, length); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Connection/ISecurityContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Net.Security; 5 | using Knet.Kudu.Client.Protobuf.Security; 6 | 7 | namespace Knet.Kudu.Client.Connection; 8 | 9 | /// 10 | /// Stores security-related infrastructure, credentials, and trusted certificates. 11 | /// Implementations of this should be thread safe. 12 | /// 13 | public interface ISecurityContext 14 | { 15 | /// 16 | /// True if the user imported an authentication token to use. 17 | /// 18 | public bool IsAuthenticationTokenImported { get; } 19 | 20 | /// 21 | /// Set the token received from connecting to the leader master. 22 | /// 23 | /// The token to set. 24 | public void SetAuthenticationToken(SignedTokenPB token); 25 | 26 | /// 27 | /// Get the current authentication token, or null if we have no valid token. 28 | /// 29 | public SignedTokenPB? GetAuthenticationToken(); 30 | 31 | /// 32 | /// Export serialized authentication data that may be passed to a different 33 | /// client instance and imported to provide that client the ability to connect 34 | /// to the cluster. 35 | /// 36 | public ReadOnlyMemory ExportAuthenticationCredentials(); 37 | 38 | /// 39 | /// Import data allowing this client to authenticate to the cluster. 40 | /// 41 | /// The authentication token. 42 | public void ImportAuthenticationCredentials(ReadOnlySpan token); 43 | 44 | /// 45 | /// Mark the given CA cert (provided in DER form) as the trusted CA cert for the 46 | /// client. Replaces any previously trusted cert. 47 | /// 48 | /// The certificates to trust. 49 | public void TrustCertificates(IEnumerable> certDers); 50 | 51 | /// 52 | /// Creates a that trusts the certificates provided 53 | /// by . 54 | /// 55 | /// The stream to wrap. 56 | public SslStream CreateTlsStream(Stream innerStream); 57 | 58 | /// 59 | /// Creates a that trusts all certificates. 60 | /// 61 | /// The stream to wrap. 62 | public SslStream CreateTlsStreamTrustAll(Stream innerStream); 63 | } 64 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Tablet/KeyRange.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Knet.Kudu.Client.Tablet; 4 | 5 | /// 6 | /// Class used to represent primary key range in tablet. 7 | /// 8 | public class KeyRange 9 | { 10 | /// 11 | /// The tablet which the key range belongs to. 12 | /// 13 | public RemoteTablet Tablet { get; } 14 | 15 | /// 16 | /// The encoded primary key where the key range starts (inclusive). 17 | /// 18 | public ReadOnlyMemory PrimaryKeyStart { get; } 19 | 20 | /// 21 | /// The encoded primary key where the key range stops (exclusive). 22 | /// 23 | public ReadOnlyMemory PrimaryKeyEnd { get; } 24 | 25 | /// 26 | /// The estimated data size of the key range. 27 | /// 28 | public long DataSizeBytes { get; } 29 | 30 | /// 31 | /// Create a new key range [primaryKeyStart, primaryKeyEnd). 32 | /// 33 | /// The tablet which the key range belongs to. 34 | /// 35 | /// The encoded primary key where to start in the key range (inclusive). 36 | /// 37 | /// 38 | /// The encoded primary key where to stop in the key range (exclusive). 39 | /// 40 | /// The estimated data size of the key range. 41 | public KeyRange( 42 | RemoteTablet tablet, 43 | ReadOnlyMemory primaryKeyStart, 44 | ReadOnlyMemory primaryKeyEnd, 45 | long dataSizeBytes) 46 | { 47 | Tablet = tablet; 48 | PrimaryKeyStart = primaryKeyStart; 49 | PrimaryKeyEnd = primaryKeyEnd; 50 | DataSizeBytes = dataSizeBytes; 51 | } 52 | 53 | /// 54 | /// The start partition key. 55 | /// 56 | public byte[] PartitionKeyStart => Tablet.Partition.PartitionKeyStart; 57 | 58 | /// 59 | /// The end partition key. 60 | /// 61 | public byte[] PartitionKeyEnd => Tablet.Partition.PartitionKeyEnd; 62 | 63 | public override string ToString() 64 | { 65 | var start = PrimaryKeyStart.Length == 0 66 | ? "" 67 | : BitConverter.ToString(PrimaryKeyStart.ToArray()); 68 | 69 | var end = PrimaryKeyEnd.Length == 0 70 | ? "" 71 | : BitConverter.ToString(PrimaryKeyEnd.ToArray()); 72 | 73 | return $"[{start}, {end}), {DataSizeBytes}, {Tablet.TabletId} {Tablet.Partition}"; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.Tests/KuduStatusTests.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Exceptions; 2 | using Xunit; 3 | using static Knet.Kudu.Client.Protobuf.AppStatusPB.Types; 4 | 5 | namespace Knet.Kudu.Client.Tests; 6 | 7 | public class KuduStatusTests 8 | { 9 | [Fact] 10 | public void TestOkStatus() 11 | { 12 | var status = KuduStatus.Ok; 13 | Assert.True(status.IsOk); 14 | Assert.False(status.IsNotAuthorized); 15 | Assert.Equal(-1, status.PosixCode); 16 | Assert.Equal("Ok", status.ToString()); 17 | } 18 | 19 | [Fact] 20 | public void TestStatusNonPosix() 21 | { 22 | var status = KuduStatus.Aborted("foo"); 23 | Assert.False(status.IsOk); 24 | Assert.True(status.IsAborted); 25 | Assert.Equal(ErrorCode.Aborted, status.Code); 26 | Assert.Equal("foo", status.Message); 27 | Assert.Equal(-1, status.PosixCode); 28 | Assert.Equal("Aborted: foo", status.ToString()); 29 | } 30 | 31 | [Fact] 32 | public void TestPosixCode() 33 | { 34 | var status = KuduStatus.NotFound("File not found", 2); 35 | Assert.False(status.IsOk); 36 | Assert.False(status.IsAborted); 37 | Assert.True(status.IsNotFound); 38 | Assert.Equal(2, status.PosixCode); 39 | Assert.Equal("Not found: File not found (error 2)", status.ToString()); 40 | } 41 | 42 | [Fact] 43 | public void TestMessageTooLong() 44 | { 45 | int maxMessageLength = KuduStatus.MaxMessageLength; 46 | string abbreviation = KuduStatus.Abbreviation; 47 | int abbreviationLength = abbreviation.Length; 48 | 49 | // Test string that will not get abbreviated. 50 | var str = new string('a', maxMessageLength); 51 | var status = KuduStatus.Corruption(str); 52 | Assert.Equal(str, status.Message); 53 | 54 | // Test string just over the limit that will get abbreviated. 55 | str = new string('a', maxMessageLength + 1); 56 | status = KuduStatus.Corruption(str); 57 | Assert.Equal(maxMessageLength, status.Message.Length); 58 | Assert.Equal( 59 | status.Message.Substring(maxMessageLength - abbreviationLength), 60 | abbreviation); 61 | 62 | // Test string that's way too big that will get abbreviated. 63 | str = new string('a', maxMessageLength * 2); 64 | status = KuduStatus.Corruption(str); 65 | Assert.Equal(maxMessageLength, status.Message.Length); 66 | Assert.Equal( 67 | status.Message.Substring(maxMessageLength - abbreviationLength), 68 | abbreviation); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Knet.Kudu.Client.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net7.0;net6.0;netstandard2.1;netstandard2.0 5 | latest 6 | enable 7 | true 8 | CS1591;SYSLIB1015 9 | 10 | $(AssemblyName) 11 | 0.2.0 12 | xqrzd 13 | true 14 | .NET client for Apache Kudu 15 | icon.png 16 | https://github.com/xqrzd/kudu-client-net 17 | Apache-2.0 18 | Kudu;Apache.Kudu 19 | 20 | true 21 | true 22 | true 23 | embedded 24 | true 25 | true 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Internal/KuduTypeValidation.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics.CodeAnalysis; 3 | using System.Runtime.CompilerServices; 4 | 5 | namespace Knet.Kudu.Client.Internal; 6 | 7 | internal static class KuduTypeValidation 8 | { 9 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 10 | public static bool IsOfType(this KuduType type, KuduTypeFlags types) 11 | { 12 | int typeFlag = 1 << (int)type; 13 | return (typeFlag & (int)types) != 0; 14 | } 15 | 16 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 17 | public static void ValidateColumnType( 18 | this ColumnSchema column, KuduTypeFlags types) 19 | { 20 | if (!column.Type.IsOfType(types)) 21 | ThrowException(column, types); 22 | } 23 | 24 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 25 | public static void ValidateColumnType( 26 | this ColumnSchema column, KuduType type) 27 | { 28 | if (column.Type != type) 29 | ThrowException(column, type); 30 | } 31 | 32 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 33 | public static void ValidateColumnIsFixedLengthType(this ColumnSchema column) 34 | { 35 | if (!column.IsFixedSize) 36 | ThrowNotFixedLengthException(column); 37 | } 38 | 39 | [DoesNotReturn] 40 | public static void ThrowException(ColumnSchema column, KuduTypeFlags types) 41 | { 42 | throw new ArgumentException( 43 | $"Expected column {column} to be one of ({types})"); 44 | } 45 | 46 | // TODO: Remove this method 47 | [DoesNotReturn] 48 | public static T ThrowException(ColumnSchema column, KuduTypeFlags types) 49 | { 50 | throw new ArgumentException( 51 | $"Expected column {column} to be one of ({types})"); 52 | } 53 | 54 | [DoesNotReturn] 55 | public static void ThrowException(ColumnSchema column, KuduType type) 56 | { 57 | throw new ArgumentException( 58 | $"Expected column {column} to be of {type}"); 59 | } 60 | 61 | [DoesNotReturn] 62 | public static void ThrowNullException(ColumnSchema column) 63 | { 64 | throw new ArgumentException($"Column {column} is null"); 65 | } 66 | 67 | [DoesNotReturn] 68 | public static void ThrowNotNullableException(ColumnSchema column) 69 | { 70 | throw new ArgumentException($"Column {column} is not nullable"); 71 | } 72 | 73 | [DoesNotReturn] 74 | public static void ThrowNotFixedLengthException(ColumnSchema column) 75 | { 76 | throw new ArgumentException($"Column {column} is not a fixed length type"); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/ReadMode.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client; 2 | 3 | /// 4 | /// The possible read modes for scanners. 5 | /// 6 | public enum ReadMode 7 | { 8 | /// 9 | /// 10 | /// When READ_LATEST is specified the server will always return committed writes at 11 | /// the time the request was received. This type of read does not return a snapshot 12 | /// timestamp and is not repeatable. 13 | /// 14 | /// 15 | /// 16 | /// In ACID terms this corresponds to Isolation mode: "Read Committed". 17 | /// 18 | /// 19 | /// 20 | /// This is the default mode. 21 | /// 22 | /// 23 | ReadLatest = Protobuf.ReadMode.ReadLatest, 24 | /// 25 | /// 26 | /// When READ_AT_SNAPSHOT is specified the server will attempt to perform a read 27 | /// at the provided timestamp. If no timestamp is provided the server will take the 28 | /// current time as the snapshot timestamp. In this mode reads are repeatable, i.e. 29 | /// all future reads at the same timestamp will yield the same data. This is performed 30 | /// at the expense of waiting for in-flight transactions whose timestamp is lower 31 | /// than the snapshot's timestamp to complete, so it might incur a latency penalty. 32 | /// 33 | /// 34 | /// 35 | /// In ACID terms this, by itself, corresponds to Isolation mode "Repeatable 36 | /// Read". If all writes to the scanned tablet are made externally consistent, 37 | /// then this corresponds to Isolation mode "Strict-Serializable". 38 | /// 39 | /// 40 | ReadAtSnapshot = Protobuf.ReadMode.ReadAtSnapshot, 41 | /// 42 | /// 43 | /// When READ_YOUR_WRITES is specified, the client will perform a read 44 | /// such that it follows all previously known writes and reads from this client. 45 | /// Specifically this mode: 46 | /// 47 | /// 48 | /// 49 | /// 50 | /// Ensures read-your-writes and read-your-reads session guarantees, 51 | /// 52 | /// 53 | /// Minimizes latency caused by waiting for outstanding write transactions to complete. 54 | /// 55 | /// 56 | /// 57 | /// 58 | /// Reads in this mode are not repeatable: two READ_YOUR_WRITES reads, even if 59 | /// they provide the same propagated timestamp bound, can execute at different 60 | /// timestamps and thus may return different results. 61 | /// 62 | /// 63 | ReadYourWrites = Protobuf.ReadMode.ReadYourWrites 64 | } 65 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.FunctionalTests/MasterFailoverTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using Knet.Kudu.Client.FunctionalTests.MiniCluster; 4 | using McMaster.Extensions.Xunit; 5 | using Xunit; 6 | 7 | namespace Knet.Kudu.Client.FunctionalTests; 8 | 9 | [MiniKuduClusterTest] 10 | public class MasterFailoverTests 11 | { 12 | [SkippableTheory] 13 | [InlineData(KillBefore.CreateClient, true)] 14 | [InlineData(KillBefore.CreateClient, false)] 15 | [InlineData(KillBefore.CreateTable, true)] 16 | [InlineData(KillBefore.CreateTable, false)] 17 | [InlineData(KillBefore.OpenTable, true)] 18 | [InlineData(KillBefore.OpenTable, false)] 19 | [InlineData(KillBefore.ScanTable, true)] 20 | [InlineData(KillBefore.ScanTable, false)] 21 | public async Task TestMasterFailover(KillBefore killBefore, bool restart) 22 | { 23 | await using var harness = await new MiniKuduClusterBuilder() 24 | .NumMasters(3) 25 | .NumTservers(3) 26 | .BuildHarnessAsync(); 27 | 28 | if (killBefore == KillBefore.CreateClient) 29 | await DoActionAsync(); 30 | 31 | await using var client = harness.CreateClientBuilder() 32 | .SetDefaultOperationTimeout(TimeSpan.FromMinutes(1)) 33 | .Build(); 34 | 35 | if (killBefore == KillBefore.CreateTable) 36 | await DoActionAsync(); 37 | 38 | var tableName = $"TestMasterFailover-killBefore={killBefore}"; 39 | var builder = new TableBuilder() 40 | .SetTableName(tableName) 41 | .SetNumReplicas(1) 42 | .AddColumn("column_x", KuduType.Int32, opt => opt.Key(true)) 43 | .AddColumn("column_y", KuduType.String) 44 | .AddHashPartitions(buckets: 4, seed: 777, "column_x"); 45 | 46 | var table = await client.CreateTableAsync(builder); 47 | Assert.Equal(tableName, table.TableName); 48 | 49 | if (killBefore == KillBefore.OpenTable) 50 | await DoActionAsync(); 51 | 52 | var table2 = await client.OpenTableAsync(tableName); 53 | 54 | if (killBefore == KillBefore.ScanTable) 55 | await DoActionAsync(); 56 | 57 | var scanner = client.NewScanBuilder(table2) 58 | .Build(); 59 | 60 | var numRows = await scanner.CountAsync(); 61 | Assert.Equal(0, numRows); // We didn't write any rows. 62 | 63 | Task DoActionAsync() 64 | { 65 | if (restart) 66 | return harness.RestartLeaderMasterAsync(); 67 | else 68 | return harness.KillLeaderMasterServerAsync(); 69 | } 70 | } 71 | 72 | public enum KillBefore 73 | { 74 | CreateClient, 75 | CreateTable, 76 | OpenTable, 77 | ScanTable 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.FunctionalTests/ClientTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading; 4 | using System.Threading.Tasks; 5 | using Knet.Kudu.Client.FunctionalTests.MiniCluster; 6 | using Knet.Kudu.Client.FunctionalTests.Util; 7 | using McMaster.Extensions.Xunit; 8 | using Xunit; 9 | 10 | namespace Knet.Kudu.Client.FunctionalTests; 11 | 12 | [MiniKuduClusterTest] 13 | public class ClientTests : IAsyncLifetime 14 | { 15 | private KuduTestHarness _harness; 16 | private KuduClient _client; 17 | 18 | public async Task InitializeAsync() 19 | { 20 | _harness = await new MiniKuduClusterBuilder().BuildHarnessAsync(); 21 | _client = _harness.CreateClient(); 22 | } 23 | 24 | public async Task DisposeAsync() 25 | { 26 | await _client.DisposeAsync(); 27 | await _harness.DisposeAsync(); 28 | } 29 | 30 | [SkippableFact] 31 | public async Task TestClusterId() 32 | { 33 | var clusterId = await _client.GetClusterIdAsync(); 34 | Assert.NotEmpty(clusterId); 35 | 36 | // Test cached path. 37 | var cachedClusterId = await _client.GetClusterIdAsync(); 38 | Assert.Equal(clusterId, cachedClusterId); 39 | } 40 | 41 | /// 42 | /// Stress test which performs upserts from many sessions on different threads 43 | /// sharing the same KuduClient and KuduTable instance. 44 | /// 45 | [SkippableFact] 46 | public async Task TestMultipleSessions() 47 | { 48 | int numTasks = 60; 49 | var tasks = new List(numTasks); 50 | 51 | var builder = ClientTestUtil.GetBasicSchema() 52 | .SetTableName("TestMultipleSessions") 53 | .CreateBasicRangePartition(); 54 | 55 | var table = await _client.CreateTableAsync(builder); 56 | 57 | using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); 58 | var token = cts.Token; 59 | 60 | for (int i = 0; i < numTasks; i++) 61 | { 62 | var task = Task.Run(async () => 63 | { 64 | while (!token.IsCancellationRequested) 65 | { 66 | await using var session = _client.NewSession(); 67 | 68 | for (int j = 0; j < 100; j++) 69 | { 70 | var row = table.NewUpsert(); 71 | row.SetInt32(0, j); 72 | row.SetInt32(1, 12345); 73 | row.SetInt32(2, 3); 74 | row.SetNull(3); 75 | row.SetBool(4, false); 76 | await session.EnqueueAsync(row); 77 | } 78 | } 79 | }); 80 | 81 | tasks.Add(task); 82 | } 83 | 84 | await Task.WhenAll(tasks); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Tablet/RemoteTabletExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Runtime.CompilerServices; 4 | 5 | namespace Knet.Kudu.Client.Tablet; 6 | 7 | internal static class RemoteTabletExtensions 8 | { 9 | public static FindTabletResult FindTablet( 10 | this List tablets, ReadOnlySpan partitionKey) 11 | { 12 | int lo = 0; 13 | int hi = tablets.Count - 1; 14 | 15 | // If length == 0, hi == -1, and loop will not be entered 16 | while (lo <= hi) 17 | { 18 | // PERF: `lo` or `hi` will never be negative inside the loop, 19 | // so computing median using uints is safe since we know 20 | // `length <= int.MaxValue`, and indices are >= 0 21 | // and thus cannot overflow an uint. 22 | // Saves one subtraction per loop compared to 23 | // `int i = lo + ((hi - lo) >> 1);` 24 | int i = (int)(((uint)hi + (uint)lo) >> 1); 25 | 26 | var tablet = tablets[i]; 27 | int c = partitionKey.SequenceCompareTo(tablet.Partition.PartitionKeyStart); 28 | if (c == 0) 29 | { 30 | return new FindTabletResult(tablet, i); 31 | } 32 | else if (c > 0) 33 | { 34 | lo = i + 1; 35 | } 36 | else 37 | { 38 | hi = i - 1; 39 | } 40 | } 41 | 42 | if (hi >= 0) 43 | { 44 | var tablet = tablets[hi]; 45 | if (tablet.Partition.ContainsPartitionKey(partitionKey)) 46 | { 47 | return new FindTabletResult(tablet, hi); 48 | } 49 | 50 | return HandleMissingTablet(tablets, lo, tablet); 51 | } 52 | 53 | // The key is before the first partition. 54 | return HandleMissingTablet(tablets); 55 | } 56 | 57 | [MethodImpl(MethodImplOptions.NoInlining)] 58 | private static FindTabletResult HandleMissingTablet(List tablets) 59 | { 60 | var nonCoveredRangeEnd = tablets.Count == 0 61 | ? Array.Empty() 62 | : tablets[0].Partition.PartitionKeyStart; 63 | 64 | return new FindTabletResult(Array.Empty(), nonCoveredRangeEnd); 65 | } 66 | 67 | [MethodImpl(MethodImplOptions.NoInlining)] 68 | private static FindTabletResult HandleMissingTablet( 69 | List tablets, int nextIndex, RemoteTablet tablet) 70 | { 71 | var nonCoveredRangeStart = tablet.Partition.PartitionKeyEnd; 72 | 73 | var nonCoveredRangeEnd = tablets.Count == nextIndex 74 | ? Array.Empty() 75 | : tablets[nextIndex].Partition.PartitionKeyStart; 76 | 77 | return new FindTabletResult(nonCoveredRangeStart, nonCoveredRangeEnd); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.Tests/TableBuilderTests.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Protobuf; 2 | using Xunit; 3 | 4 | namespace Knet.Kudu.Client.Tests; 5 | 6 | public class TableBuilderTests 7 | { 8 | [Fact] 9 | public void CanSetBasicProperties() 10 | { 11 | var builder = new TableBuilder() 12 | .SetTableName("table_name") 13 | .SetNumReplicas(3); 14 | 15 | var request = builder.Build(); 16 | 17 | Assert.Equal("table_name", request.Name); 18 | Assert.Equal(3, request.NumReplicas); 19 | } 20 | 21 | [Fact] 22 | public void CanAddColumns() 23 | { 24 | var builder = new TableBuilder() 25 | .AddColumn("c1", KuduType.Int32, opt => opt.Key(true)) 26 | .AddColumn("c2", KuduType.String, opt => opt 27 | .Nullable(false) 28 | .Encoding(EncodingType.DictEncoding) 29 | .Compression(CompressionType.Snappy)); 30 | 31 | var request = builder.Build(); 32 | 33 | Assert.Collection(request.Schema.Columns, c => 34 | { 35 | Assert.Equal("c1", c.Name); 36 | Assert.Equal(DataType.Int32, c.Type); 37 | Assert.True(c.IsKey); 38 | Assert.False(c.IsNullable); 39 | Assert.Equal(Protobuf.EncodingType.AutoEncoding, c.Encoding); 40 | Assert.Equal(Protobuf.CompressionType.DefaultCompression, c.Compression); 41 | Assert.Null(c.TypeAttributes); 42 | }, c => 43 | { 44 | Assert.Equal("c2", c.Name); 45 | Assert.Equal(DataType.String, c.Type); 46 | Assert.Equal(Protobuf.EncodingType.DictEncoding, c.Encoding); 47 | Assert.Equal(Protobuf.CompressionType.Snappy, c.Compression); 48 | }); 49 | } 50 | 51 | [Fact] 52 | public void CanSetColumnTypeAttributes() 53 | { 54 | var builder = new TableBuilder() 55 | .AddColumn("dec32", KuduType.Decimal32, opt => opt 56 | .DecimalAttributes(4, 3)); 57 | 58 | var request = builder.Build(); 59 | 60 | Assert.Collection(request.Schema.Columns, c => 61 | { 62 | Assert.Equal(4, c.TypeAttributes.Precision); 63 | Assert.Equal(3, c.TypeAttributes.Scale); 64 | }); 65 | } 66 | 67 | [Fact] 68 | public void CanAddHashPartitions() 69 | { 70 | var builder = new TableBuilder() 71 | .AddHashPartitions(5, "a", "b", "c"); 72 | 73 | var request = builder.Build(); 74 | 75 | Assert.Collection(request.PartitionSchema.HashSchema, h => 76 | { 77 | Assert.Equal(5, h.NumBuckets); 78 | Assert.Equal((uint)0, h.Seed); 79 | 80 | Assert.Collection(h.Columns, 81 | c => Assert.Equal("a", c.Name), 82 | c => Assert.Equal("b", c.Name), 83 | c => Assert.Equal("c", c.Name)); 84 | }); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # .NET Client for Apache Kudu 2 | 3 | ![Apache Kudu](https://d3dr9sfxru4sde.cloudfront.net/i/k/apachekudu_logo_0716_345px.png) 4 | 5 | ## Package 6 | You can get the package on Nuget: https://www.nuget.org/packages/Knet.Kudu.Client 7 | 8 | ## Supported Kudu Versions 9 | This library supports Apache Kudu 1.3 and newer. The newest version of this library 10 | should always be used, regardless of the Apache Kudu version. 11 | This client tries to maintain feature parity with the official C++ and Java clients. 12 | 13 | ## Quickstart 14 | 15 | ### Docker 16 | Follow the [Apache Kudu Quickstart](https://kudu.apache.org/docs/quickstart.html) guide to get Kudu running in Docker. 17 | 18 | ### Create a Client 19 | 20 | ```csharp 21 | KuduClient client = KuduClient.NewBuilder("localhost:7051,localhost:7151,localhost:7251") 22 | .Build(); 23 | ``` 24 | 25 | > Note: KuduClient is intended to be a singleton. It is inefficient to create multiple clients. 26 | 27 | ### Create a Table 28 | 29 | ```csharp 30 | var tableBuilder = new TableBuilder("twitter_firehose") 31 | .AddColumn("tweet_id", KuduType.Int64, opt => opt.Key(true)) 32 | .AddColumn("user_name", KuduType.String) 33 | .AddColumn("created_at", KuduType.UnixtimeMicros) 34 | .AddColumn("text", KuduType.String); 35 | 36 | await client.CreateTableAsync(tableBuilder); 37 | ``` 38 | 39 | See more table options in the [wiki](https://github.com/xqrzd/kudu-client-net/wiki/Create-Table). 40 | 41 | ### Open a Table 42 | 43 | ```csharp 44 | KuduTable table = await client.OpenTableAsync("twitter_firehose"); 45 | ``` 46 | 47 | > Note: This table can be cached and reused concurrently. It simply stores the table schema. 48 | 49 | ### Insert Data 50 | 51 | ```csharp 52 | var rows = Enumerable.Range(0, 100).Select(i => 53 | { 54 | var row = table.NewInsert(); 55 | row.SetInt64("tweet_id", i); 56 | row.SetString("user_name", $"user_{i}"); 57 | row.SetDateTime("created_at", DateTime.UtcNow); 58 | row.SetString("text", $"sample tweet {i}"); 59 | return row; 60 | }); 61 | 62 | await client.WriteAsync(rows); 63 | ``` 64 | 65 | ### Query Data 66 | 67 | ```csharp 68 | KuduScanner scanner = client.NewScanBuilder(table) 69 | .SetProjectedColumns("tweet_id", "user_name", "created_at") 70 | .Build(); 71 | 72 | await foreach (ResultSet resultSet in scanner) 73 | { 74 | Console.WriteLine($"Received {resultSet.Count} rows"); 75 | 76 | foreach (RowResult row in resultSet) 77 | { 78 | var tweetId = row.GetInt64("tweet_id"); 79 | var userName = row.GetString("user_name"); 80 | var createdAtUtc = row.GetDateTime("created_at"); 81 | 82 | Console.WriteLine($"tweet_id: {tweetId}, user_name: {userName}, created_at: {createdAtUtc}"); 83 | } 84 | } 85 | ``` 86 | 87 | This client includes a simple object mapper, 88 | 89 | ```csharp 90 | record Tweet(long TweetId, string Username, DateTime CreatedAt); 91 | 92 | var tweets = await scanner.ScanToListAsync(); 93 | ``` 94 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/KuduPartitioner.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using Knet.Kudu.Client.Exceptions; 4 | using Knet.Kudu.Client.Tablet; 5 | 6 | namespace Knet.Kudu.Client; 7 | 8 | /// 9 | /// 10 | /// A allows clients to determine the target 11 | /// partition of a row without actually performing a write. The set of 12 | /// partitions is eagerly fetched when the KuduPartitioner is constructed 13 | /// so that the actual partitioning step can be performed synchronously 14 | /// without any network trips. 15 | /// 16 | /// 17 | /// 18 | /// Note: Because this operates on a metadata snapshot retrieved at 19 | /// construction time, it will not reflect any metadata changes to the 20 | /// table that have occurred since its creation. 21 | /// 22 | /// 23 | public class KuduPartitioner 24 | { 25 | private readonly PartitionSchema _partitionSchema; 26 | private readonly List _tablets; 27 | 28 | public KuduPartitioner(KuduTable table, List tablets) 29 | { 30 | _partitionSchema = table.PartitionSchema; 31 | _tablets = tablets; 32 | } 33 | 34 | /// 35 | /// The number of partitions known by this partitioner. 36 | /// 37 | public int NumPartitions => _tablets.Count; 38 | 39 | /// 40 | /// Determine if the given row falls into a valid partition. 41 | /// 42 | /// The row to check. 43 | public bool IsCovered(PartialRow row) 44 | { 45 | var result = GetResult(row); 46 | return result.IsCoveredRange; 47 | } 48 | 49 | /// 50 | /// Determine the partition index that the given row falls into. 51 | /// 52 | /// The row to be partitioned. 53 | /// 54 | /// The resulting partition index. 55 | /// The result will be less than . 56 | /// 57 | public int PartitionRow(PartialRow row) 58 | { 59 | var result = GetResult(row); 60 | 61 | if (result.IsNonCoveredRange) 62 | { 63 | throw new NonCoveredRangeException( 64 | result.NonCoveredRangeStart, 65 | result.NonCoveredRangeEnd); 66 | } 67 | 68 | return result.Index; 69 | } 70 | 71 | private FindTabletResult GetResult(PartialRow row) 72 | { 73 | var partitionSchema = _partitionSchema; 74 | int maxSize = KeyEncoder.CalculateMaxPartitionKeySize(row, partitionSchema); 75 | Span buffer = stackalloc byte[maxSize]; 76 | 77 | KeyEncoder.EncodePartitionKey( 78 | row, 79 | partitionSchema, 80 | buffer, 81 | out int bytesWritten); 82 | 83 | var partitionKey = buffer.Slice(0, bytesWritten); 84 | 85 | return RemoteTabletExtensions.FindTablet(_tablets, partitionKey); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Tablet/KeyEncoder.sse.cs: -------------------------------------------------------------------------------- 1 | #if NETCOREAPP3_1_OR_GREATER 2 | using System; 3 | using System.Diagnostics.CodeAnalysis; 4 | using System.Runtime.CompilerServices; 5 | using System.Runtime.Intrinsics; 6 | using System.Runtime.Intrinsics.X86; 7 | 8 | namespace Knet.Kudu.Client.Tablet; 9 | 10 | public static partial class KeyEncoder 11 | { 12 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 13 | private static int EncodeBinary( 14 | ReadOnlySpan source, Span destination) 15 | { 16 | if (Sse41.IsSupported) 17 | { 18 | return EncodeBinarySse(source, destination); 19 | } 20 | 21 | return EncodeBinaryStandard(source, destination); 22 | } 23 | 24 | private static unsafe int EncodeBinarySse( 25 | ReadOnlySpan source, Span destination) 26 | { 27 | var length = (uint)source.Length; 28 | 29 | if ((uint)destination.Length < length * 2) 30 | ThrowException(); 31 | 32 | fixed (byte* src = source) 33 | fixed (byte* dest = destination) 34 | { 35 | var srcCurrent = src; 36 | var destCurrent = dest; 37 | 38 | var end = src + length; 39 | var simdEnd = end - (length % 16); 40 | 41 | while (srcCurrent < simdEnd) 42 | { 43 | // Load 16 bytes (unaligned) into the XMM register. 44 | var data = Sse2.LoadVector128(srcCurrent); 45 | 46 | // Compare each byte of the input with '\0'. This results in a vector 47 | // where each byte is either \x00 or \xFF, depending on whether the 48 | // input had a '\x00' in the corresponding position. 49 | var zeroBytes = Sse2.CompareEqual(data, Vector128.Zero); 50 | 51 | // Check whether the resulting vector is all-zero. 52 | // If it's all zero, we can just store the entire chunk. 53 | if (Sse41.TestZ(zeroBytes, zeroBytes)) 54 | { 55 | Sse2.Store(destCurrent, data); 56 | } 57 | else 58 | { 59 | break; 60 | } 61 | 62 | srcCurrent += 16; 63 | destCurrent += 16; 64 | } 65 | 66 | while (srcCurrent < end) 67 | { 68 | byte value = *srcCurrent++; 69 | if (value == 0) 70 | { 71 | *destCurrent++ = 0; 72 | *destCurrent++ = 1; 73 | } 74 | else 75 | { 76 | *destCurrent++ = value; 77 | } 78 | } 79 | 80 | var written = destCurrent - dest; 81 | return (int)written; 82 | } 83 | } 84 | 85 | [DoesNotReturn] 86 | private static void ThrowException() => 87 | throw new ArgumentException("Destination must be at least double source"); 88 | } 89 | #endif 90 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Internal/Netstandard2Extensions.cs: -------------------------------------------------------------------------------- 1 | #if NETSTANDARD2_0 2 | 3 | using System; 4 | using System.Collections.Concurrent; 5 | using System.Collections.Generic; 6 | using System.IO; 7 | using System.Runtime.CompilerServices; 8 | using System.Text; 9 | using System.Threading; 10 | using System.Threading.Tasks; 11 | 12 | namespace Knet.Kudu.Client.Internal; 13 | 14 | internal static class Netstandard2Extensions 15 | { 16 | public static async ValueTask WriteAsync( 17 | this Stream stream, 18 | ReadOnlyMemory buffer, 19 | CancellationToken cancellationToken = default) 20 | { 21 | await stream.WriteAsync(buffer.ToArray(), 0, buffer.Length, cancellationToken) 22 | .ConfigureAwait(false); 23 | } 24 | 25 | public static async ValueTask ReadAsync( 26 | this Stream stream, 27 | Memory buffer, 28 | CancellationToken cancellationToken = default) 29 | { 30 | var tempBuffer = new byte[buffer.Length]; 31 | var read = await stream.ReadAsync(tempBuffer, 0, tempBuffer.Length, cancellationToken) 32 | .ConfigureAwait(false); 33 | 34 | tempBuffer.AsMemory(0, read).CopyTo(buffer); 35 | return read; 36 | } 37 | 38 | public static void Write(this Stream stream, ReadOnlySpan buffer) 39 | { 40 | stream.Write(buffer.ToArray(), 0, buffer.Length); 41 | } 42 | 43 | public static int Read(this Stream stream, Span buffer) 44 | { 45 | var tempBuffer = new byte[buffer.Length]; 46 | var read = stream.Read(tempBuffer, 0, tempBuffer.Length); 47 | tempBuffer.AsSpan(0, read).CopyTo(buffer); 48 | return read; 49 | } 50 | 51 | public static string GetString(this Encoding encoding, ReadOnlySpan bytes) 52 | { 53 | return encoding.GetString(bytes.ToArray()); 54 | } 55 | 56 | public static int GetBytes(this Encoding encoding, string s, Span bytes) 57 | { 58 | var result = encoding.GetBytes(s); 59 | result.CopyTo(bytes); 60 | return result.Length; 61 | } 62 | 63 | public static bool Remove( 64 | this Dictionary dictionary, TKey key, out TValue value) 65 | { 66 | dictionary.TryGetValue(key, out value); 67 | return dictionary.Remove(key); 68 | } 69 | 70 | public static TValue GetOrAdd( 71 | this ConcurrentDictionary dictionary, 72 | TKey key, Func valueFactory, TArg factoryArgument) 73 | { 74 | return dictionary.GetOrAdd(key, key => valueFactory(key, factoryArgument)); 75 | } 76 | 77 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 78 | public static unsafe int SingleToInt32Bits(float value) 79 | { 80 | return *(int*)&value; 81 | } 82 | 83 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 84 | public static unsafe float Int32BitsToSingle(int value) 85 | { 86 | return *(float*)&value; 87 | } 88 | } 89 | 90 | #endif 91 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.Tests/PartitionTests.cs: -------------------------------------------------------------------------------- 1 | using Knet.Kudu.Client.Tablet; 2 | using Xunit; 3 | 4 | namespace Knet.Kudu.Client.Tests; 5 | 6 | public class PartitionTests 7 | { 8 | [Fact] 9 | public void EmptyPartition() 10 | { 11 | var partition = new Partition( 12 | new byte[0], 13 | new byte[0], 14 | new int[0]); 15 | 16 | Assert.True(partition.IsEndPartition); 17 | Assert.Empty(partition.PartitionKeyStart); 18 | Assert.Empty(partition.PartitionKeyEnd); 19 | 20 | Assert.Empty(partition.RangeKeyStart); 21 | Assert.Empty(partition.RangeKeyEnd); 22 | 23 | Assert.Empty(partition.HashBuckets); 24 | } 25 | 26 | [Fact] 27 | public void EmptyPartitionsAreEqual() 28 | { 29 | var partition1 = new Partition( 30 | new byte[0], 31 | new byte[0], 32 | new int[0]); 33 | 34 | var partition2 = new Partition( 35 | new byte[0], 36 | new byte[0], 37 | new int[0]); 38 | 39 | Assert.Equal(partition1, partition2); 40 | Assert.Equal(0, partition1.CompareTo(partition2)); 41 | Assert.Equal(partition1.GetHashCode(), partition2.GetHashCode()); 42 | } 43 | 44 | [Fact] 45 | public void SimpleHashPartitionStart() 46 | { 47 | var partition = new Partition( 48 | new byte[0], 49 | new byte[] { 0, 0, 0, 1 }, 50 | new int[] { 0 }); 51 | 52 | Assert.False(partition.IsEndPartition); 53 | Assert.Empty(partition.PartitionKeyStart); 54 | Assert.Equal(new byte[] { 0, 0, 0, 1 }, partition.PartitionKeyEnd); 55 | 56 | Assert.Empty(partition.RangeKeyStart); 57 | Assert.Empty(partition.RangeKeyEnd); 58 | 59 | Assert.Equal(new int[] { 0 }, partition.HashBuckets); 60 | } 61 | 62 | [Fact] 63 | public void SimpleHashPartitionMiddle() 64 | { 65 | var partition = new Partition( 66 | new byte[] { 0, 0, 0, 1 }, 67 | new byte[] { 0, 0, 0, 2 }, 68 | new int[] { 1 }); 69 | 70 | Assert.False(partition.IsEndPartition); 71 | Assert.Equal(new byte[] { 0, 0, 0, 1 }, partition.PartitionKeyStart); 72 | Assert.Equal(new byte[] { 0, 0, 0, 2 }, partition.PartitionKeyEnd); 73 | 74 | Assert.Empty(partition.RangeKeyStart); 75 | Assert.Empty(partition.RangeKeyEnd); 76 | 77 | Assert.Equal(new int[] { 1 }, partition.HashBuckets); 78 | } 79 | 80 | [Fact] 81 | public void SimpleHashPartitionEnd() 82 | { 83 | var partition = new Partition( 84 | new byte[] { 0, 0, 0, 3 }, 85 | new byte[0], 86 | new int[] { 3 }); 87 | 88 | Assert.True(partition.IsEndPartition); 89 | Assert.Equal(new byte[] { 0, 0, 0, 3 }, partition.PartitionKeyStart); 90 | Assert.Empty(partition.PartitionKeyEnd); 91 | 92 | Assert.Empty(partition.RangeKeyStart); 93 | Assert.Empty(partition.RangeKeyEnd); 94 | 95 | Assert.Equal(new int[] { 3 }, partition.HashBuckets); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/ExternalConsistencyMode.cs: -------------------------------------------------------------------------------- 1 | namespace Knet.Kudu.Client; 2 | 3 | /// 4 | /// 5 | /// The external consistency mode for client requests. 6 | /// This defines how transactions and/or sequences of operations that touch 7 | /// several TabletServers, in different machines, can be observed by external 8 | /// clients. 9 | /// 10 | /// 11 | /// 12 | /// Note that ExternalConsistencyMode makes no guarantee on atomicity, i.e. 13 | /// no sequence of operations is made atomic (or transactional) just because 14 | /// an external consistency mode is set. 15 | /// Note also that ExternalConsistencyMode has no implication on the 16 | /// consistency between replicas of the same tablet. 17 | /// 18 | /// 19 | public enum ExternalConsistencyMode 20 | { 21 | /// 22 | /// 23 | /// The response to any write will contain a timestamp. Any further calls 24 | /// from the same client to other servers will update those servers 25 | /// with that timestamp. Following write operations from the same client 26 | /// will be assigned timestamps that are strictly higher, enforcing external 27 | /// consistency without having to wait or incur any latency penalties. 28 | /// 29 | /// 30 | /// 31 | /// In order to maintain external consistency for writes between 32 | /// two different clients in this mode, the user must forward the timestamp 33 | /// from the first client to the second by using 34 | /// . 35 | /// 36 | /// 37 | /// 38 | /// This is the default external consistency mode. 39 | /// 40 | /// 41 | /// 42 | /// Failure to propagate timestamp information through back-channels 43 | /// between two different clients will negate any external consistency 44 | /// guarantee under this mode. 45 | /// 46 | /// 47 | ClientPropagated = Protobuf.ExternalConsistencyMode.ClientPropagated, 48 | 49 | /// 50 | /// 51 | /// The server will guarantee that write operations from the same or from 52 | /// other client are externally consistent, without the need to propagate 53 | /// timestamps across clients. This is done by making write operations 54 | /// wait until there is certainty that all follow up write operations 55 | /// (operations that start after the previous one finishes) 56 | /// will be assigned a timestamp that is strictly higher, enforcing external 57 | /// consistency. 58 | /// 59 | /// 60 | /// 61 | /// Depending on the clock synchronization state of TabletServers this may 62 | /// imply considerable latency. Moreover operations in COMMIT_WAIT 63 | /// external consistency mode will outright fail if TabletServer clocks 64 | /// are either unsynchronized or synchronized but with a maximum error 65 | /// which surpasses a pre-configured threshold. 66 | /// 67 | /// 68 | CommitWait = Protobuf.ExternalConsistencyMode.CommitWait 69 | } 70 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.FunctionalTests/MultipleLeaderFailoverTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using Knet.Kudu.Client.FunctionalTests.MiniCluster; 4 | using Knet.Kudu.Client.FunctionalTests.Util; 5 | using McMaster.Extensions.Xunit; 6 | using Xunit; 7 | 8 | namespace Knet.Kudu.Client.FunctionalTests; 9 | 10 | [MiniKuduClusterTest] 11 | public class MultipleLeaderFailoverTests 12 | { 13 | /// 14 | /// This test writes 3 rows. Then in a loop, it kills the leader, then 15 | /// tries to write inner_row rows, and finally restarts the tablet server 16 | /// it killed. Verifying with a read as it goes. Finally it counts to make 17 | /// sure we have total_rows_to_insert of them. 18 | /// 19 | [SkippableTheory] 20 | [InlineData(true)] 21 | [InlineData(false)] 22 | public async Task TestMultipleFailover(bool restart) 23 | { 24 | int rowsPerIteration = 3; 25 | int numIterations = 10; 26 | int totalRowsToInsert = rowsPerIteration + numIterations * rowsPerIteration; 27 | 28 | await using var harness = await new MiniKuduClusterBuilder() 29 | .NumMasters(3) 30 | .NumTservers(3) 31 | .BuildHarnessAsync(); 32 | 33 | await using var client = harness.CreateClient(); 34 | 35 | var builder = ClientTestUtil.GetBasicSchema() 36 | .SetTableName("MultipleLeaderFailoverTest"); 37 | 38 | var table = await client.CreateTableAsync(builder); 39 | await using var session = client.NewSession(); 40 | 41 | for (int i = 0; i < rowsPerIteration; i++) 42 | { 43 | var row = ClientTestUtil.CreateBasicSchemaInsert(table, i); 44 | await session.EnqueueAsync(row); 45 | } 46 | 47 | await session.FlushAsync(); 48 | 49 | var rowCount = await ClientTestUtil.CountRowsAsync(client, table); 50 | Assert.Equal(rowsPerIteration, rowCount); 51 | 52 | int currentRows = rowsPerIteration; 53 | for (int i = 0; i < numIterations; i++) 54 | { 55 | var tablets = await client.GetTableLocationsAsync( 56 | table.TableId, Array.Empty(), 1); 57 | Assert.Single(tablets); 58 | 59 | if (restart) 60 | await harness.RestartTabletServerAsync(tablets[0]); 61 | else 62 | await harness.KillTabletLeaderAsync(tablets[0]); 63 | 64 | for (int j = 0; j < rowsPerIteration; j++) 65 | { 66 | var row = ClientTestUtil.CreateBasicSchemaInsert(table, currentRows); 67 | await session.EnqueueAsync(row); 68 | currentRows++; 69 | } 70 | 71 | await session.FlushAsync(); 72 | 73 | if (!restart) 74 | await harness.StartAllTabletServersAsync(); 75 | 76 | rowCount = await ClientTestUtil.CountRowsAsync(client, table); 77 | Assert.Equal(currentRows, rowCount); 78 | } 79 | 80 | rowCount = await ClientTestUtil.CountRowsAsync(client, table); 81 | Assert.Equal(totalRowsToInsert, rowCount); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Util/FastHash.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | namespace Knet.Kudu.Client.Util; 6 | 7 | /// 8 | /// FastHash is simple, robust, and efficient general-purpose hash function from Google. 9 | /// Implementation is adapted from https://code.google.com/archive/p/fast-hash/ 10 | /// 11 | public static class FastHash 12 | { 13 | // Help distinguish zero-length objects like empty strings. 14 | // These values must match those used by Apache Kudu. 15 | private static ReadOnlySpan HashValEmpty => new byte[] { 0xee, 0x7e, 0xca, 0x7d }; 16 | 17 | /// 18 | /// Compute 64-bit FastHash. 19 | /// 20 | /// The data to hash. 21 | /// Seed to compute the hash. 22 | public static ulong Hash64(ReadOnlySpan source, ulong seed) 23 | { 24 | if (source.Length == 0) 25 | { 26 | source = HashValEmpty; 27 | } 28 | 29 | uint length = (uint)source.Length; 30 | const ulong kMultiplier = 0x880355f21e6d1965; 31 | ulong h = seed ^ (length * kMultiplier); 32 | 33 | ReadOnlySpan data = MemoryMarshal.Cast(source); 34 | 35 | foreach (ulong value in data) 36 | { 37 | h ^= FastHashMix(value); 38 | h *= kMultiplier; 39 | } 40 | 41 | ReadOnlySpan data2 = source.Slice(data.Length * sizeof(ulong)); 42 | ulong v = 0; 43 | 44 | switch (length & 7) 45 | { 46 | case 7: v ^= (ulong)data2[6] << 48; goto case 6; 47 | case 6: v ^= (ulong)data2[5] << 40; goto case 5; 48 | case 5: v ^= (ulong)data2[4] << 32; goto case 4; 49 | case 4: v ^= (ulong)data2[3] << 24; goto case 3; 50 | case 3: v ^= (ulong)data2[2] << 16; goto case 2; 51 | case 2: v ^= (ulong)data2[1] << 8; goto case 1; 52 | case 1: 53 | v ^= data2[0]; 54 | h ^= FastHashMix(v); 55 | h *= kMultiplier; 56 | break; 57 | } 58 | 59 | return FastHashMix(h); 60 | } 61 | 62 | /// 63 | /// Compute 32-bit FastHash. 64 | /// 65 | /// The data to hash. 66 | /// Seed to compute the hash. 67 | public static uint Hash32(ReadOnlySpan source, uint seed) 68 | { 69 | // The following trick converts the 64-bit hashcode to Fermat 70 | // residue, which shall retain information from both the higher 71 | // and lower parts of hashcode. 72 | ulong h = Hash64(source, seed); 73 | return (uint)(h - (h >> 32)); 74 | } 75 | 76 | /// 77 | /// Compression function for Merkle-Damgard construction. 78 | /// 79 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 80 | private static ulong FastHashMix(ulong h) 81 | { 82 | h ^= h >> 23; 83 | h *= 0x2127599bf4325c37; 84 | h ^= h >> 47; 85 | return h; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /test/Knet.Kudu.Client.Tests/RequestTrackerTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using Knet.Kudu.Client.Connection; 4 | using Xunit; 5 | 6 | namespace Knet.Kudu.Client.Tests; 7 | 8 | public class RequestTrackerTests 9 | { 10 | [Fact] 11 | public void Test() 12 | { 13 | var tracker = new RequestTracker("test"); 14 | 15 | // A new tracker should have no incomplete RPCs. 16 | Assert.Equal(RequestTracker.NoSeqNo, tracker.FirstIncomplete); 17 | 18 | int max = 10; 19 | 20 | for (int i = 0; i < max; i++) 21 | { 22 | tracker.GetNewSeqNo(); 23 | } 24 | 25 | // The first RPC is the incomplete one. 26 | Assert.Equal(1, tracker.FirstIncomplete); 27 | 28 | // Mark the first as complete, incomplete should advance by 1. 29 | tracker.CompleteRpc(1); 30 | Assert.Equal(2, tracker.FirstIncomplete); 31 | 32 | // Mark the RPC in the middle as complete, first incomplete doesn't change. 33 | tracker.CompleteRpc(5); 34 | Assert.Equal(2, tracker.FirstIncomplete); 35 | 36 | // Mark 2-4 inclusive as complete. 37 | for (int i = 2; i <= 4; i++) 38 | { 39 | tracker.CompleteRpc(i); 40 | } 41 | 42 | Assert.Equal(6, tracker.FirstIncomplete); 43 | 44 | // Get a few more sequence numbers. 45 | long lastSeqNo = 0; 46 | for (int i = max / 2; i <= max; i++) 47 | { 48 | lastSeqNo = tracker.GetNewSeqNo(); 49 | } 50 | 51 | // Mark them all as complete except the last one. 52 | while (tracker.FirstIncomplete != lastSeqNo) 53 | { 54 | tracker.CompleteRpc(tracker.FirstIncomplete); 55 | } 56 | 57 | Assert.Equal(lastSeqNo, tracker.FirstIncomplete); 58 | tracker.CompleteRpc(lastSeqNo); 59 | 60 | // Test that we get back to NO_SEQ_NO after marking them all. 61 | Assert.Equal(RequestTracker.NoSeqNo, tracker.FirstIncomplete); 62 | } 63 | 64 | [Fact] 65 | public async Task TestMultiThreaded() 66 | { 67 | var rt = new RequestTracker("fake id"); 68 | var checker = new Checker(); 69 | var tasks = new Task[16]; 70 | 71 | for (int i = 0; i < tasks.Length; i++) 72 | { 73 | tasks[i] = Task.Run(() => 74 | { 75 | for (int n = 0; n < 1000; n++) 76 | { 77 | long seqNo = rt.GetNewSeqNo(); 78 | long incomplete = rt.FirstIncomplete; 79 | checker.Check(seqNo, incomplete); 80 | rt.CompleteRpc(seqNo); 81 | } 82 | }); 83 | } 84 | 85 | await Task.WhenAll(tasks); 86 | } 87 | 88 | private class Checker 89 | { 90 | private long _curIncomplete = 0; 91 | 92 | public void Check(long seqNo, long firstIncomplete) 93 | { 94 | lock (this) 95 | { 96 | Assert.True(seqNo >= _curIncomplete, 97 | "should not send a seq number that was previously marked complete"); 98 | _curIncomplete = Math.Max(firstIncomplete, _curIncomplete); 99 | } 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Util/EndpointParser.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics.CodeAnalysis; 2 | using System.Globalization; 3 | using Knet.Kudu.Client.Connection; 4 | 5 | namespace Knet.Kudu.Client.Util; 6 | 7 | /// 8 | /// https://github.com/StackExchange/StackExchange.Redis/blob/master/src/StackExchange.Redis/Format.cs 9 | /// 10 | public static class EndpointParser 11 | { 12 | public static bool TryParseInt32([NotNullWhen(true)] string? s, out int value) 13 | { 14 | return int.TryParse(s, NumberStyles.Integer, NumberFormatInfo.InvariantInfo, out value); 15 | } 16 | 17 | public static bool TryParse( 18 | [NotNullWhen(true)] string? addressWithPort, 19 | int defaultPort, 20 | [MaybeNullWhen(false)] out HostAndPort result) 21 | { 22 | // Adapted from IPEndPointParser in Microsoft.AspNetCore 23 | // Link: https://github.com/aspnet/BasicMiddleware/blob/f320511b63da35571e890d53f3906c7761cd00a1/src/Microsoft.AspNetCore.HttpOverrides/Internal/IPEndPointParser.cs#L8 24 | // Copyright (c) .NET Foundation. All rights reserved. 25 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 26 | if (string.IsNullOrEmpty(addressWithPort)) 27 | { 28 | result = null; 29 | return false; 30 | } 31 | 32 | string? addressPart; 33 | string? portPart = null; 34 | 35 | var lastColonIndex = addressWithPort!.LastIndexOf(':'); 36 | if (lastColonIndex > 0) 37 | { 38 | // IPv4 with port or IPv6 39 | var closingIndex = addressWithPort.LastIndexOf(']'); 40 | if (closingIndex > 0) 41 | { 42 | // IPv6 with brackets 43 | addressPart = addressWithPort.Substring(1, closingIndex - 1); 44 | if (closingIndex < lastColonIndex) 45 | { 46 | // IPv6 with port [::1]:80 47 | portPart = addressWithPort.Substring(lastColonIndex + 1); 48 | } 49 | } 50 | else 51 | { 52 | // IPv6 without port or IPv4 53 | var firstColonIndex = addressWithPort.IndexOf(':'); 54 | if (firstColonIndex != lastColonIndex) 55 | { 56 | // IPv6 ::1 57 | addressPart = addressWithPort; 58 | } 59 | else 60 | { 61 | // IPv4 with port 127.0.0.1:123 62 | addressPart = addressWithPort.Substring(0, firstColonIndex); 63 | portPart = addressWithPort.Substring(firstColonIndex + 1); 64 | } 65 | } 66 | } 67 | else 68 | { 69 | // IPv4 without port 70 | addressPart = addressWithPort; 71 | } 72 | 73 | int? port = null; 74 | if (portPart is not null) 75 | { 76 | if (TryParseInt32(portPart, out var portVal)) 77 | { 78 | port = portVal; 79 | } 80 | else 81 | { 82 | // Invalid port, return 83 | result = null; 84 | return false; 85 | } 86 | } 87 | 88 | result = new HostAndPort(addressPart, port ?? defaultPort); 89 | return true; 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Util/EpochTime.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Knet.Kudu.Client.Util; 4 | 5 | /// 6 | /// Provides methods for converting to/from Kudu's unixtime_micros. 7 | /// 8 | public static class EpochTime 9 | { 10 | /// 11 | /// Represents the number of ticks in 1 microsecond. This field is constant. 12 | /// 13 | private const long TicksPerMicrosecond = TimeSpan.TicksPerMillisecond / 1000; 14 | 15 | /// 16 | /// The minimum value (0001-01-01) for a Kudu date column. 17 | /// 18 | public const int MinDateValue = -719162; 19 | 20 | /// 21 | /// The maximum value (9999-12-31) for a Kudu date column. 22 | /// 23 | public const int MaxDateValue = 2932896; 24 | 25 | /// 26 | /// Unix epoch zero-point: January 1, 1970 (midnight UTC/GMT). 27 | /// 28 | public static readonly DateTime UnixEpoch = new(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); 29 | 30 | /// 31 | /// Converts the given to microseconds since the 32 | /// Unix epoch (1970-01-01T00:00:00Z). The value is converted to UTC time. 33 | /// 34 | /// The timestamp to convert to microseconds. 35 | public static long ToUnixTimeMicros(DateTime value) 36 | { 37 | var utcValue = value.ToUniversalTime(); 38 | var epochValue = utcValue - UnixEpoch; 39 | var micros = epochValue.Ticks / TicksPerMicrosecond; 40 | return micros; 41 | } 42 | 43 | /// 44 | /// Converts a microsecond offset from the Unix epoch (1970-01-01T00:00:00Z) 45 | /// to a . 46 | /// 47 | /// The offset in microseconds since the Unix epoch. 48 | public static DateTime FromUnixTimeMicros(long micros) 49 | { 50 | var ticks = UnixEpoch.Ticks + (micros * TicksPerMicrosecond); 51 | return new DateTime(ticks, DateTimeKind.Utc); 52 | } 53 | 54 | /// 55 | /// Converts the given to days since the 56 | /// Unix epoch (1970-01-01T00:00:00Z). The value is converted to UTC time. 57 | /// 58 | /// The timestamp to convert to days. 59 | public static int ToUnixTimeDays(DateTime value) 60 | { 61 | var utcValue = value.ToUniversalTime().Date; 62 | var epochValue = utcValue - UnixEpoch; 63 | var days = (int)(epochValue.Ticks / TimeSpan.TicksPerDay); 64 | CheckDateWithinRange(days); 65 | return days; 66 | } 67 | 68 | /// 69 | /// Converts a day offset from the Unix epoch (1970-01-01T00:00:00Z) 70 | /// to a . 71 | /// 72 | /// The offset in days since the Unix epoch. 73 | public static DateTime FromUnixTimeDays(int days) 74 | { 75 | CheckDateWithinRange(days); 76 | var ticks = UnixEpoch.Ticks + (days * TimeSpan.TicksPerDay); 77 | return new DateTime(ticks, DateTimeKind.Utc); 78 | } 79 | 80 | public static void CheckDateWithinRange(int days) 81 | { 82 | if (days < MinDateValue || days > MaxDateValue) 83 | { 84 | throw new ArgumentException( 85 | $"Date value '{days}' is out of range '0001-01-01':'9999-12-31'"); 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/Knet.Kudu.Client/Tablet/RemoteTablet.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Knet.Kudu.Client.Connection; 5 | 6 | namespace Knet.Kudu.Client.Tablet; 7 | 8 | /// 9 | /// 10 | /// This class encapsulates the information regarding a tablet and its locations. 11 | /// 12 | /// 13 | /// 14 | /// RemoteTablet's main function is to keep track of where the leader for this 15 | /// tablet is. For example, an RPC might call GetServerInfo, contact that TS, find 16 | /// it's not the leader anymore, and then re-fetch the tablet locations. This 17 | /// class is immutable. 18 | /// 19 | /// 20 | /// 21 | /// A RemoteTablet's life is expected to be long in a cluster where roles aren't 22 | /// changing often, and short when they do since the Kudu client will replace the 23 | /// RemoteTablet it caches with new ones after getting tablet locations from the master. 24 | /// 25 | /// 26 | public class RemoteTablet : IEquatable 27 | { 28 | private readonly ServerInfoCache _cache; 29 | 30 | public string TableId { get; } 31 | 32 | public string TabletId { get; } 33 | 34 | public Partition Partition { get; } 35 | 36 | public RemoteTablet( 37 | string tableId, 38 | string tabletId, 39 | Partition partition, 40 | ServerInfoCache cache) 41 | { 42 | TableId = tableId; 43 | TabletId = tabletId; 44 | Partition = partition; 45 | _cache = cache; 46 | } 47 | 48 | public IReadOnlyList Servers => _cache.Servers; 49 | 50 | public IReadOnlyList Replicas => _cache.Replicas; 51 | 52 | public ServerInfo? GetServerInfo( 53 | ReplicaSelection replicaSelection, string? location = null) 54 | { 55 | return _cache.GetServerInfo(replicaSelection, location); 56 | } 57 | 58 | public ServerInfo? GetLeaderServerInfo() => _cache.GetLeaderServerInfo(); 59 | 60 | /// 61 | /// Return the current leader, or null if there is none. 62 | /// 63 | public KuduReplica? GetLeaderReplica() => _cache.GetLeaderReplica(); 64 | 65 | public RemoteTablet DemoteLeader(string uuid) 66 | { 67 | var cache = _cache.DemoteLeader(uuid); 68 | return new RemoteTablet(TableId, TabletId, Partition, cache); 69 | } 70 | 71 | public RemoteTablet RemoveTabletServer(string uuid) 72 | { 73 | var cache = _cache.RemoveTabletServer(uuid); 74 | return new RemoteTablet(TableId, TabletId, Partition, cache); 75 | } 76 | 77 | public bool Equals(RemoteTablet? other) 78 | { 79 | if (other is null) 80 | return false; 81 | 82 | return TabletId == other.TabletId; 83 | } 84 | 85 | public override bool Equals(object? obj) => Equals(obj as RemoteTablet); 86 | 87 | public override int GetHashCode() => TabletId.GetHashCode(); 88 | 89 | public override string ToString() 90 | { 91 | var leader = _cache.GetLeaderServerInfo(); 92 | 93 | var tabletServers = _cache.Servers 94 | .Select(e => $"{e}{(e == leader ? "[L]" : "")}") 95 | // Sort so that we have a consistent iteration order. 96 | .OrderBy(e => e); 97 | 98 | return $"{TabletId}@[{string.Join(",", tabletServers)}]"; 99 | } 100 | } 101 | --------------------------------------------------------------------------------