├── CHANGELOG.md ├── src ├── LaravelDuckdb.php ├── Query │ ├── Builder.php │ ├── Processor.php │ └── Grammar.php ├── Schema │ ├── Blueprint.php │ ├── Grammar.php │ └── Builder.php ├── LaravelDuckdbModel.php ├── LaravelDuckdbPackageServiceProvider.php ├── LaravelDuckdbServiceProvider.php ├── Commands │ ├── ConnectDuckdbCliCommand.php │ └── DownloadDuckDBCliCommand.php └── LaravelDuckdbConnection.php ├── .gitignore ├── data-generator.sh ├── phpunit.xml.dist ├── LICENSE.md ├── tests ├── TestCase.php └── Feature │ ├── DuckDBBasicTest.php │ ├── DuckDBSchemaStatementTest.php │ └── DuckDBBigDataTest.php ├── composer.json └── README.md /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to `laravel-duckdb` will be documented in this file. 4 | -------------------------------------------------------------------------------- /src/LaravelDuckdb.php: -------------------------------------------------------------------------------- 1 | $2 -------------------------------------------------------------------------------- /src/LaravelDuckdbPackageServiceProvider.php: -------------------------------------------------------------------------------- 1 | name('laravel-duckdb') 21 | ->hasCommand(DownloadDuckDBCliCommand::class) 22 | ->hasCommand(ConnectDuckdbCliCommand::class); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/Schema/Builder.php: -------------------------------------------------------------------------------- 1 | blueprintResolver(function ($table, $callback, $prefix) { 26 | return new Blueprint($table, $callback, $prefix); 27 | }); 28 | return parent::createBlueprint($table, $callback); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /phpunit.xml.dist: -------------------------------------------------------------------------------- 1 | 2 | 13 | 14 | 15 | tests 16 | 17 | 18 | 19 | 20 | ./src 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) harish81 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/Query/Grammar.php: -------------------------------------------------------------------------------- 1 | isExpression($table)) { 14 | return parent::compileFrom($query, $table); 15 | } 16 | if (stripos($table, ' as ') !== false) { 17 | $segments = preg_split('/\s+as\s+/i', $table); 18 | return "from ".$this->wrapFromClause($segments[0], true) 19 | ." as " 20 | .$this->wrapFromClause($segments[1]); 21 | } 22 | 23 | return "from ".$this->wrapFromClause($table, true); 24 | 25 | } 26 | 27 | private function wrapFromClause($value, $prefixAlias = false){ 28 | if(!Str::endsWith($value, ')')){//is function 29 | return $this->quoteString(($prefixAlias?$this->tablePrefix:'').$value); 30 | } 31 | return ($prefixAlias?$this->tablePrefix:'').$value; 32 | } 33 | 34 | public function compileTruncate(Builder $query) 35 | { 36 | return ['truncate '.$this->wrapTable($query->from) => []]; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /tests/TestCase.php: -------------------------------------------------------------------------------- 1 | set('app.key', 'ZsZewWyUJ5FsKp9lMwv4tYbNlegQilM7'); 30 | 31 | $app['config']->set('database.connections.my_duckdb', [ 32 | 'driver' => 'duckdb', 33 | 'cli_path' => base_path('vendor/bin/duckdb'), 34 | 'cli_timeout' => 0, 35 | 'dbfile' => '/tmp/duck_main.db', 36 | ]); 37 | 38 | //default database just for schema testing, no need for production 39 | $app['config']->set('database.default', 'my_duckdb'); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/LaravelDuckdbServiceProvider.php: -------------------------------------------------------------------------------- 1 | base_path('vendor/bin/duckdb'), 14 | 'cli_timeout' => 60, 15 | 'dbfile' => storage_path('app/duckdb/duck_main.db'), 16 | //'database' => 'duck_main' //default to filename of dbfile, in most case no need to specify manually 17 | 'schema' => 'main', 18 | 'read_only' => false, 19 | 'pre_queries' => [], 20 | 'extensions' => [] 21 | ]; 22 | 23 | $this->app->resolving('db', function ($db) use ($defaultConfig) { 24 | $db->extend('duckdb', function ($config, $name) use ($defaultConfig) { 25 | $defaultConfig['database'] = pathinfo($config['dbfile'], PATHINFO_FILENAME); 26 | 27 | $config = array_merge($defaultConfig, $config); 28 | $config['name'] = $name; 29 | return new LaravelDuckdbConnection($config); 30 | }); 31 | }); 32 | } 33 | 34 | public function boot(): void 35 | { 36 | Model::setConnectionResolver($this->app['db']); 37 | 38 | Model::setEventDispatcher($this->app['events']); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/Commands/ConnectDuckdbCliCommand.php: -------------------------------------------------------------------------------- 1 | argument('connection_name')); 19 | $isReadonly = filter_var($this->option('readonly'), FILTER_VALIDATE_BOOLEAN); 20 | if(!$connection || ($connection['driver']??'') !== 'duckdb') throw new \Exception("DuckDB connection named `".$this->argument('connection_name')."` not found!"); 21 | 22 | $cmd = [ 23 | $connection['cli_path'], 24 | $connection['dbfile'] 25 | ]; 26 | if($isReadonly) array_splice($cmd, 1, 0, '--readonly'); 27 | 28 | $this->info('Connecting to duckdb cli `'.implode(" ", $cmd).'`'); 29 | $this->process = new Process($cmd); 30 | $this->process->setTimeout(0); 31 | $this->process->setIdleTimeout(0); 32 | $this->process->setTty(Process::isTtySupported()); 33 | 34 | $this->process->run(); 35 | } 36 | 37 | public function getSubscribedSignals(): array 38 | { 39 | return [SIGINT, SIGTERM]; 40 | } 41 | 42 | public function handleSignal(int $signal): void 43 | { 44 | $this->info('stopping...'); 45 | $this->process->signal($signal); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "harish81/laravel-duckdb", 3 | "description": "DuckDB CLI wrapper to interact with duckdb databases through laravel query builder.", 4 | "keywords": [ 5 | "harish81", 6 | "laravel", 7 | "laravel-duckdb" 8 | ], 9 | "homepage": "https://github.com/harish81/laravel-duckdb", 10 | "license": "MIT", 11 | "authors": [ 12 | { 13 | "name": "harish", 14 | "email": "nandoliyaharish@gmail.com", 15 | "role": "Developer" 16 | } 17 | ], 18 | "require": { 19 | "php": "^8.1", 20 | "spatie/laravel-package-tools": "^1.14.0", 21 | "illuminate/contracts": "^10.0", 22 | "illuminate/support": "^10.0", 23 | "illuminate/database": "^10.0", 24 | "illuminate/events": "^10.0", 25 | "illuminate/container": "^10.0", 26 | "guzzlehttp/guzzle": "^7.2", 27 | "illuminate/http": "^10.0" 28 | }, 29 | "require-dev": { 30 | "laravel/pint": "^1.0", 31 | "nunomaduro/collision": "^7.9", 32 | "orchestra/testbench": "^8.0", 33 | "pestphp/pest": "^2.0", 34 | "pestphp/pest-plugin-arch": "^2.0", 35 | "pestphp/pest-plugin-laravel": "^2.0", 36 | "phpunit/phpunit": "^10.0" 37 | }, 38 | "autoload": { 39 | "psr-4": { 40 | "Harish\\LaravelDuckdb\\": "src/" 41 | } 42 | }, 43 | "autoload-dev": { 44 | "psr-4": { 45 | "Harish\\LaravelDuckdb\\Tests\\": "tests/" 46 | } 47 | }, 48 | "scripts": { 49 | "post-autoload-dump": "@php ./vendor/bin/testbench package:discover --ansi", 50 | "analyse": "vendor/bin/phpstan analyse", 51 | "test": "vendor/bin/pest", 52 | "test-coverage": "vendor/bin/pest --coverage", 53 | "format": "vendor/bin/pint" 54 | }, 55 | "config": { 56 | "sort-packages": true, 57 | "allow-plugins": { 58 | "pestphp/pest-plugin": true, 59 | "phpstan/extension-installer": true 60 | } 61 | }, 62 | "extra": { 63 | "laravel": { 64 | "providers": [ 65 | "Harish\\LaravelDuckdb\\LaravelDuckdbPackageServiceProvider", 66 | "Harish\\LaravelDuckdb\\LaravelDuckdbServiceProvider" 67 | ] 68 | } 69 | }, 70 | "minimum-stability": "dev", 71 | "prefer-stable": true 72 | } 73 | -------------------------------------------------------------------------------- /src/Commands/DownloadDuckDBCliCommand.php: -------------------------------------------------------------------------------- 1 | option('ver'); 20 | 21 | $this->info("OS: $os, Architecture: $arch"); 22 | $this->newLine(); 23 | 24 | if(in_array($os, ['linux'])){ //linux 25 | if(in_array($arch, ['x86_64', 'amd64'])){ 26 | $this->downloadCli('linux', 'amd64', $version); 27 | }else{ 28 | $this->downloadCli('linux', $arch, $version); 29 | } 30 | } elseif (in_array($os, ['darwin'])){ 31 | $this->downloadCli('osx', 'universal', $version); 32 | }else{ 33 | throw new \Exception('Not Supported! Currently Only linux, mac supported. Try manually downloading cli from: https://duckdb.org/docs/installation/'); 34 | return; 35 | } 36 | } 37 | 38 | private function downloadCli($os, $arch, $version = null){ 39 | 40 | $duck_base_url = "https://github.com/duckdb/duckdb/releases/latest/download/duckdb_cli-__OS__-__PLATEFORM__.zip"; 41 | 42 | if ($version) { 43 | $duck_base_url = "https://github.com/duckdb/duckdb/releases/download/v$version/duckdb_cli-__OS__-__PLATEFORM__.zip"; 44 | } 45 | 46 | $url = str_replace(array('__OS__', '__PLATEFORM__'), array($os, $arch), $duck_base_url); 47 | 48 | $this->info("Downloading cli($url)..."); 49 | $this->newLine(); 50 | $res = Http::timeout(10*60) 51 | ->retry(2, 100) 52 | ->get($url); 53 | 54 | $content = $res->body(); 55 | //'vendor/bin/duckdb' 56 | file_put_contents('/tmp/duckdb_cli.zip', $content); 57 | 58 | $this->info('Extracting cli...'); 59 | $this->newLine(); 60 | $zip = new \ZipArchive(); 61 | $zipRes = $zip->open('/tmp/duckdb_cli.zip'); 62 | if($zipRes){ 63 | $zip->extractTo(base_path('vendor/bin/')); 64 | $zip->close(); 65 | 66 | chmod(base_path('vendor/bin/duckdb'), 0755); 67 | $this->info('Done! cli located at `'.base_path('vendor/bin/duckdb').'`'); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /tests/Feature/DuckDBBasicTest.php: -------------------------------------------------------------------------------- 1 | table = realpath(__DIR__.'/../../_test-data/test.csv'); 18 | } 19 | } 20 | 21 | class DuckDBBasicTest extends TestCase 22 | { 23 | public function test_cli_download_specific_version() 24 | { 25 | $version = '0.7.1'; 26 | Artisan::call('laravel-duckdb:download-cli --ver='.$version); 27 | $process = Process::fromShellCommandline(base_path('vendor/bin/duckdb').' --version'); 28 | $process->run(); 29 | 30 | $this->assertTrue(str_contains($process->getOutput(), $version)); 31 | } 32 | 33 | public function test_cli_download(){ 34 | Artisan::call('laravel-duckdb:download-cli'); 35 | $this->assertFileExists(base_path('vendor/bin/duckdb')); 36 | } 37 | 38 | /*public function test_connect_command_download(){ 39 | $opt = Artisan::call('laravel-duckdb:connect', ['connection_name' => 'my_duckdb']); 40 | $this->assertEquals(1, $opt); 41 | }*/ 42 | 43 | public function test_simple() 44 | { 45 | $rs = DB::connection('my_duckdb')->selectOne('select 1'); 46 | $this->assertArrayHasKey(1, $rs); 47 | } 48 | 49 | public function test_binding_escape_str(){ 50 | $str = "Co'mpl''ex` \"st'\"ring \\0 \\n \\r \\t `myworld`"; 51 | $rs = DB::connection('my_duckdb')->selectOne('select ? as one', [$str]); 52 | 53 | $this->assertEquals($str, $rs['one']); 54 | } 55 | 56 | public function test_read_csv(){ 57 | $rs = DB::connection('my_duckdb') 58 | ->table($this->getPackageBasePath('_test-data/test.csv')) 59 | ->get(); 60 | 61 | $this->assertNotEmpty($rs); 62 | } 63 | 64 | public function test_eloquent_model(){ 65 | 66 | $rs = DuckTestDataModel::where('VALUE','>',59712) 67 | ->first()->toArray(); 68 | $this->assertNotEmpty($rs); 69 | } 70 | 71 | public function test_query_exception(){ 72 | $this->expectException(QueryException::class); 73 | $rs = DB::connection('my_duckdb')->selectOne('select * from non_existing_tbl01 where foo=1 limit 1'); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /tests/Feature/DuckDBSchemaStatementTest.php: -------------------------------------------------------------------------------- 1 | fake()->name(), 18 | 'age' => fake()->numberBetween(13, 50), 19 | 'rank' => fake()->numberBetween(1, 10), 20 | 'salary' => fake()->randomFloat(null, 10000, 90000) 21 | ]; 22 | } 23 | } 24 | class Person extends \Harish\LaravelDuckdb\LaravelDuckdbModel{ 25 | use \Illuminate\Database\Eloquent\Factories\HasFactory; 26 | protected $connection = 'my_duckdb'; 27 | protected $table = 'people'; 28 | protected $guarded = ['id']; 29 | 30 | protected static function newFactory() 31 | { 32 | return PersonFactory::new(); 33 | } 34 | } 35 | class DuckDBSchemaStatementTest extends TestCase 36 | { 37 | public function test_migration(){ 38 | 39 | Schema::connection('my_duckdb')->dropIfExists('people'); 40 | DB::connection('my_duckdb')->statement('DROP SEQUENCE IF EXISTS people_sequence'); 41 | 42 | DB::connection('my_duckdb')->statement('CREATE SEQUENCE people_sequence'); 43 | Schema::connection('my_duckdb')->create('people', function (Blueprint $table) { 44 | $table->id()->default(new \Illuminate\Database\Query\Expression("nextval('people_sequence')")); 45 | $table->string('name'); 46 | $table->integer('age'); 47 | $table->integer('rank'); 48 | $table->unsignedDecimal('salary')->nullable(); 49 | $table->timestamps(); 50 | }); 51 | 52 | $this->assertTrue(Schema::hasTable('people')); 53 | } 54 | 55 | 56 | public function test_model(){ 57 | //truncate 58 | Person::truncate(); 59 | 60 | //create 61 | $singlePerson = Person::factory()->make()->toArray(); 62 | $newPerson = Person::create($singlePerson); 63 | 64 | //batch insert 65 | $manyPersons = Person::factory()->count(10)->make()->toArray(); 66 | Person::insert($manyPersons); 67 | 68 | //update 69 | $personToUpdate = Person::where('id', $newPerson->id)->first(); 70 | $personToUpdate->name = 'Harish81'; 71 | $personToUpdate->save(); 72 | $this->assertSame(Person::where('name', 'Harish81')->count(), 1); 73 | 74 | //delete 75 | Person::where('name', 'Harish81')->delete(); 76 | 77 | //assertion count 78 | $this->assertCount( 10, Person::all()); 79 | } 80 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DuckDB CLI wrapper to interact with duckdb databases through laravel query builder. 2 | 3 | [![Latest Version on Packagist](https://img.shields.io/packagist/v/harish81/laravel-duckdb.svg?style=flat-square)](https://packagist.org/packages/harish81/laravel-duckdb) 4 | [![Total Downloads](https://img.shields.io/packagist/dt/harish81/laravel-duckdb.svg?style=flat-square)](https://packagist.org/packages/harish81/laravel-duckdb) 5 | 6 | https://github.com/duckdb/duckdb 7 | - Download CLI (either) 8 | - https://duckdb.org/docs/installation/ 9 | - https://github.com/duckdb/duckdb/releases/latest 10 | - run `php artisan laravel-duckdb:download-cli` (Experimental) 11 | - You can also pass argument `--ver` for specific version like `php artisan laravel-duckdb:download-cli --ver=0.7.1` 12 | 13 | ## Support us 14 | 15 | ## Installation 16 | 17 | You can install the package via composer: 18 | 19 | ```bash 20 | composer require harish81/laravel-duckdb 21 | ``` 22 | 23 | ## Usage 24 | 25 | - Connect 26 | ```php 27 | 'connections' => [ 28 | 'my_duckdb' => [ 29 | 'driver' => 'duckdb', 30 | 'cli_path' => env('DUCKDB_CLI_PATH', base_path('vendor/bin/duckdb')), 31 | //'dbfile' => env('DUCKDB_DB_FILE', '/tmp/duck_main.db'), 32 | ], 33 | ... 34 | ``` 35 | 36 | - Examples 37 | ```php 38 | # Using DB facade 39 | DB::connection('my_duckdb') 40 | ->table(base_path('genderdata.csv')) 41 | ->where('Gender', '=', 'M') 42 | ->limit(10) 43 | ->get(); 44 | ``` 45 | ```php 46 | # Using Raw queries 47 | DB::connection('my_duckdb') 48 | ->select("select * from '".base_path('genderdata.csv')."' limit 5") 49 | ``` 50 | 51 | ```php 52 | # Using Eloquent Model 53 | class GenderDataModel extends \Harish\LaravelDuckdb\LaravelDuckdbModel 54 | { 55 | protected $connection = 'my_duckdb'; 56 | public function __construct() 57 | { 58 | $this->table = base_path('genderdata.csv'); 59 | } 60 | } 61 | ... 62 | GenderDataModel::where('Gender','M')->first() 63 | ``` 64 | 65 | ## Advanced Usage 66 | You can install duckdb extensions too. 67 | 68 | ### Query data from s3 files directly. 69 | 70 | - in `database.php` 71 | ```php 72 | 'connections' => [ 73 | 'my_duckdb' => [ 74 | 'driver' => 'duckdb', 75 | 'cli_path' => env('DUCKDB_CLI_PATH', base_path('vendor/bin/duckdb')), 76 | 'cli_timeout' => 0, //0 to disable timeout, default to 1 Minute (60s) 77 | 'dbfile' => env('DUCKDB_DB_FILE', storage_path('app/duckdb/duck_main.db')), 78 | 'pre_queries' => [ 79 | "SET s3_region='".env('AWS_DEFAULT_REGION')."'", 80 | "SET s3_access_key_id='".env('AWS_ACCESS_KEY_ID')."'", 81 | "SET s3_secret_access_key='".env('AWS_SECRET_ACCESS_KEY')."'", 82 | ], 83 | 'extensions' => ['httpfs'], 84 | ], 85 | ... 86 | ``` 87 | 88 | - Query data 89 | ```php 90 | DB::connection('my_duckdb') 91 | ->select("SELECT * FROM read_csv_auto('s3://my-bucket/test-datasets/example1/us-gender-data-2022.csv') LIMIT 10") 92 | ``` 93 | ### Writing a migration 94 | ```php 95 | return new class extends Migration { 96 | protected $connection = 'my_duckdb'; 97 | public function up(): void 98 | { 99 | DB::connection('my_duckdb')->statement('CREATE SEQUENCE people_sequence'); 100 | Schema::create('people', function (Blueprint $table) { 101 | $table->id()->default(new \Illuminate\Database\Query\Expression("nextval('people_sequence')")); 102 | $table->string('name'); 103 | $table->integer('age'); 104 | $table->integer('rank'); 105 | $table->timestamps(); 106 | }); 107 | } 108 | 109 | public function down(): void 110 | { 111 | Schema::dropIfExists('people'); 112 | DB::connection('my_duckdb')->statement('DROP SEQUENCE people_sequence'); 113 | } 114 | }; 115 | ``` 116 | 117 | ### Readonly Connection - A solution to concurrent query. 118 | - in `database.php` 119 | ```php 120 | 'connections' => [ 121 | 'my_duckdb' => [ 122 | 'driver' => 'duckdb', 123 | 'cli_path' => env('DUCKDB_CLI_PATH', base_path('vendor/bin/duckdb')), 124 | 'cli_timeout' => 0, 125 | 'dbfile' => env('DUCKDB_DB_FILE', storage_path('app/duckdb/duck_main.db')), 126 | 'schema' => 'main', 127 | 'read_only' => true, 128 | 'pre_queries' => [ 129 | "SET s3_region='".env('AWS_DEFAULT_REGION')."'", 130 | "SET s3_access_key_id='".env('AWS_ACCESS_KEY_ID')."'", 131 | "SET s3_secret_access_key='".env('AWS_SECRET_ACCESS_KEY')."'", 132 | ], 133 | 'extensions' => ['httpfs', 'postgres_scanner'], 134 | ], 135 | ... 136 | ``` 137 | 138 | 139 | ## Testing 140 | 141 | - Generate test data 142 | ```bash 143 | # Syntax: ./data-generator.sh 144 | ./data-generator.sh 100 _test-data/test.csv 145 | ./data-generator.sh 90000000 _test-data/test_big_file.csv 146 | ``` 147 | 148 | - Run Test case 149 | ```bash 150 | composer test 151 | ``` 152 | 153 | ## Limitations & FAQ 154 | 155 | - https://duckdb.org/faq 156 | 157 | ## Changelog 158 | 159 | Please see [CHANGELOG](CHANGELOG.md) for more information on what has changed recently. 160 | 161 | ## Contributing 162 | 163 | Please see [CONTRIBUTING](CONTRIBUTING.md) for details. 164 | 165 | ## Security Vulnerabilities 166 | 167 | Please review [our security policy](../../security/policy) on how to report security vulnerabilities. 168 | 169 | ## Credits 170 | 171 | - [harish](https://github.com/harish81) 172 | - [All Contributors](../../contributors) 173 | 174 | ## License 175 | 176 | The MIT License (MIT). Please see [License File](LICENSE.md) for more information. 177 | -------------------------------------------------------------------------------- /tests/Feature/DuckDBBigDataTest.php: -------------------------------------------------------------------------------- 1 | assertTrue( 13 | DB::connection('my_duckdb')->statement("create or replace table test_big_file as select * from '".$this->getPackageBasePath('_test-data/test_big_file.csv')."'") 14 | ); 15 | 16 | $distinct_foo_codes = DB::connection('my_duckdb')->select("select DISTINCT FOO_CODE as FOO_CODE from test_big_file"); 17 | $distinct_foo_codes = collect($distinct_foo_codes)->flatten()->toArray(); 18 | $countries = array("Afghanistan", "Albania", "Algeria", "American Samoa", "Andorra", "Angola", "Anguilla", "Antarctica", "Antigua and Barbuda", "Argentina", "Armenia", "Aruba", "Australia", "Austria", "Azerbaijan", "Bahamas", "Bahrain", "Bangladesh", "Barbados", "Belarus", "Belgium", "Belize", "Benin", "Bermuda", "Bhutan", "Bolivia", "Bosnia and Herzegowina", "Botswana", "Bouvet Island", "Brazil", "British Indian Ocean Territory", "Brunei Darussalam", "Bulgaria", "Burkina Faso", "Burundi", "Cambodia", "Cameroon", "Canada", "Cape Verde", "Cayman Islands", "Central African Republic", "Chad", "Chile", "China", "Christmas Island", "Cocos (Keeling) Islands", "Colombia", "Comoros", "Congo", "Congo, the Democratic Republic of the", "Cook Islands", "Costa Rica", "Cote d'Ivoire", "Croatia (Hrvatska)", "Cuba", "Cyprus", "Czech Republic", "Denmark", "Djibouti", "Dominica", "Dominican Republic", "East Timor", "Ecuador", "Egypt", "El Salvador", "Equatorial Guinea", "Eritrea", "Estonia", "Ethiopia", "Falkland Islands (Malvinas)", "Faroe Islands", "Fiji", "Finland", "France", "France Metropolitan", "French Guiana", "French Polynesia", "French Southern Territories", "Gabon", "Gambia", "Georgia", "Germany", "Ghana", "Gibraltar", "Greece", "Greenland", "Grenada", "Guadeloupe", "Guam", "Guatemala", "Guinea", "Guinea-Bissau", "Guyana", "Haiti", "Heard and Mc Donald Islands", "Holy See (Vatican City State)", "Honduras", "Hong Kong", "Hungary", "Iceland", "India", "Indonesia", "Iran (Islamic Republic of)", "Iraq", "Ireland", "Israel", "Italy", "Jamaica", "Japan", "Jordan", "Kazakhstan", "Kenya", "Kiribati", "Korea, Democratic People's Republic of", "Korea, Republic of", "Kuwait", "Kyrgyzstan", "Lao, People's Democratic Republic", "Latvia", "Lebanon", "Lesotho", "Liberia", "Libyan Arab Jamahiriya", "Liechtenstein", "Lithuania", "Luxembourg", "Macau", "Macedonia, The Former Yugoslav Republic of", "Madagascar", "Malawi", "Malaysia", "Maldives", "Mali", "Malta", "Marshall Islands", "Martinique", "Mauritania", "Mauritius", "Mayotte", "Mexico", "Micronesia, Federated States of", "Moldova, Republic of", "Monaco", "Mongolia", "Montserrat", "Morocco", "Mozambique", "Myanmar", "Namibia", "Nauru", "Nepal", "Netherlands", "Netherlands Antilles", "New Caledonia", "New Zealand", "Nicaragua", "Niger", "Nigeria", "Niue", "Norfolk Island", "Northern Mariana Islands", "Norway", "Oman", "Pakistan", "Palau", "Panama", "Papua New Guinea", "Paraguay", "Peru", "Philippines", "Pitcairn", "Poland", "Portugal", "Puerto Rico", "Qatar", "Reunion", "Romania", "Russian Federation", "Rwanda", "Saint Kitts and Nevis", "Saint Lucia", "Saint Vincent and the Grenadines", "Samoa", "San Marino", "Sao Tome and Principe", "Saudi Arabia", "Senegal", "Seychelles", "Sierra Leone", "Singapore", "Slovakia (Slovak Republic)", "Slovenia", "Solomon Islands", "Somalia", "South Africa", "South Georgia and the South Sandwich Islands", "Spain", "Sri Lanka", "St. Helena", "St. Pierre and Miquelon", "Sudan", "Suriname", "Svalbard and Jan Mayen Islands", "Swaziland", "Sweden", "Switzerland", "Syrian Arab Republic", "Taiwan, Province of China", "Tajikistan", "Tanzania, United Republic of", "Thailand", "Togo", "Tokelau", "Tonga", "Trinidad and Tobago", "Tunisia", "Turkey", "Turkmenistan", "Turks and Caicos Islands", "Tuvalu", "Uganda", "Ukraine", "United Arab Emirates", "United Kingdom", "United States", "United States Minor Outlying Islands", "Uruguay", "Uzbekistan", "Vanuatu", "Venezuela", "Vietnam", "Virgin Islands (British)", "Virgin Islands (U.S.)", "Wallis and Futuna Islands", "Western Sahara", "Yemen", "Yugoslavia", "Zambia", "Zimbabwe"); 19 | 20 | $final_foo_tbl = []; 21 | 22 | foreach ($distinct_foo_codes as $foo_code) { 23 | $final_foo_tbl[] = [ 24 | 'CODE' => $foo_code, 25 | 'COUNTRY' => $countries[array_rand($countries)], 26 | ]; 27 | } 28 | 29 | DB::connection('my_duckdb')->statement("drop table if exists foo_locations"); 30 | DB::connection('my_duckdb')->statement("create table foo_locations(CODE text, COUNTRY text)"); 31 | DB::connection('my_duckdb')->table('foo_locations')->insert($final_foo_tbl); 32 | } 33 | 34 | public function test_simple_query(){ 35 | $rawQueryRs = DB::connection('my_duckdb')->select("select * from test_big_file limit 1000"); 36 | $this->assertCount(1000, $rawQueryRs); 37 | 38 | $dbFQuery = DB::connection('my_duckdb') 39 | ->table('test_big_file') 40 | ->limit(1000) 41 | ->get()->toArray(); 42 | $this->assertCount(1000, $dbFQuery); 43 | } 44 | 45 | public function test_count_query(){ 46 | $rs = DB::connection('my_duckdb')->selectOne("select count(*) as total_count from test_big_file"); 47 | $this->assertGreaterThan(1000000, $rs['total_count']); 48 | } 49 | 50 | public function test_groupby_query(){ 51 | $rs = DB::connection('my_duckdb') 52 | ->select("select upper(PERSON) as person_name, CAST(SUM(VALUE) as hugeint) as sum_value, count(*) as total_rec from test_big_file 53 | group by person_name 54 | order by person_name"); 55 | 56 | $this->assertLessThanOrEqual(10, count($rs)); 57 | } 58 | 59 | public function test_join_query(){ 60 | $rs = DB::connection('my_duckdb') 61 | ->select("select FOO_CODE, CAST(SUM(VALUE) as hugeint) as sum_value, count(*) as total_rec, COUNTRY 62 | from test_big_file 63 | left join foo_locations on FOO_CODE = CODE 64 | group by FOO_CODE,COUNTRY 65 | order by FOO_CODE"); 66 | 67 | $this->assertNotEmpty($rs); 68 | } 69 | 70 | public function test_summarize_table(){ 71 | $rs = DB::connection('my_duckdb') 72 | ->select("SUMMARIZE test_big_file"); 73 | 74 | $this->assertTrue(count($rs)>0 && array_key_exists('approx_unique', $rs[0])); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/LaravelDuckdbConnection.php: -------------------------------------------------------------------------------- 1 | database = $config['database']; 23 | $this->config = $config; 24 | $this->config['dbfile'] = $config['dbfile']; 25 | 26 | $this->useDefaultPostProcessor(); 27 | $this->useDefaultSchemaGrammar(); 28 | $this->useDefaultQueryGrammar(); 29 | 30 | $this->ensureDuckdbDirectory(); 31 | $this->ensureDuckCliExists(); 32 | $this->installExtensions(); 33 | } 34 | 35 | public function query() 36 | { 37 | return $this->getDefaultQueryBuilder(); 38 | } 39 | 40 | public function table($table, $as = null) 41 | { 42 | return $this->query()->from($table, $as); 43 | } 44 | 45 | private function quote($str) 46 | { 47 | if(extension_loaded('sqlite3')){ 48 | return "'".\SQLite3::escapeString($str)."'"; 49 | } 50 | if(extension_loaded('pdo_sqlite')){ 51 | return (new \PDO('sqlite::memory:'))->quote($str); 52 | } 53 | 54 | return "'".preg_replace("/'/m", "''", $str)."'"; 55 | } 56 | 57 | private function getDuckDBCommand($query, $bindings = [], $safeMode=false){ 58 | $escapeQuery = $query; 59 | $countBindings = count($bindings??[]); 60 | if($countBindings>0){ 61 | foreach ($bindings as $index => $val) { 62 | $escapeQuery = Str::replaceFirst('?', $this->quote($val), $escapeQuery); 63 | } 64 | } 65 | 66 | //disable progressbar on long queries 67 | $disable_progressbar = "SET enable_progress_bar=false"; 68 | $preQueries = [$disable_progressbar]; 69 | foreach ($this->installed_extensions as $extension) { 70 | $preQueries[] = "LOAD '$extension'"; 71 | } 72 | 73 | $preQueries = array_merge($preQueries, $this->config['pre_queries']??[]); 74 | $cmdParams = [ 75 | $this->config['cli_path'], 76 | $this->config['dbfile'], 77 | ]; 78 | if($this->config['read_only']) array_splice($cmdParams, 1, 0, '--readonly'); 79 | if(!$safeMode) $cmdParams = array_merge($cmdParams, $preQueries); 80 | $cmdParams = array_merge($cmdParams, [ 81 | "$escapeQuery", 82 | "-json" 83 | ]); 84 | return $cmdParams; 85 | } 86 | 87 | private function installExtensions(){ 88 | if(empty($this->config['extensions']??[])) return; 89 | 90 | $cacheKey = $this->config['name'].'_duckdb_extensions'; 91 | $duckdb_extensions = Cache::rememberForever($cacheKey, function (){ 92 | return $this->executeDuckCliSql("select * from duckdb_extensions()", [], true); 93 | }); 94 | $sql = []; 95 | $tobe_installed_extensions = []; 96 | foreach ($this->config['extensions'] as $extension_name) { 97 | $ext = collect($duckdb_extensions)->where('extension_name', $extension_name)->first(); 98 | if($ext){ 99 | if(!$ext['installed']) 100 | $sql[$extension_name] = "INSTALL '$extension_name'"; 101 | 102 | $tobe_installed_extensions[] = $extension_name; 103 | } 104 | } 105 | if(!empty($sql)) Cache::forget($cacheKey); 106 | foreach ($sql as $ext_name=>$sExtQuery) { 107 | $this->executeDuckCliSql($sExtQuery, [], true); 108 | } 109 | $this->installed_extensions=$tobe_installed_extensions; 110 | } 111 | 112 | private function ensureDuckCliExists(){ 113 | if(!file_exists($this->config['cli_path'])){ 114 | throw new FileNotFoundException("DuckDB CLI Not Found. Make sure DuckDB CLI exists and provide valid `cli_path`. Download CLI From https://duckdb.org/docs/installation/index or run `artisan laravel-duckdb:download-cli`"); 115 | } 116 | } 117 | 118 | private function ensureDuckdbDirectory(){ 119 | if(!is_dir(storage_path('app/duckdb'))){ 120 | if (!mkdir($duckDirectory = storage_path('app/duckdb')) && !is_dir($duckDirectory)) { 121 | throw new \RuntimeException(sprintf('Directory "%s" was not created', $duckDirectory)); 122 | } 123 | } 124 | } 125 | 126 | private function executeDuckCliSql($sql, $bindings = [], $safeMode=false){ 127 | 128 | $command = $this->getDuckDBCommand($sql, $bindings, $safeMode); 129 | $process = new Process($command); 130 | $process->setTimeout($this->config['cli_timeout']); 131 | $process->setIdleTimeout(0); 132 | $process->run(); 133 | 134 | if (!$process->isSuccessful()) { 135 | $err = $process->getErrorOutput(); 136 | if(str_starts_with($err, 'Error:')){ 137 | $finalErr = trim(substr_replace($err, '', 0, strlen('Error:'))); 138 | throw new QueryException($this->getName(), $sql, $bindings, new \Exception($finalErr)); 139 | } 140 | 141 | throw new ProcessFailedException($process); 142 | } 143 | 144 | $raw_output = trim($process->getOutput()); 145 | return json_decode($raw_output, true)??[]; 146 | } 147 | 148 | private function runQueryWithLog($query, $bindings=[]){ 149 | $start = microtime(true); 150 | 151 | //execute 152 | $result = $this->executeDuckCliSql($query, $bindings); 153 | 154 | $this->logQuery( 155 | $query, [], $this->getElapsedTime($start) 156 | ); 157 | 158 | return $result; 159 | } 160 | 161 | public function statement($query, $bindings = []) 162 | { 163 | $this->runQueryWithLog($query, $bindings); 164 | 165 | return true; 166 | } 167 | 168 | public function select($query, $bindings = [], $useReadPdo = true) 169 | { 170 | return $this->runQueryWithLog($query, $bindings); 171 | } 172 | 173 | public function affectingStatement($query, $bindings = []) 174 | { 175 | //for update/delete 176 | //todo: we have to use : returning * to get list of affected rows; currently causing error; 177 | return $this->runQueryWithLog($query, $bindings); 178 | } 179 | 180 | private function getDefaultQueryBuilder(){ 181 | return new Builder($this, $this->getDefaultQueryGrammar(), $this->getDefaultPostProcessor()); 182 | } 183 | 184 | public function getDefaultQueryGrammar() 185 | { 186 | return $this->withTablePrefix(new QueryGrammar); 187 | } 188 | 189 | public function useDefaultPostProcessor() 190 | { 191 | $this->postProcessor = $this->getDefaultPostProcessor(); 192 | } 193 | 194 | public function getDefaultPostProcessor() 195 | { 196 | return new Processor(); 197 | } 198 | 199 | public function useDefaultQueryGrammar() 200 | { 201 | $this->queryGrammar = $this->getDefaultQueryGrammar(); 202 | } 203 | 204 | public function getSchemaBuilder() 205 | { 206 | if (is_null($this->schemaGrammar)) { 207 | $this->useDefaultSchemaGrammar(); 208 | } 209 | 210 | return new \Harish\LaravelDuckdb\Schema\Builder($this); 211 | } 212 | 213 | public function useDefaultSchemaGrammar() 214 | { 215 | $this->schemaGrammar = $this->getDefaultSchemaGrammar(); 216 | } 217 | 218 | protected function getDefaultSchemaGrammar() 219 | { 220 | return new SchemaGrammar; 221 | } 222 | 223 | /** 224 | * Get the schema grammar used by the connection. 225 | * 226 | * @return \Illuminate\Database\Schema\Grammars\Grammar 227 | */ 228 | public function getSchemaGrammar() 229 | { 230 | return $this->schemaGrammar; 231 | } 232 | } 233 | --------------------------------------------------------------------------------