├── .gitignore ├── SECURITY.md ├── src ├── Support │ ├── ConfigResolver.php │ └── QueryBuilder.php ├── Commands │ ├── VectorifyStatus.php │ └── VectorifyUpsert.php ├── Provider.php └── Jobs │ ├── ProcessCollection.php │ └── UpsertItems.php ├── .github └── workflows │ └── tests.yml ├── phpunit.xml ├── LICENSE.md ├── composer.json ├── tests ├── TestCase.php └── Unit │ ├── CollectionObjectTest.php │ ├── ItemObjectTest.php │ └── UpsertObjectTest.php ├── config └── vectorify.php └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /vendor 2 | /composer.lock 3 | /.phpunit.result.cache 4 | /.idea 5 | /.history 6 | /.vscode 7 | /.vagrant 8 | /.phpunit.cache 9 | _ide_helper.php 10 | .phpstorm.meta.php 11 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | **PLEASE DON'T DISCLOSE SECURITY-RELATED ISSUES PUBLICLY, [SEE BELOW](#reporting-a-vulnerability).** 4 | 5 | ## Reporting a Vulnerability 6 | 7 | If you discover any security related issues, please email info@vectorify.ai instead of using the issue tracker. 8 | -------------------------------------------------------------------------------- /src/Support/ConfigResolver.php: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | tests/Unit 10 | 11 | 12 | 13 | 14 | src 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) Vectorify 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "vectorifyai/vectorify-laravel", 3 | "description": "Vectorify package for Laravel. The fastest way to ask AI about your data.", 4 | "keywords": [ 5 | "laravel", 6 | "ai", 7 | "llm", 8 | "rag", 9 | "mcp", 10 | "vector", 11 | "vector database", 12 | "agent", 13 | "openai" 14 | ], 15 | "type": "library", 16 | "license": "MIT", 17 | "require": { 18 | "php": ">=8.2", 19 | "laravel/framework": "^10.0|^11.0|^12.0", 20 | "vectorifyai/vectorify-php": "^1.0" 21 | }, 22 | "require-dev": { 23 | "phpunit/phpunit": "^11.5" 24 | }, 25 | "autoload": { 26 | "psr-4": { 27 | "Vectorify\\Laravel\\": "src/" 28 | } 29 | }, 30 | "autoload-dev": { 31 | "psr-4": { 32 | "Vectorify\\Laravel\\Tests\\": "tests/" 33 | } 34 | }, 35 | "extra": { 36 | "laravel": { 37 | "providers": [ 38 | "Vectorify\\Laravel\\Provider" 39 | ] 40 | } 41 | }, 42 | "scripts": { 43 | "test": "vendor/bin/phpunit --colors=always", 44 | "psalm": "vendor/bin/psalm" 45 | }, 46 | "config": { 47 | "preferred-install": "dist", 48 | "sort-packages": true, 49 | "allow-plugins": { 50 | "php-http/discovery": true 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /tests/TestCase.php: -------------------------------------------------------------------------------- 1 | 'datetime', 40 | ]; 41 | 42 | public function customer(): BelongsTo 43 | { 44 | return $this->belongsTo(Customer::class); 45 | } 46 | 47 | public function scopeVectorify(Builder $query): Builder 48 | { 49 | return $query->where('status', '!=', 'cancelled'); 50 | } 51 | } 52 | 53 | class Customer extends Model 54 | { 55 | protected $table = 'customers'; 56 | 57 | protected $fillable = [ 58 | 'name', 59 | 'email', 60 | 'phone', 61 | 'address', 62 | ]; 63 | 64 | public function invoices(): HasMany 65 | { 66 | return $this->hasMany(Invoice::class); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /tests/Unit/CollectionObjectTest.php: -------------------------------------------------------------------------------- 1 | slug = 'invoices'; 19 | 20 | $this->metadata = [ 21 | 'customer_name' => [ 22 | 'type' => 'string', 23 | ], 24 | 'status' => [ 25 | 'type' => 'enum', 26 | 'options' => ['draft', 'sent', 'paid'], 27 | ], 28 | 'due_at' => [ 29 | 'type' => 'datetime', 30 | ], 31 | ]; 32 | } 33 | 34 | #[Test] 35 | public function collection_object_creation_with_slug_only(): void 36 | { 37 | $collection = new CollectionObject($this->slug); 38 | 39 | $this->assertEquals($this->slug, $collection->slug); 40 | $this->assertEquals([], $collection->metadata); 41 | } 42 | 43 | #[Test] 44 | public function collection_object_creation_with_metadata(): void 45 | { 46 | $collection = new CollectionObject($this->slug, $this->metadata); 47 | 48 | $this->assertEquals($this->slug, $collection->slug); 49 | $this->assertEquals($this->metadata, $collection->metadata); 50 | } 51 | 52 | #[Test] 53 | public function collection_returns_correct_structure(): void 54 | { 55 | $collection = new CollectionObject($this->slug, $this->metadata); 56 | 57 | $expected = [ 58 | 'slug' => $this->slug, 59 | 'metadata' => $this->metadata, 60 | ]; 61 | 62 | $this->assertEquals($expected, $collection->toArray()); 63 | $this->assertEquals($expected, $collection->toPayload()); 64 | $this->assertEquals($collection->toArray(), $collection->toPayload()); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/Commands/VectorifyStatus.php: -------------------------------------------------------------------------------- 1 | info('📊 Vectorify Upsert Status'); 19 | $this->newLine(); 20 | 21 | $collections = config('vectorify.collections'); 22 | 23 | if (empty($collections)) { 24 | $this->warn('No collections configured.'); 25 | } else { 26 | $headers = ['Collection', 'Last Upsert', 'Status']; 27 | $rows = []; 28 | 29 | foreach ($collections as $collection => $config) { 30 | $collectionId = is_int($collection) ? $config : $collection; 31 | 32 | $collectionSlug = ConfigResolver::getCollectionSlug($collectionId); 33 | 34 | $lastUpsert = Cache::get("vectorify:last_upsert:{$collectionSlug}"); 35 | 36 | $rows[] = [ 37 | $collectionSlug, 38 | $lastUpsert ? Carbon::parse($lastUpsert)->diffForHumans() : 'Never', 39 | $lastUpsert ? '✅ Upserted' : '⏳ Pending' 40 | ]; 41 | } 42 | 43 | $this->table($headers, $rows); 44 | } 45 | 46 | if ($rateLimitInfo = Cache::get('vectorify:api:rate_limit')) { 47 | $this->newLine(); 48 | 49 | $this->info('🔄 Vectorify API Rate Limit Status:'); 50 | 51 | $remaining = $rateLimitInfo['remaining'] ?? 'Unknown'; 52 | $resetTime = isset($rateLimitInfo['reset_time']) 53 | ? Carbon::parse($rateLimitInfo['reset_time'])->diffForHumans() 54 | : 'Unknown'; 55 | 56 | $this->line("• Remaining requests: {$remaining}"); 57 | $this->line("• Reset time: {$resetTime}"); 58 | } 59 | 60 | return self::SUCCESS; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/Provider.php: -------------------------------------------------------------------------------- 1 | mergeConfigFrom( 19 | __DIR__ . '/../config/vectorify.php', 'vectorify' 20 | ); 21 | 22 | $this->app->singleton(Vectorify::class, function (Application $app) { 23 | $apiKey = config('vectorify.api_key'); 24 | $timeout = config('vectorify.timeout', 300); 25 | 26 | if (empty($apiKey)) { 27 | throw new \InvalidArgumentException( 28 | message: 'Vectorify API key is required. Please set VECTORIFY_API_KEY environment variable.' 29 | ); 30 | } 31 | 32 | // Create Laravel cache store for rate limiting 33 | $store = new LaravelStore( 34 | $app->make('cache')->store(), 35 | 'vectorify:api:rate_limit' 36 | ); 37 | 38 | return new Vectorify($apiKey, $timeout, $store); 39 | }); 40 | } 41 | 42 | public function boot() 43 | { 44 | $this->publishes([ 45 | __DIR__ . '/../config/vectorify.php' => config_path('vectorify.php'), 46 | ], 'vectorify'); 47 | 48 | if (! $this->app->runningInConsole()) { 49 | return; 50 | } 51 | 52 | $this->commands([ 53 | VectorifyStatus::class, 54 | VectorifyUpsert::class, 55 | ]); 56 | 57 | $this->app->make(Schedule::class) 58 | ->command(VectorifyUpsert::class) 59 | ->everySixHours() 60 | ->runInBackground() 61 | ->onFailure(function (Stringable $output) { 62 | report('Vectorify Upsert failed: ' . $output); 63 | }); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /config/vectorify.php: -------------------------------------------------------------------------------- 1 | env('VECTORIFY_API_KEY'), 6 | 7 | 'tenancy' => env('VECTORIFY_TENANCY', 'single'), 8 | 9 | 'queue' => env('VECTORIFY_QUEUE', 'default'), 10 | 11 | 'timeout' => env('VECTORIFY_TIMEOUT', 300), 12 | 13 | 'collections' => [ 14 | // \App\Models\Invoice::class, 15 | // 'invoices' => [ 16 | // 'query' => fn () => \App\Models\Invoice::query()->with('customer'), 17 | // 'resource' => \App\Http\Resources\InvoiceResource::class, 18 | // 'metadata' => [ 19 | // 'customer_name' => [ 20 | // 'type' => 'string', 21 | // ], 22 | // 'status' => [ 23 | // 'type' => 'enum', 24 | // 'options' => ['draft', 'sent', 'paid'], 25 | // ], 26 | // 'due_date' => [ 27 | // 'type' => 'datetime', 28 | // ], 29 | // ], 30 | // ], 31 | // 'invoices' => [ 32 | // 'query' => fn () => \App\Models\Invoice::query()->with('customer'), 33 | // 'columns' => [ 34 | // 'customer' => [ 35 | // 'relationship' => true, 36 | // 'columns' => [ 37 | // 'name' => [ 38 | // 'alias' => 'customer_name', 39 | // 'metadata' => true, 40 | // 'type' => 'string', 41 | // ], 42 | // ], 43 | // ], 44 | // 'status' => [ 45 | // 'metadata' => true, 46 | // 'type' => 'enum', 47 | // 'options' => ['draft', 'sent', 'paid'], 48 | // ], 49 | // 'amount', 50 | // 'currency_code' => [ 51 | // 'alias' => 'currency', 52 | // ], 53 | // 'due_at' => [ 54 | // 'alias' => 'due_date', 55 | // 'format' => 'Y-m-d', 56 | // 'metadata' => true, 57 | // 'type' => 'datetime', 58 | // ], 59 | // ], 60 | // ], 61 | ], 62 | 63 | ]; 64 | -------------------------------------------------------------------------------- /src/Jobs/ProcessCollection.php: -------------------------------------------------------------------------------- 1 | collectionId); 34 | 35 | $config = ConfigResolver::getConfig($this->collectionId); 36 | 37 | $builder = new QueryBuilder($config, $this->since); 38 | 39 | $builder->getQuery()->chunk( 40 | count: 90, 41 | callback: function (EloquentCollection $items) use (&$totalChunks) { 42 | $totalChunks++; 43 | 44 | dispatch(new UpsertItems( 45 | collectionId: $this->collectionId, 46 | items: $items, 47 | ))->onQueue($this->queue); 48 | 49 | // Free memory 50 | unset($items); 51 | } 52 | ); 53 | 54 | Log::info("[Vectorify] Successfully processed chunks for collection: {$collectionSlug}", [ 55 | 'package' => 'vectorify', 56 | 'total_chunks' => $totalChunks, 57 | ]); 58 | 59 | if ($totalChunks > 0) { 60 | Cache::put( 61 | "vectorify:last_upsert:{$collectionSlug}", 62 | now()->toDateTimeString(), 63 | now()->addDays(30) 64 | ); 65 | } 66 | } 67 | 68 | public function backoff(): array 69 | { 70 | return [30, 60, 120]; 71 | } 72 | 73 | public function failed(Throwable $exception): void 74 | { 75 | $collectionSlug = ConfigResolver::getCollectionSlug($this->collectionId); 76 | 77 | Log::error("[Vectorify] Collection processing permanently failed for {$collectionSlug}", [ 78 | 'package' => 'vectorify', 79 | 'error' => $exception->getMessage(), 80 | ]); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/Jobs/UpsertItems.php: -------------------------------------------------------------------------------- 1 | collectionId); 36 | 37 | $config = ConfigResolver::getConfig($this->collectionId); 38 | 39 | Log::info("[Vectorify] Upserting items for collection: {$collectionSlug}", [ 40 | 'package' => 'vectorify', 41 | 'chunk_size' => $this->items->count(), 42 | ]); 43 | 44 | try { 45 | $builder = new QueryBuilder($config, null); 46 | 47 | $items = $this->items->map(function (Model $item) use ($builder) { 48 | $object = new ItemObject( 49 | id: $item->getKey(), 50 | data: $builder->getItemData($item), 51 | metadata: $builder->getItemMetadata(), 52 | tenant: $builder->getItemTenant(), 53 | url: null, 54 | ); 55 | 56 | $builder->resetItemData(); 57 | 58 | return $object; 59 | }); 60 | 61 | $object = new UpsertObject( 62 | collection: new CollectionObject( 63 | slug: $collectionSlug, 64 | metadata: $builder->metadata, 65 | ), 66 | items: $items->toArray(), 67 | ); 68 | 69 | $response = app(Vectorify::class)->upserts->create($object); 70 | 71 | if (! $response) { 72 | throw new \RuntimeException("Failed to upsert chunk for collection: {$collectionSlug}"); 73 | } 74 | 75 | Log::info("[Vectorify] Successfully upserted items for collection: {$collectionSlug}", [ 76 | 'package' => 'vectorify', 77 | 'chunk_size' => $this->items->count(), 78 | ]); 79 | } catch (Throwable $e) { 80 | Log::error("[Vectorify] Upserting failed for collection: {$collectionSlug}", [ 81 | 'package' => 'vectorify', 82 | 'chunk_size' => $this->items->count(), 83 | 'error' => $e->getMessage(), 84 | 'trace' => $e->getTraceAsString(), 85 | ]); 86 | 87 | throw $e; 88 | } 89 | } 90 | 91 | public function backoff(): array 92 | { 93 | return [30, 60, 120]; 94 | } 95 | 96 | public function failed(Throwable $exception): void 97 | { 98 | $collectionSlug = ConfigResolver::getCollectionSlug($this->collectionId); 99 | 100 | Log::error("[Vectorify] Upserting permanently failed for collection: {$collectionSlug}", [ 101 | 'package' => 'vectorify', 102 | 'chunk_size' => $this->items->count(), 103 | 'error' => $exception->getMessage(), 104 | ]); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Vectorify package for Laravel 2 | 3 | [![Latest Version](https://img.shields.io/packagist/v/vectorifyai/vectorify-laravel.svg?label=latest&style=flat)](https://packagist.org/packages/vectorifyai/vectorify-laravel) 4 | [![Total Downloads](https://img.shields.io/packagist/dt/vectorifyai/vectorify-laravel.svg?style=flat)](https://packagist.org/packages/vectorifyai/vectorify-laravel) 5 | [![Tests](https://img.shields.io/github/actions/workflow/status/vectorifyai/vectorify-laravel/tests.yml?label=tests&style=flat)](https://github.com/vectorifyai/vectorify-laravel/actions/workflows/tests.yml) 6 | [![License](https://img.shields.io/packagist/l/vectorifyai/vectorify-laravel.svg?style=flat)](LICENSE.md) 7 | 8 | Vectorify is the end-to-end AI connector for Laravel, letting you query and explore your data in natural language in seconds. 9 | 10 | Laravel is famous for turning complex web app chores into elegant, artisan-friendly code. Vectorify brings that same spirit to AI-powered data exploration: with one `composer install` and a single `config` file, your Laravel app becomes a conversational knowledge base you (and your customers) can query in natural language. 11 | 12 |
13 | vectorify setup 14 |

15 |
16 | 17 | To interact with your data, you have four primary methods to choose from: 18 | 19 | 1. Use the [Chats](https://app.vectorify.ai/) page within our platform (fastest) 20 | 2. Embed the [Chatbot](https://docs.vectorify.ai/project/chatbot) into your Laravel app (turn data querying into a product feature) 21 | 3. Add the [MCP](https://docs.vectorify.ai/mcp-server) server to ChatGPT, Claude, etc. (use your data anywhere you work) 22 | 4. Call the REST [API](https://docs.vectorify.ai/api-reference) endpoints (build custom integrations and workflows) 23 | 24 | Unlike text-to-SQL tools that expose your entire database and take 30+ seconds per query, Vectorify uses proven RAG technology to deliver accurate answers in <4 seconds while keeping your database secure. Head to our [blog](https://vectorify.ai/blog/vectorify-laravel-unlock-ai-ready-data-in-60-seconds) to learn more about Vectorify. 25 | 26 | This package provides seamless integration to automatically extract, transform, and upsert your Laravel application data to Vectorify. 27 | 28 | ## Requirements 29 | 30 | - PHP 8.2 or higher 31 | - Laravel 10 or higher 32 | 33 | ## Installation 34 | 35 | Install the package via Composer: 36 | 37 | ```bash 38 | composer require vectorifyai/vectorify-laravel 39 | ``` 40 | 41 | The package automatically registers itself with Laravel through package auto-discovery. 42 | 43 | ## Configuration 44 | 45 | ### 1. Publish configuration file 46 | 47 | Publish the configuration file to define your collections: 48 | 49 | ```bash 50 | php artisan vendor:publish --tag=vectorify 51 | ``` 52 | 53 | This will create a `config/vectorify.php` file in your application. 54 | 55 | ### 2. Environment variables 56 | 57 | Add the API Key to your `.env` file: 58 | 59 | ```env 60 | VECTORIFY_API_KEY=your_api_key_here 61 | ``` 62 | 63 | You can get your API Key from Vectorify's [dashboard](https://app.vectorify.ai). 64 | 65 | ### 3. Configure collections 66 | 67 | Edit the `config/vectorify.php` file to define which models (collections) and columns you want to upsert. The simplest collection configuration references a model class: 68 | 69 | ```php 70 | 'collections' => [ 71 | \App\Models\Invoice::class, 72 | ] 73 | ``` 74 | 75 | This approach uses the model's `$fillable` or a custom `$vectorify` property as the column list. Read the [documentation](https://docs.vectorify.ai/configuration) to learn more about defining the collections. 76 | 77 | ## Upsert 78 | 79 | ### Manual synchronisation 80 | 81 | Run the upsert command manually to sync your data: 82 | 83 | ```bash 84 | php artisan vectorify:upsert 85 | ``` 86 | 87 | ### Automatic synchronisation 88 | 89 | The package automatically schedules the upsert command to run every 6 hours. 90 | 91 | ## Changelog 92 | 93 | Please see [Releases](../../releases) for more information on what has changed recently. 94 | 95 | ## Contributing 96 | 97 | Pull requests are more than welcome. You must follow the PSR coding standards. 98 | 99 | ## Security 100 | 101 | Please review [our security policy](https://github.com/vectorifyai/laravel-vectorify/security/policy) on how to report security vulnerabilities. 102 | 103 | ## License 104 | 105 | The MIT License (MIT). Please see [LICENSE](LICENSE.md) for more information. 106 | -------------------------------------------------------------------------------- /tests/Unit/ItemObjectTest.php: -------------------------------------------------------------------------------- 1 | data = [ 21 | 'customer_id' => 1, 22 | 'amount' => 100, 23 | 'due_at' => '2025-06-30', 24 | ]; 25 | 26 | $this->metadata = [ 27 | 'customer_name' => 'John Doe', 28 | 'status' => 'draft', 29 | 'due_at' => '2025-06-30', 30 | ]; 31 | 32 | $this->tenant = 123; 33 | 34 | $this->url = 'https://example.com/invoices/1'; 35 | } 36 | 37 | #[Test] 38 | public function item_object_creation_with_required_parameters(): void 39 | { 40 | $item = new ItemObject(1, $this->data); 41 | 42 | $this->assertEquals(1, $item->id); 43 | $this->assertEquals($this->data, $item->data); 44 | $this->assertEquals([], $item->metadata); 45 | $this->assertNull($item->tenant); 46 | $this->assertNull($item->url); 47 | } 48 | 49 | #[Test] 50 | public function item_object_creation_with_all_parameters(): void 51 | { 52 | $item = new ItemObject( 53 | id: 1, 54 | data: $this->data, 55 | metadata: $this->metadata, 56 | tenant: $this->tenant, 57 | url: $this->url, 58 | ); 59 | 60 | $this->assertEquals(1, $item->id); 61 | $this->assertEquals($this->data, $item->data); 62 | $this->assertEquals($this->metadata, $item->metadata); 63 | $this->assertEquals($this->tenant, $item->tenant); 64 | $this->assertEquals($this->url, $item->url); 65 | } 66 | 67 | #[Test] 68 | public function item_returns_correct_structure(): void 69 | { 70 | $item = new ItemObject( 71 | id: 1, 72 | data: $this->data, 73 | metadata: $this->metadata, 74 | tenant: $this->tenant, 75 | url: $this->url, 76 | ); 77 | 78 | $expected = [ 79 | 'id' => 1, 80 | 'data' => $this->data, 81 | 'metadata' => $this->metadata, 82 | 'tenant' => $this->tenant, 83 | 'url' => $this->url, 84 | ]; 85 | 86 | $this->assertEquals($expected, $item->toArray()); 87 | $this->assertEquals($expected, $item->toPayload()); 88 | $this->assertEquals($item->toArray(), $item->toPayload()); 89 | } 90 | 91 | #[Test] 92 | public function item_object_with_empty_data(): void 93 | { 94 | $item = new ItemObject(1, []); 95 | 96 | $this->assertEquals(1, $item->id); 97 | $this->assertEquals([], $item->data); 98 | $this->assertEquals([], $item->metadata); 99 | } 100 | 101 | #[Test] 102 | public function item_object_with_complex_data_structure(): void 103 | { 104 | $complexInvoiceData = [ 105 | 'amount' => 7776.00, 106 | 'due_at' => '2025-08-15', 107 | 'customer_id' => 10, 108 | 'customer_details' => [ 109 | 'name' => 'Acme Corporation', 110 | 'email' => 'billing@acme.com', 111 | 'phone' => '+1-555-0123', 112 | 'address' => '123 Business Ave, Suite 100', 113 | ], 114 | 'line_items' => [ 115 | [ 116 | 'description' => 'Professional Services', 117 | 'quantity' => 40, 118 | 'rate' => 150.00, 119 | 'amount' => 6000.00, 120 | ], 121 | [ 122 | 'description' => 'Software License', 123 | 'quantity' => 1, 124 | 'rate' => 1200.00, 125 | 'amount' => 1200.00, 126 | ], 127 | ], 128 | 'totals' => [ 129 | 'subtotal' => 7200.00, 130 | 'tax' => 576.00, 131 | 'total' => 7776.00, 132 | ], 133 | ]; 134 | 135 | $item = new ItemObject(1, $complexInvoiceData); 136 | 137 | $this->assertEquals($complexInvoiceData, $item->data); 138 | $this->assertIsArray($item->data['customer_details']); 139 | $this->assertIsArray($item->data['line_items']); 140 | $this->assertIsArray($item->data['totals']); 141 | $this->assertEquals('Acme Corporation', $item->data['customer_details']['name']); 142 | $this->assertEquals(6000.00, $item->data['line_items'][0]['amount']); 143 | $this->assertEquals(7776.00, $item->data['totals']['total']); 144 | $this->assertEquals(10, $item->data['customer_id']); 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /tests/Unit/UpsertObjectTest.php: -------------------------------------------------------------------------------- 1 | collection = new CollectionObject( 21 | slug: 'invoices', 22 | metadata: [ 23 | 'status' => ['type' => 'enum', 'options' => ['draft', 'sent', 'paid']], 24 | 'due_at' => ['type' => 'datetime'], 25 | ] 26 | ); 27 | 28 | $this->items = [ 29 | new ItemObject( 30 | id: 1, 31 | data: [ 32 | 'customer_id' => 10, 33 | 'amount' => 1500.00, 34 | 'due_at' => '2025-07-15', 35 | ], 36 | metadata: ['status' => 'sent'] 37 | ), 38 | new ItemObject( 39 | id: 2, 40 | data: [ 41 | 'customer_id' => 5, 42 | 'amount' => 850.00, 43 | 'due_at' => '2025-07-01', 44 | ], 45 | metadata: ['status' => 'paid'] 46 | ), 47 | new ItemObject( 48 | id: 3, 49 | data: [ 50 | 'customer_id' => 8, 51 | 'amount' => 2200.00, 52 | 'due_at' => '2025-08-01', 53 | ], 54 | metadata: ['status' => 'draft'] 55 | ), 56 | ]; 57 | } 58 | 59 | #[Test] 60 | public function upsert_object_creation(): void 61 | { 62 | $upsert = new UpsertObject($this->collection, $this->items); 63 | 64 | $this->assertEquals($this->collection, $upsert->collection); 65 | $this->assertEquals($this->items, $upsert->items); 66 | } 67 | 68 | #[Test] 69 | public function upsert_to_array_returns_correct_structure(): void 70 | { 71 | $upsert = new UpsertObject($this->collection, $this->items); 72 | 73 | $expected = [ 74 | 'collection' => $this->collection, 75 | 'items' => $this->items, 76 | ]; 77 | 78 | $this->assertEquals($expected, $upsert->toArray()); 79 | } 80 | 81 | #[Test] 82 | public function upsert_to_payload_returns_correct_structure(): void 83 | { 84 | $upsert = new UpsertObject($this->collection, $this->items); 85 | 86 | $result = $upsert->toPayload(); 87 | 88 | $this->assertIsArray($result); 89 | $this->assertArrayHasKey('collection', $result); 90 | $this->assertArrayHasKey('items', $result); 91 | 92 | // Test collection payload 93 | $this->assertEquals($this->collection->toPayload(), $result['collection']); 94 | 95 | // Test items payload structure 96 | $this->assertIsArray($result['items']); 97 | $this->assertCount(3, $result['items']); 98 | 99 | foreach ($result['items'] as $index => $itemPayload) { 100 | $this->assertEquals($this->items[$index]->toPayload(), $itemPayload); 101 | } 102 | } 103 | 104 | #[Test] 105 | public function upsert_with_empty_items_array(): void 106 | { 107 | $upsert = new UpsertObject($this->collection, []); 108 | 109 | $result = $upsert->toPayload(); 110 | 111 | $this->assertEquals([], $result['items']); 112 | $this->assertEquals($this->collection->toPayload(), $result['collection']); 113 | } 114 | 115 | #[Test] 116 | public function upsert_with_single_item(): void 117 | { 118 | $singleItem = [new ItemObject(1, ['name' => 'John Doe'])]; 119 | $upsert = new UpsertObject($this->collection, $singleItem); 120 | 121 | $result = $upsert->toPayload(); 122 | 123 | $this->assertCount(1, $result['items']); 124 | $this->assertEquals($singleItem[0]->toPayload(), $result['items'][0]); 125 | } 126 | 127 | #[Test] 128 | public function upsert_with_complex_collection_metadata(): void 129 | { 130 | $complexMetadata = [ 131 | 'status' => [ 132 | 'type' => 'enum', 133 | 'options' => ['draft', 'review', 'published', 'archived'], 134 | ], 135 | 'priority' => [ 136 | 'type' => 'enum', 137 | 'options' => ['low', 'medium', 'high', 'urgent'], 138 | ], 139 | 'created_at' => [ 140 | 'type' => 'datetime', 141 | ], 142 | 'author_id' => [ 143 | 'type' => 'string', 144 | ], 145 | ]; 146 | 147 | $collection = new CollectionObject('articles', $complexMetadata); 148 | $items = [ 149 | new ItemObject(1, 150 | ['title' => 'Article 1', 'content' => 'Content 1'], 151 | ['status' => 'published', 'priority' => 'high'] 152 | ), 153 | ]; 154 | 155 | $upsert = new UpsertObject($collection, $items); 156 | $payload = $upsert->toPayload(); 157 | 158 | $this->assertEquals($complexMetadata, $payload['collection']['metadata']); 159 | $this->assertEquals('articles', $payload['collection']['slug']); 160 | } 161 | 162 | #[Test] 163 | public function upsert_with_items_containing_tenant_information(): void 164 | { 165 | $items = [ 166 | new ItemObject(1, ['data' => 'value1'], [], 100), 167 | new ItemObject(2, ['data' => 'value2'], [], 200), 168 | new ItemObject(3, ['data' => 'value3'], [], 100), 169 | ]; 170 | 171 | $upsert = new UpsertObject($this->collection, $items); 172 | $payload = $upsert->toPayload(); 173 | 174 | $this->assertEquals(100, $payload['items'][0]['tenant']); 175 | $this->assertEquals(200, $payload['items'][1]['tenant']); 176 | $this->assertEquals(100, $payload['items'][2]['tenant']); 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /src/Commands/VectorifyUpsert.php: -------------------------------------------------------------------------------- 1 | info('🚀 Starting the upsert process for collections...'); 30 | 31 | if (! config('vectorify.api_key')) { 32 | $this->error('❌ API key is not set.'); 33 | 34 | return self::FAILURE; 35 | } 36 | 37 | $collections = $this->getCollections(); 38 | 39 | if (is_int($collections)) { 40 | return $collections; 41 | } 42 | 43 | $queue = $this->option('queue') ?: config('vectorify.queue'); 44 | 45 | foreach ($collections as $collection => $config) { 46 | $collectionId = is_int($collection) ? $config : $collection; 47 | 48 | $collectionSlug = ConfigResolver::getCollectionSlug($collectionId); 49 | 50 | $since = $this->getSince($collectionSlug); 51 | 52 | $this->info("Upserting collection: {$collectionSlug}" . ($since ? " (since: {$since})" : " (full)")); 53 | 54 | if (config('queue.default') === 'sync') { 55 | $this->processCollection($collectionSlug, $config, $since); 56 | } else { 57 | dispatch(new ProcessCollection( 58 | collectionId: $collectionId, 59 | since: $since, 60 | ))->onQueue($queue); 61 | 62 | $this->info("➡️ Collection {$collectionSlug} queued for processing"); 63 | } 64 | } 65 | 66 | $this->info('✅ All collections upserted successfully!'); 67 | 68 | return self::SUCCESS; 69 | } 70 | 71 | public function processCollection( 72 | string $collectionSlug, 73 | mixed $config, 74 | ?string $since 75 | ): void { 76 | $totalChunks = 0; 77 | 78 | $builder = new QueryBuilder($config, $since); 79 | 80 | $builder->getQuery()->chunk( 81 | count: 90, 82 | callback: function (EloquentCollection $items) use ($collectionSlug, $builder, &$totalChunks) { 83 | $totalChunks++; 84 | 85 | $this->upsert( 86 | collectionSlug: $collectionSlug, 87 | builder: $builder, 88 | items: $items, 89 | ); 90 | 91 | $this->info("➡️ {$items->count()} items processed for collection: {$collectionSlug}"); 92 | 93 | // Free memory 94 | unset($items); 95 | } 96 | ); 97 | 98 | $this->info("➡️ {$totalChunks} chunks processed for collection: {$collectionSlug}"); 99 | 100 | if ($totalChunks > 0) { 101 | Cache::put( 102 | "vectorify:last_upsert:{$collectionSlug}", 103 | now()->toDateTimeString(), 104 | now()->addDays(30) 105 | ); 106 | } 107 | } 108 | 109 | public function upsert( 110 | string $collectionSlug, 111 | QueryBuilder $builder, 112 | EloquentCollection $items, 113 | ): bool { 114 | $items = $items->map(function (Model $item) use ($builder) { 115 | $object = new ItemObject( 116 | id: $item->getKey(), 117 | data: $builder->getItemData($item), 118 | metadata: $builder->getItemMetadata(), 119 | tenant: $builder->getItemTenant(), 120 | url: null, 121 | ); 122 | 123 | $builder->resetItemData(); 124 | 125 | return $object; 126 | }); 127 | 128 | $object = new UpsertObject( 129 | collection: new CollectionObject( 130 | slug: $collectionSlug, 131 | metadata: $builder->metadata, 132 | ), 133 | items: $items->toArray(), 134 | ); 135 | 136 | $response = app(Vectorify::class)->upserts->create($object); 137 | 138 | if (! $response) { 139 | $this->error("❌ Failed to upsert collection: {$collectionSlug}"); 140 | 141 | return false; 142 | } 143 | 144 | return true; 145 | } 146 | 147 | public function getCollections(): array|int 148 | { 149 | $collections = config('vectorify.collections'); 150 | 151 | if (empty($collections)) { 152 | $this->warn('⚠️ No collections found in the configuration file.'); 153 | 154 | return self::SUCCESS; 155 | } 156 | 157 | // Filter collections if specific collection requested 158 | if ($collection = $this->option('collection')) { 159 | if (! isset($collections[$collection])) { 160 | $this->error("❌ Collection '{$collection}' not found."); 161 | 162 | return self::FAILURE; 163 | } 164 | 165 | $collections = [$collection => $collections[$collection]]; 166 | } 167 | 168 | return $collections; 169 | } 170 | 171 | public function getSince(string $collectionSlug): ?string 172 | { 173 | if ($this->option('force')) { 174 | return null; // Full upsert 175 | } 176 | 177 | if ($since = $this->option('since')) { 178 | return $since; 179 | } 180 | 181 | return Cache::get("vectorify:last_upsert:{$collectionSlug}"); 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /src/Support/QueryBuilder.php: -------------------------------------------------------------------------------- 1 | query = is_string($config) 40 | ? $this->resolveStringQuery($config) 41 | : $this->resolveCallbackQuery($config['query']); 42 | 43 | $this->configColumns = $config['columns'] ?? []; 44 | 45 | $this->model = $this->query->getModel(); 46 | 47 | $this->resource = $this->resolveResource(); 48 | 49 | $this->columns = $this->resolveColumns(); 50 | 51 | $this->relations = $this->resolveRelations(); 52 | 53 | $this->metadata = $this->resolveMetadata(); 54 | 55 | $this->tenant = $this->resolveTenant(); 56 | } 57 | 58 | public function getQuery(): EloquentBuilder 59 | { 60 | if (method_exists($this->model, 'scopeVectorify')) { 61 | $this->query->vectorify(); 62 | } 63 | 64 | if (! empty($this->since) && $this->model->usesTimestamps()) { 65 | $this->query->where( 66 | column: $this->query->qualifyColumn($this->model->getUpdatedAtColumn()), 67 | operator: '>=', 68 | value: $this->since, 69 | ); 70 | } 71 | 72 | return ! empty($this->columns) 73 | ? $this->query->select($this->columns) 74 | : $this->query; 75 | } 76 | 77 | public function getItemData(EloquentModel $item): array 78 | { 79 | if (! empty($this->resource)) { 80 | $data = $this->resource::make($item) 81 | ->response() 82 | ->getData(); 83 | 84 | $data = json_decode(json_encode($data), true); 85 | 86 | $this->itemData = ! empty($data['data']) ? $data['data'] : $data; 87 | } else { 88 | $item->setAppends([]); // Without appends 89 | $this->itemData = $item->withoutRelations()->toArray(); 90 | } 91 | 92 | $this->unsetItemData[$this->getKeyName()] = $this->itemData[$this->getKeyName()]; 93 | unset($this->itemData[$this->getKeyName()]); 94 | 95 | foreach ($this->configColumns as $key => $col) { 96 | if (! is_array($col)) { 97 | if (is_null($this->itemData[$col])) { 98 | unset($this->itemData[$col]); 99 | } 100 | 101 | continue; 102 | } 103 | 104 | $name = $key; 105 | 106 | if (! empty($col['alias'])) { 107 | $name = $col['alias']; 108 | 109 | $this->itemData[$name] = $this->itemData[$key]; 110 | 111 | $this->unsetItemData[$key] = $this->itemData[$key]; 112 | unset($this->itemData[$key]); 113 | } 114 | 115 | if (isset($col['data']) && $col['data'] === false) { 116 | $this->unsetItemData[$name] = $this->itemData[$key]; 117 | unset($this->itemData[$name]); 118 | 119 | continue; 120 | } 121 | 122 | if (array_key_exists($name, $this->itemData) && is_null($this->itemData[$name])) { 123 | unset($this->itemData[$name]); 124 | 125 | continue; 126 | } 127 | 128 | if (empty($col['type'])) { 129 | continue; 130 | } 131 | 132 | if ($col['type'] === 'datetime' && ! empty($col['format'])) { 133 | $this->itemData[$name] = Carbon::parse($this->itemData[$name])->format($col['format']); 134 | } 135 | } 136 | 137 | if (empty($this->relations)) { 138 | return $this->itemData; 139 | } 140 | 141 | foreach ($this->relations as $rel) { 142 | $relColumns = $this->configColumns[$rel]['columns']; 143 | $relForeignKey = $this->configColumns[$rel]['foreign_key'] ?? $rel . '_id'; 144 | 145 | $relData = $item->{$rel}?->toArray() ?? []; 146 | 147 | foreach ($relColumns as $relKey => $relValue) { 148 | if (is_array($relValue)) { 149 | if (! array_key_exists($relKey, $relData)) { 150 | continue; 151 | } 152 | 153 | if (is_null($relData[$relKey])) { 154 | continue; 155 | } 156 | 157 | $name = $relKey; 158 | 159 | if (! empty($relValue['alias'])) { 160 | $name = $relValue['alias']; 161 | } 162 | 163 | $this->itemData[$name] = $relData[$relKey]; 164 | 165 | continue; 166 | } 167 | 168 | if (! array_key_exists($relValue, $relData)) { 169 | continue; 170 | } 171 | 172 | if (is_null($relData[$relValue])) { 173 | continue; 174 | } 175 | 176 | $this->itemData[$relValue] = $relData[$relValue]; 177 | } 178 | 179 | if (array_key_exists($relForeignKey, $this->itemData)) { 180 | $this->unsetItemData[$relForeignKey] = $this->itemData[$relForeignKey]; 181 | 182 | unset($this->itemData[$relForeignKey]); 183 | } 184 | } 185 | 186 | return $this->itemData; 187 | } 188 | 189 | public function getItemMetadata(): array 190 | { 191 | $metadata = []; 192 | 193 | if (empty($this->metadata)) { 194 | return $metadata; 195 | } 196 | 197 | foreach ($this->metadata as $key => $meta) { 198 | $metaKey = is_string($key) ? $key : $meta; 199 | 200 | if (array_key_exists($metaKey, $this->itemData) && ! is_null($this->itemData[$metaKey])) { 201 | $metadata[$metaKey] = $this->itemData[$metaKey]; 202 | } 203 | 204 | if (array_key_exists($metaKey, $this->unsetItemData) && ! is_null($this->unsetItemData[$metaKey])) { 205 | $metadata[$metaKey] = $this->unsetItemData[$metaKey]; 206 | } 207 | } 208 | 209 | return $metadata; 210 | } 211 | 212 | public function getItemTenant(): ?int 213 | { 214 | $tenant = null; 215 | 216 | if (empty($this->tenant)) { 217 | return $tenant; 218 | } 219 | 220 | if (array_key_exists($this->tenant, $this->itemData)) { 221 | $tenant = $this->itemData[$this->tenant]; 222 | } 223 | 224 | if (! $tenant && array_key_exists($this->tenant, $this->unsetItemData)) { 225 | $tenant = $this->unsetItemData[$this->tenant]; 226 | } 227 | 228 | return $tenant; 229 | } 230 | 231 | public function getKeyName(): ?string 232 | { 233 | // if (! in_array($this->model->getKey(), $this->model->toArray())) { 234 | // throw new InvalidArgumentException('ID field is required.'); 235 | // } 236 | 237 | return $this->model->getKeyName(); 238 | } 239 | 240 | public function resolveStringQuery(string $model): EloquentBuilder 241 | { 242 | if (! is_subclass_of($model, EloquentModel::class)) { 243 | throw new InvalidArgumentException('Collection must extend Eloquent\Model class.'); 244 | } 245 | 246 | return $model::query(); 247 | } 248 | 249 | public function resolveCallbackQuery(callable $configQuery): EloquentBuilder 250 | { 251 | $query = $configQuery(); 252 | 253 | if (! $query instanceof EloquentBuilder) { 254 | throw new InvalidArgumentException('Callback must return an Eloquent\Builder instance.'); 255 | } 256 | 257 | return $query; 258 | } 259 | 260 | public function resolveResource(): ?string 261 | { 262 | if (empty($this->config['resource'])) { 263 | return null; 264 | } 265 | 266 | $resource = $this->config['resource']; 267 | 268 | if (! class_exists($resource)) { 269 | throw new InvalidArgumentException('Resource class does not exist.'); 270 | } 271 | 272 | $reflection = new ReflectionClass($resource); 273 | 274 | if (! $reflection->isSubclassOf(JsonResource::class)) { 275 | throw new InvalidArgumentException('Resource must be an instance of JsonResource.'); 276 | } 277 | 278 | return $resource; 279 | } 280 | 281 | public function resolveColumns(): array 282 | { 283 | if (! empty($this->resource)) { 284 | return []; 285 | } 286 | 287 | if (! empty($this->configColumns)) { 288 | $columns = $this->configColumns; 289 | } else { 290 | $columns = property_exists($this->model, 'vectorify') 291 | ? $this->model->vectorify 292 | : $this->model->getFillable(); 293 | } 294 | 295 | $columns[] = $this->getKeyName(); 296 | 297 | return collect($columns) 298 | ->map(function ($col, $key) { 299 | if (is_string($col)) { 300 | return $col; 301 | } 302 | 303 | if (is_array($col) && array_key_exists('relationship', $col)) { 304 | return $col['foreign_key'] ?? $key . '_id'; 305 | } 306 | 307 | return $key; 308 | }) 309 | ->flatten() 310 | ->toArray(); 311 | } 312 | 313 | public function resolveRelations(): array 314 | { 315 | return collect($this->configColumns) 316 | ->filter(fn ($col) => is_array($col) && array_key_exists('relationship', $col)) 317 | ->map(fn ($col, $key) => $key) 318 | ->flatten() 319 | ->toArray(); 320 | } 321 | 322 | public function resolveMetadata(): array 323 | { 324 | if (! empty($this->config['metadata'])) { 325 | return $this->config['metadata']; 326 | } 327 | 328 | $metadata = collect($this->configColumns) 329 | ->filter(fn ($col) => is_array($col) && array_key_exists('metadata', $col)) 330 | ->mapWithKeys(function ($col, $key) { 331 | $name = array_key_exists('alias', $col) 332 | ? $col['alias'] 333 | : $key; 334 | 335 | $meta = [ 336 | 'type' => ! empty($col['type']) ? $col['type'] : 'string', 337 | ]; 338 | 339 | if ($meta['type'] === 'enum') { 340 | $meta['options'] = $col['options']; 341 | } 342 | 343 | return [$name => $meta]; 344 | }) 345 | ->toArray(); 346 | 347 | foreach ($this->relations as $rel) { 348 | $relColumns = $this->configColumns[$rel]['columns']; 349 | 350 | foreach ($relColumns as $relKey => $relValue) { 351 | if (! is_array($relValue)) { 352 | continue; 353 | } 354 | 355 | if (! array_key_exists('metadata', $relValue)) { 356 | continue; 357 | } 358 | 359 | $name = array_key_exists('alias', $relValue) 360 | ? $relValue['alias'] 361 | : $relKey; 362 | 363 | $meta = [ 364 | 'type' => ! empty($relValue['type']) ? $relValue['type'] : 'string', 365 | ]; 366 | 367 | if ($meta['type'] === 'enum') { 368 | $meta['options'] = $relValue['options']; 369 | } 370 | 371 | $metadata[$name] = $meta; 372 | } 373 | } 374 | 375 | return $metadata; 376 | } 377 | 378 | public function resolveTenant(): ?string 379 | { 380 | if (config('vectorify.tenancy') === 'single') { 381 | return null; 382 | } 383 | 384 | if (! empty($this->config['tenant'])) { 385 | return $this->config['tenant']; 386 | } 387 | 388 | $tenant = collect($this->configColumns) 389 | ->filter(fn ($col) => is_array($col) && array_key_exists('tenant', $col)) 390 | ->map(fn ($col, $key) => ! empty($col['alias']) ? $col['alias'] : $key) 391 | ->flatten() 392 | ->first(); 393 | 394 | if (empty($tenant)) { 395 | throw new InvalidArgumentException('Tenant column is required.'); 396 | } 397 | 398 | return $tenant; 399 | } 400 | 401 | public function resolveArrayColumn(string $name): SupportCollection 402 | { 403 | return collect($this->configColumns) 404 | ->filter(fn ($col) => is_array($col) && array_key_exists($name, $col)) 405 | ->map(fn ($col, $key) => $key) 406 | ->flatten(); 407 | } 408 | 409 | public function resetItemData(): void 410 | { 411 | $this->itemData = []; 412 | $this->unsetItemData = []; 413 | } 414 | } 415 | --------------------------------------------------------------------------------