├── .styleci.yml ├── src ├── MetadataRepository │ ├── Factory.php │ ├── Base.php │ ├── Json.php │ ├── Memory.php │ └── Database.php ├── Exceptions │ └── BackingAdapterException.php ├── Traits │ └── Uuid.php ├── Casts │ └── BackingData.php ├── AdapterStrategy │ ├── Factory.php │ ├── BaseAdapterStrategy.php │ └── Basic.php ├── Config.php ├── Models │ └── Metadata.php ├── Contracts │ ├── AdapterStrategy.php │ └── MetadataRepository.php ├── BackingData.php ├── FilerServiceProvider.php ├── Console │ └── Commands │ │ └── ImportMetadata.php ├── Metadata.php └── Flysystem │ └── FilerAdapter.php ├── tests ├── Metadata │ ├── DatabaseTests.php │ ├── MemoryTests.php │ ├── JsonTests.php │ └── MetadataBaseTestCase.php └── TestCase.php ├── .gitignore ├── phpunit.xml ├── config └── filer.php ├── composer.json ├── database └── migrations │ └── 2021_04_08_105954_create_filer_tables.php └── README.md /.styleci.yml: -------------------------------------------------------------------------------- 1 | preset: laravel 2 | 3 | risky: false 4 | 5 | #enabled: 6 | # - phpdoc_separation 7 | -------------------------------------------------------------------------------- /src/MetadataRepository/Factory.php: -------------------------------------------------------------------------------- 1 | id = (string) Str::uuid(); 15 | }); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/Metadata/DatabaseTests.php: -------------------------------------------------------------------------------- 1 | repository = new Database('testbench'); 17 | $this->repository->setStorageId('test'); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Casts/BackingData.php: -------------------------------------------------------------------------------- 1 | toJson(); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /tests/Metadata/MemoryTests.php: -------------------------------------------------------------------------------- 1 | repository = new Memory; 12 | $this->repository->setStorageId('test'); 13 | } 14 | 15 | public function test_it_returns_its_internal_state() 16 | { 17 | $this->assertEquals(['test' => []], $this->repository->getData()); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/AdapterStrategy/Factory.php: -------------------------------------------------------------------------------- 1 | storageId = $id; 14 | 15 | return $this; 16 | } 17 | 18 | protected function valueOrFalse($path, $value) 19 | { 20 | if ($metadata = $this->getMetadata($path)) { 21 | return $metadata->{$value}; 22 | } 23 | 24 | return false; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | /public/hot 3 | /public/storage 4 | /public/*.svg 5 | /public/svg 6 | /public/js/* 7 | /public/css/* 8 | /public/assets 9 | /storage/*.key 10 | /storage/debugbar 11 | /vendor 12 | /.idea 13 | /.vscode 14 | /.vagrant 15 | Homestead.json 16 | Homestead.yaml 17 | npm-debug.log 18 | yarn-error.log 19 | .env 20 | .env.testing 21 | .env.* 22 | !.env.example 23 | !.env.travis 24 | .phpunit.result.cache 25 | /public/dist 26 | .phpstorm.meta.php 27 | _ide_helper.php 28 | _ide_helper_models.php 29 | public/mix-manifest.json 30 | .DS_Store 31 | /public/vendor/horizon 32 | disable-xdebug.ini 33 | xdebug-filter.php 34 | coverage_out* 35 | build/ 36 | -------------------------------------------------------------------------------- /src/Config.php: -------------------------------------------------------------------------------- 1 | id = $id; 32 | $this->adapterClass = $adapterClass; 33 | $this->backingDisks = $backingDisks; 34 | $this->originalDisks = $originalDisks ?? []; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /phpunit.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | ./src 10 | 11 | 12 | 13 | 14 | ./tests/ 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/Models/Metadata.php: -------------------------------------------------------------------------------- 1 | 'integer', 33 | 'backing_data' => BackingData::class, 34 | ]; 35 | 36 | protected $dates = [ 37 | 'timestamp', 38 | ]; 39 | 40 | public function __construct(array $attributes = []) 41 | { 42 | parent::__construct($attributes); 43 | 44 | $this->setConnection(config('filer.database.connection')); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/Contracts/AdapterStrategy.php: -------------------------------------------------------------------------------- 1 | false, 16 | ] 17 | ) { 18 | $this->backingAdapters = $backingAdapters; 19 | $this->originalDisks = $originalDisks; 20 | $this->options = $options; 21 | } 22 | 23 | public function setOriginalDisks(array $originalDisks) 24 | { 25 | $this->originalDisks = $originalDisks; 26 | 27 | return $this; 28 | } 29 | 30 | public function setBackingDisks(array $backingDisks) 31 | { 32 | $this->backingAdapters = $backingDisks; 33 | 34 | return $this; 35 | } 36 | 37 | public function setOptions(array $options) 38 | { 39 | $this->options = $options; 40 | 41 | return $this; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /tests/TestCase.php: -------------------------------------------------------------------------------- 1 | set('database.default', 'testbench'); 33 | 34 | $app['config']->set('filer.metadata', 'database'); 35 | $app['config']->set('filer.database.connection', 'testbench'); 36 | 37 | $app['config']->set('database.connections.testbench', [ 38 | 'driver' => 'sqlite', 39 | 'database' => ':memory:', 40 | ]); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Contracts/MetadataRepository.php: -------------------------------------------------------------------------------- 1 | 'json', 17 | 18 | /* 19 | |-------------------------------------------------------------------------- 20 | | Database connection 21 | |-------------------------------------------------------------------------- 22 | | 23 | | Specify the database connection you wish to use. Defaults to the 24 | | application default connection. 25 | | 26 | */ 27 | 'database' => [ 28 | 'connection' => env('DB_CONNECTION', 'mysql'), 29 | ], 30 | 31 | /* 32 | |-------------------------------------------------------------------------- 33 | | JSON Storage 34 | |-------------------------------------------------------------------------- 35 | | 36 | | Path to where the JSON data will be stored after it is serialized. 37 | | 38 | */ 39 | 'json' => [ 40 | 'storage_path' => 'file-storage-metadata.json', 41 | ], 42 | 43 | ]; 44 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nvahalik/laravel-filer", 3 | "type": "library", 4 | "version": "1.0.0", 5 | "license": "MIT", 6 | "keywords": ["filesystem","flysystem"], 7 | "description": "An advanced wrapper over Flysystem for Laravel.", 8 | "authors": [ 9 | { 10 | "name": "Nick Vahalik", 11 | "email": "nick@nickvahalik.com" 12 | } 13 | ], 14 | "require": { 15 | "php": ">=7.4", 16 | "illuminate/database": "^8.2", 17 | "illuminate/support": "^8.2", 18 | "league/flysystem": "^1" 19 | }, 20 | "suggest": { 21 | "ext-json": "*" 22 | }, 23 | "autoload": { 24 | "psr-4": { 25 | "Nvahalik\\Filer\\": "src/" 26 | } 27 | }, 28 | "autoload-dev": { 29 | "psr-4": { 30 | "Tests\\": "tests/" 31 | } 32 | }, 33 | "scripts": { 34 | "test": [ 35 | "@phpunit" 36 | ], 37 | "phpunit": "phpunit --verbose" 38 | }, 39 | "extra": { 40 | "laravel": { 41 | "providers": [ 42 | "Nvahalik\\Filer\\FilerServiceProvider" 43 | ], 44 | "aliases": { 45 | "Filer": "Nvahalik\\Filer\\Facade" 46 | } 47 | } 48 | }, 49 | "require-dev": { 50 | "phpunit/phpunit": "^9.5", 51 | "orchestra/testbench": "^6.17" 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /database/migrations/2021_04_08_105954_create_filer_tables.php: -------------------------------------------------------------------------------- 1 | create('filer_metadata', function (Blueprint $table) { 17 | $table->uuid('id')->comment('Internal ID.'); 18 | $table->string('disk')->comment('The disk ID of the file.'); 19 | $table->string('path')->comment('The path of the file "on-disk".'); 20 | $table->unsignedInteger('size')->comment('The size of the file.'); 21 | $table->string('mimetype')->default('application/octet-stream'); 22 | $table->string('etag')->nullable()->comment('The Etag for the file.'); 23 | $table->enum('visibility', ['public', 'private'])->comment('Visibiilty of the file.'); 24 | $table->json('backing_data')->comment('The information about where the file is stored and how.'); 25 | $table->timestamp('timestamp')->useCurrent(); 26 | $table->unique([ 27 | 'disk', 'path', 28 | ]); 29 | $table->primary('id'); 30 | }); 31 | } 32 | 33 | /** 34 | * Reverse the migrations. 35 | * 36 | * @return void 37 | */ 38 | public function down() 39 | { 40 | Schema::connection(config('filer.database.connection'))->dropIfExists('filer_metadata'); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /tests/Metadata/JsonTests.php: -------------------------------------------------------------------------------- 1 | repository = new Json('test.json'); 15 | $this->repository->setStorageId('test'); 16 | } 17 | 18 | public function test_it_loads_existing_data_from_file() 19 | { 20 | Storage::fake()->put('test.json', '{ 21 | "test": { 22 | "example.txt": { 23 | "path": "example.txt", 24 | "etag": "ce114e4501d2f4e2dcea3e17b546f339", 25 | "mimetype": "text\/plain", 26 | "visibility": "private", 27 | "size": 14, 28 | "backing_data": [], 29 | "created_at": 1618368967, 30 | "updated_at": 1618368967 31 | } 32 | } 33 | }'); 34 | $this->repository = new Json('test.json'); 35 | $this->repository->setStorageId('test'); 36 | 37 | $this->assertTrue($this->repository->has('example.txt')); 38 | $md = $this->repository->getMetadata('example.txt'); 39 | $this->assertIsObject($md); 40 | 41 | $arrayData = $md->toArray(); 42 | 43 | $this->assertEquals($arrayData['path'], 'example.txt'); 44 | $this->assertEquals($arrayData['etag'], 'ce114e4501d2f4e2dcea3e17b546f339'); 45 | $this->assertEquals($arrayData['mimetype'], 'text/plain'); 46 | $this->assertEquals($arrayData['visibility'], 'private'); 47 | $this->assertEquals($arrayData['size'], 14); 48 | $this->assertIsObject($arrayData['backing_data']); 49 | $this->assertEquals($arrayData['timestamp'], 1618368967); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/BackingData.php: -------------------------------------------------------------------------------- 1 | data[$disk] = $data; 15 | 16 | return $this; 17 | } 18 | 19 | public function fill($diskData) 20 | { 21 | $this->data = $diskData; 22 | 23 | return $this; 24 | } 25 | 26 | public function getDisk($disk) 27 | { 28 | return $this->data[$disk] ?? null; 29 | } 30 | 31 | public function removeDisk($disk) 32 | { 33 | unset($this->data[$disk]); 34 | 35 | return $this; 36 | } 37 | 38 | public function reset() 39 | { 40 | $this->data = []; 41 | 42 | return $this; 43 | } 44 | 45 | public function updateDisk($disk, $data) 46 | { 47 | $this->data[$disk] = $data; 48 | 49 | return $this; 50 | } 51 | 52 | public function disks() 53 | { 54 | return array_keys($this->data); 55 | } 56 | 57 | public function toArray() 58 | { 59 | return $this->data; 60 | } 61 | 62 | public static function diskAndPath($disk, $path) 63 | { 64 | return (new static())->addDisk($disk, [ 65 | 'path' => $path, 66 | ]); 67 | } 68 | 69 | public static function unserialize($data) 70 | { 71 | if (is_string($data)) { 72 | $unserializedData = json_decode($data, true, 4); 73 | } else { 74 | $unserializedData = $data; 75 | } 76 | 77 | return (new static)->fill($unserializedData); 78 | } 79 | 80 | public function toJson($options = 0) 81 | { 82 | return json_encode($this->toArray(), JSON_THROW_ON_ERROR | $options, 4); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/MetadataRepository/Json.php: -------------------------------------------------------------------------------- 1 | filename = $filename; 16 | 17 | try { 18 | $data = Storage::get($filename); 19 | $this->data = json_decode($data, true, 10); 20 | foreach ($this->data as $storageId => $contents) { 21 | $this->data[$storageId] = array_map(function ($array) { 22 | return Metadata::deserialize($array); 23 | }, $contents); 24 | } 25 | } catch (FileNotFoundException $e) { 26 | } 27 | } 28 | 29 | public function setVisibility(string $path, string $visibility) 30 | { 31 | $update = parent::setVisibility($path, $visibility); 32 | 33 | $this->persist(); 34 | 35 | return $update; 36 | } 37 | 38 | public function record(Metadata $metadata) 39 | { 40 | parent::record($metadata); 41 | 42 | $this->persist(); 43 | } 44 | 45 | public function delete(string $path) 46 | { 47 | parent::delete($path); 48 | 49 | $this->persist(); 50 | } 51 | 52 | public function setBackingData(string $path, $backingData) 53 | { 54 | parent::setBackingData($path, $backingData); 55 | 56 | $this->persist(); 57 | } 58 | 59 | public function rename(string $path, string $newPath) 60 | { 61 | parent::rename($path, $newPath); 62 | 63 | $this->persist(); 64 | } 65 | 66 | private function persist() 67 | { 68 | Storage::put($this->filename, json_encode($this->toArray(), JSON_PRETTY_PRINT | 0, 10)); 69 | } 70 | 71 | public function toArray() 72 | { 73 | return array_map(function ($entries) { 74 | return array_map(function (Metadata $entry) { 75 | return $entry->serialize(); 76 | }, $entries); 77 | }, $this->data); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/MetadataRepository/Memory.php: -------------------------------------------------------------------------------- 1 | valueOrFalse($path, 'size'); 16 | } 17 | 18 | public function setStorageId(string $id): MetadataRepository 19 | { 20 | if (! isset($this->data[$id])) { 21 | $this->data[$id] = []; 22 | } 23 | 24 | return parent::setStorageId($id); 25 | } 26 | 27 | public function getData() 28 | { 29 | return $this->data; 30 | } 31 | 32 | public function getMimetype($path) 33 | { 34 | return $this->valueOrFalse($path, 'mimetype'); 35 | } 36 | 37 | public function getVisibility($path) 38 | { 39 | return $this->valueOrFalse($path, 'visibility'); 40 | } 41 | 42 | public function getTimestamp($path) 43 | { 44 | return $this->valueOrFalse($path, 'timestamp'); 45 | } 46 | 47 | /** 48 | * @param $path 49 | */ 50 | public function getMetadata($path): ?Metadata 51 | { 52 | if (! isset($this->data[$this->storageId][$path])) { 53 | return null; 54 | } 55 | 56 | $metadata = $this->data[$this->storageId][$path]; 57 | 58 | return $this->data[$this->storageId][$path]; 59 | } 60 | 61 | public function listContents($directory = '', $recursive = false) 62 | { 63 | $directory = ltrim(rtrim($directory, '/').'/', '/'); 64 | 65 | $directoryOffset = strlen($directory); 66 | 67 | $matchingFiles = array_filter(array_keys($this->data[$this->storageId]), function ($path) use ( 68 | $recursive, $directory, $directoryOffset 69 | ) { 70 | $matchesPath = $directory === '' ? true : stripos($path, $directory) === 0; 71 | $hasTrailingDirectories = strpos($path, '/', $directoryOffset) === false; 72 | 73 | return $matchesPath && ($recursive ? true : $hasTrailingDirectories); 74 | }); 75 | 76 | $contents = []; 77 | 78 | foreach ($matchingFiles as $file) { 79 | $contents[] = $this->data[$this->storageId] + [ 80 | 'path' => str_replace($directory, '', $file), 81 | ]; 82 | } 83 | 84 | return $contents; 85 | } 86 | 87 | public function has(string $path): bool 88 | { 89 | return isset($this->data[$this->storageId][$path]); 90 | } 91 | 92 | public function setVisibility(string $path, string $visibility) 93 | { 94 | if ($this->has($path)) { 95 | $this->data[$this->storageId][$path]->visibility = $visibility; 96 | } 97 | 98 | return $this->getMetadata($path); 99 | } 100 | 101 | public function record(Metadata $metadata) 102 | { 103 | $this->data[$this->storageId][$metadata->path] = $metadata; 104 | } 105 | 106 | public function delete(string $path) 107 | { 108 | unset($this->data[$this->storageId][$path]); 109 | } 110 | 111 | public function rename(string $path, string $newPath) 112 | { 113 | $this->data[$this->storageId][$newPath] = $this->data[$this->storageId][$path]; 114 | $this->data[$this->storageId][$path] = null; 115 | unset($this->data[$this->storageId][$path]); 116 | } 117 | 118 | public function setBackingData(string $path, BackingData $backingData) 119 | { 120 | $this->data[$this->storageId][$path]->setBackingData($backingData); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/FilerServiceProvider.php: -------------------------------------------------------------------------------- 1 | app->runningInConsole()) { 22 | $this->registerMigrations(); 23 | $this->registerCommand(); 24 | } 25 | 26 | $this->mergeConfigFrom( 27 | __DIR__.'/../config/filer.php', 'filer' 28 | ); 29 | 30 | $this->publishes([ 31 | __DIR__.'/../database/migrations' => database_path('migrations'), 32 | ], 'filer-migrations'); 33 | 34 | $this->publishes([ 35 | __DIR__.'/../config/filer.php' => config_path('filer.php'), 36 | ], 'filer-config'); 37 | 38 | Storage::extend('filer', function (Application $app, $config) { 39 | $backing_disks = array_combine($config['backing_disks'], array_map(function ($backing_disk) use ($app) { 40 | return $app->make('filesystem')->disk($backing_disk); 41 | }, $config['backing_disks'])); 42 | 43 | $config['original_disks'] = $config['original_disks'] ?? []; 44 | 45 | $original_disks = array_combine($config['original_disks'], array_map(function ($backing_disk) use ($app) { 46 | return $app->make('filesystem')->disk($backing_disk); 47 | }, $config['original_disks'])); 48 | 49 | $filerConfig = new Config( 50 | $config['id'], 51 | $backing_disks, 52 | $config['strategy'] ?? 'priority', 53 | $original_disks 54 | ); 55 | 56 | $adapterStrategy = Factory::make( 57 | $config['adapter_strategy'], 58 | $filerConfig->backingDisks, 59 | $filerConfig->originalDisks, 60 | $config['adapter_strategy_config'] ?? [] 61 | ); 62 | 63 | return new Filesystem( 64 | new FilerAdapter( 65 | $filerConfig, 66 | $app->make(MetadataRepository::class)->setStorageId($config['id']), 67 | $adapterStrategy 68 | ), 69 | $config 70 | ); 71 | }); 72 | } 73 | 74 | public function register() 75 | { 76 | $this->app->bind(MetadataRepository::class, function ($app, $config) { 77 | switch ($app['config']['filer']['metadata']) { 78 | case 'json': 79 | return new Json($app['config']['filer']['json']['storage_path']); 80 | case 'database': 81 | return new Database($app['config']['filer']['database']['connection']); 82 | case 'memory': 83 | default: 84 | return new Memory(); 85 | } 86 | }); 87 | } 88 | 89 | /** 90 | * Register Passport's migration files. 91 | * 92 | * @return void 93 | */ 94 | protected function registerMigrations() 95 | { 96 | $this->loadMigrationsFrom(__DIR__.'/../database/migrations'); 97 | } 98 | 99 | private function registerCommand() 100 | { 101 | $this->commands([ 102 | ImportMetadata::class, 103 | ]); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/MetadataRepository/Database.php: -------------------------------------------------------------------------------- 1 | connection = $connection ?? 'default'; 23 | } 24 | 25 | /** 26 | * @inheritdoc 27 | */ 28 | public function getVisibility(string $path) 29 | { 30 | return $this->valueOrFalse($path, 'visibility'); 31 | } 32 | 33 | private function newQuery() 34 | { 35 | return DB::connection($this->connection) 36 | ->table($this->table) 37 | ->where('disk', '=', $this->storageId); 38 | } 39 | 40 | private function read($path) 41 | { 42 | return $this->newQuery() 43 | ->where('path', '=', $path) 44 | ->first() ?? false; 45 | } 46 | 47 | /** 48 | * @inheritdoc 49 | */ 50 | public function getTimestamp(string $path) 51 | { 52 | return $this->valueOrFalse($path, 'timestamp'); 53 | } 54 | 55 | /** 56 | * @inheritdoc 57 | */ 58 | public function getMimetype(string $path) 59 | { 60 | return $this->valueOrFalse($path, 'mimetype'); 61 | } 62 | 63 | /** 64 | * @inheritdoc 65 | */ 66 | public function getSize(string $path) 67 | { 68 | return $this->valueOrFalse($path, 'size'); 69 | } 70 | 71 | /** 72 | * @inheritdoc 73 | */ 74 | public function getMetadata(string $path): ?Metadata 75 | { 76 | if ($metadata = $this->read($path)) { 77 | $metadata = Metadata::deserialize((array) $metadata); 78 | } 79 | 80 | return $metadata ?: null; 81 | } 82 | 83 | public function listContents(string $directory = '', bool $recursive = false) 84 | { 85 | return $this->newQuery() 86 | ->when($recursive, function ($query) use ($directory) { 87 | $query->where('path', 'LIKE', "$directory%"); 88 | }, function ($query) use ($directory) { 89 | $query->where('path', 'LIKE', "$directory%") 90 | ->where('path', 'NOT LIKE', "$directory%/%"); 91 | })->cursor() 92 | ->map(function ($record) { 93 | $record->timestamp = Carbon::parse($record->timestamp)->format('U'); 94 | $record->backing_data = BackingData::unserialize($record->backing_data); 95 | 96 | return $record; 97 | }); 98 | } 99 | 100 | public function has(string $path): bool 101 | { 102 | return $this->newQuery() 103 | ->where('path', '=', $path) 104 | ->exists(); 105 | } 106 | 107 | public function setVisibility(string $path, string $visibility) 108 | { 109 | $this->newQuery() 110 | ->where('path', '=', $path) 111 | ->update(['visibility' => $visibility]); 112 | 113 | return $this; 114 | } 115 | 116 | public function record(Metadata $metadata): Metadata 117 | { 118 | $updatePayload = $metadata->serialize(); 119 | 120 | $updates = Arr::where(array_keys($updatePayload), fn ($a) => $a !== 'path'); 121 | 122 | $updatePayload['timestamp'] = Carbon::parse($updatePayload['timestamp'])->toDateTimeString(); 123 | $updatePayload['disk'] = $this->storageId; 124 | $updatePayload['id'] = $updatePayload['id'] ?? Str::uuid(); 125 | 126 | $this->newQuery() 127 | ->upsert($updatePayload, ['id'], $updates); 128 | 129 | return $metadata; 130 | } 131 | 132 | public function delete(string $path) 133 | { 134 | $this->newQuery() 135 | ->where('path', '=', $path) 136 | ->delete(); 137 | 138 | return $this; 139 | } 140 | 141 | public function setBackingData(string $path, BackingData $backingData) 142 | { 143 | $this->newQuery() 144 | ->where('path', '=', $path) 145 | ->update(['backing_data' => $backingData->toJson()]); 146 | 147 | return $this; 148 | } 149 | 150 | public function rename(string $oldPath, string $newPath) 151 | { 152 | $this->newQuery() 153 | ->where('path', '=', $oldPath) 154 | ->update(['path' => $newPath]); 155 | 156 | return $this; 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Laravel Filer 2 | 3 | This project was started to scratch my itch on our growing Laravel site: 4 | 5 | * **Metadata for all files is stored in a local repository** - Supported backing systems are `json`, `database`, and 6 | `memory` (for testing). This is designed to speed up certain operations which normally call out to the remote 7 | filesystems. 8 | * **Handles fallback to the original disk** - If you provide an "original disk" list, this adapter will attempt to 9 | import data from those original disks into the metadata repository. 10 | * **Pluggable Strategies** - While the current version ships with a single strategy, you can replace the `Basic` 11 | implementation which allows for 1 + async, mirror, or other interactions with backing storage adapters. 12 | * **Manage data + files** - Coming soon: query and manage the metadata to do things like: 13 | * Find files stored on a single service and mirror them 14 | * Migrate files between stores while still maintaining continuity 15 | * **Abstract data from metadata** - Planning at some point on allow things like deduplication and copy-on-write to make 16 | copies, renames, and deletions work better. 17 | 18 | # Getting Started 19 | 20 | To get started, require the project. 21 | 22 | Laravel 7, 8 (Flysystem V1): 23 | 24 | composer require nvahalik/laravel-filer@^1 25 | 26 | Laravel 9 (Flysystem V3): 27 | 28 | composer require nvahalik/laravel-filer@dev-laravel-9 29 | 30 | Once that's done, you'll need to edit the filer config file and then update your filesystem configuration. 31 | 32 | # Config File 33 | 34 | By default, the metadata is stored in a JSON file. You can edit `config/filer.php` to change the default storage 35 | mechanism from `json` to `database` or `memory`. Note that memory is really a null adapter. The JSON adapter wraps 36 | memory and just serializes and saves it after each operation. 37 | 38 | # Filesystem Configuration 39 | 40 | The configuration is very similar to other disks: 41 | 42 | 'example' => [ 43 | 'driver' => 'filer', 44 | 'original_disks' => [ 45 | 'test:s3-original', 46 | ], 47 | 'id' => 'test', 48 | 'disk_strategy' => 'basic', 49 | 'backing_disks' => [ 50 | 'test:s3-new', 51 | 'test:s3-original', 52 | ], 53 | 'visibility' => 'private', 54 | ], 55 | 56 | The `original_disks` is an option if you are migrating from an existing disk or disks to the filer system. Effectively, 57 | this is a fallback so that files which are not found in the local metadata store will be searched for 58 | in `original_disks`. If they are found, their metadata will be imported. If not, the file will be treated as missing. 59 | We'll cover doing mass importing of metadata later on. 60 | 61 | **Note: that this will slow the filesystem down until the cache is filled. Once the cache is loaded, you can remove 62 | these original_disks and those extra operations looking for files will be eliminated.** 63 | 64 | **Note 2: files which are truly missing do not get cached. Therefore, if a file is missing, and you repeatedly attempt 65 | to assert its existence, it will search over and over again. This could be improved by caching the results or likewise 66 | having some sort of `missing-files` cache.** 67 | 68 | `id` is just an internal ID for the metadata store. File duplications are not allowed within the metadata of a single 69 | `id`, for example, but would be allowed for different `id`s. 70 | 71 | `disk_strategy` has only a single option currently: `'basic'` but will be pluggable to allow for different strategies to 72 | be added and used. The `basic` strategy simply writes to the first available disk from the list of 73 | provided `backing_disks`. 74 | 75 | `backing_disks` allows you to define multiple flysystem disks to use. Want to use multiple S3-compatible adapters? You 76 | can. Note that for the basic adapter, the order of the disks determines the order in which they are tried. 77 | 78 | ## A couple of examples 79 | 80 | Given the configuration above, if the following code is run: 81 | 82 | ``` 83 | Storage::disk('example')->has('does/not/exist.txt'); 84 | ``` 85 | 86 | 1. The existing metadata repo will be searched. 87 | 2. Then, the single 'original_disks' will be searched. 88 | 3. Finally, the operation will fail. 89 | 90 | ``` 91 | Storage::disk('example')->put('does/not/exist.txt', 'content'); 92 | ``` 93 | 94 | 1. The existing metadata repo will be searched. 95 | 2. Then, the single 'original_disks' will be searched. 96 | 3. Then, a write will be attempted on `test:s3-new`. 97 | 4. If that fails, then a write will be attempted on `test:s3-original`. 98 | 5. If any of the writes succeeds, that adapter's backing information will be returned and the entries metadata updated. 99 | 6. If any of them fails, then false will be returned and the operation will have failed. 100 | 101 | # Importing Metadata 102 | 103 | If you already have a ton of files on S3, you can use the `filer:import-s3-metadata` command to import that data into 104 | your metadata repository: 105 | 106 | ```bash 107 | # Grab the existing contents. 108 | s3cmd ls s3://bucket-name -rl > s3output.txt 109 | 110 | # Import that data into the "example" storageId. 111 | php artisan filer:import-s3-metadata example s3output.txt 112 | ``` 113 | 114 | The importer uses `File::lines()` to load its data, and therefore should not consume a lot of memory. Additionally, it 115 | will look at the bucket name in the URL which is present in the output and attempt to find that within your existing 116 | filesystems config. 117 | 118 | ## Visibility 119 | 120 | By default, it will grab this from the filesystem configuration. If none is found nor provided with `--visibility`, it 121 | will default to `private`. 122 | 123 | ## Filename stripping 124 | 125 | You can strip a string from the filenames by specifying the `--strip` option. 126 | 127 | ## Disk 128 | 129 | If you need to specify the disk directly or want to otherwise override it, just pass it in with `--disk`. This is not 130 | checked, so don't mess it up. 131 | 132 | ## Example 133 | 134 | ```bash 135 | php artisan filer:import-s3-metadata example s3output.txt --disk=some-disk --visibility=public --strip=prefix-dir/ 136 | ``` 137 | 138 | The above command would strip `prefix-dir/` from the imported URLs, set their visibility to public, and mark their 139 | default backing-disk to `some-disk`. 140 | -------------------------------------------------------------------------------- /tests/Metadata/MetadataBaseTestCase.php: -------------------------------------------------------------------------------- 1 | assertNull($this->repository->getMetadata('does/not/exist.txt')); 20 | } 21 | 22 | public function test_it_returns_false_if_size_does_not_exist() 23 | { 24 | $this->assertFalse($this->repository->getSize('does/not/exist.txt')); 25 | } 26 | 27 | public function test_it_returns_false_if_timestamp_does_not_exist() 28 | { 29 | $this->assertFalse($this->repository->getTimestamp('does/not/exist.txt')); 30 | } 31 | 32 | public function test_it_returns_false_if_visibility_does_not_exist() 33 | { 34 | $this->assertFalse($this->repository->getVisibility('does/not/exist.txt')); 35 | } 36 | 37 | public function test_it_returns_false_if_mimetype_does_not_exist() 38 | { 39 | $this->assertFalse($this->repository->getMimetype('does/not/exist.txt')); 40 | } 41 | 42 | public function test_it_stores_metadata() 43 | { 44 | $this->repository->record(Metadata::generate('example.txt', 'This is a test')); 45 | 46 | $this->assertNotFalse($this->repository->getMetadata('example.txt')); 47 | } 48 | 49 | public function test_it_returns_data_if_metadata_does_exists() 50 | { 51 | $this->repository->record(Metadata::generate('example.txt', 'This is a test')); 52 | 53 | $this->assertNotFalse($this->repository->getMetadata('example.txt')); 54 | } 55 | 56 | public function test_it_returns_false_if_size_exists() 57 | { 58 | $this->repository->record(Metadata::generate('example.txt', 'This is a test')); 59 | 60 | $this->assertEquals(14, $this->repository->getSize('example.txt')); 61 | } 62 | 63 | public function test_it_returns_false_if_timestamp_exists() 64 | { 65 | $this->repository->record(Metadata::generate('example.txt', 'This is a test')); 66 | 67 | $this->assertNotFalse($this->repository->getTimestamp('example.txt')); 68 | } 69 | 70 | public function test_it_returns_false_if_visibility_exists() 71 | { 72 | $this->repository->record(Metadata::generate('example.txt', 'This is a test')); 73 | 74 | $this->assertEquals('private', $this->repository->getVisibility('example.txt')); 75 | } 76 | 77 | public function test_it_returns_false_if_mimetype_exists() 78 | { 79 | $this->repository->record(Metadata::generate('example.txt', 'This is a test')); 80 | 81 | $this->assertEquals('text/plain', $this->repository->getMimetype('example.txt')); 82 | } 83 | 84 | public function test_it_renames_a_file() 85 | { 86 | $this->repository->record(Metadata::generate('example.txt', 'This is a test')); 87 | 88 | $this->assertTrue($this->repository->has('example.txt')); 89 | $this->assertFalse($this->repository->has('example2.txt')); 90 | 91 | $this->repository->rename('example.txt', 'example2.txt'); 92 | 93 | $this->assertFalse($this->repository->has('example.txt')); 94 | $this->assertTrue($this->repository->has('example2.txt')); 95 | } 96 | 97 | public function test_it_sets_the_visibility_of_a_file() 98 | { 99 | $this->repository->record(Metadata::generate('example.txt', 'This is a test')); 100 | 101 | $this->assertEquals('private', $this->repository->getVisibility('example.txt')); 102 | 103 | $this->repository->setVisibility('example.txt', 'public'); 104 | 105 | $this->assertEquals('public', $this->repository->getVisibility('example.txt')); 106 | } 107 | 108 | public function test_it_lists_a_directory() 109 | { 110 | $contents = $this->repository->listContents(); 111 | 112 | $this->assertCount(0, $contents); 113 | 114 | $this->repository->record(Metadata::generate('example.txt', 'abc')); 115 | $this->repository->record(Metadata::generate('abc/example.txt', 'abc')); 116 | $this->repository->record(Metadata::generate('abc/example2.txt', 'abc')); 117 | $this->repository->record(Metadata::generate('abc/def/example.txt', 'abc')); 118 | $this->repository->record(Metadata::generate('abc/def/example2.txt', 'abc')); 119 | $this->repository->record(Metadata::generate('def/example.txt', 'abc')); 120 | $this->repository->record(Metadata::generate('def/example2.txt', 'abc')); 121 | $this->repository->record(Metadata::generate('def/abc/example.txt', 'abc')); 122 | $this->repository->record(Metadata::generate('def/abc/example2.txt', 'abc')); 123 | 124 | $contents = $this->repository->listContents(); 125 | 126 | $this->assertCount(1, $contents); 127 | 128 | $contents = $this->repository->listContents('', true); 129 | 130 | $this->assertCount(9, $contents); 131 | 132 | $contents = $this->repository->listContents('abc/'); 133 | 134 | $this->assertCount(2, $contents); 135 | 136 | $contents = $this->repository->listContents('abc/', true); 137 | 138 | $this->assertCount(4, $contents); 139 | } 140 | 141 | public function test_it_deletes_a_file() 142 | { 143 | $this->repository->record(Metadata::generate('example.txt', 'This is a test')); 144 | 145 | $this->assertTrue($this->repository->has('example.txt')); 146 | 147 | $this->repository->delete('example.txt'); 148 | 149 | $this->assertFalse($this->repository->has('example.txt')); 150 | } 151 | 152 | public function test_it_adds_backing_data() 153 | { 154 | $this->repository->record(Metadata::generate('example.txt', 'This is a test')); 155 | 156 | $md = $this->repository->getMetadata('example.txt'); 157 | 158 | $this->assertIsObject($md->backingData); 159 | $this->assertCount(0, $md->backingData->toArray()); 160 | 161 | $this->repository->setBackingData('example.txt', BackingData::diskAndPath('temp', 'example.txt')); 162 | 163 | $md = $this->repository->getMetadata('example.txt'); 164 | 165 | $this->assertIsObject($md->backingData); 166 | $this->assertCount(1, $md->backingData->toArray()); 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /src/Console/Commands/ImportMetadata.php: -------------------------------------------------------------------------------- 1 | repository = app(MetadataRepository::class); 55 | 56 | $filename = $this->ensureFile($this->argument('file')); 57 | $storageId = $this->argument('storageId'); 58 | $mode = $this->option('mode'); 59 | 60 | if (! $filename) { 61 | return 0; 62 | } 63 | 64 | if ($this->option('progress')) { 65 | $bar = $this->output->createProgressBar(); 66 | $bar->setFormat('verbose_nomax'); 67 | $bar->start(); 68 | } 69 | 70 | $this->repository->setStorageId($storageId); 71 | 72 | $this->diskName = $this->option('disk'); 73 | $this->defaultVisibility = $this->option('visibility'); 74 | $this->stripFromFilename = $this->option('strip'); 75 | 76 | $entries = File::lines($filename) 77 | ->map(fn ($line) => Arr::flatten($this->parseLine($line))) 78 | ->filter() 79 | ->filter(fn ($line) => $line[1] !== '0') 80 | ->map(fn ($parsed) => $this->generateMetadata($parsed)); 81 | 82 | /** @var Metadata $entry */ 83 | foreach ($entries as $entry) { 84 | if ($this->repository->has($entry->path)) { 85 | if ($mode === 'ignore') { 86 | continue; 87 | } elseif ($mode === 'overwrite') { 88 | $existingEntry = $this->repository->getMetadata($entry->path); 89 | $existingEntry->setBackingData($entry->backingData); 90 | $this->repository->record($existingEntry); 91 | } elseif ($mode === 'append') { 92 | $existingEntry = $this->repository->getMetadata($entry->path); 93 | $this->appendBackingData($existingEntry, $entry); 94 | $this->repository->record($existingEntry); 95 | } 96 | } else { 97 | $this->repository->record($entry); 98 | } 99 | 100 | $entry = null; 101 | 102 | if ($bar) { 103 | $bar->advance(); 104 | } 105 | } 106 | 107 | if ($bar) { 108 | $bar->finish(); 109 | } 110 | 111 | return 0; 112 | } 113 | 114 | private function ensureFile(?string $argument) 115 | { 116 | if (file_exists($argument) && is_readable($argument)) { 117 | return $argument; 118 | } 119 | 120 | $this->warn('Unable to find or read file: '.$argument); 121 | } 122 | 123 | private function parseLine($line) 124 | { 125 | $matches = []; 126 | preg_match_all('#^(\d{4}-\d{2}-\d{2} \d{2}:\d{2})\s+(\d+)\s+([\w-]+)\s+(\w+)\s+(.+)$#', $line, $matches); 127 | // Date- YYYY-MM-DD Time HH:MM ^Size^ ^Etag^ ^Vis^ ^Path^ 128 | 129 | return array_slice($matches, 1); 130 | } 131 | 132 | private function generateMetadata($parsed) 133 | { 134 | $path = implode('/', array_slice(explode('/', $parsed[4]), 3)); 135 | 136 | if ($this->stripFromFilename) { 137 | $path = str_replace($this->stripFromFilename, '', $path); 138 | } 139 | $disk = $this->diskName ?? $this->detectOriginalDisk($parsed[4]); 140 | 141 | return new Metadata( 142 | $path, 143 | MimeType::detectByFilename($path), 144 | (int) $parsed[1], 145 | $parsed[2], 146 | Carbon::parse($parsed[0])->format('U'), 147 | $this->getVisibility($parsed[4]), 148 | BackingData::diskAndPath($disk, $path) 149 | ); 150 | } 151 | 152 | private function detectOriginalDisk($path) 153 | { 154 | $bucketName = array_slice(explode('/', $path), 2, 1)[0]; 155 | 156 | if (! isset($this->buckets[$bucketName])) { 157 | $disks = config('filesystems.disks'); 158 | 159 | foreach ($disks as $name => $config) { 160 | if ($config['driver'] === 's3' && $config['bucket'] === $bucketName) { 161 | $this->buckets[$bucketName] = $name; 162 | } 163 | } 164 | } 165 | 166 | return $this->buckets[$bucketName]; 167 | } 168 | 169 | private function getVisibility($path) 170 | { 171 | if ($this->defaultVisibility) { 172 | return $this->defaultVisibility; 173 | } 174 | 175 | $diskName = $this->detectOriginalDisk($path); 176 | 177 | return config('filesystems.disks')[$diskName]['visibility'] ?? 'private'; 178 | } 179 | 180 | private function appendBackingData(Metadata $existingEntry, Metadata $entry) 181 | { 182 | foreach ($entry->backingData->disks() as $disk) { 183 | $existingEntry->backingData->addDisk($disk, $entry->backingData->getDisk($disk)); 184 | } 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /src/Metadata.php: -------------------------------------------------------------------------------- 1 | format('U'), 45 | $array['visibility'] ?? 'private', 46 | BackingData::unserialize($array['backing_data'] ?? []), 47 | $array['id'] ?? null 48 | ); 49 | } 50 | 51 | public static function import($array) 52 | { 53 | return new static( 54 | $array['path'], 55 | $array['mimetype'] ?? 'application/octet-stream', 56 | $array['size'], 57 | $array['etag'] ?? '', 58 | $array['timestamp'] ?? $array['updated_at'] ?? $array['created_at'], 59 | $array['visibility'] ?? 'private', 60 | $array['id'] ?? null 61 | ); 62 | } 63 | 64 | /** 65 | * @param resource|string $contents 66 | * @return false|int 67 | */ 68 | public static function getSize($contents) 69 | { 70 | if (is_resource($contents)) { 71 | fseek($contents, 0, SEEK_END); 72 | $size = ftell($contents); 73 | rewind($contents); 74 | } else { 75 | $size = Util::contentSize($contents); 76 | } 77 | 78 | return $size; 79 | } 80 | 81 | /** 82 | * @param string $path 83 | * @return Metadata 84 | */ 85 | public function setPath(string $path): self 86 | { 87 | $this->path = $path; 88 | 89 | return $this; 90 | } 91 | 92 | public function setTimestamp(int $timestamp): self 93 | { 94 | $this->timestamp = $timestamp; 95 | 96 | return $this; 97 | } 98 | 99 | /** 100 | * @param mixed|string $filename 101 | * @return Metadata 102 | */ 103 | public function setFilename(string $filename): self 104 | { 105 | $this->filename = $filename; 106 | 107 | return $this; 108 | } 109 | 110 | /** 111 | * @param string|null $mimetype 112 | * @return Metadata 113 | */ 114 | public function setMimetype(string $mimetype = null): self 115 | { 116 | $this->mimetype = $mimetype; 117 | 118 | return $this; 119 | } 120 | 121 | /** 122 | * @param string $visibility 123 | * @return Metadata 124 | */ 125 | public function setVisibility(string $visibility): self 126 | { 127 | $this->visibility = $visibility; 128 | 129 | return $this; 130 | } 131 | 132 | /** 133 | * @param int $size 134 | * @return Metadata 135 | */ 136 | public function setSize(int $size): self 137 | { 138 | $this->size = $size; 139 | 140 | return $this; 141 | } 142 | 143 | public function setId(string $id): self 144 | { 145 | $this->id = $id; 146 | 147 | return $this; 148 | } 149 | 150 | /** 151 | * @param BackingData $backingData 152 | * @return Metadata 153 | */ 154 | public function setBackingData(BackingData $backingData): self 155 | { 156 | $this->backingData = $backingData; 157 | 158 | return $this; 159 | } 160 | 161 | public int $size; 162 | 163 | public BackingData $backingData; 164 | 165 | public function __construct( 166 | string $path, 167 | string $mimetype = 'application/octet-stream', 168 | int $size = 0, 169 | string $etag = null, 170 | int $timestamp = null, 171 | string $visibility = null, 172 | BackingData $backingData = null, 173 | string $id = null 174 | ) { 175 | $this->path = $path; 176 | $this->mimetype = $mimetype; 177 | $this->size = $size; 178 | $this->etag = $etag; 179 | $this->backingData = $backingData ?? new BackingData(); 180 | $this->visibility = $visibility ?? 'private'; 181 | $this->created_at = $timestamp ?? time(); 182 | $this->updated_at = $timestamp ?? time(); 183 | $this->timestamp = $timestamp ?? time(); 184 | $this->id = $id; 185 | } 186 | 187 | public static function generateEtag($content) 188 | { 189 | if (is_resource($content)) { 190 | // Use incremental hashing to generate the MD5 sum without running out of memory for big files. 191 | $hash = hash_init('md5'); 192 | $location = ftell($content); 193 | rewind($content); 194 | hash_update_stream($hash, $content); 195 | fseek($content, $location); 196 | 197 | return hash_final($hash); 198 | } 199 | 200 | return md5($content); 201 | } 202 | 203 | public static function generate($path, $contents): Metadata 204 | { 205 | $mimetype = Util::guessMimeType($path, $contents); 206 | $size = self::getSize($contents); 207 | $etag = static::generateEtag($contents); 208 | 209 | return new static( 210 | $path, 211 | $mimetype, 212 | $size, 213 | $etag, 214 | ); 215 | } 216 | 217 | public function toArray(): array 218 | { 219 | return [ 220 | 'id' => $this->id, 221 | 'path' => $this->path, 222 | 'etag' => $this->etag, 223 | 'mimetype' => $this->mimetype, 224 | 'visibility' => $this->visibility, 225 | 'size' => $this->size, 226 | 'backing_data' => $this->backingData, 227 | 'timestamp' => $this->timestamp, 228 | ]; 229 | } 230 | 231 | public function serialize() 232 | { 233 | $data = $this->toArray(); 234 | 235 | $data['backing_data'] = $data['backing_data']->toJson(); 236 | 237 | return $data; 238 | } 239 | 240 | public function updateContents($contents) 241 | { 242 | $this->mimetype = Util::guessMimeType($this->path, $contents); 243 | $this->size = $this->getSize($contents); 244 | $this->etag = $this->generateEtag($contents); 245 | $this->updated_at = time(); 246 | 247 | return $this; 248 | } 249 | 250 | public function toJson($options = 0) 251 | { 252 | // TODO: Implement toJson() method. 253 | } 254 | } 255 | -------------------------------------------------------------------------------- /src/AdapterStrategy/Basic.php: -------------------------------------------------------------------------------- 1 | backingAdapters + $this->config['allow_new_files_on_original_disks'] ? 17 | $this->originalDisks : []; 18 | } 19 | 20 | public function getReadAdapters(): array 21 | { 22 | return array_merge($this->backingAdapters, $this->originalDisks); 23 | } 24 | 25 | public function getOriginalDiskMetadata($path) 26 | { 27 | $metadata = []; 28 | 29 | foreach ($this->originalDisks as $name => $adapter) { 30 | try { 31 | $adapterMetadata = $adapter->getMetadata($path); 32 | if (isset($adapterMetadata['etag'])) { 33 | $adapterMetadata['etag'] = rtrim(ltrim($adapterMetadata['etag'], '"'), '"'); 34 | } 35 | $metadata[$name] = $adapterMetadata; 36 | } catch (\Exception $exception) { 37 | // Ignore any exceptions, at least for now. 38 | } 39 | } 40 | 41 | return $metadata; 42 | } 43 | 44 | public function has($path) 45 | { 46 | foreach ($this->originalDisks as $name => $adapter) { 47 | if ($adapter->has($path)) { 48 | return true; 49 | } 50 | } 51 | 52 | return false; 53 | } 54 | 55 | public function writeStream($path, $stream, Config $config): ?BackingData 56 | { 57 | /** 58 | * @var string $diskId 59 | * @var FilesystemAdapter $backingAdapter 60 | */ 61 | foreach ($this->backingAdapters as $diskId => $backingAdapter) { 62 | try { 63 | $originalConfig = $backingAdapter->getConfig(); 64 | $originalConfig->setFallback($config); 65 | 66 | if ($backingAdapter->getAdapter()->writeStream($path, $stream, $originalConfig)) { 67 | return BackingData::diskAndPath($diskId, $path); 68 | } 69 | } catch (FileExistsException $e) { 70 | // Ignore. We'll try the next one. 71 | } 72 | } 73 | 74 | return null; 75 | } 76 | 77 | public function write($path, $contents, Config $config): ?BackingData 78 | { 79 | /** 80 | * @var string $diskId 81 | * @var FilesystemAdapter $backingAdapter 82 | */ 83 | foreach ($this->backingAdapters as $diskId => $backingAdapter) { 84 | try { 85 | $originalConfig = $backingAdapter->getConfig(); 86 | $originalConfig->setFallback($config); 87 | 88 | if ($backingAdapter->getAdapter()->write($path, $contents, $originalConfig)) { 89 | return BackingData::diskAndPath($diskId, $path); 90 | } 91 | } catch (FileExistsException $e) { 92 | // Ignore. We'll try the next one. 93 | } 94 | } 95 | 96 | return null; 97 | } 98 | 99 | public function readStream($backingData) 100 | { 101 | foreach ($this->getMatchingReadAdapters($backingData) as $id => $adapter) { 102 | try { 103 | if ($object = $adapter->getAdapter()->readStream($this->readAdapterPath($id, $backingData))) { 104 | return $object['stream']; 105 | } 106 | } catch (\Exception $e) { 107 | } 108 | } 109 | 110 | return false; 111 | } 112 | 113 | /** 114 | * @param BackingData $backingData An array of backing data. 115 | * @return false | string 116 | */ 117 | public function read(BackingData $backingData) 118 | { 119 | foreach ($this->getMatchingReadAdapters($backingData) as $id => $adapter) { 120 | try { 121 | if ($response = $adapter->getAdapter()->read($this->readAdapterPath($id, $backingData))) { 122 | return $response['contents']; 123 | } 124 | } catch (\Exception $e) { 125 | } 126 | } 127 | 128 | return false; 129 | } 130 | 131 | private function getMatchingReadAdapters(BackingData $backingData): array 132 | { 133 | return array_filter($this->getReadAdapters(), static function ($name) use ($backingData) { 134 | return in_array($name, $backingData->disks(), true); 135 | }, ARRAY_FILTER_USE_KEY); 136 | } 137 | 138 | public function delete(string $path, $backingData) 139 | { 140 | foreach ($this->getMatchingReadAdapters($backingData) as $id => $adapter) { 141 | try { 142 | return $adapter->getAdapter()->delete($this->readAdapterPath($id, $backingData)); 143 | } catch (\Exception $e) { 144 | throw new BackingAdapterException("Unable to delete ($path) on disk ($id)."); 145 | } 146 | } 147 | 148 | return true; 149 | } 150 | 151 | private function readAdapterPath(string $id, BackingData $backingData) 152 | { 153 | return $backingData->getDisk($id)['path']; 154 | } 155 | 156 | public function update($path, $contents, Config $config, $backingData): BackingData 157 | { 158 | foreach ($this->getMatchingReadAdapters($backingData) as $id => $adapter) { 159 | try { 160 | $originalConfig = $adapter->getConfig(); 161 | $originalConfig->setFallback($config); 162 | 163 | $adapter->getAdapter()->update($this->readAdapterPath($id, $backingData), $contents, $originalConfig); 164 | } catch (\Exception $e) { 165 | throw new BackingAdapterException('Unable to write to remote adapter: '.$id.' path ('.$path.')'); 166 | } 167 | } 168 | 169 | return $backingData; 170 | } 171 | 172 | public function updateStream($path, $stream, Config $config, $backingData): BackingData 173 | { 174 | foreach ($this->getMatchingReadAdapters($backingData) as $id => $adapter) { 175 | try { 176 | $originalConfig = $adapter->getConfig(); 177 | $originalConfig->setFallback($config); 178 | 179 | $adapter->getAdapter()->updateStream($this->readAdapterPath($id, $backingData), $stream, $originalConfig); 180 | } catch (\Exception $e) { 181 | throw new BackingAdapterException('Unable to write to remote adapter: '.$id.' path ('.$path.')'); 182 | } 183 | } 184 | 185 | return $backingData; 186 | } 187 | 188 | public function copy(BackingData $source, string $destination): ?BackingData 189 | { 190 | try { 191 | $stream = $this->readStream($source); 192 | 193 | return $this->writeStream($destination, $stream); 194 | } catch (\Exception $e) { 195 | return null; 196 | } 197 | } 198 | 199 | public function getDisk(string $disk) 200 | { 201 | $adapters = $this->getReadAdapters(); 202 | 203 | if (! array_key_exists($disk, $adapters)) { 204 | throw new BackingAdapterException(sprintf('The backing adapter (%s) does not exist on the adapter.', 205 | $disk)); 206 | } 207 | 208 | return $adapters[$disk]; 209 | } 210 | } 211 | -------------------------------------------------------------------------------- /src/Flysystem/FilerAdapter.php: -------------------------------------------------------------------------------- 1 | config = $config; 29 | $this->storageMetadata = $storageMetadata; 30 | $this->adapterManager = $adapterManager; 31 | } 32 | 33 | public function getStorageMetadata() 34 | { 35 | return $this->storageMetadata; 36 | } 37 | 38 | /** 39 | * @inheritDoc 40 | */ 41 | public function write($path, $contents, Config $config, $isStream = false) 42 | { 43 | // Create the initial entry. 44 | $backingData = $isStream 45 | ? $this->adapterManager->writeStream($path, $contents, $config) 46 | : $this->adapterManager->write($path, $contents, $config); 47 | 48 | if ($isStream) { 49 | Util::rewindStream($contents); 50 | } 51 | 52 | // Write the data out somewhere. 53 | if ($backingData) { 54 | $metadata = Metadata::generate($path, $contents); 55 | $metadata->setBackingData($backingData); 56 | 57 | // Update the entry to ensure that we've recorded what actually happened with the data. 58 | $this->storageMetadata->record($metadata); 59 | 60 | return $this->storageMetadata->getMetadata($path); 61 | } 62 | 63 | return false; 64 | } 65 | 66 | /** 67 | * @inheritDoc 68 | */ 69 | public function writeStream($path, $resource, Config $config) 70 | { 71 | return $this->write($path, $resource, $config, true); 72 | } 73 | 74 | /** 75 | * @inheritDoc 76 | */ 77 | public function update($path, $contents, Config $config, $isStream = false) 78 | { 79 | // Figure out where the data actually is... 80 | $metadata = $this->pathMetadata($path); 81 | 82 | // Update it. 83 | try { 84 | $backingData = $isStream 85 | ? $this->adapterManager->updateStream($path, $contents, $config, $metadata->backingData) 86 | : $this->adapterManager->update($path, $contents, $config, $metadata->backingData); 87 | 88 | if ($isStream) { 89 | Util::rewindStream($contents); 90 | } 91 | 92 | $metadata->updateContents($contents) 93 | ->setBackingData($backingData); 94 | 95 | // Update metadata with new size and timestamp? 96 | $this->storageMetadata->record($metadata); 97 | 98 | return $metadata->toArray(); 99 | } catch (BackingAdapterException $e) { 100 | return false; 101 | } 102 | } 103 | 104 | /** 105 | * @inheritDoc 106 | */ 107 | public function updateStream($path, $resource, Config $config) 108 | { 109 | return $this->update($path, $resource, $config, true); 110 | } 111 | 112 | /** 113 | * @inheritDoc 114 | */ 115 | public function rename($originalPath, $newPath) 116 | { 117 | // We don't really need to do anything. The on-disk doesn't have to change. 118 | $this->storageMetadata->rename($originalPath, $newPath); 119 | 120 | return true; 121 | } 122 | 123 | /** 124 | * @inheritDoc 125 | */ 126 | public function copy($originalPath, $newPath) 127 | { 128 | // Grab a copy of the metadata and save it with the new path information. 129 | $originalMetadata = $this->pathMetadata($originalPath); 130 | 131 | if (! $originalMetadata) { 132 | return false; 133 | } 134 | 135 | try { 136 | // Copy the file. 137 | $copyBackingData = $this->adapterManager->copy($originalMetadata->backingData, $newPath); 138 | 139 | if (! $copyBackingData) { 140 | return false; 141 | } 142 | 143 | $copyMetadata = (clone $originalMetadata) 144 | ->setPath($newPath) 145 | ->setBackingData($copyBackingData); 146 | 147 | $this->storageMetadata->record($copyMetadata); 148 | } catch (\Exception $e) { 149 | return false; 150 | } 151 | 152 | return true; 153 | } 154 | 155 | /** 156 | * @inheritDoc 157 | */ 158 | public function delete($path) 159 | { 160 | // Grab a copy of the metadata and save it with the new path information. 161 | $metadata = $this->pathMetadata($path); 162 | 163 | try { 164 | // Copy the file. 165 | $this->adapterManager->delete($path, $metadata->backingData); 166 | $this->storageMetadata->delete($path); 167 | } catch (BackingAdapterException $e) { 168 | return false; 169 | } 170 | 171 | return true; 172 | } 173 | 174 | /** 175 | * @inheritDoc 176 | */ 177 | public function deleteDir($dirname) 178 | { 179 | // Figure out what files are under the directory, if any, and then delete them. 180 | // This is gonna be tough. 181 | 182 | return true; 183 | } 184 | 185 | /** 186 | * @inheritDoc 187 | */ 188 | public function createDir($dirname, Config $config) 189 | { 190 | return ['path' => $dirname, 'type' => 'dir']; 191 | } 192 | 193 | /** 194 | * @inheritDoc 195 | */ 196 | public function setVisibility($path, $visibility) 197 | { 198 | return $this->storageMetadata->setVisibility($path, $visibility); 199 | } 200 | 201 | /** 202 | * @inheritDoc 203 | */ 204 | public function has($path) 205 | { 206 | return $this->storageMetadata->has($path) || $this->hasOriginalDiskFile($path); 207 | } 208 | 209 | /** 210 | * @inheritDoc 211 | */ 212 | public function read($path) 213 | { 214 | // Get the metadata. Where is this file? 215 | $metadata = $this->pathMetadata($path); 216 | 217 | if ($contents = $this->adapterManager->read($metadata->backingData)) { 218 | return ['type' => 'file', 'path' => $path, 'contents' => $contents]; 219 | } 220 | 221 | return false; 222 | } 223 | 224 | /** 225 | * Grab the metadata from the store. If it isn't there, try it from the original disks, if they are set. 226 | * 227 | * @param $path 228 | * @return Metadata 229 | */ 230 | protected function pathMetadata($path): Metadata 231 | { 232 | // Get the metadata. Where is this file? 233 | $metadata = $this->storageMetadata->getMetadata($path); 234 | 235 | // We didn't find it in the metadata store. Is there an original disk? 236 | // If so, let's reach out to the disk and see if there is data there. 237 | if (! $metadata && $this->config->originalDisks) { 238 | $metadata = $this->migrateFromOriginalDisk($path); 239 | } 240 | 241 | return $metadata; 242 | } 243 | 244 | /** 245 | * @inheritDoc 246 | */ 247 | public function readStream($path) 248 | { 249 | // Get the metadata. Where is this file? 250 | $metadata = $this->pathMetadata($path); 251 | 252 | if ($stream = $this->adapterManager->readStream($metadata->backingData)) { 253 | return ['type' => 'file', 'path' => $path, 'stream' => $stream]; 254 | } 255 | 256 | return false; 257 | } 258 | 259 | /** 260 | * @inheritDoc 261 | */ 262 | public function listContents($directory = '', $recursive = false) 263 | { 264 | // We don't need to reach out to the storage provider because we have it all cached. 265 | return $this->storageMetadata->listContents($directory, $recursive); 266 | } 267 | 268 | /** 269 | * @inheritDoc 270 | */ 271 | public function getMetadata($path, $asObject = false) 272 | { 273 | // Convert our metadata to an array. 274 | $metadata = $this->pathMetadata($path); 275 | 276 | if ($metadata) { 277 | return $asObject ? $metadata : $metadata->toArray(); 278 | } 279 | 280 | return false; 281 | } 282 | 283 | /** 284 | * @inheritDoc 285 | */ 286 | public function getSize($path) 287 | { 288 | return $this->getMetadata($path); 289 | } 290 | 291 | /** 292 | * @inheritDoc 293 | */ 294 | public function getMimetype($path) 295 | { 296 | return $this->getMetadata($path); 297 | } 298 | 299 | /** 300 | * @inheritDoc 301 | */ 302 | public function getTimestamp($path) 303 | { 304 | return $this->getMetadata($path); 305 | } 306 | 307 | /** 308 | * @inheritDoc 309 | */ 310 | public function getVisibility($path) 311 | { 312 | return $this->getMetadata($path); 313 | } 314 | 315 | private function migrateFromOriginalDisk(string $path): ?Metadata 316 | { 317 | // Did we find it? 318 | $originalMetadata = $this->adapterManager->getOriginalDiskMetadata($path); 319 | 320 | if (count($originalMetadata) > 0) { 321 | $backingData = new BackingData(); 322 | 323 | foreach ($originalMetadata as $disk => $data) { 324 | $backingData->addDisk($disk, ['path' => $path]); 325 | } 326 | 327 | $metadata = Metadata::import(current($originalMetadata)); 328 | $metadata->setBackingData($backingData); 329 | $this->storageMetadata->record($metadata); 330 | 331 | return $metadata; 332 | } 333 | 334 | return null; 335 | } 336 | 337 | private function hasOriginalDiskFile(string $path) 338 | { 339 | if ($this->config->originalDisks) { 340 | return $this->adapterManager->has($path); 341 | } 342 | 343 | return false; 344 | } 345 | 346 | public function getTemporaryUrl(string $path, $expiration, array $options = []) 347 | { 348 | $data = $this->getBackingAdapter($path); 349 | $adapter = $data['adapter']; 350 | 351 | return $adapter->temporaryUrl($path, $expiration, $options); 352 | } 353 | 354 | public function getBackingAdapter(string $path) 355 | { 356 | $metadata = $this->pathMetadata($path)->backingData->toArray(); 357 | 358 | $disk = key($metadata); 359 | 360 | return [ 361 | 'disk' => $disk, 362 | 'adapter' => $this->adapterManager->getDisk($disk), 363 | 'metadata' => current($metadata), 364 | ]; 365 | } 366 | } 367 | --------------------------------------------------------------------------------