From 76ca4dfd78b9a6c65084fdb9e47aeffa13c9a5f6 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Sun, 27 Sep 2026 04:50:06 +0000 Subject: [PATCH 1/2] Add read-through filesystems with pooled cloud support Add Laravel's read-through disk with primary writes, fallback reads, optional promotion, deletion on both disks, fallback copy/move and visibility handling. Port the current upstream tests and documentation. Adapt Flysystem operations to pooled disk lifetimes. Streams retain their leases until closed; fallback copies release the source before borrowing the primary, allowing both sides to share a capacity-one pool. Materialize pooled and native cloud listings before returning them. Preserve native cloud stream options and range requests while applying the composite disk's failure policy once, including through Sentry decorators. Preserve outer and side prefixes for paths and URLs, honor composite URL callbacks, reject circular construction with coroutine-local state, and close owned resources on errors and cancellation. Upstream: https://github.com/laravel/framework/pull/61140 https://github.com/laravel/framework/pull/61155 https://github.com/laravel/framework/pull/61272 https://github.com/laravel/framework/pull/61375 Deletion follow-up: a833cea6ad444f64833d65f075b2a73e34b8872b Source: laravel/framework master cd6e81dff3ba7a4564ac88c3949698c728d20109 Docs: laravel/docs 13.x eff8739e9090c2a0216fefac8e33dacdd689f8f6 Validation: full parallel suite, full static analysis and formatting pass. After final corrections, the affected parallel suite also passes, including pooled and non-pooled cloud reads, cleanup, Sentry storage and generated facade checks. --- src/docs/filesystem.md | 20 +- src/filesystem/README.md | 2 + src/filesystem/src/AwsS3V3Adapter.php | 51 +- src/filesystem/src/ClientPooledFilesystem.php | 21 + .../InteractsWithPooledFilesystem.php | 21 + src/filesystem/src/FilesystemManager.php | 97 +++- .../src/FilesystemOperatorAdapter.php | 210 ++++++++ .../src/GoogleCloudStorageAdapter.php | 69 +-- src/filesystem/src/ReadThroughFilesystem.php | 129 +++++ .../src/ReadThroughFilesystemAdapter.php | 340 +++++++++++++ .../Features/Storage/SentryS3V3Adapter.php | 10 + src/support/src/Facades/Storage.php | 2 + tests/Filesystem/AwsS3V3AdapterTest.php | 127 ++++- .../Filesystem/ClientPooledFilesystemTest.php | 71 +++ tests/Filesystem/FilesystemManagerTest.php | 470 ++++++++++++++++++ tests/Filesystem/FilesystemPoolProxyTest.php | 23 + 16 files changed, 1613 insertions(+), 50 deletions(-) create mode 100644 src/filesystem/src/FilesystemOperatorAdapter.php create mode 100644 src/filesystem/src/ReadThroughFilesystem.php create mode 100644 src/filesystem/src/ReadThroughFilesystemAdapter.php diff --git a/src/docs/filesystem.md b/src/docs/filesystem.md index ac0dd29cc4..15b84066cb 100644 --- a/src/docs/filesystem.md +++ b/src/docs/filesystem.md @@ -6,7 +6,7 @@ - [The Public Disk](#the-public-disk) - [Driver Prerequisites](#driver-prerequisites) - [Driver Pools](#driver-pools) - - [Scoped and Read-Only Filesystems](#scoped-and-read-only-filesystems) + - [Scoped, Read-Only, and Read-Through Filesystems](#scoped-and-read-only-filesystems) - [Amazon S3 Compatible Filesystems](#amazon-s3-compatible-filesystems) - [Obtaining Disk Instances](#obtaining-disk-instances) - [On-Demand Disks](#on-demand-disks) @@ -288,7 +288,7 @@ $result = Storage::disk('s3')->withClient(function ($client) { S3 and Google Cloud Storage streams are read lazily by default, which keeps memory usage bounded and makes data available before the entire file has downloaded. This applies to `readStream()` and `readStreamRange()`; methods such as `get()` retain their normal behavior. Streaming requests close their HTTP connection after the read, so applications that open many small streams may prefer connection reuse and set the disk's `stream_reads` option to `false`. -### Scoped and Read-Only Filesystems +### Scoped, Read-Only, and Read-Through Filesystems Scoped disks allow you to define a filesystem where all paths are automatically prefixed with a given path prefix. @@ -360,6 +360,22 @@ Dynamic scoped filesystems fail closed when the resolved prefix is empty. Pass ` Failed writes follow the scoped disk's `throw` and `report` options. +Read-through disks allow you to migrate files between disks without downtime. When reading a file, Hypervel checks the primary disk first. If the file only exists on the fallback disk, Hypervel reads it from the fallback disk and copies it to the primary disk for future requests: + +```php +'assets' => [ + 'driver' => 'read-through', + 'primary' => 's3', + 'fallback' => 'legacy-s3', +], +``` + +New files and directory listings use the primary disk. File existence and metadata checks use either disk without copying files. Deletions remove files or directories from both disks, and visibility changes apply to the disk containing the file. The `primary` and `fallback` options may also contain inline disk configurations. + +Fallback reads promote files by default. Set `copy` to `false` to read fallback files without copying them. With promotion enabled, fallback stream reads finish copying the file before returning the stream. + +If promotion fails, the read still succeeds by default. Set `throw_on_promotion_failure` to `true` to treat promotion failures as read failures; set the disk's `throw` option to `true` to receive those failures as exceptions. + ### Amazon S3 Compatible Filesystems diff --git a/src/filesystem/README.md b/src/filesystem/README.md index cdabe25954..41110d010b 100644 --- a/src/filesystem/README.md +++ b/src/filesystem/README.md @@ -15,6 +15,8 @@ The configured disk name `ondemand` is reserved. Disks explicitly set under that Hypervel pools S3 and Google Cloud Storage SDK clients rather than complete disk adapters. Disks with equivalent client construction config share the expensive client pool while retaining their own bucket, root, visibility, and callback behavior. Pooled disks expose raw internals only through borrow-scoped `withClient()`, `withDriver()`, and `withAdapter()` callbacks. +On read-through disks, raw `listContents()` results from pooled, S3 and Google Cloud Storage sides are fully loaded instead of streamed. + Filesystem construction differs from Laravel in how it carries logical disk identity. `callCustomCreator()` accepts the logical disk name as an optional second parameter, so existing one-argument calls remain valid while overrides must adopt the parameter. The public `build()` method uses a logical-name-aware construction path rather than the protected `resolve()` method because anonymous builds must pass a null name to creators; `resolve()` remains the configured-disk seam. `createScopedDriver()` resolves its prepared descriptor directly rather than through `build()`, so override `createScopedDriver()` for scoped construction customization. Customize on-demand construction through `Storage::extend()` or the public driver creator methods. Creator callbacks may accept the nullable name as a third argument after the application and configuration, while existing two-argument callbacks remain valid. Hypervel carries the name through scoped reconstruction and whole-driver pool fingerprints. A matching explicit fingerprint declares that every construction detail is equivalent, including serving-route ownership. Hypervel registers signed file-serving routes for any configured disk whose `serve` option is exactly `true`, while Laravel limits these routes to local disks and accepts truthy values. Every served disk must use a unique URL or application boot will fail. Custom drivers that opt in must provide the filesystem response methods used by these routes. diff --git a/src/filesystem/src/AwsS3V3Adapter.php b/src/filesystem/src/AwsS3V3Adapter.php index 03a0f71960..11a279d5db 100644 --- a/src/filesystem/src/AwsS3V3Adapter.php +++ b/src/filesystem/src/AwsS3V3Adapter.php @@ -135,7 +135,14 @@ public function temporaryUploadUrl(string $path, DateTimeInterface $expiration, */ public function readStream(string $path): mixed { - return $this->readStreamWithOptions($path); + try { + return $this->readStreamWithOptions($path); + } catch (UnableToReadFile $exception) { + throw_if($this->throwsExceptions(), $exception); + $this->report($exception); + + return null; + } } /** @@ -151,15 +158,35 @@ public function readStreamRange(string $path, ?int $start, ?int $end): mixed return $this->readStream($path); } - return $this->readStreamWithOptions($path, [ - 'Range' => "bytes={$start}-{$end}", - ]); + try { + return $this->readStreamRangeOrFail($path, $start, $end); + } catch (UnableToReadFile $exception) { + throw_if($this->throwsExceptions(), $exception); + $this->report($exception); + + return null; + } + } + + /** + * Open a whole-object or ranged stream without applying the disk's failure policy. + * + * @return resource + */ + public function readStreamRangeOrFail(string $path, ?int $start = null, ?int $end = null): mixed + { + [$start, $end] = $this->normalizeStreamRange($start, $end); + + return $this->readStreamWithOptions( + $path, + $start === null && $end === null ? [] : ['Range' => "bytes={$start}-{$end}"], + ); } /** * Read an object while preserving configured options and operation-owned keys. * - * @return null|resource + * @return resource */ private function readStreamWithOptions(string $path, array $operationOptions = []): mixed { @@ -181,24 +208,14 @@ private function readStreamWithOptions(string $path, array $operationOptions = [ } catch (CanceledException $exception) { throw $exception; } catch (Throwable $exception) { - $exception = UnableToReadFile::fromLocation($path, $exception->getMessage(), $exception); - - throw_if($this->throwsExceptions(), $exception); - $this->report($exception); - - return null; + throw UnableToReadFile::fromLocation($path, $exception->getMessage(), $exception); } if (! is_resource($stream)) { - $exception = UnableToReadFile::fromLocation( + throw UnableToReadFile::fromLocation( $path, 'Downloaded object does not contain a file resource.', ); - - throw_if($this->throwsExceptions(), $exception); - $this->report($exception); - - return null; } return $stream; diff --git a/src/filesystem/src/ClientPooledFilesystem.php b/src/filesystem/src/ClientPooledFilesystem.php index d24ac330ee..7fb3359656 100644 --- a/src/filesystem/src/ClientPooledFilesystem.php +++ b/src/filesystem/src/ClientPooledFilesystem.php @@ -11,6 +11,7 @@ use Hypervel\Filesystem\Concerns\InteractsWithPooledFilesystem; use Hypervel\ObjectPool\Lease; use Hypervel\ObjectPool\PoolDefinition; +use League\Flysystem\FilesystemOperator; use RuntimeException; use Throwable; @@ -51,6 +52,26 @@ public function getPoolName(): string return $this->definition->identity; } + /** + * Get an operator that preserves native cloud reads and their client leases. + */ + public function getOperator(): FilesystemOperator + { + return new FilesystemOperatorAdapter( + $this->withDriver(...), + fn (string $path): mixed => $this->leasedStream(static function (FilesystemAdapter $stack) use ($path): mixed { + return $stack instanceof AwsS3V3Adapter || $stack instanceof GoogleCloudStorageAdapter + ? $stack->readStreamRangeOrFail($path) + : $stack->getDriver()->readStream($path); + }), + fn (string $path, ?int $start, ?int $end): mixed => $this->leasedStream( + static fn (FilesystemAdapter $stack): mixed => $stack instanceof AwsS3V3Adapter || $stack instanceof GoogleCloudStorageAdapter + ? $stack->readStreamRangeOrFail($path, $start, $end) + : null, + ), + ); + } + /** * Remove and close the current client pool. */ diff --git a/src/filesystem/src/Concerns/InteractsWithPooledFilesystem.php b/src/filesystem/src/Concerns/InteractsWithPooledFilesystem.php index 30f6795dc8..a59b0085e6 100644 --- a/src/filesystem/src/Concerns/InteractsWithPooledFilesystem.php +++ b/src/filesystem/src/Concerns/InteractsWithPooledFilesystem.php @@ -10,12 +10,14 @@ use Hypervel\Container\Container; use Hypervel\Contracts\Filesystem\Filesystem as FilesystemContract; use Hypervel\Filesystem\FileResponseBuilder; +use Hypervel\Filesystem\FilesystemOperatorAdapter; use Hypervel\Http\File; use Hypervel\Http\Request; use Hypervel\Http\UploadedFile; use Hypervel\Image\Image; use Hypervel\Image\ImageException; use Hypervel\Support\Traits\Conditionable; +use League\Flysystem\FilesystemOperator; use Psr\Http\Message\StreamInterface; use RuntimeException; use Symfony\Component\HttpFoundation\Response; @@ -526,6 +528,25 @@ public function withDriver(Closure $callback): mixed return $this->withBorrowedAccessor('getDriver', $callback); } + /** + * Get an operator that owns each operation's borrow and each stream's lease. + */ + public function getOperator(): FilesystemOperator + { + return new FilesystemOperatorAdapter( + $this->withDriver(...), + fn (string $path): mixed => $this->leasedStream(static function (FilesystemContract $filesystem) use ($path): mixed { + if (! method_exists($filesystem, 'getDriver')) { + throw new RuntimeException( + 'Pooled filesystem driver [' . $filesystem::class . '] does not support [getDriver] access.', + ); + } + + return $filesystem->getDriver()->readStream($path); + }), + ); + } + /** * Run a callback with borrow-scoped access to the Flysystem adapter. */ diff --git a/src/filesystem/src/FilesystemManager.php b/src/filesystem/src/FilesystemManager.php index fc962ee95c..922075b3d6 100644 --- a/src/filesystem/src/FilesystemManager.php +++ b/src/filesystem/src/FilesystemManager.php @@ -7,6 +7,7 @@ use Aws\S3\S3Client; use Closure; use Google\Cloud\Storage\StorageClient as GcsClient; +use Hypervel\Context\CoroutineContext; use Hypervel\Contracts\Container\Container; use Hypervel\Contracts\Filesystem\Cloud; use Hypervel\Contracts\Filesystem\Factory as FactoryContract; @@ -58,6 +59,11 @@ class FilesystemManager implements FactoryContract */ protected const string ON_DEMAND_DISK_NAME = 'ondemand'; + /** + * The coroutine-local construction stack prefix for each manager. + */ + protected const string READ_THROUGH_CONTEXT_KEY_PREFIX = '__filesystem.read-through.construction.'; + /** * Google Cloud Storage client constructor options supported by the installed SDK. */ @@ -244,7 +250,7 @@ private function resolveConstructionDescriptor( return $this->createClientPooledDisk($driver, $config); } - $driverMethod = 'create' . ucfirst($driver) . 'Driver'; + $driverMethod = 'create' . Str::studly($driver) . 'Driver'; if (! method_exists($this, $driverMethod)) { throw new InvalidArgumentException("Driver [{$driver}] is not supported."); @@ -434,6 +440,95 @@ public function createS3Driver(array $config): Cloud ); } + /** + * Create an instance of the read-through driver. + */ + public function createReadThroughDriver(array $config, string $name = 'read-through'): Filesystem + { + if (! isset($config['primary']) || $config['primary'] === '' || $config['primary'] === []) { + throw new InvalidArgumentException('Read-through disk is missing "primary" configuration option.'); + } + if (! isset($config['fallback']) || $config['fallback'] === '' || $config['fallback'] === []) { + throw new InvalidArgumentException('Read-through disk is missing "fallback" configuration option.'); + } + if ($config['primary'] === $config['fallback']) { + throw new InvalidArgumentException('Read-through disk requires distinct "primary" and "fallback" disks.'); + } + + // Scoped inline sides can re-enter construction without resolving a named disk. + $contextKey = self::READ_THROUGH_CONTEXT_KEY_PREFIX . spl_object_id($this); + $stack = CoroutineContext::get($contextKey, []); + $label = $name === self::ON_DEMAND_DISK_NAME ? '(on-demand)' : $name; + + foreach ($stack as $entry) { + if ($entry['config'] !== $config) { + continue; + } + + if (count($stack) === 1) { + throw new InvalidArgumentException("Read-through disk [{$label}] cannot reference itself."); + } + + $cycle = [...array_column($stack, 'name'), $label]; + + throw new InvalidArgumentException('Circular read-through disk definition detected: ' . implode(' -> ', $cycle) . '.'); + } + + CoroutineContext::set($contextKey, [...$stack, ['config' => $config, 'name' => $label]]); + + try { + $primary = is_array($config['primary']) + ? $this->resolveWithLogicalName(self::ON_DEMAND_DISK_NAME, $config['primary'], null) + : $this->disk($config['primary']); + $fallback = is_array($config['fallback']) + ? $this->resolveWithLogicalName(self::ON_DEMAND_DISK_NAME, $config['fallback'], null) + : $this->disk($config['fallback']); + + if (! $primary instanceof Cloud || ! $fallback instanceof Cloud) { + throw new InvalidArgumentException('Read-through disks must implement the cloud filesystem contract.'); + } + + $adapter = new ReadThroughFilesystemAdapter( + $this->readThroughOperator($primary), + $this->readThroughOperator($fallback), + $config['throw_on_promotion_failure'] ?? false, + $config['copy'] ?? true, + ); + + return new ReadThroughFilesystem( + $this->createFlysystem($adapter, $config), + $primary instanceof FilesystemAdapter ? $primary->getAdapter() : $adapter, + array_replace($primary->getConfig(), $config), // @phpstan-ignore method.notFound (Pooled decorators forward adapter accessors.) + $primary, + $fallback, + $config['prefix'] ?? '', + $adapter, + ); + } finally { + if ($stack === []) { + CoroutineContext::forget($contextKey); + } else { + CoroutineContext::set($contextKey, $stack); + } + } + } + + /** + * Get a side operator without exposing borrowed clients or losing native cloud reads. + */ + protected function readThroughOperator(Cloud $disk): FilesystemOperator + { + if ($disk instanceof AwsS3V3Adapter || $disk instanceof GoogleCloudStorageAdapter) { + return new FilesystemOperatorAdapter( + static fn (Closure $operation): mixed => $operation($disk->getDriver()), + $disk->readStreamRangeOrFail(...), + $disk->readStreamRangeOrFail(...), + ); + } + + return $disk instanceof FilesystemAdapter ? $disk->getDriver() : $disk->getOperator(); // @phpstan-ignore method.notFound (Pooled decorators forward the borrow-safe accessor.) + } + /** * Derive the S3 client construction config from a disk config. */ diff --git a/src/filesystem/src/FilesystemOperatorAdapter.php b/src/filesystem/src/FilesystemOperatorAdapter.php new file mode 100644 index 0000000000..9cc43b402d --- /dev/null +++ b/src/filesystem/src/FilesystemOperatorAdapter.php @@ -0,0 +1,210 @@ +operation)(static fn (FilesystemOperator $driver): bool => $driver->fileExists($location)); + } + + /** + * Determine if a directory exists. + */ + public function directoryExists(string $location): bool + { + return ($this->operation)(static fn (FilesystemOperator $driver): bool => $driver->directoryExists($location)); + } + + /** + * Determine if a file or directory exists. + */ + public function has(string $location): bool + { + return ($this->operation)(static fn (FilesystemOperator $driver): bool => $driver->has($location)); + } + + /** + * Read a file. + */ + public function read(string $location): string + { + return ($this->operation)(static fn (FilesystemOperator $driver): string => $driver->read($location)); + } + + /** + * Open a stream that retains any required lease until closure. + * + * @return resource + */ + public function readStream(string $location): mixed + { + return ($this->stream)($location); + } + + /** + * Open a native ranged stream, or return null when ranges are unsupported. + * + * @return null|resource + */ + public function readStreamRange(string $path, ?int $start, ?int $end): mixed + { + return $this->range === null ? null : ($this->range)($path, $start, $end); + } + + /** + * List contents without holding a lease while the caller processes entries. + * + * @return DirectoryListing + */ + public function listContents(string $location, bool $deep = self::LIST_SHALLOW): DirectoryListing + { + return new DirectoryListing(($this->operation)( + static fn (FilesystemOperator $driver): array => $driver->listContents($location, $deep)->toArray(), + )); + } + + /** + * Get a file's last modification time. + */ + public function lastModified(string $path): int + { + return ($this->operation)(static fn (FilesystemOperator $driver): int => $driver->lastModified($path)); + } + + /** + * Get a file's size. + */ + public function fileSize(string $path): int + { + return ($this->operation)(static fn (FilesystemOperator $driver): int => $driver->fileSize($path)); + } + + /** + * Get a file's MIME type. + */ + public function mimeType(string $path): string + { + return ($this->operation)(static fn (FilesystemOperator $driver): string => $driver->mimeType($path)); + } + + /** + * Get a file's visibility. + */ + public function visibility(string $path): string + { + return ($this->operation)(static fn (FilesystemOperator $driver): string => $driver->visibility($path)); + } + + /** + * Write a file. + */ + public function write(string $location, string $contents, array $config = []): void + { + ($this->operation)(static function (FilesystemOperator $driver) use ($location, $contents, $config): void { + $driver->write($location, $contents, $config); + }); + } + + /** + * Write a file stream. + * + * @param resource $contents + */ + public function writeStream(string $location, mixed $contents, array $config = []): void + { + ($this->operation)(static function (FilesystemOperator $driver) use ($location, $contents, $config): void { + $driver->writeStream($location, $contents, $config); + }); + } + + /** + * Set a file's visibility. + */ + public function setVisibility(string $path, string $visibility): void + { + ($this->operation)(static function (FilesystemOperator $driver) use ($path, $visibility): void { + $driver->setVisibility($path, $visibility); + }); + } + + /** + * Delete a file. + */ + public function delete(string $location): void + { + ($this->operation)(static function (FilesystemOperator $driver) use ($location): void { + $driver->delete($location); + }); + } + + /** + * Delete a directory. + */ + public function deleteDirectory(string $location): void + { + ($this->operation)(static function (FilesystemOperator $driver) use ($location): void { + $driver->deleteDirectory($location); + }); + } + + /** + * Create a directory. + */ + public function createDirectory(string $location, array $config = []): void + { + ($this->operation)(static function (FilesystemOperator $driver) use ($location, $config): void { + $driver->createDirectory($location, $config); + }); + } + + /** + * Move a file. + */ + public function move(string $source, string $destination, array $config = []): void + { + ($this->operation)(static function (FilesystemOperator $driver) use ($source, $destination, $config): void { + $driver->move($source, $destination, $config); + }); + } + + /** + * Copy a file. + */ + public function copy(string $source, string $destination, array $config = []): void + { + ($this->operation)(static function (FilesystemOperator $driver) use ($source, $destination, $config): void { + $driver->copy($source, $destination, $config); + }); + } +} diff --git a/src/filesystem/src/GoogleCloudStorageAdapter.php b/src/filesystem/src/GoogleCloudStorageAdapter.php index 6ea0d3347e..1615d4d7cb 100644 --- a/src/filesystem/src/GoogleCloudStorageAdapter.php +++ b/src/filesystem/src/GoogleCloudStorageAdapter.php @@ -72,10 +72,14 @@ public function temporaryUploadUrl(string $path, DateTimeInterface $expiration, */ public function readStream(string $path): mixed { - return $this->readStreamWithOptions( - $path, - ($this->config['stream_reads'] ?? true) ? ['restOptions' => ['stream' => true]] : [] - ); + try { + return $this->readStreamRangeOrFail($path); + } catch (UnableToReadFile $exception) { + throw_if($this->throwsExceptions(), $exception); + $this->report($exception); + + return null; + } } /** @@ -91,17 +95,31 @@ public function readStreamRange(string $path, ?int $start, ?int $end): mixed return $this->readStream($path); } - return $this->readStreamWithOptions( - $path, - [ - 'restOptions' => [ - 'headers' => [ - 'Range' => "bytes={$start}-{$end}", - ], - ...(($this->config['stream_reads'] ?? true) ? ['stream' => true] : []), - ], - ] - ); + try { + return $this->readStreamRangeOrFail($path, $start, $end); + } catch (UnableToReadFile $exception) { + throw_if($this->throwsExceptions(), $exception); + $this->report($exception); + + return null; + } + } + + /** + * Open a whole-object or ranged stream without applying the disk's failure policy. + * + * @return resource + */ + public function readStreamRangeOrFail(string $path, ?int $start = null, ?int $end = null): mixed + { + [$start, $end] = $this->normalizeStreamRange($start, $end); + $options = ($this->config['stream_reads'] ?? true) ? ['restOptions' => ['stream' => true]] : []; + + if ($start !== null || $end !== null) { + $options['restOptions']['headers']['Range'] = "bytes={$start}-{$end}"; + } + + return $this->readStreamWithOptions($path, $options); } /** @@ -112,6 +130,11 @@ public function getClient(): StorageClient return $this->client; } + /** + * Read an object without applying the disk's failure policy. + * + * @return resource + */ private function readStreamWithOptions(string $path, array $options): mixed { $prefixedPath = $this->prefixer->prefixPath($path); @@ -121,26 +144,14 @@ private function readStreamWithOptions(string $path, array $options): mixed } catch (CanceledException $exception) { throw $exception; } catch (Throwable $exception) { - $exception = UnableToReadFile::fromLocation($path, $exception->getMessage(), $exception); - - throw_if($this->throwsExceptions(), $exception); - - $this->report($exception); - - return null; + throw UnableToReadFile::fromLocation($path, $exception->getMessage(), $exception); } if (! is_resource($stream)) { - $exception = UnableToReadFile::fromLocation( + throw UnableToReadFile::fromLocation( $path, 'Downloaded object does not contain a file resource.', ); - - throw_if($this->throwsExceptions(), $exception); - - $this->report($exception); - - return null; } return $stream; diff --git a/src/filesystem/src/ReadThroughFilesystem.php b/src/filesystem/src/ReadThroughFilesystem.php new file mode 100644 index 0000000000..f9d08ae515 --- /dev/null +++ b/src/filesystem/src/ReadThroughFilesystem.php @@ -0,0 +1,129 @@ +readThroughPrefixer = new PathPrefixer($prefix); + } + + /** + * Get the primary filesystem path, including both disks' prefixes. + */ + public function path(string $path): string + { + return $this->primary->path($this->readThroughPrefixer->prefixPath($path)); + } + + /** + * Get the URL for the file at the given path. + */ + public function url(string $path): string + { + return $this->readerFor($path)->url($this->readThroughPrefixer->prefixPath($path)); + } + + /** + * Get a temporary URL for the file at the given path. + */ + public function temporaryUrl(string $path, DateTimeInterface $expiration, array $options = []): string + { + return isset($this->temporaryUrlCallback) + ? ($this->temporaryUrlCallback)($path, $expiration, $options) + : $this->readerFor($path)->temporaryUrl($this->readThroughPrefixer->prefixPath($path), $expiration, $options); // @phpstan-ignore method.notFound + } + + /** + * Determine if temporary upload URLs can be generated. + */ + public function providesTemporaryUploadUrls(): bool + { + return isset($this->temporaryUploadUrlCallback) || $this->primary->providesTemporaryUploadUrls(); // @phpstan-ignore method.notFound + } + + /** + * Get a temporary upload URL for the file at the given path. + */ + public function temporaryUploadUrl(string $path, DateTimeInterface $expiration, array $options = []): array|string + { + return isset($this->temporaryUploadUrlCallback) + ? ($this->temporaryUploadUrlCallback)($path, $expiration, $options) + : $this->primary->temporaryUploadUrl($this->readThroughPrefixer->prefixPath($path), $expiration, $options); // @phpstan-ignore method.notFound + } + + /** + * Determine if temporary URLs can be generated. + */ + public function providesTemporaryUrls(): bool + { + return isset($this->temporaryUrlCallback) + || $this->primary->providesTemporaryUrls() // @phpstan-ignore method.notFound + || $this->fallback->providesTemporaryUrls(); // @phpstan-ignore method.notFound + } + + /** + * Read a byte range using native cloud requests when promotion is unnecessary. + * + * @return null|resource + */ + public function readStreamRange(string $path, ?int $start, ?int $end): mixed + { + [$start, $end] = $this->normalizeStreamRange($start, $end); + + if ($start === null && $end === null) { + return $this->readStream($path); + } + + try { + $stream = $this->readThroughAdapter?->readStreamRange( + $this->readThroughPrefixer->prefixPath($path), + $start, + $end, + ); + } catch (UnableToReadFile $exception) { + throw_if($this->throwsExceptions(), $exception); + $this->report($exception); + + return null; + } + + return $stream ?? parent::readStreamRange($path, $start, $end); + } + + /** + * Get the filesystem that contains the given path. + */ + protected function readerFor(string $path): Cloud + { + return $this->primary->fileExists($this->readThroughPrefixer->prefixPath($path)) // @phpstan-ignore method.notFound + ? $this->primary + : $this->fallback; + } +} diff --git a/src/filesystem/src/ReadThroughFilesystemAdapter.php b/src/filesystem/src/ReadThroughFilesystemAdapter.php new file mode 100644 index 0000000000..3bb1b58618 --- /dev/null +++ b/src/filesystem/src/ReadThroughFilesystemAdapter.php @@ -0,0 +1,340 @@ +primary->fileExists($path) || $this->fallback->fileExists($path); + } + + /** + * Determine if a directory exists. + */ + public function directoryExists(string $path): bool + { + return $this->primary->directoryExists($path) || $this->fallback->directoryExists($path); + } + + /** + * Write a file to the primary filesystem. + */ + public function write(string $path, string $contents, Config $config): void + { + $this->primary->write($path, $contents, $config->toArray()); + } + + /** + * Write a file stream to the primary filesystem. + * + * @param resource $contents + */ + public function writeStream(string $path, mixed $contents, Config $config): void + { + $this->primary->writeStream($path, $contents, $config->toArray()); + } + + /** + * Read a file, promoting it from the fallback filesystem when necessary. + */ + public function read(string $path): string + { + if ($this->primary->fileExists($path)) { + return $this->primary->read($path); + } + + $contents = $this->fallback->read($path); + + if (! $this->copy) { + return $contents; + } + + if ($this->primary->fileExists($path)) { // @phpstan-ignore if.alwaysFalse (Fallback I/O may yield while another operation promotes the file.) + return $this->primary->read($path); + } + + try { + $this->primary->write($path, $contents); + } catch (FilesystemException $exception) { + $this->handlePromotionFailure($path, $exception); + } + + return $contents; + } + + /** + * Read a file stream, promoting it from the fallback filesystem when necessary. + * + * @return resource + */ + public function readStream(string $path): mixed + { + if ($this->primary->fileExists($path)) { + return $this->primary->readStream($path); + } + + if (! $this->copy) { + return $this->fallback->readStream($path); + } + + $temporary = $this->spoolFallback($path); + $returned = false; + + try { + if ($this->primary->fileExists($path)) { // @phpstan-ignore if.alwaysFalse (Spooling may yield while another operation promotes the file.) + return $this->primary->readStream($path); + } + + try { + $this->primary->writeStream($path, $temporary); + } catch (FilesystemException $exception) { + $this->handlePromotionFailure($path, $exception); + } + + rewind($temporary); + $returned = true; + + return $temporary; + } finally { + if (! $returned) { + fclose($temporary); + } + } + } + + /** + * Open a native range when no fallback promotion is needed. + * + * @return null|resource null when the normal read-and-seek path is required + */ + public function readStreamRange(string $path, ?int $start, ?int $end): mixed + { + $reader = $this->primary->fileExists($path) + ? $this->primary + : ($this->copy ? null : $this->fallback); + + return $reader instanceof FilesystemOperatorAdapter + ? $reader->readStreamRange($path, $start, $end) + : null; + } + + /** + * Delete a file from both filesystems. + */ + public function delete(string $path): void + { + if ($this->fallback->fileExists($path)) { + $this->fallback->delete($path); + } + + $this->primary->delete($path); + } + + /** + * Delete a directory from both filesystems. + */ + public function deleteDirectory(string $path): void + { + if ($this->fallback->directoryExists($path)) { + $this->fallback->deleteDirectory($path); + } + + $this->primary->deleteDirectory($path); + } + + /** + * Create a directory on the primary filesystem. + */ + public function createDirectory(string $path, Config $config): void + { + $this->primary->createDirectory($path, $config->toArray()); + } + + /** + * Set a file's visibility on the filesystem containing it. + */ + public function setVisibility(string $path, string $visibility): void + { + $this->readerFor($path)->setVisibility($path, $visibility); + } + + /** + * Retrieve a file's visibility. + */ + public function visibility(string $path): FileAttributes + { + return new FileAttributes($path, visibility: $this->readerFor($path)->visibility($path)); + } + + /** + * Retrieve a file's MIME type. + */ + public function mimeType(string $path): FileAttributes + { + return new FileAttributes($path, mimeType: $this->readerFor($path)->mimeType($path)); + } + + /** + * Retrieve a file's last modified time. + */ + public function lastModified(string $path): FileAttributes + { + return new FileAttributes($path, lastModified: $this->readerFor($path)->lastModified($path)); + } + + /** + * Retrieve a file's size. + */ + public function fileSize(string $path): FileAttributes + { + return new FileAttributes($path, fileSize: $this->readerFor($path)->fileSize($path)); + } + + /** + * List the contents of the primary filesystem. + */ + public function listContents(string $path, bool $deep): iterable + { + return $this->primary->listContents($path, $deep); + } + + /** + * Move a file on the primary filesystem and remove its fallback source. + */ + public function move(string $source, string $destination, Config $config): void + { + $this->moveOnPrimary($source, $destination, $config); + + try { + if ($this->fallback->fileExists($source)) { + $this->fallback->delete($source); + } + } catch (FilesystemException $exception) { + throw UnableToMoveFile::fromLocationTo($source, $destination, $exception); + } + } + + /** + * Copy a file on the primary filesystem, reading it from the fallback filesystem when necessary. + */ + public function copy(string $source, string $destination, Config $config): void + { + if ($this->primary->fileExists($source)) { + $this->primary->copy($source, $destination, $config->toArray()); + + return; + } + + try { + $this->copyFromFallback($source, $destination, $config); + } catch (FilesystemException $exception) { + throw UnableToCopyFile::fromLocationTo($source, $destination, $exception); + } + } + + /** + * Move a file to its destination on the primary filesystem. + */ + protected function moveOnPrimary(string $source, string $destination, Config $config): void + { + if ($this->primary->fileExists($source)) { + $this->primary->move($source, $destination, $config->toArray()); + + return; + } + + try { + $this->copyFromFallback($source, $destination, $config); + } catch (FilesystemException $exception) { + throw UnableToMoveFile::fromLocationTo($source, $destination, $exception); + } + } + + /** + * Copy a file from the fallback filesystem to the primary filesystem. + */ + protected function copyFromFallback(string $source, string $destination, Config $config): void + { + $stream = $this->spoolFallback($source); + + try { + $this->primary->writeStream($destination, $stream, $config->toArray()); + } finally { + fclose($stream); + } + } + + /** + * Spool the fallback file and release its lease before borrowing the primary. + * + * @return resource + */ + protected function spoolFallback(string $path): mixed + { + $source = $this->fallback->readStream($path); + $temporary = null; + + try { + $temporary = fopen('php://temp', 'w+b'); + + if ($temporary === false || stream_copy_to_stream($source, $temporary) === false || ! rewind($temporary)) { + throw UnableToReadFile::fromLocation($path); + } + + return $temporary; + } catch (Throwable $exception) { + if (is_resource($temporary)) { + fclose($temporary); + } + + throw $exception; + } finally { + fclose($source); + } + } + + /** + * Get the filesystem that contains the given path. + */ + protected function readerFor(string $path): FilesystemOperator + { + return $this->primary->fileExists($path) ? $this->primary : $this->fallback; + } + + /** + * Handle an exception encountered while promoting a file. + */ + protected function handlePromotionFailure(string $path, FilesystemException $exception): void + { + if ($this->throwOnPromotionFailure) { + throw UnableToReadFile::fromLocation($path, 'Unable to promote file to the primary filesystem.', $exception); + } + } +} diff --git a/src/sentry/src/Features/Storage/SentryS3V3Adapter.php b/src/sentry/src/Features/Storage/SentryS3V3Adapter.php index 0f0697e284..b59c3f6f7f 100644 --- a/src/sentry/src/Features/Storage/SentryS3V3Adapter.php +++ b/src/sentry/src/Features/Storage/SentryS3V3Adapter.php @@ -29,4 +29,14 @@ public function __construct( $this->recordSpans = $recordSpans; $this->recordBreadcrumbs = $recordBreadcrumbs; } + + /** + * Open a whole-object or ranged stream without applying the disk's failure policy. + * + * @return resource + */ + public function readStreamRangeOrFail(string $path, ?int $start = null, ?int $end = null): mixed + { + return $this->withSentry(__FUNCTION__, func_get_args(), $path, compact('path', 'start', 'end')); + } } diff --git a/src/support/src/Facades/Storage.php b/src/support/src/Facades/Storage.php index 2433927354..5d40705f85 100644 --- a/src/support/src/Facades/Storage.php +++ b/src/support/src/Facades/Storage.php @@ -17,6 +17,7 @@ * @method static \Hypervel\Contracts\Filesystem\Filesystem createFtpDriver(array $config) * @method static \Hypervel\Contracts\Filesystem\Cloud createGcsDriver(array $config) * @method static \Hypervel\Contracts\Filesystem\Filesystem createLocalDriver(array $config, string $name = 'local') + * @method static \Hypervel\Contracts\Filesystem\Filesystem createReadThroughDriver(array $config, string $name = 'read-through') * @method static \Hypervel\Contracts\Filesystem\Cloud createS3Driver(array $config) * @method static \Hypervel\Contracts\Filesystem\Filesystem createScopedDriver(array $config, string|null $name = null) * @method static \Hypervel\Contracts\Filesystem\Filesystem createSftpDriver(array $config) @@ -93,6 +94,7 @@ * @method static string visibility(string $path) * @method static void write(string $location, string $contents, array $config = []) * @method static \Hypervel\ObjectPool\PoolDefinition getDefinition() + * @method static \League\Flysystem\FilesystemOperator getOperator() * @method static string getPoolName() * @method static bool invalidatePool() * @method static mixed withAdapter(\Closure $callback) diff --git a/tests/Filesystem/AwsS3V3AdapterTest.php b/tests/Filesystem/AwsS3V3AdapterTest.php index fcc9b2ea2d..a1e184424f 100644 --- a/tests/Filesystem/AwsS3V3AdapterTest.php +++ b/tests/Filesystem/AwsS3V3AdapterTest.php @@ -12,6 +12,12 @@ use Hypervel\Container\Container; use Hypervel\Contracts\Debug\ExceptionHandler; use Hypervel\Filesystem\AwsS3V3Adapter; +use Hypervel\Filesystem\ClientPooledFilesystem; +use Hypervel\Filesystem\FilesystemAdapter; +use Hypervel\Filesystem\FilesystemManager; +use Hypervel\ObjectPool\PoolDefinition; +use Hypervel\ObjectPool\PoolManager; +use Hypervel\ObjectPool\PoolOptions; use Hypervel\Tests\TestCase; use InvalidArgumentException; use League\Flysystem\FilesystemAdapter as FlysystemAdapter; @@ -224,6 +230,11 @@ function (CommandInterface $command) use (&$commands): Result { return new Result(['Body' => Utils::streamFor('range')]); }, + function (CommandInterface $command) use (&$commands): Result { + $commands[] = $command; + + return new Result(['Body' => Utils::streamFor('pooled')]); + }, ]); $adapter = $this->adapter($handler, [ 'bucket' => 'bucket', @@ -244,7 +255,28 @@ function (CommandInterface $command) use (&$commands): Result { $this->assertIsResource($range); fclose($plain); fclose($range); - $this->assertCount(2, $commands); + $pools = new PoolManager; + $pooled = new ClientPooledFilesystem( + new PoolDefinition('filesystem:http-options', 's3', 'test', PoolOptions::fromArray([])), + $adapter->getClient(...), + static fn (object $client): AwsS3V3Adapter => $adapter, + $pools, + $adapter->getConfig(), + ); + + try { + $stream = $pooled->getOperator()->readStream('pooled.txt'); + + try { + $this->assertSame('pooled', stream_get_contents($stream)); + } finally { + fclose($stream); + } + } finally { + $pools->purgeAll(); + } + + $this->assertCount(3, $commands); foreach ($commands as $command) { $this->assertSame('bucket', $command['Bucket']); @@ -255,6 +287,99 @@ function (CommandInterface $command) use (&$commands): Result { $this->assertSame('tenant/plain.txt', $commands[0]['Key']); $this->assertSame('tenant/range.txt', $commands[1]['Key']); $this->assertSame('bytes=2-4', $commands[1]['Range']); + $this->assertSame('tenant/pooled.txt', $commands[2]['Key']); + } + + #[DataProvider('readThroughCloudSides')] + public function testReadThroughRangesPreserveNativeRequestsAndTheOuterFailurePolicy(bool $primary, bool $pooled): void + { + $command = null; + $failure = new RuntimeException('S3 failed'); + $handler = new MockHandler([ + function (CommandInterface $request) use (&$command): Result { + $command = $request; + + return new Result(['Body' => Utils::streamFor('range')]); + }, + $failure, + ]); + $adapter = $this->adapter($handler, [ + 'bucket' => 'bucket', + 'root' => 'tenant', + 'throw' => false, + 'report' => true, + 'options' => ['@http' => ['timeout' => 12]], + ]); + $adapter->getDriver()->shouldReceive('fileExists')->andReturn(true); + $emptyDriver = m::mock(FilesystemOperator::class); + $emptyDriver->shouldReceive('fileExists')->andReturn(false); + $empty = new FilesystemAdapter($emptyDriver, m::mock(FlysystemAdapter::class)); + $pools = new PoolManager; + $cloud = $pooled ? new ClientPooledFilesystem( + new PoolDefinition('filesystem:read-through-range', 's3', 'test', PoolOptions::fromArray([])), + $adapter->getClient(...), + static fn (object $client): AwsS3V3Adapter => $adapter, + $pools, + $adapter->getConfig(), + ) : $adapter; + $exceptionHandler = m::mock(ExceptionHandler::class); + $exceptionHandler->shouldNotReceive('report'); + Container::getInstance()->instance(ExceptionHandler::class, $exceptionHandler); + $manager = new FilesystemManager(Container::getInstance()); + $manager->set('primary', $primary ? $cloud : $empty); + $manager->set('fallback', $primary ? $empty : $cloud); + $readThrough = $manager->build([ + 'driver' => 'read-through', + 'primary' => 'primary', + 'fallback' => 'fallback', + 'copy' => $primary, + 'throw' => true, + 'report' => false, + 'prefix' => 'outer', + ]); + + try { + $stream = $readThrough->readStreamRange('file.txt', 2, 4); + + try { + $this->assertSame('range', stream_get_contents($stream)); + + if ($pooled) { + $this->assertSame(1, $pools->get($cloud->getPoolName())->getBorrowedCount()); + } + } finally { + fclose($stream); + } + + $this->assertSame('bytes=2-4', $command['Range']); + $this->assertSame('tenant/outer/file.txt', $command['Key']); + $this->assertSame(12, $command['@http']['timeout']); + + try { + $readThrough->readStreamRange('file.txt', 2, 4); + $this->fail('Expected the composite failure policy to throw.'); + } catch (UnableToReadFile $exception) { + $this->assertSame($failure, $exception->getPrevious()); + } + + if ($pooled) { + $this->assertSame(0, $pools->get($cloud->getPoolName())->getBorrowedCount()); + } + } finally { + $pools->purgeAll(); + } + } + + /** + * Provide read-through sides that can serve native ranges. + */ + public static function readThroughCloudSides(): array + { + return [ + 'pooled primary' => [true, true], + 'pooled fallback without promotion' => [false, true], + 'non-pooled primary' => [true, false], + ]; } public function testReadStreamRangeWrapsClientFailures(): void diff --git a/tests/Filesystem/ClientPooledFilesystemTest.php b/tests/Filesystem/ClientPooledFilesystemTest.php index 9b95957ce8..d78ae8f2f9 100644 --- a/tests/Filesystem/ClientPooledFilesystemTest.php +++ b/tests/Filesystem/ClientPooledFilesystemTest.php @@ -15,6 +15,7 @@ use Hypervel\Contracts\ObjectPool\ObjectPool as ObjectPoolContract; use Hypervel\Filesystem\ClientPooledFilesystem; use Hypervel\Filesystem\FilesystemAdapter; +use Hypervel\Filesystem\FilesystemManager; use Hypervel\Http\IterableStreamedResponse; use Hypervel\Http\Request; use Hypervel\Http\Response; @@ -22,6 +23,7 @@ use Hypervel\ObjectPool\PoolDefinition; use Hypervel\ObjectPool\PoolManager; use Hypervel\ObjectPool\PoolOptions; +use Hypervel\Sentry\Features\Storage\SentryCloudFilesystem; use Hypervel\Testbench\TestCase; use Hypervel\Testing\ParallelTesting; use League\Flysystem\Filesystem; @@ -68,6 +70,75 @@ protected function tearDown(): void parent::tearDown(); } + public function testReadThroughDisksReleaseSharedClientBorrowsBeforeCopyingAndProcessingListings(): void + { + $definition = new PoolDefinition('filesystem:shared-read-through', 'local-client', 'shared', PoolOptions::fromArray([ + 'max_objects' => 1, + 'wait_timeout' => 0.02, + ])); + $makeDisk = fn (string $root): ClientPooledFilesystem => new ClientPooledFilesystem( + $definition, + static fn (): object => new stdClass, + static function (object $client) use ($root): FilesystemAdapter { + $adapter = new LocalFilesystemAdapter($root); + + return new FilesystemAdapter(new Filesystem($adapter), $adapter, ['root' => $root]); + }, + $this->pools, + ['root' => $root], + ); + $primary = $makeDisk($this->tempDir . '/primary'); + $fallback = $makeDisk($this->tempDir . '/fallback'); + $manager = new FilesystemManager($this->app); + $manager->set('read-primary', new SentryCloudFilesystem($primary, [], false, false)); + $manager->set('read-fallback', new SentryCloudFilesystem($fallback, [], false, false)); + $readThrough = $manager->build([ + 'driver' => 'read-through', + 'primary' => 'read-primary', + 'fallback' => 'read-fallback', + ]); + $fallback->put('source.txt', 'contents'); + + $this->assertTrue($readThrough->copy('source.txt', 'destination.txt')); + $this->assertSame('contents', $primary->get('destination.txt')); + $this->assertSame('contents', $fallback->get('source.txt')); + $this->assertSame(0, $this->pools->get($definition->identity)->getBorrowedCount()); + + foreach ($readThrough->getDriver()->listContents('', false) as $entry) { + $this->assertSame(0, $this->pools->get($definition->identity)->getBorrowedCount()); + $this->assertSame('contents', $readThrough->get($entry->path())); + } + + $stream = $readThrough->readStream('source.txt'); + + try { + $this->assertSame('contents', stream_get_contents($stream)); + $this->assertSame(0, $this->pools->get($definition->identity)->getBorrowedCount()); + $this->assertSame('contents', $primary->get('source.txt')); + } finally { + fclose($stream); + } + + $fallback->put('unpromoted.txt', 'fallback contents'); + $uncopied = $manager->build([ + 'driver' => 'read-through', + 'primary' => 'read-primary', + 'fallback' => 'read-fallback', + 'copy' => false, + ]); + $stream = $uncopied->readStream('unpromoted.txt'); + + try { + $this->assertSame('fallback contents', stream_get_contents($stream)); + $this->assertSame(1, $this->pools->get($definition->identity)->getBorrowedCount()); + } finally { + fclose($stream); + } + + $this->assertSame(0, $this->pools->get($definition->identity)->getBorrowedCount()); + $this->assertFalse($primary->exists('unpromoted.txt')); + } + public function testSynchronousOperationsBuildFreshStacksAroundOnePooledClient(): void { $clientCreations = 0; diff --git a/tests/Filesystem/FilesystemManagerTest.php b/tests/Filesystem/FilesystemManagerTest.php index abae5babef..13f905ffee 100644 --- a/tests/Filesystem/FilesystemManagerTest.php +++ b/tests/Filesystem/FilesystemManagerTest.php @@ -6,6 +6,7 @@ use Aws\S3\S3Client; use DateTimeImmutable; +use DateTimeInterface; use Google\Cloud\Storage\Bucket; use Google\Cloud\Storage\StorageClient as GcsClient; use Hypervel\Config\Repository; @@ -20,13 +21,16 @@ use Hypervel\Filesystem\FilesystemManager; use Hypervel\Filesystem\FilesystemPoolProxy; use Hypervel\Filesystem\GoogleCloudStorageAdapter; +use Hypervel\Filesystem\ReadThroughFilesystemAdapter; use Hypervel\ObjectPool\PoolFingerprint; use Hypervel\ObjectPool\PoolManager; +use Hypervel\Support\CarbonImmutable; use Hypervel\Testing\ParallelTesting; use Hypervel\Tests\TestCase; use InvalidArgumentException; use League\Flysystem\AwsS3V3\AwsS3V3Adapter as FlysystemS3Adapter; use League\Flysystem\Filesystem as Flysystem; +use League\Flysystem\FilesystemOperator; use League\Flysystem\GoogleCloudStorage\GoogleCloudStorageAdapter as FlysystemGcsAdapter; use League\Flysystem\Local\LocalFilesystemAdapter; use League\Flysystem\PathPrefixing\PathPrefixedAdapter; @@ -34,10 +38,12 @@ use League\Flysystem\UnableToReadFile; use League\Flysystem\UnableToWriteFile; use Mockery as m; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\RequiresOperatingSystem; use ReflectionProperty; use RuntimeException; use stdClass; +use Swoole\Coroutine\CanceledException; enum FilesystemTestStringBackedDisk: string { @@ -600,6 +606,443 @@ public function testCanBuildInlineScopedDisks(): void } } + public function testCanBuildReadThroughDisks(): void + { + $filesystem = $this->readThroughFilesystemManager(); + $primary = $filesystem->disk('primary'); + $fallback = $filesystem->disk('fallback'); + $readThrough = $filesystem->disk('read-through'); + + $fallback->put('fallback.txt', 'fallback contents'); + $fallback->put('hidden-from-listing.txt', 'contents'); + $primary->put('primary.txt', 'primary contents'); + $primary->put('preferred.txt', 'primary version'); + $fallback->put('preferred.txt', 'fallback version'); + + $this->assertTrue($readThrough->exists('fallback.txt')); + $this->assertSame(strlen('fallback contents'), $readThrough->size('fallback.txt')); + $this->assertTrue($primary->missing('fallback.txt')); + $this->assertSame(['preferred.txt', 'primary.txt'], $readThrough->files()); + + $this->assertSame('fallback contents', $readThrough->get('fallback.txt')); + $this->assertSame('fallback contents', $primary->get('fallback.txt')); + $this->assertSame('primary version', $readThrough->get('preferred.txt')); + + $readThrough->put('written.txt', 'written contents'); + + $this->assertSame('written contents', $primary->get('written.txt')); + $this->assertTrue($fallback->missing('written.txt')); + } + + public function testReadThroughDisksPromoteStreams(): void + { + $filesystem = $this->readThroughFilesystemManager(); + $primary = $filesystem->disk('primary'); + $fallback = $filesystem->disk('fallback'); + $readThrough = $filesystem->disk('read-through'); + + $fallback->put('stream.txt', 'stream contents'); + $stream = $readThrough->readStream('stream.txt'); + + try { + $this->assertSame('stream contents', stream_get_contents($stream)); + $this->assertSame('stream contents', $primary->get('stream.txt')); + } finally { + fclose($stream); + } + } + + public function testReadThroughDisksDoNotCopyWhenDisabled(): void + { + $filesystem = $this->readThroughFilesystemManager(['copy' => false]); + $primary = $filesystem->disk('primary'); + $fallback = $filesystem->disk('fallback'); + $readThrough = $filesystem->disk('read-through'); + + $fallback->put('fallback.txt', 'fallback contents'); + + $this->assertSame('fallback contents', $readThrough->get('fallback.txt')); + $this->assertTrue($primary->missing('fallback.txt')); + + $fallback->put('stream.txt', 'stream contents'); + $stream = $readThrough->readStream('stream.txt'); + + try { + $this->assertSame('stream contents', stream_get_contents($stream)); + $this->assertTrue($primary->missing('stream.txt')); + } finally { + fclose($stream); + } + } + + public function testReadThroughDisksDeleteFilesFromBothDisks(): void + { + $filesystem = $this->readThroughFilesystemManager(); + $primary = $filesystem->disk('primary'); + $fallback = $filesystem->disk('fallback'); + $readThrough = $filesystem->disk('read-through'); + + $fallback->put('file.txt', 'contents'); + $readThrough->get('file.txt'); + + $this->assertTrue($readThrough->delete('file.txt')); + $this->assertTrue($primary->missing('file.txt')); + $this->assertTrue($fallback->missing('file.txt')); + $this->assertTrue($readThrough->missing('file.txt')); + } + + public function testReadThroughDisksDeleteDirectoriesFromBothDisks(): void + { + $filesystem = $this->readThroughFilesystemManager(); + $primary = $filesystem->disk('primary'); + $fallback = $filesystem->disk('fallback'); + $readThrough = $filesystem->disk('read-through'); + + $primary->put('directory/primary.txt', 'primary contents'); + $fallback->put('directory/fallback.txt', 'fallback contents'); + + $this->assertTrue($readThrough->deleteDirectory('directory')); + $this->assertTrue($primary->directoryMissing('directory')); + $this->assertTrue($fallback->directoryMissing('directory')); + $this->assertTrue($readThrough->directoryMissing('directory')); + } + + public function testReadThroughDisksDoNotResurrectMovedFiles(): void + { + $filesystem = $this->readThroughFilesystemManager(); + $primary = $filesystem->disk('primary'); + $fallback = $filesystem->disk('fallback'); + $readThrough = $filesystem->disk('read-through'); + + $fallback->put('source.txt', 'contents'); + $readThrough->get('source.txt'); + + $this->assertTrue($readThrough->move('source.txt', 'destination.txt')); + $this->assertSame('contents', $primary->get('destination.txt')); + $this->assertTrue($primary->missing('source.txt')); + $this->assertTrue($fallback->missing('source.txt')); + $this->assertTrue($readThrough->missing('source.txt')); + } + + public function testReadThroughDisksMoveFilesThatOnlyExistOnTheFallbackDisk(): void + { + $filesystem = $this->readThroughFilesystemManager(); + $primary = $filesystem->disk('primary'); + $fallback = $filesystem->disk('fallback'); + $readThrough = $filesystem->disk('read-through'); + + $fallback->put('source.txt', 'contents'); + + $this->assertTrue($readThrough->move('source.txt', 'destination.txt')); + $this->assertSame('contents', $primary->get('destination.txt')); + $this->assertTrue($fallback->missing('source.txt')); + $this->assertTrue($readThrough->missing('source.txt')); + } + + public function testReadThroughDisksCopyFilesThatOnlyExistOnTheFallbackDisk(): void + { + $filesystem = $this->readThroughFilesystemManager(); + $primary = $filesystem->disk('primary'); + $fallback = $filesystem->disk('fallback'); + $readThrough = $filesystem->disk('read-through'); + + $fallback->put('source.txt', 'contents'); + + $this->assertTrue($readThrough->copy('source.txt', 'destination.txt')); + $this->assertSame('contents', $primary->get('destination.txt')); + $this->assertSame('contents', $fallback->get('source.txt')); + $this->assertSame('contents', $readThrough->get('source.txt')); + } + + public function testReadThroughDisksCopyFromTheFallbackDiskWithoutPromotingTheSourceWhenDisabled(): void + { + $filesystem = $this->readThroughFilesystemManager(['copy' => false]); + $primary = $filesystem->disk('primary'); + $fallback = $filesystem->disk('fallback'); + $readThrough = $filesystem->disk('read-through'); + + $fallback->put('source.txt', 'contents'); + + $this->assertTrue($readThrough->copy('source.txt', 'destination.txt')); + $this->assertSame('contents', $primary->get('destination.txt')); + $this->assertTrue($primary->missing('source.txt')); + $this->assertSame('contents', $fallback->get('source.txt')); + + $this->assertTrue($readThrough->move('source.txt', 'moved.txt')); + $this->assertSame('contents', $primary->get('moved.txt')); + $this->assertTrue($primary->missing('source.txt')); + $this->assertTrue($fallback->missing('source.txt')); + } + + public function testReadThroughDisksFailToMoveOrCopyMissingFiles(): void + { + $readThrough = $this->readThroughFilesystemManager()->disk('read-through'); + + $this->assertFalse($readThrough->move('missing.txt', 'destination.txt')); + $this->assertFalse($readThrough->copy('missing.txt', 'destination.txt')); + } + + public function testReadThroughDisksFailToMoveOrCopyWhenThePrimaryDiskIsUnwritable(): void + { + $filesystem = $this->readThroughFilesystemManager([ + 'primary' => [ + 'driver' => 'local', + 'root' => $this->tempDir . '/read-only-primary', + 'read-only' => true, + ], + ]); + $fallback = $filesystem->disk('fallback'); + $readThrough = $filesystem->disk('read-through'); + $fallback->put('source.txt', 'contents'); + + $this->assertFalse($readThrough->move('source.txt', 'destination.txt')); + $this->assertFalse($readThrough->copy('source.txt', 'destination.txt')); + $this->assertSame('contents', $fallback->get('source.txt')); + } + + public function testReadThroughDiskPromotionFailuresAreBestEffortByDefault(): void + { + $filesystem = $this->readThroughFilesystemManager([ + 'primary' => [ + 'driver' => 'local', + 'root' => $this->tempDir . '/read-only-primary', + 'read-only' => true, + ], + ]); + $filesystem->disk('fallback')->put('fallback.txt', 'fallback contents'); + + $this->assertSame('fallback contents', $filesystem->disk('read-through')->get('fallback.txt')); + + $stream = $filesystem->disk('read-through')->readStream('fallback.txt'); + + try { + $this->assertSame('fallback contents', stream_get_contents($stream)); + } finally { + fclose($stream); + } + } + + public function testReadThroughPromotionCancellationClosesOwnedStreams(): void + { + $source = fopen('php://temp', 'w+b'); + fwrite($source, 'contents'); + rewind($source); + $temporary = null; + $cancellation = new CanceledException('promotion canceled'); + $primary = m::mock(FilesystemOperator::class); + $primary->shouldReceive('fileExists')->twice()->with('file.txt')->andReturn(false); + $primary->shouldReceive('writeStream')->once()->andReturnUsing( + static function (string $path, mixed $stream) use (&$temporary, $cancellation): never { + $temporary = $stream; + + throw $cancellation; + }, + ); + $fallback = m::mock(FilesystemOperator::class); + $fallback->shouldReceive('readStream')->once()->with('file.txt')->andReturn($source); + $adapter = new ReadThroughFilesystemAdapter($primary, $fallback); + + try { + $adapter->readStream('file.txt'); + $this->fail('Expected cancellation to escape promotion.'); + } catch (CanceledException $exception) { + $this->assertSame($cancellation, $exception); + $this->assertFalse(is_resource($source)); + $this->assertNotNull($temporary); + $this->assertFalse(is_resource($temporary)); + } finally { + if (is_resource($source)) { + fclose($source); + } + + if (is_resource($temporary)) { + fclose($temporary); + } + } + } + + public function testReadThroughDiskCanThrowOnPromotionFailures(): void + { + $filesystem = $this->readThroughFilesystemManager([ + 'primary' => [ + 'driver' => 'local', + 'root' => $this->tempDir . '/read-only-primary', + 'read-only' => true, + ], + 'throw' => true, + 'throw_on_promotion_failure' => true, + ]); + $filesystem->disk('fallback')->put('fallback.txt', 'fallback contents'); + + $this->expectException(UnableToReadFile::class); + + $filesystem->disk('read-through')->get('fallback.txt'); + } + + #[RequiresOperatingSystem('Linux|Darwin')] + public function testReadThroughDisksSetVisibilityOnTheDiskContainingTheFile(): void + { + $filesystem = $this->readThroughFilesystemManager(); + $primary = $filesystem->disk('primary'); + $fallback = $filesystem->disk('fallback'); + $readThrough = $filesystem->disk('read-through'); + $fallback->put('fallback.txt', 'fallback contents'); + $primary->put('primary.txt', 'primary contents'); + + $this->assertTrue($readThrough->setVisibility('fallback.txt', 'private')); + $this->assertSame('private', $fallback->getVisibility('fallback.txt')); + $this->assertSame('private', $readThrough->getVisibility('fallback.txt')); + $this->assertTrue($primary->missing('fallback.txt')); + $this->assertTrue($readThrough->setVisibility('primary.txt', 'private')); + $this->assertSame('private', $primary->getVisibility('primary.txt')); + } + + public function testReadThroughDisksDelegateUrlsToTheDiskContainingTheFile(): void + { + $filesystem = $this->readThroughFilesystemManager([], [ + 'url' => 'https://primary.test', + ], [ + 'url' => 'https://fallback.test', + ]); + $primary = $filesystem->disk('primary'); + $fallback = $filesystem->disk('fallback'); + $readThrough = $filesystem->disk('read-through'); + $expiration = CarbonImmutable::create(2026, 8, 11); + + $primary->put('primary.txt', 'primary contents'); + $fallback->put('fallback.txt', 'fallback contents'); + $primary->buildTemporaryUrlsUsing(fn (string $path, DateTimeInterface $expiration, array $options): string => 'primary/' . $path . '/' . $options['version']); + $fallback->buildTemporaryUrlsUsing(fn (string $path, DateTimeInterface $expiration, array $options): string => 'fallback/' . $path . '/' . $options['version']); + $primary->buildTemporaryUploadUrlsUsing(fn (string $path, DateTimeInterface $expiration, array $options): array => [ + 'url' => 'upload/' . $path . '/' . $options['version'], + 'headers' => ['X-Test' => 'header'], + ]); + + $this->assertSame('https://primary.test/primary.txt', $readThrough->url('primary.txt')); + $this->assertSame('https://fallback.test/fallback.txt', $readThrough->url('fallback.txt')); + $this->assertTrue($readThrough->providesTemporaryUrls()); + $this->assertSame('primary/primary.txt/1', $readThrough->temporaryUrl('primary.txt', $expiration, ['version' => 1])); + $this->assertSame('fallback/fallback.txt/1', $readThrough->temporaryUrl('fallback.txt', $expiration, ['version' => 1])); + $this->assertTrue($readThrough->providesTemporaryUploadUrls()); + $this->assertSame([ + 'url' => 'upload/file.txt/1', + 'headers' => ['X-Test' => 'header'], + ], $readThrough->temporaryUploadUrl('file.txt', $expiration, ['version' => 1])); + } + + public function testReadThroughDisksPreserveOuterAndSidePrefixes(): void + { + $filesystem = $this->readThroughFilesystemManager(['prefix' => 'outer', 'copy' => false], [ + 'prefix' => 'primary-prefix', + 'url' => 'https://primary.test', + ], [ + 'prefix' => 'fallback-prefix', + 'url' => 'https://fallback.test', + ]); + $primary = $filesystem->disk('primary'); + $fallback = $filesystem->disk('fallback'); + $primary->put('outer/primary.txt', 'primary contents'); + $fallback->put('outer/fallback.txt', 'fallback contents'); + $primary->buildTemporaryUrlsUsing(fn (string $path): string => 'primary/' . $path); + $fallback->buildTemporaryUrlsUsing(fn (string $path): string => 'fallback/' . $path); + $primary->buildTemporaryUploadUrlsUsing(fn (string $path): array => ['url' => 'upload/' . $path]); + $readThrough = $filesystem->disk('read-through'); + $expiration = new DateTimeImmutable('+1 hour'); + + $this->assertSame('primary contents', $readThrough->get('primary.txt')); + $this->assertSame('fallback contents', $readThrough->get('fallback.txt')); + $this->assertSame('https://primary.test/primary-prefix/outer/primary.txt', $readThrough->url('primary.txt')); + $this->assertSame('https://fallback.test/fallback-prefix/outer/fallback.txt', $readThrough->url('fallback.txt')); + $this->assertSame($primary->path('outer/primary.txt'), $readThrough->path('primary.txt')); + $this->assertSame('primary/outer/primary.txt', $readThrough->temporaryUrl('primary.txt', $expiration)); + $this->assertSame('fallback/outer/fallback.txt', $readThrough->temporaryUrl('fallback.txt', $expiration)); + $this->assertSame(['url' => 'upload/outer/new.txt'], $readThrough->temporaryUploadUrl('new.txt', $expiration)); + } + + public function testReadThroughDisksUseTheirOwnUrlCallbacksBeforeRawAdapterHooks(): void + { + $filesystem = $this->readThroughFilesystemManager(['prefix' => 'outer'], ['driver' => 'custom-url']); + $root = $this->tempDir . '/primary'; + $filesystem->extend('custom-url', static function () use ($root): FilesystemAdapter { + $adapter = new class($root) extends LocalFilesystemAdapter { + /** + * Provide a raw adapter URL that must not override the composite callback. + */ + public function getTemporaryUrl(string $path, DateTimeInterface $expiration, array $options): string + { + return 'raw-adapter-url'; + } + }; + + return new FilesystemAdapter(new Flysystem($adapter), $adapter, ['root' => $root]); + }); + $readThrough = $filesystem->disk('read-through'); + $readThrough->buildTemporaryUrlsUsing(fn (string $path): string => 'composite/' . $path); + + $this->assertSame('composite/file.txt', $readThrough->temporaryUrl('file.txt', new DateTimeImmutable('+1 hour'))); + } + + #[DataProvider('readThroughCycles')] + public function testReadThroughDisksRejectCircularConfiguration(array $disks, string $message): void + { + $disks['legacy'] = ['driver' => 'local', 'root' => $this->tempDir . '/legacy']; + $filesystem = new FilesystemManager($this->getContainer(['disks' => $disks])); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage($message); + + $filesystem->disk('assets'); + } + + /** + * Provide circular read-through configurations. + */ + public static function readThroughCycles(): array + { + return [ + 'self' => [[ + 'assets' => ['driver' => 'read-through', 'primary' => 'assets', 'fallback' => 'legacy'], + ], 'Read-through disk [assets] cannot reference itself.'], + 'two disks' => [[ + 'assets' => ['driver' => 'read-through', 'primary' => 'other', 'fallback' => 'legacy'], + 'other' => ['driver' => 'read-through', 'primary' => 'assets', 'fallback' => 'legacy'], + ], 'Circular read-through disk definition detected: assets -> other -> assets.'], + 'inline scoped side' => [[ + 'assets' => [ + 'driver' => 'read-through', + 'primary' => ['driver' => 'scoped', 'disk' => 'assets', 'prefix' => 'cache'], + 'fallback' => 'legacy', + ], + ], 'Circular read-through disk definition detected: assets -> (on-demand) -> (on-demand).'], + ]; + } + + public function testReadThroughConstructionCanRetryAfterASideFails(): void + { + $filesystem = $this->readThroughFilesystemManager([], ['driver' => 'retry']); + $attempts = 0; + $root = $this->tempDir . '/primary'; + $filesystem->extend('retry', static function () use (&$attempts, $root): FilesystemAdapter { + if (++$attempts === 1) { + throw new RuntimeException('side unavailable'); + } + + $adapter = new LocalFilesystemAdapter($root); + + return new FilesystemAdapter(new Flysystem($adapter), $adapter); + }); + + try { + $filesystem->disk('read-through'); + $this->fail('Expected the first side construction to fail.'); + } catch (RuntimeException $exception) { + $this->assertSame('side unavailable', $exception->getMessage()); + } + + $this->assertTrue($filesystem->disk('read-through')->put('file.txt', 'contents')); + $this->assertSame('contents', $filesystem->disk('primary')->get('file.txt')); + } + public function testCustomDriverClosureBoundObjectIsFilesystemManager(): void { $container = $this->getContainer([ @@ -1722,6 +2165,33 @@ private function s3Config(string $bucket, string $key = 'test-key'): array ]; } + /** + * Create a read-through manager with isolated local disks. + */ + protected function readThroughFilesystemManager( + array $readThroughConfig = [], + array $primaryConfig = [], + array $fallbackConfig = [], + ): FilesystemManager { + return new FilesystemManager($this->getContainer([ + 'disks' => [ + 'primary' => array_replace([ + 'driver' => 'local', + 'root' => $this->tempDir . '/primary', + ], $primaryConfig), + 'fallback' => array_replace([ + 'driver' => 'local', + 'root' => $this->tempDir . '/fallback', + ], $fallbackConfig), + 'read-through' => array_replace([ + 'driver' => 'read-through', + 'primary' => 'primary', + 'fallback' => 'fallback', + ], $readThroughConfig), + ], + ])); + } + protected function getContainer(array $config = []): Container { $config = new Repository(['filesystems' => $config]); diff --git a/tests/Filesystem/FilesystemPoolProxyTest.php b/tests/Filesystem/FilesystemPoolProxyTest.php index fdef382ff1..303605ef5e 100644 --- a/tests/Filesystem/FilesystemPoolProxyTest.php +++ b/tests/Filesystem/FilesystemPoolProxyTest.php @@ -21,6 +21,7 @@ use Hypervel\Testing\ParallelTesting; use League\Flysystem\Filesystem; use League\Flysystem\Local\LocalFilesystemAdapter; +use League\Flysystem\UnableToReadFile; use Mockery as m; use PHPUnit\Framework\Attributes\DataProvider; use RuntimeException; @@ -218,6 +219,28 @@ function (object $filesystem) use (&$releaseCalls): void { $this->assertSame(1, $releaseCalls); } + public function testOperatorReadsPropagateFailuresAndReleaseTheWholeDriver(): void + { + $proxy = $this->proxy(fn (): FilesystemAdapter => $this->filesystem()); + $operator = $proxy->getOperator(); + $operator->write('file.txt', 'contents'); + $stream = $operator->readStream('file.txt'); + + try { + $this->assertSame('contents', stream_get_contents($stream)); + $this->assertSame(1, $this->pools->get('filesystem:driver')->getBorrowedCount()); + } finally { + fclose($stream); + } + + try { + $operator->readStream('missing.txt'); + $this->fail('Expected a raw read failure even though the disk does not throw.'); + } catch (UnableToReadFile $exception) { + $this->assertSame(0, $this->pools->get('filesystem:driver')->getBorrowedCount()); + } + } + public function testBoundedReadStreamKeepsTheWholeDriverBorrowedUntilClose(): void { $this->driver->write('file.txt', '0123456789'); From c78a17de0785fd89eeed5987bc8256c4b64ed223 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:57:13 +0000 Subject: [PATCH 2/2] Preserve upload streams and safe cloud defaults Keep the read-through promotion stream open when an upload adapter closes its input, including the Google Cloud Storage client. Give uploads a separate resource handle and detach its wrapper after use. Close streams conditionally in fallback copy/move and putFileAs so a completed upload cannot fail during cleanup. Use one pooled filesystem operator implementation for both client and whole-driver pools. Preserve native cloud reads, range requests, HTTP options and stream leases without applying the inner disk's error policy. Declare the filesystem package's direct context dependency. Remove the shipped public visibility overrides for S3 and GCS, keeping private adapter defaults and explicit public opt-in. Clarify destination visibility, dynamic scoping, concurrent mutation and promotion failure semantics in the read-through documentation. Extend coverage for closing upload adapters, actual Google client stream ownership with a fake HTTP transport, whole-driver cloud ranges and leases, shipped-config write ACLs, and cancellation cleanup. Keep these corrections together because their regression cases share complete test files. Follow-up to https://github.com/laravel/framework/pull/61140 Source: laravel/framework master cd6e81dff3ba7a4564ac88c3949698c728d20109 Validation: changed test files and affected parallel filesystem, storage, Sentry, facade and foundation suites pass. Full static analysis, formatting, facade regeneration and diff checks pass. --- src/docs/filesystem.md | 11 +- src/filesystem/composer.json | 1 + src/filesystem/src/ClientPooledFilesystem.php | 21 --- .../InteractsWithPooledFilesystem.php | 11 ++ src/filesystem/src/FilesystemAdapter.php | 4 +- .../src/ReadThroughFilesystemAdapter.php | 19 ++- src/foundation/config/filesystems.php | 4 +- tests/Filesystem/AwsS3V3AdapterTest.php | 58 +++++--- tests/Filesystem/FilesystemAdapterTest.php | 27 +++- tests/Filesystem/FilesystemManagerTest.php | 131 ++++++++++++++++++ .../GoogleCloudStorageAdapterTest.php | 58 ++++++++ tests/Foundation/FoundationConfigTest.php | 15 +- 12 files changed, 302 insertions(+), 58 deletions(-) diff --git a/src/docs/filesystem.md b/src/docs/filesystem.md index 15b84066cb..1ea6e9d732 100644 --- a/src/docs/filesystem.md +++ b/src/docs/filesystem.md @@ -207,7 +207,6 @@ If you need to configure a Google Cloud Storage filesystem manually, you may use 'path_prefix' => env('GOOGLE_CLOUD_STORAGE_PATH_PREFIX', ''), 'storage_api_uri' => env('GOOGLE_CLOUD_STORAGE_API_URI'), 'api_endpoint' => env('GOOGLE_CLOUD_STORAGE_API_ENDPOINT'), - 'visibility' => 'public', 'visibility_handler' => null, 'metadata' => ['cacheControl' => 'public,max-age=86400'], 'throw' => false, @@ -370,11 +369,17 @@ Read-through disks allow you to migrate files between disks without downtime. Wh ], ``` -New files and directory listings use the primary disk. File existence and metadata checks use either disk without copying files. Deletions remove files or directories from both disks, and visibility changes apply to the disk containing the file. The `primary` and `fallback` options may also contain inline disk configurations. +New files and directory listings use the primary disk. URLs, file existence checks, and metadata use the disk containing the file without copying it. Deletions remove files or directories from both disks, and visibility changes apply to the disk containing the file. The `primary` and `fallback` options may also contain inline disk configurations. + +To scope a read-through disk per request or tenant, wrap it in `ScopedCloudFilesystemProxy`. Dynamic scoped proxies cannot be used as its primary or fallback disk. Fallback reads promote files by default. Set `copy` to `false` to read fallback files without copying them. With promotion enabled, fallback stream reads finish copying the file before returning the stream. -If promotion fails, the read still succeeds by default. Set `throw_on_promotion_failure` to `true` to treat promotion failures as read failures; set the disk's `throw` option to `true` to receive those failures as exceptions. +Promoted files use the primary disk's default visibility rather than inheriting the fallback file's visibility. Fallback copy and move operations use the same default. Configure a private primary disk when migrating private files. + +Promotion does not lock files across the two disks. Coordinate writes and deletions to a path while it is being copied; otherwise, promotion can overwrite a concurrent write or restore a deleted file. + +By default, a `FilesystemException` raised while writing the promoted copy does not fail the read. Set `throw_on_promotion_failure` to `true` to treat it as a read failure; set the disk's `throw` option to `true` to receive that failure as an exception. Other errors, including pool wait timeouts, still propagate. ### Amazon S3 Compatible Filesystems diff --git a/src/filesystem/composer.json b/src/filesystem/composer.json index 77b1f6baad..ab7cbf7f68 100644 --- a/src/filesystem/composer.json +++ b/src/filesystem/composer.json @@ -30,6 +30,7 @@ "hypervel/collections": "^0.4", "hypervel/conditionable": "^0.4", "hypervel/container": "^0.4", + "hypervel/context": "^0.4", "hypervel/contracts": "^0.4", "hypervel/coroutine": "^0.4", "hypervel/http": "^0.4", diff --git a/src/filesystem/src/ClientPooledFilesystem.php b/src/filesystem/src/ClientPooledFilesystem.php index 7fb3359656..d24ac330ee 100644 --- a/src/filesystem/src/ClientPooledFilesystem.php +++ b/src/filesystem/src/ClientPooledFilesystem.php @@ -11,7 +11,6 @@ use Hypervel\Filesystem\Concerns\InteractsWithPooledFilesystem; use Hypervel\ObjectPool\Lease; use Hypervel\ObjectPool\PoolDefinition; -use League\Flysystem\FilesystemOperator; use RuntimeException; use Throwable; @@ -52,26 +51,6 @@ public function getPoolName(): string return $this->definition->identity; } - /** - * Get an operator that preserves native cloud reads and their client leases. - */ - public function getOperator(): FilesystemOperator - { - return new FilesystemOperatorAdapter( - $this->withDriver(...), - fn (string $path): mixed => $this->leasedStream(static function (FilesystemAdapter $stack) use ($path): mixed { - return $stack instanceof AwsS3V3Adapter || $stack instanceof GoogleCloudStorageAdapter - ? $stack->readStreamRangeOrFail($path) - : $stack->getDriver()->readStream($path); - }), - fn (string $path, ?int $start, ?int $end): mixed => $this->leasedStream( - static fn (FilesystemAdapter $stack): mixed => $stack instanceof AwsS3V3Adapter || $stack instanceof GoogleCloudStorageAdapter - ? $stack->readStreamRangeOrFail($path, $start, $end) - : null, - ), - ); - } - /** * Remove and close the current client pool. */ diff --git a/src/filesystem/src/Concerns/InteractsWithPooledFilesystem.php b/src/filesystem/src/Concerns/InteractsWithPooledFilesystem.php index a59b0085e6..2d98031201 100644 --- a/src/filesystem/src/Concerns/InteractsWithPooledFilesystem.php +++ b/src/filesystem/src/Concerns/InteractsWithPooledFilesystem.php @@ -9,8 +9,10 @@ use DateTimeInterface; use Hypervel\Container\Container; use Hypervel\Contracts\Filesystem\Filesystem as FilesystemContract; +use Hypervel\Filesystem\AwsS3V3Adapter; use Hypervel\Filesystem\FileResponseBuilder; use Hypervel\Filesystem\FilesystemOperatorAdapter; +use Hypervel\Filesystem\GoogleCloudStorageAdapter; use Hypervel\Http\File; use Hypervel\Http\Request; use Hypervel\Http\UploadedFile; @@ -536,6 +538,10 @@ public function getOperator(): FilesystemOperator return new FilesystemOperatorAdapter( $this->withDriver(...), fn (string $path): mixed => $this->leasedStream(static function (FilesystemContract $filesystem) use ($path): mixed { + if ($filesystem instanceof AwsS3V3Adapter || $filesystem instanceof GoogleCloudStorageAdapter) { + return $filesystem->readStreamRangeOrFail($path); + } + if (! method_exists($filesystem, 'getDriver')) { throw new RuntimeException( 'Pooled filesystem driver [' . $filesystem::class . '] does not support [getDriver] access.', @@ -544,6 +550,11 @@ public function getOperator(): FilesystemOperator return $filesystem->getDriver()->readStream($path); }), + fn (string $path, ?int $start, ?int $end): mixed => $this->leasedStream( + static fn (FilesystemContract $filesystem): mixed => $filesystem instanceof AwsS3V3Adapter || $filesystem instanceof GoogleCloudStorageAdapter + ? $filesystem->readStreamRangeOrFail($path, $start, $end) + : null, + ), ); } diff --git a/src/filesystem/src/FilesystemAdapter.php b/src/filesystem/src/FilesystemAdapter.php index 0e79407bf9..b86abc813a 100644 --- a/src/filesystem/src/FilesystemAdapter.php +++ b/src/filesystem/src/FilesystemAdapter.php @@ -451,7 +451,9 @@ public function putFileAs(string|File|UploadedFile $path, array|string|File|Uplo try { $result = $this->put($path, $stream, $options); } finally { - @fclose($stream); + if (is_resource($stream)) { + fclose($stream); + } } return $result ? $path : false; diff --git a/src/filesystem/src/ReadThroughFilesystemAdapter.php b/src/filesystem/src/ReadThroughFilesystemAdapter.php index 3bb1b58618..d81032a2d2 100644 --- a/src/filesystem/src/ReadThroughFilesystemAdapter.php +++ b/src/filesystem/src/ReadThroughFilesystemAdapter.php @@ -4,6 +4,8 @@ namespace Hypervel\Filesystem; +use GuzzleHttp\Psr7\StreamWrapper; +use GuzzleHttp\Psr7\Utils; use League\Flysystem\Config; use League\Flysystem\FileAttributes; use League\Flysystem\FilesystemAdapter; @@ -112,10 +114,21 @@ public function readStream(string $path): mixed return $this->primary->readStream($path); } + // Upload adapters may close their input; retain our copy for the caller. + $upload = Utils::streamFor($temporary); + $uploadStream = null; + try { - $this->primary->writeStream($path, $temporary); + $uploadStream = StreamWrapper::getResource($upload); + $this->primary->writeStream($path, $uploadStream); } catch (FilesystemException $exception) { $this->handlePromotionFailure($path, $exception); + } finally { + if (is_resource($uploadStream)) { + fclose($uploadStream); + } + + $upload->detach(); } rewind($temporary); @@ -287,7 +300,9 @@ protected function copyFromFallback(string $source, string $destination, Config try { $this->primary->writeStream($destination, $stream, $config->toArray()); } finally { - fclose($stream); + if (is_resource($stream)) { + fclose($stream); + } } } diff --git a/src/foundation/config/filesystems.php b/src/foundation/config/filesystems.php index 876195d40f..330e5a079f 100644 --- a/src/foundation/config/filesystems.php +++ b/src/foundation/config/filesystems.php @@ -27,7 +27,7 @@ | | Supported drivers: "local", "ftp", "sftp", "s3", "gcs" | - | The built-in disks declare their default visibility and whether storage + | Disk options control default visibility and whether storage | failures should be thrown or reported. S3-compatible services may also | require a custom endpoint, path-style URLs, or provider-specific region. | @@ -64,7 +64,6 @@ 'url' => env('AWS_URL'), 'endpoint' => env('AWS_ENDPOINT'), 'use_path_style_endpoint' => (bool) env('AWS_USE_PATH_STYLE_ENDPOINT', false), - 'visibility' => 'public', 'throw' => false, 'report' => false, 'stream_reads' => true, @@ -87,7 +86,6 @@ 'path_prefix' => env('GOOGLE_CLOUD_STORAGE_PATH_PREFIX', ''), // optional: /default/path/to/apply/in/bucket 'storage_api_uri' => env('GOOGLE_CLOUD_STORAGE_API_URI', null), // see: Public URLs below 'api_endpoint' => env('GOOGLE_CLOUD_STORAGE_API_ENDPOINT', null), // set storageClient apiEndpoint - 'visibility' => 'public', // optional: public|private 'visibility_handler' => null, // optional: set to \League\Flysystem\GoogleCloudStorage\UniformBucketLevelAccessVisibility::class to enable uniform bucket level access 'metadata' => ['cacheControl' => 'public,max-age=86400'], // optional: default metadata 'throw' => false, diff --git a/tests/Filesystem/AwsS3V3AdapterTest.php b/tests/Filesystem/AwsS3V3AdapterTest.php index a1e184424f..84b7a913ac 100644 --- a/tests/Filesystem/AwsS3V3AdapterTest.php +++ b/tests/Filesystem/AwsS3V3AdapterTest.php @@ -15,6 +15,7 @@ use Hypervel\Filesystem\ClientPooledFilesystem; use Hypervel\Filesystem\FilesystemAdapter; use Hypervel\Filesystem\FilesystemManager; +use Hypervel\Filesystem\FilesystemPoolProxy; use Hypervel\ObjectPool\PoolDefinition; use Hypervel\ObjectPool\PoolManager; use Hypervel\ObjectPool\PoolOptions; @@ -25,6 +26,7 @@ use League\Flysystem\UnableToReadFile; use Mockery as m; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\TestWith; use Psr\Http\Message\StreamInterface; use RuntimeException; use Swoole\Coroutine\CanceledException; @@ -216,7 +218,9 @@ function (CommandInterface $command) use (&$captured): Result { $this->assertSame(12, $captured['@http']['timeout']); } - public function testStreamReadsPreserveConfiguredHttpSiblingsForPlainAndRangeReads(): void + #[TestWith([false])] + #[TestWith([true])] + public function testStreamReadsPreserveConfiguredHttpSiblingsForPlainAndRangeReads(bool $wholeDriver): void { $commands = []; $handler = new MockHandler([ @@ -256,8 +260,14 @@ function (CommandInterface $command) use (&$commands): Result { fclose($plain); fclose($range); $pools = new PoolManager; - $pooled = new ClientPooledFilesystem( - new PoolDefinition('filesystem:http-options', 's3', 'test', PoolOptions::fromArray([])), + $definition = new PoolDefinition('filesystem:http-options', 's3', 'test', PoolOptions::fromArray([])); + $pooled = $wholeDriver ? new FilesystemPoolProxy( + $definition, + static fn (): AwsS3V3Adapter => $adapter, + $pools, + $adapter->getConfig(), + ) : new ClientPooledFilesystem( + $definition, $adapter->getClient(...), static fn (object $client): AwsS3V3Adapter => $adapter, $pools, @@ -291,7 +301,7 @@ function (CommandInterface $command) use (&$commands): Result { } #[DataProvider('readThroughCloudSides')] - public function testReadThroughRangesPreserveNativeRequestsAndTheOuterFailurePolicy(bool $primary, bool $pooled): void + public function testReadThroughRangesPreserveNativeRequestsAndTheOuterFailurePolicy(bool $primary, ?string $poolType): void { $command = null; $failure = new RuntimeException('S3 failed'); @@ -315,13 +325,23 @@ function (CommandInterface $request) use (&$command): Result { $emptyDriver->shouldReceive('fileExists')->andReturn(false); $empty = new FilesystemAdapter($emptyDriver, m::mock(FlysystemAdapter::class)); $pools = new PoolManager; - $cloud = $pooled ? new ClientPooledFilesystem( - new PoolDefinition('filesystem:read-through-range', 's3', 'test', PoolOptions::fromArray([])), - $adapter->getClient(...), - static fn (object $client): AwsS3V3Adapter => $adapter, - $pools, - $adapter->getConfig(), - ) : $adapter; + $definition = new PoolDefinition('filesystem:read-through-range', 's3', 'test', PoolOptions::fromArray([])); + $cloud = match ($poolType) { + 'client' => new ClientPooledFilesystem( + $definition, + $adapter->getClient(...), + static fn (object $client): AwsS3V3Adapter => $adapter, + $pools, + $adapter->getConfig(), + ), + 'driver' => new FilesystemPoolProxy( + $definition, + static fn (): AwsS3V3Adapter => $adapter, + $pools, + $adapter->getConfig(), + ), + null => $adapter, + }; $exceptionHandler = m::mock(ExceptionHandler::class); $exceptionHandler->shouldNotReceive('report'); Container::getInstance()->instance(ExceptionHandler::class, $exceptionHandler); @@ -344,8 +364,8 @@ function (CommandInterface $request) use (&$command): Result { try { $this->assertSame('range', stream_get_contents($stream)); - if ($pooled) { - $this->assertSame(1, $pools->get($cloud->getPoolName())->getBorrowedCount()); + if ($poolType !== null) { + $this->assertSame(1, $pools->get($definition->identity)->getBorrowedCount()); } } finally { fclose($stream); @@ -362,8 +382,8 @@ function (CommandInterface $request) use (&$command): Result { $this->assertSame($failure, $exception->getPrevious()); } - if ($pooled) { - $this->assertSame(0, $pools->get($cloud->getPoolName())->getBorrowedCount()); + if ($poolType !== null) { + $this->assertSame(0, $pools->get($definition->identity)->getBorrowedCount()); } } finally { $pools->purgeAll(); @@ -376,9 +396,11 @@ function (CommandInterface $request) use (&$command): Result { public static function readThroughCloudSides(): array { return [ - 'pooled primary' => [true, true], - 'pooled fallback without promotion' => [false, true], - 'non-pooled primary' => [true, false], + 'client-pooled primary' => [true, 'client'], + 'client-pooled fallback without promotion' => [false, 'client'], + 'driver-pooled primary' => [true, 'driver'], + 'driver-pooled fallback without promotion' => [false, 'driver'], + 'non-pooled primary' => [true, null], ]; } diff --git a/tests/Filesystem/FilesystemAdapterTest.php b/tests/Filesystem/FilesystemAdapterTest.php index 7d87a4ffa7..9b7fa6bfe7 100644 --- a/tests/Filesystem/FilesystemAdapterTest.php +++ b/tests/Filesystem/FilesystemAdapterTest.php @@ -883,16 +883,22 @@ public function testPutFileAsThrowsWhenTheSourceCannotBeOpenedAndExceptionsAreEn $filesystemAdapter->putFileAs('/', $this->tempDir . '/missing.txt', 'new.txt'); } - public function testPutFileAsClosesTheSourceWhenWritingThrows(): void + #[TestWith([false])] + #[TestWith([true])] + public function testPutFileAsClosesTheSourceWhenWritingThrows(bool $writerClosesStream): void { file_put_contents($filePath = $this->tempDir . '/foo.txt', 'normal file content'); $stream = null; $filesystemAdapter = m::mock(FilesystemAdapter::class, [$this->filesystem, $this->adapter])->makePartial(); $filesystemAdapter->shouldReceive('put')->once()->andReturnUsing( - function (string $path, mixed $contents, mixed $options) use (&$stream): never { + function (string $path, mixed $contents, mixed $options) use (&$stream, $writerClosesStream): never { $stream = $contents; + if ($writerClosesStream) { + fclose($stream); + } + throw UnableToWriteFile::atLocation($path); } ); @@ -905,6 +911,23 @@ function (string $path, mixed $contents, mixed $options) use (&$stream): never { } } + public function testPutFileAsAllowsTheDriverToCloseTheSource(): void + { + file_put_contents($filePath = $this->tempDir . '/foo.txt', 'normal file content'); + $filesystemAdapter = m::mock(FilesystemAdapter::class, [$this->filesystem, $this->adapter])->makePartial(); + $filesystemAdapter->shouldReceive('put')->once()->andReturnUsing( + function (string $path, mixed $contents, mixed $options): bool { + $this->assertSame('new.txt', $path); + $this->assertSame('normal file content', stream_get_contents($contents)); + fclose($contents); + + return true; + }, + ); + + $this->assertSame('new.txt', $filesystemAdapter->putFileAs('/', $filePath, 'new.txt')); + } + public function testPutFile() { file_put_contents($filePath = $this->tempDir . '/foo.txt', 'uploaded file content'); diff --git a/tests/Filesystem/FilesystemManagerTest.php b/tests/Filesystem/FilesystemManagerTest.php index 13f905ffee..3c4e475ea3 100644 --- a/tests/Filesystem/FilesystemManagerTest.php +++ b/tests/Filesystem/FilesystemManagerTest.php @@ -4,6 +4,9 @@ namespace Hypervel\Tests\Filesystem; +use Aws\CommandInterface; +use Aws\MockHandler; +use Aws\Result; use Aws\S3\S3Client; use DateTimeImmutable; use DateTimeInterface; @@ -22,6 +25,7 @@ use Hypervel\Filesystem\FilesystemPoolProxy; use Hypervel\Filesystem\GoogleCloudStorageAdapter; use Hypervel\Filesystem\ReadThroughFilesystemAdapter; +use Hypervel\Foundation\Application; use Hypervel\ObjectPool\PoolFingerprint; use Hypervel\ObjectPool\PoolManager; use Hypervel\Support\CarbonImmutable; @@ -40,6 +44,7 @@ use Mockery as m; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\RequiresOperatingSystem; +use PHPUnit\Framework\Attributes\TestWith; use ReflectionProperty; use RuntimeException; use stdClass; @@ -824,6 +829,7 @@ public function testReadThroughDiskPromotionFailuresAreBestEffortByDefault(): vo public function testReadThroughPromotionCancellationClosesOwnedStreams(): void { + $streamCount = count(get_resources('stream')); $source = fopen('php://temp', 'w+b'); fwrite($source, 'contents'); rewind($source); @@ -850,6 +856,7 @@ static function (string $path, mixed $stream) use (&$temporary, $cancellation): $this->assertFalse(is_resource($source)); $this->assertNotNull($temporary); $this->assertFalse(is_resource($temporary)); + $this->assertCount($streamCount, get_resources('stream')); } finally { if (is_resource($source)) { fclose($source); @@ -861,6 +868,77 @@ static function (string $path, mixed $stream) use (&$temporary, $cancellation): } } + #[DataProvider('closingPromotionResults')] + public function testReadThroughPromotionRetainsItsStreamWhenThePrimaryClosesItsInput(bool $fails): void + { + $filesystem = $this->readThroughFilesystemManager(); + $fallback = $filesystem->disk('fallback'); + $fallback->put('file.txt', 'fallback contents'); + $primary = m::mock(FilesystemOperator::class); + $primary->shouldReceive('fileExists')->twice()->with('file.txt')->andReturn(false); + $primary->shouldReceive('writeStream')->once()->andReturnUsing( + function (string $path, mixed $stream) use ($fails): void { + $this->assertSame('file.txt', $path); + $this->assertSame('fallback contents', stream_get_contents($stream)); + fclose($stream); + + if ($fails) { + throw UnableToWriteFile::atLocation($path); + } + }, + ); + $adapter = new ReadThroughFilesystemAdapter($primary, $fallback->getDriver()); + $stream = $adapter->readStream('file.txt'); + + try { + $this->assertIsResource($stream); + $this->assertSame('fallback contents', stream_get_contents($stream)); + } finally { + if (is_resource($stream)) { + fclose($stream); + } + } + } + + /** + * Provide upload outcomes that close the supplied stream. + */ + public static function closingPromotionResults(): array + { + return ['success' => [false], 'failure' => [true]]; + } + + #[DataProvider('fallbackTransferMethods')] + public function testReadThroughFallbackTransfersAllowThePrimaryToCloseItsInput(string $method): void + { + $filesystem = $this->readThroughFilesystemManager(); + $fallback = $filesystem->disk('fallback'); + $fallback->put('source.txt', 'contents'); + $primary = m::mock(FilesystemOperator::class); + $primary->shouldReceive('fileExists')->once()->with('source.txt')->andReturn(false); + $primary->shouldReceive('writeStream')->once()->andReturnUsing( + function (string $path, mixed $stream, array $config): void { + $this->assertSame('destination.txt', $path); + $this->assertSame('contents', stream_get_contents($stream)); + fclose($stream); + }, + ); + $adapter = new ReadThroughFilesystemAdapter($primary, $fallback->getDriver()); + $driver = new Flysystem($adapter); + + $driver->{$method}('source.txt', 'destination.txt'); + + $this->assertSame($method === 'copy', $fallback->exists('source.txt')); + } + + /** + * Provide transfers from a fallback disk. + */ + public static function fallbackTransferMethods(): array + { + return [['copy'], ['move']]; + } + public function testReadThroughDiskCanThrowOnPromotionFailures(): void { $filesystem = $this->readThroughFilesystemManager([ @@ -1962,6 +2040,59 @@ public function testS3DiskDefaultsStreamingReadsAndAllowsOptOut(): void $this->assertFalse((new ReflectionProperty(FlysystemS3Adapter::class, 'streamReads'))->getValue($disabled->getAdapter())); } + #[TestWith(['s3', null, 'private'])] + #[TestWith(['s3', 'public', 'public-read'])] + #[TestWith(['gcs', null, 'projectPrivate'])] + #[TestWith(['gcs', 'public', 'publicRead'])] + public function testShippedCloudDisksWritePrivatelyUnlessPublicVisibilityIsRequested( + string $driver, + ?string $visibility, + string $expectedAcl, + ): void { + // The shipped configuration resolves storage_path() through the application. + new Application($this->tempDir); + $config = (require __DIR__ . '/../../src/foundation/config/filesystems.php')['disks'][$driver]; + $config['bucket'] = 'documents'; + + if ($visibility !== null) { + $config['visibility'] = $visibility; + } + + $filesystem = new InspectableFilesystemManager($this->getContainer()); + + if ($driver === 's3') { + $client = new S3Client([ + 'credentials' => false, + 'region' => 'us-east-1', + 'version' => 'latest', + 'handler' => new MockHandler([ + function (CommandInterface $command) use ($expectedAcl): Result { + $this->assertSame('PutObject', $command->getName()); + $this->assertSame($expectedAcl, $command['ACL']); + + return new Result; + }, + ]), + ]); + $disk = $filesystem->buildS3DiskForTest($client, $config); + } else { + $bucket = m::mock(Bucket::class); + $bucket->shouldReceive('upload')->once()->withArgs( + function (string $contents, array $options) use ($expectedAcl): bool { + $this->assertSame('contents', $contents); + $this->assertSame($expectedAcl, $options['predefinedAcl']); + + return true; + }, + ); + $client = m::mock(GcsClient::class); + $client->shouldReceive('bucket')->once()->with('documents')->andReturn($bucket); + $disk = $filesystem->buildGcsDiskForTest($client, $config); + } + + $this->assertTrue($disk->put('file.txt', 'contents')); + } + public function testGcsClientConfigSupportsFlatKeysAndTheFullExplicitSdkSurface(): void { $filesystem = new InspectableFilesystemManager($this->getContainer()); diff --git a/tests/Filesystem/GoogleCloudStorageAdapterTest.php b/tests/Filesystem/GoogleCloudStorageAdapterTest.php index c50c4cf1b0..443123e60e 100644 --- a/tests/Filesystem/GoogleCloudStorageAdapterTest.php +++ b/tests/Filesystem/GoogleCloudStorageAdapterTest.php @@ -4,19 +4,26 @@ namespace Hypervel\Tests\Filesystem; +use Google\Cloud\Core\AnonymousCredentials; use Google\Cloud\Storage\Bucket; use Google\Cloud\Storage\StorageClient; use Google\Cloud\Storage\StorageObject; +use GuzzleHttp\Exception\RequestException; +use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Utils; use Hypervel\Container\Container; use Hypervel\Contracts\Debug\ExceptionHandler; use Hypervel\Filesystem\GoogleCloudStorageAdapter; +use Hypervel\Filesystem\ReadThroughFilesystemAdapter; use Hypervel\Tests\TestCase; use InvalidArgumentException; +use League\Flysystem\Filesystem; use League\Flysystem\FilesystemOperator; use League\Flysystem\GoogleCloudStorage\GoogleCloudStorageAdapter as FlysystemGoogleCloudAdapter; use League\Flysystem\UnableToReadFile; use Mockery as m; +use PHPUnit\Framework\Attributes\TestWith; +use Psr\Http\Message\RequestInterface; use Psr\Http\Message\StreamInterface; use RuntimeException; use Swoole\Coroutine\CanceledException; @@ -25,6 +32,57 @@ class GoogleCloudStorageAdapterTest extends TestCase { + #[TestWith([false])] + #[TestWith([true])] + public function testReadThroughPromotionRetainsItsStreamAfterGoogleUploads(bool $fails): void + { + $uploads = 0; + $client = new StorageClient([ + 'projectId' => 'test-project', + 'credentialsFetcher' => new AnonymousCredentials, + 'retries' => 0, + 'httpHandler' => function (RequestInterface $request, array $options) use (&$uploads, $fails): Response { + if ($request->getMethod() === 'HEAD') { + throw RequestException::create($request, new Response(404)); + } + + $this->assertSame('POST', $request->getMethod()); + $this->assertStringContainsString('fallback contents', (string) $request->getBody()); + ++$uploads; + + if ($fails) { + throw RequestException::create($request, new Response(400)); + } + + return new Response(200, ['Content-Type' => 'application/json'], '{"name":"file.txt","generation":"1"}'); + }, + ]); + $primary = new Filesystem(new FlysystemGoogleCloudAdapter($client->bucket('test-bucket'))); + $source = fopen('php://temp', 'w+b'); + fwrite($source, 'fallback contents'); + rewind($source); + $fallback = m::mock(FilesystemOperator::class); + $fallback->shouldReceive('readStream')->once()->with('file.txt')->andReturn($source); + $adapter = new ReadThroughFilesystemAdapter($primary, $fallback); + $stream = null; + + try { + $stream = $adapter->readStream('file.txt'); + + $this->assertSame(1, $uploads); + $this->assertIsResource($stream); + $this->assertSame('fallback contents', stream_get_contents($stream)); + } finally { + if (is_resource($stream)) { + fclose($stream); + } + + if (is_resource($source)) { + fclose($source); + } + } + } + public function testUrlUsesTheConfiguredApiUriOrBucketEndpoint(): void { $client = m::mock(StorageClient::class); diff --git a/tests/Foundation/FoundationConfigTest.php b/tests/Foundation/FoundationConfigTest.php index accc2bd276..70bdc15994 100644 --- a/tests/Foundation/FoundationConfigTest.php +++ b/tests/Foundation/FoundationConfigTest.php @@ -327,17 +327,16 @@ public static function inheritedPoolDurations(): array ]; } - public function testShippedFilesystemDisksDeclareVisibilityAndFailurePolicy(): void + public function testShippedFilesystemDisksUseSafeVisibilityDefaultsAndDeclareFailurePolicy(): void { $disks = $this->filesystemConfig()['disks']; - foreach ([ - 'local' => 'private', - 'public' => 'public', - 's3' => 'public', - 'gcs' => 'public', - ] as $name => $visibility) { - $this->assertSame($visibility, $disks[$name]['visibility']); + $this->assertSame('private', $disks['local']['visibility']); + $this->assertSame('public', $disks['public']['visibility']); + $this->assertArrayNotHasKey('visibility', $disks['s3']); + $this->assertArrayNotHasKey('visibility', $disks['gcs']); + + foreach (['local', 'public', 's3', 'gcs'] as $name) { $this->assertFalse($disks[$name]['throw']); $this->assertFalse($disks[$name]['report']); }