feat: streaming Put/Get, conditional writes and aws-sdk-go-v2 (#33)

.github/workflows/go.yml

@@ -1,28 +1,48 @@
-# This workflow will build a golang project
-# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
-
 name: Go
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "v2" ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "v2" ]
 
 jobs:
-
+  test:
-  build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
 
       - name: Set up Go
-      uses: actions/setup-go@v4
+        uses: actions/setup-go@v7
         with:
-        go-version: '1.21.0'
+          go-version: '1.25'
 
       - name: Build
         run: go build -v ./...
 
-    - name: Test
+      - name: Vet
-      run: go test -v ./...
+        run: go vet ./... && go vet -tags integration ./...
+
+      - name: Unit tests
+        run: go test -race -count=1 ./...
+
+  integration:
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.25'
+
+      - name: Pull images
+        run: |
+          docker pull pgsty/minio:latest
+          docker pull mcr.microsoft.com/azure-storage/azurite:latest
+          docker pull fsouza/fake-gcs-server:latest
+          docker pull delfer/alpine-ftp-server:latest
+
+      - name: Integration tests (MinIO, Azurite, fake-gcs-server, FTP)
+        run: go test -tags integration -count=1 -v -run Integration ./...

.github/workflows/lint.yml

@@ -15,9 +15,9 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: '~1.24'
+          go-version: '1.25'
 
       - name: Run linter
         uses: golangci/golangci-lint-action@v7
         with:
-          version: v2.1.2
+          version: v2.13.2

.golangci.yml

@@ -32,6 +32,10 @@ linters:
           - dupl
           - lll
         path: /*
+      # Legacy v0.1 API: deprecated as a whole and frozen.
+      - linters:
+          - staticcheck
+        path: ^pkg/
     paths:
       - third_party$
       - builtin$

README.md

@@ -1,126 +1,200 @@
 # Go Storage
-A simple Go storage client
 
-Supported storage:
+A streaming storage abstraction for Go backup tools.
-- Local
-- S3
-- SSH
-- FTP
-- Azure Blob
 
-```go
+Supported backends:
+
+- Local filesystem
+- S3 and S3-compatible services (MinIO, Ceph, Wasabi, ...) via aws-sdk-go-v2
+- Azure Blob Storage (and Azurite)
+- Google Cloud Storage
+- SFTP
+- FTP / explicit FTPS
+
+```shell
 go get github.com/jkaninda/go-storage
 ```
 
-### Local Storage
+## Streaming API
+
+Every backend implements the same `Storage` interface:
+
 ```go
-	localStorage := local.NewStorage(local.Config{
+type Storage interface {
-		LocalPath:  tmpPath,
+	Upload(ctx context.Context, localPath, remotePath string) error
-		RemotePath: backupDestination,
+	Download(ctx context.Context, remotePath, localPath string) error
-	})
+	List(ctx context.Context, prefix string) ([]Item, error) // recursive
-	err = localStorage.Copy(finalFileName)
+	Remove(ctx context.Context, remotePath string) error     // missing -> ErrNotFound
-	if err != nil {
+	Close() error
-		log.Fatal("Error copying file, error %v", err)
+
+	Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error)
+	Get(ctx context.Context, key string) (io.ReadCloser, Item, error)
+	Stat(ctx context.Context, key string) (Item, error)
+	PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error)
+	ListDir(ctx context.Context, dir string) ([]Item, error) // non-recursive, files only
 }
 ```
 
-### S3 Storage
+- **`Put`** streams a reader of unknown length with bounded memory (no temp
+  files, no whole-file buffering). If the reader fails or the context is
+  cancelled, `Put` returns an error and **no object (and no partial object)
+  is left at the key**: S3 aborts the multipart upload, Azure never commits
+  the block list, GCS cancels the resumable upload, and SFTP/FTP/local write
+  to `<key>.<random>.partial` and rename on success.
+- **`Get`** returns a streaming body; always `Close` it. On FTP the body holds
+  the single control connection, so close it before making another call.
+- **`PutIf`** is a conditional write for small files such as a backup index
+  (optimistic concurrency): `Condition{IfNoneMatch: true}` creates only if the
+  key does not exist, `Condition{IfMatch: item.ETag}` replaces only if the
+  object is unchanged. It returns `ErrPreconditionFailed` when the condition
+  does not hold and `ErrConditionNotSupported` when the backend cannot
+  evaluate it (fall back to locking).
+- Errors wrap the sentinels `ErrNotFound`, `ErrPreconditionFailed` and
+  `ErrConditionNotSupported`; test them with `errors.Is`.
+- Keys are relative to the backend base (bucket, container, remote directory
+  or local base path), use `/` separators and may not escape the base (`..`).
+
+| Backend | `Item.ETag`                    | `PutIf` IfNoneMatch | `PutIf` IfMatch |
+|---------|--------------------------------|---------------------|-----------------|
+| S3      | S3 ETag                        | yes                 | yes             |
+| Azure   | blob ETag                      | yes                 | yes             |
+| GCS     | object generation              | yes                 | yes             |
+| Local   | `<mtime-ns hex>-<size hex>`    | yes (lock file)     | yes (lock file) |
+| SFTP    | empty                          | yes (`O_EXCL`)      | not supported   |
+| FTP     | empty                          | not supported       | not supported   |
+
+S3 `PutIf` uses a single `PutObject` request and is limited to 16 MiB.
+
+### Example: dump, compress and upload without temp files
+
 ```go
-s3Storage, err := s3.NewStorage(s3.Config{
+package main
-    Endpoint:       "",
+
-    Bucket:         "",
+import (
-    AccessKey:      "",
+	"compress/gzip"
-    SecretKey:      "",
+	"context"
-    Region:         "",
+	"errors"
-    DisableSsl:     "",
+	"io"
-    ForcePathStyle: "",
+	"log"
-    RemotePath:     "",
+	"os/exec"
-    LocalPath:      "",
+	"strings"
+
+	storage "github.com/jkaninda/go-storage"
+)
+
+func main() {
+	ctx := context.Background()
+	s, err := storage.NewStorage(storage.StorageTypeS3, storage.Config{
+		S3Bucket:    "backups",
+		S3Region:    "us-east-1",
+		S3KeyID:     "...",
+		S3Secret:    "...",
+		S3Endpoint:  "https://minio.example.com", // optional, for S3-compatible services
+		S3ForcePath: true,
 	})
 	if err != nil {
-    log.Fatalf("Error creating s3 storage: %s", err)
+		log.Fatal(err)
 	}
-// Copy file to S3
+	defer s.Close()
-err = s3Storage.Copy(finalFileName)
+
-if err != nil {
+	pr, pw := io.Pipe()
-    log.Fatalf("Error copying file, error %v", err)
+	go func() {
+		gz := gzip.NewWriter(pw)
+		cmd := exec.CommandContext(ctx, "pg_dump", "mydb")
+		cmd.Stdout = gz
+		err := cmd.Run()
+		if err == nil {
+			err = gz.Close()
 		}
-// Download file from S3
+		pw.CloseWithError(err) // a non-nil error aborts the upload
-err = s3Storage.CopyFrom(finalFileName)
+	}()
+
+	item, err := s.Put(ctx, "db/mydb-2026-01-01.sql.gz", pr, &storage.PutOptions{ContentType: "application/gzip"})
 	if err != nil {
-    log.Fatalf("Error copying file, error %v", err)
+		log.Fatal(err)
+	}
+	log.Printf("uploaded %s (%d bytes, etag %s)", item.Key, item.Size, item.ETag)
+
+	// Restore: stream it back.
+	body, _, err := s.Get(ctx, item.Key)
+	if errors.Is(err, storage.ErrNotFound) {
+		log.Fatal("backup not found")
+	} else if err != nil {
+		log.Fatal(err)
+	}
+	defer body.Close()
+	zr, _ := gzip.NewReader(body)
+	_, _ = io.Copy(io.Discard, zr)
+
+	// Update an index file with optimistic concurrency.
+	idx, err := s.Stat(ctx, "db/index.json")
+	cond := storage.Condition{IfMatch: idx.ETag}
+	if errors.Is(err, storage.ErrNotFound) {
+		cond = storage.Condition{IfNoneMatch: true}
+	}
+	_, err = s.PutIf(ctx, "db/index.json", strings.NewReader(`{"backups":[]}`), cond, nil)
+	if errors.Is(err, storage.ErrPreconditionFailed) {
+		// someone else updated the index: re-read, merge and retry
+	}
 }
 ```
 
-### SSH Storage
+## Configuration
-```go
+
-sshStorage, err := ssh.NewStorage(ssh.Config{
+`storage.NewStorage(storageType, storage.Config{...})` with `storageType` one
-		Host:       "",
+of `s3`, `gcs`, `azure`, `sftp`, `ftp`, `local`.
-		Port:       22,
+
-		User:       "",
+| Field | Backend | Description |
-		Password:   "",
+|-------|---------|-------------|
-		RemotePath: "",
+| `LocalPath` | local | Base directory (default `/backup`) |
-		LocalPath:  "",
+| `S3Bucket` | s3 | Bucket (required) |
-	})
+| `S3Region` | s3 | Region (default from AWS config, else `us-east-1`) |
-	if err != nil {
+| `S3KeyID`, `S3Secret`, `S3SessionToken` | s3 | Static credentials; when empty the default AWS chain is used (env, shared profile, IRSA/web identity, ECS/EC2 roles) |
-        log.Fatalf("Error creating SSH storage: %s", err)
+| `S3Profile` | s3 | Shared config profile |
-	}
+| `S3Endpoint` | s3 | Custom endpoint (MinIO etc.); without a scheme `https://` is assumed (`http://` with `S3DisableTLS`) |
-	// Copy file to the remote server
+| `S3ForcePath` | s3 | Path-style addressing |
-	err = sshStorage.Copy(finalFileName)
+| `S3DisableTLS` | s3 | Use `http://` for scheme-less endpoints |
-	if err != nil {
+| `S3StorageClass` | s3 | e.g. `STANDARD_IA`, `GLACIER_IR` |
-        log.Fatalf("Error copying backup file: %s", err)
+| `S3SSE`, `S3SSEKMSKeyID` | s3 | Server-side encryption: `AES256` or `aws:kms` (+ key id) |
-	}
+| `S3PartSize` | s3 | Multipart part size (default 64 MiB, min 5 MiB) |
-// Download file from SSH remote server
+| `S3Concurrency` | s3 | Parallel part uploads (default 2); memory ≈ part size × concurrency |
-err = sshStorage.CopyFrom(finalFileName)
+| `AzureAccount`, `AzureKey` | azure | Shared key credentials |
-if err != nil {
+| `AzureSASToken` | azure | SAS token (used instead of the shared key when set) |
-log.Fatalf("Error copying file, error %v", err)
+| `AzureContainer` | azure | Container (required) |
-}
+| `AzureEndpoint` | azure | Domain suffix (default `blob.core.windows.net`) or a full service URL such as `http://127.0.0.1:10000/devstoreaccount1` (Azurite) |
-```
+| `GCSBucket` | gcs | Bucket (required) |
-### FTP Storage
+| `GCSCredentialsFile`, `GCSCredentialsJSON` | gcs | Credentials; default is Application Default Credentials |
-```go
+| `GCSEndpoint` | gcs | Custom endpoint |
-	ftpStorage, err := ftp.NewStorage(ftp.Config{
+| `SFTPHost`, `SFTPPort` | sftp | Server (port default 22) |
-		Host:       "",
+| `SFTPUsername`, `SFTPPassword` | sftp | Password authentication |
-		Port:       21,
+| `SFTPIdentityFile`, `SFTPIdentityPassphrase` | sftp | Private key authentication (an SSH agent is used too when available) |
-		User:       "",
+| `SFTPKnownHostsFile` | sftp | known_hosts file (default `~/.ssh/known_hosts`); host keys are always verified unless... |
-		Password:   "",
+| `SFTPIgnoreKnownHosts` | sftp | ...this is `true` (insecure) |
-		RemotePath: "",
+| `SFTPDirectory` | sftp | Remote base directory (default: login directory) |
-		LocalPath:  "",
+| `FTPHost`, `FTPPort`, `FTPUsername`, `FTPPassword` | ftp | Server and credentials (port default 21) |
-	})
+| `FTPDirectory` | ftp | Remote base directory (default: login directory) |
-	if err != nil {
+| `FTPTLS` | ftp | Explicit FTPS (`AUTH TLS`) |
-        log.Fatalf("Error creating FTP storage: %s", err)
+
-	}
+The library never reads environment variables itself (except what the cloud
-	err = ftpStorage.Copy(finalFileName)
+SDKs do for their default credential chains, and `SSH_AUTH_SOCK` for the SSH agent).
-	if err != nil {
+
-        log.Fatalf("Error copying backup file: %s", err)
+## Testing
-	}
+
-// Download file from ftp remote server
+```shell
-err = ftpStorage.CopyFrom(finalFileName)
+go test ./...                       # unit tests: local + in-process SFTP server
-if err != nil {
+go test -tags integration ./...     # + MinIO, Azurite, fake-gcs-server, FTP via docker
-log.Fatalf("Error copying file, error %v", err)
-}
 ```
 
-### Azure Blob storage
+Integration tests skip when docker is unavailable. Images can be overridden
+with `GO_STORAGE_MINIO_IMAGE`, `GO_STORAGE_AZURITE_IMAGE`,
+`GO_STORAGE_FAKEGCS_IMAGE` and `GO_STORAGE_FTP_IMAGE`.
 
-```go
+## Deprecated: `pkg/` (v0.1 API)
-azureStorage, err := azure.NewStorage(azure.Config{
-		ContainerName: '',
-		AccountName:   '',
-		AccountKey:    '',
-		RemotePath:    '',
-		LocalPath:     '',
-	})
-	if err != nil {
-		log.Fatal("Error creating Azure Blob storage storage: %s", err)
-	}
-	err = azureStorage.Copy(finalFileName)
-	if err != nil {
-		log.Fatal("Error copying file: %s", err)
-	}
 
-// Download file from Azure Blob remote server
+The packages under `pkg/` (`pkg/local`, `pkg/s3`, `pkg/ssh`, `pkg/ftp`,
-err = azureStorage.CopyFrom(finalFileName)
+`pkg/azure`) are the legacy v0.1 API. They still compile but are deprecated
-if err != nil {
+and will be removed in a future release; use the root package instead.
-log.Fatalf("Error copying file, error %v", err)
+
-}
+## License
-```
+
+MIT License, Copyright (c) 2024 Jonas Kaninda. See [LICENSE](LICENSE).

azure.go

@@ -1,11 +1,50 @@
+/*
+MIT License
+
+Copyright (c) 2024 Jonas Kaninda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
 package go_storage
 
 import (
 	"context"
+	"errors"
 	"fmt"
-	"os"
+	"io"
+	"net/http"
+	"strings"
+	"time"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
+)
+
+// Azure streaming upload tuning: 8 MiB blocks allow blobs up to ~390 GiB
+// (50,000 blocks) while keeping memory bounded to blockSize*concurrency.
+const (
+	azureBlockSize   int64 = 8 << 20
+	azureConcurrency       = 2
 )
 
 // AzureStorage implements Storage interface for Azure Blob Storage.
@@ -15,6 +54,9 @@ type AzureStorage struct {
 }
 
 // NewAzureStorage creates a new Azure Blob Storage instance.
+//
+// Authentication: AzureSASToken if set, otherwise shared key
+// (AzureAccount/AzureKey), otherwise anonymous.
 func NewAzureStorage(config Config) (*AzureStorage, error) {
 	if err := validateConfig(config, map[string]string{
 		"container": config.AzureContainer,
@@ -22,134 +64,286 @@ func NewAzureStorage(config Config) (*AzureStorage, error) {
 		return nil, fmt.Errorf("azure: %w", err)
 	}
 
-	// Try to get credentials from environment if not provided
+	serviceURL, err := azureServiceURL(config)
-	account := config.AzureAccount
+	if err != nil {
-	if account == "" {
+		return nil, err
-		account = os.Getenv("AZURE_STORAGE_ACCOUNT")
 	}
 
-	key := config.AzureKey
+	var client *azblob.Client
-	if key == "" {
+	switch {
-		key = os.Getenv("AZURE_STORAGE_KEY")
+	case config.AzureSASToken != "":
+		u := serviceURL + "?" + strings.TrimPrefix(config.AzureSASToken, "?")
+		client, err = azblob.NewClientWithNoCredential(u, nil)
+	case config.AzureAccount != "" && config.AzureKey != "":
+		cred, cerr := azblob.NewSharedKeyCredential(config.AzureAccount, config.AzureKey)
+		if cerr != nil {
+			return nil, fmt.Errorf("azure: failed to create credentials: %w", cerr)
+		}
+		client, err = azblob.NewClientWithSharedKeyCredential(serviceURL, cred, nil)
+	default:
+		client, err = azblob.NewClientWithNoCredential(serviceURL, nil)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("azure: failed to create client: %w", err)
 	}
 
+	return &AzureStorage{client: client, container: config.AzureContainer}, nil
+}
+
+func azureServiceURL(config Config) (string, error) {
 	endpoint := config.AzureEndpoint
 	if endpoint == "" {
 		endpoint = "blob.core.windows.net"
 	}
+	if strings.Contains(endpoint, "://") {
+		return strings.TrimSuffix(endpoint, "/") + "/", nil
+	}
+	if config.AzureAccount == "" {
+		return "", fmt.Errorf("azure: missing required configuration: account")
+	}
+	return fmt.Sprintf("https://%s.%s/", config.AzureAccount, strings.Trim(endpoint, "/")), nil
+}
 
-	var client *azblob.Client
+// Client returns the underlying azblob client.
-	var err error
+func (a *AzureStorage) Client() *azblob.Client { return a.client }
 
-	if account == "" {
+func (a *AzureStorage) put(ctx context.Context, key string, r io.Reader, opts *PutOptions, ac *blob.AccessConditions) (Item, error) {
-		// Anonymous access
+	k, err := cleanKey(key)
-		client, err = azblob.NewClientWithNoCredential(endpoint, nil)
 	if err != nil {
-			return nil, fmt.Errorf("azure: failed to create anonymous client: %w", err)
+		return Item{}, fmt.Errorf("azure: %w", err)
 	}
-	} else {
+	o := &azblob.UploadStreamOptions{
-		// Authenticated access
+		BlockSize:        azureBlockSize,
-		credential, err := azblob.NewSharedKeyCredential(account, key)
+		Concurrency:      azureConcurrency,
+		AccessConditions: ac,
+	}
+	if opts != nil {
+		if opts.ContentType != "" {
+			o.HTTPHeaders = &blob.HTTPHeaders{BlobContentType: &opts.ContentType}
+		}
+		if len(opts.Metadata) > 0 {
+			o.Metadata = make(map[string]*string, len(opts.Metadata))
+			for mk, mv := range opts.Metadata {
+				o.Metadata[mk] = &mv
+			}
+		}
+	}
+	cr := &countingReader{r: ctxReader{ctx: ctx, r: r}}
+	resp, err := a.client.UploadStream(ctx, a.container, k, cr, o)
 	if err != nil {
-			return nil, fmt.Errorf("azure: failed to create credentials: %w", err)
+		if ac != nil && (isAzurePrecondition(err) || bloberror.HasCode(err, bloberror.BlobNotFound)) {
+			return Item{}, fmt.Errorf("azure: conditional put of %s: %w", k, ErrPreconditionFailed)
+		}
+		return Item{}, fmt.Errorf("azure: failed to upload %s to container %s: %w", k, a.container, err)
+	}
+	it := Item{Key: k, Size: cr.n, ModifiedTime: time.Now().UTC()}
+	if resp.ETag != nil {
+		it.ETag = string(*resp.ETag)
+	}
+	if resp.LastModified != nil {
+		it.ModifiedTime = *resp.LastModified
+	}
+	if resp.VersionID != nil {
+		it.VersionID = *resp.VersionID
+	}
+	return it, nil
 }
 
-		url := fmt.Sprintf("https://%s.%s", account, endpoint)
+// Put streams r to key with a block upload; the block list is only
+// committed when the whole stream was read successfully.
+func (a *AzureStorage) Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error) {
+	return a.put(ctx, key, r, opts, nil)
+}
 
-		client, err = azblob.NewClientWithSharedKeyCredential(url, credential, nil)
+// PutIf writes r to key with If-Match / If-None-Match: * access conditions.
+func (a *AzureStorage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error) {
+	if err := validateCondition(cond); err != nil {
+		return Item{}, fmt.Errorf("azure: %w", err)
+	}
+	mac := &blob.ModifiedAccessConditions{}
+	if cond.IfMatch != "" {
+		etag := azcore.ETag(cond.IfMatch)
+		mac.IfMatch = &etag
+	}
+	if cond.IfNoneMatch {
+		etag := azcore.ETagAny
+		mac.IfNoneMatch = &etag
+	}
+	return a.put(ctx, key, r, opts, &blob.AccessConditions{ModifiedAccessConditions: mac})
+}
+
+// Get streams the blob body.
+func (a *AzureStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
+	k, err := cleanKey(key)
 	if err != nil {
-			return nil, fmt.Errorf("azure: failed to create client: %w", err)
+		return nil, Item{}, fmt.Errorf("azure: %w", err)
 	}
+	resp, err := a.client.DownloadStream(ctx, a.container, k, nil)
+	if err != nil {
+		return nil, Item{}, a.mapErr("get", k, err)
+	}
+	it := Item{Key: k}
+	if resp.ContentLength != nil {
+		it.Size = *resp.ContentLength
+	}
+	if resp.LastModified != nil {
+		it.ModifiedTime = *resp.LastModified
+	}
+	if resp.ETag != nil {
+		it.ETag = string(*resp.ETag)
+	}
+	if resp.VersionID != nil {
+		it.VersionID = *resp.VersionID
+	}
+	return resp.Body, it, nil
 }
 
-	return &AzureStorage{
+// Stat returns blob properties.
-		client:    client,
+func (a *AzureStorage) Stat(ctx context.Context, key string) (Item, error) {
-		container: config.AzureContainer,
+	k, err := cleanKey(key)
-	}, nil
+	if err != nil {
+		return Item{}, fmt.Errorf("azure: %w", err)
+	}
+	bc := a.client.ServiceClient().NewContainerClient(a.container).NewBlobClient(k)
+	resp, err := bc.GetProperties(ctx, nil)
+	if err != nil {
+		return Item{}, a.mapErr("stat", k, err)
+	}
+	it := Item{Key: k}
+	if resp.ContentLength != nil {
+		it.Size = *resp.ContentLength
+	}
+	if resp.LastModified != nil {
+		it.ModifiedTime = *resp.LastModified
+	}
+	if resp.ETag != nil {
+		it.ETag = string(*resp.ETag)
+	}
+	if resp.VersionID != nil {
+		it.VersionID = *resp.VersionID
+	}
+	return it, nil
 }
 
 // Upload uploads a file to Azure Blob Storage.
 func (a *AzureStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
-	file, err := os.Open(localPath)
+	if err := uploadFile(ctx, a, localPath, remotePath); err != nil {
-	if err != nil {
+		return fmt.Errorf("azure: %w", err)
-		return fmt.Errorf("azure: failed to open local file: %w", err)
 	}
-	defer func(file *os.File) {
-		err = file.Close()
-		if err != nil {
-			fmt.Printf("azure: failed to close file: %v\n", err)
-		}
-	}(file)
-
-	_, err = a.client.UploadFile(ctx, a.container, normalizePathSeparators(remotePath), file, nil)
-	if err != nil {
-		return fmt.Errorf("azure: failed to upload to container %s: %w", a.container, err)
-	}
-
 	return nil
 }
 
 // Download downloads a file from Azure Blob Storage.
 func (a *AzureStorage) Download(ctx context.Context, remotePath string, localPath string) error {
-	file, err := os.Create(localPath)
+	if err := downloadFile(ctx, a, remotePath, localPath); err != nil {
-	if err != nil {
+		return fmt.Errorf("azure: %w", err)
-		return fmt.Errorf("azure: failed to create local file: %w", err)
 	}
-	defer func(file *os.File) {
-		err = file.Close()
-		if err != nil {
-			fmt.Printf("azure: failed to close file: %v\n", err)
-		}
-	}(file)
-
-	_, err = a.client.DownloadFile(ctx, a.container, normalizePathSeparators(remotePath), file, nil)
-	if err != nil {
-		return fmt.Errorf("azure: failed to download from container %s: %w", a.container, err)
-	}
-
 	return nil
 }
 
-// List lists blobs in Azure Blob Storage with the given prefix.
+// List lists blobs (recursively) with the given prefix.
 func (a *AzureStorage) List(ctx context.Context, prefix string) ([]Item, error) {
-	normalizedPrefix := normalizePathSeparators(prefix)
+	p, err := cleanPrefix(prefix)
-
+	if err != nil {
-	pager := a.client.NewListBlobsFlatPager(a.container, &azblob.ListBlobsFlatOptions{
+		return nil, fmt.Errorf("azure: %w", err)
-		Prefix: &normalizedPrefix,
+	}
-	})
+	pager := a.client.NewListBlobsFlatPager(a.container, &azblob.ListBlobsFlatOptions{Prefix: &p})
-
 	items := make([]Item, 0)
-
 	for pager.More() {
 		resp, err := pager.NextPage(ctx)
 		if err != nil {
 			return nil, fmt.Errorf("azure: failed to list blobs in container %s: %w", a.container, err)
 		}
-
+		for _, b := range resp.Segment.BlobItems {
-		for _, blob := range resp.Segment.BlobItems {
+			items = append(items, azureItem(b))
-			items = append(items, Item{
-				Key:          *blob.Name,
-				ModifiedTime: *blob.Properties.LastModified,
-				Size:         *blob.Properties.ContentLength,
-				IsDirectory:  false,
-			})
 		}
 	}
-
 	return items, nil
 }
 
-// Remove deletes a blob from Azure Blob Storage.
+// ListDir lists the blobs directly under dir.
-func (a *AzureStorage) Remove(ctx context.Context, remotePath string) error {
+func (a *AzureStorage) ListDir(ctx context.Context, dir string) ([]Item, error) {
-	_, err := a.client.DeleteBlob(ctx, a.container, normalizePathSeparators(remotePath), nil)
+	d, err := cleanDir(dir)
 	if err != nil {
-		return fmt.Errorf("azure: failed to remove blob from container %s: %w", a.container, err)
+		return nil, fmt.Errorf("azure: %w", err)
+	}
+	p := dirPrefix(d)
+	cc := a.client.ServiceClient().NewContainerClient(a.container)
+	pager := cc.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{Prefix: &p})
+	items := make([]Item, 0)
+	for pager.More() {
+		resp, err := pager.NextPage(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("azure: failed to list blobs in container %s: %w", a.container, err)
+		}
+		for _, b := range resp.Segment.BlobItems {
+			if b.Name == nil || strings.HasSuffix(*b.Name, "/") {
+				continue
+			}
+			items = append(items, azureItem(b))
+		}
+	}
+	return items, nil
 }
 
+func azureItem(b *container.BlobItem) Item {
+	it := Item{}
+	if b.Name != nil {
+		it.Key = *b.Name
+	}
+	if b.VersionID != nil {
+		it.VersionID = *b.VersionID
+	}
+	if p := b.Properties; p != nil {
+		if p.LastModified != nil {
+			it.ModifiedTime = *p.LastModified
+		}
+		if p.ContentLength != nil {
+			it.Size = *p.ContentLength
+		}
+		if p.ETag != nil {
+			it.ETag = string(*p.ETag)
+		}
+	}
+	return it
+}
+
+// Remove deletes a blob.
+func (a *AzureStorage) Remove(ctx context.Context, remotePath string) error {
+	k, err := cleanKey(remotePath)
+	if err != nil {
+		return fmt.Errorf("azure: %w", err)
+	}
+	if _, err := a.client.DeleteBlob(ctx, a.container, k, nil); err != nil {
+		return a.mapErr("remove", k, err)
+	}
 	return nil
 }
 
-// Close releases resources.
+// Close releases resources (no-op).
 func (a *AzureStorage) Close() error {
 	return nil
 }
+
+func (a *AzureStorage) mapErr(op, key string, err error) error {
+	if bloberror.HasCode(err, bloberror.BlobNotFound, bloberror.ResourceNotFound) {
+		return fmt.Errorf("azure: %s %s in container %s: %w", op, key, a.container, ErrNotFound)
+	}
+	var re *azcore.ResponseError
+	if errors.As(err, &re) && re.StatusCode == http.StatusNotFound && re.ErrorCode != string(bloberror.ContainerNotFound) {
+		return fmt.Errorf("azure: %s %s in container %s: %w", op, key, a.container, ErrNotFound)
+	}
+	return fmt.Errorf("azure: %s %s in container %s: %w", op, key, a.container, err)
+}
+
+func isAzurePrecondition(err error) bool {
+	if bloberror.HasCode(err, bloberror.ConditionNotMet, bloberror.BlobAlreadyExists, bloberror.TargetConditionNotMet) {
+		return true
+	}
+	var re *azcore.ResponseError
+	if errors.As(err, &re) {
+		return re.StatusCode == http.StatusPreconditionFailed ||
+			(re.StatusCode == http.StatusConflict && re.ErrorCode == string(bloberror.BlobAlreadyExists))
+	}
+	return false
+}

conformance_test.go

@@ -0,0 +1,364 @@
+/*
+MIT License
+
+Copyright (c) 2024 Jonas Kaninda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+package go_storage
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+)
+
+const conformanceStreamSize = 20 << 20
+
+// deterministicStream writes n pseudo-random bytes into an io.Pipe and
+// returns the reader plus the expected SHA-256.
+func deterministicStream(n int64, seed int64) (io.Reader, [32]byte) {
+	h := sha256.New()
+	_, _ = io.Copy(h, io.LimitReader(rand.New(rand.NewSource(seed)), n))
+	var sum [32]byte
+	copy(sum[:], h.Sum(nil))
+
+	pr, pw := io.Pipe()
+	go func() {
+		_, err := io.Copy(pw, io.LimitReader(rand.New(rand.NewSource(seed)), n))
+		_ = pw.CloseWithError(err)
+	}()
+	return pr, sum
+}
+
+// failingReader returns ok bytes, then err.
+type failingReader struct {
+	r   io.Reader
+	n   int64
+	err error
+}
+
+func (f *failingReader) Read(p []byte) (int, error) {
+	if f.n <= 0 {
+		return 0, f.err
+	}
+	if int64(len(p)) > f.n {
+		p = p[:f.n]
+	}
+	n, err := f.r.Read(p)
+	f.n -= int64(n)
+	return n, err
+}
+
+func keysOf(items []Item, filesOnly bool) []string {
+	out := make([]string, 0, len(items))
+	for _, it := range items {
+		if filesOnly && it.IsDirectory {
+			continue
+		}
+		out = append(out, it.Key)
+	}
+	sort.Strings(out)
+	return out
+}
+
+func putString(t *testing.T, s Storage, key, content string) Item {
+	t.Helper()
+	it, err := s.Put(context.Background(), key, strings.NewReader(content), &PutOptions{ContentType: "text/plain"})
+	if err != nil {
+		t.Fatalf("Put(%s): %v", key, err)
+	}
+	return it
+}
+
+func getString(t *testing.T, s Storage, key string) string {
+	t.Helper()
+	body, _, err := s.Get(context.Background(), key)
+	if err != nil {
+		t.Fatalf("Get(%s): %v", key, err)
+	}
+	defer func() { _ = body.Close() }()
+	b, err := io.ReadAll(body)
+	if err != nil {
+		t.Fatalf("read %s: %v", key, err)
+	}
+	return string(b)
+}
+
+// runConformance exercises the full Storage contract against s.
+func runConformance(t *testing.T, s Storage) {
+	root := fmt.Sprintf("conf-%d", time.Now().UnixNano())
+	k := func(p string) string { return root + "/" + p }
+	for _, tc := range []struct {
+		name string
+		fn   func(*testing.T, Storage, func(string) string)
+	}{
+		{"StreamRoundTrip", confStreamRoundTrip},
+		{"FailingReaderLeavesNothing", confFailingReaderLeavesNothing},
+		{"ListDirBoundary", confListDirBoundary},
+		{"ListRecursive", confListRecursive},
+		{"Remove", confRemove},
+		{"PathEscape", confPathEscape},
+		{"PutIf", confPutIf},
+		{"UploadDownload", confUploadDownload},
+	} {
+		t.Run(tc.name, func(t *testing.T) { tc.fn(t, s, k) })
+	}
+}
+
+func confStreamRoundTrip(t *testing.T, s Storage, k func(string) string) {
+	ctx := context.Background()
+	r, want := deterministicStream(conformanceStreamSize, 42)
+	it, err := s.Put(ctx, k("stream/big.bin"), r, &PutOptions{ContentType: "application/octet-stream", Size: -1})
+	if err != nil {
+		t.Fatalf("Put: %v", err)
+	}
+	if it.Size != conformanceStreamSize {
+		t.Errorf("Put size = %d, want %d", it.Size, conformanceStreamSize)
+	}
+	st, err := s.Stat(ctx, k("stream/big.bin"))
+	if err != nil {
+		t.Fatalf("Stat: %v", err)
+	}
+	if st.Size != conformanceStreamSize {
+		t.Errorf("Stat size = %d", st.Size)
+	}
+	if st.Key != k("stream/big.bin") {
+		t.Errorf("Stat key = %q", st.Key)
+	}
+	if st.ETag != it.ETag {
+		t.Errorf("Stat ETag %q != Put ETag %q", st.ETag, it.ETag)
+	}
+	body, gi, err := s.Get(ctx, k("stream/big.bin"))
+	if err != nil {
+		t.Fatalf("Get: %v", err)
+	}
+	h := sha256.New()
+	_, err = io.Copy(h, body)
+	_ = body.Close()
+	if err != nil {
+		t.Fatalf("read body: %v", err)
+	}
+	if !bytes.Equal(h.Sum(nil), want[:]) {
+		t.Fatal("content mismatch")
+	}
+	if gi.Size != conformanceStreamSize {
+		t.Errorf("Get item size = %d", gi.Size)
+	}
+}
+
+func confFailingReaderLeavesNothing(t *testing.T, s Storage, k func(string) string) {
+	ctx := context.Background()
+	key := k("fail/broken.bin")
+	boom := errors.New("boom")
+	r := &failingReader{r: rand.New(rand.NewSource(1)), n: 18 << 20, err: boom}
+	if _, err := s.Put(ctx, key, r, nil); err == nil {
+		t.Fatal("Put with failing reader succeeded")
+	}
+	if _, err := s.Stat(ctx, key); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("Stat after failed Put: %v, want ErrNotFound", err)
+	}
+	items, err := s.ListDir(ctx, k("fail"))
+	if err != nil {
+		t.Fatalf("ListDir: %v", err)
+	}
+	for _, it := range items {
+		if strings.HasSuffix(it.Key, ".partial") || strings.HasSuffix(it.Key, "broken.bin") {
+			t.Errorf("leftover after failed Put: %s", it.Key)
+		}
+	}
+
+	cctx, cancel := context.WithCancel(ctx)
+	pr, pw := io.Pipe()
+	go func() {
+		_, _ = pw.Write(make([]byte, 1<<20))
+		cancel()
+		_, _ = pw.Write(make([]byte, 1<<20))
+		_ = pw.Close()
+	}()
+	if _, err := s.Put(cctx, k("fail/cancelled.bin"), pr, nil); err == nil {
+		t.Fatal("Put with cancelled context succeeded")
+	}
+	_ = pr.Close()
+	if _, err := s.Stat(ctx, k("fail/cancelled.bin")); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("Stat after cancelled Put: %v, want ErrNotFound", err)
+	}
+}
+
+func confListDirBoundary(t *testing.T, s Storage, k func(string) string) {
+	ctx := context.Background()
+	for _, p := range []string{"ld/a/b/1.txt", "ld/a/b/2.txt", "ld/a/b/sub/deep.txt", "ld/a/bc/x.txt", "ld/top.txt"} {
+		putString(t, s, k(p), "x")
+	}
+	want := []string{k("ld/a/b/1.txt"), k("ld/a/b/2.txt")}
+	for _, dir := range []string{k("ld/a/b"), k("ld/a/b/")} {
+		items, err := s.ListDir(ctx, dir)
+		if err != nil {
+			t.Fatalf("ListDir(%s): %v", dir, err)
+		}
+		if got := keysOf(items, false); strings.Join(got, ",") != strings.Join(want, ",") {
+			t.Errorf("ListDir(%s) = %v, want %v", dir, got, want)
+		}
+		for _, it := range items {
+			if it.IsDirectory || it.Size != 1 {
+				t.Errorf("ListDir item %+v", it)
+			}
+		}
+	}
+	items, err := s.ListDir(ctx, k("ld"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := keysOf(items, false); len(got) != 1 || got[0] != k("ld/top.txt") {
+		t.Errorf("ListDir(ld) = %v", got)
+	}
+	items, err = s.ListDir(ctx, k("ld/missing"))
+	if err != nil || len(items) != 0 {
+		t.Errorf("ListDir(missing) = %v, %v", items, err)
+	}
+}
+
+func confListRecursive(t *testing.T, s Storage, k func(string) string) {
+	ctx := context.Background()
+	items, err := s.List(ctx, k("ld/a/b/"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []string{k("ld/a/b/1.txt"), k("ld/a/b/2.txt"), k("ld/a/b/sub/deep.txt")}
+	if got := keysOf(items, true); strings.Join(got, ",") != strings.Join(want, ",") {
+		t.Errorf("List(ld/a/b/) = %v, want %v", got, want)
+	}
+	items, err = s.List(ctx, k("ld/"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := keysOf(items, true); len(got) != 5 {
+		t.Errorf("List(ld/) = %v, want 5 files", got)
+	}
+}
+
+func confRemove(t *testing.T, s Storage, k func(string) string) {
+	ctx := context.Background()
+	if err := s.Remove(ctx, k("does/not/exist.txt")); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("Remove missing: %v, want ErrNotFound", err)
+	}
+	putString(t, s, k("rm/x.txt"), "x")
+	if err := s.Remove(ctx, k("rm/x.txt")); err != nil {
+		t.Fatalf("Remove: %v", err)
+	}
+	if _, err := s.Stat(ctx, k("rm/x.txt")); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("Stat after Remove: %v", err)
+	}
+	if _, _, err := s.Get(ctx, k("rm/x.txt")); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("Get after Remove: %v", err)
+	}
+}
+
+func confPathEscape(t *testing.T, s Storage, k func(string) string) {
+	ctx := context.Background()
+	if _, err := s.Put(ctx, "../escape.txt", strings.NewReader("x"), nil); err == nil {
+		t.Fatal("Put ../escape.txt succeeded")
+	}
+	if _, err := s.Stat(ctx, "a/../../escape.txt"); err == nil || errors.Is(err, ErrNotFound) {
+		t.Fatalf("Stat escaping key: %v", err)
+	}
+}
+
+func confPutIf(t *testing.T, s Storage, k func(string) string) {
+	ctx := context.Background()
+	key := k("index/index.json")
+	it1, err := s.PutIf(ctx, key, strings.NewReader(`{"v":1}`), Condition{IfNoneMatch: true}, &PutOptions{ContentType: "application/json"})
+	if errors.Is(err, ErrConditionNotSupported) {
+		t.Skip("backend does not support conditional writes")
+	}
+	if err != nil {
+		t.Fatalf("PutIf IfNoneMatch (new): %v", err)
+	}
+	if _, err := s.PutIf(ctx, key, strings.NewReader(`{"v":2}`), Condition{IfNoneMatch: true}, nil); !errors.Is(err, ErrPreconditionFailed) {
+		t.Fatalf("PutIf IfNoneMatch (exists): %v, want ErrPreconditionFailed", err)
+	}
+	if got := getString(t, s, key); got != `{"v":1}` {
+		t.Fatalf("content after rejected PutIf = %q", got)
+	}
+
+	it2, err := s.PutIf(ctx, key, strings.NewReader(`{"v":2}`), Condition{IfMatch: it1.ETag}, nil)
+	if errors.Is(err, ErrConditionNotSupported) {
+		t.Log("IfMatch not supported; skipping IfMatch checks")
+		return
+	}
+	if err != nil {
+		t.Fatalf("PutIf IfMatch (fresh): %v", err)
+	}
+	if it2.ETag == "" || it2.ETag == it1.ETag {
+		t.Fatalf("ETag did not change: %q -> %q", it1.ETag, it2.ETag)
+	}
+	if _, err := s.PutIf(ctx, key, strings.NewReader(`{"v":3}`), Condition{IfMatch: it1.ETag}, nil); !errors.Is(err, ErrPreconditionFailed) {
+		t.Fatalf("PutIf IfMatch (stale): %v, want ErrPreconditionFailed", err)
+	}
+	if got := getString(t, s, key); got != `{"v":2}` {
+		t.Fatalf("content = %q, want v2", got)
+	}
+	st, err := s.Stat(ctx, key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if st.ETag != it2.ETag {
+		t.Fatalf("Stat ETag %q != PutIf ETag %q", st.ETag, it2.ETag)
+	}
+	if _, err := s.PutIf(ctx, k("index/missing.json"), strings.NewReader("x"), Condition{IfMatch: it2.ETag}, nil); !errors.Is(err, ErrPreconditionFailed) {
+		t.Fatalf("PutIf IfMatch on missing key: %v, want ErrPreconditionFailed", err)
+	}
+}
+
+func confUploadDownload(t *testing.T, s Storage, k func(string) string) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	src := filepath.Join(dir, "src.txt")
+	if err := os.WriteFile(src, []byte("hello world"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.Upload(ctx, src, k("ud/file.txt")); err != nil {
+		t.Fatalf("Upload: %v", err)
+	}
+	dst := filepath.Join(dir, "out", "dst.txt")
+	if err := s.Download(ctx, k("ud/file.txt"), dst); err != nil {
+		t.Fatalf("Download: %v", err)
+	}
+	b, err := os.ReadFile(dst)
+	if err != nil || string(b) != "hello world" {
+		t.Fatalf("downloaded %q, %v", b, err)
+	}
+	if err := s.Download(ctx, k("ud/missing.txt"), filepath.Join(dir, "m.txt")); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("Download missing: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "m.txt")); !os.IsNotExist(err) {
+		t.Fatal("Download of missing key left a local file")
+	}
+}

ftp.go

@@ -1,22 +1,57 @@
+/*
+MIT License
+
+Copyright (c) 2024 Jonas Kaninda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
 package go_storage
 
 import (
 	"context"
+	"crypto/tls"
+	"errors"
 	"fmt"
 	"io"
-	"os"
+	"net"
-	"path/filepath"
+	"net/textproto"
+	"path"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/jlaffaye/ftp"
 )
 
-// FTPStorage implements Storage interface for FTP.
+// FTPStorage implements Storage interface for FTP (optionally explicit FTPS).
+//
+// FTP uses a single control connection, so all operations are serialised by a
+// mutex. The body returned by Get holds that connection until it is closed:
+// a Get body MUST be closed before any other call on the same FTPStorage,
+// otherwise that call blocks.
 type FTPStorage struct {
 	host    string
 	port    string
 	baseDir string
+
+	mu     sync.Mutex
 	client *ftp.ServerConn
 }
 
@@ -32,220 +67,330 @@ func NewFTPStorage(config Config) (*FTPStorage, error) {
 	if port == "" {
 		port = "21"
 	}
+	hostPort := net.JoinHostPort(config.FTPHost, port)
 
-	// Connect to FTP server
+	opts := []ftp.DialOption{ftp.DialWithTimeout(30 * time.Second)}
-	hostPort := fmt.Sprintf("%s:%s", config.FTPHost, port)
+	if config.FTPTLS {
-	conn, err := ftp.Dial(hostPort, ftp.DialWithTimeout(30*time.Second))
+		opts = append(opts, ftp.DialWithExplicitTLS(&tls.Config{
+			ServerName: config.FTPHost,
+			MinVersion: tls.VersionTLS12,
+		}))
+	}
+	conn, err := ftp.Dial(hostPort, opts...)
 	if err != nil {
 		return nil, fmt.Errorf("ftp: failed to connect to %s: %w", hostPort, err)
 	}
 
-	// Login
 	if err := conn.Login(config.FTPUsername, config.FTPPassword); err != nil {
-		conn.Quit()
+		_ = conn.Quit()
 		return nil, fmt.Errorf("ftp: login failed: %w", err)
 	}
 
+	baseDir := normalizePathSeparators(config.FTPDirectory)
+	if baseDir == "" {
+		if wd, err := conn.CurrentDir(); err == nil && wd != "" {
+			baseDir = wd
+		} else {
+			baseDir = "/"
+		}
+	}
+
 	return &FTPStorage{
 		host:    config.FTPHost,
 		port:    port,
-		baseDir: config.FTPDirectory,
+		baseDir: path.Clean(baseDir),
 		client:  conn,
 	}, nil
 }
 
+func (f *FTPStorage) resolve(key string) (string, string, error) {
+	k, err := cleanKey(key)
+	if err != nil {
+		return "", "", fmt.Errorf("ftp: %w", err)
+	}
+	return k, path.Join(f.baseDir, k), nil
+}
+
+func ftpCode(err error) int {
+	var te *textproto.Error
+	if errors.As(err, &te) {
+		return te.Code
+	}
+	return 0
+}
+
+func ftpNotFound(err error) bool {
+	return ftpCode(err) == ftp.StatusFileUnavailable
+}
+
+// Put streams r to <key>.<rand>.partial and renames it on success.
+func (f *FTPStorage) Put(ctx context.Context, key string, r io.Reader, _ *PutOptions) (Item, error) {
+	k, full, err := f.resolve(key)
+	if err != nil {
+		return Item{}, err
+	}
+	if err := ctx.Err(); err != nil {
+		return Item{}, err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	f.ensureDirectory(path.Dir(full))
+	tmp := partialName(full)
+	err = f.client.Stor(tmp, ctxReader{ctx: ctx, r: r})
+	if err == nil {
+		err = ctx.Err()
+	}
+	if err != nil {
+		_ = f.client.Delete(tmp)
+		return Item{}, fmt.Errorf("ftp: failed to upload %s: %w", k, err)
+	}
+	if err := f.client.Rename(tmp, full); err != nil {
+		// Some servers refuse to rename onto an existing file.
+		_ = f.client.Delete(full)
+		if err := f.client.Rename(tmp, full); err != nil {
+			_ = f.client.Delete(tmp)
+			return Item{}, fmt.Errorf("ftp: failed to rename %s: %w", k, err)
+		}
+	}
+	return f.stat(k, full)
+}
+
+// PutIf is not supported on FTP: it always returns ErrConditionNotSupported.
+// Callers should fall back to a lock-based protocol.
+func (f *FTPStorage) PutIf(_ context.Context, key string, _ io.Reader, _ Condition, _ *PutOptions) (Item, error) {
+	return Item{}, fmt.Errorf("ftp: conditional put of %s: %w", key, ErrConditionNotSupported)
+}
+
+// ftpBody releases the connection lock when closed.
+type ftpBody struct {
+	ctx  context.Context
+	resp *ftp.Response
+	once sync.Once
+	mu   *sync.Mutex
+}
+
+func (b *ftpBody) Read(p []byte) (int, error) {
+	if err := b.ctx.Err(); err != nil {
+		return 0, err
+	}
+	return b.resp.Read(p)
+}
+
+func (b *ftpBody) Close() error {
+	var err error
+	b.once.Do(func() {
+		err = b.resp.Close()
+		b.mu.Unlock()
+	})
+	return err
+}
+
+// Get opens key for streaming reads. The returned body holds the FTP control
+// connection: close it before issuing any other call on this storage.
+func (f *FTPStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
+	k, full, err := f.resolve(key)
+	if err != nil {
+		return nil, Item{}, err
+	}
+	if err := ctx.Err(); err != nil {
+		return nil, Item{}, err
+	}
+	f.mu.Lock()
+	it, err := f.stat(k, full)
+	if err != nil {
+		f.mu.Unlock()
+		return nil, Item{}, err
+	}
+	if it.IsDirectory {
+		f.mu.Unlock()
+		return nil, Item{}, fmt.Errorf("ftp: %s is a directory", k)
+	}
+	resp, err := f.client.Retr(full)
+	if err != nil {
+		f.mu.Unlock()
+		if ftpNotFound(err) {
+			return nil, Item{}, fmt.Errorf("ftp: get %s: %w", k, ErrNotFound)
+		}
+		return nil, Item{}, fmt.Errorf("ftp: failed to retrieve %s: %w", k, err)
+	}
+	return &ftpBody{ctx: ctx, resp: resp, mu: &f.mu}, it, nil
+}
+
+// Stat returns metadata for key (from a listing of its parent directory).
+func (f *FTPStorage) Stat(ctx context.Context, key string) (Item, error) {
+	k, full, err := f.resolve(key)
+	if err != nil {
+		return Item{}, err
+	}
+	if err := ctx.Err(); err != nil {
+		return Item{}, err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.stat(k, full)
+}
+
+func (f *FTPStorage) stat(key, full string) (Item, error) {
+	entries, err := f.client.List(path.Dir(full))
+	if err != nil {
+		if ftpNotFound(err) {
+			return Item{}, fmt.Errorf("ftp: stat %s: %w", key, ErrNotFound)
+		}
+		return Item{}, fmt.Errorf("ftp: stat %s: %w", key, err)
+	}
+	name := path.Base(full)
+	for _, e := range entries {
+		if e.Name == name {
+			return ftpItem(key, e), nil
+		}
+	}
+	return Item{}, fmt.Errorf("ftp: stat %s: %w", key, ErrNotFound)
+}
+
+func ftpItem(key string, e *ftp.Entry) Item {
+	it := Item{Key: key, ModifiedTime: e.Time, IsDirectory: e.Type == ftp.EntryTypeFolder}
+	if !it.IsDirectory {
+		it.Size = int64(e.Size) //nolint:gosec // sizes fit in int64
+	}
+	return it
+}
+
 // Upload uploads a file via FTP.
 func (f *FTPStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
-	select {
+	return uploadFile(ctx, f, localPath, remotePath)
-	case <-ctx.Done():
-		return ctx.Err()
-	default:
-	}
-
-	src, err := os.Open(localPath)
-	if err != nil {
-		return fmt.Errorf("ftp: failed to open local file: %w", err)
-	}
-	defer src.Close()
-
-	fullRemotePath := filepath.Join(f.baseDir, remotePath)
-	fullRemotePath = normalizePathSeparators(fullRemotePath)
-
-	// Create parent directory if needed
-	remoteDir := filepath.Dir(fullRemotePath)
-	remoteDir = normalizePathSeparators(remoteDir)
-
-	if remoteDir != "." && remoteDir != "/" {
-		f.ensureDirectory(remoteDir)
-
-	}
-
-	// Upload file
-	if err := f.client.Stor(fullRemotePath, src); err != nil {
-		return fmt.Errorf("ftp: failed to upload file %s: %w", fullRemotePath, err)
-	}
-
-	return nil
 }
 
 // Download downloads a file via FTP.
 func (f *FTPStorage) Download(ctx context.Context, remotePath string, localPath string) error {
-	select {
+	return downloadFile(ctx, f, remotePath, localPath)
-	case <-ctx.Done():
-		return ctx.Err()
-	default:
 }
 
-	fullRemotePath := filepath.Join(f.baseDir, remotePath)
+// List lists files and directories (recursively) whose key starts with prefix.
-	fullRemotePath = normalizePathSeparators(fullRemotePath)
-
-	// Get the file from FTP server
-	resp, err := f.client.Retr(fullRemotePath)
-	if err != nil {
-		return fmt.Errorf("ftp: failed to retrieve file %s: %w", fullRemotePath, err)
-	}
-	defer resp.Close()
-
-	// Create local file
-	dst, err := os.Create(localPath)
-	if err != nil {
-		return fmt.Errorf("ftp: failed to create local file: %w", err)
-	}
-	defer dst.Close()
-
-	// Copy data
-	if _, err := io.Copy(dst, resp); err != nil {
-		return fmt.Errorf("ftp: failed to transfer data: %w", err)
-	}
-
-	return nil
-}
-
-// List lists files via FTP with the given prefix.
 func (f *FTPStorage) List(ctx context.Context, prefix string) ([]Item, error) {
-	select {
+	p, err := cleanPrefix(prefix)
-	case <-ctx.Done():
+	if err != nil {
-		return nil, ctx.Err()
+		return nil, fmt.Errorf("ftp: %w", err)
-	default:
 	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
 
 	items := make([]Item, 0)
-
+	startRel := ""
-	// Normalize the base directory
+	if i := strings.LastIndex(p, "/"); i >= 0 {
-	baseDir := normalizePathSeparators(f.baseDir)
+		startRel = p[:i]
-	if baseDir == "" {
-		baseDir = "/"
 	}
-
+	if err := f.walk(ctx, startRel, p, &items); err != nil {
-	// Walk the directory
-	if err := f.walkDirectory(baseDir, prefix, &items); err != nil {
 		return nil, err
 	}
-
 	return items, nil
 }
 
-// walkDirectory recursively walks FTP directory structure.
+func (f *FTPStorage) walk(ctx context.Context, rel, prefix string, items *[]Item) error {
-func (f *FTPStorage) walkDirectory(dir string, prefix string, items *[]Item) error {
+	if err := ctx.Err(); err != nil {
-	entries, err := f.client.List(dir)
+		return err
+	}
+	entries, err := f.client.List(path.Join(f.baseDir, rel))
 	if err != nil {
-		return fmt.Errorf("ftp: failed to list directory %s: %w", dir, err)
+		if ftpNotFound(err) {
-	}
-
-	for _, entry := range entries {
-		// Get relative path
-		var relPath string
-		if f.baseDir == "" || f.baseDir == "/" {
-			relPath = filepath.Join(strings.TrimPrefix(dir, "/"), entry.Name)
-		} else {
-			fullPath := filepath.Join(dir, entry.Name)
-			relPath = strings.TrimPrefix(fullPath, f.baseDir)
-			relPath = strings.TrimPrefix(relPath, "/")
-		}
-
-		// Normalize separators for comparison
-		relPath = normalizePathSeparators(relPath)
-
-		// Skip if doesn't match prefix
-		if prefix != "" && !strings.HasPrefix(relPath, prefix) {
-			// For directories, we might need to walk them to find matching files
-			if entry.Type == ftp.EntryTypeFolder {
-				fullPath := filepath.Join(dir, entry.Name)
-				fullPath = normalizePathSeparators(fullPath)
-				if err := f.walkDirectory(fullPath, prefix, items); err != nil {
-					// Log but don't fail on subdirectory errors
-					continue
-				}
-			}
-			continue
-		}
-
-		item := Item{
-			Key:          relPath,
-			ModifiedTime: entry.Time,
-			Size:         int64(entry.Size),
-			IsDirectory:  entry.Type == ftp.EntryTypeFolder,
-		}
-		*items = append(*items, item)
-
-		// Recursively walk directories
-		if entry.Type == ftp.EntryTypeFolder {
-			fullPath := filepath.Join(dir, entry.Name)
-			fullPath = normalizePathSeparators(fullPath)
-			if err := f.walkDirectory(fullPath, prefix, items); err != nil {
-				// Log but don't fail on subdirectory errors
-				continue
-			}
-		}
-	}
-
 			return nil
 		}
+		return fmt.Errorf("ftp: failed to list directory %s: %w", rel, err)
+	}
+	for _, e := range entries {
+		if e.Name == "." || e.Name == ".." || e.Type == ftp.EntryTypeLink {
+			continue
+		}
+		key := path.Join(rel, e.Name)
+		isDir := e.Type == ftp.EntryTypeFolder
+		if strings.HasPrefix(key, prefix) {
+			*items = append(*items, ftpItem(key, e))
+		} else if !isDir || !strings.HasPrefix(prefix, key+"/") {
+			continue
+		}
+		if isDir {
+			if err := f.walk(ctx, key, prefix, items); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// ListDir lists the files directly inside dir.
+func (f *FTPStorage) ListDir(ctx context.Context, dir string) ([]Item, error) {
+	d, err := cleanDir(dir)
+	if err != nil {
+		return nil, fmt.Errorf("ftp: %w", err)
+	}
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	entries, err := f.client.List(path.Join(f.baseDir, d))
+	if err != nil {
+		if ftpNotFound(err) {
+			return []Item{}, nil
+		}
+		return nil, fmt.Errorf("ftp: failed to list directory %s: %w", dir, err)
+	}
+	items := make([]Item, 0, len(entries))
+	for _, e := range entries {
+		if e.Type != ftp.EntryTypeFile {
+			continue
+		}
+		items = append(items, ftpItem(path.Join(d, e.Name), e))
+	}
+	return items, nil
+}
 
 // Remove deletes a file via FTP.
 func (f *FTPStorage) Remove(ctx context.Context, remotePath string) error {
-	select {
+	k, full, err := f.resolve(remotePath)
-	case <-ctx.Done():
+	if err != nil {
-		return ctx.Err()
+		return err
-	default:
 	}
-
+	if err := ctx.Err(); err != nil {
-	fullRemotePath := filepath.Join(f.baseDir, remotePath)
+		return err
-	fullRemotePath = normalizePathSeparators(fullRemotePath)
+	}
-
+	f.mu.Lock()
-	if err := f.client.Delete(fullRemotePath); err != nil {
+	defer f.mu.Unlock()
-		return fmt.Errorf("ftp: failed to remove %s: %w", fullRemotePath, err)
+	if err := f.client.Delete(full); err != nil {
+		if ftpNotFound(err) {
+			return fmt.Errorf("ftp: remove %s: %w", k, ErrNotFound)
+		}
+		return fmt.Errorf("ftp: failed to remove %s: %w", k, err)
 	}
-
 	return nil
 }
 
 // Close releases resources.
 func (f *FTPStorage) Close() error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
 	if err := f.client.Quit(); err != nil {
 		return fmt.Errorf("ftp: failed to close connection: %w", err)
 	}
 	return nil
 }
 
-// ensureDirectory creates a directory path if it doesn't exist.
+// ensureDirectory creates a directory path if it doesn't exist (errors are
-func (f *FTPStorage) ensureDirectory(path string) {
+// ignored: the directory usually exists already, and a real failure surfaces
-	parts := strings.Split(path, "/")
+// on the following STOR).
+func (f *FTPStorage) ensureDirectory(dir string) {
+	if dir == "." || dir == "/" || dir == "" {
+		return
+	}
 	current := ""
-
+	if strings.HasPrefix(dir, "/") {
-	for _, part := range parts {
+		current = "/"
+	}
+	for _, part := range strings.Split(dir, "/") {
 		if part == "" {
 			continue
 		}
-
+		current = path.Join(current, part)
-		if current == "" {
-			current = part
-		} else {
-			current = current + "/" + part
-		}
-
-		// Try to create directory (ignore errors if it already exists)
 		_ = f.client.MakeDir(current)
 	}
-
 }

gcs.go

@@ -1,12 +1,42 @@
+/*
+MIT License
+
+Copyright (c) 2024 Jonas Kaninda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
 package go_storage
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
+	"strconv"
+	"strings"
 
 	"cloud.google.com/go/storage"
+	"google.golang.org/api/googleapi"
 	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
 )
@@ -18,6 +48,10 @@ type GCSStorage struct {
 }
 
 // NewGCSStorage creates a new Google Cloud Storage instance.
+//
+// Credentials: GCSCredentialsJSON, else GCSCredentialsFile, else Application
+// Default Credentials. The STORAGE_EMULATOR_HOST environment variable is
+// honoured by the Google client library itself.
 func NewGCSStorage(config Config) (*GCSStorage, error) {
 	if err := validateConfig(config, map[string]string{
 		"bucket": config.GCSBucket,
@@ -25,128 +59,222 @@ func NewGCSStorage(config Config) (*GCSStorage, error) {
 		return nil, fmt.Errorf("gcs: %w", err)
 	}
 
-	options := make([]option.ClientOption, 0)
+	options := make([]option.ClientOption, 0, 2)
-
 	if config.GCSEndpoint != "" {
 		options = append(options, option.WithEndpoint(config.GCSEndpoint))
 	}
-
+	switch {
-	if config.GCSCredentialsFile != "" {
+	case config.GCSCredentialsJSON != "":
-		options = append(options, option.WithCredentialsFile(config.GCSCredentialsFile))
+		data := []byte(config.GCSCredentialsJSON)
+		options = append(options, option.WithAuthCredentialsJSON(gcsCredType(data), data))
+	case config.GCSCredentialsFile != "":
+		data, err := os.ReadFile(config.GCSCredentialsFile)
+		if err != nil {
+			return nil, fmt.Errorf("gcs: failed to read credentials file: %w", err)
+		}
+		options = append(options, option.WithAuthCredentialsFile(gcsCredType(data), config.GCSCredentialsFile))
+	}
+	return newGCSStorage(context.Background(), config.GCSBucket, options...)
 }
 
-	client, err := storage.NewClient(context.Background(), options...)
+func newGCSStorage(ctx context.Context, bucket string, options ...option.ClientOption) (*GCSStorage, error) {
+	client, err := storage.NewClient(ctx, options...)
 	if err != nil {
 		return nil, fmt.Errorf("gcs: failed to create client: %w", err)
 	}
+	return &GCSStorage{client: client, bucket: bucket}, nil
+}
 
-	return &GCSStorage{
+// gcsCredType detects the credential type from the JSON "type" field.
-		client: client,
+func gcsCredType(data []byte) option.CredentialsType {
-		bucket: config.GCSBucket,
+	var v struct {
+		Type string `json:"type"`
+	}
+	_ = json.Unmarshal(data, &v)
+	switch v.Type {
+	case "authorized_user":
+		return option.AuthorizedUser
+	case "impersonated_service_account":
+		return option.ImpersonatedServiceAccount
+	case "external_account":
+		return option.ExternalAccount
+	default:
+		return option.ServiceAccount
+	}
+}
+
+// Client returns the underlying GCS client.
+func (g *GCSStorage) Client() *storage.Client { return g.client }
+
+func gcsItem(attrs *storage.ObjectAttrs) Item {
+	return Item{
+		Key:          attrs.Name,
+		ModifiedTime: attrs.Updated,
+		Size:         attrs.Size,
+		ETag:         strconv.FormatInt(attrs.Generation, 10),
+	}
+}
+
+func (g *GCSStorage) put(ctx context.Context, key string, r io.Reader, opts *PutOptions, cond *storage.Conditions) (Item, error) {
+	k, err := cleanKey(key)
+	if err != nil {
+		return Item{}, fmt.Errorf("gcs: %w", err)
+	}
+	obj := g.client.Bucket(g.bucket).Object(k)
+	if cond != nil {
+		obj = obj.If(*cond)
+	}
+	// A child context: cancelling it before Close aborts the upload so the
+	// object is never finalised.
+	wctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	w := obj.NewWriter(wctx)
+	if opts != nil {
+		w.ContentType = opts.ContentType
+		if len(opts.Metadata) > 0 {
+			w.Metadata = opts.Metadata
+		}
+	}
+	if _, err := io.Copy(w, ctxReader{ctx: ctx, r: r}); err != nil {
+		cancel()
+		_ = w.Close()
+		if cond != nil && isGCSPrecondition(err) {
+			return Item{}, fmt.Errorf("gcs: conditional put of %s: %w", k, ErrPreconditionFailed)
+		}
+		return Item{}, fmt.Errorf("gcs: failed to upload %s: %w", k, err)
+	}
+	if err := ctx.Err(); err != nil {
+		cancel()
+		_ = w.Close()
+		return Item{}, fmt.Errorf("gcs: failed to upload %s: %w", k, err)
+	}
+	if err := w.Close(); err != nil {
+		if cond != nil && isGCSPrecondition(err) {
+			return Item{}, fmt.Errorf("gcs: conditional put of %s: %w", k, ErrPreconditionFailed)
+		}
+		return Item{}, fmt.Errorf("gcs: failed to finalize upload of %s: %w", k, err)
+	}
+	return gcsItem(w.Attrs()), nil
+}
+
+// Put streams r to key.
+func (g *GCSStorage) Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error) {
+	return g.put(ctx, key, r, opts, nil)
+}
+
+// PutIf writes r to key with a generation precondition (IfMatch is the
+// generation as returned in Item.ETag).
+func (g *GCSStorage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error) {
+	if err := validateCondition(cond); err != nil {
+		return Item{}, fmt.Errorf("gcs: %w", err)
+	}
+	c := storage.Conditions{}
+	switch {
+	case cond.IfNoneMatch:
+		c.DoesNotExist = true
+	default:
+		gen, err := strconv.ParseInt(cond.IfMatch, 10, 64)
+		if err != nil || gen <= 0 {
+			return Item{}, fmt.Errorf("gcs: invalid generation %q: %w", cond.IfMatch, ErrPreconditionFailed)
+		}
+		c.GenerationMatch = gen
+	}
+	return g.put(ctx, key, r, opts, &c)
+}
+
+// Get streams the object body.
+func (g *GCSStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
+	k, err := cleanKey(key)
+	if err != nil {
+		return nil, Item{}, fmt.Errorf("gcs: %w", err)
+	}
+	rd, err := g.client.Bucket(g.bucket).Object(k).NewReader(ctx)
+	if err != nil {
+		return nil, Item{}, g.mapErr("get", k, err)
+	}
+	return rd, Item{
+		Key:          k,
+		Size:         rd.Attrs.Size,
+		ModifiedTime: rd.Attrs.LastModified,
+		ETag:         strconv.FormatInt(rd.Attrs.Generation, 10),
 	}, nil
 }
 
+// Stat returns object attributes.
+func (g *GCSStorage) Stat(ctx context.Context, key string) (Item, error) {
+	k, err := cleanKey(key)
+	if err != nil {
+		return Item{}, fmt.Errorf("gcs: %w", err)
+	}
+	attrs, err := g.client.Bucket(g.bucket).Object(k).Attrs(ctx)
+	if err != nil {
+		return Item{}, g.mapErr("stat", k, err)
+	}
+	return gcsItem(attrs), nil
+}
+
 // Upload uploads a file to GCS.
 func (g *GCSStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
-	file, err := os.Open(localPath)
+	if err := uploadFile(ctx, g, localPath, remotePath); err != nil {
-	if err != nil {
+		return fmt.Errorf("gcs: %w", err)
-		return fmt.Errorf("gcs: failed to open local file: %w", err)
 	}
-	defer func(file *os.File) {
-		err = file.Close()
-		if err != nil {
-			fmt.Printf("gcs: failed to close file: %v\n", err)
-		}
-	}(file)
-
-	obj := g.client.Bucket(g.bucket).Object(normalizePathSeparators(remotePath)).NewWriter(ctx)
-	defer func(obj *storage.Writer) {
-		err = obj.Close()
-		if err != nil {
-			fmt.Printf("gcs: failed to close object writer: %v\n", err)
-		}
-	}(obj)
-
-	if _, err := io.Copy(obj, file); err != nil {
-		return fmt.Errorf("gcs: failed to write data: %w", err)
-	}
-
-	if err = obj.Close(); err != nil {
-		return fmt.Errorf("gcs: failed to finalize upload: %w", err)
-	}
-
 	return nil
 }
 
 // Download downloads a file from GCS.
 func (g *GCSStorage) Download(ctx context.Context, remotePath string, localPath string) error {
-	file, err := os.Create(localPath)
+	if err := downloadFile(ctx, g, remotePath, localPath); err != nil {
-	if err != nil {
+		return fmt.Errorf("gcs: %w", err)
-		return fmt.Errorf("gcs: failed to create local file: %w", err)
 	}
-	defer func(file *os.File) {
-		err = file.Close()
-		if err != nil {
-			fmt.Printf("gcs: failed to close file: %v\n", err)
-		}
-	}(file)
-
-	obj, err := g.client.Bucket(g.bucket).Object(normalizePathSeparators(remotePath)).NewReader(ctx)
-	if err != nil {
-		return fmt.Errorf("gcs: failed to create reader: %w", err)
-	}
-	defer func(obj *storage.Reader) {
-		err = obj.Close()
-		if err != nil {
-			fmt.Printf("gcs: failed to close object reader: %v\n", err)
-		}
-	}(obj)
-
-	if _, err = io.Copy(file, obj); err != nil {
-		return fmt.Errorf("gcs: failed to read data: %w", err)
-	}
-
 	return nil
 }
 
-// List lists objects in GCS with the given prefix.
+// List lists objects (recursively) with the given prefix.
 func (g *GCSStorage) List(ctx context.Context, prefix string) ([]Item, error) {
-	items := make([]Item, 0)
+	p, err := cleanPrefix(prefix)
-
+	if err != nil {
-	query := &storage.Query{
+		return nil, fmt.Errorf("gcs: %w", err)
-		Prefix: normalizePathSeparators(prefix),
+	}
+	return g.list(ctx, &storage.Query{Prefix: p})
 }
 
-	it := g.client.Bucket(g.bucket).Objects(ctx, query)
+// ListDir lists the objects directly under dir.
+func (g *GCSStorage) ListDir(ctx context.Context, dir string) ([]Item, error) {
+	d, err := cleanDir(dir)
+	if err != nil {
+		return nil, fmt.Errorf("gcs: %w", err)
+	}
+	return g.list(ctx, &storage.Query{Prefix: dirPrefix(d), Delimiter: "/"})
+}
 
+func (g *GCSStorage) list(ctx context.Context, q *storage.Query) ([]Item, error) {
+	items := make([]Item, 0)
+	it := g.client.Bucket(g.bucket).Objects(ctx, q)
 	for {
 		attrs, err := it.Next()
-		if err == iterator.Done {
+		if errors.Is(err, iterator.Done) {
 			break
 		}
 		if err != nil {
-			return nil, fmt.Errorf("gcs: failed to iterate objects: %w", err)
+			return nil, fmt.Errorf("gcs: failed to list objects in bucket %s: %w", g.bucket, err)
 		}
-
+		if attrs.Name == "" || (q.Delimiter != "" && strings.HasSuffix(attrs.Name, "/")) {
-		items = append(items, Item{
+			continue // synthetic prefix or directory marker
-			Key:          attrs.Name,
+		}
-			ModifiedTime: attrs.Updated,
+		items = append(items, gcsItem(attrs))
-			Size:         attrs.Size,
-			IsDirectory:  false,
-		})
 	}
-
 	return items, nil
 }
 
-// Remove deletes an object from GCS.
+// Remove deletes an object.
 func (g *GCSStorage) Remove(ctx context.Context, remotePath string) error {
-	obj := g.client.Bucket(g.bucket).Object(normalizePathSeparators(remotePath))
+	k, err := cleanKey(remotePath)
-
+	if err != nil {
-	if err := obj.Delete(ctx); err != nil {
+		return fmt.Errorf("gcs: %w", err)
-		return fmt.Errorf("gcs: failed to remove object from bucket %s: %w", g.bucket, err)
+	}
+	if err := g.client.Bucket(g.bucket).Object(k).Delete(ctx); err != nil {
+		return g.mapErr("remove", k, err)
 	}
-
 	return nil
 }
 
@@ -154,3 +282,16 @@ func (g *GCSStorage) Remove(ctx context.Context, remotePath string) error {
 func (g *GCSStorage) Close() error {
 	return g.client.Close()
 }
+
+func (g *GCSStorage) mapErr(op, key string, err error) error {
+	var ge *googleapi.Error
+	if errors.Is(err, storage.ErrObjectNotExist) || (errors.As(err, &ge) && ge.Code == http.StatusNotFound) {
+		return fmt.Errorf("gcs: %s %s in bucket %s: %w", op, key, g.bucket, ErrNotFound)
+	}
+	return fmt.Errorf("gcs: %s %s in bucket %s: %w", op, key, g.bucket, err)
+}
+
+func isGCSPrecondition(err error) bool {
+	var ge *googleapi.Error
+	return errors.As(err, &ge) && (ge.Code == http.StatusPreconditionFailed || ge.Code == http.StatusConflict)
+}

go.mod

@@ -1,11 +1,18 @@
 module github.com/jkaninda/go-storage
 
-go 1.24.0
+go 1.25
 
 require (
 	cloud.google.com/go/storage v1.60.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
 	github.com/aws/aws-sdk-go v1.55.7
+	github.com/aws/aws-sdk-go-v2 v1.47.3
+	github.com/aws/aws-sdk-go-v2/config v1.33.4
+	github.com/aws/aws-sdk-go-v2/credentials v1.20.9
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.43
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3
+	github.com/aws/smithy-go v1.28.5
 	github.com/bramvdbogaerde/go-scp v1.5.0
 	github.com/jlaffaye/ftp v0.2.0
 	github.com/pkg/sftp v1.13.10
@@ -21,11 +28,23 @@ require (
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	cloud.google.com/go/iam v1.5.3 // indirect
 	cloud.google.com/go/monitoring v1.24.3 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
 	github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect

go.sum

@@ -42,6 +42,44 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapp
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc=
 github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE=
 github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
+github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk=
+github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 h1:oiEmPz/YMMeh/JvAE8xSxNCIsvWE6lpFe4ZPlyFykQI=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22/go.mod h1:QWUDuRi9PFjBHIU2iH1Pkldbe7zqLfXJC4wWLbF8nuQ=
+github.com/aws/aws-sdk-go-v2/config v1.33.4 h1:FzvkXKSzwqHni4U7nDigHg4jjtqMpVUuHgmZfSoJVQ0=
+github.com/aws/aws-sdk-go-v2/config v1.33.4/go.mod h1:VZqGZnZsCWVfK/iGPptJIyNIX3XEX6iQU2Rel4sLrr8=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.43 h1:Fhx4uwBshQF0+jmhV/FKDM4LPj7L9iUy5n3MvLxLy00=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.43/go.mod h1:4Tyfc1eIPEyrAMysLCVkcYL1Hm/hkGWbElLo8pejHPo=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 h1:CwvFlKw0uUScfgOMflXSRZa+ayXgOf6wk7lhyMemQYU=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7/go.mod h1:4SbD8BB+p+QEnG0aX6DoWwrtk9SufyYXRsUv0nve8LE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 h1:NyeuyXkZL476uoAowNActm4XJO1QPKf7aassgXHpiqA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3/go.mod h1:GfoBeKMDJecgRUsGf/0RGdx0NJAh81ObTd922h8pQys=
+github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A=
+github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M=
+github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA=
+github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 h1:OKEfbf2tcfsF2pBk/S9QqNpo6alaAtiBs4hysCsBjPw=
+github.com/aws/aws-sdk-go-v2/service/sts v1.51.4/go.mod h1:mCd6qGo9b8hTtfjgsmNkFlzwjyce4owd7HDVFnCLXhM=
+github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k=
+github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/bramvdbogaerde/go-scp v1.5.0 h1:a9BinAjTfQh273eh7vd3qUgmBC+bx+3TRDtkZWmIpzM=
 github.com/bramvdbogaerde/go-scp v1.5.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=

integration_test.go

@@ -0,0 +1,263 @@
+//go:build integration
+
+/*
+MIT License
+
+Copyright (c) 2024 Jonas Kaninda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+package go_storage
+
+// Integration tests against real services started with the docker CLI.
+//
+//	go test -tags integration ./...
+//
+// Tests skip when docker is unavailable. Images can be overridden with
+// GO_STORAGE_MINIO_IMAGE, GO_STORAGE_AZURITE_IMAGE, GO_STORAGE_FAKEGCS_IMAGE
+// and GO_STORAGE_FTP_IMAGE.
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand"
+	"net"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"cloud.google.com/go/storage"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"google.golang.org/api/option"
+)
+
+func image(env, def string) string {
+	if v := os.Getenv(env); v != "" {
+		return v
+	}
+	return def
+}
+
+func requireDocker(t *testing.T) {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	if err := exec.CommandContext(ctx, "docker", "info").Run(); err != nil {
+		t.Skipf("docker not available: %v", err)
+	}
+}
+
+func docker(t *testing.T, args ...string) string {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+	out, err := exec.CommandContext(ctx, "docker", args...).CombinedOutput()
+	if err != nil {
+		t.Fatalf("docker %s: %v\n%s", strings.Join(args, " "), err, out)
+	}
+	return strings.TrimSpace(string(out))
+}
+
+// runContainer starts a detached container and removes it at cleanup.
+func runContainer(t *testing.T, args ...string) string {
+	t.Helper()
+	id := docker(t, append([]string{"run", "-d"}, args...)...)
+	t.Cleanup(func() {
+		if t.Failed() {
+			out, _ := exec.Command("docker", "logs", "--tail", "50", id).CombinedOutput()
+			t.Logf("container logs:\n%s", out)
+		}
+		_ = exec.Command("docker", "rm", "-f", "-v", id).Run()
+	})
+	return id
+}
+
+// hostPort returns the host port mapped to the container port.
+func hostPort(t *testing.T, id, containerPort string) string {
+	t.Helper()
+	out := docker(t, "port", id, containerPort)
+	line := strings.Split(out, "\n")[0]
+	_, port, err := net.SplitHostPort(strings.TrimSpace(line))
+	if err != nil {
+		t.Fatalf("parse docker port %q: %v", out, err)
+	}
+	return port
+}
+
+func freePort(t *testing.T) int {
+	t.Helper()
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = l.Close() }()
+	return l.Addr().(*net.TCPAddr).Port
+}
+
+const readyTimeout = 60 * time.Second
+
+// eventually retries fn until it succeeds or readyTimeout expires.
+func eventually(t *testing.T, what string, fn func() error) {
+	t.Helper()
+	deadline := time.Now().Add(readyTimeout)
+	var err error
+	for time.Now().Before(deadline) {
+		if err = fn(); err == nil {
+			return
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	t.Fatalf("%s: %v", what, err)
+}
+
+func TestIntegrationMinIO(t *testing.T) {
+	requireDocker(t)
+	id := runContainer(t, "-p", "127.0.0.1::9000",
+		"-e", "MINIO_ROOT_USER=minioadmin", "-e", "MINIO_ROOT_PASSWORD=minioadmin",
+		image("GO_STORAGE_MINIO_IMAGE", "pgsty/minio:latest"), "server", "/data")
+	port := hostPort(t, id, "9000/tcp")
+
+	s, err := NewS3Storage(Config{
+		S3Bucket:      "conformance",
+		S3Endpoint:    "127.0.0.1:" + port,
+		S3DisableTLS:  true,
+		S3ForcePath:   true,
+		S3KeyID:       "minioadmin",
+		S3Secret:      "minioadmin",
+		S3PartSize:    MinS3PartSize, // 20 MiB stream => real multipart upload
+		S3Concurrency: 2,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.Background()
+	eventually(t, "create bucket", func() error {
+		_, err := s.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String("conformance")})
+		return err
+	})
+	runConformance(t, s)
+
+	// Failed multipart uploads must have been aborted.
+	out, err := s.client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{Bucket: aws.String("conformance")})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(out.Uploads) != 0 {
+		t.Errorf("%d multipart uploads left behind", len(out.Uploads))
+	}
+}
+
+func TestIntegrationAzurite(t *testing.T) {
+	requireDocker(t)
+	id := runContainer(t, "-p", "127.0.0.1::10000",
+		image("GO_STORAGE_AZURITE_IMAGE", "mcr.microsoft.com/azure-storage/azurite:latest"),
+		"azurite-blob", "--blobHost", "0.0.0.0", "--skipApiVersionCheck", "--loose")
+	port := hostPort(t, id, "10000/tcp")
+
+	s, err := NewAzureStorage(Config{
+		AzureAccount:   "devstoreaccount1",
+		AzureKey:       "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
+		AzureContainer: "conformance",
+		AzureEndpoint:  "http://127.0.0.1:" + port + "/devstoreaccount1",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	eventually(t, "create container", func() error {
+		_, err := s.client.CreateContainer(context.Background(), "conformance", nil)
+		return err
+	})
+	runConformance(t, s)
+}
+
+func TestIntegrationFakeGCS(t *testing.T) {
+	requireDocker(t)
+	port := freePort(t)
+	addr := "127.0.0.1:" + strconv.Itoa(port)
+	runContainer(t, "-p", fmt.Sprintf("%s:4443", addr),
+		image("GO_STORAGE_FAKEGCS_IMAGE", "fsouza/fake-gcs-server:latest"),
+		"-scheme", "http", "-port", "4443", "-public-host", addr, "-external-url", "http://"+addr, "-backend", "memory")
+
+	ctx := context.Background()
+	s, err := newGCSStorage(ctx, "conformance",
+		option.WithEndpoint("http://"+addr+"/storage/v1/"),
+		option.WithoutAuthentication())
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = s.Close() })
+	eventually(t, "create bucket", func() error {
+		err := s.client.Bucket("conformance").Create(ctx, "test-project", &storage.BucketAttrs{})
+		var exists bool
+		if err != nil && strings.Contains(err.Error(), "409") {
+			exists = true
+		}
+		if exists {
+			return nil
+		}
+		return err
+	})
+	runConformance(t, s)
+}
+
+func TestIntegrationFTP(t *testing.T) {
+	requireDocker(t)
+	// Passive ports must be identical inside and outside the container.
+	minPort := 30000 + rand.Intn(20000)
+	maxPort := minPort + 9
+	ctrlPort := freePort(t)
+	runContainer(t,
+		"-p", fmt.Sprintf("127.0.0.1:%d:21", ctrlPort),
+		"-p", fmt.Sprintf("127.0.0.1:%d-%d:%d-%d", minPort, maxPort, minPort, maxPort),
+		"-e", "USERS=backup|s3cret|/ftp/backup",
+		"-e", "ADDRESS=127.0.0.1",
+		"-e", "MIN_PORT="+strconv.Itoa(minPort), "-e", "MAX_PORT="+strconv.Itoa(maxPort),
+		image("GO_STORAGE_FTP_IMAGE", "delfer/alpine-ftp-server:latest"))
+
+	var s *FTPStorage
+	eventually(t, "connect ftp", func() error {
+		var err error
+		s, err = NewFTPStorage(Config{
+			FTPHost:     "127.0.0.1",
+			FTPPort:     strconv.Itoa(ctrlPort),
+			FTPUsername: "backup",
+			FTPPassword: "s3cret",
+		})
+		if err != nil {
+			return err
+		}
+		if _, err := s.ListDir(context.Background(), ""); err != nil {
+			_ = s.Close()
+			return err
+		}
+		return nil
+	})
+	t.Cleanup(func() { _ = s.Close() })
+	runConformance(t, s)
+
+	if _, err := s.PutIf(context.Background(), "x", strings.NewReader("x"), Condition{IfNoneMatch: true}, nil); !errors.Is(err, ErrConditionNotSupported) {
+		t.Fatalf("FTP PutIf: %v, want ErrConditionNotSupported", err)
+	}
+}

local.go

@@ -1,343 +1,428 @@
+/*
+MIT License
+
+Copyright (c) 2024 Jonas Kaninda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
 package go_storage
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
+	"time"
 )
 
 // LocalStorage implements the Storage interface using the local filesystem.
-// It is safe for concurrent use as it relies on filesystem operations which
+// It is safe for concurrent use.
-// are handled by the OS.
 type LocalStorage struct {
 	basePath string
 }
 
+// Lock tuning for PutIf / Put on the local backend.
+const (
+	localLockTimeout = 10 * time.Second
+	localLockRetry   = 10 * time.Millisecond
+	// A lock older than this is considered stale (the holder crashed). The
+	// lock is only held around a stat + rename, never while streaming data.
+	localLockStale = 30 * time.Second
+)
+
 // NewLocalStorage creates a new LocalStorage instance with the given base directory.
 // The base directory will be created if it doesn't exist.
 func NewLocalStorage(basePath string) (*LocalStorage, error) {
 	absPath, err := filepath.Abs(basePath)
 	if err != nil {
-		return nil, fmt.Errorf("failed to resolve absolute path: %w", err)
+		return nil, fmt.Errorf("local: failed to resolve absolute path: %w", err)
 	}
 
-	if err := os.MkdirAll(absPath, 0755); err != nil {
+	if err := os.MkdirAll(absPath, 0o755); err != nil {
-		return nil, fmt.Errorf("failed to create base directory: %w", err)
+		return nil, fmt.Errorf("local: failed to create base directory: %w", err)
 	}
 
-	return &LocalStorage{
+	return &LocalStorage{basePath: absPath}, nil
-		basePath: absPath,
-	}, nil
 }
 
-// resolvePath safely resolves a remote path to an absolute local path,
+// resolvePath safely resolves a key to an absolute local path, ensuring it
-// ensuring it stays within the base directory.
+// stays within the base directory.
 func (s *LocalStorage) resolvePath(remotePath string) (string, error) {
-	// Clean the path to remove any .. or . components
+	key, err := cleanKey(remotePath)
-	cleanPath := filepath.Clean(remotePath)
+	if err != nil {
-
+		return "", fmt.Errorf("local: %w", err)
-	// Remove leading slashes to make it relative
+	}
-	cleanPath = strings.TrimPrefix(cleanPath, string(filepath.Separator))
+	return s.join(key)
-
-	// Join with base path
-	fullPath := filepath.Join(s.basePath, cleanPath)
-
-	// Verify the resolved path is still within basePath
-	if !strings.HasPrefix(fullPath, s.basePath) {
-		return "", fmt.Errorf("path traversal attempt detected: %s", remotePath)
 }
 
+func (s *LocalStorage) join(key string) (string, error) {
+	fullPath := filepath.Join(s.basePath, filepath.FromSlash(key))
+	rel, err := filepath.Rel(s.basePath, fullPath)
+	if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
+		return "", fmt.Errorf("local: path traversal attempt detected: %s", key)
+	}
 	return fullPath, nil
 }
 
+// localETag derives the version token from mtime and size.
+func localETag(fi fs.FileInfo) string {
+	return strconv.FormatInt(fi.ModTime().UnixNano(), 16) + "-" + strconv.FormatInt(fi.Size(), 16)
+}
+
+func (s *LocalStorage) item(full string, fi fs.FileInfo) Item {
+	rel, err := filepath.Rel(s.basePath, full)
+	if err != nil {
+		rel = full
+	}
+	it := Item{
+		Key:          filepath.ToSlash(rel),
+		ModifiedTime: fi.ModTime(),
+		IsDirectory:  fi.IsDir(),
+	}
+	if !fi.IsDir() {
+		it.Size = fi.Size()
+		it.ETag = localETag(fi)
+	}
+	return it
+}
+
 // Upload uploads a local file to the storage with the given target name.
 func (s *LocalStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
-	select {
+	return uploadFile(ctx, s, localPath, remotePath)
-	case <-ctx.Done():
-		return ctx.Err()
-	default:
-	}
-
-	targetPath, err := s.resolvePath(remotePath)
-	if err != nil {
-		return err
-	}
-
-	// Open source file
-	srcFile, err := os.Open(localPath)
-	if err != nil {
-		return fmt.Errorf("failed to open source file: %w", err)
-	}
-	defer srcFile.Close()
-
-	// Create parent directories if needed
-	if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
-		return fmt.Errorf("failed to create parent directories: %w", err)
-	}
-
-	// Create destination file
-	dstFile, err := os.Create(targetPath)
-	if err != nil {
-		return fmt.Errorf("failed to create destination file: %w", err)
-	}
-	defer dstFile.Close()
-
-	// Copy with context cancellation checks
-	buf := make([]byte, 32*1024) // 32KB buffer
-	for {
-		select {
-		case <-ctx.Done():
-			// Clean up partial file on cancellation
-			os.Remove(targetPath)
-			return ctx.Err()
-		default:
-		}
-
-		n, readErr := srcFile.Read(buf)
-		if n > 0 {
-			if _, writeErr := dstFile.Write(buf[:n]); writeErr != nil {
-				os.Remove(targetPath)
-				return fmt.Errorf("failed to write to destination: %w", writeErr)
-			}
-		}
-		if readErr == io.EOF {
-			break
-		}
-		if readErr != nil {
-			os.Remove(targetPath)
-			return fmt.Errorf("failed to read source file: %w", readErr)
-		}
-	}
-
-	return nil
 }
 
 // Download downloads a file from storage to a local path.
 func (s *LocalStorage) Download(ctx context.Context, remotePath string, localPath string) error {
-	select {
+	return downloadFile(ctx, s, remotePath, localPath)
-	case <-ctx.Done():
-		return ctx.Err()
-	default:
 }
 
-	sourcePath, err := s.resolvePath(remotePath)
+// writeTemp streams r into a new temporary file next to full and returns its path.
+func writeTemp(ctx context.Context, full string, r io.Reader) (string, error) {
+	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+		return "", fmt.Errorf("local: failed to create parent directories: %w", err)
+	}
+	tmp := partialName(full)
+	f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
 	if err != nil {
-		return err
+		return "", fmt.Errorf("local: failed to create temporary file: %w", err)
+	}
+	_, err = io.Copy(f, ctxReader{ctx: ctx, r: r})
+	if err == nil {
+		err = f.Sync()
+	}
+	if cerr := f.Close(); err == nil {
+		err = cerr
+	}
+	if err == nil {
+		err = ctx.Err()
 	}
-
-	// Open source file
-	srcFile, err := os.Open(sourcePath)
 	if err != nil {
-		return fmt.Errorf("failed to open remote file: %w", err)
+		_ = os.Remove(tmp)
+		return "", fmt.Errorf("local: failed to write data: %w", err)
 	}
-	defer srcFile.Close()
+	return tmp, nil
-
-	// Create parent directories for local path if needed
-	if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil {
-		return fmt.Errorf("failed to create local parent directories: %w", err)
 }
 
-	// Create destination file
+// Put streams r to key via a temporary file that is renamed on success.
-	dstFile, err := os.Create(localPath)
+func (s *LocalStorage) Put(ctx context.Context, key string, r io.Reader, _ *PutOptions) (Item, error) {
-	if err != nil {
+	return s.put(ctx, key, r, nil)
-		return fmt.Errorf("failed to create local file: %w", err)
-	}
-	defer dstFile.Close()
-
-	// Copy with context cancellation checks
-	buf := make([]byte, 32*1024) // 32KB buffer
-	for {
-		select {
-		case <-ctx.Done():
-			// Clean up partial file on cancellation
-			os.Remove(localPath)
-			return ctx.Err()
-		default:
 }
 
-		n, readErr := srcFile.Read(buf)
+// PutIf writes r to key only if cond holds. It uses an exclusive lock file
-		if n > 0 {
+// (<key>.lock) so it is atomic with respect to other PutIf/Put calls on the
-			if _, writeErr := dstFile.Write(buf[:n]); writeErr != nil {
+// same filesystem.
-				os.Remove(localPath)
+func (s *LocalStorage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, _ *PutOptions) (Item, error) {
-				return fmt.Errorf("failed to write to local file: %w", writeErr)
+	if err := validateCondition(cond); err != nil {
-			}
+		return Item{}, fmt.Errorf("local: %w", err)
-		}
-		if readErr == io.EOF {
-			break
-		}
-		if readErr != nil {
-			os.Remove(localPath)
-			return fmt.Errorf("failed to read remote file: %w", readErr)
 	}
+	return s.put(ctx, key, r, &cond)
 }
 
-	return nil
+func (s *LocalStorage) put(ctx context.Context, key string, r io.Reader, cond *Condition) (Item, error) {
-}
+	full, err := s.resolvePath(key)
-
-// List returns all items in the storage that match the given prefix.
-func (s *LocalStorage) List(ctx context.Context, prefix string) ([]Item, error) {
-	select {
-	case <-ctx.Done():
-		return nil, ctx.Err()
-	default:
-	}
-
-	var items []Item
-
-	// Clean and resolve the prefix path
-	searchPath := s.basePath
-	cleanPrefix := strings.TrimPrefix(filepath.Clean(prefix), string(filepath.Separator))
-
-	if cleanPrefix != "" && cleanPrefix != "." {
-		searchPath = filepath.Join(s.basePath, cleanPrefix)
-	}
-
-	// Check if the search path exists
-	info, err := os.Stat(searchPath)
-	if os.IsNotExist(err) {
-		// If the exact path doesn't exist, try to find files with this prefix
-		// by walking the parent directory
-		parentDir := filepath.Dir(searchPath)
-		baseName := filepath.Base(searchPath)
-
-		if _, err := os.Stat(parentDir); os.IsNotExist(err) {
-			return items, nil // Return empty list if parent doesn't exist
-		}
-
-		entries, err := os.ReadDir(parentDir)
-		if err != nil {
-			return nil, fmt.Errorf("failed to read directory: %w", err)
-		}
-
-		for _, entry := range entries {
-			select {
-			case <-ctx.Done():
-				return nil, ctx.Err()
-			default:
-			}
-
-			if strings.HasPrefix(entry.Name(), baseName) {
-				fullPath := filepath.Join(parentDir, entry.Name())
-				item, err := s.fileInfoToItem(fullPath)
-				if err != nil {
-					continue // Skip files we can't stat
-				}
-				items = append(items, item)
-			}
-		}
-		return items, nil
-	}
-
-	if err != nil {
-		return nil, fmt.Errorf("failed to stat path: %w", err)
-	}
-
-	// If it's a file, return just that file
-	if !info.IsDir() {
-		item, err := s.fileInfoToItem(searchPath)
-		if err != nil {
-			return nil, err
-		}
-		return []Item{item}, nil
-	}
-
-	// Walk the directory
-	err = filepath.WalkDir(searchPath, func(path string, d os.DirEntry, err error) error {
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		default:
-		}
-
-		if err != nil {
-			return nil // Skip entries with errors
-		}
-
-		item, err := s.fileInfoToItem(path)
-		if err != nil {
-			return nil // Skip files we can't stat
-		}
-		items = append(items, item)
-		return nil
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("failed to walk directory: %w", err)
-	}
-
-	return items, nil
-}
-
-// fileInfoToItem converts a file path to an Item.
-func (s *LocalStorage) fileInfoToItem(path string) (Item, error) {
-	info, err := os.Stat(path)
 	if err != nil {
 		return Item{}, err
 	}
-
+	tmp, err := writeTemp(ctx, full, r)
-	// Get relative path from base
-	relPath, err := filepath.Rel(s.basePath, path)
 	if err != nil {
-		relPath = path
+		return Item{}, err
+	}
+	committed := false
+	defer func() {
+		if !committed {
+			_ = os.Remove(tmp)
+		}
+	}()
+
+	unlock, err := acquireLock(ctx, full+".lock")
+	if err != nil {
+		return Item{}, err
+	}
+	defer unlock()
+
+	cur, statErr := os.Stat(full)
+	exists := statErr == nil
+	if statErr != nil && !errors.Is(statErr, fs.ErrNotExist) {
+		return Item{}, fmt.Errorf("local: failed to stat %s: %w", key, statErr)
+	}
+	if cond != nil {
+		switch {
+		case cond.IfNoneMatch && exists:
+			return Item{}, fmt.Errorf("local: %s already exists: %w", key, ErrPreconditionFailed)
+		case cond.IfMatch != "" && (!exists || localETag(cur) != cond.IfMatch):
+			return Item{}, fmt.Errorf("local: %s etag mismatch: %w", key, ErrPreconditionFailed)
+		}
+	}
+	if exists {
+		if err := bumpMtime(tmp, cur.ModTime()); err != nil {
+			return Item{}, err
+		}
+	}
+	if err := ctx.Err(); err != nil {
+		return Item{}, err
+	}
+	if err := os.Rename(tmp, full); err != nil {
+		return Item{}, fmt.Errorf("local: failed to rename temporary file: %w", err)
+	}
+	committed = true
+	fi, err := os.Stat(full)
+	if err != nil {
+		return Item{}, fmt.Errorf("local: failed to stat %s: %w", key, err)
+	}
+	return s.item(full, fi), nil
 }
 
-	// Normalize to forward slashes for consistency
+// bumpMtime makes sure the mtime of tmp is strictly after prev so that the
-	relPath = filepath.ToSlash(relPath)
+// ETag changes even on filesystems with coarse timestamps.
-
+func bumpMtime(tmp string, prev time.Time) error {
-	size := info.Size()
+	for _, step := range []time.Duration{time.Microsecond, time.Millisecond, time.Second, 2 * time.Second} {
-	if info.IsDir() {
+		fi, err := os.Stat(tmp)
-		size = 0
+		if err != nil {
+			return fmt.Errorf("local: failed to stat temporary file: %w", err)
+		}
+		if fi.ModTime().After(prev) {
+			return nil
+		}
+		t := prev.Add(step)
+		if err := os.Chtimes(tmp, t, t); err != nil {
+			return fmt.Errorf("local: failed to set mtime: %w", err)
+		}
+	}
+	return nil
 }
 
-	return Item{
+// acquireLock creates lockPath exclusively, retrying until localLockTimeout.
-		Key:          relPath,
+func acquireLock(ctx context.Context, lockPath string) (func(), error) {
-		ModifiedTime: info.ModTime(),
+	deadline := time.Now().Add(localLockTimeout)
-		Size:         size,
+	for {
-		IsDirectory:  info.IsDir(),
+		f, err := os.OpenFile(lockPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
-	}, nil
+		if err == nil {
+			_, _ = fmt.Fprintf(f, "%d\n", os.Getpid())
+			_ = f.Close()
+			return func() { _ = os.Remove(lockPath) }, nil
+		}
+		if !errors.Is(err, fs.ErrExist) {
+			return nil, fmt.Errorf("local: failed to create lock %s: %w", lockPath, err)
+		}
+		if fi, serr := os.Stat(lockPath); serr == nil && time.Since(fi.ModTime()) > localLockStale {
+			_ = os.Remove(lockPath)
+			continue
+		}
+		if time.Now().After(deadline) {
+			return nil, fmt.Errorf("local: timed out waiting for lock %s", lockPath)
 		}
-
-// Remove deletes a file from storage.
-func (s *LocalStorage) Remove(ctx context.Context, remotePath string) error {
 		select {
 		case <-ctx.Done():
-		return ctx.Err()
+			return nil, ctx.Err()
-	default:
+		case <-time.After(localLockRetry):
+		}
+	}
 }
 
+// Get opens key for reading.
+func (s *LocalStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, Item{}, err
+	}
+	full, err := s.resolvePath(key)
+	if err != nil {
+		return nil, Item{}, err
+	}
+	f, err := os.Open(full)
+	if err != nil {
+		return nil, Item{}, mapLocalErr(key, err)
+	}
+	fi, err := f.Stat()
+	if err != nil {
+		_ = f.Close()
+		return nil, Item{}, mapLocalErr(key, err)
+	}
+	if fi.IsDir() {
+		_ = f.Close()
+		return nil, Item{}, fmt.Errorf("local: %s is a directory", key)
+	}
+	return f, s.item(full, fi), nil
+}
+
+// Stat returns metadata for key.
+func (s *LocalStorage) Stat(ctx context.Context, key string) (Item, error) {
+	if err := ctx.Err(); err != nil {
+		return Item{}, err
+	}
+	full, err := s.resolvePath(key)
+	if err != nil {
+		return Item{}, err
+	}
+	fi, err := os.Stat(full)
+	if err != nil {
+		return Item{}, mapLocalErr(key, err)
+	}
+	return s.item(full, fi), nil
+}
+
+func mapLocalErr(key string, err error) error {
+	if errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("local: %s: %w", key, ErrNotFound)
+	}
+	return fmt.Errorf("local: %s: %w", key, err)
+}
+
+// ListDir lists the files directly inside dir.
+func (s *LocalStorage) ListDir(ctx context.Context, dir string) ([]Item, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	d, err := cleanDir(dir)
+	if err != nil {
+		return nil, fmt.Errorf("local: %w", err)
+	}
+	full, err := s.join(d)
+	if err != nil {
+		return nil, err
+	}
+	entries, err := os.ReadDir(full)
+	if errors.Is(err, fs.ErrNotExist) {
+		return []Item{}, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("local: failed to read directory %s: %w", dir, err)
+	}
+	items := make([]Item, 0, len(entries))
+	for _, e := range entries {
+		if e.IsDir() {
+			continue
+		}
+		fi, err := e.Info()
+		if err != nil || !fi.Mode().IsRegular() {
+			continue
+		}
+		items = append(items, s.item(filepath.Join(full, e.Name()), fi))
+	}
+	return items, nil
+}
+
+// List returns all items (files and directories, recursively) whose key
+// starts with prefix.
+func (s *LocalStorage) List(ctx context.Context, prefix string) ([]Item, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	p, err := cleanPrefix(prefix)
+	if err != nil {
+		return nil, fmt.Errorf("local: %w", err)
+	}
+
+	// Walk from the deepest directory fully contained in the prefix.
+	root := s.basePath
+	if i := strings.LastIndex(p, "/"); i >= 0 {
+		root = filepath.Join(s.basePath, filepath.FromSlash(p[:i]))
+	}
+	items := make([]Item, 0)
+	err = filepath.WalkDir(root, func(full string, d fs.DirEntry, err error) error {
+		if cerr := ctx.Err(); cerr != nil {
+			return cerr
+		}
+		if err != nil {
+			if full == root && errors.Is(err, fs.ErrNotExist) {
+				return fs.SkipAll
+			}
+			return nil // skip unreadable entries
+		}
+		if full == s.basePath {
+			return nil
+		}
+		rel, rerr := filepath.Rel(s.basePath, full)
+		if rerr != nil {
+			return nil
+		}
+		rel = filepath.ToSlash(rel)
+		if !strings.HasPrefix(rel, p) {
+			// Prune directories that cannot contain matches.
+			if d.IsDir() && !strings.HasPrefix(p, rel+"/") {
+				return fs.SkipDir
+			}
+			return nil
+		}
+		fi, ierr := d.Info()
+		if ierr != nil {
+			return nil
+		}
+		items = append(items, s.item(full, fi))
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("local: failed to walk directory: %w", err)
+	}
+	return items, nil
+}
+
+// Remove deletes a file (or directory tree) from storage.
+func (s *LocalStorage) Remove(ctx context.Context, remotePath string) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
 	targetPath, err := s.resolvePath(remotePath)
 	if err != nil {
 		return err
 	}
-
-	// Check if file exists
 	info, err := os.Stat(targetPath)
-	if os.IsNotExist(err) {
+	if err != nil {
-		return fmt.Errorf("file not found: %s", remotePath)
+		return mapLocalErr(remotePath, err)
+	}
+	if info.IsDir() {
+		err = os.RemoveAll(targetPath)
+	} else {
+		err = os.Remove(targetPath)
 	}
 	if err != nil {
-		return fmt.Errorf("failed to stat file: %w", err)
+		return mapLocalErr(remotePath, err)
 	}
-
-	// Remove file or directory
-	if info.IsDir() {
-		if err := os.RemoveAll(targetPath); err != nil {
-			return fmt.Errorf("failed to remove directory: %w", err)
-		}
-	} else {
-		if err := os.Remove(targetPath); err != nil {
-			return fmt.Errorf("failed to remove file: %w", err)
-		}
-	}
-
 	return nil
 }
 
-// Close releases any resources held by the storage implementation.
+// Close releases any resources held by the storage implementation (no-op).
-// For LocalStorage, this is a no-op as there are no persistent resources.
 func (s *LocalStorage) Close() error {
 	return nil
 }

pkg/azure/azure.go

@@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 */
 
+// Package azure is the legacy v0.1 API of go-storage.
+//
+// Deprecated: use the root package github.com/jkaninda/go-storage
+// (NewStorage and the streaming Storage interface) instead.
 package azure
 
 import (

pkg/ftp/ftp.go

@@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 */
 
+// Package ftp is the legacy v0.1 API of go-storage.
+//
+// Deprecated: use the root package github.com/jkaninda/go-storage
+// (NewStorage and the streaming Storage interface) instead.
 package ftp
 
 import (

pkg/local/local.go

@@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 */
 
+// Package local is the legacy v0.1 API of go-storage.
+//
+// Deprecated: use the root package github.com/jkaninda/go-storage
+// (NewStorage and the streaming Storage interface) instead.
 package local
 
 import (

pkg/local/local_test.go

@@ -1,3 +1,27 @@
+/*
+MIT License
+
+Copyright (c) 2024 Jonas Kaninda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
 package local
 
 import (

pkg/s3/s3.go

@@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 */
 
+// Package s3 is the legacy v0.1 API of go-storage.
+//
+// Deprecated: use the root package github.com/jkaninda/go-storage
+// (NewStorage and the streaming Storage interface) instead.
 package s3
 
 import (

pkg/ssh/ssh.go

@@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 */
 
+// Package ssh is the legacy v0.1 API of go-storage.
+//
+// Deprecated: use the root package github.com/jkaninda/go-storage
+// (NewStorage and the streaming Storage interface) instead.
 package ssh
 
 import (

pkg/ssh/ssh_test.go

@@ -1 +1,25 @@
+/*
+MIT License
+
+Copyright (c) 2024 Jonas Kaninda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
 package ssh

pkg/storage.go

@@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 */
 
+// Package pkg is the legacy v0.1 API of go-storage.
+//
+// Deprecated: use the root package github.com/jkaninda/go-storage
+// (NewStorage and the streaming Storage interface) instead.
 package pkg
 
 type Storage interface {

s3.go

@@ -1,25 +1,74 @@
+/*
+MIT License
+
+Copyright (c) 2024 Jonas Kaninda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
 package go_storage
 
 import (
+	"bytes"
 	"context"
+	"errors"
 	"fmt"
-	"os"
+	"io"
+	"net/http"
+	"strings"
+	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
+	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
-	"github.com/aws/aws-sdk-go/aws/session"
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
-	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
 )
 
-// S3Storage implements Storage interface for Amazon S3.
+// S3 defaults.
+const (
+	DefaultS3PartSize    int64 = 64 << 20
+	MinS3PartSize        int64 = 5 << 20
+	DefaultS3Concurrency       = 2
+	DefaultS3Region            = "us-east-1"
+)
+
+// S3Storage implements Storage for Amazon S3 and S3-compatible services
+// (MinIO, Ceph, Wasabi, ...).
 type S3Storage struct {
-	session *session.Session
+	client       *s3.Client
+	uploader     *manager.Uploader //nolint:staticcheck // see NewS3Storage
 	bucket       string
 	region       string
+	storageClass types.StorageClass
+	sse          types.ServerSideEncryption
+	sseKMSKeyID  string
 }
 
 // NewS3Storage creates a new S3 storage instance.
+//
+// Credentials: static keys (S3KeyID/S3Secret/S3SessionToken) when set,
+// otherwise the default AWS credential chain (environment, shared profile
+// S3Profile, web identity/IRSA, ECS/EC2 roles).
 func NewS3Storage(config Config) (*S3Storage, error) {
 	if err := validateConfig(config, map[string]string{
 		"bucket": config.S3Bucket,
@@ -27,161 +76,330 @@ func NewS3Storage(config Config) (*S3Storage, error) {
 		return nil, fmt.Errorf("s3: %w", err)
 	}
 
-	awsConfig := aws.NewConfig()
+	opts := []func(*awsconfig.LoadOptions) error{
-
+		awsconfig.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired),
+		awsconfig.WithResponseChecksumValidation(aws.ResponseChecksumValidationWhenRequired),
+	}
 	if config.S3Region != "" {
-		awsConfig = awsConfig.WithRegion(config.S3Region)
+		opts = append(opts, awsconfig.WithRegion(config.S3Region))
 	}
-
-	if config.S3KeyID != "" && config.S3Secret != "" {
-		awsConfig = awsConfig.WithCredentials(
-			credentials.NewStaticCredentials(config.S3KeyID, config.S3Secret, ""),
-		)
-	}
-
-	if config.S3Endpoint != "" {
-		awsConfig = awsConfig.WithEndpoint(config.S3Endpoint)
-	}
-
-	if config.S3ForcePath {
-		awsConfig = awsConfig.WithS3ForcePathStyle(true)
-	}
-
-	if config.S3DisableTLS {
-		awsConfig = awsConfig.WithDisableSSL(true)
-	}
-
-	sessionOpts := session.Options{
-		Config:            *awsConfig,
-		SharedConfigState: session.SharedConfigEnable,
-	}
-
 	if config.S3Profile != "" {
-		sessionOpts.Profile = config.S3Profile
+		opts = append(opts, awsconfig.WithSharedConfigProfile(config.S3Profile))
+	}
+	if config.S3KeyID != "" && config.S3Secret != "" {
+		opts = append(opts, awsconfig.WithCredentialsProvider(
+			credentials.NewStaticCredentialsProvider(config.S3KeyID, config.S3Secret, config.S3SessionToken)))
 	}
 
-	sess, err := session.NewSessionWithOptions(sessionOpts)
+	cfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...)
 	if err != nil {
-		return nil, fmt.Errorf("s3: failed to create session: %w", err)
+		return nil, fmt.Errorf("s3: failed to load AWS configuration: %w", err)
+	}
+	if cfg.Region == "" {
+		cfg.Region = DefaultS3Region
 	}
 
-	return &S3Storage{
+	endpoint := config.S3Endpoint
-		session: sess,
+	if endpoint != "" && !strings.Contains(endpoint, "://") {
+		if config.S3DisableTLS {
+			endpoint = "http://" + endpoint
+		} else {
+			endpoint = "https://" + endpoint
+		}
+	}
+
+	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
+		if endpoint != "" {
+			o.BaseEndpoint = aws.String(endpoint)
+		}
+		o.UsePathStyle = config.S3ForcePath
+	})
+
+	partSize := config.S3PartSize
+	if partSize <= 0 {
+		partSize = DefaultS3PartSize
+	}
+	if partSize < MinS3PartSize {
+		partSize = MinS3PartSize
+	}
+	concurrency := config.S3Concurrency
+	if concurrency <= 0 {
+		concurrency = DefaultS3Concurrency
+	}
+
+	//nolint:staticcheck // feature/s3/manager is deprecated in favour of the (pre-1.0) transfermanager.
+	uploader := manager.NewUploader(client, func(u *manager.Uploader) {
+		u.PartSize = partSize
+		u.Concurrency = concurrency
+		u.LeavePartsOnError = false
+		u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
+	})
+
+	s := &S3Storage{
+		client:       client,
+		uploader:     uploader,
 		bucket:       config.S3Bucket,
-		region:  config.S3Region,
+		region:       cfg.Region,
+		storageClass: types.StorageClass(config.S3StorageClass),
+	}
+	switch strings.ToLower(config.S3SSE) {
+	case "":
+	case "aes256":
+		s.sse = types.ServerSideEncryptionAes256
+	case "aws:kms":
+		s.sse = types.ServerSideEncryptionAwsKms
+		s.sseKMSKeyID = config.S3SSEKMSKeyID
+	default:
+		s.sse = types.ServerSideEncryption(config.S3SSE)
+		s.sseKMSKeyID = config.S3SSEKMSKeyID
+	}
+	return s, nil
+}
+
+// Client returns the underlying S3 client.
+func (s *S3Storage) Client() *s3.Client { return s.client }
+
+func (s *S3Storage) putInput(key string, body io.Reader, opts *PutOptions) *s3.PutObjectInput {
+	in := &s3.PutObjectInput{
+		Bucket: aws.String(s.bucket),
+		Key:    aws.String(key),
+		Body:   body,
+	}
+	if opts != nil {
+		if opts.ContentType != "" {
+			in.ContentType = aws.String(opts.ContentType)
+		}
+		if len(opts.Metadata) > 0 {
+			in.Metadata = opts.Metadata
+		}
+	}
+	if s.storageClass != "" {
+		in.StorageClass = s.storageClass
+	}
+	if s.sse != "" {
+		in.ServerSideEncryption = s.sse
+		if s.sseKMSKeyID != "" {
+			in.SSEKMSKeyId = aws.String(s.sseKMSKeyID)
+		}
+	}
+	return in
+}
+
+// Put streams r to key using a multipart upload (aborted on error).
+func (s *S3Storage) Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error) {
+	k, err := cleanKey(key)
+	if err != nil {
+		return Item{}, fmt.Errorf("s3: %w", err)
+	}
+	cr := &countingReader{r: ctxReader{ctx: ctx, r: r}}
+	//nolint:staticcheck // see NewS3Storage
+	out, err := s.uploader.Upload(ctx, s.putInput(k, cr, opts))
+	if err != nil {
+		return Item{}, fmt.Errorf("s3: failed to upload %s to bucket %s: %w", k, s.bucket, err)
+	}
+	return Item{
+		Key:          k,
+		Size:         cr.n,
+		ModifiedTime: time.Now().UTC(),
+		ETag:         aws.ToString(out.ETag),
+		VersionID:    aws.ToString(out.VersionID),
+	}, nil
+}
+
+// PutIf performs a conditional single-request PutObject (If-Match /
+// If-None-Match: *). The body is buffered and limited to 16 MiB.
+func (s *S3Storage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error) {
+	k, err := cleanKey(key)
+	if err != nil {
+		return Item{}, fmt.Errorf("s3: %w", err)
+	}
+	if err := validateCondition(cond); err != nil {
+		return Item{}, fmt.Errorf("s3: %w", err)
+	}
+	data, err := io.ReadAll(io.LimitReader(ctxReader{ctx: ctx, r: r}, maxConditionalPutSize+1))
+	if err != nil {
+		return Item{}, fmt.Errorf("s3: failed to read data: %w", err)
+	}
+	if len(data) > maxConditionalPutSize {
+		return Item{}, fmt.Errorf("s3: conditional put of %s exceeds %d bytes", k, maxConditionalPutSize)
+	}
+	in := s.putInput(k, bytes.NewReader(data), opts)
+	in.ContentLength = aws.Int64(int64(len(data)))
+	if cond.IfMatch != "" {
+		in.IfMatch = aws.String(cond.IfMatch)
+	}
+	if cond.IfNoneMatch {
+		in.IfNoneMatch = aws.String("*")
+	}
+	out, err := s.client.PutObject(ctx, in)
+	if err != nil {
+		// If-Match on a missing key yields 404 NoSuchKey on AWS.
+		if isS3Precondition(err) || (cond.IfMatch != "" && s3StatusCode(err) == http.StatusNotFound) {
+			return Item{}, fmt.Errorf("s3: conditional put of %s: %w", k, ErrPreconditionFailed)
+		}
+		return Item{}, fmt.Errorf("s3: failed to put %s: %w", k, err)
+	}
+	return Item{
+		Key:          k,
+		Size:         int64(len(data)),
+		ModifiedTime: time.Now().UTC(),
+		ETag:         aws.ToString(out.ETag),
+		VersionID:    aws.ToString(out.VersionId),
+	}, nil
+}
+
+// Get streams the object body.
+func (s *S3Storage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
+	k, err := cleanKey(key)
+	if err != nil {
+		return nil, Item{}, fmt.Errorf("s3: %w", err)
+	}
+	out, err := s.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(k)})
+	if err != nil {
+		return nil, Item{}, s.mapErr("get", k, err)
+	}
+	return out.Body, Item{
+		Key:          k,
+		Size:         aws.ToInt64(out.ContentLength),
+		ModifiedTime: aws.ToTime(out.LastModified),
+		ETag:         aws.ToString(out.ETag),
+		VersionID:    aws.ToString(out.VersionId),
+	}, nil
+}
+
+// Stat returns object metadata (HeadObject).
+func (s *S3Storage) Stat(ctx context.Context, key string) (Item, error) {
+	k, err := cleanKey(key)
+	if err != nil {
+		return Item{}, fmt.Errorf("s3: %w", err)
+	}
+	out, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(k)})
+	if err != nil {
+		return Item{}, s.mapErr("stat", k, err)
+	}
+	return Item{
+		Key:          k,
+		Size:         aws.ToInt64(out.ContentLength),
+		ModifiedTime: aws.ToTime(out.LastModified),
+		ETag:         aws.ToString(out.ETag),
+		VersionID:    aws.ToString(out.VersionId),
 	}, nil
 }
 
 // Upload uploads a file to S3.
 func (s *S3Storage) Upload(ctx context.Context, localPath string, remotePath string) error {
-	file, err := os.Open(localPath)
+	if err := uploadFile(ctx, s, localPath, remotePath); err != nil {
-	if err != nil {
+		return fmt.Errorf("s3: %w", err)
-		return fmt.Errorf("s3: failed to open local file: %w", err)
 	}
-	defer func(file *os.File) {
-		err = file.Close()
-		if err != nil {
-			fmt.Printf("s3: warning: failed to close file: %v\n", err)
-		}
-	}(file)
-
-	uploader := s3manager.NewUploader(s.session)
-
-	_, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(normalizePathSeparators(remotePath)),
-		Body:   file,
-	})
-
-	if err != nil {
-		return fmt.Errorf("s3: failed to upload to bucket %s: %w", s.bucket, err)
-	}
-
 	return nil
 }
 
 // Download downloads a file from S3.
 func (s *S3Storage) Download(ctx context.Context, remotePath string, localPath string) error {
-	file, err := os.Create(localPath)
+	if err := downloadFile(ctx, s, remotePath, localPath); err != nil {
-	if err != nil {
+		return fmt.Errorf("s3: %w", err)
-		return fmt.Errorf("s3: failed to create local file: %w", err)
 	}
-	defer func(file *os.File) {
-		err = file.Close()
-		if err != nil {
-			fmt.Printf("s3: warning: failed to close file: %v\n", err)
-		}
-	}(file)
-
-	downloader := s3manager.NewDownloader(s.session)
-
-	_, err = downloader.DownloadWithContext(ctx, file, &s3.GetObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(normalizePathSeparators(remotePath)),
-	})
-
-	if err != nil {
-		return fmt.Errorf("s3: failed to download from bucket %s: %w", s.bucket, err)
-	}
-
 	return nil
 }
 
-// List lists objects in S3 with the given prefix.
+// List lists objects (recursively) with the given prefix.
 func (s *S3Storage) List(ctx context.Context, prefix string) ([]Item, error) {
-	svc := s3.New(s.session)
+	p, err := cleanPrefix(prefix)
-	items := make([]Item, 0)
+	if err != nil {
-
+		return nil, fmt.Errorf("s3: %w", err)
-	var continuationToken *string
+	}
-
+	return s.list(ctx, p, "")
-	for {
-		input := &s3.ListObjectsV2Input{
-			Bucket:            aws.String(s.bucket),
-			Prefix:            aws.String(normalizePathSeparators(prefix)),
-			ContinuationToken: continuationToken,
 }
 
-		resp, err := svc.ListObjectsV2WithContext(ctx, input)
+// ListDir lists the objects directly under dir.
+func (s *S3Storage) ListDir(ctx context.Context, dir string) ([]Item, error) {
+	d, err := cleanDir(dir)
+	if err != nil {
+		return nil, fmt.Errorf("s3: %w", err)
+	}
+	return s.list(ctx, dirPrefix(d), "/")
+}
+
+func (s *S3Storage) list(ctx context.Context, prefix, delimiter string) ([]Item, error) {
+	in := &s3.ListObjectsV2Input{Bucket: aws.String(s.bucket), Prefix: aws.String(prefix)}
+	if delimiter != "" {
+		in.Delimiter = aws.String(delimiter)
+	}
+	items := make([]Item, 0)
+	p := s3.NewListObjectsV2Paginator(s.client, in)
+	for p.HasMorePages() {
+		page, err := p.NextPage(ctx)
 		if err != nil {
 			return nil, fmt.Errorf("s3: failed to list objects in bucket %s: %w", s.bucket, err)
 		}
-
+		for _, obj := range page.Contents {
-		for _, obj := range resp.Contents {
+			key := aws.ToString(obj.Key)
+			if delimiter != "" && strings.HasSuffix(key, "/") {
+				continue // directory marker
+			}
 			items = append(items, Item{
-				Key:          *obj.Key,
+				Key:          key,
-				ModifiedTime: *obj.LastModified,
+				ModifiedTime: aws.ToTime(obj.LastModified),
-				Size:         *obj.Size,
+				Size:         aws.ToInt64(obj.Size),
-				IsDirectory:  false,
+				ETag:         aws.ToString(obj.ETag),
 			})
 		}
-
-		if resp.IsTruncated == nil || !*resp.IsTruncated {
-			break
 	}
-
-		continuationToken = resp.NextContinuationToken
-	}
-
 	return items, nil
 }
 
-// Remove deletes an object from S3.
+// Remove deletes an object. A missing key returns ErrNotFound (checked with
+// HeadObject because DeleteObject is idempotent on S3).
 func (s *S3Storage) Remove(ctx context.Context, remotePath string) error {
-	svc := s3.New(s.session)
+	k, err := cleanKey(remotePath)
-
-	_, err := svc.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(normalizePathSeparators(remotePath)),
-	})
-
 	if err != nil {
-		return fmt.Errorf("s3: failed to remove object from bucket %s: %w", s.bucket, err)
+		return fmt.Errorf("s3: %w", err)
+	}
+	if _, err := s.Stat(ctx, k); err != nil {
+		return err
+	}
+	if _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(k)}); err != nil {
+		return s.mapErr("remove", k, err)
 	}
-
 	return nil
 }
 
-// Close releases resources.
+// Close releases resources (no-op).
 func (s *S3Storage) Close() error {
 	return nil
 }
+
+func (s *S3Storage) mapErr(op, key string, err error) error {
+	if s3StatusCode(err) == http.StatusNotFound {
+		return fmt.Errorf("s3: %s %s in bucket %s: %w", op, key, s.bucket, ErrNotFound)
+	}
+	var nsk *types.NoSuchKey
+	var nf *types.NotFound
+	if errors.As(err, &nsk) || errors.As(err, &nf) {
+		return fmt.Errorf("s3: %s %s in bucket %s: %w", op, key, s.bucket, ErrNotFound)
+	}
+	return fmt.Errorf("s3: %s %s in bucket %s: %w", op, key, s.bucket, err)
+}
+
+func s3StatusCode(err error) int {
+	var re *awshttp.ResponseError
+	if errors.As(err, &re) {
+		return re.HTTPStatusCode()
+	}
+	return 0
+}
+
+func isS3Precondition(err error) bool {
+	var apiErr smithy.APIError
+	if errors.As(err, &apiErr) {
+		switch apiErr.ErrorCode() {
+		case "PreconditionFailed", "ConditionalRequestConflict":
+			return true
+		}
+	}
+	switch s3StatusCode(err) {
+	case http.StatusPreconditionFailed, http.StatusConflict:
+		return true
+	}
+	return false
+}

sftp.go

@@ -1,3 +1,27 @@
+/*
+MIT License
+
+Copyright (c) 2024 Jonas Kaninda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
 package go_storage
 
 import (
@@ -5,11 +29,14 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"net"
 	"os"
 	"os/user"
+	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/pkg/sftp"
 
@@ -27,6 +54,9 @@ type SFTPStorage struct {
 }
 
 // NewSFTPStorage creates a new SFTP storage instance.
+//
+// Host keys are verified against SFTPKnownHostsFile (or ~/.ssh/known_hosts
+// and /etc/ssh/ssh_known_hosts) unless SFTPIgnoreKnownHosts is true.
 func NewSFTPStorage(config Config) (*SFTPStorage, error) {
 	if err := validateConfig(config, map[string]string{
 		"host": config.SFTPHost,
@@ -38,6 +68,7 @@ func NewSFTPStorage(config Config) (*SFTPStorage, error) {
 	if port == "" {
 		port = "22"
 	}
+	hostPort := net.JoinHostPort(config.SFTPHost, port)
 
 	username := config.SFTPUsername
 	if username == "" {
@@ -48,21 +79,15 @@ func NewSFTPStorage(config Config) (*SFTPStorage, error) {
 		username = currentUser.Username
 	}
 
-	password := config.SFTPPassword
+	authMethods := make([]ssh.AuthMethod, 0, 2)
-	if password == "" {
+	if config.SFTPPassword != "" {
-		password = os.Getenv("PGBK_SSH_PASS")
+		authMethods = append(authMethods, ssh.Password(config.SFTPPassword))
 	}
-
+	passphrase := config.SFTPIdentityPassphrase
-	// Build authentication methods
+	if passphrase == "" {
-	authMethods := make([]ssh.AuthMethod, 0)
+		passphrase = config.SFTPPassword // backwards compatibility
-
-	// Password authentication (if no identity file is specified)
-	if config.SFTPIdentityFile == "" && password != "" {
-		authMethods = append(authMethods, ssh.Password(password))
 	}
-
+	signers, err := getSSHSigners(config.SFTPIdentityFile, passphrase)
-	// Public key authentication
-	signers, err := getSSHSigners(config.SFTPIdentityFile, password)
 	if err != nil {
 		return nil, fmt.Errorf("sftp: %w", err)
 	}
@@ -70,33 +95,36 @@ func NewSFTPStorage(config Config) (*SFTPStorage, error) {
 		authMethods = append(authMethods, ssh.PublicKeys(signers...))
 	}
 
-	// Build SSH client config
+	hostKeyCallback, hostKeyAlgos, err := getHostKeyCallback(config, hostPort)
+	if err != nil {
+		return nil, fmt.Errorf("sftp: %w", err)
+	}
+
 	sshConfig := &ssh.ClientConfig{
 		User:              username,
 		Auth:              authMethods,
-		HostKeyCallback: getHostKeyCallback(config.SFTPIgnoreKnownHosts),
+		HostKeyCallback:   hostKeyCallback,
+		HostKeyAlgorithms: hostKeyAlgos,
+		Timeout:           30 * time.Second,
 	}
 
-	// Connect to SSH server
-	hostPort := fmt.Sprintf("%s:%s", config.SFTPHost, port)
 	sshClient, err := ssh.Dial("tcp", hostPort, sshConfig)
 	if err != nil {
 		return nil, fmt.Errorf("sftp: failed to connect to %s: %w", hostPort, err)
 	}
 
-	// Open SFTP session
 	sftpClient, err := sftp.NewClient(sshClient)
 	if err != nil {
-		sshClient.Close()
+		_ = sshClient.Close()
 		return nil, fmt.Errorf("sftp: failed to open sftp session: %w", err)
 	}
 
-	baseDir := config.SFTPDirectory
+	baseDir := normalizePathSeparators(config.SFTPDirectory)
 	if baseDir == "" {
 		wd, err := sftpClient.Getwd()
 		if err != nil {
-			sftpClient.Close()
+			_ = sftpClient.Close()
-			sshClient.Close()
+			_ = sshClient.Close()
 			return nil, fmt.Errorf("sftp: failed to get working directory: %w", err)
 		}
 		baseDir = wd
@@ -104,169 +132,305 @@ func NewSFTPStorage(config Config) (*SFTPStorage, error) {
 
 	return &SFTPStorage{
 		host:      config.SFTPHost,
-		baseDir:   baseDir,
+		baseDir:   path.Clean(baseDir),
 		sshClient: sshClient,
 		client:    sftpClient,
 	}, nil
 }
 
+// Client returns the underlying sftp client.
+func (s *SFTPStorage) Client() *sftp.Client { return s.client }
+
+func (s *SFTPStorage) resolve(key string) (string, string, error) {
+	k, err := cleanKey(key)
+	if err != nil {
+		return "", "", fmt.Errorf("sftp: %w", err)
+	}
+	return k, path.Join(s.baseDir, k), nil
+}
+
+func (s *SFTPStorage) item(key string, fi os.FileInfo) Item {
+	it := Item{Key: key, ModifiedTime: fi.ModTime(), IsDirectory: fi.IsDir()}
+	if !fi.IsDir() {
+		it.Size = fi.Size()
+	}
+	return it
+}
+
+func (s *SFTPStorage) mkdirParent(full string) error {
+	dir := path.Dir(full)
+	if dir == "." || dir == "/" {
+		return nil
+	}
+	if err := s.client.MkdirAll(dir); err != nil {
+		return fmt.Errorf("sftp: failed to create directory %s: %w", dir, err)
+	}
+	return nil
+}
+
+// Put streams r to <key>.<rand>.partial and renames it to key on success.
+func (s *SFTPStorage) Put(ctx context.Context, key string, r io.Reader, _ *PutOptions) (Item, error) {
+	k, full, err := s.resolve(key)
+	if err != nil {
+		return Item{}, err
+	}
+	if err := s.mkdirParent(full); err != nil {
+		return Item{}, err
+	}
+	tmp := partialName(full)
+	f, err := s.client.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
+	if err != nil {
+		return Item{}, fmt.Errorf("sftp: failed to create %s: %w", tmp, err)
+	}
+	if err := copyAndClose(ctx, f, r); err != nil {
+		_ = s.client.Remove(tmp)
+		return Item{}, fmt.Errorf("sftp: failed to upload %s: %w", k, err)
+	}
+	if err := s.rename(tmp, full); err != nil {
+		_ = s.client.Remove(tmp)
+		return Item{}, err
+	}
+	return s.Stat(ctx, k)
+}
+
+func copyAndClose(ctx context.Context, w io.WriteCloser, r io.Reader) error {
+	_, err := io.Copy(w, ctxReader{ctx: ctx, r: r})
+	if cerr := w.Close(); err == nil {
+		err = cerr
+	}
+	if err == nil {
+		err = ctx.Err()
+	}
+	return err
+}
+
+func (s *SFTPStorage) rename(from, to string) error {
+	if _, ok := s.client.HasExtension("posix-rename@openssh.com"); ok {
+		if err := s.client.PosixRename(from, to); err != nil {
+			return fmt.Errorf("sftp: failed to rename %s to %s: %w", from, to, err)
+		}
+		return nil
+	}
+	if err := s.client.Remove(to); err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("sftp: failed to replace %s: %w", to, err)
+	}
+	if err := s.client.Rename(from, to); err != nil {
+		return fmt.Errorf("sftp: failed to rename %s to %s: %w", from, to, err)
+	}
+	return nil
+}
+
+// PutIf supports only Condition.IfNoneMatch (via O_EXCL on the final path).
+// IfMatch returns ErrConditionNotSupported.
+func (s *SFTPStorage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, _ *PutOptions) (Item, error) {
+	if cond.IfMatch != "" || !cond.IfNoneMatch {
+		return Item{}, fmt.Errorf("sftp: IfMatch: %w", ErrConditionNotSupported)
+	}
+	k, full, err := s.resolve(key)
+	if err != nil {
+		return Item{}, err
+	}
+	if err := s.mkdirParent(full); err != nil {
+		return Item{}, err
+	}
+	f, err := s.client.OpenFile(full, os.O_WRONLY|os.O_CREATE|os.O_EXCL)
+	if err != nil {
+		if _, serr := s.client.Lstat(full); serr == nil {
+			return Item{}, fmt.Errorf("sftp: %s already exists: %w", k, ErrPreconditionFailed)
+		}
+		return Item{}, fmt.Errorf("sftp: failed to create %s: %w", k, err)
+	}
+	if err := copyAndClose(ctx, f, r); err != nil {
+		_ = s.client.Remove(full)
+		return Item{}, fmt.Errorf("sftp: failed to upload %s: %w", k, err)
+	}
+	return s.Stat(ctx, k)
+}
+
+// Get opens key for streaming reads.
+func (s *SFTPStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, Item{}, err
+	}
+	k, full, err := s.resolve(key)
+	if err != nil {
+		return nil, Item{}, err
+	}
+	f, err := s.client.Open(full)
+	if err != nil {
+		return nil, Item{}, mapSFTPErr("get", k, err)
+	}
+	fi, err := f.Stat()
+	if err != nil {
+		_ = f.Close()
+		return nil, Item{}, mapSFTPErr("get", k, err)
+	}
+	if fi.IsDir() {
+		_ = f.Close()
+		return nil, Item{}, fmt.Errorf("sftp: %s is a directory", k)
+	}
+	return f, s.item(k, fi), nil
+}
+
+// Stat returns metadata for key.
+func (s *SFTPStorage) Stat(ctx context.Context, key string) (Item, error) {
+	if err := ctx.Err(); err != nil {
+		return Item{}, err
+	}
+	k, full, err := s.resolve(key)
+	if err != nil {
+		return Item{}, err
+	}
+	fi, err := s.client.Stat(full)
+	if err != nil {
+		return Item{}, mapSFTPErr("stat", k, err)
+	}
+	return s.item(k, fi), nil
+}
+
+func mapSFTPErr(op, key string, err error) error {
+	if errors.Is(err, fs.ErrNotExist) || errors.Is(err, sftp.ErrSSHFxNoSuchFile) {
+		return fmt.Errorf("sftp: %s %s: %w", op, key, ErrNotFound)
+	}
+	var se *sftp.StatusError
+	if errors.As(err, &se) && se.FxCode() == sftp.ErrSSHFxNoSuchFile {
+		return fmt.Errorf("sftp: %s %s: %w", op, key, ErrNotFound)
+	}
+	return fmt.Errorf("sftp: %s %s: %w", op, key, err)
+}
+
 // Upload uploads a file via SFTP.
 func (s *SFTPStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
-	src, err := os.Open(localPath)
+	return uploadFile(ctx, s, localPath, remotePath)
-	if err != nil {
-		return fmt.Errorf("sftp: failed to open local file: %w", err)
-	}
-	defer src.Close()
-
-	fullRemotePath := filepath.Join(s.baseDir, remotePath)
-	fullRemotePath = normalizePathSeparators(fullRemotePath)
-
-	// Create parent directory if needed
-	remoteDir := filepath.Dir(fullRemotePath)
-	remoteDir = normalizePathSeparators(remoteDir)
-
-	if remoteDir != "." && remoteDir != "/" {
-		if err := s.client.MkdirAll(remoteDir); err != nil {
-			return fmt.Errorf("sftp: failed to create directory %s: %w", remoteDir, err)
-		}
-	}
-
-	dst, err := s.client.Create(fullRemotePath)
-	if err != nil {
-		return fmt.Errorf("sftp: failed to create remote file %s: %w", fullRemotePath, err)
-	}
-	defer dst.Close()
-
-	if _, err := io.Copy(dst, src); err != nil {
-		return fmt.Errorf("sftp: failed to transfer data: %w", err)
-	}
-
-	return nil
 }
 
 // Download downloads a file via SFTP.
 func (s *SFTPStorage) Download(ctx context.Context, remotePath string, localPath string) error {
-	dst, err := os.Create(localPath)
+	return downloadFile(ctx, s, remotePath, localPath)
-	if err != nil {
-		return fmt.Errorf("sftp: failed to create local file: %w", err)
-	}
-	defer dst.Close()
-
-	fullRemotePath := filepath.Join(s.baseDir, remotePath)
-	fullRemotePath = normalizePathSeparators(fullRemotePath)
-
-	src, err := s.client.Open(fullRemotePath)
-	if err != nil {
-		return fmt.Errorf("sftp: failed to open remote file %s: %w", fullRemotePath, err)
-	}
-	defer src.Close()
-
-	if _, err := io.Copy(dst, src); err != nil {
-		return fmt.Errorf("sftp: failed to transfer data: %w", err)
 }
 
-	return nil
+// List lists files and directories (recursively) whose key starts with prefix.
-}
-
-// List lists files via SFTP with the given prefix.
 func (s *SFTPStorage) List(ctx context.Context, prefix string) ([]Item, error) {
-	items := make([]Item, 0)
+	p, err := cleanPrefix(prefix)
-
-	baseDir := normalizePathSeparators(s.baseDir)
-
-	walker := s.client.Walk(baseDir)
-	for walker.Step() {
-		if err := walker.Err(); err != nil {
-			return nil, fmt.Errorf("sftp: walk error: %w", err)
-		}
-
-		// Get relative path
-		path := walker.Path()
-		relPath, err := filepath.Rel(baseDir, path)
 	if err != nil {
+		return nil, fmt.Errorf("sftp: %w", err)
+	}
+	items := make([]Item, 0)
+	root := s.baseDir
+	if i := strings.LastIndex(p, "/"); i >= 0 {
+		root = path.Join(s.baseDir, p[:i])
+	}
+	walker := s.client.Walk(root)
+	for walker.Step() {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+		if err := walker.Err(); err != nil {
+			if walker.Path() == root && errors.Is(err, fs.ErrNotExist) {
+				return items, nil
+			}
 			continue
 		}
-
+		full := walker.Path()
-		// Normalize separators for comparison
+		if full == s.baseDir {
-		relPath = normalizePathSeparators(relPath)
-
-		if !strings.HasPrefix(relPath, prefix) {
 			continue
 		}
-
+		rel := strings.TrimPrefix(strings.TrimPrefix(full, s.baseDir), "/")
 		stat := walker.Stat()
-		items = append(items, Item{
+		if !strings.HasPrefix(rel, p) {
-			Key:          relPath,
+			if stat.IsDir() && !strings.HasPrefix(p, rel+"/") {
-			ModifiedTime: stat.ModTime(),
+				walker.SkipDir()
-			Size:         stat.Size(),
+			}
-			IsDirectory:  stat.IsDir(),
+			continue
-		})
+		}
+		items = append(items, s.item(rel, stat))
+	}
+	return items, nil
 }
 
+// ListDir lists the files directly inside dir.
+func (s *SFTPStorage) ListDir(ctx context.Context, dir string) ([]Item, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	d, err := cleanDir(dir)
+	if err != nil {
+		return nil, fmt.Errorf("sftp: %w", err)
+	}
+	entries, err := s.client.ReadDir(path.Join(s.baseDir, d))
+	if errors.Is(err, fs.ErrNotExist) {
+		return []Item{}, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("sftp: failed to read directory %s: %w", dir, err)
+	}
+	items := make([]Item, 0, len(entries))
+	for _, fi := range entries {
+		if !fi.Mode().IsRegular() {
+			continue
+		}
+		items = append(items, s.item(path.Join(d, fi.Name()), fi))
+	}
 	return items, nil
 }
 
 // Remove deletes a file via SFTP.
 func (s *SFTPStorage) Remove(ctx context.Context, remotePath string) error {
-	fullRemotePath := filepath.Join(s.baseDir, remotePath)
+	if err := ctx.Err(); err != nil {
-	fullRemotePath = normalizePathSeparators(fullRemotePath)
+		return err
-
+	}
-	if err := s.client.Remove(fullRemotePath); err != nil {
+	k, full, err := s.resolve(remotePath)
-		return fmt.Errorf("sftp: failed to remove %s: %w", fullRemotePath, err)
+	if err != nil {
+		return err
+	}
+	if err := s.client.Remove(full); err != nil {
+		return mapSFTPErr("remove", k, err)
 	}
-
 	return nil
 }
 
 // Close releases resources.
 func (s *SFTPStorage) Close() error {
-	if err := s.client.Close(); err != nil {
+	err := s.client.Close()
-		s.sshClient.Close()
+	if cerr := s.sshClient.Close(); err == nil {
-		return err
+		err = cerr
 	}
-	return s.sshClient.Close()
+	return err
 }
 
 // getSSHSigners returns SSH signers from identity file and SSH agent.
 func getSSHSigners(identityFile string, passphrase string) ([]ssh.Signer, error) {
 	signers := make([]ssh.Signer, 0)
 
-	// Load identity file if provided
 	if identityFile != "" {
-		path, err := expandHomeDir(identityFile)
+		p, err := expandHomeDir(identityFile)
 		if err != nil {
 			return nil, fmt.Errorf("failed to expand identity file path: %w", err)
 		}
 
-		keyData, err := os.ReadFile(path)
+		keyData, err := os.ReadFile(p)
 		if err != nil {
-			return nil, fmt.Errorf("failed to read identity file %s: %w", path, err)
+			return nil, fmt.Errorf("failed to read identity file %s: %w", p, err)
 		}
 
 		signer, err := ssh.ParsePrivateKey(keyData)
 		if err != nil {
 			var passErr *ssh.PassphraseMissingError
-			if errors.As(err, &passErr) {
+			if !errors.As(err, &passErr) {
+				return nil, fmt.Errorf("failed to parse identity file: %w", err)
+			}
 			signer, err = ssh.ParsePrivateKeyWithPassphrase(keyData, []byte(passphrase))
 			if err != nil {
 				return nil, fmt.Errorf("failed to decrypt identity file: %w", err)
 			}
-			} else {
-				return nil, fmt.Errorf("failed to parse identity file: %w", err)
 		}
-		}
-
 		signers = append(signers, signer)
 	}
 
-	// Try to get keys from SSH agent
+	// Keys from a running SSH agent, if any.
-	socket := os.Getenv("SSH_AUTH_SOCK")
+	if socket := os.Getenv("SSH_AUTH_SOCK"); socket != "" {
-	if socket != "" {
+		if conn, err := net.Dial("unix", socket); err == nil {
-		conn, err := net.Dial("unix", socket)
+			if agentSigners, err := agent.NewClient(conn).Signers(); err == nil {
-		if err == nil {
-			agentClient := agent.NewClient(conn)
-			agentSigners, err := agentClient.Signers()
-			if err == nil {
 				signers = append(signers, agentSigners...)
 			}
 		}
@@ -275,61 +439,108 @@ func getSSHSigners(identityFile string, passphrase string) ([]ssh.Signer, error)
 	return signers, nil
 }
 
-// getHostKeyCallback returns appropriate host key callback.
+// getHostKeyCallback builds the host key verification callback and the list
-func getHostKeyCallback(ignoreHostKey bool) ssh.HostKeyCallback {
+// of host key algorithms known for hostPort (so the server negotiates a key
-	if ignoreHostKey {
+// type present in known_hosts).
-		return ssh.InsecureIgnoreHostKey()
+func getHostKeyCallback(config Config, hostPort string) (ssh.HostKeyCallback, []string, error) {
+	if config.SFTPIgnoreKnownHosts {
+		return ssh.InsecureIgnoreHostKey(), nil, nil //nolint:gosec // explicitly requested
 	}
 
-	knownHostsFiles := make([]string, 0)
+	files := make([]string, 0, 2)
-	for _, p := range []string{"/etc/ssh/ssh_known_hosts", "~/.ssh/known_hosts"} {
+	if config.SFTPKnownHostsFile != "" {
-		path, err := expandHomeDir(p)
+		p, err := expandHomeDir(config.SFTPKnownHostsFile)
+		if err != nil {
+			return nil, nil, err
+		}
+		if _, err := os.Stat(p); err != nil {
+			return nil, nil, fmt.Errorf("known_hosts file %s: %w", p, err)
+		}
+		files = append(files, p)
+	} else {
+		for _, candidate := range []string{"~/.ssh/known_hosts", "/etc/ssh/ssh_known_hosts"} {
+			p, err := expandHomeDir(candidate)
 			if err != nil {
 				continue
 			}
-
+			if _, err := os.Stat(p); err == nil {
-		if _, err := os.Stat(path); err == nil {
+				files = append(files, p)
-			knownHostsFiles = append(knownHostsFiles, path)
 			}
 		}
+	}
+	if len(files) == 0 {
+		return nil, nil, fmt.Errorf("no known_hosts file available for host key verification " +
+			"(set SSH_KNOWN_HOSTS or SSH_INSECURE_IGNORE_HOST_KEY=true)")
+	}
 
-	if len(knownHostsFiles) == 0 {
+	callback, err := knownhosts.New(files...)
-		return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
-			return fmt.Errorf("no known_hosts files found for host key verification")
-		}
-	}
-
-	callback, err := knownhosts.New(knownHostsFiles...)
 	if err != nil {
-		return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
+		return nil, nil, fmt.Errorf("failed to load known_hosts: %w", err)
-			return fmt.Errorf("failed to load known_hosts: %w", err)
 	}
+	return callback, knownHostAlgorithms(callback, hostPort), nil
 }
 
-	return callback
+// knownHostAlgorithms returns the host key algorithms recorded for hostPort,
+// using the KeyError returned for a deliberately unknown key.
+func knownHostAlgorithms(cb ssh.HostKeyCallback, hostPort string) []string {
+	addr, err := net.ResolveTCPAddr("tcp", hostPort)
+	if err != nil {
+		addr = &net.TCPAddr{}
 	}
+	var keyErr *knownhosts.KeyError
+	if err := cb(hostPort, addr, probeKey{}); !errors.As(err, &keyErr) {
+		return nil
+	}
+	algos := make([]string, 0, len(keyErr.Want))
+	seen := map[string]bool{}
+	for _, k := range keyErr.Want {
+		for _, a := range algorithmsForKeyType(k.Key.Type()) {
+			if !seen[a] {
+				seen[a] = true
+				algos = append(algos, a)
+			}
+		}
+	}
+	if len(algos) == 0 {
+		return nil
+	}
+	return algos
+}
+
+func algorithmsForKeyType(t string) []string {
+	if t == ssh.KeyAlgoRSA {
+		return []string{ssh.KeyAlgoRSASHA512, ssh.KeyAlgoRSASHA256, ssh.KeyAlgoRSA}
+	}
+	return []string{t}
+}
+
+// probeKey is a public key that never matches anything in known_hosts.
+type probeKey struct{}
+
+func (probeKey) Type() string                        { return "probe-key" }
+func (probeKey) Marshal() []byte                     { return []byte("probe-key") }
+func (probeKey) Verify([]byte, *ssh.Signature) error { return errors.New("probe key") }
 
 // expandHomeDir expands ~ in file paths to home directory.
-func expandHomeDir(path string) (string, error) {
+func expandHomeDir(p string) (string, error) {
-	if !strings.HasPrefix(path, "~") {
+	if !strings.HasPrefix(p, "~") {
-		return filepath.Clean(path), nil
+		return filepath.Clean(p), nil
 	}
 
-	parts := strings.SplitN(path, "/", 2)
+	parts := strings.SplitN(p, "/", 2)
 	username := strings.TrimPrefix(parts[0], "~")
 
 	var homeDir string
-	var err error
-
 	if username == "" {
-		homeDir, err = os.UserHomeDir()
+		h, err := os.UserHomeDir()
-		if err != nil || homeDir == "" {
+		if err != nil || h == "" {
 			currentUser, err := user.Current()
 			if err != nil {
 				return "", fmt.Errorf("failed to get home directory: %w", err)
 			}
-			homeDir = currentUser.HomeDir
+			h = currentUser.HomeDir
 		}
+		homeDir = h
 	} else {
 		userInfo, err := user.Lookup(username)
 		if err != nil {
@@ -341,10 +552,8 @@ func expandHomeDir(path string) (string, error) {
 	if homeDir == "" {
 		return "", fmt.Errorf("empty home directory")
 	}
-
 	if len(parts) == 1 {
 		return homeDir, nil
 	}
-
 	return filepath.Join(homeDir, parts[1]), nil
 }

sftp_test.go

@@ -0,0 +1,208 @@
+/*
+MIT License
+
+Copyright (c) 2024 Jonas Kaninda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+package go_storage
+
+import (
+	"crypto/ed25519"
+	"crypto/rand"
+	"errors"
+	"io"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
+)
+
+const (
+	testSFTPUser = "backup"
+	testSFTPPass = "s3cret"
+)
+
+// startSFTPServer runs an in-process SSH/SFTP server on 127.0.0.1:0 and
+// returns its address and a known_hosts file containing its host key.
+func startSFTPServer(t *testing.T) (host, port, knownHostsFile string) {
+	t.Helper()
+	_, priv, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	signer, err := ssh.NewSignerFromKey(priv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg := &ssh.ServerConfig{
+		PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
+			if c.User() == testSFTPUser && string(pass) == testSFTPPass {
+				return nil, nil
+			}
+			return nil, errors.New("access denied")
+		},
+	}
+	cfg.AddHostKey(signer)
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = ln.Close() })
+	go func() {
+		for {
+			conn, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			go serveSSHConn(conn, cfg)
+		}
+	}()
+
+	addr := ln.Addr().String()
+	knownHostsFile = filepath.Join(t.TempDir(), "known_hosts")
+	line := knownhosts.Line([]string{knownhosts.Normalize(addr)}, signer.PublicKey())
+	if err := os.WriteFile(knownHostsFile, []byte(line+"\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	host, port, _ = net.SplitHostPort(addr)
+	return host, port, knownHostsFile
+}
+
+func serveSSHConn(conn net.Conn, cfg *ssh.ServerConfig) {
+	sconn, chans, reqs, err := ssh.NewServerConn(conn, cfg)
+	if err != nil {
+		_ = conn.Close()
+		return
+	}
+	defer func() { _ = sconn.Close() }()
+	go ssh.DiscardRequests(reqs)
+	for nc := range chans {
+		if nc.ChannelType() != "session" {
+			_ = nc.Reject(ssh.UnknownChannelType, "unsupported")
+			continue
+		}
+		ch, chReqs, err := nc.Accept()
+		if err != nil {
+			continue
+		}
+		go func() {
+			for req := range chReqs {
+				ok := req.Type == "subsystem" && len(req.Payload) > 4 && string(req.Payload[4:]) == "sftp"
+				_ = req.Reply(ok, nil)
+				if !ok {
+					continue
+				}
+				srv, err := sftp.NewServer(ch)
+				if err != nil {
+					_ = ch.Close()
+					return
+				}
+				if err := srv.Serve(); err != nil && !errors.Is(err, io.EOF) {
+					_ = srv.Close()
+				}
+				_ = ch.Close()
+				return
+			}
+		}()
+	}
+}
+
+func newTestSFTP(t *testing.T) (*SFTPStorage, string) {
+	t.Helper()
+	t.Setenv("SSH_AUTH_SOCK", "")
+	host, port, kh := startSFTPServer(t)
+	base := t.TempDir()
+	s, err := NewSFTPStorage(Config{
+		SFTPHost:           host,
+		SFTPPort:           port,
+		SFTPUsername:       testSFTPUser,
+		SFTPPassword:       testSFTPPass,
+		SFTPDirectory:      base,
+		SFTPKnownHostsFile: kh,
+	})
+	if err != nil {
+		t.Fatalf("NewSFTPStorage: %v", err)
+	}
+	t.Cleanup(func() { _ = s.Close() })
+	return s, base
+}
+
+func TestSFTPConformance(t *testing.T) {
+	s, base := newTestSFTP(t)
+	runConformance(t, s)
+
+	// No temporary files may survive anywhere under the base directory.
+	_ = filepath.WalkDir(base, func(p string, d os.DirEntry, err error) error {
+		if err == nil && strings.HasSuffix(p, ".partial") {
+			t.Errorf("leftover partial file %s", p)
+		}
+		return nil
+	})
+}
+
+func TestSFTPHostKeyVerification(t *testing.T) {
+	t.Setenv("SSH_AUTH_SOCK", "")
+	host, port, _ := startSFTPServer(t)
+	base := Config{SFTPHost: host, SFTPPort: port, SFTPUsername: testSFTPUser, SFTPPassword: testSFTPPass, SFTPDirectory: t.TempDir()}
+
+	// A known_hosts file with a different key for this host must be rejected.
+	_, otherPriv, _ := ed25519.GenerateKey(rand.Reader)
+	otherSigner, _ := ssh.NewSignerFromKey(otherPriv)
+	wrong := filepath.Join(t.TempDir(), "known_hosts")
+	line := knownhosts.Line([]string{knownhosts.Normalize(net.JoinHostPort(host, port))}, otherSigner.PublicKey())
+	if err := os.WriteFile(wrong, []byte(line+"\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	cfg := base
+	cfg.SFTPKnownHostsFile = wrong
+	if s, err := NewSFTPStorage(cfg); err == nil {
+		_ = s.Close()
+		t.Fatal("connected despite host key mismatch")
+	}
+
+	// Missing known_hosts file is an error with a helpful message.
+	cfg = base
+	cfg.SFTPKnownHostsFile = filepath.Join(t.TempDir(), "nope")
+	if _, err := NewSFTPStorage(cfg); err == nil {
+		t.Fatal("connected without known_hosts")
+	}
+	t.Setenv("HOME", t.TempDir())
+	cfg.SFTPKnownHostsFile = ""
+	if _, err := os.Stat("/etc/ssh/ssh_known_hosts"); os.IsNotExist(err) {
+		if _, err := NewSFTPStorage(cfg); err == nil || !strings.Contains(err.Error(), "SSH_INSECURE_IGNORE_HOST_KEY") {
+			t.Fatalf("expected helpful known_hosts error, got %v", err)
+		}
+	}
+
+	// Explicitly ignoring host keys works.
+	cfg.SFTPIgnoreKnownHosts = true
+	s, err := NewSFTPStorage(cfg)
+	if err != nil {
+		t.Fatalf("SFTPIgnoreKnownHosts: %v", err)
+	}
+	_ = s.Close()
+}

storage.go

@@ -1,15 +1,64 @@
+/*
+MIT License
+
+Copyright (c) 2024 Jonas Kaninda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Package go_storage provides a uniform, streaming storage abstraction over
+// S3 (and S3-compatible services), Azure Blob Storage, Google Cloud Storage,
+// SFTP, FTP and the local filesystem.
+//
+// All backends implement [Storage]. Objects are addressed by keys that are
+// relative to the backend base (bucket, container, remote directory or local
+// base path) and always use forward slashes.
 package go_storage
 
 import (
 	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
 	"fmt"
+	"io"
 	"os"
+	"path"
+	"path/filepath"
 	"strings"
 	"time"
 )
 
+// Sentinel errors. Backends wrap them with %w, so test with errors.Is.
+var (
+	// ErrNotFound is returned (wrapped) by Stat, Get and Remove when the key does not exist.
+	ErrNotFound = errors.New("storage: object not found")
+	// ErrPreconditionFailed is returned (wrapped) by PutIf when the condition does not hold.
+	ErrPreconditionFailed = errors.New("storage: precondition failed")
+	// ErrConditionNotSupported is returned (wrapped) by PutIf when the backend cannot
+	// evaluate the requested condition atomically. Callers should fall back to locking.
+	ErrConditionNotSupported = errors.New("storage: conditional write not supported by backend")
+)
+
 // Storage defines the interface for remote file storage operations.
-// All implementations must be safe for concurrent use.
+// All implementations are safe for concurrent use (see FTPStorage for the
+// one restriction regarding open Get bodies).
 type Storage interface {
 	// Upload uploads a local file to the remote storage with the given target name.
 	Upload(ctx context.Context, localPath string, remotePath string) error
@@ -17,37 +66,91 @@ type Storage interface {
 	// Download downloads a file from remote storage to a local path.
 	Download(ctx context.Context, remotePath string, localPath string) error
 
-	// List returns all items in the storage that match the given prefix.
+	// List returns all items (recursively) in the storage that match the given prefix.
 	List(ctx context.Context, prefix string) ([]Item, error)
 
-	// Remove deletes a file from remote storage.
+	// Remove deletes a file from remote storage. A missing key returns an
+	// error wrapping ErrNotFound.
 	Remove(ctx context.Context, remotePath string) error
 
 	// Close releases any resources held by the storage implementation.
 	Close() error
+
+	// Put streams r to key. r may be of unknown length; memory use is bounded.
+	// If reading r fails or ctx is cancelled, Put returns an error and no
+	// (partial) object is left at key.
+	Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error)
+
+	// Get opens key for streaming reads. The caller must Close the body.
+	// A missing key returns an error wrapping ErrNotFound.
+	Get(ctx context.Context, key string) (io.ReadCloser, Item, error)
+
+	// Stat returns metadata for key. A missing key returns an error wrapping ErrNotFound.
+	Stat(ctx context.Context, key string) (Item, error)
+
+	// PutIf writes r to key only if cond holds, atomically with respect to
+	// other PutIf calls. It returns an error wrapping ErrPreconditionFailed when
+	// the condition does not hold, or ErrConditionNotSupported when the backend
+	// cannot evaluate it. Intended for small objects (index files).
+	PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error)
+
+	// ListDir lists only the files directly inside dir (dir is treated as a
+	// directory: "a/b" never matches "a/bc/x"; "" = root). Non-recursive, no
+	// directories are returned. A missing directory yields an empty list.
+	ListDir(ctx context.Context, dir string) ([]Item, error)
 }
 
 // Item represents a file or directory in remote storage.
 type Item struct {
-	Key          string    // The path/key of the item
+	Key          string    // The path/key of the item, relative to the backend base
 	ModifiedTime time.Time // Last modification time
 	Size         int64     // Size in bytes (0 for directories)
 	IsDirectory  bool      // Whether this item is a directory
+	// ETag is an opaque version token usable with Condition.IfMatch
+	// (S3/Azure ETag; GCS generation as decimal string; local:
+	// "<mtime-unixnano hex>-<size hex>"; SFTP/FTP: "").
+	ETag string
+	// VersionID is the S3 version id when bucket versioning is enabled.
+	VersionID string
+}
+
+// PutOptions are optional parameters for Put and PutIf.
+type PutOptions struct {
+	ContentType string
+	Metadata    map[string]string
+	Size        int64 // optional size hint; -1 or 0 = unknown
+}
+
+// Condition is a precondition for PutIf. Exactly one field must be set; a
+// zero Condition is rejected (it would otherwise silently become an
+// unconditional write, e.g. when an empty ETag is passed as IfMatch).
+type Condition struct {
+	IfMatch     string // only write if the current ETag equals this
+	IfNoneMatch bool   // only write if the object does not exist
 }
 
 // Config holds configuration options for creating storage instances.
 type Config struct {
+	// Local configuration
+	LocalPath string // base directory for the local backend (default "/backup")
+
 	// S3 configuration
 	S3Region       string
 	S3Bucket       string
 	S3Profile      string
 	S3KeyID        string
 	S3Secret       string
+	S3SessionToken string
 	S3Endpoint     string
 	S3ForcePath    bool
 	S3DisableTLS   bool
+	S3StorageClass string // e.g. STANDARD, STANDARD_IA, GLACIER_IR
+	S3SSE          string // "AES256" or "aws:kms"
+	S3SSEKMSKeyID  string // KMS key id when S3SSE is "aws:kms"
+	S3PartSize     int64  // multipart part size in bytes (default 64 MiB, min 5 MiB)
+	S3Concurrency  int    // multipart upload concurrency (default 2)
 
-	// B2 configuration
+	// B2 configuration (reserved, no backend implementation yet)
 	B2KeyID                 string
 	B2AppKey                string
 	B2Bucket                string
@@ -58,12 +161,19 @@ type Config struct {
 	GCSBucket          string
 	GCSEndpoint        string
 	GCSCredentialsFile string
+	GCSCredentialsJSON string
 
 	// Azure configuration
 	AzureAccount   string
 	AzureKey       string
 	AzureContainer string
+	// AzureEndpoint is either a domain suffix (default "blob.core.windows.net",
+	// the service URL becomes https://<account>.<endpoint>) or a full service
+	// URL containing a scheme (e.g. "http://127.0.0.1:10000/devstoreaccount1"
+	// for Azurite), used as is.
 	AzureEndpoint string
+	// AzureSASToken, when set, is used instead of the shared key.
+	AzureSASToken string
 
 	// SFTP configuration
 	SFTPHost               string
@@ -72,6 +182,8 @@ type Config struct {
 	SFTPPassword           string
 	SFTPDirectory          string
 	SFTPIdentityFile       string
+	SFTPIdentityPassphrase string
+	SFTPKnownHostsFile     string
 	SFTPIgnoreKnownHosts   bool
 
 	// FTP configuration
@@ -80,11 +192,13 @@ type Config struct {
 	FTPUsername  string
 	FTPPassword  string
 	FTPDirectory string
+	FTPTLS       bool // explicit FTPS (AUTH TLS)
 }
 
 // StorageType represents the type of storage backend.
 type StorageType string
 
+// Supported storage types.
 const (
 	StorageTypeS3    StorageType = "s3"
 	StorageTypeGCS   StorageType = "gcs"
@@ -94,6 +208,9 @@ const (
 	StorageTypeLocal StorageType = "local"
 )
 
+// DefaultLocalPath is the base directory of the local backend when Config.LocalPath is empty.
+const DefaultLocalPath = "/backup"
+
 // NewStorage creates a new Storage instance based on the specified type and configuration.
 func NewStorage(storageType StorageType, config Config) (Storage, error) {
 	switch storageType {
@@ -108,19 +225,165 @@ func NewStorage(storageType StorageType, config Config) (Storage, error) {
 	case StorageTypeFTP:
 		return NewFTPStorage(config)
 	case StorageTypeLocal:
-		return NewLocalStorage("/backups")
+		p := config.LocalPath
+		if p == "" {
+			p = DefaultLocalPath
+		}
+		return NewLocalStorage(p)
 	default:
 		return nil, fmt.Errorf("unsupported storage type: %s", storageType)
 	}
 }
 
 // normalizePathSeparators converts OS-specific path separators to forward slashes.
-// This is necessary for cloud storage services which expect forward slashes.
+func normalizePathSeparators(p string) string {
-func normalizePathSeparators(path string) string {
+	if os.PathSeparator != '/' {
-	if os.PathSeparator == '/' {
+		p = strings.ReplaceAll(p, string(os.PathSeparator), "/")
-		return path
 	}
-	return strings.ReplaceAll(path, string(os.PathSeparator), "/")
+	return strings.ReplaceAll(p, "\\", "/")
+}
+
+// cleanKey normalises an object key: separators become "/", the path is
+// cleaned and leading slashes are removed. Keys escaping the base ("..")
+// and empty keys are rejected.
+func cleanKey(key string) (string, error) {
+	k, err := cleanDir(key)
+	if err != nil {
+		return "", err
+	}
+	if k == "" {
+		return "", fmt.Errorf("storage: empty key")
+	}
+	return k, nil
+}
+
+// cleanDir is like cleanKey but allows the empty string (root).
+func cleanDir(dir string) (string, error) {
+	d := path.Clean("/" + normalizePathSeparators(dir))
+	d = strings.TrimPrefix(d, "/")
+	// path.Clean of an absolute path never yields "..", but check the raw
+	// input so that escaping attempts are reported rather than silently clamped.
+	for _, seg := range strings.Split(path.Clean(normalizePathSeparators(dir)), "/") {
+		if seg == ".." {
+			return "", fmt.Errorf("storage: key %q escapes the base directory", dir)
+		}
+	}
+	return d, nil
+}
+
+// cleanPrefix normalises a List prefix. Unlike keys, a prefix is not
+// path-cleaned (a trailing "/" is significant) but must not contain "..".
+func cleanPrefix(prefix string) (string, error) {
+	p := normalizePathSeparators(prefix)
+	for _, seg := range strings.Split(p, "/") {
+		if seg == ".." {
+			return "", fmt.Errorf("storage: prefix %q escapes the base directory", prefix)
+		}
+	}
+	return strings.TrimLeft(p, "/"), nil
+}
+
+// dirPrefix returns the object prefix for a directory ("" for root, "a/b/" otherwise).
+func dirPrefix(dir string) string {
+	if dir == "" {
+		return ""
+	}
+	return dir + "/"
+}
+
+// partialName returns a unique temporary name next to p.
+func partialName(p string) string {
+	var b [6]byte
+	_, _ = rand.Read(b[:])
+	return p + "." + hex.EncodeToString(b[:]) + ".partial"
+}
+
+// ctxReader aborts reads once ctx is done.
+type ctxReader struct {
+	ctx context.Context
+	r   io.Reader
+}
+
+func (c ctxReader) Read(p []byte) (int, error) {
+	if err := c.ctx.Err(); err != nil {
+		return 0, err
+	}
+	return c.r.Read(p)
+}
+
+// countingReader counts bytes read.
+type countingReader struct {
+	r io.Reader
+	n int64
+}
+
+func (c *countingReader) Read(p []byte) (int, error) {
+	n, err := c.r.Read(p)
+	c.n += int64(n)
+	return n, err
+}
+
+// errEmptyCondition is returned by PutIf when neither IfMatch nor IfNoneMatch is set.
+var errEmptyCondition = errors.New("storage: PutIf requires Condition.IfMatch or Condition.IfNoneMatch")
+
+func validateCondition(c Condition) error {
+	if c.IfMatch == "" && !c.IfNoneMatch {
+		return errEmptyCondition
+	}
+	if c.IfMatch != "" && c.IfNoneMatch {
+		return errors.New("storage: Condition.IfMatch and Condition.IfNoneMatch are mutually exclusive")
+	}
+	return nil
+}
+
+// maxConditionalPutSize bounds the buffer used by backends whose conditional
+// write needs a single request (S3).
+const maxConditionalPutSize = 16 << 20
+
+// uploadFile implements Storage.Upload on top of Put.
+func uploadFile(ctx context.Context, s Storage, localPath, remotePath string) error {
+	f, err := os.Open(localPath)
+	if err != nil {
+		return fmt.Errorf("failed to open local file: %w", err)
+	}
+	defer func() { _ = f.Close() }()
+	opts := &PutOptions{Size: -1}
+	if fi, err := f.Stat(); err == nil {
+		opts.Size = fi.Size()
+	}
+	_, err = s.Put(ctx, remotePath, f, opts)
+	return err
+}
+
+// downloadFile implements Storage.Download on top of Get.
+func downloadFile(ctx context.Context, s Storage, remotePath, localPath string) (err error) {
+	body, _, err := s.Get(ctx, remotePath)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = body.Close() }()
+
+	if dir := filepath.Dir(localPath); dir != "" {
+		if err := os.MkdirAll(dir, 0o755); err != nil {
+			return fmt.Errorf("failed to create local directory: %w", err)
+		}
+	}
+	f, err := os.Create(localPath)
+	if err != nil {
+		return fmt.Errorf("failed to create local file: %w", err)
+	}
+	defer func() {
+		if cerr := f.Close(); cerr != nil && err == nil {
+			err = fmt.Errorf("failed to close local file: %w", cerr)
+		}
+		if err != nil {
+			_ = os.Remove(localPath)
+		}
+	}()
+	if _, err = io.Copy(f, ctxReader{ctx: ctx, r: body}); err != nil {
+		return fmt.Errorf("failed to download %s: %w", remotePath, err)
+	}
+	return nil
 }
 
 // validateConfig checks if the required configuration fields are present.

storage_test.go

@@ -1,41 +1,212 @@
+/*
+MIT License
+
+Copyright (c) 2024 Jonas Kaninda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
 package go_storage
 
 import (
 	"context"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
 	"testing"
 	"time"
 )
 
-// Test helpers
 func TestStorageInterface(t *testing.T) {
 	var _ Storage = (*S3Storage)(nil)
 	var _ Storage = (*GCSStorage)(nil)
 	var _ Storage = (*AzureStorage)(nil)
 	var _ Storage = (*SFTPStorage)(nil)
+	var _ Storage = (*FTPStorage)(nil)
+	var _ Storage = (*LocalStorage)(nil)
 }
 
-func TestContextTimeout(t *testing.T) {
+func TestCleanKey(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	ok := map[string]string{
-	defer cancel()
+		"x/y.txt":     "x/y.txt",
+		"/a/b.txt":    "a/b.txt",
+		"a//b/./c":    "a/b/c",
+		`a\b\c.txt`:   "a/b/c.txt",
+		"a/../b.txt":  "b.txt",
+		"./x":         "x",
+		"backup.sql~": "backup.sql~",
+	}
+	for in, want := range ok {
+		got, err := cleanKey(in)
+		if err != nil || got != want {
+			t.Errorf("cleanKey(%q) = %q, %v; want %q", in, got, err, want)
+		}
+	}
+	for _, in := range []string{"", "/", ".", "..", "../x", "a/../../x", `..\x`} {
+		if got, err := cleanKey(in); err == nil {
+			t.Errorf("cleanKey(%q) = %q, want error", in, got)
+		}
+	}
+	if d, err := cleanDir(""); err != nil || d != "" {
+		t.Errorf("cleanDir(\"\") = %q, %v", d, err)
+	}
+	if _, err := cleanPrefix("a/../../b"); err == nil {
+		t.Error("cleanPrefix with .. accepted")
+	}
+}
 
-	config := Config{
+func TestNewStorageLocalPath(t *testing.T) {
-		S3Region: "us-east-1",
+	dir := filepath.Join(t.TempDir(), "base")
+	s, err := NewStorage(StorageTypeLocal, Config{LocalPath: dir})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := s.(*LocalStorage).BasePath(); got != dir {
+		t.Fatalf("base path = %q, want %q", got, dir)
+	}
+}
+
+func TestNewS3StorageOffline(t *testing.T) {
+	s, err := NewS3Storage(Config{
 		S3Bucket:     "test-bucket",
+		S3Endpoint:   "127.0.0.1:9",
+		S3DisableTLS: true,
+		S3KeyID:      "k",
+		S3Secret:     "s",
+		S3PartSize:   1,
+	})
+	if err != nil {
+		t.Fatalf("NewS3Storage: %v", err)
+	}
+	if s.region != DefaultS3Region {
+		t.Errorf("region = %q", s.region)
+	}
+	if got := *s.client.Options().BaseEndpoint; got != "http://127.0.0.1:9" {
+		t.Errorf("endpoint = %q", got)
+	}
+	//nolint:staticcheck // manager is deprecated, see s3.go
+	if s.uploader.PartSize != MinS3PartSize || s.uploader.Concurrency != DefaultS3Concurrency {
+		t.Errorf("uploader part size/concurrency = %d/%d", s.uploader.PartSize, s.uploader.Concurrency)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if err := s.Upload(ctx, filepath.Join(t.TempDir(), "missing.txt"), "remote.txt"); err == nil {
+		t.Fatal("Upload of a missing local file succeeded")
+	}
 }
 
-	s3, err := NewStorage(StorageTypeS3, config)
+func TestLocalConformance(t *testing.T) {
+	s, err := NewLocalStorage(t.TempDir())
 	if err != nil {
-		t.Fatalf("Failed to create storage: %v", err)
+		t.Fatal(err)
 	}
-	defer func(s3 Storage) {
+	defer func() { _ = s.Close() }()
-		err = s3.Close()
+	runConformance(t, s)
-		if err != nil {
-			t.Logf("Failed to close storage: %v", err)
 }
-	}(s3)
 
-	err = s3.Upload(ctx, "test.txt", "remote-test.txt")
+func TestLocalNoLeftovers(t *testing.T) {
+	base := t.TempDir()
+	s, err := NewLocalStorage(base)
 	if err != nil {
-		t.Logf("Upload failed (expected in test): %v", err)
+		t.Fatal(err)
+	}
+	ctx := context.Background()
+	_, _ = s.Put(ctx, "x/y.bin", &failingReader{r: strings.NewReader(strings.Repeat("a", 100)), n: 10, err: errors.New("boom")}, nil)
+	if _, err := s.PutIf(ctx, "x/idx.json", strings.NewReader("1"), Condition{IfNoneMatch: true}, nil); err != nil {
+		t.Fatal(err)
+	}
+	_, _ = s.PutIf(ctx, "x/idx.json", strings.NewReader("2"), Condition{IfNoneMatch: true}, nil)
+	entries, err := os.ReadDir(filepath.Join(base, "x"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, e := range entries {
+		if e.Name() != "idx.json" {
+			t.Errorf("leftover file %s", e.Name())
+		}
+	}
+}
+
+// TestLocalPutIfConcurrent checks that concurrent read-modify-write cycles
+// with IfMatch never lose an update.
+func TestLocalPutIfConcurrent(t *testing.T) {
+	s, err := NewLocalStorage(t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.Background()
+	if _, err := s.PutIf(ctx, "idx", strings.NewReader(""), Condition{IfNoneMatch: true}, nil); err != nil {
+		t.Fatal(err)
+	}
+	const workers, perWorker = 8, 10
+	var wg sync.WaitGroup
+	for w := 0; w < workers; w++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < perWorker; {
+				body, it, err := s.Get(ctx, "idx")
+				if err != nil {
+					t.Error(err)
+					return
+				}
+				b, _ := io.ReadAll(body)
+				_ = body.Close()
+				_, err = s.PutIf(ctx, "idx", strings.NewReader(string(b)+"x"), Condition{IfMatch: it.ETag}, nil)
+				if errors.Is(err, ErrPreconditionFailed) {
+					continue
+				}
+				if err != nil {
+					t.Error(err)
+					return
+				}
+				i++
+			}
+		}()
+	}
+	wg.Wait()
+	body, _, err := s.Get(ctx, "idx")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = body.Close() }()
+	b, _ := io.ReadAll(body)
+	if len(b) != workers*perWorker {
+		t.Fatalf("lost updates: got %d, want %d", len(b), workers*perWorker)
+	}
+}
+
+func TestLocalPathTraversal(t *testing.T) {
+	base := t.TempDir()
+	s, err := NewLocalStorage(filepath.Join(base, "store"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, k := range []string{"../outside.txt", "a/../../outside.txt", `..\outside.txt`} {
+		if _, err := s.Put(context.Background(), k, strings.NewReader("x"), nil); err == nil {
+			t.Errorf("Put(%q) succeeded", k)
+		}
+	}
+	if _, err := os.Stat(filepath.Join(base, "outside.txt")); !os.IsNotExist(err) {
+		t.Fatal("file written outside base")
 	}
 }