diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 4c7c144..f40cb3b 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -1,28 +1,48 @@ -# This workflow will build a golang project -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go - name: Go on: push: - branches: [ "main" ] + branches: [ "main", "v2" ] pull_request: - branches: [ "main" ] + branches: [ "main", "v2" ] jobs: - - build: + test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: '1.21.0' + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version: '1.25' - - name: Build - run: go build -v ./... + - name: Build + run: go build -v ./... - - name: Test - run: go test -v ./... + - name: Vet + run: go vet ./... && go vet -tags integration ./... + + - name: Unit tests + run: go test -race -count=1 ./... + + integration: + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.25' + + - name: Pull images + run: | + docker pull pgsty/minio:latest + docker pull mcr.microsoft.com/azure-storage/azurite:latest + docker pull fsouza/fake-gcs-server:latest + docker pull delfer/alpine-ftp-server:latest + + - name: Integration tests (MinIO, Azurite, fake-gcs-server, FTP) + run: go test -tags integration -count=1 -v -run Integration ./... diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f40d365..82524d9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,9 +15,9 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: '~1.22' + go-version: '1.25' - name: Run linter - uses: golangci/golangci-lint-action@v6 + uses: golangci/golangci-lint-action@v7 with: - version: v1.61 + version: v2.13.2 diff --git a/.golangci.yml b/.golangci.yml index ae8a9cf..0d4f268 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,44 +1,51 @@ +version: "2" run: - timeout: 5m allow-parallel-runners: true - -issues: - # don't skip warning about doc comments - # don't exclude the default set of lint - exclude-use-default: false - # restore some of the defaults - # (fill in the rest as needed) - exclude-rules: - - path: "internal/*" - linters: - - dupl - - lll - - goimports linters: - disable-all: true + default: none enable: + - copyloopvar - dupl - errcheck - - copyloopvar - ginkgolinter - goconst - gocyclo - - gofmt - - gosimple - govet - ineffassign - # - lll + #- lll - misspell - nakedret - prealloc - revive - staticcheck - - typecheck - unconvert - unparam - unused - -linters-settings: - revive: + settings: + revive: + rules: + - name: comment-spacings + exclusions: + generated: lax rules: - - name: comment-spacings + - linters: + - dupl + - lll + path: /* + # Legacy v0.1 API: deprecated as a whole and frozen. + - linters: + - staticcheck + path: ^pkg/ + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - gofmt + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/README.md b/README.md index 7f188bd..298a204 100644 --- a/README.md +++ b/README.md @@ -1,126 +1,200 @@ # Go Storage -A simple Go storage client -Supported storage: -- Local -- S3 -- SSH -- FTP -- Azure Blob +A streaming storage abstraction for Go backup tools. -```go +Supported backends: + +- Local filesystem +- S3 and S3-compatible services (MinIO, Ceph, Wasabi, ...) via aws-sdk-go-v2 +- Azure Blob Storage (and Azurite) +- Google Cloud Storage +- SFTP +- FTP / explicit FTPS + +```shell go get github.com/jkaninda/go-storage ``` -### Local Storage -```go - localStorage := local.NewStorage(local.Config{ - LocalPath: tmpPath, - RemotePath: backupDestination, - }) - err = localStorage.Copy(finalFileName) - if err != nil { - log.Fatal("Error copying file, error %v", err) - } -``` +## Streaming API + +Every backend implements the same `Storage` interface: -### S3 Storage ```go -s3Storage, err := s3.NewStorage(s3.Config{ - Endpoint: "", - Bucket: "", - AccessKey: "", - SecretKey: "", - Region: "", - DisableSsl: "", - ForcePathStyle: "", - RemotePath: "", - LocalPath: "", -}) -if err != nil { - log.Fatalf("Error creating s3 storage: %s", err) -} -// Copy file to S3 -err = s3Storage.Copy(finalFileName) -if err != nil { - log.Fatalf("Error copying file, error %v", err) -} -// Download file from S3 -err = s3Storage.CopyFrom(finalFileName) -if err != nil { - log.Fatalf("Error copying file, error %v", err) +type Storage interface { + Upload(ctx context.Context, localPath, remotePath string) error + Download(ctx context.Context, remotePath, localPath string) error + List(ctx context.Context, prefix string) ([]Item, error) // recursive + Remove(ctx context.Context, remotePath string) error // missing -> ErrNotFound + Close() error + + Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error) + Get(ctx context.Context, key string) (io.ReadCloser, Item, error) + Stat(ctx context.Context, key string) (Item, error) + PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error) + ListDir(ctx context.Context, dir string) ([]Item, error) // non-recursive, files only } ``` -### SSH Storage +- **`Put`** streams a reader of unknown length with bounded memory (no temp + files, no whole-file buffering). If the reader fails or the context is + cancelled, `Put` returns an error and **no object (and no partial object) + is left at the key**: S3 aborts the multipart upload, Azure never commits + the block list, GCS cancels the resumable upload, and SFTP/FTP/local write + to `..partial` and rename on success. +- **`Get`** returns a streaming body; always `Close` it. On FTP the body holds + the single control connection, so close it before making another call. +- **`PutIf`** is a conditional write for small files such as a backup index + (optimistic concurrency): `Condition{IfNoneMatch: true}` creates only if the + key does not exist, `Condition{IfMatch: item.ETag}` replaces only if the + object is unchanged. It returns `ErrPreconditionFailed` when the condition + does not hold and `ErrConditionNotSupported` when the backend cannot + evaluate it (fall back to locking). +- Errors wrap the sentinels `ErrNotFound`, `ErrPreconditionFailed` and + `ErrConditionNotSupported`; test them with `errors.Is`. +- Keys are relative to the backend base (bucket, container, remote directory + or local base path), use `/` separators and may not escape the base (`..`). + +| Backend | `Item.ETag` | `PutIf` IfNoneMatch | `PutIf` IfMatch | +|---------|--------------------------------|---------------------|-----------------| +| S3 | S3 ETag | yes | yes | +| Azure | blob ETag | yes | yes | +| GCS | object generation | yes | yes | +| Local | `-` | yes (lock file) | yes (lock file) | +| SFTP | empty | yes (`O_EXCL`) | not supported | +| FTP | empty | not supported | not supported | + +S3 `PutIf` uses a single `PutObject` request and is limited to 16 MiB. + +### Example: dump, compress and upload without temp files + ```go -sshStorage, err := ssh.NewStorage(ssh.Config{ - Host: "", - Port: 22, - User: "", - Password: "", - RemotePath: "", - LocalPath: "", +package main + +import ( + "compress/gzip" + "context" + "errors" + "io" + "log" + "os/exec" + "strings" + + storage "github.com/jkaninda/go-storage" +) + +func main() { + ctx := context.Background() + s, err := storage.NewStorage(storage.StorageTypeS3, storage.Config{ + S3Bucket: "backups", + S3Region: "us-east-1", + S3KeyID: "...", + S3Secret: "...", + S3Endpoint: "https://minio.example.com", // optional, for S3-compatible services + S3ForcePath: true, }) if err != nil { - log.Fatalf("Error creating SSH storage: %s", err) + log.Fatal(err) } - // Copy file to the remote server - err = sshStorage.Copy(finalFileName) + defer s.Close() + + pr, pw := io.Pipe() + go func() { + gz := gzip.NewWriter(pw) + cmd := exec.CommandContext(ctx, "pg_dump", "mydb") + cmd.Stdout = gz + err := cmd.Run() + if err == nil { + err = gz.Close() + } + pw.CloseWithError(err) // a non-nil error aborts the upload + }() + + item, err := s.Put(ctx, "db/mydb-2026-01-01.sql.gz", pr, &storage.PutOptions{ContentType: "application/gzip"}) if err != nil { - log.Fatalf("Error copying backup file: %s", err) + log.Fatal(err) } -// Download file from SSH remote server -err = sshStorage.CopyFrom(finalFileName) -if err != nil { -log.Fatalf("Error copying file, error %v", err) -} -``` -### FTP Storage -```go - ftpStorage, err := ftp.NewStorage(ftp.Config{ - Host: "", - Port: 21, - User: "", - Password: "", - RemotePath: "", - LocalPath: "", - }) - if err != nil { - log.Fatalf("Error creating FTP storage: %s", err) + log.Printf("uploaded %s (%d bytes, etag %s)", item.Key, item.Size, item.ETag) + + // Restore: stream it back. + body, _, err := s.Get(ctx, item.Key) + if errors.Is(err, storage.ErrNotFound) { + log.Fatal("backup not found") + } else if err != nil { + log.Fatal(err) } - err = ftpStorage.Copy(finalFileName) - if err != nil { - log.Fatalf("Error copying backup file: %s", err) + defer body.Close() + zr, _ := gzip.NewReader(body) + _, _ = io.Copy(io.Discard, zr) + + // Update an index file with optimistic concurrency. + idx, err := s.Stat(ctx, "db/index.json") + cond := storage.Condition{IfMatch: idx.ETag} + if errors.Is(err, storage.ErrNotFound) { + cond = storage.Condition{IfNoneMatch: true} + } + _, err = s.PutIf(ctx, "db/index.json", strings.NewReader(`{"backups":[]}`), cond, nil) + if errors.Is(err, storage.ErrPreconditionFailed) { + // someone else updated the index: re-read, merge and retry } -// Download file from ftp remote server -err = ftpStorage.CopyFrom(finalFileName) -if err != nil { -log.Fatalf("Error copying file, error %v", err) } ``` -### Azure Blob storage +## Configuration -```go -azureStorage, err := azure.NewStorage(azure.Config{ - ContainerName: '', - AccountName: '', - AccountKey: '', - RemotePath: '', - LocalPath: '', - }) - if err != nil { - log.Fatal("Error creating Azure Blob storage storage: %s", err) - } - err = azureStorage.Copy(finalFileName) - if err != nil { - log.Fatal("Error copying file: %s", err) - } +`storage.NewStorage(storageType, storage.Config{...})` with `storageType` one +of `s3`, `gcs`, `azure`, `sftp`, `ftp`, `local`. -// Download file from Azure Blob remote server -err = azureStorage.CopyFrom(finalFileName) -if err != nil { -log.Fatalf("Error copying file, error %v", err) -} -``` \ No newline at end of file +| Field | Backend | Description | +|-------|---------|-------------| +| `LocalPath` | local | Base directory (default `/backup`) | +| `S3Bucket` | s3 | Bucket (required) | +| `S3Region` | s3 | Region (default from AWS config, else `us-east-1`) | +| `S3KeyID`, `S3Secret`, `S3SessionToken` | s3 | Static credentials; when empty the default AWS chain is used (env, shared profile, IRSA/web identity, ECS/EC2 roles) | +| `S3Profile` | s3 | Shared config profile | +| `S3Endpoint` | s3 | Custom endpoint (MinIO etc.); without a scheme `https://` is assumed (`http://` with `S3DisableTLS`) | +| `S3ForcePath` | s3 | Path-style addressing | +| `S3DisableTLS` | s3 | Use `http://` for scheme-less endpoints | +| `S3StorageClass` | s3 | e.g. `STANDARD_IA`, `GLACIER_IR` | +| `S3SSE`, `S3SSEKMSKeyID` | s3 | Server-side encryption: `AES256` or `aws:kms` (+ key id) | +| `S3PartSize` | s3 | Multipart part size (default 64 MiB, min 5 MiB) | +| `S3Concurrency` | s3 | Parallel part uploads (default 2); memory ≈ part size × concurrency | +| `AzureAccount`, `AzureKey` | azure | Shared key credentials | +| `AzureSASToken` | azure | SAS token (used instead of the shared key when set) | +| `AzureContainer` | azure | Container (required) | +| `AzureEndpoint` | azure | Domain suffix (default `blob.core.windows.net`) or a full service URL such as `http://127.0.0.1:10000/devstoreaccount1` (Azurite) | +| `GCSBucket` | gcs | Bucket (required) | +| `GCSCredentialsFile`, `GCSCredentialsJSON` | gcs | Credentials; default is Application Default Credentials | +| `GCSEndpoint` | gcs | Custom endpoint | +| `SFTPHost`, `SFTPPort` | sftp | Server (port default 22) | +| `SFTPUsername`, `SFTPPassword` | sftp | Password authentication | +| `SFTPIdentityFile`, `SFTPIdentityPassphrase` | sftp | Private key authentication (an SSH agent is used too when available) | +| `SFTPKnownHostsFile` | sftp | known_hosts file (default `~/.ssh/known_hosts`); host keys are always verified unless... | +| `SFTPIgnoreKnownHosts` | sftp | ...this is `true` (insecure) | +| `SFTPDirectory` | sftp | Remote base directory (default: login directory) | +| `FTPHost`, `FTPPort`, `FTPUsername`, `FTPPassword` | ftp | Server and credentials (port default 21) | +| `FTPDirectory` | ftp | Remote base directory (default: login directory) | +| `FTPTLS` | ftp | Explicit FTPS (`AUTH TLS`) | + +The library never reads environment variables itself (except what the cloud +SDKs do for their default credential chains, and `SSH_AUTH_SOCK` for the SSH agent). + +## Testing + +```shell +go test ./... # unit tests: local + in-process SFTP server +go test -tags integration ./... # + MinIO, Azurite, fake-gcs-server, FTP via docker +``` + +Integration tests skip when docker is unavailable. Images can be overridden +with `GO_STORAGE_MINIO_IMAGE`, `GO_STORAGE_AZURITE_IMAGE`, +`GO_STORAGE_FAKEGCS_IMAGE` and `GO_STORAGE_FTP_IMAGE`. + +## Deprecated: `pkg/` (v0.1 API) + +The packages under `pkg/` (`pkg/local`, `pkg/s3`, `pkg/ssh`, `pkg/ftp`, +`pkg/azure`) are the legacy v0.1 API. They still compile but are deprecated +and will be removed in a future release; use the root package instead. + +## License + +MIT License, Copyright (c) 2024 Jonas Kaninda. See [LICENSE](LICENSE). diff --git a/azure.go b/azure.go new file mode 100644 index 0000000..6cf17dc --- /dev/null +++ b/azure.go @@ -0,0 +1,349 @@ +/* +MIT License + +Copyright (c) 2024 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package go_storage + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" +) + +// Azure streaming upload tuning: 8 MiB blocks allow blobs up to ~390 GiB +// (50,000 blocks) while keeping memory bounded to blockSize*concurrency. +const ( + azureBlockSize int64 = 8 << 20 + azureConcurrency = 2 +) + +// AzureStorage implements Storage interface for Azure Blob Storage. +type AzureStorage struct { + client *azblob.Client + container string +} + +// NewAzureStorage creates a new Azure Blob Storage instance. +// +// Authentication: AzureSASToken if set, otherwise shared key +// (AzureAccount/AzureKey), otherwise anonymous. +func NewAzureStorage(config Config) (*AzureStorage, error) { + if err := validateConfig(config, map[string]string{ + "container": config.AzureContainer, + }); err != nil { + return nil, fmt.Errorf("azure: %w", err) + } + + serviceURL, err := azureServiceURL(config) + if err != nil { + return nil, err + } + + var client *azblob.Client + switch { + case config.AzureSASToken != "": + u := serviceURL + "?" + strings.TrimPrefix(config.AzureSASToken, "?") + client, err = azblob.NewClientWithNoCredential(u, nil) + case config.AzureAccount != "" && config.AzureKey != "": + cred, cerr := azblob.NewSharedKeyCredential(config.AzureAccount, config.AzureKey) + if cerr != nil { + return nil, fmt.Errorf("azure: failed to create credentials: %w", cerr) + } + client, err = azblob.NewClientWithSharedKeyCredential(serviceURL, cred, nil) + default: + client, err = azblob.NewClientWithNoCredential(serviceURL, nil) + } + if err != nil { + return nil, fmt.Errorf("azure: failed to create client: %w", err) + } + + return &AzureStorage{client: client, container: config.AzureContainer}, nil +} + +func azureServiceURL(config Config) (string, error) { + endpoint := config.AzureEndpoint + if endpoint == "" { + endpoint = "blob.core.windows.net" + } + if strings.Contains(endpoint, "://") { + return strings.TrimSuffix(endpoint, "/") + "/", nil + } + if config.AzureAccount == "" { + return "", fmt.Errorf("azure: missing required configuration: account") + } + return fmt.Sprintf("https://%s.%s/", config.AzureAccount, strings.Trim(endpoint, "/")), nil +} + +// Client returns the underlying azblob client. +func (a *AzureStorage) Client() *azblob.Client { return a.client } + +func (a *AzureStorage) put(ctx context.Context, key string, r io.Reader, opts *PutOptions, ac *blob.AccessConditions) (Item, error) { + k, err := cleanKey(key) + if err != nil { + return Item{}, fmt.Errorf("azure: %w", err) + } + o := &azblob.UploadStreamOptions{ + BlockSize: azureBlockSize, + Concurrency: azureConcurrency, + AccessConditions: ac, + } + if opts != nil { + if opts.ContentType != "" { + o.HTTPHeaders = &blob.HTTPHeaders{BlobContentType: &opts.ContentType} + } + if len(opts.Metadata) > 0 { + o.Metadata = make(map[string]*string, len(opts.Metadata)) + for mk, mv := range opts.Metadata { + o.Metadata[mk] = &mv + } + } + } + cr := &countingReader{r: ctxReader{ctx: ctx, r: r}} + resp, err := a.client.UploadStream(ctx, a.container, k, cr, o) + if err != nil { + if ac != nil && (isAzurePrecondition(err) || bloberror.HasCode(err, bloberror.BlobNotFound)) { + return Item{}, fmt.Errorf("azure: conditional put of %s: %w", k, ErrPreconditionFailed) + } + return Item{}, fmt.Errorf("azure: failed to upload %s to container %s: %w", k, a.container, err) + } + it := Item{Key: k, Size: cr.n, ModifiedTime: time.Now().UTC()} + if resp.ETag != nil { + it.ETag = string(*resp.ETag) + } + if resp.LastModified != nil { + it.ModifiedTime = *resp.LastModified + } + if resp.VersionID != nil { + it.VersionID = *resp.VersionID + } + return it, nil +} + +// Put streams r to key with a block upload; the block list is only +// committed when the whole stream was read successfully. +func (a *AzureStorage) Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error) { + return a.put(ctx, key, r, opts, nil) +} + +// PutIf writes r to key with If-Match / If-None-Match: * access conditions. +func (a *AzureStorage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error) { + if err := validateCondition(cond); err != nil { + return Item{}, fmt.Errorf("azure: %w", err) + } + mac := &blob.ModifiedAccessConditions{} + if cond.IfMatch != "" { + etag := azcore.ETag(cond.IfMatch) + mac.IfMatch = &etag + } + if cond.IfNoneMatch { + etag := azcore.ETagAny + mac.IfNoneMatch = &etag + } + return a.put(ctx, key, r, opts, &blob.AccessConditions{ModifiedAccessConditions: mac}) +} + +// Get streams the blob body. +func (a *AzureStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) { + k, err := cleanKey(key) + if err != nil { + return nil, Item{}, fmt.Errorf("azure: %w", err) + } + resp, err := a.client.DownloadStream(ctx, a.container, k, nil) + if err != nil { + return nil, Item{}, a.mapErr("get", k, err) + } + it := Item{Key: k} + if resp.ContentLength != nil { + it.Size = *resp.ContentLength + } + if resp.LastModified != nil { + it.ModifiedTime = *resp.LastModified + } + if resp.ETag != nil { + it.ETag = string(*resp.ETag) + } + if resp.VersionID != nil { + it.VersionID = *resp.VersionID + } + return resp.Body, it, nil +} + +// Stat returns blob properties. +func (a *AzureStorage) Stat(ctx context.Context, key string) (Item, error) { + k, err := cleanKey(key) + if err != nil { + return Item{}, fmt.Errorf("azure: %w", err) + } + bc := a.client.ServiceClient().NewContainerClient(a.container).NewBlobClient(k) + resp, err := bc.GetProperties(ctx, nil) + if err != nil { + return Item{}, a.mapErr("stat", k, err) + } + it := Item{Key: k} + if resp.ContentLength != nil { + it.Size = *resp.ContentLength + } + if resp.LastModified != nil { + it.ModifiedTime = *resp.LastModified + } + if resp.ETag != nil { + it.ETag = string(*resp.ETag) + } + if resp.VersionID != nil { + it.VersionID = *resp.VersionID + } + return it, nil +} + +// Upload uploads a file to Azure Blob Storage. +func (a *AzureStorage) Upload(ctx context.Context, localPath string, remotePath string) error { + if err := uploadFile(ctx, a, localPath, remotePath); err != nil { + return fmt.Errorf("azure: %w", err) + } + return nil +} + +// Download downloads a file from Azure Blob Storage. +func (a *AzureStorage) Download(ctx context.Context, remotePath string, localPath string) error { + if err := downloadFile(ctx, a, remotePath, localPath); err != nil { + return fmt.Errorf("azure: %w", err) + } + return nil +} + +// List lists blobs (recursively) with the given prefix. +func (a *AzureStorage) List(ctx context.Context, prefix string) ([]Item, error) { + p, err := cleanPrefix(prefix) + if err != nil { + return nil, fmt.Errorf("azure: %w", err) + } + pager := a.client.NewListBlobsFlatPager(a.container, &azblob.ListBlobsFlatOptions{Prefix: &p}) + items := make([]Item, 0) + for pager.More() { + resp, err := pager.NextPage(ctx) + if err != nil { + return nil, fmt.Errorf("azure: failed to list blobs in container %s: %w", a.container, err) + } + for _, b := range resp.Segment.BlobItems { + items = append(items, azureItem(b)) + } + } + return items, nil +} + +// ListDir lists the blobs directly under dir. +func (a *AzureStorage) ListDir(ctx context.Context, dir string) ([]Item, error) { + d, err := cleanDir(dir) + if err != nil { + return nil, fmt.Errorf("azure: %w", err) + } + p := dirPrefix(d) + cc := a.client.ServiceClient().NewContainerClient(a.container) + pager := cc.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{Prefix: &p}) + items := make([]Item, 0) + for pager.More() { + resp, err := pager.NextPage(ctx) + if err != nil { + return nil, fmt.Errorf("azure: failed to list blobs in container %s: %w", a.container, err) + } + for _, b := range resp.Segment.BlobItems { + if b.Name == nil || strings.HasSuffix(*b.Name, "/") { + continue + } + items = append(items, azureItem(b)) + } + } + return items, nil +} + +func azureItem(b *container.BlobItem) Item { + it := Item{} + if b.Name != nil { + it.Key = *b.Name + } + if b.VersionID != nil { + it.VersionID = *b.VersionID + } + if p := b.Properties; p != nil { + if p.LastModified != nil { + it.ModifiedTime = *p.LastModified + } + if p.ContentLength != nil { + it.Size = *p.ContentLength + } + if p.ETag != nil { + it.ETag = string(*p.ETag) + } + } + return it +} + +// Remove deletes a blob. +func (a *AzureStorage) Remove(ctx context.Context, remotePath string) error { + k, err := cleanKey(remotePath) + if err != nil { + return fmt.Errorf("azure: %w", err) + } + if _, err := a.client.DeleteBlob(ctx, a.container, k, nil); err != nil { + return a.mapErr("remove", k, err) + } + return nil +} + +// Close releases resources (no-op). +func (a *AzureStorage) Close() error { + return nil +} + +func (a *AzureStorage) mapErr(op, key string, err error) error { + if bloberror.HasCode(err, bloberror.BlobNotFound, bloberror.ResourceNotFound) { + return fmt.Errorf("azure: %s %s in container %s: %w", op, key, a.container, ErrNotFound) + } + var re *azcore.ResponseError + if errors.As(err, &re) && re.StatusCode == http.StatusNotFound && re.ErrorCode != string(bloberror.ContainerNotFound) { + return fmt.Errorf("azure: %s %s in container %s: %w", op, key, a.container, ErrNotFound) + } + return fmt.Errorf("azure: %s %s in container %s: %w", op, key, a.container, err) +} + +func isAzurePrecondition(err error) bool { + if bloberror.HasCode(err, bloberror.ConditionNotMet, bloberror.BlobAlreadyExists, bloberror.TargetConditionNotMet) { + return true + } + var re *azcore.ResponseError + if errors.As(err, &re) { + return re.StatusCode == http.StatusPreconditionFailed || + (re.StatusCode == http.StatusConflict && re.ErrorCode == string(bloberror.BlobAlreadyExists)) + } + return false +} diff --git a/conformance_test.go b/conformance_test.go new file mode 100644 index 0000000..7df9bda --- /dev/null +++ b/conformance_test.go @@ -0,0 +1,364 @@ +/* +MIT License + +Copyright (c) 2024 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package go_storage + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "math/rand" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" +) + +const conformanceStreamSize = 20 << 20 + +// deterministicStream writes n pseudo-random bytes into an io.Pipe and +// returns the reader plus the expected SHA-256. +func deterministicStream(n int64, seed int64) (io.Reader, [32]byte) { + h := sha256.New() + _, _ = io.Copy(h, io.LimitReader(rand.New(rand.NewSource(seed)), n)) + var sum [32]byte + copy(sum[:], h.Sum(nil)) + + pr, pw := io.Pipe() + go func() { + _, err := io.Copy(pw, io.LimitReader(rand.New(rand.NewSource(seed)), n)) + _ = pw.CloseWithError(err) + }() + return pr, sum +} + +// failingReader returns ok bytes, then err. +type failingReader struct { + r io.Reader + n int64 + err error +} + +func (f *failingReader) Read(p []byte) (int, error) { + if f.n <= 0 { + return 0, f.err + } + if int64(len(p)) > f.n { + p = p[:f.n] + } + n, err := f.r.Read(p) + f.n -= int64(n) + return n, err +} + +func keysOf(items []Item, filesOnly bool) []string { + out := make([]string, 0, len(items)) + for _, it := range items { + if filesOnly && it.IsDirectory { + continue + } + out = append(out, it.Key) + } + sort.Strings(out) + return out +} + +func putString(t *testing.T, s Storage, key, content string) Item { + t.Helper() + it, err := s.Put(context.Background(), key, strings.NewReader(content), &PutOptions{ContentType: "text/plain"}) + if err != nil { + t.Fatalf("Put(%s): %v", key, err) + } + return it +} + +func getString(t *testing.T, s Storage, key string) string { + t.Helper() + body, _, err := s.Get(context.Background(), key) + if err != nil { + t.Fatalf("Get(%s): %v", key, err) + } + defer func() { _ = body.Close() }() + b, err := io.ReadAll(body) + if err != nil { + t.Fatalf("read %s: %v", key, err) + } + return string(b) +} + +// runConformance exercises the full Storage contract against s. +func runConformance(t *testing.T, s Storage) { + root := fmt.Sprintf("conf-%d", time.Now().UnixNano()) + k := func(p string) string { return root + "/" + p } + for _, tc := range []struct { + name string + fn func(*testing.T, Storage, func(string) string) + }{ + {"StreamRoundTrip", confStreamRoundTrip}, + {"FailingReaderLeavesNothing", confFailingReaderLeavesNothing}, + {"ListDirBoundary", confListDirBoundary}, + {"ListRecursive", confListRecursive}, + {"Remove", confRemove}, + {"PathEscape", confPathEscape}, + {"PutIf", confPutIf}, + {"UploadDownload", confUploadDownload}, + } { + t.Run(tc.name, func(t *testing.T) { tc.fn(t, s, k) }) + } +} + +func confStreamRoundTrip(t *testing.T, s Storage, k func(string) string) { + ctx := context.Background() + r, want := deterministicStream(conformanceStreamSize, 42) + it, err := s.Put(ctx, k("stream/big.bin"), r, &PutOptions{ContentType: "application/octet-stream", Size: -1}) + if err != nil { + t.Fatalf("Put: %v", err) + } + if it.Size != conformanceStreamSize { + t.Errorf("Put size = %d, want %d", it.Size, conformanceStreamSize) + } + st, err := s.Stat(ctx, k("stream/big.bin")) + if err != nil { + t.Fatalf("Stat: %v", err) + } + if st.Size != conformanceStreamSize { + t.Errorf("Stat size = %d", st.Size) + } + if st.Key != k("stream/big.bin") { + t.Errorf("Stat key = %q", st.Key) + } + if st.ETag != it.ETag { + t.Errorf("Stat ETag %q != Put ETag %q", st.ETag, it.ETag) + } + body, gi, err := s.Get(ctx, k("stream/big.bin")) + if err != nil { + t.Fatalf("Get: %v", err) + } + h := sha256.New() + _, err = io.Copy(h, body) + _ = body.Close() + if err != nil { + t.Fatalf("read body: %v", err) + } + if !bytes.Equal(h.Sum(nil), want[:]) { + t.Fatal("content mismatch") + } + if gi.Size != conformanceStreamSize { + t.Errorf("Get item size = %d", gi.Size) + } +} + +func confFailingReaderLeavesNothing(t *testing.T, s Storage, k func(string) string) { + ctx := context.Background() + key := k("fail/broken.bin") + boom := errors.New("boom") + r := &failingReader{r: rand.New(rand.NewSource(1)), n: 18 << 20, err: boom} + if _, err := s.Put(ctx, key, r, nil); err == nil { + t.Fatal("Put with failing reader succeeded") + } + if _, err := s.Stat(ctx, key); !errors.Is(err, ErrNotFound) { + t.Fatalf("Stat after failed Put: %v, want ErrNotFound", err) + } + items, err := s.ListDir(ctx, k("fail")) + if err != nil { + t.Fatalf("ListDir: %v", err) + } + for _, it := range items { + if strings.HasSuffix(it.Key, ".partial") || strings.HasSuffix(it.Key, "broken.bin") { + t.Errorf("leftover after failed Put: %s", it.Key) + } + } + + cctx, cancel := context.WithCancel(ctx) + pr, pw := io.Pipe() + go func() { + _, _ = pw.Write(make([]byte, 1<<20)) + cancel() + _, _ = pw.Write(make([]byte, 1<<20)) + _ = pw.Close() + }() + if _, err := s.Put(cctx, k("fail/cancelled.bin"), pr, nil); err == nil { + t.Fatal("Put with cancelled context succeeded") + } + _ = pr.Close() + if _, err := s.Stat(ctx, k("fail/cancelled.bin")); !errors.Is(err, ErrNotFound) { + t.Fatalf("Stat after cancelled Put: %v, want ErrNotFound", err) + } +} + +func confListDirBoundary(t *testing.T, s Storage, k func(string) string) { + ctx := context.Background() + for _, p := range []string{"ld/a/b/1.txt", "ld/a/b/2.txt", "ld/a/b/sub/deep.txt", "ld/a/bc/x.txt", "ld/top.txt"} { + putString(t, s, k(p), "x") + } + want := []string{k("ld/a/b/1.txt"), k("ld/a/b/2.txt")} + for _, dir := range []string{k("ld/a/b"), k("ld/a/b/")} { + items, err := s.ListDir(ctx, dir) + if err != nil { + t.Fatalf("ListDir(%s): %v", dir, err) + } + if got := keysOf(items, false); strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("ListDir(%s) = %v, want %v", dir, got, want) + } + for _, it := range items { + if it.IsDirectory || it.Size != 1 { + t.Errorf("ListDir item %+v", it) + } + } + } + items, err := s.ListDir(ctx, k("ld")) + if err != nil { + t.Fatal(err) + } + if got := keysOf(items, false); len(got) != 1 || got[0] != k("ld/top.txt") { + t.Errorf("ListDir(ld) = %v", got) + } + items, err = s.ListDir(ctx, k("ld/missing")) + if err != nil || len(items) != 0 { + t.Errorf("ListDir(missing) = %v, %v", items, err) + } +} + +func confListRecursive(t *testing.T, s Storage, k func(string) string) { + ctx := context.Background() + items, err := s.List(ctx, k("ld/a/b/")) + if err != nil { + t.Fatal(err) + } + want := []string{k("ld/a/b/1.txt"), k("ld/a/b/2.txt"), k("ld/a/b/sub/deep.txt")} + if got := keysOf(items, true); strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("List(ld/a/b/) = %v, want %v", got, want) + } + items, err = s.List(ctx, k("ld/")) + if err != nil { + t.Fatal(err) + } + if got := keysOf(items, true); len(got) != 5 { + t.Errorf("List(ld/) = %v, want 5 files", got) + } +} + +func confRemove(t *testing.T, s Storage, k func(string) string) { + ctx := context.Background() + if err := s.Remove(ctx, k("does/not/exist.txt")); !errors.Is(err, ErrNotFound) { + t.Fatalf("Remove missing: %v, want ErrNotFound", err) + } + putString(t, s, k("rm/x.txt"), "x") + if err := s.Remove(ctx, k("rm/x.txt")); err != nil { + t.Fatalf("Remove: %v", err) + } + if _, err := s.Stat(ctx, k("rm/x.txt")); !errors.Is(err, ErrNotFound) { + t.Fatalf("Stat after Remove: %v", err) + } + if _, _, err := s.Get(ctx, k("rm/x.txt")); !errors.Is(err, ErrNotFound) { + t.Fatalf("Get after Remove: %v", err) + } +} + +func confPathEscape(t *testing.T, s Storage, k func(string) string) { + ctx := context.Background() + if _, err := s.Put(ctx, "../escape.txt", strings.NewReader("x"), nil); err == nil { + t.Fatal("Put ../escape.txt succeeded") + } + if _, err := s.Stat(ctx, "a/../../escape.txt"); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("Stat escaping key: %v", err) + } +} + +func confPutIf(t *testing.T, s Storage, k func(string) string) { + ctx := context.Background() + key := k("index/index.json") + it1, err := s.PutIf(ctx, key, strings.NewReader(`{"v":1}`), Condition{IfNoneMatch: true}, &PutOptions{ContentType: "application/json"}) + if errors.Is(err, ErrConditionNotSupported) { + t.Skip("backend does not support conditional writes") + } + if err != nil { + t.Fatalf("PutIf IfNoneMatch (new): %v", err) + } + if _, err := s.PutIf(ctx, key, strings.NewReader(`{"v":2}`), Condition{IfNoneMatch: true}, nil); !errors.Is(err, ErrPreconditionFailed) { + t.Fatalf("PutIf IfNoneMatch (exists): %v, want ErrPreconditionFailed", err) + } + if got := getString(t, s, key); got != `{"v":1}` { + t.Fatalf("content after rejected PutIf = %q", got) + } + + it2, err := s.PutIf(ctx, key, strings.NewReader(`{"v":2}`), Condition{IfMatch: it1.ETag}, nil) + if errors.Is(err, ErrConditionNotSupported) { + t.Log("IfMatch not supported; skipping IfMatch checks") + return + } + if err != nil { + t.Fatalf("PutIf IfMatch (fresh): %v", err) + } + if it2.ETag == "" || it2.ETag == it1.ETag { + t.Fatalf("ETag did not change: %q -> %q", it1.ETag, it2.ETag) + } + if _, err := s.PutIf(ctx, key, strings.NewReader(`{"v":3}`), Condition{IfMatch: it1.ETag}, nil); !errors.Is(err, ErrPreconditionFailed) { + t.Fatalf("PutIf IfMatch (stale): %v, want ErrPreconditionFailed", err) + } + if got := getString(t, s, key); got != `{"v":2}` { + t.Fatalf("content = %q, want v2", got) + } + st, err := s.Stat(ctx, key) + if err != nil { + t.Fatal(err) + } + if st.ETag != it2.ETag { + t.Fatalf("Stat ETag %q != PutIf ETag %q", st.ETag, it2.ETag) + } + if _, err := s.PutIf(ctx, k("index/missing.json"), strings.NewReader("x"), Condition{IfMatch: it2.ETag}, nil); !errors.Is(err, ErrPreconditionFailed) { + t.Fatalf("PutIf IfMatch on missing key: %v, want ErrPreconditionFailed", err) + } +} + +func confUploadDownload(t *testing.T, s Storage, k func(string) string) { + ctx := context.Background() + dir := t.TempDir() + src := filepath.Join(dir, "src.txt") + if err := os.WriteFile(src, []byte("hello world"), 0o644); err != nil { + t.Fatal(err) + } + if err := s.Upload(ctx, src, k("ud/file.txt")); err != nil { + t.Fatalf("Upload: %v", err) + } + dst := filepath.Join(dir, "out", "dst.txt") + if err := s.Download(ctx, k("ud/file.txt"), dst); err != nil { + t.Fatalf("Download: %v", err) + } + b, err := os.ReadFile(dst) + if err != nil || string(b) != "hello world" { + t.Fatalf("downloaded %q, %v", b, err) + } + if err := s.Download(ctx, k("ud/missing.txt"), filepath.Join(dir, "m.txt")); !errors.Is(err, ErrNotFound) { + t.Fatalf("Download missing: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "m.txt")); !os.IsNotExist(err) { + t.Fatal("Download of missing key left a local file") + } +} diff --git a/ftp.go b/ftp.go new file mode 100644 index 0000000..491a6d4 --- /dev/null +++ b/ftp.go @@ -0,0 +1,396 @@ +/* +MIT License + +Copyright (c) 2024 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package go_storage + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "net/textproto" + "path" + "strings" + "sync" + "time" + + "github.com/jlaffaye/ftp" +) + +// FTPStorage implements Storage interface for FTP (optionally explicit FTPS). +// +// FTP uses a single control connection, so all operations are serialised by a +// mutex. The body returned by Get holds that connection until it is closed: +// a Get body MUST be closed before any other call on the same FTPStorage, +// otherwise that call blocks. +type FTPStorage struct { + host string + port string + baseDir string + + mu sync.Mutex + client *ftp.ServerConn +} + +// NewFTPStorage creates a new FTP storage instance. +func NewFTPStorage(config Config) (*FTPStorage, error) { + if err := validateConfig(config, map[string]string{ + "host": config.FTPHost, + }); err != nil { + return nil, fmt.Errorf("ftp: %w", err) + } + + port := config.FTPPort + if port == "" { + port = "21" + } + hostPort := net.JoinHostPort(config.FTPHost, port) + + opts := []ftp.DialOption{ftp.DialWithTimeout(30 * time.Second)} + if config.FTPTLS { + opts = append(opts, ftp.DialWithExplicitTLS(&tls.Config{ + ServerName: config.FTPHost, + MinVersion: tls.VersionTLS12, + })) + } + conn, err := ftp.Dial(hostPort, opts...) + if err != nil { + return nil, fmt.Errorf("ftp: failed to connect to %s: %w", hostPort, err) + } + + if err := conn.Login(config.FTPUsername, config.FTPPassword); err != nil { + _ = conn.Quit() + return nil, fmt.Errorf("ftp: login failed: %w", err) + } + + baseDir := normalizePathSeparators(config.FTPDirectory) + if baseDir == "" { + if wd, err := conn.CurrentDir(); err == nil && wd != "" { + baseDir = wd + } else { + baseDir = "/" + } + } + + return &FTPStorage{ + host: config.FTPHost, + port: port, + baseDir: path.Clean(baseDir), + client: conn, + }, nil +} + +func (f *FTPStorage) resolve(key string) (string, string, error) { + k, err := cleanKey(key) + if err != nil { + return "", "", fmt.Errorf("ftp: %w", err) + } + return k, path.Join(f.baseDir, k), nil +} + +func ftpCode(err error) int { + var te *textproto.Error + if errors.As(err, &te) { + return te.Code + } + return 0 +} + +func ftpNotFound(err error) bool { + return ftpCode(err) == ftp.StatusFileUnavailable +} + +// Put streams r to ..partial and renames it on success. +func (f *FTPStorage) Put(ctx context.Context, key string, r io.Reader, _ *PutOptions) (Item, error) { + k, full, err := f.resolve(key) + if err != nil { + return Item{}, err + } + if err := ctx.Err(); err != nil { + return Item{}, err + } + f.mu.Lock() + defer f.mu.Unlock() + + f.ensureDirectory(path.Dir(full)) + tmp := partialName(full) + err = f.client.Stor(tmp, ctxReader{ctx: ctx, r: r}) + if err == nil { + err = ctx.Err() + } + if err != nil { + _ = f.client.Delete(tmp) + return Item{}, fmt.Errorf("ftp: failed to upload %s: %w", k, err) + } + if err := f.client.Rename(tmp, full); err != nil { + // Some servers refuse to rename onto an existing file. + _ = f.client.Delete(full) + if err := f.client.Rename(tmp, full); err != nil { + _ = f.client.Delete(tmp) + return Item{}, fmt.Errorf("ftp: failed to rename %s: %w", k, err) + } + } + return f.stat(k, full) +} + +// PutIf is not supported on FTP: it always returns ErrConditionNotSupported. +// Callers should fall back to a lock-based protocol. +func (f *FTPStorage) PutIf(_ context.Context, key string, _ io.Reader, _ Condition, _ *PutOptions) (Item, error) { + return Item{}, fmt.Errorf("ftp: conditional put of %s: %w", key, ErrConditionNotSupported) +} + +// ftpBody releases the connection lock when closed. +type ftpBody struct { + ctx context.Context + resp *ftp.Response + once sync.Once + mu *sync.Mutex +} + +func (b *ftpBody) Read(p []byte) (int, error) { + if err := b.ctx.Err(); err != nil { + return 0, err + } + return b.resp.Read(p) +} + +func (b *ftpBody) Close() error { + var err error + b.once.Do(func() { + err = b.resp.Close() + b.mu.Unlock() + }) + return err +} + +// Get opens key for streaming reads. The returned body holds the FTP control +// connection: close it before issuing any other call on this storage. +func (f *FTPStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) { + k, full, err := f.resolve(key) + if err != nil { + return nil, Item{}, err + } + if err := ctx.Err(); err != nil { + return nil, Item{}, err + } + f.mu.Lock() + it, err := f.stat(k, full) + if err != nil { + f.mu.Unlock() + return nil, Item{}, err + } + if it.IsDirectory { + f.mu.Unlock() + return nil, Item{}, fmt.Errorf("ftp: %s is a directory", k) + } + resp, err := f.client.Retr(full) + if err != nil { + f.mu.Unlock() + if ftpNotFound(err) { + return nil, Item{}, fmt.Errorf("ftp: get %s: %w", k, ErrNotFound) + } + return nil, Item{}, fmt.Errorf("ftp: failed to retrieve %s: %w", k, err) + } + return &ftpBody{ctx: ctx, resp: resp, mu: &f.mu}, it, nil +} + +// Stat returns metadata for key (from a listing of its parent directory). +func (f *FTPStorage) Stat(ctx context.Context, key string) (Item, error) { + k, full, err := f.resolve(key) + if err != nil { + return Item{}, err + } + if err := ctx.Err(); err != nil { + return Item{}, err + } + f.mu.Lock() + defer f.mu.Unlock() + return f.stat(k, full) +} + +func (f *FTPStorage) stat(key, full string) (Item, error) { + entries, err := f.client.List(path.Dir(full)) + if err != nil { + if ftpNotFound(err) { + return Item{}, fmt.Errorf("ftp: stat %s: %w", key, ErrNotFound) + } + return Item{}, fmt.Errorf("ftp: stat %s: %w", key, err) + } + name := path.Base(full) + for _, e := range entries { + if e.Name == name { + return ftpItem(key, e), nil + } + } + return Item{}, fmt.Errorf("ftp: stat %s: %w", key, ErrNotFound) +} + +func ftpItem(key string, e *ftp.Entry) Item { + it := Item{Key: key, ModifiedTime: e.Time, IsDirectory: e.Type == ftp.EntryTypeFolder} + if !it.IsDirectory { + it.Size = int64(e.Size) //nolint:gosec // sizes fit in int64 + } + return it +} + +// Upload uploads a file via FTP. +func (f *FTPStorage) Upload(ctx context.Context, localPath string, remotePath string) error { + return uploadFile(ctx, f, localPath, remotePath) +} + +// Download downloads a file via FTP. +func (f *FTPStorage) Download(ctx context.Context, remotePath string, localPath string) error { + return downloadFile(ctx, f, remotePath, localPath) +} + +// List lists files and directories (recursively) whose key starts with prefix. +func (f *FTPStorage) List(ctx context.Context, prefix string) ([]Item, error) { + p, err := cleanPrefix(prefix) + if err != nil { + return nil, fmt.Errorf("ftp: %w", err) + } + f.mu.Lock() + defer f.mu.Unlock() + + items := make([]Item, 0) + startRel := "" + if i := strings.LastIndex(p, "/"); i >= 0 { + startRel = p[:i] + } + if err := f.walk(ctx, startRel, p, &items); err != nil { + return nil, err + } + return items, nil +} + +func (f *FTPStorage) walk(ctx context.Context, rel, prefix string, items *[]Item) error { + if err := ctx.Err(); err != nil { + return err + } + entries, err := f.client.List(path.Join(f.baseDir, rel)) + if err != nil { + if ftpNotFound(err) { + return nil + } + return fmt.Errorf("ftp: failed to list directory %s: %w", rel, err) + } + for _, e := range entries { + if e.Name == "." || e.Name == ".." || e.Type == ftp.EntryTypeLink { + continue + } + key := path.Join(rel, e.Name) + isDir := e.Type == ftp.EntryTypeFolder + if strings.HasPrefix(key, prefix) { + *items = append(*items, ftpItem(key, e)) + } else if !isDir || !strings.HasPrefix(prefix, key+"/") { + continue + } + if isDir { + if err := f.walk(ctx, key, prefix, items); err != nil { + return err + } + } + } + return nil +} + +// ListDir lists the files directly inside dir. +func (f *FTPStorage) ListDir(ctx context.Context, dir string) ([]Item, error) { + d, err := cleanDir(dir) + if err != nil { + return nil, fmt.Errorf("ftp: %w", err) + } + if err := ctx.Err(); err != nil { + return nil, err + } + f.mu.Lock() + defer f.mu.Unlock() + entries, err := f.client.List(path.Join(f.baseDir, d)) + if err != nil { + if ftpNotFound(err) { + return []Item{}, nil + } + return nil, fmt.Errorf("ftp: failed to list directory %s: %w", dir, err) + } + items := make([]Item, 0, len(entries)) + for _, e := range entries { + if e.Type != ftp.EntryTypeFile { + continue + } + items = append(items, ftpItem(path.Join(d, e.Name), e)) + } + return items, nil +} + +// Remove deletes a file via FTP. +func (f *FTPStorage) Remove(ctx context.Context, remotePath string) error { + k, full, err := f.resolve(remotePath) + if err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + f.mu.Lock() + defer f.mu.Unlock() + if err := f.client.Delete(full); err != nil { + if ftpNotFound(err) { + return fmt.Errorf("ftp: remove %s: %w", k, ErrNotFound) + } + return fmt.Errorf("ftp: failed to remove %s: %w", k, err) + } + return nil +} + +// Close releases resources. +func (f *FTPStorage) Close() error { + f.mu.Lock() + defer f.mu.Unlock() + if err := f.client.Quit(); err != nil { + return fmt.Errorf("ftp: failed to close connection: %w", err) + } + return nil +} + +// ensureDirectory creates a directory path if it doesn't exist (errors are +// ignored: the directory usually exists already, and a real failure surfaces +// on the following STOR). +func (f *FTPStorage) ensureDirectory(dir string) { + if dir == "." || dir == "/" || dir == "" { + return + } + current := "" + if strings.HasPrefix(dir, "/") { + current = "/" + } + for _, part := range strings.Split(dir, "/") { + if part == "" { + continue + } + current = path.Join(current, part) + _ = f.client.MakeDir(current) + } +} diff --git a/gcs.go b/gcs.go new file mode 100644 index 0000000..56a99bc --- /dev/null +++ b/gcs.go @@ -0,0 +1,297 @@ +/* +MIT License + +Copyright (c) 2024 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package go_storage + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + + "cloud.google.com/go/storage" + "google.golang.org/api/googleapi" + "google.golang.org/api/iterator" + "google.golang.org/api/option" +) + +// GCSStorage implements Storage interface for Google Cloud Storage. +type GCSStorage struct { + client *storage.Client + bucket string +} + +// NewGCSStorage creates a new Google Cloud Storage instance. +// +// Credentials: GCSCredentialsJSON, else GCSCredentialsFile, else Application +// Default Credentials. The STORAGE_EMULATOR_HOST environment variable is +// honoured by the Google client library itself. +func NewGCSStorage(config Config) (*GCSStorage, error) { + if err := validateConfig(config, map[string]string{ + "bucket": config.GCSBucket, + }); err != nil { + return nil, fmt.Errorf("gcs: %w", err) + } + + options := make([]option.ClientOption, 0, 2) + if config.GCSEndpoint != "" { + options = append(options, option.WithEndpoint(config.GCSEndpoint)) + } + switch { + case config.GCSCredentialsJSON != "": + data := []byte(config.GCSCredentialsJSON) + options = append(options, option.WithAuthCredentialsJSON(gcsCredType(data), data)) + case config.GCSCredentialsFile != "": + data, err := os.ReadFile(config.GCSCredentialsFile) + if err != nil { + return nil, fmt.Errorf("gcs: failed to read credentials file: %w", err) + } + options = append(options, option.WithAuthCredentialsFile(gcsCredType(data), config.GCSCredentialsFile)) + } + return newGCSStorage(context.Background(), config.GCSBucket, options...) +} + +func newGCSStorage(ctx context.Context, bucket string, options ...option.ClientOption) (*GCSStorage, error) { + client, err := storage.NewClient(ctx, options...) + if err != nil { + return nil, fmt.Errorf("gcs: failed to create client: %w", err) + } + return &GCSStorage{client: client, bucket: bucket}, nil +} + +// gcsCredType detects the credential type from the JSON "type" field. +func gcsCredType(data []byte) option.CredentialsType { + var v struct { + Type string `json:"type"` + } + _ = json.Unmarshal(data, &v) + switch v.Type { + case "authorized_user": + return option.AuthorizedUser + case "impersonated_service_account": + return option.ImpersonatedServiceAccount + case "external_account": + return option.ExternalAccount + default: + return option.ServiceAccount + } +} + +// Client returns the underlying GCS client. +func (g *GCSStorage) Client() *storage.Client { return g.client } + +func gcsItem(attrs *storage.ObjectAttrs) Item { + return Item{ + Key: attrs.Name, + ModifiedTime: attrs.Updated, + Size: attrs.Size, + ETag: strconv.FormatInt(attrs.Generation, 10), + } +} + +func (g *GCSStorage) put(ctx context.Context, key string, r io.Reader, opts *PutOptions, cond *storage.Conditions) (Item, error) { + k, err := cleanKey(key) + if err != nil { + return Item{}, fmt.Errorf("gcs: %w", err) + } + obj := g.client.Bucket(g.bucket).Object(k) + if cond != nil { + obj = obj.If(*cond) + } + // A child context: cancelling it before Close aborts the upload so the + // object is never finalised. + wctx, cancel := context.WithCancel(ctx) + defer cancel() + w := obj.NewWriter(wctx) + if opts != nil { + w.ContentType = opts.ContentType + if len(opts.Metadata) > 0 { + w.Metadata = opts.Metadata + } + } + if _, err := io.Copy(w, ctxReader{ctx: ctx, r: r}); err != nil { + cancel() + _ = w.Close() + if cond != nil && isGCSPrecondition(err) { + return Item{}, fmt.Errorf("gcs: conditional put of %s: %w", k, ErrPreconditionFailed) + } + return Item{}, fmt.Errorf("gcs: failed to upload %s: %w", k, err) + } + if err := ctx.Err(); err != nil { + cancel() + _ = w.Close() + return Item{}, fmt.Errorf("gcs: failed to upload %s: %w", k, err) + } + if err := w.Close(); err != nil { + if cond != nil && isGCSPrecondition(err) { + return Item{}, fmt.Errorf("gcs: conditional put of %s: %w", k, ErrPreconditionFailed) + } + return Item{}, fmt.Errorf("gcs: failed to finalize upload of %s: %w", k, err) + } + return gcsItem(w.Attrs()), nil +} + +// Put streams r to key. +func (g *GCSStorage) Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error) { + return g.put(ctx, key, r, opts, nil) +} + +// PutIf writes r to key with a generation precondition (IfMatch is the +// generation as returned in Item.ETag). +func (g *GCSStorage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error) { + if err := validateCondition(cond); err != nil { + return Item{}, fmt.Errorf("gcs: %w", err) + } + c := storage.Conditions{} + switch { + case cond.IfNoneMatch: + c.DoesNotExist = true + default: + gen, err := strconv.ParseInt(cond.IfMatch, 10, 64) + if err != nil || gen <= 0 { + return Item{}, fmt.Errorf("gcs: invalid generation %q: %w", cond.IfMatch, ErrPreconditionFailed) + } + c.GenerationMatch = gen + } + return g.put(ctx, key, r, opts, &c) +} + +// Get streams the object body. +func (g *GCSStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) { + k, err := cleanKey(key) + if err != nil { + return nil, Item{}, fmt.Errorf("gcs: %w", err) + } + rd, err := g.client.Bucket(g.bucket).Object(k).NewReader(ctx) + if err != nil { + return nil, Item{}, g.mapErr("get", k, err) + } + return rd, Item{ + Key: k, + Size: rd.Attrs.Size, + ModifiedTime: rd.Attrs.LastModified, + ETag: strconv.FormatInt(rd.Attrs.Generation, 10), + }, nil +} + +// Stat returns object attributes. +func (g *GCSStorage) Stat(ctx context.Context, key string) (Item, error) { + k, err := cleanKey(key) + if err != nil { + return Item{}, fmt.Errorf("gcs: %w", err) + } + attrs, err := g.client.Bucket(g.bucket).Object(k).Attrs(ctx) + if err != nil { + return Item{}, g.mapErr("stat", k, err) + } + return gcsItem(attrs), nil +} + +// Upload uploads a file to GCS. +func (g *GCSStorage) Upload(ctx context.Context, localPath string, remotePath string) error { + if err := uploadFile(ctx, g, localPath, remotePath); err != nil { + return fmt.Errorf("gcs: %w", err) + } + return nil +} + +// Download downloads a file from GCS. +func (g *GCSStorage) Download(ctx context.Context, remotePath string, localPath string) error { + if err := downloadFile(ctx, g, remotePath, localPath); err != nil { + return fmt.Errorf("gcs: %w", err) + } + return nil +} + +// List lists objects (recursively) with the given prefix. +func (g *GCSStorage) List(ctx context.Context, prefix string) ([]Item, error) { + p, err := cleanPrefix(prefix) + if err != nil { + return nil, fmt.Errorf("gcs: %w", err) + } + return g.list(ctx, &storage.Query{Prefix: p}) +} + +// ListDir lists the objects directly under dir. +func (g *GCSStorage) ListDir(ctx context.Context, dir string) ([]Item, error) { + d, err := cleanDir(dir) + if err != nil { + return nil, fmt.Errorf("gcs: %w", err) + } + return g.list(ctx, &storage.Query{Prefix: dirPrefix(d), Delimiter: "/"}) +} + +func (g *GCSStorage) list(ctx context.Context, q *storage.Query) ([]Item, error) { + items := make([]Item, 0) + it := g.client.Bucket(g.bucket).Objects(ctx, q) + for { + attrs, err := it.Next() + if errors.Is(err, iterator.Done) { + break + } + if err != nil { + return nil, fmt.Errorf("gcs: failed to list objects in bucket %s: %w", g.bucket, err) + } + if attrs.Name == "" || (q.Delimiter != "" && strings.HasSuffix(attrs.Name, "/")) { + continue // synthetic prefix or directory marker + } + items = append(items, gcsItem(attrs)) + } + return items, nil +} + +// Remove deletes an object. +func (g *GCSStorage) Remove(ctx context.Context, remotePath string) error { + k, err := cleanKey(remotePath) + if err != nil { + return fmt.Errorf("gcs: %w", err) + } + if err := g.client.Bucket(g.bucket).Object(k).Delete(ctx); err != nil { + return g.mapErr("remove", k, err) + } + return nil +} + +// Close releases resources. +func (g *GCSStorage) Close() error { + return g.client.Close() +} + +func (g *GCSStorage) mapErr(op, key string, err error) error { + var ge *googleapi.Error + if errors.Is(err, storage.ErrObjectNotExist) || (errors.As(err, &ge) && ge.Code == http.StatusNotFound) { + return fmt.Errorf("gcs: %s %s in bucket %s: %w", op, key, g.bucket, ErrNotFound) + } + return fmt.Errorf("gcs: %s %s in bucket %s: %w", op, key, g.bucket, err) +} + +func isGCSPrecondition(err error) bool { + var ge *googleapi.Error + return errors.As(err, &ge) && (ge.Code == http.StatusPreconditionFailed || ge.Code == http.StatusConflict) +} diff --git a/go.mod b/go.mod index 2918cf8..545fb1f 100644 --- a/go.mod +++ b/go.mod @@ -1,22 +1,86 @@ module github.com/jkaninda/go-storage -go 1.23.0 +go 1.25 require ( + cloud.google.com/go/storage v1.60.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 github.com/aws/aws-sdk-go v1.55.7 + github.com/aws/aws-sdk-go-v2 v1.47.3 + github.com/aws/aws-sdk-go-v2/config v1.33.4 + github.com/aws/aws-sdk-go-v2/credentials v1.20.9 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.43 + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 + github.com/aws/smithy-go v1.28.5 github.com/bramvdbogaerde/go-scp v1.5.0 github.com/jlaffaye/ftp v0.2.0 - golang.org/x/crypto v0.38.0 + github.com/pkg/sftp v1.13.10 + golang.org/x/crypto v0.47.0 + google.golang.org/api v0.267.0 ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect + cel.dev/expr v0.24.0 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.18.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect + github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.1.3 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect + github.com/googleapis/gax-go/v2 v2.17.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - golang.org/x/net v0.39.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.25.0 // indirect + github.com/kr/fs v0.1.0 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.39.0 // indirect + go.opentelemetry.io/otel/metric v1.39.0 // indirect + go.opentelemetry.io/otel/sdk v1.39.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.39.0 // indirect + go.opentelemetry.io/otel/trace v1.39.0 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/oauth2 v0.35.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/text v0.33.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect + google.golang.org/grpc v1.78.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/go.sum b/go.sum index d7a4448..8f8c3e1 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,25 @@ +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs= +cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY= +cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw= +cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= +cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= +cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/storage v1.60.0 h1:oBfZrSOCimggVNz9Y/bXY35uUcts7OViubeddTTVzQ8= +cloud.google.com/go/storage v1.60.0/go.mod h1:q+5196hXfejkctrnx+VYU8RKQr/L3c0cBIlrjmiAKE0= +cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= +cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g= @@ -10,17 +32,96 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSy github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE= github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= +github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 h1:oiEmPz/YMMeh/JvAE8xSxNCIsvWE6lpFe4ZPlyFykQI= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22/go.mod h1:QWUDuRi9PFjBHIU2iH1Pkldbe7zqLfXJC4wWLbF8nuQ= +github.com/aws/aws-sdk-go-v2/config v1.33.4 h1:FzvkXKSzwqHni4U7nDigHg4jjtqMpVUuHgmZfSoJVQ0= +github.com/aws/aws-sdk-go-v2/config v1.33.4/go.mod h1:VZqGZnZsCWVfK/iGPptJIyNIX3XEX6iQU2Rel4sLrr8= +github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0= +github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.43 h1:Fhx4uwBshQF0+jmhV/FKDM4LPj7L9iUy5n3MvLxLy00= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.43/go.mod h1:4Tyfc1eIPEyrAMysLCVkcYL1Hm/hkGWbElLo8pejHPo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 h1:CwvFlKw0uUScfgOMflXSRZa+ayXgOf6wk7lhyMemQYU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7/go.mod h1:4SbD8BB+p+QEnG0aX6DoWwrtk9SufyYXRsUv0nve8LE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 h1:NyeuyXkZL476uoAowNActm4XJO1QPKf7aassgXHpiqA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3/go.mod h1:GfoBeKMDJecgRUsGf/0RGdx0NJAh81ObTd922h8pQys= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 h1:OKEfbf2tcfsF2pBk/S9QqNpo6alaAtiBs4hysCsBjPw= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.4/go.mod h1:mCd6qGo9b8hTtfjgsmNkFlzwjyce4owd7HDVFnCLXhM= +github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= +github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bramvdbogaerde/go-scp v1.5.0 h1:a9BinAjTfQh273eh7vd3qUgmBC+bx+3TRDtkZWmIpzM= github.com/bramvdbogaerde/go-scp v1.5.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0= +github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM= +github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs= +github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo= +github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao= +github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8= +github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc= +github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -32,25 +133,74 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU= +github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= -golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs= +go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8OkCfD1j3/ER79rUuTYmCvlXBKeYL8= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= +golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= +golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE= +google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM= +google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0= +google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/integration_test.go b/integration_test.go new file mode 100644 index 0000000..f43aa7c --- /dev/null +++ b/integration_test.go @@ -0,0 +1,263 @@ +//go:build integration + +/* +MIT License + +Copyright (c) 2024 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package go_storage + +// Integration tests against real services started with the docker CLI. +// +// go test -tags integration ./... +// +// Tests skip when docker is unavailable. Images can be overridden with +// GO_STORAGE_MINIO_IMAGE, GO_STORAGE_AZURITE_IMAGE, GO_STORAGE_FAKEGCS_IMAGE +// and GO_STORAGE_FTP_IMAGE. + +import ( + "context" + "errors" + "fmt" + "math/rand" + "net" + "os" + "os/exec" + "strconv" + "strings" + "testing" + "time" + + "cloud.google.com/go/storage" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "google.golang.org/api/option" +) + +func image(env, def string) string { + if v := os.Getenv(env); v != "" { + return v + } + return def +} + +func requireDocker(t *testing.T) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + if err := exec.CommandContext(ctx, "docker", "info").Run(); err != nil { + t.Skipf("docker not available: %v", err) + } +} + +func docker(t *testing.T, args ...string) string { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + out, err := exec.CommandContext(ctx, "docker", args...).CombinedOutput() + if err != nil { + t.Fatalf("docker %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) +} + +// runContainer starts a detached container and removes it at cleanup. +func runContainer(t *testing.T, args ...string) string { + t.Helper() + id := docker(t, append([]string{"run", "-d"}, args...)...) + t.Cleanup(func() { + if t.Failed() { + out, _ := exec.Command("docker", "logs", "--tail", "50", id).CombinedOutput() + t.Logf("container logs:\n%s", out) + } + _ = exec.Command("docker", "rm", "-f", "-v", id).Run() + }) + return id +} + +// hostPort returns the host port mapped to the container port. +func hostPort(t *testing.T, id, containerPort string) string { + t.Helper() + out := docker(t, "port", id, containerPort) + line := strings.Split(out, "\n")[0] + _, port, err := net.SplitHostPort(strings.TrimSpace(line)) + if err != nil { + t.Fatalf("parse docker port %q: %v", out, err) + } + return port +} + +func freePort(t *testing.T) int { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer func() { _ = l.Close() }() + return l.Addr().(*net.TCPAddr).Port +} + +const readyTimeout = 60 * time.Second + +// eventually retries fn until it succeeds or readyTimeout expires. +func eventually(t *testing.T, what string, fn func() error) { + t.Helper() + deadline := time.Now().Add(readyTimeout) + var err error + for time.Now().Before(deadline) { + if err = fn(); err == nil { + return + } + time.Sleep(500 * time.Millisecond) + } + t.Fatalf("%s: %v", what, err) +} + +func TestIntegrationMinIO(t *testing.T) { + requireDocker(t) + id := runContainer(t, "-p", "127.0.0.1::9000", + "-e", "MINIO_ROOT_USER=minioadmin", "-e", "MINIO_ROOT_PASSWORD=minioadmin", + image("GO_STORAGE_MINIO_IMAGE", "pgsty/minio:latest"), "server", "/data") + port := hostPort(t, id, "9000/tcp") + + s, err := NewS3Storage(Config{ + S3Bucket: "conformance", + S3Endpoint: "127.0.0.1:" + port, + S3DisableTLS: true, + S3ForcePath: true, + S3KeyID: "minioadmin", + S3Secret: "minioadmin", + S3PartSize: MinS3PartSize, // 20 MiB stream => real multipart upload + S3Concurrency: 2, + }) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + eventually(t, "create bucket", func() error { + _, err := s.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String("conformance")}) + return err + }) + runConformance(t, s) + + // Failed multipart uploads must have been aborted. + out, err := s.client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{Bucket: aws.String("conformance")}) + if err != nil { + t.Fatal(err) + } + if len(out.Uploads) != 0 { + t.Errorf("%d multipart uploads left behind", len(out.Uploads)) + } +} + +func TestIntegrationAzurite(t *testing.T) { + requireDocker(t) + id := runContainer(t, "-p", "127.0.0.1::10000", + image("GO_STORAGE_AZURITE_IMAGE", "mcr.microsoft.com/azure-storage/azurite:latest"), + "azurite-blob", "--blobHost", "0.0.0.0", "--skipApiVersionCheck", "--loose") + port := hostPort(t, id, "10000/tcp") + + s, err := NewAzureStorage(Config{ + AzureAccount: "devstoreaccount1", + AzureKey: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + AzureContainer: "conformance", + AzureEndpoint: "http://127.0.0.1:" + port + "/devstoreaccount1", + }) + if err != nil { + t.Fatal(err) + } + eventually(t, "create container", func() error { + _, err := s.client.CreateContainer(context.Background(), "conformance", nil) + return err + }) + runConformance(t, s) +} + +func TestIntegrationFakeGCS(t *testing.T) { + requireDocker(t) + port := freePort(t) + addr := "127.0.0.1:" + strconv.Itoa(port) + runContainer(t, "-p", fmt.Sprintf("%s:4443", addr), + image("GO_STORAGE_FAKEGCS_IMAGE", "fsouza/fake-gcs-server:latest"), + "-scheme", "http", "-port", "4443", "-public-host", addr, "-external-url", "http://"+addr, "-backend", "memory") + + ctx := context.Background() + s, err := newGCSStorage(ctx, "conformance", + option.WithEndpoint("http://"+addr+"/storage/v1/"), + option.WithoutAuthentication()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + eventually(t, "create bucket", func() error { + err := s.client.Bucket("conformance").Create(ctx, "test-project", &storage.BucketAttrs{}) + var exists bool + if err != nil && strings.Contains(err.Error(), "409") { + exists = true + } + if exists { + return nil + } + return err + }) + runConformance(t, s) +} + +func TestIntegrationFTP(t *testing.T) { + requireDocker(t) + // Passive ports must be identical inside and outside the container. + minPort := 30000 + rand.Intn(20000) + maxPort := minPort + 9 + ctrlPort := freePort(t) + runContainer(t, + "-p", fmt.Sprintf("127.0.0.1:%d:21", ctrlPort), + "-p", fmt.Sprintf("127.0.0.1:%d-%d:%d-%d", minPort, maxPort, minPort, maxPort), + "-e", "USERS=backup|s3cret|/ftp/backup", + "-e", "ADDRESS=127.0.0.1", + "-e", "MIN_PORT="+strconv.Itoa(minPort), "-e", "MAX_PORT="+strconv.Itoa(maxPort), + image("GO_STORAGE_FTP_IMAGE", "delfer/alpine-ftp-server:latest")) + + var s *FTPStorage + eventually(t, "connect ftp", func() error { + var err error + s, err = NewFTPStorage(Config{ + FTPHost: "127.0.0.1", + FTPPort: strconv.Itoa(ctrlPort), + FTPUsername: "backup", + FTPPassword: "s3cret", + }) + if err != nil { + return err + } + if _, err := s.ListDir(context.Background(), ""); err != nil { + _ = s.Close() + return err + } + return nil + }) + t.Cleanup(func() { _ = s.Close() }) + runConformance(t, s) + + if _, err := s.PutIf(context.Background(), "x", strings.NewReader("x"), Condition{IfNoneMatch: true}, nil); !errors.Is(err, ErrConditionNotSupported) { + t.Fatalf("FTP PutIf: %v, want ErrConditionNotSupported", err) + } +} diff --git a/local.go b/local.go new file mode 100644 index 0000000..450649f --- /dev/null +++ b/local.go @@ -0,0 +1,433 @@ +/* +MIT License + +Copyright (c) 2024 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package go_storage + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +// LocalStorage implements the Storage interface using the local filesystem. +// It is safe for concurrent use. +type LocalStorage struct { + basePath string +} + +// Lock tuning for PutIf / Put on the local backend. +const ( + localLockTimeout = 10 * time.Second + localLockRetry = 10 * time.Millisecond + // A lock older than this is considered stale (the holder crashed). The + // lock is only held around a stat + rename, never while streaming data. + localLockStale = 30 * time.Second +) + +// NewLocalStorage creates a new LocalStorage instance with the given base directory. +// The base directory will be created if it doesn't exist. +func NewLocalStorage(basePath string) (*LocalStorage, error) { + absPath, err := filepath.Abs(basePath) + if err != nil { + return nil, fmt.Errorf("local: failed to resolve absolute path: %w", err) + } + + if err := os.MkdirAll(absPath, 0o755); err != nil { + return nil, fmt.Errorf("local: failed to create base directory: %w", err) + } + + return &LocalStorage{basePath: absPath}, nil +} + +// resolvePath safely resolves a key to an absolute local path, ensuring it +// stays within the base directory. +func (s *LocalStorage) resolvePath(remotePath string) (string, error) { + key, err := cleanKey(remotePath) + if err != nil { + return "", fmt.Errorf("local: %w", err) + } + return s.join(key) +} + +func (s *LocalStorage) join(key string) (string, error) { + fullPath := filepath.Join(s.basePath, filepath.FromSlash(key)) + rel, err := filepath.Rel(s.basePath, fullPath) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("local: path traversal attempt detected: %s", key) + } + return fullPath, nil +} + +// localETag derives the version token from mtime and size. +func localETag(fi fs.FileInfo) string { + return strconv.FormatInt(fi.ModTime().UnixNano(), 16) + "-" + strconv.FormatInt(fi.Size(), 16) +} + +func (s *LocalStorage) item(full string, fi fs.FileInfo) Item { + rel, err := filepath.Rel(s.basePath, full) + if err != nil { + rel = full + } + it := Item{ + Key: filepath.ToSlash(rel), + ModifiedTime: fi.ModTime(), + IsDirectory: fi.IsDir(), + } + if !fi.IsDir() { + it.Size = fi.Size() + it.ETag = localETag(fi) + } + return it +} + +// Upload uploads a local file to the storage with the given target name. +func (s *LocalStorage) Upload(ctx context.Context, localPath string, remotePath string) error { + return uploadFile(ctx, s, localPath, remotePath) +} + +// Download downloads a file from storage to a local path. +func (s *LocalStorage) Download(ctx context.Context, remotePath string, localPath string) error { + return downloadFile(ctx, s, remotePath, localPath) +} + +// writeTemp streams r into a new temporary file next to full and returns its path. +func writeTemp(ctx context.Context, full string, r io.Reader) (string, error) { + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + return "", fmt.Errorf("local: failed to create parent directories: %w", err) + } + tmp := partialName(full) + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return "", fmt.Errorf("local: failed to create temporary file: %w", err) + } + _, err = io.Copy(f, ctxReader{ctx: ctx, r: r}) + if err == nil { + err = f.Sync() + } + if cerr := f.Close(); err == nil { + err = cerr + } + if err == nil { + err = ctx.Err() + } + if err != nil { + _ = os.Remove(tmp) + return "", fmt.Errorf("local: failed to write data: %w", err) + } + return tmp, nil +} + +// Put streams r to key via a temporary file that is renamed on success. +func (s *LocalStorage) Put(ctx context.Context, key string, r io.Reader, _ *PutOptions) (Item, error) { + return s.put(ctx, key, r, nil) +} + +// PutIf writes r to key only if cond holds. It uses an exclusive lock file +// (.lock) so it is atomic with respect to other PutIf/Put calls on the +// same filesystem. +func (s *LocalStorage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, _ *PutOptions) (Item, error) { + if err := validateCondition(cond); err != nil { + return Item{}, fmt.Errorf("local: %w", err) + } + return s.put(ctx, key, r, &cond) +} + +func (s *LocalStorage) put(ctx context.Context, key string, r io.Reader, cond *Condition) (Item, error) { + full, err := s.resolvePath(key) + if err != nil { + return Item{}, err + } + tmp, err := writeTemp(ctx, full, r) + if err != nil { + return Item{}, err + } + committed := false + defer func() { + if !committed { + _ = os.Remove(tmp) + } + }() + + unlock, err := acquireLock(ctx, full+".lock") + if err != nil { + return Item{}, err + } + defer unlock() + + cur, statErr := os.Stat(full) + exists := statErr == nil + if statErr != nil && !errors.Is(statErr, fs.ErrNotExist) { + return Item{}, fmt.Errorf("local: failed to stat %s: %w", key, statErr) + } + if cond != nil { + switch { + case cond.IfNoneMatch && exists: + return Item{}, fmt.Errorf("local: %s already exists: %w", key, ErrPreconditionFailed) + case cond.IfMatch != "" && (!exists || localETag(cur) != cond.IfMatch): + return Item{}, fmt.Errorf("local: %s etag mismatch: %w", key, ErrPreconditionFailed) + } + } + if exists { + if err := bumpMtime(tmp, cur.ModTime()); err != nil { + return Item{}, err + } + } + if err := ctx.Err(); err != nil { + return Item{}, err + } + if err := os.Rename(tmp, full); err != nil { + return Item{}, fmt.Errorf("local: failed to rename temporary file: %w", err) + } + committed = true + fi, err := os.Stat(full) + if err != nil { + return Item{}, fmt.Errorf("local: failed to stat %s: %w", key, err) + } + return s.item(full, fi), nil +} + +// bumpMtime makes sure the mtime of tmp is strictly after prev so that the +// ETag changes even on filesystems with coarse timestamps. +func bumpMtime(tmp string, prev time.Time) error { + for _, step := range []time.Duration{time.Microsecond, time.Millisecond, time.Second, 2 * time.Second} { + fi, err := os.Stat(tmp) + if err != nil { + return fmt.Errorf("local: failed to stat temporary file: %w", err) + } + if fi.ModTime().After(prev) { + return nil + } + t := prev.Add(step) + if err := os.Chtimes(tmp, t, t); err != nil { + return fmt.Errorf("local: failed to set mtime: %w", err) + } + } + return nil +} + +// acquireLock creates lockPath exclusively, retrying until localLockTimeout. +func acquireLock(ctx context.Context, lockPath string) (func(), error) { + deadline := time.Now().Add(localLockTimeout) + for { + f, err := os.OpenFile(lockPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err == nil { + _, _ = fmt.Fprintf(f, "%d\n", os.Getpid()) + _ = f.Close() + return func() { _ = os.Remove(lockPath) }, nil + } + if !errors.Is(err, fs.ErrExist) { + return nil, fmt.Errorf("local: failed to create lock %s: %w", lockPath, err) + } + if fi, serr := os.Stat(lockPath); serr == nil && time.Since(fi.ModTime()) > localLockStale { + _ = os.Remove(lockPath) + continue + } + if time.Now().After(deadline) { + return nil, fmt.Errorf("local: timed out waiting for lock %s", lockPath) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(localLockRetry): + } + } +} + +// Get opens key for reading. +func (s *LocalStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) { + if err := ctx.Err(); err != nil { + return nil, Item{}, err + } + full, err := s.resolvePath(key) + if err != nil { + return nil, Item{}, err + } + f, err := os.Open(full) + if err != nil { + return nil, Item{}, mapLocalErr(key, err) + } + fi, err := f.Stat() + if err != nil { + _ = f.Close() + return nil, Item{}, mapLocalErr(key, err) + } + if fi.IsDir() { + _ = f.Close() + return nil, Item{}, fmt.Errorf("local: %s is a directory", key) + } + return f, s.item(full, fi), nil +} + +// Stat returns metadata for key. +func (s *LocalStorage) Stat(ctx context.Context, key string) (Item, error) { + if err := ctx.Err(); err != nil { + return Item{}, err + } + full, err := s.resolvePath(key) + if err != nil { + return Item{}, err + } + fi, err := os.Stat(full) + if err != nil { + return Item{}, mapLocalErr(key, err) + } + return s.item(full, fi), nil +} + +func mapLocalErr(key string, err error) error { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("local: %s: %w", key, ErrNotFound) + } + return fmt.Errorf("local: %s: %w", key, err) +} + +// ListDir lists the files directly inside dir. +func (s *LocalStorage) ListDir(ctx context.Context, dir string) ([]Item, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + d, err := cleanDir(dir) + if err != nil { + return nil, fmt.Errorf("local: %w", err) + } + full, err := s.join(d) + if err != nil { + return nil, err + } + entries, err := os.ReadDir(full) + if errors.Is(err, fs.ErrNotExist) { + return []Item{}, nil + } + if err != nil { + return nil, fmt.Errorf("local: failed to read directory %s: %w", dir, err) + } + items := make([]Item, 0, len(entries)) + for _, e := range entries { + if e.IsDir() { + continue + } + fi, err := e.Info() + if err != nil || !fi.Mode().IsRegular() { + continue + } + items = append(items, s.item(filepath.Join(full, e.Name()), fi)) + } + return items, nil +} + +// List returns all items (files and directories, recursively) whose key +// starts with prefix. +func (s *LocalStorage) List(ctx context.Context, prefix string) ([]Item, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + p, err := cleanPrefix(prefix) + if err != nil { + return nil, fmt.Errorf("local: %w", err) + } + + // Walk from the deepest directory fully contained in the prefix. + root := s.basePath + if i := strings.LastIndex(p, "/"); i >= 0 { + root = filepath.Join(s.basePath, filepath.FromSlash(p[:i])) + } + items := make([]Item, 0) + err = filepath.WalkDir(root, func(full string, d fs.DirEntry, err error) error { + if cerr := ctx.Err(); cerr != nil { + return cerr + } + if err != nil { + if full == root && errors.Is(err, fs.ErrNotExist) { + return fs.SkipAll + } + return nil // skip unreadable entries + } + if full == s.basePath { + return nil + } + rel, rerr := filepath.Rel(s.basePath, full) + if rerr != nil { + return nil + } + rel = filepath.ToSlash(rel) + if !strings.HasPrefix(rel, p) { + // Prune directories that cannot contain matches. + if d.IsDir() && !strings.HasPrefix(p, rel+"/") { + return fs.SkipDir + } + return nil + } + fi, ierr := d.Info() + if ierr != nil { + return nil + } + items = append(items, s.item(full, fi)) + return nil + }) + if err != nil { + return nil, fmt.Errorf("local: failed to walk directory: %w", err) + } + return items, nil +} + +// Remove deletes a file (or directory tree) from storage. +func (s *LocalStorage) Remove(ctx context.Context, remotePath string) error { + if err := ctx.Err(); err != nil { + return err + } + targetPath, err := s.resolvePath(remotePath) + if err != nil { + return err + } + info, err := os.Stat(targetPath) + if err != nil { + return mapLocalErr(remotePath, err) + } + if info.IsDir() { + err = os.RemoveAll(targetPath) + } else { + err = os.Remove(targetPath) + } + if err != nil { + return mapLocalErr(remotePath, err) + } + return nil +} + +// Close releases any resources held by the storage implementation (no-op). +func (s *LocalStorage) Close() error { + return nil +} + +// BasePath returns the base directory path of the storage. +func (s *LocalStorage) BasePath() string { + return s.basePath +} diff --git a/pkg/azure/azure.go b/pkg/azure/azure.go index 88dcab3..8ca5b41 100644 --- a/pkg/azure/azure.go +++ b/pkg/azure/azure.go @@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +// Package azure is the legacy v0.1 API of go-storage. +// +// Deprecated: use the root package github.com/jkaninda/go-storage +// (NewStorage and the streaming Storage interface) instead. package azure import ( diff --git a/pkg/ftp/ftp.go b/pkg/ftp/ftp.go index 1b84e29..4f3132c 100644 --- a/pkg/ftp/ftp.go +++ b/pkg/ftp/ftp.go @@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +// Package ftp is the legacy v0.1 API of go-storage. +// +// Deprecated: use the root package github.com/jkaninda/go-storage +// (NewStorage and the streaming Storage interface) instead. package ftp import ( diff --git a/pkg/local/local.go b/pkg/local/local.go index 0386446..c634d73 100644 --- a/pkg/local/local.go +++ b/pkg/local/local.go @@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +// Package local is the legacy v0.1 API of go-storage. +// +// Deprecated: use the root package github.com/jkaninda/go-storage +// (NewStorage and the streaming Storage interface) instead. package local import ( diff --git a/pkg/local/local_test.go b/pkg/local/local_test.go index fce970c..790dec9 100644 --- a/pkg/local/local_test.go +++ b/pkg/local/local_test.go @@ -1,3 +1,27 @@ +/* +MIT License + +Copyright (c) 2024 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + package local import ( diff --git a/pkg/s3/s3.go b/pkg/s3/s3.go index c4f0b10..0050d48 100644 --- a/pkg/s3/s3.go +++ b/pkg/s3/s3.go @@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +// Package s3 is the legacy v0.1 API of go-storage. +// +// Deprecated: use the root package github.com/jkaninda/go-storage +// (NewStorage and the streaming Storage interface) instead. package s3 import ( diff --git a/pkg/ssh/ssh.go b/pkg/ssh/ssh.go index 6328e1d..ece0eda 100644 --- a/pkg/ssh/ssh.go +++ b/pkg/ssh/ssh.go @@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +// Package ssh is the legacy v0.1 API of go-storage. +// +// Deprecated: use the root package github.com/jkaninda/go-storage +// (NewStorage and the streaming Storage interface) instead. package ssh import ( diff --git a/pkg/ssh/ssh_test.go b/pkg/ssh/ssh_test.go index 8104872..cf19277 100644 --- a/pkg/ssh/ssh_test.go +++ b/pkg/ssh/ssh_test.go @@ -1 +1,25 @@ +/* +MIT License + +Copyright (c) 2024 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + package ssh diff --git a/pkg/storage.go b/pkg/storage.go index 8e68b44..0a54a4a 100644 --- a/pkg/storage.go +++ b/pkg/storage.go @@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +// Package pkg is the legacy v0.1 API of go-storage. +// +// Deprecated: use the root package github.com/jkaninda/go-storage +// (NewStorage and the streaming Storage interface) instead. package pkg type Storage interface { diff --git a/s3.go b/s3.go new file mode 100644 index 0000000..fe0062c --- /dev/null +++ b/s3.go @@ -0,0 +1,405 @@ +/* +MIT License + +Copyright (c) 2024 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package go_storage + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" +) + +// S3 defaults. +const ( + DefaultS3PartSize int64 = 64 << 20 + MinS3PartSize int64 = 5 << 20 + DefaultS3Concurrency = 2 + DefaultS3Region = "us-east-1" +) + +// S3Storage implements Storage for Amazon S3 and S3-compatible services +// (MinIO, Ceph, Wasabi, ...). +type S3Storage struct { + client *s3.Client + uploader *manager.Uploader //nolint:staticcheck // see NewS3Storage + bucket string + region string + storageClass types.StorageClass + sse types.ServerSideEncryption + sseKMSKeyID string +} + +// NewS3Storage creates a new S3 storage instance. +// +// Credentials: static keys (S3KeyID/S3Secret/S3SessionToken) when set, +// otherwise the default AWS credential chain (environment, shared profile +// S3Profile, web identity/IRSA, ECS/EC2 roles). +func NewS3Storage(config Config) (*S3Storage, error) { + if err := validateConfig(config, map[string]string{ + "bucket": config.S3Bucket, + }); err != nil { + return nil, fmt.Errorf("s3: %w", err) + } + + opts := []func(*awsconfig.LoadOptions) error{ + awsconfig.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired), + awsconfig.WithResponseChecksumValidation(aws.ResponseChecksumValidationWhenRequired), + } + if config.S3Region != "" { + opts = append(opts, awsconfig.WithRegion(config.S3Region)) + } + if config.S3Profile != "" { + opts = append(opts, awsconfig.WithSharedConfigProfile(config.S3Profile)) + } + if config.S3KeyID != "" && config.S3Secret != "" { + opts = append(opts, awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(config.S3KeyID, config.S3Secret, config.S3SessionToken))) + } + + cfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...) + if err != nil { + return nil, fmt.Errorf("s3: failed to load AWS configuration: %w", err) + } + if cfg.Region == "" { + cfg.Region = DefaultS3Region + } + + endpoint := config.S3Endpoint + if endpoint != "" && !strings.Contains(endpoint, "://") { + if config.S3DisableTLS { + endpoint = "http://" + endpoint + } else { + endpoint = "https://" + endpoint + } + } + + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + if endpoint != "" { + o.BaseEndpoint = aws.String(endpoint) + } + o.UsePathStyle = config.S3ForcePath + }) + + partSize := config.S3PartSize + if partSize <= 0 { + partSize = DefaultS3PartSize + } + if partSize < MinS3PartSize { + partSize = MinS3PartSize + } + concurrency := config.S3Concurrency + if concurrency <= 0 { + concurrency = DefaultS3Concurrency + } + + //nolint:staticcheck // feature/s3/manager is deprecated in favour of the (pre-1.0) transfermanager. + uploader := manager.NewUploader(client, func(u *manager.Uploader) { + u.PartSize = partSize + u.Concurrency = concurrency + u.LeavePartsOnError = false + u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + }) + + s := &S3Storage{ + client: client, + uploader: uploader, + bucket: config.S3Bucket, + region: cfg.Region, + storageClass: types.StorageClass(config.S3StorageClass), + } + switch strings.ToLower(config.S3SSE) { + case "": + case "aes256": + s.sse = types.ServerSideEncryptionAes256 + case "aws:kms": + s.sse = types.ServerSideEncryptionAwsKms + s.sseKMSKeyID = config.S3SSEKMSKeyID + default: + s.sse = types.ServerSideEncryption(config.S3SSE) + s.sseKMSKeyID = config.S3SSEKMSKeyID + } + return s, nil +} + +// Client returns the underlying S3 client. +func (s *S3Storage) Client() *s3.Client { return s.client } + +func (s *S3Storage) putInput(key string, body io.Reader, opts *PutOptions) *s3.PutObjectInput { + in := &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + Body: body, + } + if opts != nil { + if opts.ContentType != "" { + in.ContentType = aws.String(opts.ContentType) + } + if len(opts.Metadata) > 0 { + in.Metadata = opts.Metadata + } + } + if s.storageClass != "" { + in.StorageClass = s.storageClass + } + if s.sse != "" { + in.ServerSideEncryption = s.sse + if s.sseKMSKeyID != "" { + in.SSEKMSKeyId = aws.String(s.sseKMSKeyID) + } + } + return in +} + +// Put streams r to key using a multipart upload (aborted on error). +func (s *S3Storage) Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error) { + k, err := cleanKey(key) + if err != nil { + return Item{}, fmt.Errorf("s3: %w", err) + } + cr := &countingReader{r: ctxReader{ctx: ctx, r: r}} + //nolint:staticcheck // see NewS3Storage + out, err := s.uploader.Upload(ctx, s.putInput(k, cr, opts)) + if err != nil { + return Item{}, fmt.Errorf("s3: failed to upload %s to bucket %s: %w", k, s.bucket, err) + } + return Item{ + Key: k, + Size: cr.n, + ModifiedTime: time.Now().UTC(), + ETag: aws.ToString(out.ETag), + VersionID: aws.ToString(out.VersionID), + }, nil +} + +// PutIf performs a conditional single-request PutObject (If-Match / +// If-None-Match: *). The body is buffered and limited to 16 MiB. +func (s *S3Storage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error) { + k, err := cleanKey(key) + if err != nil { + return Item{}, fmt.Errorf("s3: %w", err) + } + if err := validateCondition(cond); err != nil { + return Item{}, fmt.Errorf("s3: %w", err) + } + data, err := io.ReadAll(io.LimitReader(ctxReader{ctx: ctx, r: r}, maxConditionalPutSize+1)) + if err != nil { + return Item{}, fmt.Errorf("s3: failed to read data: %w", err) + } + if len(data) > maxConditionalPutSize { + return Item{}, fmt.Errorf("s3: conditional put of %s exceeds %d bytes", k, maxConditionalPutSize) + } + in := s.putInput(k, bytes.NewReader(data), opts) + in.ContentLength = aws.Int64(int64(len(data))) + if cond.IfMatch != "" { + in.IfMatch = aws.String(cond.IfMatch) + } + if cond.IfNoneMatch { + in.IfNoneMatch = aws.String("*") + } + out, err := s.client.PutObject(ctx, in) + if err != nil { + // If-Match on a missing key yields 404 NoSuchKey on AWS. + if isS3Precondition(err) || (cond.IfMatch != "" && s3StatusCode(err) == http.StatusNotFound) { + return Item{}, fmt.Errorf("s3: conditional put of %s: %w", k, ErrPreconditionFailed) + } + return Item{}, fmt.Errorf("s3: failed to put %s: %w", k, err) + } + return Item{ + Key: k, + Size: int64(len(data)), + ModifiedTime: time.Now().UTC(), + ETag: aws.ToString(out.ETag), + VersionID: aws.ToString(out.VersionId), + }, nil +} + +// Get streams the object body. +func (s *S3Storage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) { + k, err := cleanKey(key) + if err != nil { + return nil, Item{}, fmt.Errorf("s3: %w", err) + } + out, err := s.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(k)}) + if err != nil { + return nil, Item{}, s.mapErr("get", k, err) + } + return out.Body, Item{ + Key: k, + Size: aws.ToInt64(out.ContentLength), + ModifiedTime: aws.ToTime(out.LastModified), + ETag: aws.ToString(out.ETag), + VersionID: aws.ToString(out.VersionId), + }, nil +} + +// Stat returns object metadata (HeadObject). +func (s *S3Storage) Stat(ctx context.Context, key string) (Item, error) { + k, err := cleanKey(key) + if err != nil { + return Item{}, fmt.Errorf("s3: %w", err) + } + out, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(k)}) + if err != nil { + return Item{}, s.mapErr("stat", k, err) + } + return Item{ + Key: k, + Size: aws.ToInt64(out.ContentLength), + ModifiedTime: aws.ToTime(out.LastModified), + ETag: aws.ToString(out.ETag), + VersionID: aws.ToString(out.VersionId), + }, nil +} + +// Upload uploads a file to S3. +func (s *S3Storage) Upload(ctx context.Context, localPath string, remotePath string) error { + if err := uploadFile(ctx, s, localPath, remotePath); err != nil { + return fmt.Errorf("s3: %w", err) + } + return nil +} + +// Download downloads a file from S3. +func (s *S3Storage) Download(ctx context.Context, remotePath string, localPath string) error { + if err := downloadFile(ctx, s, remotePath, localPath); err != nil { + return fmt.Errorf("s3: %w", err) + } + return nil +} + +// List lists objects (recursively) with the given prefix. +func (s *S3Storage) List(ctx context.Context, prefix string) ([]Item, error) { + p, err := cleanPrefix(prefix) + if err != nil { + return nil, fmt.Errorf("s3: %w", err) + } + return s.list(ctx, p, "") +} + +// ListDir lists the objects directly under dir. +func (s *S3Storage) ListDir(ctx context.Context, dir string) ([]Item, error) { + d, err := cleanDir(dir) + if err != nil { + return nil, fmt.Errorf("s3: %w", err) + } + return s.list(ctx, dirPrefix(d), "/") +} + +func (s *S3Storage) list(ctx context.Context, prefix, delimiter string) ([]Item, error) { + in := &s3.ListObjectsV2Input{Bucket: aws.String(s.bucket), Prefix: aws.String(prefix)} + if delimiter != "" { + in.Delimiter = aws.String(delimiter) + } + items := make([]Item, 0) + p := s3.NewListObjectsV2Paginator(s.client, in) + for p.HasMorePages() { + page, err := p.NextPage(ctx) + if err != nil { + return nil, fmt.Errorf("s3: failed to list objects in bucket %s: %w", s.bucket, err) + } + for _, obj := range page.Contents { + key := aws.ToString(obj.Key) + if delimiter != "" && strings.HasSuffix(key, "/") { + continue // directory marker + } + items = append(items, Item{ + Key: key, + ModifiedTime: aws.ToTime(obj.LastModified), + Size: aws.ToInt64(obj.Size), + ETag: aws.ToString(obj.ETag), + }) + } + } + return items, nil +} + +// Remove deletes an object. A missing key returns ErrNotFound (checked with +// HeadObject because DeleteObject is idempotent on S3). +func (s *S3Storage) Remove(ctx context.Context, remotePath string) error { + k, err := cleanKey(remotePath) + if err != nil { + return fmt.Errorf("s3: %w", err) + } + if _, err := s.Stat(ctx, k); err != nil { + return err + } + if _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(k)}); err != nil { + return s.mapErr("remove", k, err) + } + return nil +} + +// Close releases resources (no-op). +func (s *S3Storage) Close() error { + return nil +} + +func (s *S3Storage) mapErr(op, key string, err error) error { + if s3StatusCode(err) == http.StatusNotFound { + return fmt.Errorf("s3: %s %s in bucket %s: %w", op, key, s.bucket, ErrNotFound) + } + var nsk *types.NoSuchKey + var nf *types.NotFound + if errors.As(err, &nsk) || errors.As(err, &nf) { + return fmt.Errorf("s3: %s %s in bucket %s: %w", op, key, s.bucket, ErrNotFound) + } + return fmt.Errorf("s3: %s %s in bucket %s: %w", op, key, s.bucket, err) +} + +func s3StatusCode(err error) int { + var re *awshttp.ResponseError + if errors.As(err, &re) { + return re.HTTPStatusCode() + } + return 0 +} + +func isS3Precondition(err error) bool { + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch apiErr.ErrorCode() { + case "PreconditionFailed", "ConditionalRequestConflict": + return true + } + } + switch s3StatusCode(err) { + case http.StatusPreconditionFailed, http.StatusConflict: + return true + } + return false +} diff --git a/sftp.go b/sftp.go new file mode 100644 index 0000000..9daa844 --- /dev/null +++ b/sftp.go @@ -0,0 +1,559 @@ +/* +MIT License + +Copyright (c) 2024 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package go_storage + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "net" + "os" + "os/user" + "path" + "path/filepath" + "strings" + "time" + + "github.com/pkg/sftp" + + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/agent" + "golang.org/x/crypto/ssh/knownhosts" +) + +// SFTPStorage implements Storage interface for SFTP. +type SFTPStorage struct { + host string + baseDir string + sshClient *ssh.Client + client *sftp.Client +} + +// NewSFTPStorage creates a new SFTP storage instance. +// +// Host keys are verified against SFTPKnownHostsFile (or ~/.ssh/known_hosts +// and /etc/ssh/ssh_known_hosts) unless SFTPIgnoreKnownHosts is true. +func NewSFTPStorage(config Config) (*SFTPStorage, error) { + if err := validateConfig(config, map[string]string{ + "host": config.SFTPHost, + }); err != nil { + return nil, fmt.Errorf("sftp: %w", err) + } + + port := config.SFTPPort + if port == "" { + port = "22" + } + hostPort := net.JoinHostPort(config.SFTPHost, port) + + username := config.SFTPUsername + if username == "" { + currentUser, err := user.Current() + if err != nil { + return nil, fmt.Errorf("sftp: failed to get current user: %w", err) + } + username = currentUser.Username + } + + authMethods := make([]ssh.AuthMethod, 0, 2) + if config.SFTPPassword != "" { + authMethods = append(authMethods, ssh.Password(config.SFTPPassword)) + } + passphrase := config.SFTPIdentityPassphrase + if passphrase == "" { + passphrase = config.SFTPPassword // backwards compatibility + } + signers, err := getSSHSigners(config.SFTPIdentityFile, passphrase) + if err != nil { + return nil, fmt.Errorf("sftp: %w", err) + } + if len(signers) > 0 { + authMethods = append(authMethods, ssh.PublicKeys(signers...)) + } + + hostKeyCallback, hostKeyAlgos, err := getHostKeyCallback(config, hostPort) + if err != nil { + return nil, fmt.Errorf("sftp: %w", err) + } + + sshConfig := &ssh.ClientConfig{ + User: username, + Auth: authMethods, + HostKeyCallback: hostKeyCallback, + HostKeyAlgorithms: hostKeyAlgos, + Timeout: 30 * time.Second, + } + + sshClient, err := ssh.Dial("tcp", hostPort, sshConfig) + if err != nil { + return nil, fmt.Errorf("sftp: failed to connect to %s: %w", hostPort, err) + } + + sftpClient, err := sftp.NewClient(sshClient) + if err != nil { + _ = sshClient.Close() + return nil, fmt.Errorf("sftp: failed to open sftp session: %w", err) + } + + baseDir := normalizePathSeparators(config.SFTPDirectory) + if baseDir == "" { + wd, err := sftpClient.Getwd() + if err != nil { + _ = sftpClient.Close() + _ = sshClient.Close() + return nil, fmt.Errorf("sftp: failed to get working directory: %w", err) + } + baseDir = wd + } + + return &SFTPStorage{ + host: config.SFTPHost, + baseDir: path.Clean(baseDir), + sshClient: sshClient, + client: sftpClient, + }, nil +} + +// Client returns the underlying sftp client. +func (s *SFTPStorage) Client() *sftp.Client { return s.client } + +func (s *SFTPStorage) resolve(key string) (string, string, error) { + k, err := cleanKey(key) + if err != nil { + return "", "", fmt.Errorf("sftp: %w", err) + } + return k, path.Join(s.baseDir, k), nil +} + +func (s *SFTPStorage) item(key string, fi os.FileInfo) Item { + it := Item{Key: key, ModifiedTime: fi.ModTime(), IsDirectory: fi.IsDir()} + if !fi.IsDir() { + it.Size = fi.Size() + } + return it +} + +func (s *SFTPStorage) mkdirParent(full string) error { + dir := path.Dir(full) + if dir == "." || dir == "/" { + return nil + } + if err := s.client.MkdirAll(dir); err != nil { + return fmt.Errorf("sftp: failed to create directory %s: %w", dir, err) + } + return nil +} + +// Put streams r to ..partial and renames it to key on success. +func (s *SFTPStorage) Put(ctx context.Context, key string, r io.Reader, _ *PutOptions) (Item, error) { + k, full, err := s.resolve(key) + if err != nil { + return Item{}, err + } + if err := s.mkdirParent(full); err != nil { + return Item{}, err + } + tmp := partialName(full) + f, err := s.client.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC) + if err != nil { + return Item{}, fmt.Errorf("sftp: failed to create %s: %w", tmp, err) + } + if err := copyAndClose(ctx, f, r); err != nil { + _ = s.client.Remove(tmp) + return Item{}, fmt.Errorf("sftp: failed to upload %s: %w", k, err) + } + if err := s.rename(tmp, full); err != nil { + _ = s.client.Remove(tmp) + return Item{}, err + } + return s.Stat(ctx, k) +} + +func copyAndClose(ctx context.Context, w io.WriteCloser, r io.Reader) error { + _, err := io.Copy(w, ctxReader{ctx: ctx, r: r}) + if cerr := w.Close(); err == nil { + err = cerr + } + if err == nil { + err = ctx.Err() + } + return err +} + +func (s *SFTPStorage) rename(from, to string) error { + if _, ok := s.client.HasExtension("posix-rename@openssh.com"); ok { + if err := s.client.PosixRename(from, to); err != nil { + return fmt.Errorf("sftp: failed to rename %s to %s: %w", from, to, err) + } + return nil + } + if err := s.client.Remove(to); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("sftp: failed to replace %s: %w", to, err) + } + if err := s.client.Rename(from, to); err != nil { + return fmt.Errorf("sftp: failed to rename %s to %s: %w", from, to, err) + } + return nil +} + +// PutIf supports only Condition.IfNoneMatch (via O_EXCL on the final path). +// IfMatch returns ErrConditionNotSupported. +func (s *SFTPStorage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, _ *PutOptions) (Item, error) { + if cond.IfMatch != "" || !cond.IfNoneMatch { + return Item{}, fmt.Errorf("sftp: IfMatch: %w", ErrConditionNotSupported) + } + k, full, err := s.resolve(key) + if err != nil { + return Item{}, err + } + if err := s.mkdirParent(full); err != nil { + return Item{}, err + } + f, err := s.client.OpenFile(full, os.O_WRONLY|os.O_CREATE|os.O_EXCL) + if err != nil { + if _, serr := s.client.Lstat(full); serr == nil { + return Item{}, fmt.Errorf("sftp: %s already exists: %w", k, ErrPreconditionFailed) + } + return Item{}, fmt.Errorf("sftp: failed to create %s: %w", k, err) + } + if err := copyAndClose(ctx, f, r); err != nil { + _ = s.client.Remove(full) + return Item{}, fmt.Errorf("sftp: failed to upload %s: %w", k, err) + } + return s.Stat(ctx, k) +} + +// Get opens key for streaming reads. +func (s *SFTPStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) { + if err := ctx.Err(); err != nil { + return nil, Item{}, err + } + k, full, err := s.resolve(key) + if err != nil { + return nil, Item{}, err + } + f, err := s.client.Open(full) + if err != nil { + return nil, Item{}, mapSFTPErr("get", k, err) + } + fi, err := f.Stat() + if err != nil { + _ = f.Close() + return nil, Item{}, mapSFTPErr("get", k, err) + } + if fi.IsDir() { + _ = f.Close() + return nil, Item{}, fmt.Errorf("sftp: %s is a directory", k) + } + return f, s.item(k, fi), nil +} + +// Stat returns metadata for key. +func (s *SFTPStorage) Stat(ctx context.Context, key string) (Item, error) { + if err := ctx.Err(); err != nil { + return Item{}, err + } + k, full, err := s.resolve(key) + if err != nil { + return Item{}, err + } + fi, err := s.client.Stat(full) + if err != nil { + return Item{}, mapSFTPErr("stat", k, err) + } + return s.item(k, fi), nil +} + +func mapSFTPErr(op, key string, err error) error { + if errors.Is(err, fs.ErrNotExist) || errors.Is(err, sftp.ErrSSHFxNoSuchFile) { + return fmt.Errorf("sftp: %s %s: %w", op, key, ErrNotFound) + } + var se *sftp.StatusError + if errors.As(err, &se) && se.FxCode() == sftp.ErrSSHFxNoSuchFile { + return fmt.Errorf("sftp: %s %s: %w", op, key, ErrNotFound) + } + return fmt.Errorf("sftp: %s %s: %w", op, key, err) +} + +// Upload uploads a file via SFTP. +func (s *SFTPStorage) Upload(ctx context.Context, localPath string, remotePath string) error { + return uploadFile(ctx, s, localPath, remotePath) +} + +// Download downloads a file via SFTP. +func (s *SFTPStorage) Download(ctx context.Context, remotePath string, localPath string) error { + return downloadFile(ctx, s, remotePath, localPath) +} + +// List lists files and directories (recursively) whose key starts with prefix. +func (s *SFTPStorage) List(ctx context.Context, prefix string) ([]Item, error) { + p, err := cleanPrefix(prefix) + if err != nil { + return nil, fmt.Errorf("sftp: %w", err) + } + items := make([]Item, 0) + root := s.baseDir + if i := strings.LastIndex(p, "/"); i >= 0 { + root = path.Join(s.baseDir, p[:i]) + } + walker := s.client.Walk(root) + for walker.Step() { + if err := ctx.Err(); err != nil { + return nil, err + } + if err := walker.Err(); err != nil { + if walker.Path() == root && errors.Is(err, fs.ErrNotExist) { + return items, nil + } + continue + } + full := walker.Path() + if full == s.baseDir { + continue + } + rel := strings.TrimPrefix(strings.TrimPrefix(full, s.baseDir), "/") + stat := walker.Stat() + if !strings.HasPrefix(rel, p) { + if stat.IsDir() && !strings.HasPrefix(p, rel+"/") { + walker.SkipDir() + } + continue + } + items = append(items, s.item(rel, stat)) + } + return items, nil +} + +// ListDir lists the files directly inside dir. +func (s *SFTPStorage) ListDir(ctx context.Context, dir string) ([]Item, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + d, err := cleanDir(dir) + if err != nil { + return nil, fmt.Errorf("sftp: %w", err) + } + entries, err := s.client.ReadDir(path.Join(s.baseDir, d)) + if errors.Is(err, fs.ErrNotExist) { + return []Item{}, nil + } + if err != nil { + return nil, fmt.Errorf("sftp: failed to read directory %s: %w", dir, err) + } + items := make([]Item, 0, len(entries)) + for _, fi := range entries { + if !fi.Mode().IsRegular() { + continue + } + items = append(items, s.item(path.Join(d, fi.Name()), fi)) + } + return items, nil +} + +// Remove deletes a file via SFTP. +func (s *SFTPStorage) Remove(ctx context.Context, remotePath string) error { + if err := ctx.Err(); err != nil { + return err + } + k, full, err := s.resolve(remotePath) + if err != nil { + return err + } + if err := s.client.Remove(full); err != nil { + return mapSFTPErr("remove", k, err) + } + return nil +} + +// Close releases resources. +func (s *SFTPStorage) Close() error { + err := s.client.Close() + if cerr := s.sshClient.Close(); err == nil { + err = cerr + } + return err +} + +// getSSHSigners returns SSH signers from identity file and SSH agent. +func getSSHSigners(identityFile string, passphrase string) ([]ssh.Signer, error) { + signers := make([]ssh.Signer, 0) + + if identityFile != "" { + p, err := expandHomeDir(identityFile) + if err != nil { + return nil, fmt.Errorf("failed to expand identity file path: %w", err) + } + + keyData, err := os.ReadFile(p) + if err != nil { + return nil, fmt.Errorf("failed to read identity file %s: %w", p, err) + } + + signer, err := ssh.ParsePrivateKey(keyData) + if err != nil { + var passErr *ssh.PassphraseMissingError + if !errors.As(err, &passErr) { + return nil, fmt.Errorf("failed to parse identity file: %w", err) + } + signer, err = ssh.ParsePrivateKeyWithPassphrase(keyData, []byte(passphrase)) + if err != nil { + return nil, fmt.Errorf("failed to decrypt identity file: %w", err) + } + } + signers = append(signers, signer) + } + + // Keys from a running SSH agent, if any. + if socket := os.Getenv("SSH_AUTH_SOCK"); socket != "" { + if conn, err := net.Dial("unix", socket); err == nil { + if agentSigners, err := agent.NewClient(conn).Signers(); err == nil { + signers = append(signers, agentSigners...) + } + } + } + + return signers, nil +} + +// getHostKeyCallback builds the host key verification callback and the list +// of host key algorithms known for hostPort (so the server negotiates a key +// type present in known_hosts). +func getHostKeyCallback(config Config, hostPort string) (ssh.HostKeyCallback, []string, error) { + if config.SFTPIgnoreKnownHosts { + return ssh.InsecureIgnoreHostKey(), nil, nil //nolint:gosec // explicitly requested + } + + files := make([]string, 0, 2) + if config.SFTPKnownHostsFile != "" { + p, err := expandHomeDir(config.SFTPKnownHostsFile) + if err != nil { + return nil, nil, err + } + if _, err := os.Stat(p); err != nil { + return nil, nil, fmt.Errorf("known_hosts file %s: %w", p, err) + } + files = append(files, p) + } else { + for _, candidate := range []string{"~/.ssh/known_hosts", "/etc/ssh/ssh_known_hosts"} { + p, err := expandHomeDir(candidate) + if err != nil { + continue + } + if _, err := os.Stat(p); err == nil { + files = append(files, p) + } + } + } + if len(files) == 0 { + return nil, nil, fmt.Errorf("no known_hosts file available for host key verification " + + "(set SSH_KNOWN_HOSTS or SSH_INSECURE_IGNORE_HOST_KEY=true)") + } + + callback, err := knownhosts.New(files...) + if err != nil { + return nil, nil, fmt.Errorf("failed to load known_hosts: %w", err) + } + return callback, knownHostAlgorithms(callback, hostPort), nil +} + +// knownHostAlgorithms returns the host key algorithms recorded for hostPort, +// using the KeyError returned for a deliberately unknown key. +func knownHostAlgorithms(cb ssh.HostKeyCallback, hostPort string) []string { + addr, err := net.ResolveTCPAddr("tcp", hostPort) + if err != nil { + addr = &net.TCPAddr{} + } + var keyErr *knownhosts.KeyError + if err := cb(hostPort, addr, probeKey{}); !errors.As(err, &keyErr) { + return nil + } + algos := make([]string, 0, len(keyErr.Want)) + seen := map[string]bool{} + for _, k := range keyErr.Want { + for _, a := range algorithmsForKeyType(k.Key.Type()) { + if !seen[a] { + seen[a] = true + algos = append(algos, a) + } + } + } + if len(algos) == 0 { + return nil + } + return algos +} + +func algorithmsForKeyType(t string) []string { + if t == ssh.KeyAlgoRSA { + return []string{ssh.KeyAlgoRSASHA512, ssh.KeyAlgoRSASHA256, ssh.KeyAlgoRSA} + } + return []string{t} +} + +// probeKey is a public key that never matches anything in known_hosts. +type probeKey struct{} + +func (probeKey) Type() string { return "probe-key" } +func (probeKey) Marshal() []byte { return []byte("probe-key") } +func (probeKey) Verify([]byte, *ssh.Signature) error { return errors.New("probe key") } + +// expandHomeDir expands ~ in file paths to home directory. +func expandHomeDir(p string) (string, error) { + if !strings.HasPrefix(p, "~") { + return filepath.Clean(p), nil + } + + parts := strings.SplitN(p, "/", 2) + username := strings.TrimPrefix(parts[0], "~") + + var homeDir string + if username == "" { + h, err := os.UserHomeDir() + if err != nil || h == "" { + currentUser, err := user.Current() + if err != nil { + return "", fmt.Errorf("failed to get home directory: %w", err) + } + h = currentUser.HomeDir + } + homeDir = h + } else { + userInfo, err := user.Lookup(username) + if err != nil { + return "", fmt.Errorf("failed to lookup user %s: %w", username, err) + } + homeDir = userInfo.HomeDir + } + + if homeDir == "" { + return "", fmt.Errorf("empty home directory") + } + if len(parts) == 1 { + return homeDir, nil + } + return filepath.Join(homeDir, parts[1]), nil +} diff --git a/sftp_test.go b/sftp_test.go new file mode 100644 index 0000000..cddfb38 --- /dev/null +++ b/sftp_test.go @@ -0,0 +1,208 @@ +/* +MIT License + +Copyright (c) 2024 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package go_storage + +import ( + "crypto/ed25519" + "crypto/rand" + "errors" + "io" + "net" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" +) + +const ( + testSFTPUser = "backup" + testSFTPPass = "s3cret" +) + +// startSFTPServer runs an in-process SSH/SFTP server on 127.0.0.1:0 and +// returns its address and a known_hosts file containing its host key. +func startSFTPServer(t *testing.T) (host, port, knownHostsFile string) { + t.Helper() + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + signer, err := ssh.NewSignerFromKey(priv) + if err != nil { + t.Fatal(err) + } + cfg := &ssh.ServerConfig{ + PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) { + if c.User() == testSFTPUser && string(pass) == testSFTPPass { + return nil, nil + } + return nil, errors.New("access denied") + }, + } + cfg.AddHostKey(signer) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go serveSSHConn(conn, cfg) + } + }() + + addr := ln.Addr().String() + knownHostsFile = filepath.Join(t.TempDir(), "known_hosts") + line := knownhosts.Line([]string{knownhosts.Normalize(addr)}, signer.PublicKey()) + if err := os.WriteFile(knownHostsFile, []byte(line+"\n"), 0o600); err != nil { + t.Fatal(err) + } + host, port, _ = net.SplitHostPort(addr) + return host, port, knownHostsFile +} + +func serveSSHConn(conn net.Conn, cfg *ssh.ServerConfig) { + sconn, chans, reqs, err := ssh.NewServerConn(conn, cfg) + if err != nil { + _ = conn.Close() + return + } + defer func() { _ = sconn.Close() }() + go ssh.DiscardRequests(reqs) + for nc := range chans { + if nc.ChannelType() != "session" { + _ = nc.Reject(ssh.UnknownChannelType, "unsupported") + continue + } + ch, chReqs, err := nc.Accept() + if err != nil { + continue + } + go func() { + for req := range chReqs { + ok := req.Type == "subsystem" && len(req.Payload) > 4 && string(req.Payload[4:]) == "sftp" + _ = req.Reply(ok, nil) + if !ok { + continue + } + srv, err := sftp.NewServer(ch) + if err != nil { + _ = ch.Close() + return + } + if err := srv.Serve(); err != nil && !errors.Is(err, io.EOF) { + _ = srv.Close() + } + _ = ch.Close() + return + } + }() + } +} + +func newTestSFTP(t *testing.T) (*SFTPStorage, string) { + t.Helper() + t.Setenv("SSH_AUTH_SOCK", "") + host, port, kh := startSFTPServer(t) + base := t.TempDir() + s, err := NewSFTPStorage(Config{ + SFTPHost: host, + SFTPPort: port, + SFTPUsername: testSFTPUser, + SFTPPassword: testSFTPPass, + SFTPDirectory: base, + SFTPKnownHostsFile: kh, + }) + if err != nil { + t.Fatalf("NewSFTPStorage: %v", err) + } + t.Cleanup(func() { _ = s.Close() }) + return s, base +} + +func TestSFTPConformance(t *testing.T) { + s, base := newTestSFTP(t) + runConformance(t, s) + + // No temporary files may survive anywhere under the base directory. + _ = filepath.WalkDir(base, func(p string, d os.DirEntry, err error) error { + if err == nil && strings.HasSuffix(p, ".partial") { + t.Errorf("leftover partial file %s", p) + } + return nil + }) +} + +func TestSFTPHostKeyVerification(t *testing.T) { + t.Setenv("SSH_AUTH_SOCK", "") + host, port, _ := startSFTPServer(t) + base := Config{SFTPHost: host, SFTPPort: port, SFTPUsername: testSFTPUser, SFTPPassword: testSFTPPass, SFTPDirectory: t.TempDir()} + + // A known_hosts file with a different key for this host must be rejected. + _, otherPriv, _ := ed25519.GenerateKey(rand.Reader) + otherSigner, _ := ssh.NewSignerFromKey(otherPriv) + wrong := filepath.Join(t.TempDir(), "known_hosts") + line := knownhosts.Line([]string{knownhosts.Normalize(net.JoinHostPort(host, port))}, otherSigner.PublicKey()) + if err := os.WriteFile(wrong, []byte(line+"\n"), 0o600); err != nil { + t.Fatal(err) + } + cfg := base + cfg.SFTPKnownHostsFile = wrong + if s, err := NewSFTPStorage(cfg); err == nil { + _ = s.Close() + t.Fatal("connected despite host key mismatch") + } + + // Missing known_hosts file is an error with a helpful message. + cfg = base + cfg.SFTPKnownHostsFile = filepath.Join(t.TempDir(), "nope") + if _, err := NewSFTPStorage(cfg); err == nil { + t.Fatal("connected without known_hosts") + } + t.Setenv("HOME", t.TempDir()) + cfg.SFTPKnownHostsFile = "" + if _, err := os.Stat("/etc/ssh/ssh_known_hosts"); os.IsNotExist(err) { + if _, err := NewSFTPStorage(cfg); err == nil || !strings.Contains(err.Error(), "SSH_INSECURE_IGNORE_HOST_KEY") { + t.Fatalf("expected helpful known_hosts error, got %v", err) + } + } + + // Explicitly ignoring host keys works. + cfg.SFTPIgnoreKnownHosts = true + s, err := NewSFTPStorage(cfg) + if err != nil { + t.Fatalf("SFTPIgnoreKnownHosts: %v", err) + } + _ = s.Close() +} diff --git a/storage.go b/storage.go new file mode 100644 index 0000000..a8143fb --- /dev/null +++ b/storage.go @@ -0,0 +1,403 @@ +/* +MIT License + +Copyright (c) 2024 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Package go_storage provides a uniform, streaming storage abstraction over +// S3 (and S3-compatible services), Azure Blob Storage, Google Cloud Storage, +// SFTP, FTP and the local filesystem. +// +// All backends implement [Storage]. Objects are addressed by keys that are +// relative to the backend base (bucket, container, remote directory or local +// base path) and always use forward slashes. +package go_storage + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" + "time" +) + +// Sentinel errors. Backends wrap them with %w, so test with errors.Is. +var ( + // ErrNotFound is returned (wrapped) by Stat, Get and Remove when the key does not exist. + ErrNotFound = errors.New("storage: object not found") + // ErrPreconditionFailed is returned (wrapped) by PutIf when the condition does not hold. + ErrPreconditionFailed = errors.New("storage: precondition failed") + // ErrConditionNotSupported is returned (wrapped) by PutIf when the backend cannot + // evaluate the requested condition atomically. Callers should fall back to locking. + ErrConditionNotSupported = errors.New("storage: conditional write not supported by backend") +) + +// Storage defines the interface for remote file storage operations. +// All implementations are safe for concurrent use (see FTPStorage for the +// one restriction regarding open Get bodies). +type Storage interface { + // Upload uploads a local file to the remote storage with the given target name. + Upload(ctx context.Context, localPath string, remotePath string) error + + // Download downloads a file from remote storage to a local path. + Download(ctx context.Context, remotePath string, localPath string) error + + // List returns all items (recursively) in the storage that match the given prefix. + List(ctx context.Context, prefix string) ([]Item, error) + + // Remove deletes a file from remote storage. A missing key returns an + // error wrapping ErrNotFound. + Remove(ctx context.Context, remotePath string) error + + // Close releases any resources held by the storage implementation. + Close() error + + // Put streams r to key. r may be of unknown length; memory use is bounded. + // If reading r fails or ctx is cancelled, Put returns an error and no + // (partial) object is left at key. + Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error) + + // Get opens key for streaming reads. The caller must Close the body. + // A missing key returns an error wrapping ErrNotFound. + Get(ctx context.Context, key string) (io.ReadCloser, Item, error) + + // Stat returns metadata for key. A missing key returns an error wrapping ErrNotFound. + Stat(ctx context.Context, key string) (Item, error) + + // PutIf writes r to key only if cond holds, atomically with respect to + // other PutIf calls. It returns an error wrapping ErrPreconditionFailed when + // the condition does not hold, or ErrConditionNotSupported when the backend + // cannot evaluate it. Intended for small objects (index files). + PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error) + + // ListDir lists only the files directly inside dir (dir is treated as a + // directory: "a/b" never matches "a/bc/x"; "" = root). Non-recursive, no + // directories are returned. A missing directory yields an empty list. + ListDir(ctx context.Context, dir string) ([]Item, error) +} + +// Item represents a file or directory in remote storage. +type Item struct { + Key string // The path/key of the item, relative to the backend base + ModifiedTime time.Time // Last modification time + Size int64 // Size in bytes (0 for directories) + IsDirectory bool // Whether this item is a directory + // ETag is an opaque version token usable with Condition.IfMatch + // (S3/Azure ETag; GCS generation as decimal string; local: + // "-"; SFTP/FTP: ""). + ETag string + // VersionID is the S3 version id when bucket versioning is enabled. + VersionID string +} + +// PutOptions are optional parameters for Put and PutIf. +type PutOptions struct { + ContentType string + Metadata map[string]string + Size int64 // optional size hint; -1 or 0 = unknown +} + +// Condition is a precondition for PutIf. Exactly one field must be set; a +// zero Condition is rejected (it would otherwise silently become an +// unconditional write, e.g. when an empty ETag is passed as IfMatch). +type Condition struct { + IfMatch string // only write if the current ETag equals this + IfNoneMatch bool // only write if the object does not exist +} + +// Config holds configuration options for creating storage instances. +type Config struct { + // Local configuration + LocalPath string // base directory for the local backend (default "/backup") + + // S3 configuration + S3Region string + S3Bucket string + S3Profile string + S3KeyID string + S3Secret string + S3SessionToken string + S3Endpoint string + S3ForcePath bool + S3DisableTLS bool + S3StorageClass string // e.g. STANDARD, STANDARD_IA, GLACIER_IR + S3SSE string // "AES256" or "aws:kms" + S3SSEKMSKeyID string // KMS key id when S3SSE is "aws:kms" + S3PartSize int64 // multipart part size in bytes (default 64 MiB, min 5 MiB) + S3Concurrency int // multipart upload concurrency (default 2) + + // B2 configuration (reserved, no backend implementation yet) + B2KeyID string + B2AppKey string + B2Bucket string + B2ConcurrentConnections int + B2ForcePath bool + + // Google Cloud Storage configuration + GCSBucket string + GCSEndpoint string + GCSCredentialsFile string + GCSCredentialsJSON string + + // Azure configuration + AzureAccount string + AzureKey string + AzureContainer string + // AzureEndpoint is either a domain suffix (default "blob.core.windows.net", + // the service URL becomes https://.) or a full service + // URL containing a scheme (e.g. "http://127.0.0.1:10000/devstoreaccount1" + // for Azurite), used as is. + AzureEndpoint string + // AzureSASToken, when set, is used instead of the shared key. + AzureSASToken string + + // SFTP configuration + SFTPHost string + SFTPPort string + SFTPUsername string + SFTPPassword string + SFTPDirectory string + SFTPIdentityFile string + SFTPIdentityPassphrase string + SFTPKnownHostsFile string + SFTPIgnoreKnownHosts bool + + // FTP configuration + FTPHost string + FTPPort string + FTPUsername string + FTPPassword string + FTPDirectory string + FTPTLS bool // explicit FTPS (AUTH TLS) +} + +// StorageType represents the type of storage backend. +type StorageType string + +// Supported storage types. +const ( + StorageTypeS3 StorageType = "s3" + StorageTypeGCS StorageType = "gcs" + StorageTypeAzure StorageType = "azure" + StorageTypeSFTP StorageType = "sftp" + StorageTypeFTP StorageType = "ftp" + StorageTypeLocal StorageType = "local" +) + +// DefaultLocalPath is the base directory of the local backend when Config.LocalPath is empty. +const DefaultLocalPath = "/backup" + +// NewStorage creates a new Storage instance based on the specified type and configuration. +func NewStorage(storageType StorageType, config Config) (Storage, error) { + switch storageType { + case StorageTypeS3: + return NewS3Storage(config) + case StorageTypeGCS: + return NewGCSStorage(config) + case StorageTypeAzure: + return NewAzureStorage(config) + case StorageTypeSFTP: + return NewSFTPStorage(config) + case StorageTypeFTP: + return NewFTPStorage(config) + case StorageTypeLocal: + p := config.LocalPath + if p == "" { + p = DefaultLocalPath + } + return NewLocalStorage(p) + default: + return nil, fmt.Errorf("unsupported storage type: %s", storageType) + } +} + +// normalizePathSeparators converts OS-specific path separators to forward slashes. +func normalizePathSeparators(p string) string { + if os.PathSeparator != '/' { + p = strings.ReplaceAll(p, string(os.PathSeparator), "/") + } + return strings.ReplaceAll(p, "\\", "/") +} + +// cleanKey normalises an object key: separators become "/", the path is +// cleaned and leading slashes are removed. Keys escaping the base ("..") +// and empty keys are rejected. +func cleanKey(key string) (string, error) { + k, err := cleanDir(key) + if err != nil { + return "", err + } + if k == "" { + return "", fmt.Errorf("storage: empty key") + } + return k, nil +} + +// cleanDir is like cleanKey but allows the empty string (root). +func cleanDir(dir string) (string, error) { + d := path.Clean("/" + normalizePathSeparators(dir)) + d = strings.TrimPrefix(d, "/") + // path.Clean of an absolute path never yields "..", but check the raw + // input so that escaping attempts are reported rather than silently clamped. + for _, seg := range strings.Split(path.Clean(normalizePathSeparators(dir)), "/") { + if seg == ".." { + return "", fmt.Errorf("storage: key %q escapes the base directory", dir) + } + } + return d, nil +} + +// cleanPrefix normalises a List prefix. Unlike keys, a prefix is not +// path-cleaned (a trailing "/" is significant) but must not contain "..". +func cleanPrefix(prefix string) (string, error) { + p := normalizePathSeparators(prefix) + for _, seg := range strings.Split(p, "/") { + if seg == ".." { + return "", fmt.Errorf("storage: prefix %q escapes the base directory", prefix) + } + } + return strings.TrimLeft(p, "/"), nil +} + +// dirPrefix returns the object prefix for a directory ("" for root, "a/b/" otherwise). +func dirPrefix(dir string) string { + if dir == "" { + return "" + } + return dir + "/" +} + +// partialName returns a unique temporary name next to p. +func partialName(p string) string { + var b [6]byte + _, _ = rand.Read(b[:]) + return p + "." + hex.EncodeToString(b[:]) + ".partial" +} + +// ctxReader aborts reads once ctx is done. +type ctxReader struct { + ctx context.Context + r io.Reader +} + +func (c ctxReader) Read(p []byte) (int, error) { + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.r.Read(p) +} + +// countingReader counts bytes read. +type countingReader struct { + r io.Reader + n int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += int64(n) + return n, err +} + +// errEmptyCondition is returned by PutIf when neither IfMatch nor IfNoneMatch is set. +var errEmptyCondition = errors.New("storage: PutIf requires Condition.IfMatch or Condition.IfNoneMatch") + +func validateCondition(c Condition) error { + if c.IfMatch == "" && !c.IfNoneMatch { + return errEmptyCondition + } + if c.IfMatch != "" && c.IfNoneMatch { + return errors.New("storage: Condition.IfMatch and Condition.IfNoneMatch are mutually exclusive") + } + return nil +} + +// maxConditionalPutSize bounds the buffer used by backends whose conditional +// write needs a single request (S3). +const maxConditionalPutSize = 16 << 20 + +// uploadFile implements Storage.Upload on top of Put. +func uploadFile(ctx context.Context, s Storage, localPath, remotePath string) error { + f, err := os.Open(localPath) + if err != nil { + return fmt.Errorf("failed to open local file: %w", err) + } + defer func() { _ = f.Close() }() + opts := &PutOptions{Size: -1} + if fi, err := f.Stat(); err == nil { + opts.Size = fi.Size() + } + _, err = s.Put(ctx, remotePath, f, opts) + return err +} + +// downloadFile implements Storage.Download on top of Get. +func downloadFile(ctx context.Context, s Storage, remotePath, localPath string) (err error) { + body, _, err := s.Get(ctx, remotePath) + if err != nil { + return err + } + defer func() { _ = body.Close() }() + + if dir := filepath.Dir(localPath); dir != "" { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("failed to create local directory: %w", err) + } + } + f, err := os.Create(localPath) + if err != nil { + return fmt.Errorf("failed to create local file: %w", err) + } + defer func() { + if cerr := f.Close(); cerr != nil && err == nil { + err = fmt.Errorf("failed to close local file: %w", cerr) + } + if err != nil { + _ = os.Remove(localPath) + } + }() + if _, err = io.Copy(f, ctxReader{ctx: ctx, r: body}); err != nil { + return fmt.Errorf("failed to download %s: %w", remotePath, err) + } + return nil +} + +// validateConfig checks if the required configuration fields are present. +func validateConfig(_ Config, requiredFields map[string]string) error { + var missing []string + for field, value := range requiredFields { + if value == "" { + missing = append(missing, field) + } + } + + if len(missing) > 0 { + return fmt.Errorf("missing required configuration: %s", strings.Join(missing, ", ")) + } + + return nil +} diff --git a/storage_test.go b/storage_test.go new file mode 100644 index 0000000..ba8e738 --- /dev/null +++ b/storage_test.go @@ -0,0 +1,212 @@ +/* +MIT License + +Copyright (c) 2024 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package go_storage + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func TestStorageInterface(t *testing.T) { + var _ Storage = (*S3Storage)(nil) + var _ Storage = (*GCSStorage)(nil) + var _ Storage = (*AzureStorage)(nil) + var _ Storage = (*SFTPStorage)(nil) + var _ Storage = (*FTPStorage)(nil) + var _ Storage = (*LocalStorage)(nil) +} + +func TestCleanKey(t *testing.T) { + ok := map[string]string{ + "x/y.txt": "x/y.txt", + "/a/b.txt": "a/b.txt", + "a//b/./c": "a/b/c", + `a\b\c.txt`: "a/b/c.txt", + "a/../b.txt": "b.txt", + "./x": "x", + "backup.sql~": "backup.sql~", + } + for in, want := range ok { + got, err := cleanKey(in) + if err != nil || got != want { + t.Errorf("cleanKey(%q) = %q, %v; want %q", in, got, err, want) + } + } + for _, in := range []string{"", "/", ".", "..", "../x", "a/../../x", `..\x`} { + if got, err := cleanKey(in); err == nil { + t.Errorf("cleanKey(%q) = %q, want error", in, got) + } + } + if d, err := cleanDir(""); err != nil || d != "" { + t.Errorf("cleanDir(\"\") = %q, %v", d, err) + } + if _, err := cleanPrefix("a/../../b"); err == nil { + t.Error("cleanPrefix with .. accepted") + } +} + +func TestNewStorageLocalPath(t *testing.T) { + dir := filepath.Join(t.TempDir(), "base") + s, err := NewStorage(StorageTypeLocal, Config{LocalPath: dir}) + if err != nil { + t.Fatal(err) + } + if got := s.(*LocalStorage).BasePath(); got != dir { + t.Fatalf("base path = %q, want %q", got, dir) + } +} + +func TestNewS3StorageOffline(t *testing.T) { + s, err := NewS3Storage(Config{ + S3Bucket: "test-bucket", + S3Endpoint: "127.0.0.1:9", + S3DisableTLS: true, + S3KeyID: "k", + S3Secret: "s", + S3PartSize: 1, + }) + if err != nil { + t.Fatalf("NewS3Storage: %v", err) + } + if s.region != DefaultS3Region { + t.Errorf("region = %q", s.region) + } + if got := *s.client.Options().BaseEndpoint; got != "http://127.0.0.1:9" { + t.Errorf("endpoint = %q", got) + } + //nolint:staticcheck // manager is deprecated, see s3.go + if s.uploader.PartSize != MinS3PartSize || s.uploader.Concurrency != DefaultS3Concurrency { + t.Errorf("uploader part size/concurrency = %d/%d", s.uploader.PartSize, s.uploader.Concurrency) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.Upload(ctx, filepath.Join(t.TempDir(), "missing.txt"), "remote.txt"); err == nil { + t.Fatal("Upload of a missing local file succeeded") + } +} + +func TestLocalConformance(t *testing.T) { + s, err := NewLocalStorage(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = s.Close() }() + runConformance(t, s) +} + +func TestLocalNoLeftovers(t *testing.T) { + base := t.TempDir() + s, err := NewLocalStorage(base) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + _, _ = s.Put(ctx, "x/y.bin", &failingReader{r: strings.NewReader(strings.Repeat("a", 100)), n: 10, err: errors.New("boom")}, nil) + if _, err := s.PutIf(ctx, "x/idx.json", strings.NewReader("1"), Condition{IfNoneMatch: true}, nil); err != nil { + t.Fatal(err) + } + _, _ = s.PutIf(ctx, "x/idx.json", strings.NewReader("2"), Condition{IfNoneMatch: true}, nil) + entries, err := os.ReadDir(filepath.Join(base, "x")) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if e.Name() != "idx.json" { + t.Errorf("leftover file %s", e.Name()) + } + } +} + +// TestLocalPutIfConcurrent checks that concurrent read-modify-write cycles +// with IfMatch never lose an update. +func TestLocalPutIfConcurrent(t *testing.T) { + s, err := NewLocalStorage(t.TempDir()) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + if _, err := s.PutIf(ctx, "idx", strings.NewReader(""), Condition{IfNoneMatch: true}, nil); err != nil { + t.Fatal(err) + } + const workers, perWorker = 8, 10 + var wg sync.WaitGroup + for w := 0; w < workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < perWorker; { + body, it, err := s.Get(ctx, "idx") + if err != nil { + t.Error(err) + return + } + b, _ := io.ReadAll(body) + _ = body.Close() + _, err = s.PutIf(ctx, "idx", strings.NewReader(string(b)+"x"), Condition{IfMatch: it.ETag}, nil) + if errors.Is(err, ErrPreconditionFailed) { + continue + } + if err != nil { + t.Error(err) + return + } + i++ + } + }() + } + wg.Wait() + body, _, err := s.Get(ctx, "idx") + if err != nil { + t.Fatal(err) + } + defer func() { _ = body.Close() }() + b, _ := io.ReadAll(body) + if len(b) != workers*perWorker { + t.Fatalf("lost updates: got %d, want %d", len(b), workers*perWorker) + } +} + +func TestLocalPathTraversal(t *testing.T) { + base := t.TempDir() + s, err := NewLocalStorage(filepath.Join(base, "store")) + if err != nil { + t.Fatal(err) + } + for _, k := range []string{"../outside.txt", "a/../../outside.txt", `..\outside.txt`} { + if _, err := s.Put(context.Background(), k, strings.NewReader("x"), nil); err == nil { + t.Errorf("Put(%q) succeeded", k) + } + } + if _, err := os.Stat(filepath.Join(base, "outside.txt")); !os.IsNotExist(err) { + t.Fatal("file written outside base") + } +}