mirror of
https://github.com/jkaninda/go-storage.git
synced 2026-11-13 16:47:15 +00:00
Compare commits
1
Commits
main
..
2ef6e55c5a
| Author | SHA1 | Date | |
|---|---|---|---|
 dependabot[bot]
|
2ef6e55c5a |
No files matched your search
+13
-33
@@ -1,48 +1,28 @@
|
|||||||
|
# This workflow will build a golang project
|
||||||
|
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
|
||||||
|
|
||||||
name: Go
|
name: Go
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: [ "main", "v2" ]
|
branches: [ "main" ]
|
||||||
pull_request:
|
pull_request:
|
||||||
branches: [ "main", "v2" ]
|
branches: [ "main" ]
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test:
|
|
||||||
|
build:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v7
|
|
||||||
|
|
||||||
- name: Set up Go
|
|
||||||
uses: actions/setup-go@v7
|
|
||||||
with:
|
|
||||||
go-version: '1.25'
|
|
||||||
|
|
||||||
- name: Build
|
|
||||||
run: go build -v ./...
|
|
||||||
|
|
||||||
- name: Vet
|
|
||||||
run: go vet ./... && go vet -tags integration ./...
|
|
||||||
|
|
||||||
- name: Unit tests
|
|
||||||
run: go test -race -count=1 ./...
|
|
||||||
|
|
||||||
integration:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: test
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up Go
|
- name: Set up Go
|
||||||
uses: actions/setup-go@v5
|
uses: actions/setup-go@v4
|
||||||
with:
|
with:
|
||||||
go-version: '1.25'
|
go-version: '1.21.0'
|
||||||
|
|
||||||
- name: Pull images
|
- name: Build
|
||||||
run: |
|
run: go build -v ./...
|
||||||
docker pull pgsty/minio:latest
|
|
||||||
docker pull mcr.microsoft.com/azure-storage/azurite:latest
|
|
||||||
docker pull fsouza/fake-gcs-server:latest
|
|
||||||
docker pull delfer/alpine-ftp-server:latest
|
|
||||||
|
|
||||||
- name: Integration tests (MinIO, Azurite, fake-gcs-server, FTP)
|
- name: Test
|
||||||
run: go test -tags integration -count=1 -v -run Integration ./...
|
run: go test -v ./...
|
||||||
@@ -15,9 +15,9 @@ jobs:
|
|||||||
- name: Setup Go
|
- name: Setup Go
|
||||||
uses: actions/setup-go@v5
|
uses: actions/setup-go@v5
|
||||||
with:
|
with:
|
||||||
go-version: '1.25'
|
go-version: '~1.22'
|
||||||
|
|
||||||
- name: Run linter
|
- name: Run linter
|
||||||
uses: golangci/golangci-lint-action@v7
|
uses: golangci/golangci-lint-action@v6
|
||||||
with:
|
with:
|
||||||
version: v2.13.2
|
version: v1.61
|
||||||
+22
-29
@@ -1,51 +1,44 @@
|
|||||||
version: "2"
|
|
||||||
run:
|
run:
|
||||||
|
timeout: 5m
|
||||||
allow-parallel-runners: true
|
allow-parallel-runners: true
|
||||||
|
|
||||||
|
issues:
|
||||||
|
# don't skip warning about doc comments
|
||||||
|
# don't exclude the default set of lint
|
||||||
|
exclude-use-default: false
|
||||||
|
# restore some of the defaults
|
||||||
|
# (fill in the rest as needed)
|
||||||
|
exclude-rules:
|
||||||
|
- path: "internal/*"
|
||||||
|
linters:
|
||||||
|
- dupl
|
||||||
|
- lll
|
||||||
|
- goimports
|
||||||
linters:
|
linters:
|
||||||
default: none
|
disable-all: true
|
||||||
enable:
|
enable:
|
||||||
- copyloopvar
|
|
||||||
- dupl
|
- dupl
|
||||||
- errcheck
|
- errcheck
|
||||||
|
- copyloopvar
|
||||||
- ginkgolinter
|
- ginkgolinter
|
||||||
- goconst
|
- goconst
|
||||||
- gocyclo
|
- gocyclo
|
||||||
|
- gofmt
|
||||||
|
- gosimple
|
||||||
- govet
|
- govet
|
||||||
- ineffassign
|
- ineffassign
|
||||||
#- lll
|
# - lll
|
||||||
- misspell
|
- misspell
|
||||||
- nakedret
|
- nakedret
|
||||||
- prealloc
|
- prealloc
|
||||||
- revive
|
- revive
|
||||||
- staticcheck
|
- staticcheck
|
||||||
|
- typecheck
|
||||||
- unconvert
|
- unconvert
|
||||||
- unparam
|
- unparam
|
||||||
- unused
|
- unused
|
||||||
settings:
|
|
||||||
|
linters-settings:
|
||||||
revive:
|
revive:
|
||||||
rules:
|
rules:
|
||||||
- name: comment-spacings
|
- name: comment-spacings
|
||||||
exclusions:
|
|
||||||
generated: lax
|
|
||||||
rules:
|
|
||||||
- linters:
|
|
||||||
- dupl
|
|
||||||
- lll
|
|
||||||
path: /*
|
|
||||||
# Legacy v0.1 API: deprecated as a whole and frozen.
|
|
||||||
- linters:
|
|
||||||
- staticcheck
|
|
||||||
path: ^pkg/
|
|
||||||
paths:
|
|
||||||
- third_party$
|
|
||||||
- builtin$
|
|
||||||
- examples$
|
|
||||||
formatters:
|
|
||||||
enable:
|
|
||||||
- gofmt
|
|
||||||
exclusions:
|
|
||||||
generated: lax
|
|
||||||
paths:
|
|
||||||
- third_party$
|
|
||||||
- builtin$
|
|
||||||
- examples$
|
|
||||||
@@ -1,200 +1,126 @@
|
|||||||
# Go Storage
|
# Go Storage
|
||||||
|
A simple Go storage client
|
||||||
|
|
||||||
A streaming storage abstraction for Go backup tools.
|
Supported storage:
|
||||||
|
- Local
|
||||||
|
- S3
|
||||||
|
- SSH
|
||||||
|
- FTP
|
||||||
|
- Azure Blob
|
||||||
|
|
||||||
Supported backends:
|
```go
|
||||||
|
|
||||||
- Local filesystem
|
|
||||||
- S3 and S3-compatible services (MinIO, Ceph, Wasabi, ...) via aws-sdk-go-v2
|
|
||||||
- Azure Blob Storage (and Azurite)
|
|
||||||
- Google Cloud Storage
|
|
||||||
- SFTP
|
|
||||||
- FTP / explicit FTPS
|
|
||||||
|
|
||||||
```shell
|
|
||||||
go get github.com/jkaninda/go-storage
|
go get github.com/jkaninda/go-storage
|
||||||
```
|
```
|
||||||
|
|
||||||
## Streaming API
|
### Local Storage
|
||||||
|
|
||||||
Every backend implements the same `Storage` interface:
|
|
||||||
|
|
||||||
```go
|
```go
|
||||||
type Storage interface {
|
localStorage := local.NewStorage(local.Config{
|
||||||
Upload(ctx context.Context, localPath, remotePath string) error
|
LocalPath: tmpPath,
|
||||||
Download(ctx context.Context, remotePath, localPath string) error
|
RemotePath: backupDestination,
|
||||||
List(ctx context.Context, prefix string) ([]Item, error) // recursive
|
})
|
||||||
Remove(ctx context.Context, remotePath string) error // missing -> ErrNotFound
|
err = localStorage.Copy(finalFileName)
|
||||||
Close() error
|
if err != nil {
|
||||||
|
log.Fatal("Error copying file, error %v", err)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error)
|
### S3 Storage
|
||||||
Get(ctx context.Context, key string) (io.ReadCloser, Item, error)
|
```go
|
||||||
Stat(ctx context.Context, key string) (Item, error)
|
s3Storage, err := s3.NewStorage(s3.Config{
|
||||||
PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error)
|
Endpoint: "",
|
||||||
ListDir(ctx context.Context, dir string) ([]Item, error) // non-recursive, files only
|
Bucket: "",
|
||||||
|
AccessKey: "",
|
||||||
|
SecretKey: "",
|
||||||
|
Region: "",
|
||||||
|
DisableSsl: "",
|
||||||
|
ForcePathStyle: "",
|
||||||
|
RemotePath: "",
|
||||||
|
LocalPath: "",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Error creating s3 storage: %s", err)
|
||||||
|
}
|
||||||
|
// Copy file to S3
|
||||||
|
err = s3Storage.Copy(finalFileName)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Error copying file, error %v", err)
|
||||||
|
}
|
||||||
|
// Download file from S3
|
||||||
|
err = s3Storage.CopyFrom(finalFileName)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Error copying file, error %v", err)
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
- **`Put`** streams a reader of unknown length with bounded memory (no temp
|
### SSH Storage
|
||||||
files, no whole-file buffering). If the reader fails or the context is
|
|
||||||
cancelled, `Put` returns an error and **no object (and no partial object)
|
|
||||||
is left at the key**: S3 aborts the multipart upload, Azure never commits
|
|
||||||
the block list, GCS cancels the resumable upload, and SFTP/FTP/local write
|
|
||||||
to `<key>.<random>.partial` and rename on success.
|
|
||||||
- **`Get`** returns a streaming body; always `Close` it. On FTP the body holds
|
|
||||||
the single control connection, so close it before making another call.
|
|
||||||
- **`PutIf`** is a conditional write for small files such as a backup index
|
|
||||||
(optimistic concurrency): `Condition{IfNoneMatch: true}` creates only if the
|
|
||||||
key does not exist, `Condition{IfMatch: item.ETag}` replaces only if the
|
|
||||||
object is unchanged. It returns `ErrPreconditionFailed` when the condition
|
|
||||||
does not hold and `ErrConditionNotSupported` when the backend cannot
|
|
||||||
evaluate it (fall back to locking).
|
|
||||||
- Errors wrap the sentinels `ErrNotFound`, `ErrPreconditionFailed` and
|
|
||||||
`ErrConditionNotSupported`; test them with `errors.Is`.
|
|
||||||
- Keys are relative to the backend base (bucket, container, remote directory
|
|
||||||
or local base path), use `/` separators and may not escape the base (`..`).
|
|
||||||
|
|
||||||
| Backend | `Item.ETag` | `PutIf` IfNoneMatch | `PutIf` IfMatch |
|
|
||||||
|---------|--------------------------------|---------------------|-----------------|
|
|
||||||
| S3 | S3 ETag | yes | yes |
|
|
||||||
| Azure | blob ETag | yes | yes |
|
|
||||||
| GCS | object generation | yes | yes |
|
|
||||||
| Local | `<mtime-ns hex>-<size hex>` | yes (lock file) | yes (lock file) |
|
|
||||||
| SFTP | empty | yes (`O_EXCL`) | not supported |
|
|
||||||
| FTP | empty | not supported | not supported |
|
|
||||||
|
|
||||||
S3 `PutIf` uses a single `PutObject` request and is limited to 16 MiB.
|
|
||||||
|
|
||||||
### Example: dump, compress and upload without temp files
|
|
||||||
|
|
||||||
```go
|
```go
|
||||||
package main
|
sshStorage, err := ssh.NewStorage(ssh.Config{
|
||||||
|
Host: "",
|
||||||
import (
|
Port: 22,
|
||||||
"compress/gzip"
|
User: "",
|
||||||
"context"
|
Password: "",
|
||||||
"errors"
|
RemotePath: "",
|
||||||
"io"
|
LocalPath: "",
|
||||||
"log"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
storage "github.com/jkaninda/go-storage"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
ctx := context.Background()
|
|
||||||
s, err := storage.NewStorage(storage.StorageTypeS3, storage.Config{
|
|
||||||
S3Bucket: "backups",
|
|
||||||
S3Region: "us-east-1",
|
|
||||||
S3KeyID: "...",
|
|
||||||
S3Secret: "...",
|
|
||||||
S3Endpoint: "https://minio.example.com", // optional, for S3-compatible services
|
|
||||||
S3ForcePath: true,
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatalf("Error creating SSH storage: %s", err)
|
||||||
}
|
}
|
||||||
defer s.Close()
|
// Copy file to the remote server
|
||||||
|
err = sshStorage.Copy(finalFileName)
|
||||||
pr, pw := io.Pipe()
|
|
||||||
go func() {
|
|
||||||
gz := gzip.NewWriter(pw)
|
|
||||||
cmd := exec.CommandContext(ctx, "pg_dump", "mydb")
|
|
||||||
cmd.Stdout = gz
|
|
||||||
err := cmd.Run()
|
|
||||||
if err == nil {
|
|
||||||
err = gz.Close()
|
|
||||||
}
|
|
||||||
pw.CloseWithError(err) // a non-nil error aborts the upload
|
|
||||||
}()
|
|
||||||
|
|
||||||
item, err := s.Put(ctx, "db/mydb-2026-01-01.sql.gz", pr, &storage.PutOptions{ContentType: "application/gzip"})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatalf("Error copying backup file: %s", err)
|
||||||
}
|
}
|
||||||
log.Printf("uploaded %s (%d bytes, etag %s)", item.Key, item.Size, item.ETag)
|
// Download file from SSH remote server
|
||||||
|
err = sshStorage.CopyFrom(finalFileName)
|
||||||
// Restore: stream it back.
|
if err != nil {
|
||||||
body, _, err := s.Get(ctx, item.Key)
|
log.Fatalf("Error copying file, error %v", err)
|
||||||
if errors.Is(err, storage.ErrNotFound) {
|
}
|
||||||
log.Fatal("backup not found")
|
```
|
||||||
} else if err != nil {
|
### FTP Storage
|
||||||
log.Fatal(err)
|
```go
|
||||||
|
ftpStorage, err := ftp.NewStorage(ftp.Config{
|
||||||
|
Host: "",
|
||||||
|
Port: 21,
|
||||||
|
User: "",
|
||||||
|
Password: "",
|
||||||
|
RemotePath: "",
|
||||||
|
LocalPath: "",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Error creating FTP storage: %s", err)
|
||||||
}
|
}
|
||||||
defer body.Close()
|
err = ftpStorage.Copy(finalFileName)
|
||||||
zr, _ := gzip.NewReader(body)
|
if err != nil {
|
||||||
_, _ = io.Copy(io.Discard, zr)
|
log.Fatalf("Error copying backup file: %s", err)
|
||||||
|
|
||||||
// Update an index file with optimistic concurrency.
|
|
||||||
idx, err := s.Stat(ctx, "db/index.json")
|
|
||||||
cond := storage.Condition{IfMatch: idx.ETag}
|
|
||||||
if errors.Is(err, storage.ErrNotFound) {
|
|
||||||
cond = storage.Condition{IfNoneMatch: true}
|
|
||||||
}
|
|
||||||
_, err = s.PutIf(ctx, "db/index.json", strings.NewReader(`{"backups":[]}`), cond, nil)
|
|
||||||
if errors.Is(err, storage.ErrPreconditionFailed) {
|
|
||||||
// someone else updated the index: re-read, merge and retry
|
|
||||||
}
|
}
|
||||||
|
// Download file from ftp remote server
|
||||||
|
err = ftpStorage.CopyFrom(finalFileName)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Error copying file, error %v", err)
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
## Configuration
|
### Azure Blob storage
|
||||||
|
|
||||||
`storage.NewStorage(storageType, storage.Config{...})` with `storageType` one
|
```go
|
||||||
of `s3`, `gcs`, `azure`, `sftp`, `ftp`, `local`.
|
azureStorage, err := azure.NewStorage(azure.Config{
|
||||||
|
ContainerName: '',
|
||||||
|
AccountName: '',
|
||||||
|
AccountKey: '',
|
||||||
|
RemotePath: '',
|
||||||
|
LocalPath: '',
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Error creating Azure Blob storage storage: %s", err)
|
||||||
|
}
|
||||||
|
err = azureStorage.Copy(finalFileName)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Error copying file: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
| Field | Backend | Description |
|
// Download file from Azure Blob remote server
|
||||||
|-------|---------|-------------|
|
err = azureStorage.CopyFrom(finalFileName)
|
||||||
| `LocalPath` | local | Base directory (default `/backup`) |
|
if err != nil {
|
||||||
| `S3Bucket` | s3 | Bucket (required) |
|
log.Fatalf("Error copying file, error %v", err)
|
||||||
| `S3Region` | s3 | Region (default from AWS config, else `us-east-1`) |
|
}
|
||||||
| `S3KeyID`, `S3Secret`, `S3SessionToken` | s3 | Static credentials; when empty the default AWS chain is used (env, shared profile, IRSA/web identity, ECS/EC2 roles) |
|
|
||||||
| `S3Profile` | s3 | Shared config profile |
|
|
||||||
| `S3Endpoint` | s3 | Custom endpoint (MinIO etc.); without a scheme `https://` is assumed (`http://` with `S3DisableTLS`) |
|
|
||||||
| `S3ForcePath` | s3 | Path-style addressing |
|
|
||||||
| `S3DisableTLS` | s3 | Use `http://` for scheme-less endpoints |
|
|
||||||
| `S3StorageClass` | s3 | e.g. `STANDARD_IA`, `GLACIER_IR` |
|
|
||||||
| `S3SSE`, `S3SSEKMSKeyID` | s3 | Server-side encryption: `AES256` or `aws:kms` (+ key id) |
|
|
||||||
| `S3PartSize` | s3 | Multipart part size (default 64 MiB, min 5 MiB) |
|
|
||||||
| `S3Concurrency` | s3 | Parallel part uploads (default 2); memory ≈ part size × concurrency |
|
|
||||||
| `AzureAccount`, `AzureKey` | azure | Shared key credentials |
|
|
||||||
| `AzureSASToken` | azure | SAS token (used instead of the shared key when set) |
|
|
||||||
| `AzureContainer` | azure | Container (required) |
|
|
||||||
| `AzureEndpoint` | azure | Domain suffix (default `blob.core.windows.net`) or a full service URL such as `http://127.0.0.1:10000/devstoreaccount1` (Azurite) |
|
|
||||||
| `GCSBucket` | gcs | Bucket (required) |
|
|
||||||
| `GCSCredentialsFile`, `GCSCredentialsJSON` | gcs | Credentials; default is Application Default Credentials |
|
|
||||||
| `GCSEndpoint` | gcs | Custom endpoint |
|
|
||||||
| `SFTPHost`, `SFTPPort` | sftp | Server (port default 22) |
|
|
||||||
| `SFTPUsername`, `SFTPPassword` | sftp | Password authentication |
|
|
||||||
| `SFTPIdentityFile`, `SFTPIdentityPassphrase` | sftp | Private key authentication (an SSH agent is used too when available) |
|
|
||||||
| `SFTPKnownHostsFile` | sftp | known_hosts file (default `~/.ssh/known_hosts`); host keys are always verified unless... |
|
|
||||||
| `SFTPIgnoreKnownHosts` | sftp | ...this is `true` (insecure) |
|
|
||||||
| `SFTPDirectory` | sftp | Remote base directory (default: login directory) |
|
|
||||||
| `FTPHost`, `FTPPort`, `FTPUsername`, `FTPPassword` | ftp | Server and credentials (port default 21) |
|
|
||||||
| `FTPDirectory` | ftp | Remote base directory (default: login directory) |
|
|
||||||
| `FTPTLS` | ftp | Explicit FTPS (`AUTH TLS`) |
|
|
||||||
|
|
||||||
The library never reads environment variables itself (except what the cloud
|
|
||||||
SDKs do for their default credential chains, and `SSH_AUTH_SOCK` for the SSH agent).
|
|
||||||
|
|
||||||
## Testing
|
|
||||||
|
|
||||||
```shell
|
|
||||||
go test ./... # unit tests: local + in-process SFTP server
|
|
||||||
go test -tags integration ./... # + MinIO, Azurite, fake-gcs-server, FTP via docker
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Integration tests skip when docker is unavailable. Images can be overridden
|
|
||||||
with `GO_STORAGE_MINIO_IMAGE`, `GO_STORAGE_AZURITE_IMAGE`,
|
|
||||||
`GO_STORAGE_FAKEGCS_IMAGE` and `GO_STORAGE_FTP_IMAGE`.
|
|
||||||
|
|
||||||
## Deprecated: `pkg/` (v0.1 API)
|
|
||||||
|
|
||||||
The packages under `pkg/` (`pkg/local`, `pkg/s3`, `pkg/ssh`, `pkg/ftp`,
|
|
||||||
`pkg/azure`) are the legacy v0.1 API. They still compile but are deprecated
|
|
||||||
and will be removed in a future release; use the root package instead.
|
|
||||||
|
|
||||||
## License
|
|
||||||
|
|
||||||
MIT License, Copyright (c) 2024 Jonas Kaninda. See [LICENSE](LICENSE).
|
|
||||||
@@ -1,349 +0,0 @@
|
|||||||
/*
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2024 Jonas Kaninda
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package go_storage
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
|
||||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
|
||||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
|
||||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
|
|
||||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Azure streaming upload tuning: 8 MiB blocks allow blobs up to ~390 GiB
|
|
||||||
// (50,000 blocks) while keeping memory bounded to blockSize*concurrency.
|
|
||||||
const (
|
|
||||||
azureBlockSize int64 = 8 << 20
|
|
||||||
azureConcurrency = 2
|
|
||||||
)
|
|
||||||
|
|
||||||
// AzureStorage implements Storage interface for Azure Blob Storage.
|
|
||||||
type AzureStorage struct {
|
|
||||||
client *azblob.Client
|
|
||||||
container string
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewAzureStorage creates a new Azure Blob Storage instance.
|
|
||||||
//
|
|
||||||
// Authentication: AzureSASToken if set, otherwise shared key
|
|
||||||
// (AzureAccount/AzureKey), otherwise anonymous.
|
|
||||||
func NewAzureStorage(config Config) (*AzureStorage, error) {
|
|
||||||
if err := validateConfig(config, map[string]string{
|
|
||||||
"container": config.AzureContainer,
|
|
||||||
}); err != nil {
|
|
||||||
return nil, fmt.Errorf("azure: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
serviceURL, err := azureServiceURL(config)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var client *azblob.Client
|
|
||||||
switch {
|
|
||||||
case config.AzureSASToken != "":
|
|
||||||
u := serviceURL + "?" + strings.TrimPrefix(config.AzureSASToken, "?")
|
|
||||||
client, err = azblob.NewClientWithNoCredential(u, nil)
|
|
||||||
case config.AzureAccount != "" && config.AzureKey != "":
|
|
||||||
cred, cerr := azblob.NewSharedKeyCredential(config.AzureAccount, config.AzureKey)
|
|
||||||
if cerr != nil {
|
|
||||||
return nil, fmt.Errorf("azure: failed to create credentials: %w", cerr)
|
|
||||||
}
|
|
||||||
client, err = azblob.NewClientWithSharedKeyCredential(serviceURL, cred, nil)
|
|
||||||
default:
|
|
||||||
client, err = azblob.NewClientWithNoCredential(serviceURL, nil)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("azure: failed to create client: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &AzureStorage{client: client, container: config.AzureContainer}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func azureServiceURL(config Config) (string, error) {
|
|
||||||
endpoint := config.AzureEndpoint
|
|
||||||
if endpoint == "" {
|
|
||||||
endpoint = "blob.core.windows.net"
|
|
||||||
}
|
|
||||||
if strings.Contains(endpoint, "://") {
|
|
||||||
return strings.TrimSuffix(endpoint, "/") + "/", nil
|
|
||||||
}
|
|
||||||
if config.AzureAccount == "" {
|
|
||||||
return "", fmt.Errorf("azure: missing required configuration: account")
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("https://%s.%s/", config.AzureAccount, strings.Trim(endpoint, "/")), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Client returns the underlying azblob client.
|
|
||||||
func (a *AzureStorage) Client() *azblob.Client { return a.client }
|
|
||||||
|
|
||||||
func (a *AzureStorage) put(ctx context.Context, key string, r io.Reader, opts *PutOptions, ac *blob.AccessConditions) (Item, error) {
|
|
||||||
k, err := cleanKey(key)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, fmt.Errorf("azure: %w", err)
|
|
||||||
}
|
|
||||||
o := &azblob.UploadStreamOptions{
|
|
||||||
BlockSize: azureBlockSize,
|
|
||||||
Concurrency: azureConcurrency,
|
|
||||||
AccessConditions: ac,
|
|
||||||
}
|
|
||||||
if opts != nil {
|
|
||||||
if opts.ContentType != "" {
|
|
||||||
o.HTTPHeaders = &blob.HTTPHeaders{BlobContentType: &opts.ContentType}
|
|
||||||
}
|
|
||||||
if len(opts.Metadata) > 0 {
|
|
||||||
o.Metadata = make(map[string]*string, len(opts.Metadata))
|
|
||||||
for mk, mv := range opts.Metadata {
|
|
||||||
o.Metadata[mk] = &mv
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
cr := &countingReader{r: ctxReader{ctx: ctx, r: r}}
|
|
||||||
resp, err := a.client.UploadStream(ctx, a.container, k, cr, o)
|
|
||||||
if err != nil {
|
|
||||||
if ac != nil && (isAzurePrecondition(err) || bloberror.HasCode(err, bloberror.BlobNotFound)) {
|
|
||||||
return Item{}, fmt.Errorf("azure: conditional put of %s: %w", k, ErrPreconditionFailed)
|
|
||||||
}
|
|
||||||
return Item{}, fmt.Errorf("azure: failed to upload %s to container %s: %w", k, a.container, err)
|
|
||||||
}
|
|
||||||
it := Item{Key: k, Size: cr.n, ModifiedTime: time.Now().UTC()}
|
|
||||||
if resp.ETag != nil {
|
|
||||||
it.ETag = string(*resp.ETag)
|
|
||||||
}
|
|
||||||
if resp.LastModified != nil {
|
|
||||||
it.ModifiedTime = *resp.LastModified
|
|
||||||
}
|
|
||||||
if resp.VersionID != nil {
|
|
||||||
it.VersionID = *resp.VersionID
|
|
||||||
}
|
|
||||||
return it, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Put streams r to key with a block upload; the block list is only
|
|
||||||
// committed when the whole stream was read successfully.
|
|
||||||
func (a *AzureStorage) Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error) {
|
|
||||||
return a.put(ctx, key, r, opts, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
// PutIf writes r to key with If-Match / If-None-Match: * access conditions.
|
|
||||||
func (a *AzureStorage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error) {
|
|
||||||
if err := validateCondition(cond); err != nil {
|
|
||||||
return Item{}, fmt.Errorf("azure: %w", err)
|
|
||||||
}
|
|
||||||
mac := &blob.ModifiedAccessConditions{}
|
|
||||||
if cond.IfMatch != "" {
|
|
||||||
etag := azcore.ETag(cond.IfMatch)
|
|
||||||
mac.IfMatch = &etag
|
|
||||||
}
|
|
||||||
if cond.IfNoneMatch {
|
|
||||||
etag := azcore.ETagAny
|
|
||||||
mac.IfNoneMatch = &etag
|
|
||||||
}
|
|
||||||
return a.put(ctx, key, r, opts, &blob.AccessConditions{ModifiedAccessConditions: mac})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get streams the blob body.
|
|
||||||
func (a *AzureStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
|
|
||||||
k, err := cleanKey(key)
|
|
||||||
if err != nil {
|
|
||||||
return nil, Item{}, fmt.Errorf("azure: %w", err)
|
|
||||||
}
|
|
||||||
resp, err := a.client.DownloadStream(ctx, a.container, k, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, Item{}, a.mapErr("get", k, err)
|
|
||||||
}
|
|
||||||
it := Item{Key: k}
|
|
||||||
if resp.ContentLength != nil {
|
|
||||||
it.Size = *resp.ContentLength
|
|
||||||
}
|
|
||||||
if resp.LastModified != nil {
|
|
||||||
it.ModifiedTime = *resp.LastModified
|
|
||||||
}
|
|
||||||
if resp.ETag != nil {
|
|
||||||
it.ETag = string(*resp.ETag)
|
|
||||||
}
|
|
||||||
if resp.VersionID != nil {
|
|
||||||
it.VersionID = *resp.VersionID
|
|
||||||
}
|
|
||||||
return resp.Body, it, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stat returns blob properties.
|
|
||||||
func (a *AzureStorage) Stat(ctx context.Context, key string) (Item, error) {
|
|
||||||
k, err := cleanKey(key)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, fmt.Errorf("azure: %w", err)
|
|
||||||
}
|
|
||||||
bc := a.client.ServiceClient().NewContainerClient(a.container).NewBlobClient(k)
|
|
||||||
resp, err := bc.GetProperties(ctx, nil)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, a.mapErr("stat", k, err)
|
|
||||||
}
|
|
||||||
it := Item{Key: k}
|
|
||||||
if resp.ContentLength != nil {
|
|
||||||
it.Size = *resp.ContentLength
|
|
||||||
}
|
|
||||||
if resp.LastModified != nil {
|
|
||||||
it.ModifiedTime = *resp.LastModified
|
|
||||||
}
|
|
||||||
if resp.ETag != nil {
|
|
||||||
it.ETag = string(*resp.ETag)
|
|
||||||
}
|
|
||||||
if resp.VersionID != nil {
|
|
||||||
it.VersionID = *resp.VersionID
|
|
||||||
}
|
|
||||||
return it, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Upload uploads a file to Azure Blob Storage.
|
|
||||||
func (a *AzureStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
|
||||||
if err := uploadFile(ctx, a, localPath, remotePath); err != nil {
|
|
||||||
return fmt.Errorf("azure: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Download downloads a file from Azure Blob Storage.
|
|
||||||
func (a *AzureStorage) Download(ctx context.Context, remotePath string, localPath string) error {
|
|
||||||
if err := downloadFile(ctx, a, remotePath, localPath); err != nil {
|
|
||||||
return fmt.Errorf("azure: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// List lists blobs (recursively) with the given prefix.
|
|
||||||
func (a *AzureStorage) List(ctx context.Context, prefix string) ([]Item, error) {
|
|
||||||
p, err := cleanPrefix(prefix)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("azure: %w", err)
|
|
||||||
}
|
|
||||||
pager := a.client.NewListBlobsFlatPager(a.container, &azblob.ListBlobsFlatOptions{Prefix: &p})
|
|
||||||
items := make([]Item, 0)
|
|
||||||
for pager.More() {
|
|
||||||
resp, err := pager.NextPage(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("azure: failed to list blobs in container %s: %w", a.container, err)
|
|
||||||
}
|
|
||||||
for _, b := range resp.Segment.BlobItems {
|
|
||||||
items = append(items, azureItem(b))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return items, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListDir lists the blobs directly under dir.
|
|
||||||
func (a *AzureStorage) ListDir(ctx context.Context, dir string) ([]Item, error) {
|
|
||||||
d, err := cleanDir(dir)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("azure: %w", err)
|
|
||||||
}
|
|
||||||
p := dirPrefix(d)
|
|
||||||
cc := a.client.ServiceClient().NewContainerClient(a.container)
|
|
||||||
pager := cc.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{Prefix: &p})
|
|
||||||
items := make([]Item, 0)
|
|
||||||
for pager.More() {
|
|
||||||
resp, err := pager.NextPage(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("azure: failed to list blobs in container %s: %w", a.container, err)
|
|
||||||
}
|
|
||||||
for _, b := range resp.Segment.BlobItems {
|
|
||||||
if b.Name == nil || strings.HasSuffix(*b.Name, "/") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
items = append(items, azureItem(b))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return items, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func azureItem(b *container.BlobItem) Item {
|
|
||||||
it := Item{}
|
|
||||||
if b.Name != nil {
|
|
||||||
it.Key = *b.Name
|
|
||||||
}
|
|
||||||
if b.VersionID != nil {
|
|
||||||
it.VersionID = *b.VersionID
|
|
||||||
}
|
|
||||||
if p := b.Properties; p != nil {
|
|
||||||
if p.LastModified != nil {
|
|
||||||
it.ModifiedTime = *p.LastModified
|
|
||||||
}
|
|
||||||
if p.ContentLength != nil {
|
|
||||||
it.Size = *p.ContentLength
|
|
||||||
}
|
|
||||||
if p.ETag != nil {
|
|
||||||
it.ETag = string(*p.ETag)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return it
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove deletes a blob.
|
|
||||||
func (a *AzureStorage) Remove(ctx context.Context, remotePath string) error {
|
|
||||||
k, err := cleanKey(remotePath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("azure: %w", err)
|
|
||||||
}
|
|
||||||
if _, err := a.client.DeleteBlob(ctx, a.container, k, nil); err != nil {
|
|
||||||
return a.mapErr("remove", k, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close releases resources (no-op).
|
|
||||||
func (a *AzureStorage) Close() error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *AzureStorage) mapErr(op, key string, err error) error {
|
|
||||||
if bloberror.HasCode(err, bloberror.BlobNotFound, bloberror.ResourceNotFound) {
|
|
||||||
return fmt.Errorf("azure: %s %s in container %s: %w", op, key, a.container, ErrNotFound)
|
|
||||||
}
|
|
||||||
var re *azcore.ResponseError
|
|
||||||
if errors.As(err, &re) && re.StatusCode == http.StatusNotFound && re.ErrorCode != string(bloberror.ContainerNotFound) {
|
|
||||||
return fmt.Errorf("azure: %s %s in container %s: %w", op, key, a.container, ErrNotFound)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("azure: %s %s in container %s: %w", op, key, a.container, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func isAzurePrecondition(err error) bool {
|
|
||||||
if bloberror.HasCode(err, bloberror.ConditionNotMet, bloberror.BlobAlreadyExists, bloberror.TargetConditionNotMet) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
var re *azcore.ResponseError
|
|
||||||
if errors.As(err, &re) {
|
|
||||||
return re.StatusCode == http.StatusPreconditionFailed ||
|
|
||||||
(re.StatusCode == http.StatusConflict && re.ErrorCode == string(bloberror.BlobAlreadyExists))
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
@@ -1,364 +0,0 @@
|
|||||||
/*
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2024 Jonas Kaninda
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package go_storage
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"math/rand"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const conformanceStreamSize = 20 << 20
|
|
||||||
|
|
||||||
// deterministicStream writes n pseudo-random bytes into an io.Pipe and
|
|
||||||
// returns the reader plus the expected SHA-256.
|
|
||||||
func deterministicStream(n int64, seed int64) (io.Reader, [32]byte) {
|
|
||||||
h := sha256.New()
|
|
||||||
_, _ = io.Copy(h, io.LimitReader(rand.New(rand.NewSource(seed)), n))
|
|
||||||
var sum [32]byte
|
|
||||||
copy(sum[:], h.Sum(nil))
|
|
||||||
|
|
||||||
pr, pw := io.Pipe()
|
|
||||||
go func() {
|
|
||||||
_, err := io.Copy(pw, io.LimitReader(rand.New(rand.NewSource(seed)), n))
|
|
||||||
_ = pw.CloseWithError(err)
|
|
||||||
}()
|
|
||||||
return pr, sum
|
|
||||||
}
|
|
||||||
|
|
||||||
// failingReader returns ok bytes, then err.
|
|
||||||
type failingReader struct {
|
|
||||||
r io.Reader
|
|
||||||
n int64
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *failingReader) Read(p []byte) (int, error) {
|
|
||||||
if f.n <= 0 {
|
|
||||||
return 0, f.err
|
|
||||||
}
|
|
||||||
if int64(len(p)) > f.n {
|
|
||||||
p = p[:f.n]
|
|
||||||
}
|
|
||||||
n, err := f.r.Read(p)
|
|
||||||
f.n -= int64(n)
|
|
||||||
return n, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func keysOf(items []Item, filesOnly bool) []string {
|
|
||||||
out := make([]string, 0, len(items))
|
|
||||||
for _, it := range items {
|
|
||||||
if filesOnly && it.IsDirectory {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, it.Key)
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func putString(t *testing.T, s Storage, key, content string) Item {
|
|
||||||
t.Helper()
|
|
||||||
it, err := s.Put(context.Background(), key, strings.NewReader(content), &PutOptions{ContentType: "text/plain"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Put(%s): %v", key, err)
|
|
||||||
}
|
|
||||||
return it
|
|
||||||
}
|
|
||||||
|
|
||||||
func getString(t *testing.T, s Storage, key string) string {
|
|
||||||
t.Helper()
|
|
||||||
body, _, err := s.Get(context.Background(), key)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Get(%s): %v", key, err)
|
|
||||||
}
|
|
||||||
defer func() { _ = body.Close() }()
|
|
||||||
b, err := io.ReadAll(body)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read %s: %v", key, err)
|
|
||||||
}
|
|
||||||
return string(b)
|
|
||||||
}
|
|
||||||
|
|
||||||
// runConformance exercises the full Storage contract against s.
|
|
||||||
func runConformance(t *testing.T, s Storage) {
|
|
||||||
root := fmt.Sprintf("conf-%d", time.Now().UnixNano())
|
|
||||||
k := func(p string) string { return root + "/" + p }
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
fn func(*testing.T, Storage, func(string) string)
|
|
||||||
}{
|
|
||||||
{"StreamRoundTrip", confStreamRoundTrip},
|
|
||||||
{"FailingReaderLeavesNothing", confFailingReaderLeavesNothing},
|
|
||||||
{"ListDirBoundary", confListDirBoundary},
|
|
||||||
{"ListRecursive", confListRecursive},
|
|
||||||
{"Remove", confRemove},
|
|
||||||
{"PathEscape", confPathEscape},
|
|
||||||
{"PutIf", confPutIf},
|
|
||||||
{"UploadDownload", confUploadDownload},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) { tc.fn(t, s, k) })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func confStreamRoundTrip(t *testing.T, s Storage, k func(string) string) {
|
|
||||||
ctx := context.Background()
|
|
||||||
r, want := deterministicStream(conformanceStreamSize, 42)
|
|
||||||
it, err := s.Put(ctx, k("stream/big.bin"), r, &PutOptions{ContentType: "application/octet-stream", Size: -1})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Put: %v", err)
|
|
||||||
}
|
|
||||||
if it.Size != conformanceStreamSize {
|
|
||||||
t.Errorf("Put size = %d, want %d", it.Size, conformanceStreamSize)
|
|
||||||
}
|
|
||||||
st, err := s.Stat(ctx, k("stream/big.bin"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Stat: %v", err)
|
|
||||||
}
|
|
||||||
if st.Size != conformanceStreamSize {
|
|
||||||
t.Errorf("Stat size = %d", st.Size)
|
|
||||||
}
|
|
||||||
if st.Key != k("stream/big.bin") {
|
|
||||||
t.Errorf("Stat key = %q", st.Key)
|
|
||||||
}
|
|
||||||
if st.ETag != it.ETag {
|
|
||||||
t.Errorf("Stat ETag %q != Put ETag %q", st.ETag, it.ETag)
|
|
||||||
}
|
|
||||||
body, gi, err := s.Get(ctx, k("stream/big.bin"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Get: %v", err)
|
|
||||||
}
|
|
||||||
h := sha256.New()
|
|
||||||
_, err = io.Copy(h, body)
|
|
||||||
_ = body.Close()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read body: %v", err)
|
|
||||||
}
|
|
||||||
if !bytes.Equal(h.Sum(nil), want[:]) {
|
|
||||||
t.Fatal("content mismatch")
|
|
||||||
}
|
|
||||||
if gi.Size != conformanceStreamSize {
|
|
||||||
t.Errorf("Get item size = %d", gi.Size)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func confFailingReaderLeavesNothing(t *testing.T, s Storage, k func(string) string) {
|
|
||||||
ctx := context.Background()
|
|
||||||
key := k("fail/broken.bin")
|
|
||||||
boom := errors.New("boom")
|
|
||||||
r := &failingReader{r: rand.New(rand.NewSource(1)), n: 18 << 20, err: boom}
|
|
||||||
if _, err := s.Put(ctx, key, r, nil); err == nil {
|
|
||||||
t.Fatal("Put with failing reader succeeded")
|
|
||||||
}
|
|
||||||
if _, err := s.Stat(ctx, key); !errors.Is(err, ErrNotFound) {
|
|
||||||
t.Fatalf("Stat after failed Put: %v, want ErrNotFound", err)
|
|
||||||
}
|
|
||||||
items, err := s.ListDir(ctx, k("fail"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ListDir: %v", err)
|
|
||||||
}
|
|
||||||
for _, it := range items {
|
|
||||||
if strings.HasSuffix(it.Key, ".partial") || strings.HasSuffix(it.Key, "broken.bin") {
|
|
||||||
t.Errorf("leftover after failed Put: %s", it.Key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
cctx, cancel := context.WithCancel(ctx)
|
|
||||||
pr, pw := io.Pipe()
|
|
||||||
go func() {
|
|
||||||
_, _ = pw.Write(make([]byte, 1<<20))
|
|
||||||
cancel()
|
|
||||||
_, _ = pw.Write(make([]byte, 1<<20))
|
|
||||||
_ = pw.Close()
|
|
||||||
}()
|
|
||||||
if _, err := s.Put(cctx, k("fail/cancelled.bin"), pr, nil); err == nil {
|
|
||||||
t.Fatal("Put with cancelled context succeeded")
|
|
||||||
}
|
|
||||||
_ = pr.Close()
|
|
||||||
if _, err := s.Stat(ctx, k("fail/cancelled.bin")); !errors.Is(err, ErrNotFound) {
|
|
||||||
t.Fatalf("Stat after cancelled Put: %v, want ErrNotFound", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func confListDirBoundary(t *testing.T, s Storage, k func(string) string) {
|
|
||||||
ctx := context.Background()
|
|
||||||
for _, p := range []string{"ld/a/b/1.txt", "ld/a/b/2.txt", "ld/a/b/sub/deep.txt", "ld/a/bc/x.txt", "ld/top.txt"} {
|
|
||||||
putString(t, s, k(p), "x")
|
|
||||||
}
|
|
||||||
want := []string{k("ld/a/b/1.txt"), k("ld/a/b/2.txt")}
|
|
||||||
for _, dir := range []string{k("ld/a/b"), k("ld/a/b/")} {
|
|
||||||
items, err := s.ListDir(ctx, dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ListDir(%s): %v", dir, err)
|
|
||||||
}
|
|
||||||
if got := keysOf(items, false); strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Errorf("ListDir(%s) = %v, want %v", dir, got, want)
|
|
||||||
}
|
|
||||||
for _, it := range items {
|
|
||||||
if it.IsDirectory || it.Size != 1 {
|
|
||||||
t.Errorf("ListDir item %+v", it)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
items, err := s.ListDir(ctx, k("ld"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := keysOf(items, false); len(got) != 1 || got[0] != k("ld/top.txt") {
|
|
||||||
t.Errorf("ListDir(ld) = %v", got)
|
|
||||||
}
|
|
||||||
items, err = s.ListDir(ctx, k("ld/missing"))
|
|
||||||
if err != nil || len(items) != 0 {
|
|
||||||
t.Errorf("ListDir(missing) = %v, %v", items, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func confListRecursive(t *testing.T, s Storage, k func(string) string) {
|
|
||||||
ctx := context.Background()
|
|
||||||
items, err := s.List(ctx, k("ld/a/b/"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
want := []string{k("ld/a/b/1.txt"), k("ld/a/b/2.txt"), k("ld/a/b/sub/deep.txt")}
|
|
||||||
if got := keysOf(items, true); strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Errorf("List(ld/a/b/) = %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
items, err = s.List(ctx, k("ld/"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := keysOf(items, true); len(got) != 5 {
|
|
||||||
t.Errorf("List(ld/) = %v, want 5 files", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func confRemove(t *testing.T, s Storage, k func(string) string) {
|
|
||||||
ctx := context.Background()
|
|
||||||
if err := s.Remove(ctx, k("does/not/exist.txt")); !errors.Is(err, ErrNotFound) {
|
|
||||||
t.Fatalf("Remove missing: %v, want ErrNotFound", err)
|
|
||||||
}
|
|
||||||
putString(t, s, k("rm/x.txt"), "x")
|
|
||||||
if err := s.Remove(ctx, k("rm/x.txt")); err != nil {
|
|
||||||
t.Fatalf("Remove: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := s.Stat(ctx, k("rm/x.txt")); !errors.Is(err, ErrNotFound) {
|
|
||||||
t.Fatalf("Stat after Remove: %v", err)
|
|
||||||
}
|
|
||||||
if _, _, err := s.Get(ctx, k("rm/x.txt")); !errors.Is(err, ErrNotFound) {
|
|
||||||
t.Fatalf("Get after Remove: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func confPathEscape(t *testing.T, s Storage, k func(string) string) {
|
|
||||||
ctx := context.Background()
|
|
||||||
if _, err := s.Put(ctx, "../escape.txt", strings.NewReader("x"), nil); err == nil {
|
|
||||||
t.Fatal("Put ../escape.txt succeeded")
|
|
||||||
}
|
|
||||||
if _, err := s.Stat(ctx, "a/../../escape.txt"); err == nil || errors.Is(err, ErrNotFound) {
|
|
||||||
t.Fatalf("Stat escaping key: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func confPutIf(t *testing.T, s Storage, k func(string) string) {
|
|
||||||
ctx := context.Background()
|
|
||||||
key := k("index/index.json")
|
|
||||||
it1, err := s.PutIf(ctx, key, strings.NewReader(`{"v":1}`), Condition{IfNoneMatch: true}, &PutOptions{ContentType: "application/json"})
|
|
||||||
if errors.Is(err, ErrConditionNotSupported) {
|
|
||||||
t.Skip("backend does not support conditional writes")
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("PutIf IfNoneMatch (new): %v", err)
|
|
||||||
}
|
|
||||||
if _, err := s.PutIf(ctx, key, strings.NewReader(`{"v":2}`), Condition{IfNoneMatch: true}, nil); !errors.Is(err, ErrPreconditionFailed) {
|
|
||||||
t.Fatalf("PutIf IfNoneMatch (exists): %v, want ErrPreconditionFailed", err)
|
|
||||||
}
|
|
||||||
if got := getString(t, s, key); got != `{"v":1}` {
|
|
||||||
t.Fatalf("content after rejected PutIf = %q", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
it2, err := s.PutIf(ctx, key, strings.NewReader(`{"v":2}`), Condition{IfMatch: it1.ETag}, nil)
|
|
||||||
if errors.Is(err, ErrConditionNotSupported) {
|
|
||||||
t.Log("IfMatch not supported; skipping IfMatch checks")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("PutIf IfMatch (fresh): %v", err)
|
|
||||||
}
|
|
||||||
if it2.ETag == "" || it2.ETag == it1.ETag {
|
|
||||||
t.Fatalf("ETag did not change: %q -> %q", it1.ETag, it2.ETag)
|
|
||||||
}
|
|
||||||
if _, err := s.PutIf(ctx, key, strings.NewReader(`{"v":3}`), Condition{IfMatch: it1.ETag}, nil); !errors.Is(err, ErrPreconditionFailed) {
|
|
||||||
t.Fatalf("PutIf IfMatch (stale): %v, want ErrPreconditionFailed", err)
|
|
||||||
}
|
|
||||||
if got := getString(t, s, key); got != `{"v":2}` {
|
|
||||||
t.Fatalf("content = %q, want v2", got)
|
|
||||||
}
|
|
||||||
st, err := s.Stat(ctx, key)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if st.ETag != it2.ETag {
|
|
||||||
t.Fatalf("Stat ETag %q != PutIf ETag %q", st.ETag, it2.ETag)
|
|
||||||
}
|
|
||||||
if _, err := s.PutIf(ctx, k("index/missing.json"), strings.NewReader("x"), Condition{IfMatch: it2.ETag}, nil); !errors.Is(err, ErrPreconditionFailed) {
|
|
||||||
t.Fatalf("PutIf IfMatch on missing key: %v, want ErrPreconditionFailed", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func confUploadDownload(t *testing.T, s Storage, k func(string) string) {
|
|
||||||
ctx := context.Background()
|
|
||||||
dir := t.TempDir()
|
|
||||||
src := filepath.Join(dir, "src.txt")
|
|
||||||
if err := os.WriteFile(src, []byte("hello world"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := s.Upload(ctx, src, k("ud/file.txt")); err != nil {
|
|
||||||
t.Fatalf("Upload: %v", err)
|
|
||||||
}
|
|
||||||
dst := filepath.Join(dir, "out", "dst.txt")
|
|
||||||
if err := s.Download(ctx, k("ud/file.txt"), dst); err != nil {
|
|
||||||
t.Fatalf("Download: %v", err)
|
|
||||||
}
|
|
||||||
b, err := os.ReadFile(dst)
|
|
||||||
if err != nil || string(b) != "hello world" {
|
|
||||||
t.Fatalf("downloaded %q, %v", b, err)
|
|
||||||
}
|
|
||||||
if err := s.Download(ctx, k("ud/missing.txt"), filepath.Join(dir, "m.txt")); !errors.Is(err, ErrNotFound) {
|
|
||||||
t.Fatalf("Download missing: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(filepath.Join(dir, "m.txt")); !os.IsNotExist(err) {
|
|
||||||
t.Fatal("Download of missing key left a local file")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,396 +0,0 @@
|
|||||||
/*
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2024 Jonas Kaninda
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package go_storage
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/tls"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/textproto"
|
|
||||||
"path"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/jlaffaye/ftp"
|
|
||||||
)
|
|
||||||
|
|
||||||
// FTPStorage implements Storage interface for FTP (optionally explicit FTPS).
|
|
||||||
//
|
|
||||||
// FTP uses a single control connection, so all operations are serialised by a
|
|
||||||
// mutex. The body returned by Get holds that connection until it is closed:
|
|
||||||
// a Get body MUST be closed before any other call on the same FTPStorage,
|
|
||||||
// otherwise that call blocks.
|
|
||||||
type FTPStorage struct {
|
|
||||||
host string
|
|
||||||
port string
|
|
||||||
baseDir string
|
|
||||||
|
|
||||||
mu sync.Mutex
|
|
||||||
client *ftp.ServerConn
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewFTPStorage creates a new FTP storage instance.
|
|
||||||
func NewFTPStorage(config Config) (*FTPStorage, error) {
|
|
||||||
if err := validateConfig(config, map[string]string{
|
|
||||||
"host": config.FTPHost,
|
|
||||||
}); err != nil {
|
|
||||||
return nil, fmt.Errorf("ftp: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
port := config.FTPPort
|
|
||||||
if port == "" {
|
|
||||||
port = "21"
|
|
||||||
}
|
|
||||||
hostPort := net.JoinHostPort(config.FTPHost, port)
|
|
||||||
|
|
||||||
opts := []ftp.DialOption{ftp.DialWithTimeout(30 * time.Second)}
|
|
||||||
if config.FTPTLS {
|
|
||||||
opts = append(opts, ftp.DialWithExplicitTLS(&tls.Config{
|
|
||||||
ServerName: config.FTPHost,
|
|
||||||
MinVersion: tls.VersionTLS12,
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
conn, err := ftp.Dial(hostPort, opts...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("ftp: failed to connect to %s: %w", hostPort, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := conn.Login(config.FTPUsername, config.FTPPassword); err != nil {
|
|
||||||
_ = conn.Quit()
|
|
||||||
return nil, fmt.Errorf("ftp: login failed: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
baseDir := normalizePathSeparators(config.FTPDirectory)
|
|
||||||
if baseDir == "" {
|
|
||||||
if wd, err := conn.CurrentDir(); err == nil && wd != "" {
|
|
||||||
baseDir = wd
|
|
||||||
} else {
|
|
||||||
baseDir = "/"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return &FTPStorage{
|
|
||||||
host: config.FTPHost,
|
|
||||||
port: port,
|
|
||||||
baseDir: path.Clean(baseDir),
|
|
||||||
client: conn,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *FTPStorage) resolve(key string) (string, string, error) {
|
|
||||||
k, err := cleanKey(key)
|
|
||||||
if err != nil {
|
|
||||||
return "", "", fmt.Errorf("ftp: %w", err)
|
|
||||||
}
|
|
||||||
return k, path.Join(f.baseDir, k), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func ftpCode(err error) int {
|
|
||||||
var te *textproto.Error
|
|
||||||
if errors.As(err, &te) {
|
|
||||||
return te.Code
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func ftpNotFound(err error) bool {
|
|
||||||
return ftpCode(err) == ftp.StatusFileUnavailable
|
|
||||||
}
|
|
||||||
|
|
||||||
// Put streams r to <key>.<rand>.partial and renames it on success.
|
|
||||||
func (f *FTPStorage) Put(ctx context.Context, key string, r io.Reader, _ *PutOptions) (Item, error) {
|
|
||||||
k, full, err := f.resolve(key)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
|
|
||||||
f.ensureDirectory(path.Dir(full))
|
|
||||||
tmp := partialName(full)
|
|
||||||
err = f.client.Stor(tmp, ctxReader{ctx: ctx, r: r})
|
|
||||||
if err == nil {
|
|
||||||
err = ctx.Err()
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
_ = f.client.Delete(tmp)
|
|
||||||
return Item{}, fmt.Errorf("ftp: failed to upload %s: %w", k, err)
|
|
||||||
}
|
|
||||||
if err := f.client.Rename(tmp, full); err != nil {
|
|
||||||
// Some servers refuse to rename onto an existing file.
|
|
||||||
_ = f.client.Delete(full)
|
|
||||||
if err := f.client.Rename(tmp, full); err != nil {
|
|
||||||
_ = f.client.Delete(tmp)
|
|
||||||
return Item{}, fmt.Errorf("ftp: failed to rename %s: %w", k, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return f.stat(k, full)
|
|
||||||
}
|
|
||||||
|
|
||||||
// PutIf is not supported on FTP: it always returns ErrConditionNotSupported.
|
|
||||||
// Callers should fall back to a lock-based protocol.
|
|
||||||
func (f *FTPStorage) PutIf(_ context.Context, key string, _ io.Reader, _ Condition, _ *PutOptions) (Item, error) {
|
|
||||||
return Item{}, fmt.Errorf("ftp: conditional put of %s: %w", key, ErrConditionNotSupported)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ftpBody releases the connection lock when closed.
|
|
||||||
type ftpBody struct {
|
|
||||||
ctx context.Context
|
|
||||||
resp *ftp.Response
|
|
||||||
once sync.Once
|
|
||||||
mu *sync.Mutex
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *ftpBody) Read(p []byte) (int, error) {
|
|
||||||
if err := b.ctx.Err(); err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
return b.resp.Read(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *ftpBody) Close() error {
|
|
||||||
var err error
|
|
||||||
b.once.Do(func() {
|
|
||||||
err = b.resp.Close()
|
|
||||||
b.mu.Unlock()
|
|
||||||
})
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get opens key for streaming reads. The returned body holds the FTP control
|
|
||||||
// connection: close it before issuing any other call on this storage.
|
|
||||||
func (f *FTPStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
|
|
||||||
k, full, err := f.resolve(key)
|
|
||||||
if err != nil {
|
|
||||||
return nil, Item{}, err
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, Item{}, err
|
|
||||||
}
|
|
||||||
f.mu.Lock()
|
|
||||||
it, err := f.stat(k, full)
|
|
||||||
if err != nil {
|
|
||||||
f.mu.Unlock()
|
|
||||||
return nil, Item{}, err
|
|
||||||
}
|
|
||||||
if it.IsDirectory {
|
|
||||||
f.mu.Unlock()
|
|
||||||
return nil, Item{}, fmt.Errorf("ftp: %s is a directory", k)
|
|
||||||
}
|
|
||||||
resp, err := f.client.Retr(full)
|
|
||||||
if err != nil {
|
|
||||||
f.mu.Unlock()
|
|
||||||
if ftpNotFound(err) {
|
|
||||||
return nil, Item{}, fmt.Errorf("ftp: get %s: %w", k, ErrNotFound)
|
|
||||||
}
|
|
||||||
return nil, Item{}, fmt.Errorf("ftp: failed to retrieve %s: %w", k, err)
|
|
||||||
}
|
|
||||||
return &ftpBody{ctx: ctx, resp: resp, mu: &f.mu}, it, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stat returns metadata for key (from a listing of its parent directory).
|
|
||||||
func (f *FTPStorage) Stat(ctx context.Context, key string) (Item, error) {
|
|
||||||
k, full, err := f.resolve(key)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
return f.stat(k, full)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *FTPStorage) stat(key, full string) (Item, error) {
|
|
||||||
entries, err := f.client.List(path.Dir(full))
|
|
||||||
if err != nil {
|
|
||||||
if ftpNotFound(err) {
|
|
||||||
return Item{}, fmt.Errorf("ftp: stat %s: %w", key, ErrNotFound)
|
|
||||||
}
|
|
||||||
return Item{}, fmt.Errorf("ftp: stat %s: %w", key, err)
|
|
||||||
}
|
|
||||||
name := path.Base(full)
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.Name == name {
|
|
||||||
return ftpItem(key, e), nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Item{}, fmt.Errorf("ftp: stat %s: %w", key, ErrNotFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
func ftpItem(key string, e *ftp.Entry) Item {
|
|
||||||
it := Item{Key: key, ModifiedTime: e.Time, IsDirectory: e.Type == ftp.EntryTypeFolder}
|
|
||||||
if !it.IsDirectory {
|
|
||||||
it.Size = int64(e.Size) //nolint:gosec // sizes fit in int64
|
|
||||||
}
|
|
||||||
return it
|
|
||||||
}
|
|
||||||
|
|
||||||
// Upload uploads a file via FTP.
|
|
||||||
func (f *FTPStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
|
||||||
return uploadFile(ctx, f, localPath, remotePath)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Download downloads a file via FTP.
|
|
||||||
func (f *FTPStorage) Download(ctx context.Context, remotePath string, localPath string) error {
|
|
||||||
return downloadFile(ctx, f, remotePath, localPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
// List lists files and directories (recursively) whose key starts with prefix.
|
|
||||||
func (f *FTPStorage) List(ctx context.Context, prefix string) ([]Item, error) {
|
|
||||||
p, err := cleanPrefix(prefix)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("ftp: %w", err)
|
|
||||||
}
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
|
|
||||||
items := make([]Item, 0)
|
|
||||||
startRel := ""
|
|
||||||
if i := strings.LastIndex(p, "/"); i >= 0 {
|
|
||||||
startRel = p[:i]
|
|
||||||
}
|
|
||||||
if err := f.walk(ctx, startRel, p, &items); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return items, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *FTPStorage) walk(ctx context.Context, rel, prefix string, items *[]Item) error {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
entries, err := f.client.List(path.Join(f.baseDir, rel))
|
|
||||||
if err != nil {
|
|
||||||
if ftpNotFound(err) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return fmt.Errorf("ftp: failed to list directory %s: %w", rel, err)
|
|
||||||
}
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.Name == "." || e.Name == ".." || e.Type == ftp.EntryTypeLink {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
key := path.Join(rel, e.Name)
|
|
||||||
isDir := e.Type == ftp.EntryTypeFolder
|
|
||||||
if strings.HasPrefix(key, prefix) {
|
|
||||||
*items = append(*items, ftpItem(key, e))
|
|
||||||
} else if !isDir || !strings.HasPrefix(prefix, key+"/") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if isDir {
|
|
||||||
if err := f.walk(ctx, key, prefix, items); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListDir lists the files directly inside dir.
|
|
||||||
func (f *FTPStorage) ListDir(ctx context.Context, dir string) ([]Item, error) {
|
|
||||||
d, err := cleanDir(dir)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("ftp: %w", err)
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
entries, err := f.client.List(path.Join(f.baseDir, d))
|
|
||||||
if err != nil {
|
|
||||||
if ftpNotFound(err) {
|
|
||||||
return []Item{}, nil
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("ftp: failed to list directory %s: %w", dir, err)
|
|
||||||
}
|
|
||||||
items := make([]Item, 0, len(entries))
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.Type != ftp.EntryTypeFile {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
items = append(items, ftpItem(path.Join(d, e.Name), e))
|
|
||||||
}
|
|
||||||
return items, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove deletes a file via FTP.
|
|
||||||
func (f *FTPStorage) Remove(ctx context.Context, remotePath string) error {
|
|
||||||
k, full, err := f.resolve(remotePath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
if err := f.client.Delete(full); err != nil {
|
|
||||||
if ftpNotFound(err) {
|
|
||||||
return fmt.Errorf("ftp: remove %s: %w", k, ErrNotFound)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("ftp: failed to remove %s: %w", k, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close releases resources.
|
|
||||||
func (f *FTPStorage) Close() error {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
if err := f.client.Quit(); err != nil {
|
|
||||||
return fmt.Errorf("ftp: failed to close connection: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ensureDirectory creates a directory path if it doesn't exist (errors are
|
|
||||||
// ignored: the directory usually exists already, and a real failure surfaces
|
|
||||||
// on the following STOR).
|
|
||||||
func (f *FTPStorage) ensureDirectory(dir string) {
|
|
||||||
if dir == "." || dir == "/" || dir == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
current := ""
|
|
||||||
if strings.HasPrefix(dir, "/") {
|
|
||||||
current = "/"
|
|
||||||
}
|
|
||||||
for _, part := range strings.Split(dir, "/") {
|
|
||||||
if part == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
current = path.Join(current, part)
|
|
||||||
_ = f.client.MakeDir(current)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,297 +0,0 @@
|
|||||||
/*
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2024 Jonas Kaninda
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package go_storage
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"cloud.google.com/go/storage"
|
|
||||||
"google.golang.org/api/googleapi"
|
|
||||||
"google.golang.org/api/iterator"
|
|
||||||
"google.golang.org/api/option"
|
|
||||||
)
|
|
||||||
|
|
||||||
// GCSStorage implements Storage interface for Google Cloud Storage.
|
|
||||||
type GCSStorage struct {
|
|
||||||
client *storage.Client
|
|
||||||
bucket string
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewGCSStorage creates a new Google Cloud Storage instance.
|
|
||||||
//
|
|
||||||
// Credentials: GCSCredentialsJSON, else GCSCredentialsFile, else Application
|
|
||||||
// Default Credentials. The STORAGE_EMULATOR_HOST environment variable is
|
|
||||||
// honoured by the Google client library itself.
|
|
||||||
func NewGCSStorage(config Config) (*GCSStorage, error) {
|
|
||||||
if err := validateConfig(config, map[string]string{
|
|
||||||
"bucket": config.GCSBucket,
|
|
||||||
}); err != nil {
|
|
||||||
return nil, fmt.Errorf("gcs: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
options := make([]option.ClientOption, 0, 2)
|
|
||||||
if config.GCSEndpoint != "" {
|
|
||||||
options = append(options, option.WithEndpoint(config.GCSEndpoint))
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case config.GCSCredentialsJSON != "":
|
|
||||||
data := []byte(config.GCSCredentialsJSON)
|
|
||||||
options = append(options, option.WithAuthCredentialsJSON(gcsCredType(data), data))
|
|
||||||
case config.GCSCredentialsFile != "":
|
|
||||||
data, err := os.ReadFile(config.GCSCredentialsFile)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("gcs: failed to read credentials file: %w", err)
|
|
||||||
}
|
|
||||||
options = append(options, option.WithAuthCredentialsFile(gcsCredType(data), config.GCSCredentialsFile))
|
|
||||||
}
|
|
||||||
return newGCSStorage(context.Background(), config.GCSBucket, options...)
|
|
||||||
}
|
|
||||||
|
|
||||||
func newGCSStorage(ctx context.Context, bucket string, options ...option.ClientOption) (*GCSStorage, error) {
|
|
||||||
client, err := storage.NewClient(ctx, options...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("gcs: failed to create client: %w", err)
|
|
||||||
}
|
|
||||||
return &GCSStorage{client: client, bucket: bucket}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// gcsCredType detects the credential type from the JSON "type" field.
|
|
||||||
func gcsCredType(data []byte) option.CredentialsType {
|
|
||||||
var v struct {
|
|
||||||
Type string `json:"type"`
|
|
||||||
}
|
|
||||||
_ = json.Unmarshal(data, &v)
|
|
||||||
switch v.Type {
|
|
||||||
case "authorized_user":
|
|
||||||
return option.AuthorizedUser
|
|
||||||
case "impersonated_service_account":
|
|
||||||
return option.ImpersonatedServiceAccount
|
|
||||||
case "external_account":
|
|
||||||
return option.ExternalAccount
|
|
||||||
default:
|
|
||||||
return option.ServiceAccount
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Client returns the underlying GCS client.
|
|
||||||
func (g *GCSStorage) Client() *storage.Client { return g.client }
|
|
||||||
|
|
||||||
func gcsItem(attrs *storage.ObjectAttrs) Item {
|
|
||||||
return Item{
|
|
||||||
Key: attrs.Name,
|
|
||||||
ModifiedTime: attrs.Updated,
|
|
||||||
Size: attrs.Size,
|
|
||||||
ETag: strconv.FormatInt(attrs.Generation, 10),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *GCSStorage) put(ctx context.Context, key string, r io.Reader, opts *PutOptions, cond *storage.Conditions) (Item, error) {
|
|
||||||
k, err := cleanKey(key)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, fmt.Errorf("gcs: %w", err)
|
|
||||||
}
|
|
||||||
obj := g.client.Bucket(g.bucket).Object(k)
|
|
||||||
if cond != nil {
|
|
||||||
obj = obj.If(*cond)
|
|
||||||
}
|
|
||||||
// A child context: cancelling it before Close aborts the upload so the
|
|
||||||
// object is never finalised.
|
|
||||||
wctx, cancel := context.WithCancel(ctx)
|
|
||||||
defer cancel()
|
|
||||||
w := obj.NewWriter(wctx)
|
|
||||||
if opts != nil {
|
|
||||||
w.ContentType = opts.ContentType
|
|
||||||
if len(opts.Metadata) > 0 {
|
|
||||||
w.Metadata = opts.Metadata
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, err := io.Copy(w, ctxReader{ctx: ctx, r: r}); err != nil {
|
|
||||||
cancel()
|
|
||||||
_ = w.Close()
|
|
||||||
if cond != nil && isGCSPrecondition(err) {
|
|
||||||
return Item{}, fmt.Errorf("gcs: conditional put of %s: %w", k, ErrPreconditionFailed)
|
|
||||||
}
|
|
||||||
return Item{}, fmt.Errorf("gcs: failed to upload %s: %w", k, err)
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
cancel()
|
|
||||||
_ = w.Close()
|
|
||||||
return Item{}, fmt.Errorf("gcs: failed to upload %s: %w", k, err)
|
|
||||||
}
|
|
||||||
if err := w.Close(); err != nil {
|
|
||||||
if cond != nil && isGCSPrecondition(err) {
|
|
||||||
return Item{}, fmt.Errorf("gcs: conditional put of %s: %w", k, ErrPreconditionFailed)
|
|
||||||
}
|
|
||||||
return Item{}, fmt.Errorf("gcs: failed to finalize upload of %s: %w", k, err)
|
|
||||||
}
|
|
||||||
return gcsItem(w.Attrs()), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Put streams r to key.
|
|
||||||
func (g *GCSStorage) Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error) {
|
|
||||||
return g.put(ctx, key, r, opts, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
// PutIf writes r to key with a generation precondition (IfMatch is the
|
|
||||||
// generation as returned in Item.ETag).
|
|
||||||
func (g *GCSStorage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error) {
|
|
||||||
if err := validateCondition(cond); err != nil {
|
|
||||||
return Item{}, fmt.Errorf("gcs: %w", err)
|
|
||||||
}
|
|
||||||
c := storage.Conditions{}
|
|
||||||
switch {
|
|
||||||
case cond.IfNoneMatch:
|
|
||||||
c.DoesNotExist = true
|
|
||||||
default:
|
|
||||||
gen, err := strconv.ParseInt(cond.IfMatch, 10, 64)
|
|
||||||
if err != nil || gen <= 0 {
|
|
||||||
return Item{}, fmt.Errorf("gcs: invalid generation %q: %w", cond.IfMatch, ErrPreconditionFailed)
|
|
||||||
}
|
|
||||||
c.GenerationMatch = gen
|
|
||||||
}
|
|
||||||
return g.put(ctx, key, r, opts, &c)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get streams the object body.
|
|
||||||
func (g *GCSStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
|
|
||||||
k, err := cleanKey(key)
|
|
||||||
if err != nil {
|
|
||||||
return nil, Item{}, fmt.Errorf("gcs: %w", err)
|
|
||||||
}
|
|
||||||
rd, err := g.client.Bucket(g.bucket).Object(k).NewReader(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, Item{}, g.mapErr("get", k, err)
|
|
||||||
}
|
|
||||||
return rd, Item{
|
|
||||||
Key: k,
|
|
||||||
Size: rd.Attrs.Size,
|
|
||||||
ModifiedTime: rd.Attrs.LastModified,
|
|
||||||
ETag: strconv.FormatInt(rd.Attrs.Generation, 10),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stat returns object attributes.
|
|
||||||
func (g *GCSStorage) Stat(ctx context.Context, key string) (Item, error) {
|
|
||||||
k, err := cleanKey(key)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, fmt.Errorf("gcs: %w", err)
|
|
||||||
}
|
|
||||||
attrs, err := g.client.Bucket(g.bucket).Object(k).Attrs(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, g.mapErr("stat", k, err)
|
|
||||||
}
|
|
||||||
return gcsItem(attrs), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Upload uploads a file to GCS.
|
|
||||||
func (g *GCSStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
|
||||||
if err := uploadFile(ctx, g, localPath, remotePath); err != nil {
|
|
||||||
return fmt.Errorf("gcs: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Download downloads a file from GCS.
|
|
||||||
func (g *GCSStorage) Download(ctx context.Context, remotePath string, localPath string) error {
|
|
||||||
if err := downloadFile(ctx, g, remotePath, localPath); err != nil {
|
|
||||||
return fmt.Errorf("gcs: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// List lists objects (recursively) with the given prefix.
|
|
||||||
func (g *GCSStorage) List(ctx context.Context, prefix string) ([]Item, error) {
|
|
||||||
p, err := cleanPrefix(prefix)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("gcs: %w", err)
|
|
||||||
}
|
|
||||||
return g.list(ctx, &storage.Query{Prefix: p})
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListDir lists the objects directly under dir.
|
|
||||||
func (g *GCSStorage) ListDir(ctx context.Context, dir string) ([]Item, error) {
|
|
||||||
d, err := cleanDir(dir)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("gcs: %w", err)
|
|
||||||
}
|
|
||||||
return g.list(ctx, &storage.Query{Prefix: dirPrefix(d), Delimiter: "/"})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *GCSStorage) list(ctx context.Context, q *storage.Query) ([]Item, error) {
|
|
||||||
items := make([]Item, 0)
|
|
||||||
it := g.client.Bucket(g.bucket).Objects(ctx, q)
|
|
||||||
for {
|
|
||||||
attrs, err := it.Next()
|
|
||||||
if errors.Is(err, iterator.Done) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("gcs: failed to list objects in bucket %s: %w", g.bucket, err)
|
|
||||||
}
|
|
||||||
if attrs.Name == "" || (q.Delimiter != "" && strings.HasSuffix(attrs.Name, "/")) {
|
|
||||||
continue // synthetic prefix or directory marker
|
|
||||||
}
|
|
||||||
items = append(items, gcsItem(attrs))
|
|
||||||
}
|
|
||||||
return items, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove deletes an object.
|
|
||||||
func (g *GCSStorage) Remove(ctx context.Context, remotePath string) error {
|
|
||||||
k, err := cleanKey(remotePath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("gcs: %w", err)
|
|
||||||
}
|
|
||||||
if err := g.client.Bucket(g.bucket).Object(k).Delete(ctx); err != nil {
|
|
||||||
return g.mapErr("remove", k, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close releases resources.
|
|
||||||
func (g *GCSStorage) Close() error {
|
|
||||||
return g.client.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *GCSStorage) mapErr(op, key string, err error) error {
|
|
||||||
var ge *googleapi.Error
|
|
||||||
if errors.Is(err, storage.ErrObjectNotExist) || (errors.As(err, &ge) && ge.Code == http.StatusNotFound) {
|
|
||||||
return fmt.Errorf("gcs: %s %s in bucket %s: %w", op, key, g.bucket, ErrNotFound)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("gcs: %s %s in bucket %s: %w", op, key, g.bucket, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func isGCSPrecondition(err error) bool {
|
|
||||||
var ge *googleapi.Error
|
|
||||||
return errors.As(err, &ge) && (ge.Code == http.StatusPreconditionFailed || ge.Code == http.StatusConflict)
|
|
||||||
}
|
|
||||||
@@ -1,86 +1,23 @@
|
|||||||
module github.com/jkaninda/go-storage
|
module github.com/jkaninda/go-storage
|
||||||
|
|
||||||
go 1.25
|
go 1.22.0
|
||||||
|
toolchain go1.24.1
|
||||||
|
|
||||||
require (
|
require (
|
||||||
cloud.google.com/go/storage v1.60.0
|
|
||||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0
|
|
||||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
|
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
|
||||||
github.com/aws/aws-sdk-go v1.55.7
|
github.com/aws/aws-sdk-go v1.55.7
|
||||||
github.com/aws/aws-sdk-go-v2 v1.47.3
|
|
||||||
github.com/aws/aws-sdk-go-v2/config v1.33.4
|
|
||||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9
|
|
||||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.43
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3
|
|
||||||
github.com/aws/smithy-go v1.28.5
|
|
||||||
github.com/bramvdbogaerde/go-scp v1.5.0
|
github.com/bramvdbogaerde/go-scp v1.5.0
|
||||||
github.com/jlaffaye/ftp v0.2.0
|
github.com/jlaffaye/ftp v0.2.0
|
||||||
github.com/pkg/sftp v1.13.10
|
golang.org/x/crypto v0.37.0
|
||||||
golang.org/x/crypto v0.47.0
|
|
||||||
google.golang.org/api v0.267.0
|
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
cel.dev/expr v0.24.0 // indirect
|
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
|
||||||
cloud.google.com/go v0.123.0 // indirect
|
|
||||||
cloud.google.com/go/auth v0.18.1 // indirect
|
|
||||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
|
||||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
|
||||||
cloud.google.com/go/iam v1.5.3 // indirect
|
|
||||||
cloud.google.com/go/monitoring v1.24.3 // indirect
|
|
||||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
|
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
|
||||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
|
|
||||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
|
|
||||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
|
|
||||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 // indirect
|
|
||||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect
|
|
||||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect
|
|
||||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect
|
|
||||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect
|
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
|
||||||
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
|
|
||||||
github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
|
|
||||||
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
|
|
||||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
|
||||||
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
|
||||||
github.com/go-logr/logr v1.4.3 // indirect
|
|
||||||
github.com/go-logr/stdr v1.2.2 // indirect
|
|
||||||
github.com/google/s2a-go v0.1.9 // indirect
|
|
||||||
github.com/google/uuid v1.6.0 // indirect
|
|
||||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
|
|
||||||
github.com/googleapis/gax-go/v2 v2.17.0 // indirect
|
|
||||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||||
github.com/kr/fs v0.1.0 // indirect
|
golang.org/x/net v0.39.0 // indirect
|
||||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
|
golang.org/x/sys v0.32.0 // indirect
|
||||||
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
|
golang.org/x/text v0.24.0 // indirect
|
||||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
|
||||||
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
|
|
||||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
|
|
||||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
|
|
||||||
go.opentelemetry.io/otel v1.39.0 // indirect
|
|
||||||
go.opentelemetry.io/otel/metric v1.39.0 // indirect
|
|
||||||
go.opentelemetry.io/otel/sdk v1.39.0 // indirect
|
|
||||||
go.opentelemetry.io/otel/sdk/metric v1.39.0 // indirect
|
|
||||||
go.opentelemetry.io/otel/trace v1.39.0 // indirect
|
|
||||||
golang.org/x/net v0.49.0 // indirect
|
|
||||||
golang.org/x/oauth2 v0.35.0 // indirect
|
|
||||||
golang.org/x/sync v0.19.0 // indirect
|
|
||||||
golang.org/x/sys v0.40.0 // indirect
|
|
||||||
golang.org/x/text v0.33.0 // indirect
|
|
||||||
golang.org/x/time v0.14.0 // indirect
|
|
||||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
|
|
||||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect
|
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
|
|
||||||
google.golang.org/grpc v1.78.0 // indirect
|
|
||||||
google.golang.org/protobuf v1.36.11 // indirect
|
|
||||||
)
|
)
|
||||||
@@ -1,25 +1,3 @@
|
|||||||
cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
|
|
||||||
cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
|
|
||||||
cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
|
|
||||||
cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
|
|
||||||
cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
|
|
||||||
cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
|
|
||||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
|
||||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
|
||||||
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
|
|
||||||
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
|
|
||||||
cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc=
|
|
||||||
cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU=
|
|
||||||
cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
|
|
||||||
cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
|
|
||||||
cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8=
|
|
||||||
cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk=
|
|
||||||
cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE=
|
|
||||||
cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI=
|
|
||||||
cloud.google.com/go/storage v1.60.0 h1:oBfZrSOCimggVNz9Y/bXY35uUcts7OViubeddTTVzQ8=
|
|
||||||
cloud.google.com/go/storage v1.60.0/go.mod h1:q+5196hXfejkctrnx+VYU8RKQr/L3c0cBIlrjmiAKE0=
|
|
||||||
cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U=
|
|
||||||
cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s=
|
|
||||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=
|
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=
|
||||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
|
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
|
||||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g=
|
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g=
|
||||||
@@ -32,96 +10,17 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSy
|
|||||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8=
|
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8=
|
||||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
|
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
|
||||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=
|
|
||||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
|
|
||||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8=
|
|
||||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc=
|
|
||||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ=
|
|
||||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA=
|
|
||||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk=
|
|
||||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc=
|
|
||||||
github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE=
|
github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE=
|
||||||
github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
|
github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
|
||||||
github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk=
|
|
||||||
github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A=
|
|
||||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 h1:oiEmPz/YMMeh/JvAE8xSxNCIsvWE6lpFe4ZPlyFykQI=
|
|
||||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22/go.mod h1:QWUDuRi9PFjBHIU2iH1Pkldbe7zqLfXJC4wWLbF8nuQ=
|
|
||||||
github.com/aws/aws-sdk-go-v2/config v1.33.4 h1:FzvkXKSzwqHni4U7nDigHg4jjtqMpVUuHgmZfSoJVQ0=
|
|
||||||
github.com/aws/aws-sdk-go-v2/config v1.33.4/go.mod h1:VZqGZnZsCWVfK/iGPptJIyNIX3XEX6iQU2Rel4sLrr8=
|
|
||||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0=
|
|
||||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I=
|
|
||||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk=
|
|
||||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8=
|
|
||||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.43 h1:Fhx4uwBshQF0+jmhV/FKDM4LPj7L9iUy5n3MvLxLy00=
|
|
||||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.43/go.mod h1:4Tyfc1eIPEyrAMysLCVkcYL1Hm/hkGWbElLo8pejHPo=
|
|
||||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0=
|
|
||||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE=
|
|
||||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM=
|
|
||||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4=
|
|
||||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4=
|
|
||||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 h1:CwvFlKw0uUScfgOMflXSRZa+ayXgOf6wk7lhyMemQYU=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7/go.mod h1:4SbD8BB+p+QEnG0aX6DoWwrtk9SufyYXRsUv0nve8LE=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 h1:NyeuyXkZL476uoAowNActm4XJO1QPKf7aassgXHpiqA=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3/go.mod h1:GfoBeKMDJecgRUsGf/0RGdx0NJAh81ObTd922h8pQys=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 h1:OKEfbf2tcfsF2pBk/S9QqNpo6alaAtiBs4hysCsBjPw=
|
|
||||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.4/go.mod h1:mCd6qGo9b8hTtfjgsmNkFlzwjyce4owd7HDVFnCLXhM=
|
|
||||||
github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k=
|
|
||||||
github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
|
||||||
github.com/bramvdbogaerde/go-scp v1.5.0 h1:a9BinAjTfQh273eh7vd3qUgmBC+bx+3TRDtkZWmIpzM=
|
github.com/bramvdbogaerde/go-scp v1.5.0 h1:a9BinAjTfQh273eh7vd3qUgmBC+bx+3TRDtkZWmIpzM=
|
||||||
github.com/bramvdbogaerde/go-scp v1.5.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ=
|
github.com/bramvdbogaerde/go-scp v1.5.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ=
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
|
||||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
|
||||||
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0=
|
|
||||||
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4=
|
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM=
|
|
||||||
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs=
|
|
||||||
github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo=
|
|
||||||
github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs=
|
|
||||||
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
|
|
||||||
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
|
|
||||||
github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8=
|
|
||||||
github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU=
|
|
||||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
|
||||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
|
||||||
github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
|
|
||||||
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
|
||||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
|
||||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
|
||||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
|
||||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
|
||||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
|
||||||
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
|
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
|
||||||
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
|
||||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
|
||||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
|
||||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
|
||||||
github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
|
|
||||||
github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
|
|
||||||
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
|
|
||||||
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
|
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
|
|
||||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
|
|
||||||
github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
|
|
||||||
github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
|
|
||||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||||
@@ -133,74 +32,25 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
|
|||||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||||
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
|
|
||||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
|
||||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
||||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
||||||
github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA=
|
|
||||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
|
|
||||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
|
||||||
github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo=
|
|
||||||
github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
|
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
|
||||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
|
||||||
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs=
|
golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
|
||||||
go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts=
|
golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
|
||||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo=
|
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
|
||||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ=
|
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
|
golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o=
|
||||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
|
golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw=
|
||||||
go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
|
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
|
||||||
go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=
|
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
|
||||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8OkCfD1j3/ER79rUuTYmCvlXBKeYL8=
|
|
||||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
|
|
||||||
go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0=
|
|
||||||
go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=
|
|
||||||
go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=
|
|
||||||
go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE=
|
|
||||||
go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8=
|
|
||||||
go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=
|
|
||||||
go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
|
|
||||||
go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
|
|
||||||
golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
|
|
||||||
golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
|
|
||||||
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
|
|
||||||
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
|
|
||||||
golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
|
|
||||||
golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
|
||||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
|
||||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
|
||||||
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
|
|
||||||
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
|
||||||
golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
|
|
||||||
golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
|
|
||||||
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
|
|
||||||
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
|
|
||||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
|
||||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
|
||||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
|
||||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
|
||||||
google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
|
|
||||||
google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
|
|
||||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
|
|
||||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
|
|
||||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
|
|
||||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
|
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
|
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
|
|
||||||
google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
|
|
||||||
google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
|
|
||||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
|
||||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
||||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
|||||||
@@ -1,263 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
/*
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2024 Jonas Kaninda
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package go_storage
|
|
||||||
|
|
||||||
// Integration tests against real services started with the docker CLI.
|
|
||||||
//
|
|
||||||
// go test -tags integration ./...
|
|
||||||
//
|
|
||||||
// Tests skip when docker is unavailable. Images can be overridden with
|
|
||||||
// GO_STORAGE_MINIO_IMAGE, GO_STORAGE_AZURITE_IMAGE, GO_STORAGE_FAKEGCS_IMAGE
|
|
||||||
// and GO_STORAGE_FTP_IMAGE.
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"math/rand"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/storage"
|
|
||||||
"github.com/aws/aws-sdk-go-v2/aws"
|
|
||||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
||||||
"google.golang.org/api/option"
|
|
||||||
)
|
|
||||||
|
|
||||||
func image(env, def string) string {
|
|
||||||
if v := os.Getenv(env); v != "" {
|
|
||||||
return v
|
|
||||||
}
|
|
||||||
return def
|
|
||||||
}
|
|
||||||
|
|
||||||
func requireDocker(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
if err := exec.CommandContext(ctx, "docker", "info").Run(); err != nil {
|
|
||||||
t.Skipf("docker not available: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func docker(t *testing.T, args ...string) string {
|
|
||||||
t.Helper()
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
||||||
defer cancel()
|
|
||||||
out, err := exec.CommandContext(ctx, "docker", args...).CombinedOutput()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("docker %s: %v\n%s", strings.Join(args, " "), err, out)
|
|
||||||
}
|
|
||||||
return strings.TrimSpace(string(out))
|
|
||||||
}
|
|
||||||
|
|
||||||
// runContainer starts a detached container and removes it at cleanup.
|
|
||||||
func runContainer(t *testing.T, args ...string) string {
|
|
||||||
t.Helper()
|
|
||||||
id := docker(t, append([]string{"run", "-d"}, args...)...)
|
|
||||||
t.Cleanup(func() {
|
|
||||||
if t.Failed() {
|
|
||||||
out, _ := exec.Command("docker", "logs", "--tail", "50", id).CombinedOutput()
|
|
||||||
t.Logf("container logs:\n%s", out)
|
|
||||||
}
|
|
||||||
_ = exec.Command("docker", "rm", "-f", "-v", id).Run()
|
|
||||||
})
|
|
||||||
return id
|
|
||||||
}
|
|
||||||
|
|
||||||
// hostPort returns the host port mapped to the container port.
|
|
||||||
func hostPort(t *testing.T, id, containerPort string) string {
|
|
||||||
t.Helper()
|
|
||||||
out := docker(t, "port", id, containerPort)
|
|
||||||
line := strings.Split(out, "\n")[0]
|
|
||||||
_, port, err := net.SplitHostPort(strings.TrimSpace(line))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parse docker port %q: %v", out, err)
|
|
||||||
}
|
|
||||||
return port
|
|
||||||
}
|
|
||||||
|
|
||||||
func freePort(t *testing.T) int {
|
|
||||||
t.Helper()
|
|
||||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer func() { _ = l.Close() }()
|
|
||||||
return l.Addr().(*net.TCPAddr).Port
|
|
||||||
}
|
|
||||||
|
|
||||||
const readyTimeout = 60 * time.Second
|
|
||||||
|
|
||||||
// eventually retries fn until it succeeds or readyTimeout expires.
|
|
||||||
func eventually(t *testing.T, what string, fn func() error) {
|
|
||||||
t.Helper()
|
|
||||||
deadline := time.Now().Add(readyTimeout)
|
|
||||||
var err error
|
|
||||||
for time.Now().Before(deadline) {
|
|
||||||
if err = fn(); err == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
time.Sleep(500 * time.Millisecond)
|
|
||||||
}
|
|
||||||
t.Fatalf("%s: %v", what, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIntegrationMinIO(t *testing.T) {
|
|
||||||
requireDocker(t)
|
|
||||||
id := runContainer(t, "-p", "127.0.0.1::9000",
|
|
||||||
"-e", "MINIO_ROOT_USER=minioadmin", "-e", "MINIO_ROOT_PASSWORD=minioadmin",
|
|
||||||
image("GO_STORAGE_MINIO_IMAGE", "pgsty/minio:latest"), "server", "/data")
|
|
||||||
port := hostPort(t, id, "9000/tcp")
|
|
||||||
|
|
||||||
s, err := NewS3Storage(Config{
|
|
||||||
S3Bucket: "conformance",
|
|
||||||
S3Endpoint: "127.0.0.1:" + port,
|
|
||||||
S3DisableTLS: true,
|
|
||||||
S3ForcePath: true,
|
|
||||||
S3KeyID: "minioadmin",
|
|
||||||
S3Secret: "minioadmin",
|
|
||||||
S3PartSize: MinS3PartSize, // 20 MiB stream => real multipart upload
|
|
||||||
S3Concurrency: 2,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ctx := context.Background()
|
|
||||||
eventually(t, "create bucket", func() error {
|
|
||||||
_, err := s.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String("conformance")})
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
runConformance(t, s)
|
|
||||||
|
|
||||||
// Failed multipart uploads must have been aborted.
|
|
||||||
out, err := s.client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{Bucket: aws.String("conformance")})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(out.Uploads) != 0 {
|
|
||||||
t.Errorf("%d multipart uploads left behind", len(out.Uploads))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIntegrationAzurite(t *testing.T) {
|
|
||||||
requireDocker(t)
|
|
||||||
id := runContainer(t, "-p", "127.0.0.1::10000",
|
|
||||||
image("GO_STORAGE_AZURITE_IMAGE", "mcr.microsoft.com/azure-storage/azurite:latest"),
|
|
||||||
"azurite-blob", "--blobHost", "0.0.0.0", "--skipApiVersionCheck", "--loose")
|
|
||||||
port := hostPort(t, id, "10000/tcp")
|
|
||||||
|
|
||||||
s, err := NewAzureStorage(Config{
|
|
||||||
AzureAccount: "devstoreaccount1",
|
|
||||||
AzureKey: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
|
|
||||||
AzureContainer: "conformance",
|
|
||||||
AzureEndpoint: "http://127.0.0.1:" + port + "/devstoreaccount1",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
eventually(t, "create container", func() error {
|
|
||||||
_, err := s.client.CreateContainer(context.Background(), "conformance", nil)
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
runConformance(t, s)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIntegrationFakeGCS(t *testing.T) {
|
|
||||||
requireDocker(t)
|
|
||||||
port := freePort(t)
|
|
||||||
addr := "127.0.0.1:" + strconv.Itoa(port)
|
|
||||||
runContainer(t, "-p", fmt.Sprintf("%s:4443", addr),
|
|
||||||
image("GO_STORAGE_FAKEGCS_IMAGE", "fsouza/fake-gcs-server:latest"),
|
|
||||||
"-scheme", "http", "-port", "4443", "-public-host", addr, "-external-url", "http://"+addr, "-backend", "memory")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
s, err := newGCSStorage(ctx, "conformance",
|
|
||||||
option.WithEndpoint("http://"+addr+"/storage/v1/"),
|
|
||||||
option.WithoutAuthentication())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { _ = s.Close() })
|
|
||||||
eventually(t, "create bucket", func() error {
|
|
||||||
err := s.client.Bucket("conformance").Create(ctx, "test-project", &storage.BucketAttrs{})
|
|
||||||
var exists bool
|
|
||||||
if err != nil && strings.Contains(err.Error(), "409") {
|
|
||||||
exists = true
|
|
||||||
}
|
|
||||||
if exists {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
runConformance(t, s)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIntegrationFTP(t *testing.T) {
|
|
||||||
requireDocker(t)
|
|
||||||
// Passive ports must be identical inside and outside the container.
|
|
||||||
minPort := 30000 + rand.Intn(20000)
|
|
||||||
maxPort := minPort + 9
|
|
||||||
ctrlPort := freePort(t)
|
|
||||||
runContainer(t,
|
|
||||||
"-p", fmt.Sprintf("127.0.0.1:%d:21", ctrlPort),
|
|
||||||
"-p", fmt.Sprintf("127.0.0.1:%d-%d:%d-%d", minPort, maxPort, minPort, maxPort),
|
|
||||||
"-e", "USERS=backup|s3cret|/ftp/backup",
|
|
||||||
"-e", "ADDRESS=127.0.0.1",
|
|
||||||
"-e", "MIN_PORT="+strconv.Itoa(minPort), "-e", "MAX_PORT="+strconv.Itoa(maxPort),
|
|
||||||
image("GO_STORAGE_FTP_IMAGE", "delfer/alpine-ftp-server:latest"))
|
|
||||||
|
|
||||||
var s *FTPStorage
|
|
||||||
eventually(t, "connect ftp", func() error {
|
|
||||||
var err error
|
|
||||||
s, err = NewFTPStorage(Config{
|
|
||||||
FTPHost: "127.0.0.1",
|
|
||||||
FTPPort: strconv.Itoa(ctrlPort),
|
|
||||||
FTPUsername: "backup",
|
|
||||||
FTPPassword: "s3cret",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := s.ListDir(context.Background(), ""); err != nil {
|
|
||||||
_ = s.Close()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
t.Cleanup(func() { _ = s.Close() })
|
|
||||||
runConformance(t, s)
|
|
||||||
|
|
||||||
if _, err := s.PutIf(context.Background(), "x", strings.NewReader("x"), Condition{IfNoneMatch: true}, nil); !errors.Is(err, ErrConditionNotSupported) {
|
|
||||||
t.Fatalf("FTP PutIf: %v, want ErrConditionNotSupported", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,433 +0,0 @@
|
|||||||
/*
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2024 Jonas Kaninda
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package go_storage
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"io/fs"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// LocalStorage implements the Storage interface using the local filesystem.
|
|
||||||
// It is safe for concurrent use.
|
|
||||||
type LocalStorage struct {
|
|
||||||
basePath string
|
|
||||||
}
|
|
||||||
|
|
||||||
// Lock tuning for PutIf / Put on the local backend.
|
|
||||||
const (
|
|
||||||
localLockTimeout = 10 * time.Second
|
|
||||||
localLockRetry = 10 * time.Millisecond
|
|
||||||
// A lock older than this is considered stale (the holder crashed). The
|
|
||||||
// lock is only held around a stat + rename, never while streaming data.
|
|
||||||
localLockStale = 30 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
// NewLocalStorage creates a new LocalStorage instance with the given base directory.
|
|
||||||
// The base directory will be created if it doesn't exist.
|
|
||||||
func NewLocalStorage(basePath string) (*LocalStorage, error) {
|
|
||||||
absPath, err := filepath.Abs(basePath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("local: failed to resolve absolute path: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := os.MkdirAll(absPath, 0o755); err != nil {
|
|
||||||
return nil, fmt.Errorf("local: failed to create base directory: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &LocalStorage{basePath: absPath}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolvePath safely resolves a key to an absolute local path, ensuring it
|
|
||||||
// stays within the base directory.
|
|
||||||
func (s *LocalStorage) resolvePath(remotePath string) (string, error) {
|
|
||||||
key, err := cleanKey(remotePath)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("local: %w", err)
|
|
||||||
}
|
|
||||||
return s.join(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *LocalStorage) join(key string) (string, error) {
|
|
||||||
fullPath := filepath.Join(s.basePath, filepath.FromSlash(key))
|
|
||||||
rel, err := filepath.Rel(s.basePath, fullPath)
|
|
||||||
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
|
||||||
return "", fmt.Errorf("local: path traversal attempt detected: %s", key)
|
|
||||||
}
|
|
||||||
return fullPath, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// localETag derives the version token from mtime and size.
|
|
||||||
func localETag(fi fs.FileInfo) string {
|
|
||||||
return strconv.FormatInt(fi.ModTime().UnixNano(), 16) + "-" + strconv.FormatInt(fi.Size(), 16)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *LocalStorage) item(full string, fi fs.FileInfo) Item {
|
|
||||||
rel, err := filepath.Rel(s.basePath, full)
|
|
||||||
if err != nil {
|
|
||||||
rel = full
|
|
||||||
}
|
|
||||||
it := Item{
|
|
||||||
Key: filepath.ToSlash(rel),
|
|
||||||
ModifiedTime: fi.ModTime(),
|
|
||||||
IsDirectory: fi.IsDir(),
|
|
||||||
}
|
|
||||||
if !fi.IsDir() {
|
|
||||||
it.Size = fi.Size()
|
|
||||||
it.ETag = localETag(fi)
|
|
||||||
}
|
|
||||||
return it
|
|
||||||
}
|
|
||||||
|
|
||||||
// Upload uploads a local file to the storage with the given target name.
|
|
||||||
func (s *LocalStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
|
||||||
return uploadFile(ctx, s, localPath, remotePath)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Download downloads a file from storage to a local path.
|
|
||||||
func (s *LocalStorage) Download(ctx context.Context, remotePath string, localPath string) error {
|
|
||||||
return downloadFile(ctx, s, remotePath, localPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeTemp streams r into a new temporary file next to full and returns its path.
|
|
||||||
func writeTemp(ctx context.Context, full string, r io.Reader) (string, error) {
|
|
||||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
|
||||||
return "", fmt.Errorf("local: failed to create parent directories: %w", err)
|
|
||||||
}
|
|
||||||
tmp := partialName(full)
|
|
||||||
f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("local: failed to create temporary file: %w", err)
|
|
||||||
}
|
|
||||||
_, err = io.Copy(f, ctxReader{ctx: ctx, r: r})
|
|
||||||
if err == nil {
|
|
||||||
err = f.Sync()
|
|
||||||
}
|
|
||||||
if cerr := f.Close(); err == nil {
|
|
||||||
err = cerr
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
err = ctx.Err()
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
_ = os.Remove(tmp)
|
|
||||||
return "", fmt.Errorf("local: failed to write data: %w", err)
|
|
||||||
}
|
|
||||||
return tmp, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Put streams r to key via a temporary file that is renamed on success.
|
|
||||||
func (s *LocalStorage) Put(ctx context.Context, key string, r io.Reader, _ *PutOptions) (Item, error) {
|
|
||||||
return s.put(ctx, key, r, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
// PutIf writes r to key only if cond holds. It uses an exclusive lock file
|
|
||||||
// (<key>.lock) so it is atomic with respect to other PutIf/Put calls on the
|
|
||||||
// same filesystem.
|
|
||||||
func (s *LocalStorage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, _ *PutOptions) (Item, error) {
|
|
||||||
if err := validateCondition(cond); err != nil {
|
|
||||||
return Item{}, fmt.Errorf("local: %w", err)
|
|
||||||
}
|
|
||||||
return s.put(ctx, key, r, &cond)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *LocalStorage) put(ctx context.Context, key string, r io.Reader, cond *Condition) (Item, error) {
|
|
||||||
full, err := s.resolvePath(key)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
tmp, err := writeTemp(ctx, full, r)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
committed := false
|
|
||||||
defer func() {
|
|
||||||
if !committed {
|
|
||||||
_ = os.Remove(tmp)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
unlock, err := acquireLock(ctx, full+".lock")
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
defer unlock()
|
|
||||||
|
|
||||||
cur, statErr := os.Stat(full)
|
|
||||||
exists := statErr == nil
|
|
||||||
if statErr != nil && !errors.Is(statErr, fs.ErrNotExist) {
|
|
||||||
return Item{}, fmt.Errorf("local: failed to stat %s: %w", key, statErr)
|
|
||||||
}
|
|
||||||
if cond != nil {
|
|
||||||
switch {
|
|
||||||
case cond.IfNoneMatch && exists:
|
|
||||||
return Item{}, fmt.Errorf("local: %s already exists: %w", key, ErrPreconditionFailed)
|
|
||||||
case cond.IfMatch != "" && (!exists || localETag(cur) != cond.IfMatch):
|
|
||||||
return Item{}, fmt.Errorf("local: %s etag mismatch: %w", key, ErrPreconditionFailed)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if exists {
|
|
||||||
if err := bumpMtime(tmp, cur.ModTime()); err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
if err := os.Rename(tmp, full); err != nil {
|
|
||||||
return Item{}, fmt.Errorf("local: failed to rename temporary file: %w", err)
|
|
||||||
}
|
|
||||||
committed = true
|
|
||||||
fi, err := os.Stat(full)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, fmt.Errorf("local: failed to stat %s: %w", key, err)
|
|
||||||
}
|
|
||||||
return s.item(full, fi), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// bumpMtime makes sure the mtime of tmp is strictly after prev so that the
|
|
||||||
// ETag changes even on filesystems with coarse timestamps.
|
|
||||||
func bumpMtime(tmp string, prev time.Time) error {
|
|
||||||
for _, step := range []time.Duration{time.Microsecond, time.Millisecond, time.Second, 2 * time.Second} {
|
|
||||||
fi, err := os.Stat(tmp)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("local: failed to stat temporary file: %w", err)
|
|
||||||
}
|
|
||||||
if fi.ModTime().After(prev) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
t := prev.Add(step)
|
|
||||||
if err := os.Chtimes(tmp, t, t); err != nil {
|
|
||||||
return fmt.Errorf("local: failed to set mtime: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// acquireLock creates lockPath exclusively, retrying until localLockTimeout.
|
|
||||||
func acquireLock(ctx context.Context, lockPath string) (func(), error) {
|
|
||||||
deadline := time.Now().Add(localLockTimeout)
|
|
||||||
for {
|
|
||||||
f, err := os.OpenFile(lockPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
|
||||||
if err == nil {
|
|
||||||
_, _ = fmt.Fprintf(f, "%d\n", os.Getpid())
|
|
||||||
_ = f.Close()
|
|
||||||
return func() { _ = os.Remove(lockPath) }, nil
|
|
||||||
}
|
|
||||||
if !errors.Is(err, fs.ErrExist) {
|
|
||||||
return nil, fmt.Errorf("local: failed to create lock %s: %w", lockPath, err)
|
|
||||||
}
|
|
||||||
if fi, serr := os.Stat(lockPath); serr == nil && time.Since(fi.ModTime()) > localLockStale {
|
|
||||||
_ = os.Remove(lockPath)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
return nil, fmt.Errorf("local: timed out waiting for lock %s", lockPath)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
case <-time.After(localLockRetry):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get opens key for reading.
|
|
||||||
func (s *LocalStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, Item{}, err
|
|
||||||
}
|
|
||||||
full, err := s.resolvePath(key)
|
|
||||||
if err != nil {
|
|
||||||
return nil, Item{}, err
|
|
||||||
}
|
|
||||||
f, err := os.Open(full)
|
|
||||||
if err != nil {
|
|
||||||
return nil, Item{}, mapLocalErr(key, err)
|
|
||||||
}
|
|
||||||
fi, err := f.Stat()
|
|
||||||
if err != nil {
|
|
||||||
_ = f.Close()
|
|
||||||
return nil, Item{}, mapLocalErr(key, err)
|
|
||||||
}
|
|
||||||
if fi.IsDir() {
|
|
||||||
_ = f.Close()
|
|
||||||
return nil, Item{}, fmt.Errorf("local: %s is a directory", key)
|
|
||||||
}
|
|
||||||
return f, s.item(full, fi), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stat returns metadata for key.
|
|
||||||
func (s *LocalStorage) Stat(ctx context.Context, key string) (Item, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
full, err := s.resolvePath(key)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
fi, err := os.Stat(full)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, mapLocalErr(key, err)
|
|
||||||
}
|
|
||||||
return s.item(full, fi), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func mapLocalErr(key string, err error) error {
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return fmt.Errorf("local: %s: %w", key, ErrNotFound)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("local: %s: %w", key, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListDir lists the files directly inside dir.
|
|
||||||
func (s *LocalStorage) ListDir(ctx context.Context, dir string) ([]Item, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
d, err := cleanDir(dir)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("local: %w", err)
|
|
||||||
}
|
|
||||||
full, err := s.join(d)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
entries, err := os.ReadDir(full)
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return []Item{}, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("local: failed to read directory %s: %w", dir, err)
|
|
||||||
}
|
|
||||||
items := make([]Item, 0, len(entries))
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.IsDir() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
fi, err := e.Info()
|
|
||||||
if err != nil || !fi.Mode().IsRegular() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
items = append(items, s.item(filepath.Join(full, e.Name()), fi))
|
|
||||||
}
|
|
||||||
return items, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// List returns all items (files and directories, recursively) whose key
|
|
||||||
// starts with prefix.
|
|
||||||
func (s *LocalStorage) List(ctx context.Context, prefix string) ([]Item, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
p, err := cleanPrefix(prefix)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("local: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Walk from the deepest directory fully contained in the prefix.
|
|
||||||
root := s.basePath
|
|
||||||
if i := strings.LastIndex(p, "/"); i >= 0 {
|
|
||||||
root = filepath.Join(s.basePath, filepath.FromSlash(p[:i]))
|
|
||||||
}
|
|
||||||
items := make([]Item, 0)
|
|
||||||
err = filepath.WalkDir(root, func(full string, d fs.DirEntry, err error) error {
|
|
||||||
if cerr := ctx.Err(); cerr != nil {
|
|
||||||
return cerr
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
if full == root && errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return fs.SkipAll
|
|
||||||
}
|
|
||||||
return nil // skip unreadable entries
|
|
||||||
}
|
|
||||||
if full == s.basePath {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
rel, rerr := filepath.Rel(s.basePath, full)
|
|
||||||
if rerr != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
rel = filepath.ToSlash(rel)
|
|
||||||
if !strings.HasPrefix(rel, p) {
|
|
||||||
// Prune directories that cannot contain matches.
|
|
||||||
if d.IsDir() && !strings.HasPrefix(p, rel+"/") {
|
|
||||||
return fs.SkipDir
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
fi, ierr := d.Info()
|
|
||||||
if ierr != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
items = append(items, s.item(full, fi))
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("local: failed to walk directory: %w", err)
|
|
||||||
}
|
|
||||||
return items, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove deletes a file (or directory tree) from storage.
|
|
||||||
func (s *LocalStorage) Remove(ctx context.Context, remotePath string) error {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
targetPath, err := s.resolvePath(remotePath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
info, err := os.Stat(targetPath)
|
|
||||||
if err != nil {
|
|
||||||
return mapLocalErr(remotePath, err)
|
|
||||||
}
|
|
||||||
if info.IsDir() {
|
|
||||||
err = os.RemoveAll(targetPath)
|
|
||||||
} else {
|
|
||||||
err = os.Remove(targetPath)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return mapLocalErr(remotePath, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close releases any resources held by the storage implementation (no-op).
|
|
||||||
func (s *LocalStorage) Close() error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// BasePath returns the base directory path of the storage.
|
|
||||||
func (s *LocalStorage) BasePath() string {
|
|
||||||
return s.basePath
|
|
||||||
}
|
|
||||||
@@ -22,10 +22,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|||||||
SOFTWARE.
|
SOFTWARE.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// Package azure is the legacy v0.1 API of go-storage.
|
|
||||||
//
|
|
||||||
// Deprecated: use the root package github.com/jkaninda/go-storage
|
|
||||||
// (NewStorage and the streaming Storage interface) instead.
|
|
||||||
package azure
|
package azure
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|||||||
@@ -22,10 +22,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|||||||
SOFTWARE.
|
SOFTWARE.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// Package ftp is the legacy v0.1 API of go-storage.
|
|
||||||
//
|
|
||||||
// Deprecated: use the root package github.com/jkaninda/go-storage
|
|
||||||
// (NewStorage and the streaming Storage interface) instead.
|
|
||||||
package ftp
|
package ftp
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|||||||
@@ -22,10 +22,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|||||||
SOFTWARE.
|
SOFTWARE.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// Package local is the legacy v0.1 API of go-storage.
|
|
||||||
//
|
|
||||||
// Deprecated: use the root package github.com/jkaninda/go-storage
|
|
||||||
// (NewStorage and the streaming Storage interface) instead.
|
|
||||||
package local
|
package local
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|||||||
@@ -1,27 +1,3 @@
|
|||||||
/*
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2024 Jonas Kaninda
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package local
|
package local
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|||||||
@@ -22,10 +22,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|||||||
SOFTWARE.
|
SOFTWARE.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// Package s3 is the legacy v0.1 API of go-storage.
|
|
||||||
//
|
|
||||||
// Deprecated: use the root package github.com/jkaninda/go-storage
|
|
||||||
// (NewStorage and the streaming Storage interface) instead.
|
|
||||||
package s3
|
package s3
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|||||||
@@ -22,10 +22,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|||||||
SOFTWARE.
|
SOFTWARE.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// Package ssh is the legacy v0.1 API of go-storage.
|
|
||||||
//
|
|
||||||
// Deprecated: use the root package github.com/jkaninda/go-storage
|
|
||||||
// (NewStorage and the streaming Storage interface) instead.
|
|
||||||
package ssh
|
package ssh
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|||||||
@@ -1,25 +1 @@
|
|||||||
/*
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2024 Jonas Kaninda
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package ssh
|
package ssh
|
||||||
@@ -22,10 +22,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|||||||
SOFTWARE.
|
SOFTWARE.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// Package pkg is the legacy v0.1 API of go-storage.
|
|
||||||
//
|
|
||||||
// Deprecated: use the root package github.com/jkaninda/go-storage
|
|
||||||
// (NewStorage and the streaming Storage interface) instead.
|
|
||||||
package pkg
|
package pkg
|
||||||
|
|
||||||
type Storage interface {
|
type Storage interface {
|
||||||
|
|||||||
@@ -1,405 +0,0 @@
|
|||||||
/*
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2024 Jonas Kaninda
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package go_storage
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/aws/aws-sdk-go-v2/aws"
|
|
||||||
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
|
|
||||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
|
||||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
|
||||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
|
||||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
||||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
|
||||||
"github.com/aws/smithy-go"
|
|
||||||
)
|
|
||||||
|
|
||||||
// S3 defaults.
|
|
||||||
const (
|
|
||||||
DefaultS3PartSize int64 = 64 << 20
|
|
||||||
MinS3PartSize int64 = 5 << 20
|
|
||||||
DefaultS3Concurrency = 2
|
|
||||||
DefaultS3Region = "us-east-1"
|
|
||||||
)
|
|
||||||
|
|
||||||
// S3Storage implements Storage for Amazon S3 and S3-compatible services
|
|
||||||
// (MinIO, Ceph, Wasabi, ...).
|
|
||||||
type S3Storage struct {
|
|
||||||
client *s3.Client
|
|
||||||
uploader *manager.Uploader //nolint:staticcheck // see NewS3Storage
|
|
||||||
bucket string
|
|
||||||
region string
|
|
||||||
storageClass types.StorageClass
|
|
||||||
sse types.ServerSideEncryption
|
|
||||||
sseKMSKeyID string
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewS3Storage creates a new S3 storage instance.
|
|
||||||
//
|
|
||||||
// Credentials: static keys (S3KeyID/S3Secret/S3SessionToken) when set,
|
|
||||||
// otherwise the default AWS credential chain (environment, shared profile
|
|
||||||
// S3Profile, web identity/IRSA, ECS/EC2 roles).
|
|
||||||
func NewS3Storage(config Config) (*S3Storage, error) {
|
|
||||||
if err := validateConfig(config, map[string]string{
|
|
||||||
"bucket": config.S3Bucket,
|
|
||||||
}); err != nil {
|
|
||||||
return nil, fmt.Errorf("s3: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
opts := []func(*awsconfig.LoadOptions) error{
|
|
||||||
awsconfig.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired),
|
|
||||||
awsconfig.WithResponseChecksumValidation(aws.ResponseChecksumValidationWhenRequired),
|
|
||||||
}
|
|
||||||
if config.S3Region != "" {
|
|
||||||
opts = append(opts, awsconfig.WithRegion(config.S3Region))
|
|
||||||
}
|
|
||||||
if config.S3Profile != "" {
|
|
||||||
opts = append(opts, awsconfig.WithSharedConfigProfile(config.S3Profile))
|
|
||||||
}
|
|
||||||
if config.S3KeyID != "" && config.S3Secret != "" {
|
|
||||||
opts = append(opts, awsconfig.WithCredentialsProvider(
|
|
||||||
credentials.NewStaticCredentialsProvider(config.S3KeyID, config.S3Secret, config.S3SessionToken)))
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("s3: failed to load AWS configuration: %w", err)
|
|
||||||
}
|
|
||||||
if cfg.Region == "" {
|
|
||||||
cfg.Region = DefaultS3Region
|
|
||||||
}
|
|
||||||
|
|
||||||
endpoint := config.S3Endpoint
|
|
||||||
if endpoint != "" && !strings.Contains(endpoint, "://") {
|
|
||||||
if config.S3DisableTLS {
|
|
||||||
endpoint = "http://" + endpoint
|
|
||||||
} else {
|
|
||||||
endpoint = "https://" + endpoint
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
|
|
||||||
if endpoint != "" {
|
|
||||||
o.BaseEndpoint = aws.String(endpoint)
|
|
||||||
}
|
|
||||||
o.UsePathStyle = config.S3ForcePath
|
|
||||||
})
|
|
||||||
|
|
||||||
partSize := config.S3PartSize
|
|
||||||
if partSize <= 0 {
|
|
||||||
partSize = DefaultS3PartSize
|
|
||||||
}
|
|
||||||
if partSize < MinS3PartSize {
|
|
||||||
partSize = MinS3PartSize
|
|
||||||
}
|
|
||||||
concurrency := config.S3Concurrency
|
|
||||||
if concurrency <= 0 {
|
|
||||||
concurrency = DefaultS3Concurrency
|
|
||||||
}
|
|
||||||
|
|
||||||
//nolint:staticcheck // feature/s3/manager is deprecated in favour of the (pre-1.0) transfermanager.
|
|
||||||
uploader := manager.NewUploader(client, func(u *manager.Uploader) {
|
|
||||||
u.PartSize = partSize
|
|
||||||
u.Concurrency = concurrency
|
|
||||||
u.LeavePartsOnError = false
|
|
||||||
u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
|
|
||||||
})
|
|
||||||
|
|
||||||
s := &S3Storage{
|
|
||||||
client: client,
|
|
||||||
uploader: uploader,
|
|
||||||
bucket: config.S3Bucket,
|
|
||||||
region: cfg.Region,
|
|
||||||
storageClass: types.StorageClass(config.S3StorageClass),
|
|
||||||
}
|
|
||||||
switch strings.ToLower(config.S3SSE) {
|
|
||||||
case "":
|
|
||||||
case "aes256":
|
|
||||||
s.sse = types.ServerSideEncryptionAes256
|
|
||||||
case "aws:kms":
|
|
||||||
s.sse = types.ServerSideEncryptionAwsKms
|
|
||||||
s.sseKMSKeyID = config.S3SSEKMSKeyID
|
|
||||||
default:
|
|
||||||
s.sse = types.ServerSideEncryption(config.S3SSE)
|
|
||||||
s.sseKMSKeyID = config.S3SSEKMSKeyID
|
|
||||||
}
|
|
||||||
return s, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Client returns the underlying S3 client.
|
|
||||||
func (s *S3Storage) Client() *s3.Client { return s.client }
|
|
||||||
|
|
||||||
func (s *S3Storage) putInput(key string, body io.Reader, opts *PutOptions) *s3.PutObjectInput {
|
|
||||||
in := &s3.PutObjectInput{
|
|
||||||
Bucket: aws.String(s.bucket),
|
|
||||||
Key: aws.String(key),
|
|
||||||
Body: body,
|
|
||||||
}
|
|
||||||
if opts != nil {
|
|
||||||
if opts.ContentType != "" {
|
|
||||||
in.ContentType = aws.String(opts.ContentType)
|
|
||||||
}
|
|
||||||
if len(opts.Metadata) > 0 {
|
|
||||||
in.Metadata = opts.Metadata
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if s.storageClass != "" {
|
|
||||||
in.StorageClass = s.storageClass
|
|
||||||
}
|
|
||||||
if s.sse != "" {
|
|
||||||
in.ServerSideEncryption = s.sse
|
|
||||||
if s.sseKMSKeyID != "" {
|
|
||||||
in.SSEKMSKeyId = aws.String(s.sseKMSKeyID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return in
|
|
||||||
}
|
|
||||||
|
|
||||||
// Put streams r to key using a multipart upload (aborted on error).
|
|
||||||
func (s *S3Storage) Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error) {
|
|
||||||
k, err := cleanKey(key)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, fmt.Errorf("s3: %w", err)
|
|
||||||
}
|
|
||||||
cr := &countingReader{r: ctxReader{ctx: ctx, r: r}}
|
|
||||||
//nolint:staticcheck // see NewS3Storage
|
|
||||||
out, err := s.uploader.Upload(ctx, s.putInput(k, cr, opts))
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, fmt.Errorf("s3: failed to upload %s to bucket %s: %w", k, s.bucket, err)
|
|
||||||
}
|
|
||||||
return Item{
|
|
||||||
Key: k,
|
|
||||||
Size: cr.n,
|
|
||||||
ModifiedTime: time.Now().UTC(),
|
|
||||||
ETag: aws.ToString(out.ETag),
|
|
||||||
VersionID: aws.ToString(out.VersionID),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// PutIf performs a conditional single-request PutObject (If-Match /
|
|
||||||
// If-None-Match: *). The body is buffered and limited to 16 MiB.
|
|
||||||
func (s *S3Storage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error) {
|
|
||||||
k, err := cleanKey(key)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, fmt.Errorf("s3: %w", err)
|
|
||||||
}
|
|
||||||
if err := validateCondition(cond); err != nil {
|
|
||||||
return Item{}, fmt.Errorf("s3: %w", err)
|
|
||||||
}
|
|
||||||
data, err := io.ReadAll(io.LimitReader(ctxReader{ctx: ctx, r: r}, maxConditionalPutSize+1))
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, fmt.Errorf("s3: failed to read data: %w", err)
|
|
||||||
}
|
|
||||||
if len(data) > maxConditionalPutSize {
|
|
||||||
return Item{}, fmt.Errorf("s3: conditional put of %s exceeds %d bytes", k, maxConditionalPutSize)
|
|
||||||
}
|
|
||||||
in := s.putInput(k, bytes.NewReader(data), opts)
|
|
||||||
in.ContentLength = aws.Int64(int64(len(data)))
|
|
||||||
if cond.IfMatch != "" {
|
|
||||||
in.IfMatch = aws.String(cond.IfMatch)
|
|
||||||
}
|
|
||||||
if cond.IfNoneMatch {
|
|
||||||
in.IfNoneMatch = aws.String("*")
|
|
||||||
}
|
|
||||||
out, err := s.client.PutObject(ctx, in)
|
|
||||||
if err != nil {
|
|
||||||
// If-Match on a missing key yields 404 NoSuchKey on AWS.
|
|
||||||
if isS3Precondition(err) || (cond.IfMatch != "" && s3StatusCode(err) == http.StatusNotFound) {
|
|
||||||
return Item{}, fmt.Errorf("s3: conditional put of %s: %w", k, ErrPreconditionFailed)
|
|
||||||
}
|
|
||||||
return Item{}, fmt.Errorf("s3: failed to put %s: %w", k, err)
|
|
||||||
}
|
|
||||||
return Item{
|
|
||||||
Key: k,
|
|
||||||
Size: int64(len(data)),
|
|
||||||
ModifiedTime: time.Now().UTC(),
|
|
||||||
ETag: aws.ToString(out.ETag),
|
|
||||||
VersionID: aws.ToString(out.VersionId),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get streams the object body.
|
|
||||||
func (s *S3Storage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
|
|
||||||
k, err := cleanKey(key)
|
|
||||||
if err != nil {
|
|
||||||
return nil, Item{}, fmt.Errorf("s3: %w", err)
|
|
||||||
}
|
|
||||||
out, err := s.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(k)})
|
|
||||||
if err != nil {
|
|
||||||
return nil, Item{}, s.mapErr("get", k, err)
|
|
||||||
}
|
|
||||||
return out.Body, Item{
|
|
||||||
Key: k,
|
|
||||||
Size: aws.ToInt64(out.ContentLength),
|
|
||||||
ModifiedTime: aws.ToTime(out.LastModified),
|
|
||||||
ETag: aws.ToString(out.ETag),
|
|
||||||
VersionID: aws.ToString(out.VersionId),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stat returns object metadata (HeadObject).
|
|
||||||
func (s *S3Storage) Stat(ctx context.Context, key string) (Item, error) {
|
|
||||||
k, err := cleanKey(key)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, fmt.Errorf("s3: %w", err)
|
|
||||||
}
|
|
||||||
out, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(k)})
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, s.mapErr("stat", k, err)
|
|
||||||
}
|
|
||||||
return Item{
|
|
||||||
Key: k,
|
|
||||||
Size: aws.ToInt64(out.ContentLength),
|
|
||||||
ModifiedTime: aws.ToTime(out.LastModified),
|
|
||||||
ETag: aws.ToString(out.ETag),
|
|
||||||
VersionID: aws.ToString(out.VersionId),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Upload uploads a file to S3.
|
|
||||||
func (s *S3Storage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
|
||||||
if err := uploadFile(ctx, s, localPath, remotePath); err != nil {
|
|
||||||
return fmt.Errorf("s3: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Download downloads a file from S3.
|
|
||||||
func (s *S3Storage) Download(ctx context.Context, remotePath string, localPath string) error {
|
|
||||||
if err := downloadFile(ctx, s, remotePath, localPath); err != nil {
|
|
||||||
return fmt.Errorf("s3: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// List lists objects (recursively) with the given prefix.
|
|
||||||
func (s *S3Storage) List(ctx context.Context, prefix string) ([]Item, error) {
|
|
||||||
p, err := cleanPrefix(prefix)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("s3: %w", err)
|
|
||||||
}
|
|
||||||
return s.list(ctx, p, "")
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListDir lists the objects directly under dir.
|
|
||||||
func (s *S3Storage) ListDir(ctx context.Context, dir string) ([]Item, error) {
|
|
||||||
d, err := cleanDir(dir)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("s3: %w", err)
|
|
||||||
}
|
|
||||||
return s.list(ctx, dirPrefix(d), "/")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *S3Storage) list(ctx context.Context, prefix, delimiter string) ([]Item, error) {
|
|
||||||
in := &s3.ListObjectsV2Input{Bucket: aws.String(s.bucket), Prefix: aws.String(prefix)}
|
|
||||||
if delimiter != "" {
|
|
||||||
in.Delimiter = aws.String(delimiter)
|
|
||||||
}
|
|
||||||
items := make([]Item, 0)
|
|
||||||
p := s3.NewListObjectsV2Paginator(s.client, in)
|
|
||||||
for p.HasMorePages() {
|
|
||||||
page, err := p.NextPage(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("s3: failed to list objects in bucket %s: %w", s.bucket, err)
|
|
||||||
}
|
|
||||||
for _, obj := range page.Contents {
|
|
||||||
key := aws.ToString(obj.Key)
|
|
||||||
if delimiter != "" && strings.HasSuffix(key, "/") {
|
|
||||||
continue // directory marker
|
|
||||||
}
|
|
||||||
items = append(items, Item{
|
|
||||||
Key: key,
|
|
||||||
ModifiedTime: aws.ToTime(obj.LastModified),
|
|
||||||
Size: aws.ToInt64(obj.Size),
|
|
||||||
ETag: aws.ToString(obj.ETag),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return items, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove deletes an object. A missing key returns ErrNotFound (checked with
|
|
||||||
// HeadObject because DeleteObject is idempotent on S3).
|
|
||||||
func (s *S3Storage) Remove(ctx context.Context, remotePath string) error {
|
|
||||||
k, err := cleanKey(remotePath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("s3: %w", err)
|
|
||||||
}
|
|
||||||
if _, err := s.Stat(ctx, k); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(k)}); err != nil {
|
|
||||||
return s.mapErr("remove", k, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close releases resources (no-op).
|
|
||||||
func (s *S3Storage) Close() error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *S3Storage) mapErr(op, key string, err error) error {
|
|
||||||
if s3StatusCode(err) == http.StatusNotFound {
|
|
||||||
return fmt.Errorf("s3: %s %s in bucket %s: %w", op, key, s.bucket, ErrNotFound)
|
|
||||||
}
|
|
||||||
var nsk *types.NoSuchKey
|
|
||||||
var nf *types.NotFound
|
|
||||||
if errors.As(err, &nsk) || errors.As(err, &nf) {
|
|
||||||
return fmt.Errorf("s3: %s %s in bucket %s: %w", op, key, s.bucket, ErrNotFound)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("s3: %s %s in bucket %s: %w", op, key, s.bucket, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func s3StatusCode(err error) int {
|
|
||||||
var re *awshttp.ResponseError
|
|
||||||
if errors.As(err, &re) {
|
|
||||||
return re.HTTPStatusCode()
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func isS3Precondition(err error) bool {
|
|
||||||
var apiErr smithy.APIError
|
|
||||||
if errors.As(err, &apiErr) {
|
|
||||||
switch apiErr.ErrorCode() {
|
|
||||||
case "PreconditionFailed", "ConditionalRequestConflict":
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
switch s3StatusCode(err) {
|
|
||||||
case http.StatusPreconditionFailed, http.StatusConflict:
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
@@ -1,559 +0,0 @@
|
|||||||
/*
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2024 Jonas Kaninda
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package go_storage
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"io/fs"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"os/user"
|
|
||||||
"path"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/pkg/sftp"
|
|
||||||
|
|
||||||
"golang.org/x/crypto/ssh"
|
|
||||||
"golang.org/x/crypto/ssh/agent"
|
|
||||||
"golang.org/x/crypto/ssh/knownhosts"
|
|
||||||
)
|
|
||||||
|
|
||||||
// SFTPStorage implements Storage interface for SFTP.
|
|
||||||
type SFTPStorage struct {
|
|
||||||
host string
|
|
||||||
baseDir string
|
|
||||||
sshClient *ssh.Client
|
|
||||||
client *sftp.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewSFTPStorage creates a new SFTP storage instance.
|
|
||||||
//
|
|
||||||
// Host keys are verified against SFTPKnownHostsFile (or ~/.ssh/known_hosts
|
|
||||||
// and /etc/ssh/ssh_known_hosts) unless SFTPIgnoreKnownHosts is true.
|
|
||||||
func NewSFTPStorage(config Config) (*SFTPStorage, error) {
|
|
||||||
if err := validateConfig(config, map[string]string{
|
|
||||||
"host": config.SFTPHost,
|
|
||||||
}); err != nil {
|
|
||||||
return nil, fmt.Errorf("sftp: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
port := config.SFTPPort
|
|
||||||
if port == "" {
|
|
||||||
port = "22"
|
|
||||||
}
|
|
||||||
hostPort := net.JoinHostPort(config.SFTPHost, port)
|
|
||||||
|
|
||||||
username := config.SFTPUsername
|
|
||||||
if username == "" {
|
|
||||||
currentUser, err := user.Current()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("sftp: failed to get current user: %w", err)
|
|
||||||
}
|
|
||||||
username = currentUser.Username
|
|
||||||
}
|
|
||||||
|
|
||||||
authMethods := make([]ssh.AuthMethod, 0, 2)
|
|
||||||
if config.SFTPPassword != "" {
|
|
||||||
authMethods = append(authMethods, ssh.Password(config.SFTPPassword))
|
|
||||||
}
|
|
||||||
passphrase := config.SFTPIdentityPassphrase
|
|
||||||
if passphrase == "" {
|
|
||||||
passphrase = config.SFTPPassword // backwards compatibility
|
|
||||||
}
|
|
||||||
signers, err := getSSHSigners(config.SFTPIdentityFile, passphrase)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("sftp: %w", err)
|
|
||||||
}
|
|
||||||
if len(signers) > 0 {
|
|
||||||
authMethods = append(authMethods, ssh.PublicKeys(signers...))
|
|
||||||
}
|
|
||||||
|
|
||||||
hostKeyCallback, hostKeyAlgos, err := getHostKeyCallback(config, hostPort)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("sftp: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
sshConfig := &ssh.ClientConfig{
|
|
||||||
User: username,
|
|
||||||
Auth: authMethods,
|
|
||||||
HostKeyCallback: hostKeyCallback,
|
|
||||||
HostKeyAlgorithms: hostKeyAlgos,
|
|
||||||
Timeout: 30 * time.Second,
|
|
||||||
}
|
|
||||||
|
|
||||||
sshClient, err := ssh.Dial("tcp", hostPort, sshConfig)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("sftp: failed to connect to %s: %w", hostPort, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
sftpClient, err := sftp.NewClient(sshClient)
|
|
||||||
if err != nil {
|
|
||||||
_ = sshClient.Close()
|
|
||||||
return nil, fmt.Errorf("sftp: failed to open sftp session: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
baseDir := normalizePathSeparators(config.SFTPDirectory)
|
|
||||||
if baseDir == "" {
|
|
||||||
wd, err := sftpClient.Getwd()
|
|
||||||
if err != nil {
|
|
||||||
_ = sftpClient.Close()
|
|
||||||
_ = sshClient.Close()
|
|
||||||
return nil, fmt.Errorf("sftp: failed to get working directory: %w", err)
|
|
||||||
}
|
|
||||||
baseDir = wd
|
|
||||||
}
|
|
||||||
|
|
||||||
return &SFTPStorage{
|
|
||||||
host: config.SFTPHost,
|
|
||||||
baseDir: path.Clean(baseDir),
|
|
||||||
sshClient: sshClient,
|
|
||||||
client: sftpClient,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Client returns the underlying sftp client.
|
|
||||||
func (s *SFTPStorage) Client() *sftp.Client { return s.client }
|
|
||||||
|
|
||||||
func (s *SFTPStorage) resolve(key string) (string, string, error) {
|
|
||||||
k, err := cleanKey(key)
|
|
||||||
if err != nil {
|
|
||||||
return "", "", fmt.Errorf("sftp: %w", err)
|
|
||||||
}
|
|
||||||
return k, path.Join(s.baseDir, k), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SFTPStorage) item(key string, fi os.FileInfo) Item {
|
|
||||||
it := Item{Key: key, ModifiedTime: fi.ModTime(), IsDirectory: fi.IsDir()}
|
|
||||||
if !fi.IsDir() {
|
|
||||||
it.Size = fi.Size()
|
|
||||||
}
|
|
||||||
return it
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SFTPStorage) mkdirParent(full string) error {
|
|
||||||
dir := path.Dir(full)
|
|
||||||
if dir == "." || dir == "/" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := s.client.MkdirAll(dir); err != nil {
|
|
||||||
return fmt.Errorf("sftp: failed to create directory %s: %w", dir, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Put streams r to <key>.<rand>.partial and renames it to key on success.
|
|
||||||
func (s *SFTPStorage) Put(ctx context.Context, key string, r io.Reader, _ *PutOptions) (Item, error) {
|
|
||||||
k, full, err := s.resolve(key)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
if err := s.mkdirParent(full); err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
tmp := partialName(full)
|
|
||||||
f, err := s.client.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, fmt.Errorf("sftp: failed to create %s: %w", tmp, err)
|
|
||||||
}
|
|
||||||
if err := copyAndClose(ctx, f, r); err != nil {
|
|
||||||
_ = s.client.Remove(tmp)
|
|
||||||
return Item{}, fmt.Errorf("sftp: failed to upload %s: %w", k, err)
|
|
||||||
}
|
|
||||||
if err := s.rename(tmp, full); err != nil {
|
|
||||||
_ = s.client.Remove(tmp)
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
return s.Stat(ctx, k)
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyAndClose(ctx context.Context, w io.WriteCloser, r io.Reader) error {
|
|
||||||
_, err := io.Copy(w, ctxReader{ctx: ctx, r: r})
|
|
||||||
if cerr := w.Close(); err == nil {
|
|
||||||
err = cerr
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
err = ctx.Err()
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SFTPStorage) rename(from, to string) error {
|
|
||||||
if _, ok := s.client.HasExtension("posix-rename@openssh.com"); ok {
|
|
||||||
if err := s.client.PosixRename(from, to); err != nil {
|
|
||||||
return fmt.Errorf("sftp: failed to rename %s to %s: %w", from, to, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := s.client.Remove(to); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return fmt.Errorf("sftp: failed to replace %s: %w", to, err)
|
|
||||||
}
|
|
||||||
if err := s.client.Rename(from, to); err != nil {
|
|
||||||
return fmt.Errorf("sftp: failed to rename %s to %s: %w", from, to, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// PutIf supports only Condition.IfNoneMatch (via O_EXCL on the final path).
|
|
||||||
// IfMatch returns ErrConditionNotSupported.
|
|
||||||
func (s *SFTPStorage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, _ *PutOptions) (Item, error) {
|
|
||||||
if cond.IfMatch != "" || !cond.IfNoneMatch {
|
|
||||||
return Item{}, fmt.Errorf("sftp: IfMatch: %w", ErrConditionNotSupported)
|
|
||||||
}
|
|
||||||
k, full, err := s.resolve(key)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
if err := s.mkdirParent(full); err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
f, err := s.client.OpenFile(full, os.O_WRONLY|os.O_CREATE|os.O_EXCL)
|
|
||||||
if err != nil {
|
|
||||||
if _, serr := s.client.Lstat(full); serr == nil {
|
|
||||||
return Item{}, fmt.Errorf("sftp: %s already exists: %w", k, ErrPreconditionFailed)
|
|
||||||
}
|
|
||||||
return Item{}, fmt.Errorf("sftp: failed to create %s: %w", k, err)
|
|
||||||
}
|
|
||||||
if err := copyAndClose(ctx, f, r); err != nil {
|
|
||||||
_ = s.client.Remove(full)
|
|
||||||
return Item{}, fmt.Errorf("sftp: failed to upload %s: %w", k, err)
|
|
||||||
}
|
|
||||||
return s.Stat(ctx, k)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get opens key for streaming reads.
|
|
||||||
func (s *SFTPStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, Item{}, err
|
|
||||||
}
|
|
||||||
k, full, err := s.resolve(key)
|
|
||||||
if err != nil {
|
|
||||||
return nil, Item{}, err
|
|
||||||
}
|
|
||||||
f, err := s.client.Open(full)
|
|
||||||
if err != nil {
|
|
||||||
return nil, Item{}, mapSFTPErr("get", k, err)
|
|
||||||
}
|
|
||||||
fi, err := f.Stat()
|
|
||||||
if err != nil {
|
|
||||||
_ = f.Close()
|
|
||||||
return nil, Item{}, mapSFTPErr("get", k, err)
|
|
||||||
}
|
|
||||||
if fi.IsDir() {
|
|
||||||
_ = f.Close()
|
|
||||||
return nil, Item{}, fmt.Errorf("sftp: %s is a directory", k)
|
|
||||||
}
|
|
||||||
return f, s.item(k, fi), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stat returns metadata for key.
|
|
||||||
func (s *SFTPStorage) Stat(ctx context.Context, key string) (Item, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
k, full, err := s.resolve(key)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, err
|
|
||||||
}
|
|
||||||
fi, err := s.client.Stat(full)
|
|
||||||
if err != nil {
|
|
||||||
return Item{}, mapSFTPErr("stat", k, err)
|
|
||||||
}
|
|
||||||
return s.item(k, fi), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func mapSFTPErr(op, key string, err error) error {
|
|
||||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, sftp.ErrSSHFxNoSuchFile) {
|
|
||||||
return fmt.Errorf("sftp: %s %s: %w", op, key, ErrNotFound)
|
|
||||||
}
|
|
||||||
var se *sftp.StatusError
|
|
||||||
if errors.As(err, &se) && se.FxCode() == sftp.ErrSSHFxNoSuchFile {
|
|
||||||
return fmt.Errorf("sftp: %s %s: %w", op, key, ErrNotFound)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("sftp: %s %s: %w", op, key, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Upload uploads a file via SFTP.
|
|
||||||
func (s *SFTPStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
|
||||||
return uploadFile(ctx, s, localPath, remotePath)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Download downloads a file via SFTP.
|
|
||||||
func (s *SFTPStorage) Download(ctx context.Context, remotePath string, localPath string) error {
|
|
||||||
return downloadFile(ctx, s, remotePath, localPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
// List lists files and directories (recursively) whose key starts with prefix.
|
|
||||||
func (s *SFTPStorage) List(ctx context.Context, prefix string) ([]Item, error) {
|
|
||||||
p, err := cleanPrefix(prefix)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("sftp: %w", err)
|
|
||||||
}
|
|
||||||
items := make([]Item, 0)
|
|
||||||
root := s.baseDir
|
|
||||||
if i := strings.LastIndex(p, "/"); i >= 0 {
|
|
||||||
root = path.Join(s.baseDir, p[:i])
|
|
||||||
}
|
|
||||||
walker := s.client.Walk(root)
|
|
||||||
for walker.Step() {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := walker.Err(); err != nil {
|
|
||||||
if walker.Path() == root && errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return items, nil
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
full := walker.Path()
|
|
||||||
if full == s.baseDir {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
rel := strings.TrimPrefix(strings.TrimPrefix(full, s.baseDir), "/")
|
|
||||||
stat := walker.Stat()
|
|
||||||
if !strings.HasPrefix(rel, p) {
|
|
||||||
if stat.IsDir() && !strings.HasPrefix(p, rel+"/") {
|
|
||||||
walker.SkipDir()
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
items = append(items, s.item(rel, stat))
|
|
||||||
}
|
|
||||||
return items, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListDir lists the files directly inside dir.
|
|
||||||
func (s *SFTPStorage) ListDir(ctx context.Context, dir string) ([]Item, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
d, err := cleanDir(dir)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("sftp: %w", err)
|
|
||||||
}
|
|
||||||
entries, err := s.client.ReadDir(path.Join(s.baseDir, d))
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return []Item{}, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("sftp: failed to read directory %s: %w", dir, err)
|
|
||||||
}
|
|
||||||
items := make([]Item, 0, len(entries))
|
|
||||||
for _, fi := range entries {
|
|
||||||
if !fi.Mode().IsRegular() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
items = append(items, s.item(path.Join(d, fi.Name()), fi))
|
|
||||||
}
|
|
||||||
return items, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove deletes a file via SFTP.
|
|
||||||
func (s *SFTPStorage) Remove(ctx context.Context, remotePath string) error {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
k, full, err := s.resolve(remotePath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := s.client.Remove(full); err != nil {
|
|
||||||
return mapSFTPErr("remove", k, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close releases resources.
|
|
||||||
func (s *SFTPStorage) Close() error {
|
|
||||||
err := s.client.Close()
|
|
||||||
if cerr := s.sshClient.Close(); err == nil {
|
|
||||||
err = cerr
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// getSSHSigners returns SSH signers from identity file and SSH agent.
|
|
||||||
func getSSHSigners(identityFile string, passphrase string) ([]ssh.Signer, error) {
|
|
||||||
signers := make([]ssh.Signer, 0)
|
|
||||||
|
|
||||||
if identityFile != "" {
|
|
||||||
p, err := expandHomeDir(identityFile)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to expand identity file path: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
keyData, err := os.ReadFile(p)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read identity file %s: %w", p, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
signer, err := ssh.ParsePrivateKey(keyData)
|
|
||||||
if err != nil {
|
|
||||||
var passErr *ssh.PassphraseMissingError
|
|
||||||
if !errors.As(err, &passErr) {
|
|
||||||
return nil, fmt.Errorf("failed to parse identity file: %w", err)
|
|
||||||
}
|
|
||||||
signer, err = ssh.ParsePrivateKeyWithPassphrase(keyData, []byte(passphrase))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to decrypt identity file: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
signers = append(signers, signer)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Keys from a running SSH agent, if any.
|
|
||||||
if socket := os.Getenv("SSH_AUTH_SOCK"); socket != "" {
|
|
||||||
if conn, err := net.Dial("unix", socket); err == nil {
|
|
||||||
if agentSigners, err := agent.NewClient(conn).Signers(); err == nil {
|
|
||||||
signers = append(signers, agentSigners...)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return signers, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// getHostKeyCallback builds the host key verification callback and the list
|
|
||||||
// of host key algorithms known for hostPort (so the server negotiates a key
|
|
||||||
// type present in known_hosts).
|
|
||||||
func getHostKeyCallback(config Config, hostPort string) (ssh.HostKeyCallback, []string, error) {
|
|
||||||
if config.SFTPIgnoreKnownHosts {
|
|
||||||
return ssh.InsecureIgnoreHostKey(), nil, nil //nolint:gosec // explicitly requested
|
|
||||||
}
|
|
||||||
|
|
||||||
files := make([]string, 0, 2)
|
|
||||||
if config.SFTPKnownHostsFile != "" {
|
|
||||||
p, err := expandHomeDir(config.SFTPKnownHostsFile)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(p); err != nil {
|
|
||||||
return nil, nil, fmt.Errorf("known_hosts file %s: %w", p, err)
|
|
||||||
}
|
|
||||||
files = append(files, p)
|
|
||||||
} else {
|
|
||||||
for _, candidate := range []string{"~/.ssh/known_hosts", "/etc/ssh/ssh_known_hosts"} {
|
|
||||||
p, err := expandHomeDir(candidate)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(p); err == nil {
|
|
||||||
files = append(files, p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(files) == 0 {
|
|
||||||
return nil, nil, fmt.Errorf("no known_hosts file available for host key verification " +
|
|
||||||
"(set SSH_KNOWN_HOSTS or SSH_INSECURE_IGNORE_HOST_KEY=true)")
|
|
||||||
}
|
|
||||||
|
|
||||||
callback, err := knownhosts.New(files...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, fmt.Errorf("failed to load known_hosts: %w", err)
|
|
||||||
}
|
|
||||||
return callback, knownHostAlgorithms(callback, hostPort), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// knownHostAlgorithms returns the host key algorithms recorded for hostPort,
|
|
||||||
// using the KeyError returned for a deliberately unknown key.
|
|
||||||
func knownHostAlgorithms(cb ssh.HostKeyCallback, hostPort string) []string {
|
|
||||||
addr, err := net.ResolveTCPAddr("tcp", hostPort)
|
|
||||||
if err != nil {
|
|
||||||
addr = &net.TCPAddr{}
|
|
||||||
}
|
|
||||||
var keyErr *knownhosts.KeyError
|
|
||||||
if err := cb(hostPort, addr, probeKey{}); !errors.As(err, &keyErr) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
algos := make([]string, 0, len(keyErr.Want))
|
|
||||||
seen := map[string]bool{}
|
|
||||||
for _, k := range keyErr.Want {
|
|
||||||
for _, a := range algorithmsForKeyType(k.Key.Type()) {
|
|
||||||
if !seen[a] {
|
|
||||||
seen[a] = true
|
|
||||||
algos = append(algos, a)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(algos) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return algos
|
|
||||||
}
|
|
||||||
|
|
||||||
func algorithmsForKeyType(t string) []string {
|
|
||||||
if t == ssh.KeyAlgoRSA {
|
|
||||||
return []string{ssh.KeyAlgoRSASHA512, ssh.KeyAlgoRSASHA256, ssh.KeyAlgoRSA}
|
|
||||||
}
|
|
||||||
return []string{t}
|
|
||||||
}
|
|
||||||
|
|
||||||
// probeKey is a public key that never matches anything in known_hosts.
|
|
||||||
type probeKey struct{}
|
|
||||||
|
|
||||||
func (probeKey) Type() string { return "probe-key" }
|
|
||||||
func (probeKey) Marshal() []byte { return []byte("probe-key") }
|
|
||||||
func (probeKey) Verify([]byte, *ssh.Signature) error { return errors.New("probe key") }
|
|
||||||
|
|
||||||
// expandHomeDir expands ~ in file paths to home directory.
|
|
||||||
func expandHomeDir(p string) (string, error) {
|
|
||||||
if !strings.HasPrefix(p, "~") {
|
|
||||||
return filepath.Clean(p), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
parts := strings.SplitN(p, "/", 2)
|
|
||||||
username := strings.TrimPrefix(parts[0], "~")
|
|
||||||
|
|
||||||
var homeDir string
|
|
||||||
if username == "" {
|
|
||||||
h, err := os.UserHomeDir()
|
|
||||||
if err != nil || h == "" {
|
|
||||||
currentUser, err := user.Current()
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to get home directory: %w", err)
|
|
||||||
}
|
|
||||||
h = currentUser.HomeDir
|
|
||||||
}
|
|
||||||
homeDir = h
|
|
||||||
} else {
|
|
||||||
userInfo, err := user.Lookup(username)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to lookup user %s: %w", username, err)
|
|
||||||
}
|
|
||||||
homeDir = userInfo.HomeDir
|
|
||||||
}
|
|
||||||
|
|
||||||
if homeDir == "" {
|
|
||||||
return "", fmt.Errorf("empty home directory")
|
|
||||||
}
|
|
||||||
if len(parts) == 1 {
|
|
||||||
return homeDir, nil
|
|
||||||
}
|
|
||||||
return filepath.Join(homeDir, parts[1]), nil
|
|
||||||
}
|
|
||||||
-208
@@ -1,208 +0,0 @@
|
|||||||
/*
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2024 Jonas Kaninda
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package go_storage
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/ed25519"
|
|
||||||
"crypto/rand"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/pkg/sftp"
|
|
||||||
"golang.org/x/crypto/ssh"
|
|
||||||
"golang.org/x/crypto/ssh/knownhosts"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
testSFTPUser = "backup"
|
|
||||||
testSFTPPass = "s3cret"
|
|
||||||
)
|
|
||||||
|
|
||||||
// startSFTPServer runs an in-process SSH/SFTP server on 127.0.0.1:0 and
|
|
||||||
// returns its address and a known_hosts file containing its host key.
|
|
||||||
func startSFTPServer(t *testing.T) (host, port, knownHostsFile string) {
|
|
||||||
t.Helper()
|
|
||||||
_, priv, err := ed25519.GenerateKey(rand.Reader)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
signer, err := ssh.NewSignerFromKey(priv)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cfg := &ssh.ServerConfig{
|
|
||||||
PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
|
|
||||||
if c.User() == testSFTPUser && string(pass) == testSFTPPass {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return nil, errors.New("access denied")
|
|
||||||
},
|
|
||||||
}
|
|
||||||
cfg.AddHostKey(signer)
|
|
||||||
|
|
||||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { _ = ln.Close() })
|
|
||||||
go func() {
|
|
||||||
for {
|
|
||||||
conn, err := ln.Accept()
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
go serveSSHConn(conn, cfg)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
addr := ln.Addr().String()
|
|
||||||
knownHostsFile = filepath.Join(t.TempDir(), "known_hosts")
|
|
||||||
line := knownhosts.Line([]string{knownhosts.Normalize(addr)}, signer.PublicKey())
|
|
||||||
if err := os.WriteFile(knownHostsFile, []byte(line+"\n"), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
host, port, _ = net.SplitHostPort(addr)
|
|
||||||
return host, port, knownHostsFile
|
|
||||||
}
|
|
||||||
|
|
||||||
func serveSSHConn(conn net.Conn, cfg *ssh.ServerConfig) {
|
|
||||||
sconn, chans, reqs, err := ssh.NewServerConn(conn, cfg)
|
|
||||||
if err != nil {
|
|
||||||
_ = conn.Close()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer func() { _ = sconn.Close() }()
|
|
||||||
go ssh.DiscardRequests(reqs)
|
|
||||||
for nc := range chans {
|
|
||||||
if nc.ChannelType() != "session" {
|
|
||||||
_ = nc.Reject(ssh.UnknownChannelType, "unsupported")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ch, chReqs, err := nc.Accept()
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
go func() {
|
|
||||||
for req := range chReqs {
|
|
||||||
ok := req.Type == "subsystem" && len(req.Payload) > 4 && string(req.Payload[4:]) == "sftp"
|
|
||||||
_ = req.Reply(ok, nil)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
srv, err := sftp.NewServer(ch)
|
|
||||||
if err != nil {
|
|
||||||
_ = ch.Close()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := srv.Serve(); err != nil && !errors.Is(err, io.EOF) {
|
|
||||||
_ = srv.Close()
|
|
||||||
}
|
|
||||||
_ = ch.Close()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newTestSFTP(t *testing.T) (*SFTPStorage, string) {
|
|
||||||
t.Helper()
|
|
||||||
t.Setenv("SSH_AUTH_SOCK", "")
|
|
||||||
host, port, kh := startSFTPServer(t)
|
|
||||||
base := t.TempDir()
|
|
||||||
s, err := NewSFTPStorage(Config{
|
|
||||||
SFTPHost: host,
|
|
||||||
SFTPPort: port,
|
|
||||||
SFTPUsername: testSFTPUser,
|
|
||||||
SFTPPassword: testSFTPPass,
|
|
||||||
SFTPDirectory: base,
|
|
||||||
SFTPKnownHostsFile: kh,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewSFTPStorage: %v", err)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { _ = s.Close() })
|
|
||||||
return s, base
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSFTPConformance(t *testing.T) {
|
|
||||||
s, base := newTestSFTP(t)
|
|
||||||
runConformance(t, s)
|
|
||||||
|
|
||||||
// No temporary files may survive anywhere under the base directory.
|
|
||||||
_ = filepath.WalkDir(base, func(p string, d os.DirEntry, err error) error {
|
|
||||||
if err == nil && strings.HasSuffix(p, ".partial") {
|
|
||||||
t.Errorf("leftover partial file %s", p)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSFTPHostKeyVerification(t *testing.T) {
|
|
||||||
t.Setenv("SSH_AUTH_SOCK", "")
|
|
||||||
host, port, _ := startSFTPServer(t)
|
|
||||||
base := Config{SFTPHost: host, SFTPPort: port, SFTPUsername: testSFTPUser, SFTPPassword: testSFTPPass, SFTPDirectory: t.TempDir()}
|
|
||||||
|
|
||||||
// A known_hosts file with a different key for this host must be rejected.
|
|
||||||
_, otherPriv, _ := ed25519.GenerateKey(rand.Reader)
|
|
||||||
otherSigner, _ := ssh.NewSignerFromKey(otherPriv)
|
|
||||||
wrong := filepath.Join(t.TempDir(), "known_hosts")
|
|
||||||
line := knownhosts.Line([]string{knownhosts.Normalize(net.JoinHostPort(host, port))}, otherSigner.PublicKey())
|
|
||||||
if err := os.WriteFile(wrong, []byte(line+"\n"), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cfg := base
|
|
||||||
cfg.SFTPKnownHostsFile = wrong
|
|
||||||
if s, err := NewSFTPStorage(cfg); err == nil {
|
|
||||||
_ = s.Close()
|
|
||||||
t.Fatal("connected despite host key mismatch")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Missing known_hosts file is an error with a helpful message.
|
|
||||||
cfg = base
|
|
||||||
cfg.SFTPKnownHostsFile = filepath.Join(t.TempDir(), "nope")
|
|
||||||
if _, err := NewSFTPStorage(cfg); err == nil {
|
|
||||||
t.Fatal("connected without known_hosts")
|
|
||||||
}
|
|
||||||
t.Setenv("HOME", t.TempDir())
|
|
||||||
cfg.SFTPKnownHostsFile = ""
|
|
||||||
if _, err := os.Stat("/etc/ssh/ssh_known_hosts"); os.IsNotExist(err) {
|
|
||||||
if _, err := NewSFTPStorage(cfg); err == nil || !strings.Contains(err.Error(), "SSH_INSECURE_IGNORE_HOST_KEY") {
|
|
||||||
t.Fatalf("expected helpful known_hosts error, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Explicitly ignoring host keys works.
|
|
||||||
cfg.SFTPIgnoreKnownHosts = true
|
|
||||||
s, err := NewSFTPStorage(cfg)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("SFTPIgnoreKnownHosts: %v", err)
|
|
||||||
}
|
|
||||||
_ = s.Close()
|
|
||||||
}
|
|
||||||
-403
@@ -1,403 +0,0 @@
|
|||||||
/*
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2024 Jonas Kaninda
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
// Package go_storage provides a uniform, streaming storage abstraction over
|
|
||||||
// S3 (and S3-compatible services), Azure Blob Storage, Google Cloud Storage,
|
|
||||||
// SFTP, FTP and the local filesystem.
|
|
||||||
//
|
|
||||||
// All backends implement [Storage]. Objects are addressed by keys that are
|
|
||||||
// relative to the backend base (bucket, container, remote directory or local
|
|
||||||
// base path) and always use forward slashes.
|
|
||||||
package go_storage
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/rand"
|
|
||||||
"encoding/hex"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Sentinel errors. Backends wrap them with %w, so test with errors.Is.
|
|
||||||
var (
|
|
||||||
// ErrNotFound is returned (wrapped) by Stat, Get and Remove when the key does not exist.
|
|
||||||
ErrNotFound = errors.New("storage: object not found")
|
|
||||||
// ErrPreconditionFailed is returned (wrapped) by PutIf when the condition does not hold.
|
|
||||||
ErrPreconditionFailed = errors.New("storage: precondition failed")
|
|
||||||
// ErrConditionNotSupported is returned (wrapped) by PutIf when the backend cannot
|
|
||||||
// evaluate the requested condition atomically. Callers should fall back to locking.
|
|
||||||
ErrConditionNotSupported = errors.New("storage: conditional write not supported by backend")
|
|
||||||
)
|
|
||||||
|
|
||||||
// Storage defines the interface for remote file storage operations.
|
|
||||||
// All implementations are safe for concurrent use (see FTPStorage for the
|
|
||||||
// one restriction regarding open Get bodies).
|
|
||||||
type Storage interface {
|
|
||||||
// Upload uploads a local file to the remote storage with the given target name.
|
|
||||||
Upload(ctx context.Context, localPath string, remotePath string) error
|
|
||||||
|
|
||||||
// Download downloads a file from remote storage to a local path.
|
|
||||||
Download(ctx context.Context, remotePath string, localPath string) error
|
|
||||||
|
|
||||||
// List returns all items (recursively) in the storage that match the given prefix.
|
|
||||||
List(ctx context.Context, prefix string) ([]Item, error)
|
|
||||||
|
|
||||||
// Remove deletes a file from remote storage. A missing key returns an
|
|
||||||
// error wrapping ErrNotFound.
|
|
||||||
Remove(ctx context.Context, remotePath string) error
|
|
||||||
|
|
||||||
// Close releases any resources held by the storage implementation.
|
|
||||||
Close() error
|
|
||||||
|
|
||||||
// Put streams r to key. r may be of unknown length; memory use is bounded.
|
|
||||||
// If reading r fails or ctx is cancelled, Put returns an error and no
|
|
||||||
// (partial) object is left at key.
|
|
||||||
Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error)
|
|
||||||
|
|
||||||
// Get opens key for streaming reads. The caller must Close the body.
|
|
||||||
// A missing key returns an error wrapping ErrNotFound.
|
|
||||||
Get(ctx context.Context, key string) (io.ReadCloser, Item, error)
|
|
||||||
|
|
||||||
// Stat returns metadata for key. A missing key returns an error wrapping ErrNotFound.
|
|
||||||
Stat(ctx context.Context, key string) (Item, error)
|
|
||||||
|
|
||||||
// PutIf writes r to key only if cond holds, atomically with respect to
|
|
||||||
// other PutIf calls. It returns an error wrapping ErrPreconditionFailed when
|
|
||||||
// the condition does not hold, or ErrConditionNotSupported when the backend
|
|
||||||
// cannot evaluate it. Intended for small objects (index files).
|
|
||||||
PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error)
|
|
||||||
|
|
||||||
// ListDir lists only the files directly inside dir (dir is treated as a
|
|
||||||
// directory: "a/b" never matches "a/bc/x"; "" = root). Non-recursive, no
|
|
||||||
// directories are returned. A missing directory yields an empty list.
|
|
||||||
ListDir(ctx context.Context, dir string) ([]Item, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Item represents a file or directory in remote storage.
|
|
||||||
type Item struct {
|
|
||||||
Key string // The path/key of the item, relative to the backend base
|
|
||||||
ModifiedTime time.Time // Last modification time
|
|
||||||
Size int64 // Size in bytes (0 for directories)
|
|
||||||
IsDirectory bool // Whether this item is a directory
|
|
||||||
// ETag is an opaque version token usable with Condition.IfMatch
|
|
||||||
// (S3/Azure ETag; GCS generation as decimal string; local:
|
|
||||||
// "<mtime-unixnano hex>-<size hex>"; SFTP/FTP: "").
|
|
||||||
ETag string
|
|
||||||
// VersionID is the S3 version id when bucket versioning is enabled.
|
|
||||||
VersionID string
|
|
||||||
}
|
|
||||||
|
|
||||||
// PutOptions are optional parameters for Put and PutIf.
|
|
||||||
type PutOptions struct {
|
|
||||||
ContentType string
|
|
||||||
Metadata map[string]string
|
|
||||||
Size int64 // optional size hint; -1 or 0 = unknown
|
|
||||||
}
|
|
||||||
|
|
||||||
// Condition is a precondition for PutIf. Exactly one field must be set; a
|
|
||||||
// zero Condition is rejected (it would otherwise silently become an
|
|
||||||
// unconditional write, e.g. when an empty ETag is passed as IfMatch).
|
|
||||||
type Condition struct {
|
|
||||||
IfMatch string // only write if the current ETag equals this
|
|
||||||
IfNoneMatch bool // only write if the object does not exist
|
|
||||||
}
|
|
||||||
|
|
||||||
// Config holds configuration options for creating storage instances.
|
|
||||||
type Config struct {
|
|
||||||
// Local configuration
|
|
||||||
LocalPath string // base directory for the local backend (default "/backup")
|
|
||||||
|
|
||||||
// S3 configuration
|
|
||||||
S3Region string
|
|
||||||
S3Bucket string
|
|
||||||
S3Profile string
|
|
||||||
S3KeyID string
|
|
||||||
S3Secret string
|
|
||||||
S3SessionToken string
|
|
||||||
S3Endpoint string
|
|
||||||
S3ForcePath bool
|
|
||||||
S3DisableTLS bool
|
|
||||||
S3StorageClass string // e.g. STANDARD, STANDARD_IA, GLACIER_IR
|
|
||||||
S3SSE string // "AES256" or "aws:kms"
|
|
||||||
S3SSEKMSKeyID string // KMS key id when S3SSE is "aws:kms"
|
|
||||||
S3PartSize int64 // multipart part size in bytes (default 64 MiB, min 5 MiB)
|
|
||||||
S3Concurrency int // multipart upload concurrency (default 2)
|
|
||||||
|
|
||||||
// B2 configuration (reserved, no backend implementation yet)
|
|
||||||
B2KeyID string
|
|
||||||
B2AppKey string
|
|
||||||
B2Bucket string
|
|
||||||
B2ConcurrentConnections int
|
|
||||||
B2ForcePath bool
|
|
||||||
|
|
||||||
// Google Cloud Storage configuration
|
|
||||||
GCSBucket string
|
|
||||||
GCSEndpoint string
|
|
||||||
GCSCredentialsFile string
|
|
||||||
GCSCredentialsJSON string
|
|
||||||
|
|
||||||
// Azure configuration
|
|
||||||
AzureAccount string
|
|
||||||
AzureKey string
|
|
||||||
AzureContainer string
|
|
||||||
// AzureEndpoint is either a domain suffix (default "blob.core.windows.net",
|
|
||||||
// the service URL becomes https://<account>.<endpoint>) or a full service
|
|
||||||
// URL containing a scheme (e.g. "http://127.0.0.1:10000/devstoreaccount1"
|
|
||||||
// for Azurite), used as is.
|
|
||||||
AzureEndpoint string
|
|
||||||
// AzureSASToken, when set, is used instead of the shared key.
|
|
||||||
AzureSASToken string
|
|
||||||
|
|
||||||
// SFTP configuration
|
|
||||||
SFTPHost string
|
|
||||||
SFTPPort string
|
|
||||||
SFTPUsername string
|
|
||||||
SFTPPassword string
|
|
||||||
SFTPDirectory string
|
|
||||||
SFTPIdentityFile string
|
|
||||||
SFTPIdentityPassphrase string
|
|
||||||
SFTPKnownHostsFile string
|
|
||||||
SFTPIgnoreKnownHosts bool
|
|
||||||
|
|
||||||
// FTP configuration
|
|
||||||
FTPHost string
|
|
||||||
FTPPort string
|
|
||||||
FTPUsername string
|
|
||||||
FTPPassword string
|
|
||||||
FTPDirectory string
|
|
||||||
FTPTLS bool // explicit FTPS (AUTH TLS)
|
|
||||||
}
|
|
||||||
|
|
||||||
// StorageType represents the type of storage backend.
|
|
||||||
type StorageType string
|
|
||||||
|
|
||||||
// Supported storage types.
|
|
||||||
const (
|
|
||||||
StorageTypeS3 StorageType = "s3"
|
|
||||||
StorageTypeGCS StorageType = "gcs"
|
|
||||||
StorageTypeAzure StorageType = "azure"
|
|
||||||
StorageTypeSFTP StorageType = "sftp"
|
|
||||||
StorageTypeFTP StorageType = "ftp"
|
|
||||||
StorageTypeLocal StorageType = "local"
|
|
||||||
)
|
|
||||||
|
|
||||||
// DefaultLocalPath is the base directory of the local backend when Config.LocalPath is empty.
|
|
||||||
const DefaultLocalPath = "/backup"
|
|
||||||
|
|
||||||
// NewStorage creates a new Storage instance based on the specified type and configuration.
|
|
||||||
func NewStorage(storageType StorageType, config Config) (Storage, error) {
|
|
||||||
switch storageType {
|
|
||||||
case StorageTypeS3:
|
|
||||||
return NewS3Storage(config)
|
|
||||||
case StorageTypeGCS:
|
|
||||||
return NewGCSStorage(config)
|
|
||||||
case StorageTypeAzure:
|
|
||||||
return NewAzureStorage(config)
|
|
||||||
case StorageTypeSFTP:
|
|
||||||
return NewSFTPStorage(config)
|
|
||||||
case StorageTypeFTP:
|
|
||||||
return NewFTPStorage(config)
|
|
||||||
case StorageTypeLocal:
|
|
||||||
p := config.LocalPath
|
|
||||||
if p == "" {
|
|
||||||
p = DefaultLocalPath
|
|
||||||
}
|
|
||||||
return NewLocalStorage(p)
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("unsupported storage type: %s", storageType)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// normalizePathSeparators converts OS-specific path separators to forward slashes.
|
|
||||||
func normalizePathSeparators(p string) string {
|
|
||||||
if os.PathSeparator != '/' {
|
|
||||||
p = strings.ReplaceAll(p, string(os.PathSeparator), "/")
|
|
||||||
}
|
|
||||||
return strings.ReplaceAll(p, "\\", "/")
|
|
||||||
}
|
|
||||||
|
|
||||||
// cleanKey normalises an object key: separators become "/", the path is
|
|
||||||
// cleaned and leading slashes are removed. Keys escaping the base ("..")
|
|
||||||
// and empty keys are rejected.
|
|
||||||
func cleanKey(key string) (string, error) {
|
|
||||||
k, err := cleanDir(key)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if k == "" {
|
|
||||||
return "", fmt.Errorf("storage: empty key")
|
|
||||||
}
|
|
||||||
return k, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// cleanDir is like cleanKey but allows the empty string (root).
|
|
||||||
func cleanDir(dir string) (string, error) {
|
|
||||||
d := path.Clean("/" + normalizePathSeparators(dir))
|
|
||||||
d = strings.TrimPrefix(d, "/")
|
|
||||||
// path.Clean of an absolute path never yields "..", but check the raw
|
|
||||||
// input so that escaping attempts are reported rather than silently clamped.
|
|
||||||
for _, seg := range strings.Split(path.Clean(normalizePathSeparators(dir)), "/") {
|
|
||||||
if seg == ".." {
|
|
||||||
return "", fmt.Errorf("storage: key %q escapes the base directory", dir)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return d, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// cleanPrefix normalises a List prefix. Unlike keys, a prefix is not
|
|
||||||
// path-cleaned (a trailing "/" is significant) but must not contain "..".
|
|
||||||
func cleanPrefix(prefix string) (string, error) {
|
|
||||||
p := normalizePathSeparators(prefix)
|
|
||||||
for _, seg := range strings.Split(p, "/") {
|
|
||||||
if seg == ".." {
|
|
||||||
return "", fmt.Errorf("storage: prefix %q escapes the base directory", prefix)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return strings.TrimLeft(p, "/"), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// dirPrefix returns the object prefix for a directory ("" for root, "a/b/" otherwise).
|
|
||||||
func dirPrefix(dir string) string {
|
|
||||||
if dir == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return dir + "/"
|
|
||||||
}
|
|
||||||
|
|
||||||
// partialName returns a unique temporary name next to p.
|
|
||||||
func partialName(p string) string {
|
|
||||||
var b [6]byte
|
|
||||||
_, _ = rand.Read(b[:])
|
|
||||||
return p + "." + hex.EncodeToString(b[:]) + ".partial"
|
|
||||||
}
|
|
||||||
|
|
||||||
// ctxReader aborts reads once ctx is done.
|
|
||||||
type ctxReader struct {
|
|
||||||
ctx context.Context
|
|
||||||
r io.Reader
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c ctxReader) Read(p []byte) (int, error) {
|
|
||||||
if err := c.ctx.Err(); err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
return c.r.Read(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
// countingReader counts bytes read.
|
|
||||||
type countingReader struct {
|
|
||||||
r io.Reader
|
|
||||||
n int64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *countingReader) Read(p []byte) (int, error) {
|
|
||||||
n, err := c.r.Read(p)
|
|
||||||
c.n += int64(n)
|
|
||||||
return n, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// errEmptyCondition is returned by PutIf when neither IfMatch nor IfNoneMatch is set.
|
|
||||||
var errEmptyCondition = errors.New("storage: PutIf requires Condition.IfMatch or Condition.IfNoneMatch")
|
|
||||||
|
|
||||||
func validateCondition(c Condition) error {
|
|
||||||
if c.IfMatch == "" && !c.IfNoneMatch {
|
|
||||||
return errEmptyCondition
|
|
||||||
}
|
|
||||||
if c.IfMatch != "" && c.IfNoneMatch {
|
|
||||||
return errors.New("storage: Condition.IfMatch and Condition.IfNoneMatch are mutually exclusive")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// maxConditionalPutSize bounds the buffer used by backends whose conditional
|
|
||||||
// write needs a single request (S3).
|
|
||||||
const maxConditionalPutSize = 16 << 20
|
|
||||||
|
|
||||||
// uploadFile implements Storage.Upload on top of Put.
|
|
||||||
func uploadFile(ctx context.Context, s Storage, localPath, remotePath string) error {
|
|
||||||
f, err := os.Open(localPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to open local file: %w", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = f.Close() }()
|
|
||||||
opts := &PutOptions{Size: -1}
|
|
||||||
if fi, err := f.Stat(); err == nil {
|
|
||||||
opts.Size = fi.Size()
|
|
||||||
}
|
|
||||||
_, err = s.Put(ctx, remotePath, f, opts)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// downloadFile implements Storage.Download on top of Get.
|
|
||||||
func downloadFile(ctx context.Context, s Storage, remotePath, localPath string) (err error) {
|
|
||||||
body, _, err := s.Get(ctx, remotePath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer func() { _ = body.Close() }()
|
|
||||||
|
|
||||||
if dir := filepath.Dir(localPath); dir != "" {
|
|
||||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
||||||
return fmt.Errorf("failed to create local directory: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
f, err := os.Create(localPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to create local file: %w", err)
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
if cerr := f.Close(); cerr != nil && err == nil {
|
|
||||||
err = fmt.Errorf("failed to close local file: %w", cerr)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
_ = os.Remove(localPath)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
if _, err = io.Copy(f, ctxReader{ctx: ctx, r: body}); err != nil {
|
|
||||||
return fmt.Errorf("failed to download %s: %w", remotePath, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// validateConfig checks if the required configuration fields are present.
|
|
||||||
func validateConfig(_ Config, requiredFields map[string]string) error {
|
|
||||||
var missing []string
|
|
||||||
for field, value := range requiredFields {
|
|
||||||
if value == "" {
|
|
||||||
missing = append(missing, field)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(missing) > 0 {
|
|
||||||
return fmt.Errorf("missing required configuration: %s", strings.Join(missing, ", "))
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
-212
@@ -1,212 +0,0 @@
|
|||||||
/*
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2024 Jonas Kaninda
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package go_storage
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestStorageInterface(t *testing.T) {
|
|
||||||
var _ Storage = (*S3Storage)(nil)
|
|
||||||
var _ Storage = (*GCSStorage)(nil)
|
|
||||||
var _ Storage = (*AzureStorage)(nil)
|
|
||||||
var _ Storage = (*SFTPStorage)(nil)
|
|
||||||
var _ Storage = (*FTPStorage)(nil)
|
|
||||||
var _ Storage = (*LocalStorage)(nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCleanKey(t *testing.T) {
|
|
||||||
ok := map[string]string{
|
|
||||||
"x/y.txt": "x/y.txt",
|
|
||||||
"/a/b.txt": "a/b.txt",
|
|
||||||
"a//b/./c": "a/b/c",
|
|
||||||
`a\b\c.txt`: "a/b/c.txt",
|
|
||||||
"a/../b.txt": "b.txt",
|
|
||||||
"./x": "x",
|
|
||||||
"backup.sql~": "backup.sql~",
|
|
||||||
}
|
|
||||||
for in, want := range ok {
|
|
||||||
got, err := cleanKey(in)
|
|
||||||
if err != nil || got != want {
|
|
||||||
t.Errorf("cleanKey(%q) = %q, %v; want %q", in, got, err, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, in := range []string{"", "/", ".", "..", "../x", "a/../../x", `..\x`} {
|
|
||||||
if got, err := cleanKey(in); err == nil {
|
|
||||||
t.Errorf("cleanKey(%q) = %q, want error", in, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if d, err := cleanDir(""); err != nil || d != "" {
|
|
||||||
t.Errorf("cleanDir(\"\") = %q, %v", d, err)
|
|
||||||
}
|
|
||||||
if _, err := cleanPrefix("a/../../b"); err == nil {
|
|
||||||
t.Error("cleanPrefix with .. accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewStorageLocalPath(t *testing.T) {
|
|
||||||
dir := filepath.Join(t.TempDir(), "base")
|
|
||||||
s, err := NewStorage(StorageTypeLocal, Config{LocalPath: dir})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := s.(*LocalStorage).BasePath(); got != dir {
|
|
||||||
t.Fatalf("base path = %q, want %q", got, dir)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewS3StorageOffline(t *testing.T) {
|
|
||||||
s, err := NewS3Storage(Config{
|
|
||||||
S3Bucket: "test-bucket",
|
|
||||||
S3Endpoint: "127.0.0.1:9",
|
|
||||||
S3DisableTLS: true,
|
|
||||||
S3KeyID: "k",
|
|
||||||
S3Secret: "s",
|
|
||||||
S3PartSize: 1,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewS3Storage: %v", err)
|
|
||||||
}
|
|
||||||
if s.region != DefaultS3Region {
|
|
||||||
t.Errorf("region = %q", s.region)
|
|
||||||
}
|
|
||||||
if got := *s.client.Options().BaseEndpoint; got != "http://127.0.0.1:9" {
|
|
||||||
t.Errorf("endpoint = %q", got)
|
|
||||||
}
|
|
||||||
//nolint:staticcheck // manager is deprecated, see s3.go
|
|
||||||
if s.uploader.PartSize != MinS3PartSize || s.uploader.Concurrency != DefaultS3Concurrency {
|
|
||||||
t.Errorf("uploader part size/concurrency = %d/%d", s.uploader.PartSize, s.uploader.Concurrency)
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
if err := s.Upload(ctx, filepath.Join(t.TempDir(), "missing.txt"), "remote.txt"); err == nil {
|
|
||||||
t.Fatal("Upload of a missing local file succeeded")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLocalConformance(t *testing.T) {
|
|
||||||
s, err := NewLocalStorage(t.TempDir())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer func() { _ = s.Close() }()
|
|
||||||
runConformance(t, s)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLocalNoLeftovers(t *testing.T) {
|
|
||||||
base := t.TempDir()
|
|
||||||
s, err := NewLocalStorage(base)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ctx := context.Background()
|
|
||||||
_, _ = s.Put(ctx, "x/y.bin", &failingReader{r: strings.NewReader(strings.Repeat("a", 100)), n: 10, err: errors.New("boom")}, nil)
|
|
||||||
if _, err := s.PutIf(ctx, "x/idx.json", strings.NewReader("1"), Condition{IfNoneMatch: true}, nil); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_, _ = s.PutIf(ctx, "x/idx.json", strings.NewReader("2"), Condition{IfNoneMatch: true}, nil)
|
|
||||||
entries, err := os.ReadDir(filepath.Join(base, "x"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.Name() != "idx.json" {
|
|
||||||
t.Errorf("leftover file %s", e.Name())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLocalPutIfConcurrent checks that concurrent read-modify-write cycles
|
|
||||||
// with IfMatch never lose an update.
|
|
||||||
func TestLocalPutIfConcurrent(t *testing.T) {
|
|
||||||
s, err := NewLocalStorage(t.TempDir())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ctx := context.Background()
|
|
||||||
if _, err := s.PutIf(ctx, "idx", strings.NewReader(""), Condition{IfNoneMatch: true}, nil); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
const workers, perWorker = 8, 10
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for w := 0; w < workers; w++ {
|
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
for i := 0; i < perWorker; {
|
|
||||||
body, it, err := s.Get(ctx, "idx")
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
b, _ := io.ReadAll(body)
|
|
||||||
_ = body.Close()
|
|
||||||
_, err = s.PutIf(ctx, "idx", strings.NewReader(string(b)+"x"), Condition{IfMatch: it.ETag}, nil)
|
|
||||||
if errors.Is(err, ErrPreconditionFailed) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
i++
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
body, _, err := s.Get(ctx, "idx")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer func() { _ = body.Close() }()
|
|
||||||
b, _ := io.ReadAll(body)
|
|
||||||
if len(b) != workers*perWorker {
|
|
||||||
t.Fatalf("lost updates: got %d, want %d", len(b), workers*perWorker)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLocalPathTraversal(t *testing.T) {
|
|
||||||
base := t.TempDir()
|
|
||||||
s, err := NewLocalStorage(filepath.Join(base, "store"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, k := range []string{"../outside.txt", "a/../../outside.txt", `..\outside.txt`} {
|
|
||||||
if _, err := s.Put(context.Background(), k, strings.NewReader("x"), nil); err == nil {
|
|
||||||
t.Errorf("Put(%q) succeeded", k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(filepath.Join(base, "outside.txt")); !os.IsNotExist(err) {
|
|
||||||
t.Fatal("file written outside base")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Reference in new issue
Block a user