jkaninda/go-storage
1
0 Fork 0
mirror of https://github.com/jkaninda/go-storage.git synced 2026-11-13 08:37:16 +00:00
Code Issues Actions Projects Releases Wiki Activity

Compare commits

base: jkaninda/go-storage:v0.2.0-rc1
jkaninda/go-storage:main jkaninda/go-storage:dependabot/go_modules/golang.org/x/crypto-0.48.0 jkaninda/go-storage:dependabot/go_modules/github.com/Azure/azure-sdk-for-go/sdk/storage/azblob-1.6.4 jkaninda/go-storage:dependabot/go_modules/github.com/bramvdbogaerde/go-scp-1.6.0 jkaninda/go-storage:dependabot/go_modules/github.com/aws/aws-sdk-go-1.55.8
jkaninda/go-storage:v0.3.0 jkaninda/go-storage:v0.2.0 jkaninda/go-storage:v0.2.0-rc1 jkaninda/go-storage:v0.1.3 jkaninda/go-storage:v0.1.2 jkaninda/go-storage:v0.1.1 jkaninda/go-storage:v0.1.0
..
compare: jkaninda/go-storage:v0.2.0
jkaninda/go-storage:main jkaninda/go-storage:dependabot/go_modules/golang.org/x/crypto-0.48.0 jkaninda/go-storage:dependabot/go_modules/github.com/Azure/azure-sdk-for-go/sdk/storage/azblob-1.6.4 jkaninda/go-storage:dependabot/go_modules/github.com/bramvdbogaerde/go-scp-1.6.0 jkaninda/go-storage:dependabot/go_modules/github.com/aws/aws-sdk-go-1.55.8
jkaninda/go-storage:v0.3.0 jkaninda/go-storage:v0.2.0 jkaninda/go-storage:v0.2.0-rc1 jkaninda/go-storage:v0.1.3 jkaninda/go-storage:v0.1.2 jkaninda/go-storage:v0.1.1 jkaninda/go-storage:v0.1.0
1 Commits
v0.2.0-rc1 .. v0.2.0
Author SHA1 Message Date
jkaninda 2dc0864b35 feat: streaming Put/Get, conditional writes and aws-sdk-go-v2 (#33)
Go / test (push) Canceled after 0s
Details
Lint / Run on Ubuntu (push) Canceled after 0s
Details
Go / integration (push) Canceled after 0s
Details
2026-10-27 20:46:54 +01:00
25 changed files with 3538 additions and 1050 deletions
Expand all files Collapse all files

No files matched your search

.github/workflows/go.yml
+36 -16
View File
@@ -1,28 +1,48 @@
# This workflow will build a golang project
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
name: Go
on:
push:
branches: [ "main" ]
branches: [ "main", "v2" ]
pull_request:
branches: [ "main" ]
branches: [ "main", "v2" ]
jobs:
build:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: '1.21.0'
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version: '1.25'
- name: Build
run: go build -v ./...
- name: Build
run: go build -v ./...
- name: Test
run: go test -v ./...
- name: Vet
run: go vet ./... && go vet -tags integration ./...
- name: Unit tests
run: go test -race -count=1 ./...
integration:
runs-on: ubuntu-latest
needs: test
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.25'
- name: Pull images
run: |
docker pull pgsty/minio:latest
docker pull mcr.microsoft.com/azure-storage/azurite:latest
docker pull fsouza/fake-gcs-server:latest
docker pull delfer/alpine-ftp-server:latest
- name: Integration tests (MinIO, Azurite, fake-gcs-server, FTP)
run: go test -tags integration -count=1 -v -run Integration ./...
.github/workflows/lint.yml
+2 -2
View File
@@ -15,9 +15,9 @@ jobs:
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '~1.24'
go-version: '1.25'
- name: Run linter
uses: golangci/golangci-lint-action@v7
with:
version: v2.1.2
version: v2.13.2
.golangci.yml
+4
View File
@@ -32,6 +32,10 @@ linters:
- dupl
- lll
path: /*
# Legacy v0.1 API: deprecated as a whole and frozen.
- linters:
- staticcheck
path: ^pkg/
paths:
- third_party$
- builtin$
README.md
+176 -102
View File
@@ -1,126 +1,200 @@
# Go Storage
A simple Go storage client
Supported storage:
- Local
- S3
- SSH
- FTP
- Azure Blob
A streaming storage abstraction for Go backup tools.
```go
Supported backends:
- Local filesystem
- S3 and S3-compatible services (MinIO, Ceph, Wasabi, ...) via aws-sdk-go-v2
- Azure Blob Storage (and Azurite)
- Google Cloud Storage
- SFTP
- FTP / explicit FTPS
```shell
go get github.com/jkaninda/go-storage
```
### Local Storage
```go
localStorage := local.NewStorage(local.Config{
LocalPath: tmpPath,
RemotePath: backupDestination,
})
err = localStorage.Copy(finalFileName)
if err != nil {
log.Fatal("Error copying file, error %v", err)
}
```
## Streaming API
Every backend implements the same `Storage` interface:
### S3 Storage
```go
s3Storage, err := s3.NewStorage(s3.Config{
Endpoint: "",
Bucket: "",
AccessKey: "",
SecretKey: "",
Region: "",
DisableSsl: "",
ForcePathStyle: "",
RemotePath: "",
LocalPath: "",
})
if err != nil {
log.Fatalf("Error creating s3 storage: %s", err)
}
// Copy file to S3
err = s3Storage.Copy(finalFileName)
if err != nil {
log.Fatalf("Error copying file, error %v", err)
}
// Download file from S3
err = s3Storage.CopyFrom(finalFileName)
if err != nil {
log.Fatalf("Error copying file, error %v", err)
type Storage interface {
Upload(ctx context.Context, localPath, remotePath string) error
Download(ctx context.Context, remotePath, localPath string) error
List(ctx context.Context, prefix string) ([]Item, error) // recursive
Remove(ctx context.Context, remotePath string) error // missing -> ErrNotFound
Close() error
Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error)
Get(ctx context.Context, key string) (io.ReadCloser, Item, error)
Stat(ctx context.Context, key string) (Item, error)
PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error)
ListDir(ctx context.Context, dir string) ([]Item, error) // non-recursive, files only
}
```
### SSH Storage
- **`Put`** streams a reader of unknown length with bounded memory (no temp
files, no whole-file buffering). If the reader fails or the context is
cancelled, `Put` returns an error and **no object (and no partial object)
is left at the key**: S3 aborts the multipart upload, Azure never commits
the block list, GCS cancels the resumable upload, and SFTP/FTP/local write
to `<key>.<random>.partial` and rename on success.
- **`Get`** returns a streaming body; always `Close` it. On FTP the body holds
the single control connection, so close it before making another call.
- **`PutIf`** is a conditional write for small files such as a backup index
(optimistic concurrency): `Condition{IfNoneMatch: true}` creates only if the
key does not exist, `Condition{IfMatch: item.ETag}` replaces only if the
object is unchanged. It returns `ErrPreconditionFailed` when the condition
does not hold and `ErrConditionNotSupported` when the backend cannot
evaluate it (fall back to locking).
- Errors wrap the sentinels `ErrNotFound`, `ErrPreconditionFailed` and
`ErrConditionNotSupported`; test them with `errors.Is`.
- Keys are relative to the backend base (bucket, container, remote directory
or local base path), use `/` separators and may not escape the base (`..`).
| Backend | `Item.ETag` | `PutIf` IfNoneMatch | `PutIf` IfMatch |
|---------|--------------------------------|---------------------|-----------------|
| S3 | S3 ETag | yes | yes |
| Azure | blob ETag | yes | yes |
| GCS | object generation | yes | yes |
| Local | `<mtime-ns hex>-<size hex>` | yes (lock file) | yes (lock file) |
| SFTP | empty | yes (`O_EXCL`) | not supported |
| FTP | empty | not supported | not supported |
S3 `PutIf` uses a single `PutObject` request and is limited to 16 MiB.
### Example: dump, compress and upload without temp files
```go
sshStorage, err := ssh.NewStorage(ssh.Config{
Host: "",
Port: 22,
User: "",
Password: "",
RemotePath: "",
LocalPath: "",
package main
import (
"compress/gzip"
"context"
"errors"
"io"
"log"
"os/exec"
"strings"
storage "github.com/jkaninda/go-storage"
)
func main() {
ctx := context.Background()
s, err := storage.NewStorage(storage.StorageTypeS3, storage.Config{
S3Bucket: "backups",
S3Region: "us-east-1",
S3KeyID: "...",
S3Secret: "...",
S3Endpoint: "https://minio.example.com", // optional, for S3-compatible services
S3ForcePath: true,
})
if err != nil {
log.Fatalf("Error creating SSH storage: %s", err)
log.Fatal(err)
}
// Copy file to the remote server
err = sshStorage.Copy(finalFileName)
defer s.Close()
pr, pw := io.Pipe()
go func() {
gz := gzip.NewWriter(pw)
cmd := exec.CommandContext(ctx, "pg_dump", "mydb")
cmd.Stdout = gz
err := cmd.Run()
if err == nil {
err = gz.Close()
}
pw.CloseWithError(err) // a non-nil error aborts the upload
}()
item, err := s.Put(ctx, "db/mydb-2026-01-01.sql.gz", pr, &storage.PutOptions{ContentType: "application/gzip"})
if err != nil {
log.Fatalf("Error copying backup file: %s", err)
log.Fatal(err)
}
// Download file from SSH remote server
err = sshStorage.CopyFrom(finalFileName)
if err != nil {
log.Fatalf("Error copying file, error %v", err)
}
```
### FTP Storage
```go
ftpStorage, err := ftp.NewStorage(ftp.Config{
Host: "",
Port: 21,
User: "",
Password: "",
RemotePath: "",
LocalPath: "",
})
if err != nil {
log.Fatalf("Error creating FTP storage: %s", err)
log.Printf("uploaded %s (%d bytes, etag %s)", item.Key, item.Size, item.ETag)
// Restore: stream it back.
body, _, err := s.Get(ctx, item.Key)
if errors.Is(err, storage.ErrNotFound) {
log.Fatal("backup not found")
} else if err != nil {
log.Fatal(err)
}
err = ftpStorage.Copy(finalFileName)
if err != nil {
log.Fatalf("Error copying backup file: %s", err)
defer body.Close()
zr, _ := gzip.NewReader(body)
_, _ = io.Copy(io.Discard, zr)
// Update an index file with optimistic concurrency.
idx, err := s.Stat(ctx, "db/index.json")
cond := storage.Condition{IfMatch: idx.ETag}
if errors.Is(err, storage.ErrNotFound) {
cond = storage.Condition{IfNoneMatch: true}
}
_, err = s.PutIf(ctx, "db/index.json", strings.NewReader(`{"backups":[]}`), cond, nil)
if errors.Is(err, storage.ErrPreconditionFailed) {
// someone else updated the index: re-read, merge and retry
}
// Download file from ftp remote server
err = ftpStorage.CopyFrom(finalFileName)
if err != nil {
log.Fatalf("Error copying file, error %v", err)
}
```
### Azure Blob storage
## Configuration
```go
azureStorage, err := azure.NewStorage(azure.Config{
ContainerName: '',
AccountName: '',
AccountKey: '',
RemotePath: '',
LocalPath: '',
})
if err != nil {
log.Fatal("Error creating Azure Blob storage storage: %s", err)
}
err = azureStorage.Copy(finalFileName)
if err != nil {
log.Fatal("Error copying file: %s", err)
}
`storage.NewStorage(storageType, storage.Config{...})` with `storageType` one
of `s3`, `gcs`, `azure`, `sftp`, `ftp`, `local`.
// Download file from Azure Blob remote server
err = azureStorage.CopyFrom(finalFileName)
if err != nil {
log.Fatalf("Error copying file, error %v", err)
}
```
| Field | Backend | Description |
|-------|---------|-------------|
| `LocalPath` | local | Base directory (default `/backup`) |
| `S3Bucket` | s3 | Bucket (required) |
| `S3Region` | s3 | Region (default from AWS config, else `us-east-1`) |
| `S3KeyID`, `S3Secret`, `S3SessionToken` | s3 | Static credentials; when empty the default AWS chain is used (env, shared profile, IRSA/web identity, ECS/EC2 roles) |
| `S3Profile` | s3 | Shared config profile |
| `S3Endpoint` | s3 | Custom endpoint (MinIO etc.); without a scheme `https://` is assumed (`http://` with `S3DisableTLS`) |
| `S3ForcePath` | s3 | Path-style addressing |
| `S3DisableTLS` | s3 | Use `http://` for scheme-less endpoints |
| `S3StorageClass` | s3 | e.g. `STANDARD_IA`, `GLACIER_IR` |
| `S3SSE`, `S3SSEKMSKeyID` | s3 | Server-side encryption: `AES256` or `aws:kms` (+ key id) |
| `S3PartSize` | s3 | Multipart part size (default 64 MiB, min 5 MiB) |
| `S3Concurrency` | s3 | Parallel part uploads (default 2); memory ≈ part size × concurrency |
| `AzureAccount`, `AzureKey` | azure | Shared key credentials |
| `AzureSASToken` | azure | SAS token (used instead of the shared key when set) |
| `AzureContainer` | azure | Container (required) |
| `AzureEndpoint` | azure | Domain suffix (default `blob.core.windows.net`) or a full service URL such as `http://127.0.0.1:10000/devstoreaccount1` (Azurite) |
| `GCSBucket` | gcs | Bucket (required) |
| `GCSCredentialsFile`, `GCSCredentialsJSON` | gcs | Credentials; default is Application Default Credentials |
| `GCSEndpoint` | gcs | Custom endpoint |
| `SFTPHost`, `SFTPPort` | sftp | Server (port default 22) |
| `SFTPUsername`, `SFTPPassword` | sftp | Password authentication |
| `SFTPIdentityFile`, `SFTPIdentityPassphrase` | sftp | Private key authentication (an SSH agent is used too when available) |
| `SFTPKnownHostsFile` | sftp | known_hosts file (default `~/.ssh/known_hosts`); host keys are always verified unless... |
| `SFTPIgnoreKnownHosts` | sftp | ...this is `true` (insecure) |
| `SFTPDirectory` | sftp | Remote base directory (default: login directory) |
| `FTPHost`, `FTPPort`, `FTPUsername`, `FTPPassword` | ftp | Server and credentials (port default 21) |
| `FTPDirectory` | ftp | Remote base directory (default: login directory) |
| `FTPTLS` | ftp | Explicit FTPS (`AUTH TLS`) |
The library never reads environment variables itself (except what the cloud
SDKs do for their default credential chains, and `SSH_AUTH_SOCK` for the SSH agent).
## Testing
```shell
go test ./... # unit tests: local + in-process SFTP server
go test -tags integration ./... # + MinIO, Azurite, fake-gcs-server, FTP via docker
```
Integration tests skip when docker is unavailable. Images can be overridden
with `GO_STORAGE_MINIO_IMAGE`, `GO_STORAGE_AZURITE_IMAGE`,
`GO_STORAGE_FAKEGCS_IMAGE` and `GO_STORAGE_FTP_IMAGE`.
## Deprecated: `pkg/` (v0.1 API)
The packages under `pkg/` (`pkg/local`, `pkg/s3`, `pkg/ssh`, `pkg/ftp`,
`pkg/azure`) are the legacy v0.1 API. They still compile but are deprecated
and will be removed in a future release; use the root package instead.
## License
MIT License, Copyright (c) 2024 Jonas Kaninda. See [LICENSE](LICENSE).
azure.go
+277 -83
View File
@@ -1,11 +1,50 @@
/*
MIT License
Copyright (c) 2024 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package go_storage
import (
"context"
"errors"
"fmt"
"os"
"io"
"net/http"
"strings"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
)
// Azure streaming upload tuning: 8 MiB blocks allow blobs up to ~390 GiB
// (50,000 blocks) while keeping memory bounded to blockSize*concurrency.
const (
azureBlockSize int64 = 8 << 20
azureConcurrency = 2
)
// AzureStorage implements Storage interface for Azure Blob Storage.
@@ -15,6 +54,9 @@ type AzureStorage struct {
}
// NewAzureStorage creates a new Azure Blob Storage instance.
//
// Authentication: AzureSASToken if set, otherwise shared key
// (AzureAccount/AzureKey), otherwise anonymous.
func NewAzureStorage(config Config) (*AzureStorage, error) {
if err := validateConfig(config, map[string]string{
"container": config.AzureContainer,
@@ -22,134 +64,286 @@ func NewAzureStorage(config Config) (*AzureStorage, error) {
return nil, fmt.Errorf("azure: %w", err)
}
// Try to get credentials from environment if not provided
account := config.AzureAccount
if account == "" {
account = os.Getenv("AZURE_STORAGE_ACCOUNT")
serviceURL, err := azureServiceURL(config)
if err != nil {
return nil, err
}
key := config.AzureKey
if key == "" {
key = os.Getenv("AZURE_STORAGE_KEY")
var client *azblob.Client
switch {
case config.AzureSASToken != "":
u := serviceURL + "?" + strings.TrimPrefix(config.AzureSASToken, "?")
client, err = azblob.NewClientWithNoCredential(u, nil)
case config.AzureAccount != "" && config.AzureKey != "":
cred, cerr := azblob.NewSharedKeyCredential(config.AzureAccount, config.AzureKey)
if cerr != nil {
return nil, fmt.Errorf("azure: failed to create credentials: %w", cerr)
}
client, err = azblob.NewClientWithSharedKeyCredential(serviceURL, cred, nil)
default:
client, err = azblob.NewClientWithNoCredential(serviceURL, nil)
}
if err != nil {
return nil, fmt.Errorf("azure: failed to create client: %w", err)
}
return &AzureStorage{client: client, container: config.AzureContainer}, nil
}
func azureServiceURL(config Config) (string, error) {
endpoint := config.AzureEndpoint
if endpoint == "" {
endpoint = "blob.core.windows.net"
}
if strings.Contains(endpoint, "://") {
return strings.TrimSuffix(endpoint, "/") + "/", nil
}
if config.AzureAccount == "" {
return "", fmt.Errorf("azure: missing required configuration: account")
}
return fmt.Sprintf("https://%s.%s/", config.AzureAccount, strings.Trim(endpoint, "/")), nil
}
var client *azblob.Client
var err error
// Client returns the underlying azblob client.
func (a *AzureStorage) Client() *azblob.Client { return a.client }
if account == "" {
// Anonymous access
client, err = azblob.NewClientWithNoCredential(endpoint, nil)
if err != nil {
return nil, fmt.Errorf("azure: failed to create anonymous client: %w", err)
func (a *AzureStorage) put(ctx context.Context, key string, r io.Reader, opts *PutOptions, ac *blob.AccessConditions) (Item, error) {
k, err := cleanKey(key)
if err != nil {
return Item{}, fmt.Errorf("azure: %w", err)
}
o := &azblob.UploadStreamOptions{
BlockSize: azureBlockSize,
Concurrency: azureConcurrency,
AccessConditions: ac,
}
if opts != nil {
if opts.ContentType != "" {
o.HTTPHeaders = &blob.HTTPHeaders{BlobContentType: &opts.ContentType}
}
} else {
// Authenticated access
credential, err := azblob.NewSharedKeyCredential(account, key)
if err != nil {
return nil, fmt.Errorf("azure: failed to create credentials: %w", err)
}
url := fmt.Sprintf("https://%s.%s", account, endpoint)
client, err = azblob.NewClientWithSharedKeyCredential(url, credential, nil)
if err != nil {
return nil, fmt.Errorf("azure: failed to create client: %w", err)
if len(opts.Metadata) > 0 {
o.Metadata = make(map[string]*string, len(opts.Metadata))
for mk, mv := range opts.Metadata {
o.Metadata[mk] = &mv
}
}
}
cr := &countingReader{r: ctxReader{ctx: ctx, r: r}}
resp, err := a.client.UploadStream(ctx, a.container, k, cr, o)
if err != nil {
if ac != nil && (isAzurePrecondition(err) || bloberror.HasCode(err, bloberror.BlobNotFound)) {
return Item{}, fmt.Errorf("azure: conditional put of %s: %w", k, ErrPreconditionFailed)
}
return Item{}, fmt.Errorf("azure: failed to upload %s to container %s: %w", k, a.container, err)
}
it := Item{Key: k, Size: cr.n, ModifiedTime: time.Now().UTC()}
if resp.ETag != nil {
it.ETag = string(*resp.ETag)
}
if resp.LastModified != nil {
it.ModifiedTime = *resp.LastModified
}
if resp.VersionID != nil {
it.VersionID = *resp.VersionID
}
return it, nil
}
return &AzureStorage{
client: client,
container: config.AzureContainer,
}, nil
// Put streams r to key with a block upload; the block list is only
// committed when the whole stream was read successfully.
func (a *AzureStorage) Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error) {
return a.put(ctx, key, r, opts, nil)
}
// PutIf writes r to key with If-Match / If-None-Match: * access conditions.
func (a *AzureStorage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error) {
if err := validateCondition(cond); err != nil {
return Item{}, fmt.Errorf("azure: %w", err)
}
mac := &blob.ModifiedAccessConditions{}
if cond.IfMatch != "" {
etag := azcore.ETag(cond.IfMatch)
mac.IfMatch = &etag
}
if cond.IfNoneMatch {
etag := azcore.ETagAny
mac.IfNoneMatch = &etag
}
return a.put(ctx, key, r, opts, &blob.AccessConditions{ModifiedAccessConditions: mac})
}
// Get streams the blob body.
func (a *AzureStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
k, err := cleanKey(key)
if err != nil {
return nil, Item{}, fmt.Errorf("azure: %w", err)
}
resp, err := a.client.DownloadStream(ctx, a.container, k, nil)
if err != nil {
return nil, Item{}, a.mapErr("get", k, err)
}
it := Item{Key: k}
if resp.ContentLength != nil {
it.Size = *resp.ContentLength
}
if resp.LastModified != nil {
it.ModifiedTime = *resp.LastModified
}
if resp.ETag != nil {
it.ETag = string(*resp.ETag)
}
if resp.VersionID != nil {
it.VersionID = *resp.VersionID
}
return resp.Body, it, nil
}
// Stat returns blob properties.
func (a *AzureStorage) Stat(ctx context.Context, key string) (Item, error) {
k, err := cleanKey(key)
if err != nil {
return Item{}, fmt.Errorf("azure: %w", err)
}
bc := a.client.ServiceClient().NewContainerClient(a.container).NewBlobClient(k)
resp, err := bc.GetProperties(ctx, nil)
if err != nil {
return Item{}, a.mapErr("stat", k, err)
}
it := Item{Key: k}
if resp.ContentLength != nil {
it.Size = *resp.ContentLength
}
if resp.LastModified != nil {
it.ModifiedTime = *resp.LastModified
}
if resp.ETag != nil {
it.ETag = string(*resp.ETag)
}
if resp.VersionID != nil {
it.VersionID = *resp.VersionID
}
return it, nil
}
// Upload uploads a file to Azure Blob Storage.
func (a *AzureStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
file, err := os.Open(localPath)
if err != nil {
return fmt.Errorf("azure: failed to open local file: %w", err)
if err := uploadFile(ctx, a, localPath, remotePath); err != nil {
return fmt.Errorf("azure: %w", err)
}
defer func(file *os.File) {
err = file.Close()
if err != nil {
fmt.Printf("azure: failed to close file: %v\n", err)
}
}(file)
_, err = a.client.UploadFile(ctx, a.container, normalizePathSeparators(remotePath), file, nil)
if err != nil {
return fmt.Errorf("azure: failed to upload to container %s: %w", a.container, err)
}
return nil
}
// Download downloads a file from Azure Blob Storage.
func (a *AzureStorage) Download(ctx context.Context, remotePath string, localPath string) error {
file, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("azure: failed to create local file: %w", err)
if err := downloadFile(ctx, a, remotePath, localPath); err != nil {
return fmt.Errorf("azure: %w", err)
}
defer func(file *os.File) {
err = file.Close()
if err != nil {
fmt.Printf("azure: failed to close file: %v\n", err)
}
}(file)
_, err = a.client.DownloadFile(ctx, a.container, normalizePathSeparators(remotePath), file, nil)
if err != nil {
return fmt.Errorf("azure: failed to download from container %s: %w", a.container, err)
}
return nil
}
// List lists blobs in Azure Blob Storage with the given prefix.
// List lists blobs (recursively) with the given prefix.
func (a *AzureStorage) List(ctx context.Context, prefix string) ([]Item, error) {
normalizedPrefix := normalizePathSeparators(prefix)
pager := a.client.NewListBlobsFlatPager(a.container, &azblob.ListBlobsFlatOptions{
Prefix: &normalizedPrefix,
})
p, err := cleanPrefix(prefix)
if err != nil {
return nil, fmt.Errorf("azure: %w", err)
}
pager := a.client.NewListBlobsFlatPager(a.container, &azblob.ListBlobsFlatOptions{Prefix: &p})
items := make([]Item, 0)
for pager.More() {
resp, err := pager.NextPage(ctx)
if err != nil {
return nil, fmt.Errorf("azure: failed to list blobs in container %s: %w", a.container, err)
}
for _, blob := range resp.Segment.BlobItems {
items = append(items, Item{
Key: *blob.Name,
ModifiedTime: *blob.Properties.LastModified,
Size: *blob.Properties.ContentLength,
IsDirectory: false,
})
for _, b := range resp.Segment.BlobItems {
items = append(items, azureItem(b))
}
}
return items, nil
}
// Remove deletes a blob from Azure Blob Storage.
func (a *AzureStorage) Remove(ctx context.Context, remotePath string) error {
_, err := a.client.DeleteBlob(ctx, a.container, normalizePathSeparators(remotePath), nil)
// ListDir lists the blobs directly under dir.
func (a *AzureStorage) ListDir(ctx context.Context, dir string) ([]Item, error) {
d, err := cleanDir(dir)
if err != nil {
return fmt.Errorf("azure: failed to remove blob from container %s: %w", a.container, err)
return nil, fmt.Errorf("azure: %w", err)
}
p := dirPrefix(d)
cc := a.client.ServiceClient().NewContainerClient(a.container)
pager := cc.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{Prefix: &p})
items := make([]Item, 0)
for pager.More() {
resp, err := pager.NextPage(ctx)
if err != nil {
return nil, fmt.Errorf("azure: failed to list blobs in container %s: %w", a.container, err)
}
for _, b := range resp.Segment.BlobItems {
if b.Name == nil || strings.HasSuffix(*b.Name, "/") {
continue
}
items = append(items, azureItem(b))
}
}
return items, nil
}
func azureItem(b *container.BlobItem) Item {
it := Item{}
if b.Name != nil {
it.Key = *b.Name
}
if b.VersionID != nil {
it.VersionID = *b.VersionID
}
if p := b.Properties; p != nil {
if p.LastModified != nil {
it.ModifiedTime = *p.LastModified
}
if p.ContentLength != nil {
it.Size = *p.ContentLength
}
if p.ETag != nil {
it.ETag = string(*p.ETag)
}
}
return it
}
// Remove deletes a blob.
func (a *AzureStorage) Remove(ctx context.Context, remotePath string) error {
k, err := cleanKey(remotePath)
if err != nil {
return fmt.Errorf("azure: %w", err)
}
if _, err := a.client.DeleteBlob(ctx, a.container, k, nil); err != nil {
return a.mapErr("remove", k, err)
}
return nil
}
// Close releases resources.
// Close releases resources (no-op).
func (a *AzureStorage) Close() error {
return nil
}
func (a *AzureStorage) mapErr(op, key string, err error) error {
if bloberror.HasCode(err, bloberror.BlobNotFound, bloberror.ResourceNotFound) {
return fmt.Errorf("azure: %s %s in container %s: %w", op, key, a.container, ErrNotFound)
}
var re *azcore.ResponseError
if errors.As(err, &re) && re.StatusCode == http.StatusNotFound && re.ErrorCode != string(bloberror.ContainerNotFound) {
return fmt.Errorf("azure: %s %s in container %s: %w", op, key, a.container, ErrNotFound)
}
return fmt.Errorf("azure: %s %s in container %s: %w", op, key, a.container, err)
}
func isAzurePrecondition(err error) bool {
if bloberror.HasCode(err, bloberror.ConditionNotMet, bloberror.BlobAlreadyExists, bloberror.TargetConditionNotMet) {
return true
}
var re *azcore.ResponseError
if errors.As(err, &re) {
return re.StatusCode == http.StatusPreconditionFailed ||
(re.StatusCode == http.StatusConflict && re.ErrorCode == string(bloberror.BlobAlreadyExists))
}
return false
}
conformance_test.go
+364
View File
@@ -0,0 +1,364 @@
/*
MIT License
Copyright (c) 2024 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package go_storage
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"io"
"math/rand"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"time"
)
const conformanceStreamSize = 20 << 20
// deterministicStream writes n pseudo-random bytes into an io.Pipe and
// returns the reader plus the expected SHA-256.
func deterministicStream(n int64, seed int64) (io.Reader, [32]byte) {
h := sha256.New()
_, _ = io.Copy(h, io.LimitReader(rand.New(rand.NewSource(seed)), n))
var sum [32]byte
copy(sum[:], h.Sum(nil))
pr, pw := io.Pipe()
go func() {
_, err := io.Copy(pw, io.LimitReader(rand.New(rand.NewSource(seed)), n))
_ = pw.CloseWithError(err)
}()
return pr, sum
}
// failingReader returns ok bytes, then err.
type failingReader struct {
r io.Reader
n int64
err error
}
func (f *failingReader) Read(p []byte) (int, error) {
if f.n <= 0 {
return 0, f.err
}
if int64(len(p)) > f.n {
p = p[:f.n]
}
n, err := f.r.Read(p)
f.n -= int64(n)
return n, err
}
func keysOf(items []Item, filesOnly bool) []string {
out := make([]string, 0, len(items))
for _, it := range items {
if filesOnly && it.IsDirectory {
continue
}
out = append(out, it.Key)
}
sort.Strings(out)
return out
}
func putString(t *testing.T, s Storage, key, content string) Item {
t.Helper()
it, err := s.Put(context.Background(), key, strings.NewReader(content), &PutOptions{ContentType: "text/plain"})
if err != nil {
t.Fatalf("Put(%s): %v", key, err)
}
return it
}
func getString(t *testing.T, s Storage, key string) string {
t.Helper()
body, _, err := s.Get(context.Background(), key)
if err != nil {
t.Fatalf("Get(%s): %v", key, err)
}
defer func() { _ = body.Close() }()
b, err := io.ReadAll(body)
if err != nil {
t.Fatalf("read %s: %v", key, err)
}
return string(b)
}
// runConformance exercises the full Storage contract against s.
func runConformance(t *testing.T, s Storage) {
root := fmt.Sprintf("conf-%d", time.Now().UnixNano())
k := func(p string) string { return root + "/" + p }
for _, tc := range []struct {
name string
fn func(*testing.T, Storage, func(string) string)
}{
{"StreamRoundTrip", confStreamRoundTrip},
{"FailingReaderLeavesNothing", confFailingReaderLeavesNothing},
{"ListDirBoundary", confListDirBoundary},
{"ListRecursive", confListRecursive},
{"Remove", confRemove},
{"PathEscape", confPathEscape},
{"PutIf", confPutIf},
{"UploadDownload", confUploadDownload},
} {
t.Run(tc.name, func(t *testing.T) { tc.fn(t, s, k) })
}
}
func confStreamRoundTrip(t *testing.T, s Storage, k func(string) string) {
ctx := context.Background()
r, want := deterministicStream(conformanceStreamSize, 42)
it, err := s.Put(ctx, k("stream/big.bin"), r, &PutOptions{ContentType: "application/octet-stream", Size: -1})
if err != nil {
t.Fatalf("Put: %v", err)
}
if it.Size != conformanceStreamSize {
t.Errorf("Put size = %d, want %d", it.Size, conformanceStreamSize)
}
st, err := s.Stat(ctx, k("stream/big.bin"))
if err != nil {
t.Fatalf("Stat: %v", err)
}
if st.Size != conformanceStreamSize {
t.Errorf("Stat size = %d", st.Size)
}
if st.Key != k("stream/big.bin") {
t.Errorf("Stat key = %q", st.Key)
}
if st.ETag != it.ETag {
t.Errorf("Stat ETag %q != Put ETag %q", st.ETag, it.ETag)
}
body, gi, err := s.Get(ctx, k("stream/big.bin"))
if err != nil {
t.Fatalf("Get: %v", err)
}
h := sha256.New()
_, err = io.Copy(h, body)
_ = body.Close()
if err != nil {
t.Fatalf("read body: %v", err)
}
if !bytes.Equal(h.Sum(nil), want[:]) {
t.Fatal("content mismatch")
}
if gi.Size != conformanceStreamSize {
t.Errorf("Get item size = %d", gi.Size)
}
}
func confFailingReaderLeavesNothing(t *testing.T, s Storage, k func(string) string) {
ctx := context.Background()
key := k("fail/broken.bin")
boom := errors.New("boom")
r := &failingReader{r: rand.New(rand.NewSource(1)), n: 18 << 20, err: boom}
if _, err := s.Put(ctx, key, r, nil); err == nil {
t.Fatal("Put with failing reader succeeded")
}
if _, err := s.Stat(ctx, key); !errors.Is(err, ErrNotFound) {
t.Fatalf("Stat after failed Put: %v, want ErrNotFound", err)
}
items, err := s.ListDir(ctx, k("fail"))
if err != nil {
t.Fatalf("ListDir: %v", err)
}
for _, it := range items {
if strings.HasSuffix(it.Key, ".partial") || strings.HasSuffix(it.Key, "broken.bin") {
t.Errorf("leftover after failed Put: %s", it.Key)
}
}
cctx, cancel := context.WithCancel(ctx)
pr, pw := io.Pipe()
go func() {
_, _ = pw.Write(make([]byte, 1<<20))
cancel()
_, _ = pw.Write(make([]byte, 1<<20))
_ = pw.Close()
}()
if _, err := s.Put(cctx, k("fail/cancelled.bin"), pr, nil); err == nil {
t.Fatal("Put with cancelled context succeeded")
}
_ = pr.Close()
if _, err := s.Stat(ctx, k("fail/cancelled.bin")); !errors.Is(err, ErrNotFound) {
t.Fatalf("Stat after cancelled Put: %v, want ErrNotFound", err)
}
}
func confListDirBoundary(t *testing.T, s Storage, k func(string) string) {
ctx := context.Background()
for _, p := range []string{"ld/a/b/1.txt", "ld/a/b/2.txt", "ld/a/b/sub/deep.txt", "ld/a/bc/x.txt", "ld/top.txt"} {
putString(t, s, k(p), "x")
}
want := []string{k("ld/a/b/1.txt"), k("ld/a/b/2.txt")}
for _, dir := range []string{k("ld/a/b"), k("ld/a/b/")} {
items, err := s.ListDir(ctx, dir)
if err != nil {
t.Fatalf("ListDir(%s): %v", dir, err)
}
if got := keysOf(items, false); strings.Join(got, ",") != strings.Join(want, ",") {
t.Errorf("ListDir(%s) = %v, want %v", dir, got, want)
}
for _, it := range items {
if it.IsDirectory || it.Size != 1 {
t.Errorf("ListDir item %+v", it)
}
}
}
items, err := s.ListDir(ctx, k("ld"))
if err != nil {
t.Fatal(err)
}
if got := keysOf(items, false); len(got) != 1 || got[0] != k("ld/top.txt") {
t.Errorf("ListDir(ld) = %v", got)
}
items, err = s.ListDir(ctx, k("ld/missing"))
if err != nil || len(items) != 0 {
t.Errorf("ListDir(missing) = %v, %v", items, err)
}
}
func confListRecursive(t *testing.T, s Storage, k func(string) string) {
ctx := context.Background()
items, err := s.List(ctx, k("ld/a/b/"))
if err != nil {
t.Fatal(err)
}
want := []string{k("ld/a/b/1.txt"), k("ld/a/b/2.txt"), k("ld/a/b/sub/deep.txt")}
if got := keysOf(items, true); strings.Join(got, ",") != strings.Join(want, ",") {
t.Errorf("List(ld/a/b/) = %v, want %v", got, want)
}
items, err = s.List(ctx, k("ld/"))
if err != nil {
t.Fatal(err)
}
if got := keysOf(items, true); len(got) != 5 {
t.Errorf("List(ld/) = %v, want 5 files", got)
}
}
func confRemove(t *testing.T, s Storage, k func(string) string) {
ctx := context.Background()
if err := s.Remove(ctx, k("does/not/exist.txt")); !errors.Is(err, ErrNotFound) {
t.Fatalf("Remove missing: %v, want ErrNotFound", err)
}
putString(t, s, k("rm/x.txt"), "x")
if err := s.Remove(ctx, k("rm/x.txt")); err != nil {
t.Fatalf("Remove: %v", err)
}
if _, err := s.Stat(ctx, k("rm/x.txt")); !errors.Is(err, ErrNotFound) {
t.Fatalf("Stat after Remove: %v", err)
}
if _, _, err := s.Get(ctx, k("rm/x.txt")); !errors.Is(err, ErrNotFound) {
t.Fatalf("Get after Remove: %v", err)
}
}
func confPathEscape(t *testing.T, s Storage, k func(string) string) {
ctx := context.Background()
if _, err := s.Put(ctx, "../escape.txt", strings.NewReader("x"), nil); err == nil {
t.Fatal("Put ../escape.txt succeeded")
}
if _, err := s.Stat(ctx, "a/../../escape.txt"); err == nil || errors.Is(err, ErrNotFound) {
t.Fatalf("Stat escaping key: %v", err)
}
}
func confPutIf(t *testing.T, s Storage, k func(string) string) {
ctx := context.Background()
key := k("index/index.json")
it1, err := s.PutIf(ctx, key, strings.NewReader(`{"v":1}`), Condition{IfNoneMatch: true}, &PutOptions{ContentType: "application/json"})
if errors.Is(err, ErrConditionNotSupported) {
t.Skip("backend does not support conditional writes")
}
if err != nil {
t.Fatalf("PutIf IfNoneMatch (new): %v", err)
}
if _, err := s.PutIf(ctx, key, strings.NewReader(`{"v":2}`), Condition{IfNoneMatch: true}, nil); !errors.Is(err, ErrPreconditionFailed) {
t.Fatalf("PutIf IfNoneMatch (exists): %v, want ErrPreconditionFailed", err)
}
if got := getString(t, s, key); got != `{"v":1}` {
t.Fatalf("content after rejected PutIf = %q", got)
}
it2, err := s.PutIf(ctx, key, strings.NewReader(`{"v":2}`), Condition{IfMatch: it1.ETag}, nil)
if errors.Is(err, ErrConditionNotSupported) {
t.Log("IfMatch not supported; skipping IfMatch checks")
return
}
if err != nil {
t.Fatalf("PutIf IfMatch (fresh): %v", err)
}
if it2.ETag == "" || it2.ETag == it1.ETag {
t.Fatalf("ETag did not change: %q -> %q", it1.ETag, it2.ETag)
}
if _, err := s.PutIf(ctx, key, strings.NewReader(`{"v":3}`), Condition{IfMatch: it1.ETag}, nil); !errors.Is(err, ErrPreconditionFailed) {
t.Fatalf("PutIf IfMatch (stale): %v, want ErrPreconditionFailed", err)
}
if got := getString(t, s, key); got != `{"v":2}` {
t.Fatalf("content = %q, want v2", got)
}
st, err := s.Stat(ctx, key)
if err != nil {
t.Fatal(err)
}
if st.ETag != it2.ETag {
t.Fatalf("Stat ETag %q != PutIf ETag %q", st.ETag, it2.ETag)
}
if _, err := s.PutIf(ctx, k("index/missing.json"), strings.NewReader("x"), Condition{IfMatch: it2.ETag}, nil); !errors.Is(err, ErrPreconditionFailed) {
t.Fatalf("PutIf IfMatch on missing key: %v, want ErrPreconditionFailed", err)
}
}
func confUploadDownload(t *testing.T, s Storage, k func(string) string) {
ctx := context.Background()
dir := t.TempDir()
src := filepath.Join(dir, "src.txt")
if err := os.WriteFile(src, []byte("hello world"), 0o644); err != nil {
t.Fatal(err)
}
if err := s.Upload(ctx, src, k("ud/file.txt")); err != nil {
t.Fatalf("Upload: %v", err)
}
dst := filepath.Join(dir, "out", "dst.txt")
if err := s.Download(ctx, k("ud/file.txt"), dst); err != nil {
t.Fatalf("Download: %v", err)
}
b, err := os.ReadFile(dst)
if err != nil || string(b) != "hello world" {
t.Fatalf("downloaded %q, %v", b, err)
}
if err := s.Download(ctx, k("ud/missing.txt"), filepath.Join(dir, "m.txt")); !errors.Is(err, ErrNotFound) {
t.Fatalf("Download missing: %v", err)
}
if _, err := os.Stat(filepath.Join(dir, "m.txt")); !os.IsNotExist(err) {
t.Fatal("Download of missing key left a local file")
}
}
ftp.go
+299 -154
View File
@@ -1,23 +1,58 @@
/*
MIT License
Copyright (c) 2024 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package go_storage
import (
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"net"
"net/textproto"
"path"
"strings"
"sync"
"time"
"github.com/jlaffaye/ftp"
)
// FTPStorage implements Storage interface for FTP.
// FTPStorage implements Storage interface for FTP (optionally explicit FTPS).
//
// FTP uses a single control connection, so all operations are serialised by a
// mutex. The body returned by Get holds that connection until it is closed:
// a Get body MUST be closed before any other call on the same FTPStorage,
// otherwise that call blocks.
type FTPStorage struct {
host string
port string
baseDir string
client *ftp.ServerConn
mu sync.Mutex
client *ftp.ServerConn
}
// NewFTPStorage creates a new FTP storage instance.
@@ -32,220 +67,330 @@ func NewFTPStorage(config Config) (*FTPStorage, error) {
if port == "" {
port = "21"
}
hostPort := net.JoinHostPort(config.FTPHost, port)
// Connect to FTP server
hostPort := fmt.Sprintf("%s:%s", config.FTPHost, port)
conn, err := ftp.Dial(hostPort, ftp.DialWithTimeout(30*time.Second))
opts := []ftp.DialOption{ftp.DialWithTimeout(30 * time.Second)}
if config.FTPTLS {
opts = append(opts, ftp.DialWithExplicitTLS(&tls.Config{
ServerName: config.FTPHost,
MinVersion: tls.VersionTLS12,
}))
}
conn, err := ftp.Dial(hostPort, opts...)
if err != nil {
return nil, fmt.Errorf("ftp: failed to connect to %s: %w", hostPort, err)
}
// Login
if err := conn.Login(config.FTPUsername, config.FTPPassword); err != nil {
conn.Quit()
_ = conn.Quit()
return nil, fmt.Errorf("ftp: login failed: %w", err)
}
baseDir := normalizePathSeparators(config.FTPDirectory)
if baseDir == "" {
if wd, err := conn.CurrentDir(); err == nil && wd != "" {
baseDir = wd
} else {
baseDir = "/"
}
}
return &FTPStorage{
host: config.FTPHost,
port: port,
baseDir: config.FTPDirectory,
baseDir: path.Clean(baseDir),
client: conn,
}, nil
}
func (f *FTPStorage) resolve(key string) (string, string, error) {
k, err := cleanKey(key)
if err != nil {
return "", "", fmt.Errorf("ftp: %w", err)
}
return k, path.Join(f.baseDir, k), nil
}
func ftpCode(err error) int {
var te *textproto.Error
if errors.As(err, &te) {
return te.Code
}
return 0
}
func ftpNotFound(err error) bool {
return ftpCode(err) == ftp.StatusFileUnavailable
}
// Put streams r to <key>.<rand>.partial and renames it on success.
func (f *FTPStorage) Put(ctx context.Context, key string, r io.Reader, _ *PutOptions) (Item, error) {
k, full, err := f.resolve(key)
if err != nil {
return Item{}, err
}
if err := ctx.Err(); err != nil {
return Item{}, err
}
f.mu.Lock()
defer f.mu.Unlock()
f.ensureDirectory(path.Dir(full))
tmp := partialName(full)
err = f.client.Stor(tmp, ctxReader{ctx: ctx, r: r})
if err == nil {
err = ctx.Err()
}
if err != nil {
_ = f.client.Delete(tmp)
return Item{}, fmt.Errorf("ftp: failed to upload %s: %w", k, err)
}
if err := f.client.Rename(tmp, full); err != nil {
// Some servers refuse to rename onto an existing file.
_ = f.client.Delete(full)
if err := f.client.Rename(tmp, full); err != nil {
_ = f.client.Delete(tmp)
return Item{}, fmt.Errorf("ftp: failed to rename %s: %w", k, err)
}
}
return f.stat(k, full)
}
// PutIf is not supported on FTP: it always returns ErrConditionNotSupported.
// Callers should fall back to a lock-based protocol.
func (f *FTPStorage) PutIf(_ context.Context, key string, _ io.Reader, _ Condition, _ *PutOptions) (Item, error) {
return Item{}, fmt.Errorf("ftp: conditional put of %s: %w", key, ErrConditionNotSupported)
}
// ftpBody releases the connection lock when closed.
type ftpBody struct {
ctx context.Context
resp *ftp.Response
once sync.Once
mu *sync.Mutex
}
func (b *ftpBody) Read(p []byte) (int, error) {
if err := b.ctx.Err(); err != nil {
return 0, err
}
return b.resp.Read(p)
}
func (b *ftpBody) Close() error {
var err error
b.once.Do(func() {
err = b.resp.Close()
b.mu.Unlock()
})
return err
}
// Get opens key for streaming reads. The returned body holds the FTP control
// connection: close it before issuing any other call on this storage.
func (f *FTPStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
k, full, err := f.resolve(key)
if err != nil {
return nil, Item{}, err
}
if err := ctx.Err(); err != nil {
return nil, Item{}, err
}
f.mu.Lock()
it, err := f.stat(k, full)
if err != nil {
f.mu.Unlock()
return nil, Item{}, err
}
if it.IsDirectory {
f.mu.Unlock()
return nil, Item{}, fmt.Errorf("ftp: %s is a directory", k)
}
resp, err := f.client.Retr(full)
if err != nil {
f.mu.Unlock()
if ftpNotFound(err) {
return nil, Item{}, fmt.Errorf("ftp: get %s: %w", k, ErrNotFound)
}
return nil, Item{}, fmt.Errorf("ftp: failed to retrieve %s: %w", k, err)
}
return &ftpBody{ctx: ctx, resp: resp, mu: &f.mu}, it, nil
}
// Stat returns metadata for key (from a listing of its parent directory).
func (f *FTPStorage) Stat(ctx context.Context, key string) (Item, error) {
k, full, err := f.resolve(key)
if err != nil {
return Item{}, err
}
if err := ctx.Err(); err != nil {
return Item{}, err
}
f.mu.Lock()
defer f.mu.Unlock()
return f.stat(k, full)
}
func (f *FTPStorage) stat(key, full string) (Item, error) {
entries, err := f.client.List(path.Dir(full))
if err != nil {
if ftpNotFound(err) {
return Item{}, fmt.Errorf("ftp: stat %s: %w", key, ErrNotFound)
}
return Item{}, fmt.Errorf("ftp: stat %s: %w", key, err)
}
name := path.Base(full)
for _, e := range entries {
if e.Name == name {
return ftpItem(key, e), nil
}
}
return Item{}, fmt.Errorf("ftp: stat %s: %w", key, ErrNotFound)
}
func ftpItem(key string, e *ftp.Entry) Item {
it := Item{Key: key, ModifiedTime: e.Time, IsDirectory: e.Type == ftp.EntryTypeFolder}
if !it.IsDirectory {
it.Size = int64(e.Size) //nolint:gosec // sizes fit in int64
}
return it
}
// Upload uploads a file via FTP.
func (f *FTPStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
src, err := os.Open(localPath)
if err != nil {
return fmt.Errorf("ftp: failed to open local file: %w", err)
}
defer src.Close()
fullRemotePath := filepath.Join(f.baseDir, remotePath)
fullRemotePath = normalizePathSeparators(fullRemotePath)
// Create parent directory if needed
remoteDir := filepath.Dir(fullRemotePath)
remoteDir = normalizePathSeparators(remoteDir)
if remoteDir != "." && remoteDir != "/" {
f.ensureDirectory(remoteDir)
}
// Upload file
if err := f.client.Stor(fullRemotePath, src); err != nil {
return fmt.Errorf("ftp: failed to upload file %s: %w", fullRemotePath, err)
}
return nil
return uploadFile(ctx, f, localPath, remotePath)
}
// Download downloads a file via FTP.
func (f *FTPStorage) Download(ctx context.Context, remotePath string, localPath string) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
fullRemotePath := filepath.Join(f.baseDir, remotePath)
fullRemotePath = normalizePathSeparators(fullRemotePath)
// Get the file from FTP server
resp, err := f.client.Retr(fullRemotePath)
if err != nil {
return fmt.Errorf("ftp: failed to retrieve file %s: %w", fullRemotePath, err)
}
defer resp.Close()
// Create local file
dst, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("ftp: failed to create local file: %w", err)
}
defer dst.Close()
// Copy data
if _, err := io.Copy(dst, resp); err != nil {
return fmt.Errorf("ftp: failed to transfer data: %w", err)
}
return nil
return downloadFile(ctx, f, remotePath, localPath)
}
// List lists files via FTP with the given prefix.
// List lists files and directories (recursively) whose key starts with prefix.
func (f *FTPStorage) List(ctx context.Context, prefix string) ([]Item, error) {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
p, err := cleanPrefix(prefix)
if err != nil {
return nil, fmt.Errorf("ftp: %w", err)
}
f.mu.Lock()
defer f.mu.Unlock()
items := make([]Item, 0)
// Normalize the base directory
baseDir := normalizePathSeparators(f.baseDir)
if baseDir == "" {
baseDir = "/"
startRel := ""
if i := strings.LastIndex(p, "/"); i >= 0 {
startRel = p[:i]
}
// Walk the directory
if err := f.walkDirectory(baseDir, prefix, &items); err != nil {
if err := f.walk(ctx, startRel, p, &items); err != nil {
return nil, err
}
return items, nil
}
// walkDirectory recursively walks FTP directory structure.
func (f *FTPStorage) walkDirectory(dir string, prefix string, items *[]Item) error {
entries, err := f.client.List(dir)
if err != nil {
return fmt.Errorf("ftp: failed to list directory %s: %w", dir, err)
func (f *FTPStorage) walk(ctx context.Context, rel, prefix string, items *[]Item) error {
if err := ctx.Err(); err != nil {
return err
}
for _, entry := range entries {
// Get relative path
var relPath string
if f.baseDir == "" || f.baseDir == "/" {
relPath = filepath.Join(strings.TrimPrefix(dir, "/"), entry.Name)
} else {
fullPath := filepath.Join(dir, entry.Name)
relPath = strings.TrimPrefix(fullPath, f.baseDir)
relPath = strings.TrimPrefix(relPath, "/")
entries, err := f.client.List(path.Join(f.baseDir, rel))
if err != nil {
if ftpNotFound(err) {
return nil
}
// Normalize separators for comparison
relPath = normalizePathSeparators(relPath)
// Skip if doesn't match prefix
if prefix != "" && !strings.HasPrefix(relPath, prefix) {
// For directories, we might need to walk them to find matching files
if entry.Type == ftp.EntryTypeFolder {
fullPath := filepath.Join(dir, entry.Name)
fullPath = normalizePathSeparators(fullPath)
if err := f.walkDirectory(fullPath, prefix, items); err != nil {
// Log but don't fail on subdirectory errors
continue
}
}
return fmt.Errorf("ftp: failed to list directory %s: %w", rel, err)
}
for _, e := range entries {
if e.Name == "." || e.Name == ".." || e.Type == ftp.EntryTypeLink {
continue
}
item := Item{
Key: relPath,
ModifiedTime: entry.Time,
Size: int64(entry.Size),
IsDirectory: entry.Type == ftp.EntryTypeFolder,
key := path.Join(rel, e.Name)
isDir := e.Type == ftp.EntryTypeFolder
if strings.HasPrefix(key, prefix) {
*items = append(*items, ftpItem(key, e))
} else if !isDir || !strings.HasPrefix(prefix, key+"/") {
continue
}
*items = append(*items, item)
// Recursively walk directories
if entry.Type == ftp.EntryTypeFolder {
fullPath := filepath.Join(dir, entry.Name)
fullPath = normalizePathSeparators(fullPath)
if err := f.walkDirectory(fullPath, prefix, items); err != nil {
// Log but don't fail on subdirectory errors
continue
if isDir {
if err := f.walk(ctx, key, prefix, items); err != nil {
return err
}
}
}
return nil
}
// ListDir lists the files directly inside dir.
func (f *FTPStorage) ListDir(ctx context.Context, dir string) ([]Item, error) {
d, err := cleanDir(dir)
if err != nil {
return nil, fmt.Errorf("ftp: %w", err)
}
if err := ctx.Err(); err != nil {
return nil, err
}
f.mu.Lock()
defer f.mu.Unlock()
entries, err := f.client.List(path.Join(f.baseDir, d))
if err != nil {
if ftpNotFound(err) {
return []Item{}, nil
}
return nil, fmt.Errorf("ftp: failed to list directory %s: %w", dir, err)
}
items := make([]Item, 0, len(entries))
for _, e := range entries {
if e.Type != ftp.EntryTypeFile {
continue
}
items = append(items, ftpItem(path.Join(d, e.Name), e))
}
return items, nil
}
// Remove deletes a file via FTP.
func (f *FTPStorage) Remove(ctx context.Context, remotePath string) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
k, full, err := f.resolve(remotePath)
if err != nil {
return err
}
fullRemotePath := filepath.Join(f.baseDir, remotePath)
fullRemotePath = normalizePathSeparators(fullRemotePath)
if err := f.client.Delete(fullRemotePath); err != nil {
return fmt.Errorf("ftp: failed to remove %s: %w", fullRemotePath, err)
if err := ctx.Err(); err != nil {
return err
}
f.mu.Lock()
defer f.mu.Unlock()
if err := f.client.Delete(full); err != nil {
if ftpNotFound(err) {
return fmt.Errorf("ftp: remove %s: %w", k, ErrNotFound)
}
return fmt.Errorf("ftp: failed to remove %s: %w", k, err)
}
return nil
}
// Close releases resources.
func (f *FTPStorage) Close() error {
f.mu.Lock()
defer f.mu.Unlock()
if err := f.client.Quit(); err != nil {
return fmt.Errorf("ftp: failed to close connection: %w", err)
}
return nil
}
// ensureDirectory creates a directory path if it doesn't exist.
func (f *FTPStorage) ensureDirectory(path string) {
parts := strings.Split(path, "/")
// ensureDirectory creates a directory path if it doesn't exist (errors are
// ignored: the directory usually exists already, and a real failure surfaces
// on the following STOR).
func (f *FTPStorage) ensureDirectory(dir string) {
if dir == "." || dir == "/" || dir == "" {
return
}
current := ""
for _, part := range parts {
if strings.HasPrefix(dir, "/") {
current = "/"
}
for _, part := range strings.Split(dir, "/") {
if part == "" {
continue
}
if current == "" {
current = part
} else {
current = current + "/" + part
}
// Try to create directory (ignore errors if it already exists)
current = path.Join(current, part)
_ = f.client.MakeDir(current)
}
}
gcs.go
+223 -82
View File
@@ -1,12 +1,42 @@
/*
MIT License
Copyright (c) 2024 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package go_storage
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"strconv"
"strings"
"cloud.google.com/go/storage"
"google.golang.org/api/googleapi"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
)
@@ -18,6 +48,10 @@ type GCSStorage struct {
}
// NewGCSStorage creates a new Google Cloud Storage instance.
//
// Credentials: GCSCredentialsJSON, else GCSCredentialsFile, else Application
// Default Credentials. The STORAGE_EMULATOR_HOST environment variable is
// honoured by the Google client library itself.
func NewGCSStorage(config Config) (*GCSStorage, error) {
if err := validateConfig(config, map[string]string{
"bucket": config.GCSBucket,
@@ -25,128 +59,222 @@ func NewGCSStorage(config Config) (*GCSStorage, error) {
return nil, fmt.Errorf("gcs: %w", err)
}
options := make([]option.ClientOption, 0)
options := make([]option.ClientOption, 0, 2)
if config.GCSEndpoint != "" {
options = append(options, option.WithEndpoint(config.GCSEndpoint))
}
if config.GCSCredentialsFile != "" {
options = append(options, option.WithCredentialsFile(config.GCSCredentialsFile))
switch {
case config.GCSCredentialsJSON != "":
data := []byte(config.GCSCredentialsJSON)
options = append(options, option.WithAuthCredentialsJSON(gcsCredType(data), data))
case config.GCSCredentialsFile != "":
data, err := os.ReadFile(config.GCSCredentialsFile)
if err != nil {
return nil, fmt.Errorf("gcs: failed to read credentials file: %w", err)
}
options = append(options, option.WithAuthCredentialsFile(gcsCredType(data), config.GCSCredentialsFile))
}
return newGCSStorage(context.Background(), config.GCSBucket, options...)
}
client, err := storage.NewClient(context.Background(), options...)
func newGCSStorage(ctx context.Context, bucket string, options ...option.ClientOption) (*GCSStorage, error) {
client, err := storage.NewClient(ctx, options...)
if err != nil {
return nil, fmt.Errorf("gcs: failed to create client: %w", err)
}
return &GCSStorage{client: client, bucket: bucket}, nil
}
return &GCSStorage{
client: client,
bucket: config.GCSBucket,
// gcsCredType detects the credential type from the JSON "type" field.
func gcsCredType(data []byte) option.CredentialsType {
var v struct {
Type string `json:"type"`
}
_ = json.Unmarshal(data, &v)
switch v.Type {
case "authorized_user":
return option.AuthorizedUser
case "impersonated_service_account":
return option.ImpersonatedServiceAccount
case "external_account":
return option.ExternalAccount
default:
return option.ServiceAccount
}
}
// Client returns the underlying GCS client.
func (g *GCSStorage) Client() *storage.Client { return g.client }
func gcsItem(attrs *storage.ObjectAttrs) Item {
return Item{
Key: attrs.Name,
ModifiedTime: attrs.Updated,
Size: attrs.Size,
ETag: strconv.FormatInt(attrs.Generation, 10),
}
}
func (g *GCSStorage) put(ctx context.Context, key string, r io.Reader, opts *PutOptions, cond *storage.Conditions) (Item, error) {
k, err := cleanKey(key)
if err != nil {
return Item{}, fmt.Errorf("gcs: %w", err)
}
obj := g.client.Bucket(g.bucket).Object(k)
if cond != nil {
obj = obj.If(*cond)
}
// A child context: cancelling it before Close aborts the upload so the
// object is never finalised.
wctx, cancel := context.WithCancel(ctx)
defer cancel()
w := obj.NewWriter(wctx)
if opts != nil {
w.ContentType = opts.ContentType
if len(opts.Metadata) > 0 {
w.Metadata = opts.Metadata
}
}
if _, err := io.Copy(w, ctxReader{ctx: ctx, r: r}); err != nil {
cancel()
_ = w.Close()
if cond != nil && isGCSPrecondition(err) {
return Item{}, fmt.Errorf("gcs: conditional put of %s: %w", k, ErrPreconditionFailed)
}
return Item{}, fmt.Errorf("gcs: failed to upload %s: %w", k, err)
}
if err := ctx.Err(); err != nil {
cancel()
_ = w.Close()
return Item{}, fmt.Errorf("gcs: failed to upload %s: %w", k, err)
}
if err := w.Close(); err != nil {
if cond != nil && isGCSPrecondition(err) {
return Item{}, fmt.Errorf("gcs: conditional put of %s: %w", k, ErrPreconditionFailed)
}
return Item{}, fmt.Errorf("gcs: failed to finalize upload of %s: %w", k, err)
}
return gcsItem(w.Attrs()), nil
}
// Put streams r to key.
func (g *GCSStorage) Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error) {
return g.put(ctx, key, r, opts, nil)
}
// PutIf writes r to key with a generation precondition (IfMatch is the
// generation as returned in Item.ETag).
func (g *GCSStorage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error) {
if err := validateCondition(cond); err != nil {
return Item{}, fmt.Errorf("gcs: %w", err)
}
c := storage.Conditions{}
switch {
case cond.IfNoneMatch:
c.DoesNotExist = true
default:
gen, err := strconv.ParseInt(cond.IfMatch, 10, 64)
if err != nil || gen <= 0 {
return Item{}, fmt.Errorf("gcs: invalid generation %q: %w", cond.IfMatch, ErrPreconditionFailed)
}
c.GenerationMatch = gen
}
return g.put(ctx, key, r, opts, &c)
}
// Get streams the object body.
func (g *GCSStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
k, err := cleanKey(key)
if err != nil {
return nil, Item{}, fmt.Errorf("gcs: %w", err)
}
rd, err := g.client.Bucket(g.bucket).Object(k).NewReader(ctx)
if err != nil {
return nil, Item{}, g.mapErr("get", k, err)
}
return rd, Item{
Key: k,
Size: rd.Attrs.Size,
ModifiedTime: rd.Attrs.LastModified,
ETag: strconv.FormatInt(rd.Attrs.Generation, 10),
}, nil
}
// Stat returns object attributes.
func (g *GCSStorage) Stat(ctx context.Context, key string) (Item, error) {
k, err := cleanKey(key)
if err != nil {
return Item{}, fmt.Errorf("gcs: %w", err)
}
attrs, err := g.client.Bucket(g.bucket).Object(k).Attrs(ctx)
if err != nil {
return Item{}, g.mapErr("stat", k, err)
}
return gcsItem(attrs), nil
}
// Upload uploads a file to GCS.
func (g *GCSStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
file, err := os.Open(localPath)
if err != nil {
return fmt.Errorf("gcs: failed to open local file: %w", err)
if err := uploadFile(ctx, g, localPath, remotePath); err != nil {
return fmt.Errorf("gcs: %w", err)
}
defer func(file *os.File) {
err = file.Close()
if err != nil {
fmt.Printf("gcs: failed to close file: %v\n", err)
}
}(file)
obj := g.client.Bucket(g.bucket).Object(normalizePathSeparators(remotePath)).NewWriter(ctx)
defer func(obj *storage.Writer) {
err = obj.Close()
if err != nil {
fmt.Printf("gcs: failed to close object writer: %v\n", err)
}
}(obj)
if _, err := io.Copy(obj, file); err != nil {
return fmt.Errorf("gcs: failed to write data: %w", err)
}
if err = obj.Close(); err != nil {
return fmt.Errorf("gcs: failed to finalize upload: %w", err)
}
return nil
}
// Download downloads a file from GCS.
func (g *GCSStorage) Download(ctx context.Context, remotePath string, localPath string) error {
file, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("gcs: failed to create local file: %w", err)
if err := downloadFile(ctx, g, remotePath, localPath); err != nil {
return fmt.Errorf("gcs: %w", err)
}
defer func(file *os.File) {
err = file.Close()
if err != nil {
fmt.Printf("gcs: failed to close file: %v\n", err)
}
}(file)
obj, err := g.client.Bucket(g.bucket).Object(normalizePathSeparators(remotePath)).NewReader(ctx)
if err != nil {
return fmt.Errorf("gcs: failed to create reader: %w", err)
}
defer func(obj *storage.Reader) {
err = obj.Close()
if err != nil {
fmt.Printf("gcs: failed to close object reader: %v\n", err)
}
}(obj)
if _, err = io.Copy(file, obj); err != nil {
return fmt.Errorf("gcs: failed to read data: %w", err)
}
return nil
}
// List lists objects in GCS with the given prefix.
// List lists objects (recursively) with the given prefix.
func (g *GCSStorage) List(ctx context.Context, prefix string) ([]Item, error) {
items := make([]Item, 0)
query := &storage.Query{
Prefix: normalizePathSeparators(prefix),
p, err := cleanPrefix(prefix)
if err != nil {
return nil, fmt.Errorf("gcs: %w", err)
}
return g.list(ctx, &storage.Query{Prefix: p})
}
it := g.client.Bucket(g.bucket).Objects(ctx, query)
// ListDir lists the objects directly under dir.
func (g *GCSStorage) ListDir(ctx context.Context, dir string) ([]Item, error) {
d, err := cleanDir(dir)
if err != nil {
return nil, fmt.Errorf("gcs: %w", err)
}
return g.list(ctx, &storage.Query{Prefix: dirPrefix(d), Delimiter: "/"})
}
func (g *GCSStorage) list(ctx context.Context, q *storage.Query) ([]Item, error) {
items := make([]Item, 0)
it := g.client.Bucket(g.bucket).Objects(ctx, q)
for {
attrs, err := it.Next()
if err == iterator.Done {
if errors.Is(err, iterator.Done) {
break
}
if err != nil {
return nil, fmt.Errorf("gcs: failed to iterate objects: %w", err)
return nil, fmt.Errorf("gcs: failed to list objects in bucket %s: %w", g.bucket, err)
}
items = append(items, Item{
Key: attrs.Name,
ModifiedTime: attrs.Updated,
Size: attrs.Size,
IsDirectory: false,
})
if attrs.Name == "" || (q.Delimiter != "" && strings.HasSuffix(attrs.Name, "/")) {
continue // synthetic prefix or directory marker
}
items = append(items, gcsItem(attrs))
}
return items, nil
}
// Remove deletes an object from GCS.
// Remove deletes an object.
func (g *GCSStorage) Remove(ctx context.Context, remotePath string) error {
obj := g.client.Bucket(g.bucket).Object(normalizePathSeparators(remotePath))
if err := obj.Delete(ctx); err != nil {
return fmt.Errorf("gcs: failed to remove object from bucket %s: %w", g.bucket, err)
k, err := cleanKey(remotePath)
if err != nil {
return fmt.Errorf("gcs: %w", err)
}
if err := g.client.Bucket(g.bucket).Object(k).Delete(ctx); err != nil {
return g.mapErr("remove", k, err)
}
return nil
}
@@ -154,3 +282,16 @@ func (g *GCSStorage) Remove(ctx context.Context, remotePath string) error {
func (g *GCSStorage) Close() error {
return g.client.Close()
}
func (g *GCSStorage) mapErr(op, key string, err error) error {
var ge *googleapi.Error
if errors.Is(err, storage.ErrObjectNotExist) || (errors.As(err, &ge) && ge.Code == http.StatusNotFound) {
return fmt.Errorf("gcs: %s %s in bucket %s: %w", op, key, g.bucket, ErrNotFound)
}
return fmt.Errorf("gcs: %s %s in bucket %s: %w", op, key, g.bucket, err)
}
func isGCSPrecondition(err error) bool {
var ge *googleapi.Error
return errors.As(err, &ge) && (ge.Code == http.StatusPreconditionFailed || ge.Code == http.StatusConflict)
}
go.mod
+21 -2
View File
@@ -1,11 +1,18 @@
module github.com/jkaninda/go-storage
go 1.24.0
go 1.25
require (
cloud.google.com/go/storage v1.60.0
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
github.com/aws/aws-sdk-go v1.55.7
github.com/aws/aws-sdk-go-v2 v1.47.3
github.com/aws/aws-sdk-go-v2/config v1.33.4
github.com/aws/aws-sdk-go-v2/credentials v1.20.9
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.43
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3
github.com/aws/smithy-go v1.28.5
github.com/bramvdbogaerde/go-scp v1.5.0
github.com/jlaffaye/ftp v0.2.0
github.com/pkg/sftp v1.13.10
@@ -21,11 +28,23 @@ require (
cloud.google.com/go/compute/metadata v0.9.0 // indirect
cloud.google.com/go/iam v1.5.3 // indirect
cloud.google.com/go/monitoring v1.24.3 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
go.sum
+38
View File
@@ -42,6 +42,44 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapp
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc=
github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE=
github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk=
github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 h1:oiEmPz/YMMeh/JvAE8xSxNCIsvWE6lpFe4ZPlyFykQI=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22/go.mod h1:QWUDuRi9PFjBHIU2iH1Pkldbe7zqLfXJC4wWLbF8nuQ=
github.com/aws/aws-sdk-go-v2/config v1.33.4 h1:FzvkXKSzwqHni4U7nDigHg4jjtqMpVUuHgmZfSoJVQ0=
github.com/aws/aws-sdk-go-v2/config v1.33.4/go.mod h1:VZqGZnZsCWVfK/iGPptJIyNIX3XEX6iQU2Rel4sLrr8=
github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0=
github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.43 h1:Fhx4uwBshQF0+jmhV/FKDM4LPj7L9iUy5n3MvLxLy00=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.43/go.mod h1:4Tyfc1eIPEyrAMysLCVkcYL1Hm/hkGWbElLo8pejHPo=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 h1:CwvFlKw0uUScfgOMflXSRZa+ayXgOf6wk7lhyMemQYU=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7/go.mod h1:4SbD8BB+p+QEnG0aX6DoWwrtk9SufyYXRsUv0nve8LE=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 h1:NyeuyXkZL476uoAowNActm4XJO1QPKf7aassgXHpiqA=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3/go.mod h1:GfoBeKMDJecgRUsGf/0RGdx0NJAh81ObTd922h8pQys=
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A=
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M=
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk=
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA=
github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 h1:OKEfbf2tcfsF2pBk/S9QqNpo6alaAtiBs4hysCsBjPw=
github.com/aws/aws-sdk-go-v2/service/sts v1.51.4/go.mod h1:mCd6qGo9b8hTtfjgsmNkFlzwjyce4owd7HDVFnCLXhM=
github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k=
github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/bramvdbogaerde/go-scp v1.5.0 h1:a9BinAjTfQh273eh7vd3qUgmBC+bx+3TRDtkZWmIpzM=
github.com/bramvdbogaerde/go-scp v1.5.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
integration_test.go
+263
View File
@@ -0,0 +1,263 @@
//go:build integration
/*
MIT License
Copyright (c) 2024 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package go_storage
// Integration tests against real services started with the docker CLI.
//
// go test -tags integration ./...
//
// Tests skip when docker is unavailable. Images can be overridden with
// GO_STORAGE_MINIO_IMAGE, GO_STORAGE_AZURITE_IMAGE, GO_STORAGE_FAKEGCS_IMAGE
// and GO_STORAGE_FTP_IMAGE.
import (
"context"
"errors"
"fmt"
"math/rand"
"net"
"os"
"os/exec"
"strconv"
"strings"
"testing"
"time"
"cloud.google.com/go/storage"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"google.golang.org/api/option"
)
func image(env, def string) string {
if v := os.Getenv(env); v != "" {
return v
}
return def
}
func requireDocker(t *testing.T) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
if err := exec.CommandContext(ctx, "docker", "info").Run(); err != nil {
t.Skipf("docker not available: %v", err)
}
}
func docker(t *testing.T, args ...string) string {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
out, err := exec.CommandContext(ctx, "docker", args...).CombinedOutput()
if err != nil {
t.Fatalf("docker %s: %v\n%s", strings.Join(args, " "), err, out)
}
return strings.TrimSpace(string(out))
}
// runContainer starts a detached container and removes it at cleanup.
func runContainer(t *testing.T, args ...string) string {
t.Helper()
id := docker(t, append([]string{"run", "-d"}, args...)...)
t.Cleanup(func() {
if t.Failed() {
out, _ := exec.Command("docker", "logs", "--tail", "50", id).CombinedOutput()
t.Logf("container logs:\n%s", out)
}
_ = exec.Command("docker", "rm", "-f", "-v", id).Run()
})
return id
}
// hostPort returns the host port mapped to the container port.
func hostPort(t *testing.T, id, containerPort string) string {
t.Helper()
out := docker(t, "port", id, containerPort)
line := strings.Split(out, "\n")[0]
_, port, err := net.SplitHostPort(strings.TrimSpace(line))
if err != nil {
t.Fatalf("parse docker port %q: %v", out, err)
}
return port
}
func freePort(t *testing.T) int {
t.Helper()
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer func() { _ = l.Close() }()
return l.Addr().(*net.TCPAddr).Port
}
const readyTimeout = 60 * time.Second
// eventually retries fn until it succeeds or readyTimeout expires.
func eventually(t *testing.T, what string, fn func() error) {
t.Helper()
deadline := time.Now().Add(readyTimeout)
var err error
for time.Now().Before(deadline) {
if err = fn(); err == nil {
return
}
time.Sleep(500 * time.Millisecond)
}
t.Fatalf("%s: %v", what, err)
}
func TestIntegrationMinIO(t *testing.T) {
requireDocker(t)
id := runContainer(t, "-p", "127.0.0.1::9000",
"-e", "MINIO_ROOT_USER=minioadmin", "-e", "MINIO_ROOT_PASSWORD=minioadmin",
image("GO_STORAGE_MINIO_IMAGE", "pgsty/minio:latest"), "server", "/data")
port := hostPort(t, id, "9000/tcp")
s, err := NewS3Storage(Config{
S3Bucket: "conformance",
S3Endpoint: "127.0.0.1:" + port,
S3DisableTLS: true,
S3ForcePath: true,
S3KeyID: "minioadmin",
S3Secret: "minioadmin",
S3PartSize: MinS3PartSize, // 20 MiB stream => real multipart upload
S3Concurrency: 2,
})
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
eventually(t, "create bucket", func() error {
_, err := s.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String("conformance")})
return err
})
runConformance(t, s)
// Failed multipart uploads must have been aborted.
out, err := s.client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{Bucket: aws.String("conformance")})
if err != nil {
t.Fatal(err)
}
if len(out.Uploads) != 0 {
t.Errorf("%d multipart uploads left behind", len(out.Uploads))
}
}
func TestIntegrationAzurite(t *testing.T) {
requireDocker(t)
id := runContainer(t, "-p", "127.0.0.1::10000",
image("GO_STORAGE_AZURITE_IMAGE", "mcr.microsoft.com/azure-storage/azurite:latest"),
"azurite-blob", "--blobHost", "0.0.0.0", "--skipApiVersionCheck", "--loose")
port := hostPort(t, id, "10000/tcp")
s, err := NewAzureStorage(Config{
AzureAccount: "devstoreaccount1",
AzureKey: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
AzureContainer: "conformance",
AzureEndpoint: "http://127.0.0.1:" + port + "/devstoreaccount1",
})
if err != nil {
t.Fatal(err)
}
eventually(t, "create container", func() error {
_, err := s.client.CreateContainer(context.Background(), "conformance", nil)
return err
})
runConformance(t, s)
}
func TestIntegrationFakeGCS(t *testing.T) {
requireDocker(t)
port := freePort(t)
addr := "127.0.0.1:" + strconv.Itoa(port)
runContainer(t, "-p", fmt.Sprintf("%s:4443", addr),
image("GO_STORAGE_FAKEGCS_IMAGE", "fsouza/fake-gcs-server:latest"),
"-scheme", "http", "-port", "4443", "-public-host", addr, "-external-url", "http://"+addr, "-backend", "memory")
ctx := context.Background()
s, err := newGCSStorage(ctx, "conformance",
option.WithEndpoint("http://"+addr+"/storage/v1/"),
option.WithoutAuthentication())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = s.Close() })
eventually(t, "create bucket", func() error {
err := s.client.Bucket("conformance").Create(ctx, "test-project", &storage.BucketAttrs{})
var exists bool
if err != nil && strings.Contains(err.Error(), "409") {
exists = true
}
if exists {
return nil
}
return err
})
runConformance(t, s)
}
func TestIntegrationFTP(t *testing.T) {
requireDocker(t)
// Passive ports must be identical inside and outside the container.
minPort := 30000 + rand.Intn(20000)
maxPort := minPort + 9
ctrlPort := freePort(t)
runContainer(t,
"-p", fmt.Sprintf("127.0.0.1:%d:21", ctrlPort),
"-p", fmt.Sprintf("127.0.0.1:%d-%d:%d-%d", minPort, maxPort, minPort, maxPort),
"-e", "USERS=backup|s3cret|/ftp/backup",
"-e", "ADDRESS=127.0.0.1",
"-e", "MIN_PORT="+strconv.Itoa(minPort), "-e", "MAX_PORT="+strconv.Itoa(maxPort),
image("GO_STORAGE_FTP_IMAGE", "delfer/alpine-ftp-server:latest"))
var s *FTPStorage
eventually(t, "connect ftp", func() error {
var err error
s, err = NewFTPStorage(Config{
FTPHost: "127.0.0.1",
FTPPort: strconv.Itoa(ctrlPort),
FTPUsername: "backup",
FTPPassword: "s3cret",
})
if err != nil {
return err
}
if _, err := s.ListDir(context.Background(), ""); err != nil {
_ = s.Close()
return err
}
return nil
})
t.Cleanup(func() { _ = s.Close() })
runConformance(t, s)
if _, err := s.PutIf(context.Background(), "x", strings.NewReader("x"), Condition{IfNoneMatch: true}, nil); !errors.Is(err, ErrConditionNotSupported) {
t.Fatalf("FTP PutIf: %v, want ErrConditionNotSupported", err)
}
}
local.go
+353 -268
View File
@@ -1,343 +1,428 @@
/*
MIT License
Copyright (c) 2024 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package go_storage
import (
"context"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
// LocalStorage implements the Storage interface using the local filesystem.
// It is safe for concurrent use as it relies on filesystem operations which
// are handled by the OS.
// It is safe for concurrent use.
type LocalStorage struct {
basePath string
}
// Lock tuning for PutIf / Put on the local backend.
const (
localLockTimeout = 10 * time.Second
localLockRetry = 10 * time.Millisecond
// A lock older than this is considered stale (the holder crashed). The
// lock is only held around a stat + rename, never while streaming data.
localLockStale = 30 * time.Second
)
// NewLocalStorage creates a new LocalStorage instance with the given base directory.
// The base directory will be created if it doesn't exist.
func NewLocalStorage(basePath string) (*LocalStorage, error) {
absPath, err := filepath.Abs(basePath)
if err != nil {
return nil, fmt.Errorf("failed to resolve absolute path: %w", err)
return nil, fmt.Errorf("local: failed to resolve absolute path: %w", err)
}
if err := os.MkdirAll(absPath, 0755); err != nil {
return nil, fmt.Errorf("failed to create base directory: %w", err)
if err := os.MkdirAll(absPath, 0o755); err != nil {
return nil, fmt.Errorf("local: failed to create base directory: %w", err)
}
return &LocalStorage{
basePath: absPath,
}, nil
return &LocalStorage{basePath: absPath}, nil
}
// resolvePath safely resolves a remote path to an absolute local path,
// ensuring it stays within the base directory.
// resolvePath safely resolves a key to an absolute local path, ensuring it
// stays within the base directory.
func (s *LocalStorage) resolvePath(remotePath string) (string, error) {
// Clean the path to remove any .. or . components
cleanPath := filepath.Clean(remotePath)
// Remove leading slashes to make it relative
cleanPath = strings.TrimPrefix(cleanPath, string(filepath.Separator))
// Join with base path
fullPath := filepath.Join(s.basePath, cleanPath)
// Verify the resolved path is still within basePath
if !strings.HasPrefix(fullPath, s.basePath) {
return "", fmt.Errorf("path traversal attempt detected: %s", remotePath)
key, err := cleanKey(remotePath)
if err != nil {
return "", fmt.Errorf("local: %w", err)
}
return s.join(key)
}
func (s *LocalStorage) join(key string) (string, error) {
fullPath := filepath.Join(s.basePath, filepath.FromSlash(key))
rel, err := filepath.Rel(s.basePath, fullPath)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("local: path traversal attempt detected: %s", key)
}
return fullPath, nil
}
// localETag derives the version token from mtime and size.
func localETag(fi fs.FileInfo) string {
return strconv.FormatInt(fi.ModTime().UnixNano(), 16) + "-" + strconv.FormatInt(fi.Size(), 16)
}
func (s *LocalStorage) item(full string, fi fs.FileInfo) Item {
rel, err := filepath.Rel(s.basePath, full)
if err != nil {
rel = full
}
it := Item{
Key: filepath.ToSlash(rel),
ModifiedTime: fi.ModTime(),
IsDirectory: fi.IsDir(),
}
if !fi.IsDir() {
it.Size = fi.Size()
it.ETag = localETag(fi)
}
return it
}
// Upload uploads a local file to the storage with the given target name.
func (s *LocalStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
targetPath, err := s.resolvePath(remotePath)
if err != nil {
return err
}
// Open source file
srcFile, err := os.Open(localPath)
if err != nil {
return fmt.Errorf("failed to open source file: %w", err)
}
defer srcFile.Close()
// Create parent directories if needed
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
return fmt.Errorf("failed to create parent directories: %w", err)
}
// Create destination file
dstFile, err := os.Create(targetPath)
if err != nil {
return fmt.Errorf("failed to create destination file: %w", err)
}
defer dstFile.Close()
// Copy with context cancellation checks
buf := make([]byte, 32*1024) // 32KB buffer
for {
select {
case <-ctx.Done():
// Clean up partial file on cancellation
os.Remove(targetPath)
return ctx.Err()
default:
}
n, readErr := srcFile.Read(buf)
if n > 0 {
if _, writeErr := dstFile.Write(buf[:n]); writeErr != nil {
os.Remove(targetPath)
return fmt.Errorf("failed to write to destination: %w", writeErr)
}
}
if readErr == io.EOF {
break
}
if readErr != nil {
os.Remove(targetPath)
return fmt.Errorf("failed to read source file: %w", readErr)
}
}
return nil
return uploadFile(ctx, s, localPath, remotePath)
}
// Download downloads a file from storage to a local path.
func (s *LocalStorage) Download(ctx context.Context, remotePath string, localPath string) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
sourcePath, err := s.resolvePath(remotePath)
if err != nil {
return err
}
// Open source file
srcFile, err := os.Open(sourcePath)
if err != nil {
return fmt.Errorf("failed to open remote file: %w", err)
}
defer srcFile.Close()
// Create parent directories for local path if needed
if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil {
return fmt.Errorf("failed to create local parent directories: %w", err)
}
// Create destination file
dstFile, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("failed to create local file: %w", err)
}
defer dstFile.Close()
// Copy with context cancellation checks
buf := make([]byte, 32*1024) // 32KB buffer
for {
select {
case <-ctx.Done():
// Clean up partial file on cancellation
os.Remove(localPath)
return ctx.Err()
default:
}
n, readErr := srcFile.Read(buf)
if n > 0 {
if _, writeErr := dstFile.Write(buf[:n]); writeErr != nil {
os.Remove(localPath)
return fmt.Errorf("failed to write to local file: %w", writeErr)
}
}
if readErr == io.EOF {
break
}
if readErr != nil {
os.Remove(localPath)
return fmt.Errorf("failed to read remote file: %w", readErr)
}
}
return nil
return downloadFile(ctx, s, remotePath, localPath)
}
// List returns all items in the storage that match the given prefix.
func (s *LocalStorage) List(ctx context.Context, prefix string) ([]Item, error) {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
// writeTemp streams r into a new temporary file next to full and returns its path.
func writeTemp(ctx context.Context, full string, r io.Reader) (string, error) {
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
return "", fmt.Errorf("local: failed to create parent directories: %w", err)
}
var items []Item
// Clean and resolve the prefix path
searchPath := s.basePath
cleanPrefix := strings.TrimPrefix(filepath.Clean(prefix), string(filepath.Separator))
if cleanPrefix != "" && cleanPrefix != "." {
searchPath = filepath.Join(s.basePath, cleanPrefix)
}
// Check if the search path exists
info, err := os.Stat(searchPath)
if os.IsNotExist(err) {
// If the exact path doesn't exist, try to find files with this prefix
// by walking the parent directory
parentDir := filepath.Dir(searchPath)
baseName := filepath.Base(searchPath)
if _, err := os.Stat(parentDir); os.IsNotExist(err) {
return items, nil // Return empty list if parent doesn't exist
}
entries, err := os.ReadDir(parentDir)
if err != nil {
return nil, fmt.Errorf("failed to read directory: %w", err)
}
for _, entry := range entries {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
if strings.HasPrefix(entry.Name(), baseName) {
fullPath := filepath.Join(parentDir, entry.Name())
item, err := s.fileInfoToItem(fullPath)
if err != nil {
continue // Skip files we can't stat
}
items = append(items, item)
}
}
return items, nil
}
tmp := partialName(full)
f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
if err != nil {
return nil, fmt.Errorf("failed to stat path: %w", err)
return "", fmt.Errorf("local: failed to create temporary file: %w", err)
}
// If it's a file, return just that file
if !info.IsDir() {
item, err := s.fileInfoToItem(searchPath)
if err != nil {
return nil, err
}
return []Item{item}, nil
_, err = io.Copy(f, ctxReader{ctx: ctx, r: r})
if err == nil {
err = f.Sync()
}
if cerr := f.Close(); err == nil {
err = cerr
}
if err == nil {
err = ctx.Err()
}
// Walk the directory
err = filepath.WalkDir(searchPath, func(path string, d os.DirEntry, err error) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if err != nil {
return nil // Skip entries with errors
}
item, err := s.fileInfoToItem(path)
if err != nil {
return nil // Skip files we can't stat
}
items = append(items, item)
return nil
})
if err != nil {
return nil, fmt.Errorf("failed to walk directory: %w", err)
_ = os.Remove(tmp)
return "", fmt.Errorf("local: failed to write data: %w", err)
}
return items, nil
return tmp, nil
}
// fileInfoToItem converts a file path to an Item.
func (s *LocalStorage) fileInfoToItem(path string) (Item, error) {
info, err := os.Stat(path)
// Put streams r to key via a temporary file that is renamed on success.
func (s *LocalStorage) Put(ctx context.Context, key string, r io.Reader, _ *PutOptions) (Item, error) {
return s.put(ctx, key, r, nil)
}
// PutIf writes r to key only if cond holds. It uses an exclusive lock file
// (<key>.lock) so it is atomic with respect to other PutIf/Put calls on the
// same filesystem.
func (s *LocalStorage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, _ *PutOptions) (Item, error) {
if err := validateCondition(cond); err != nil {
return Item{}, fmt.Errorf("local: %w", err)
}
return s.put(ctx, key, r, &cond)
}
func (s *LocalStorage) put(ctx context.Context, key string, r io.Reader, cond *Condition) (Item, error) {
full, err := s.resolvePath(key)
if err != nil {
return Item{}, err
}
// Get relative path from base
relPath, err := filepath.Rel(s.basePath, path)
tmp, err := writeTemp(ctx, full, r)
if err != nil {
relPath = path
return Item{}, err
}
committed := false
defer func() {
if !committed {
_ = os.Remove(tmp)
}
}()
// Normalize to forward slashes for consistency
relPath = filepath.ToSlash(relPath)
size := info.Size()
if info.IsDir() {
size = 0
unlock, err := acquireLock(ctx, full+".lock")
if err != nil {
return Item{}, err
}
defer unlock()
return Item{
Key: relPath,
ModifiedTime: info.ModTime(),
Size: size,
IsDirectory: info.IsDir(),
}, nil
cur, statErr := os.Stat(full)
exists := statErr == nil
if statErr != nil && !errors.Is(statErr, fs.ErrNotExist) {
return Item{}, fmt.Errorf("local: failed to stat %s: %w", key, statErr)
}
if cond != nil {
switch {
case cond.IfNoneMatch && exists:
return Item{}, fmt.Errorf("local: %s already exists: %w", key, ErrPreconditionFailed)
case cond.IfMatch != "" && (!exists || localETag(cur) != cond.IfMatch):
return Item{}, fmt.Errorf("local: %s etag mismatch: %w", key, ErrPreconditionFailed)
}
}
if exists {
if err := bumpMtime(tmp, cur.ModTime()); err != nil {
return Item{}, err
}
}
if err := ctx.Err(); err != nil {
return Item{}, err
}
if err := os.Rename(tmp, full); err != nil {
return Item{}, fmt.Errorf("local: failed to rename temporary file: %w", err)
}
committed = true
fi, err := os.Stat(full)
if err != nil {
return Item{}, fmt.Errorf("local: failed to stat %s: %w", key, err)
}
return s.item(full, fi), nil
}
// Remove deletes a file from storage.
func (s *LocalStorage) Remove(ctx context.Context, remotePath string) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
// bumpMtime makes sure the mtime of tmp is strictly after prev so that the
// ETag changes even on filesystems with coarse timestamps.
func bumpMtime(tmp string, prev time.Time) error {
for _, step := range []time.Duration{time.Microsecond, time.Millisecond, time.Second, 2 * time.Second} {
fi, err := os.Stat(tmp)
if err != nil {
return fmt.Errorf("local: failed to stat temporary file: %w", err)
}
if fi.ModTime().After(prev) {
return nil
}
t := prev.Add(step)
if err := os.Chtimes(tmp, t, t); err != nil {
return fmt.Errorf("local: failed to set mtime: %w", err)
}
}
return nil
}
// acquireLock creates lockPath exclusively, retrying until localLockTimeout.
func acquireLock(ctx context.Context, lockPath string) (func(), error) {
deadline := time.Now().Add(localLockTimeout)
for {
f, err := os.OpenFile(lockPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
if err == nil {
_, _ = fmt.Fprintf(f, "%d\n", os.Getpid())
_ = f.Close()
return func() { _ = os.Remove(lockPath) }, nil
}
if !errors.Is(err, fs.ErrExist) {
return nil, fmt.Errorf("local: failed to create lock %s: %w", lockPath, err)
}
if fi, serr := os.Stat(lockPath); serr == nil && time.Since(fi.ModTime()) > localLockStale {
_ = os.Remove(lockPath)
continue
}
if time.Now().After(deadline) {
return nil, fmt.Errorf("local: timed out waiting for lock %s", lockPath)
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(localLockRetry):
}
}
}
// Get opens key for reading.
func (s *LocalStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
if err := ctx.Err(); err != nil {
return nil, Item{}, err
}
full, err := s.resolvePath(key)
if err != nil {
return nil, Item{}, err
}
f, err := os.Open(full)
if err != nil {
return nil, Item{}, mapLocalErr(key, err)
}
fi, err := f.Stat()
if err != nil {
_ = f.Close()
return nil, Item{}, mapLocalErr(key, err)
}
if fi.IsDir() {
_ = f.Close()
return nil, Item{}, fmt.Errorf("local: %s is a directory", key)
}
return f, s.item(full, fi), nil
}
// Stat returns metadata for key.
func (s *LocalStorage) Stat(ctx context.Context, key string) (Item, error) {
if err := ctx.Err(); err != nil {
return Item{}, err
}
full, err := s.resolvePath(key)
if err != nil {
return Item{}, err
}
fi, err := os.Stat(full)
if err != nil {
return Item{}, mapLocalErr(key, err)
}
return s.item(full, fi), nil
}
func mapLocalErr(key string, err error) error {
if errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("local: %s: %w", key, ErrNotFound)
}
return fmt.Errorf("local: %s: %w", key, err)
}
// ListDir lists the files directly inside dir.
func (s *LocalStorage) ListDir(ctx context.Context, dir string) ([]Item, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
d, err := cleanDir(dir)
if err != nil {
return nil, fmt.Errorf("local: %w", err)
}
full, err := s.join(d)
if err != nil {
return nil, err
}
entries, err := os.ReadDir(full)
if errors.Is(err, fs.ErrNotExist) {
return []Item{}, nil
}
if err != nil {
return nil, fmt.Errorf("local: failed to read directory %s: %w", dir, err)
}
items := make([]Item, 0, len(entries))
for _, e := range entries {
if e.IsDir() {
continue
}
fi, err := e.Info()
if err != nil || !fi.Mode().IsRegular() {
continue
}
items = append(items, s.item(filepath.Join(full, e.Name()), fi))
}
return items, nil
}
// List returns all items (files and directories, recursively) whose key
// starts with prefix.
func (s *LocalStorage) List(ctx context.Context, prefix string) ([]Item, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
p, err := cleanPrefix(prefix)
if err != nil {
return nil, fmt.Errorf("local: %w", err)
}
// Walk from the deepest directory fully contained in the prefix.
root := s.basePath
if i := strings.LastIndex(p, "/"); i >= 0 {
root = filepath.Join(s.basePath, filepath.FromSlash(p[:i]))
}
items := make([]Item, 0)
err = filepath.WalkDir(root, func(full string, d fs.DirEntry, err error) error {
if cerr := ctx.Err(); cerr != nil {
return cerr
}
if err != nil {
if full == root && errors.Is(err, fs.ErrNotExist) {
return fs.SkipAll
}
return nil // skip unreadable entries
}
if full == s.basePath {
return nil
}
rel, rerr := filepath.Rel(s.basePath, full)
if rerr != nil {
return nil
}
rel = filepath.ToSlash(rel)
if !strings.HasPrefix(rel, p) {
// Prune directories that cannot contain matches.
if d.IsDir() && !strings.HasPrefix(p, rel+"/") {
return fs.SkipDir
}
return nil
}
fi, ierr := d.Info()
if ierr != nil {
return nil
}
items = append(items, s.item(full, fi))
return nil
})
if err != nil {
return nil, fmt.Errorf("local: failed to walk directory: %w", err)
}
return items, nil
}
// Remove deletes a file (or directory tree) from storage.
func (s *LocalStorage) Remove(ctx context.Context, remotePath string) error {
if err := ctx.Err(); err != nil {
return err
}
targetPath, err := s.resolvePath(remotePath)
if err != nil {
return err
}
// Check if file exists
info, err := os.Stat(targetPath)
if os.IsNotExist(err) {
return fmt.Errorf("file not found: %s", remotePath)
if err != nil {
return mapLocalErr(remotePath, err)
}
if info.IsDir() {
err = os.RemoveAll(targetPath)
} else {
err = os.Remove(targetPath)
}
if err != nil {
return fmt.Errorf("failed to stat file: %w", err)
return mapLocalErr(remotePath, err)
}
// Remove file or directory
if info.IsDir() {
if err := os.RemoveAll(targetPath); err != nil {
return fmt.Errorf("failed to remove directory: %w", err)
}
} else {
if err := os.Remove(targetPath); err != nil {
return fmt.Errorf("failed to remove file: %w", err)
}
}
return nil
}
// Close releases any resources held by the storage implementation.
// For LocalStorage, this is a no-op as there are no persistent resources.
// Close releases any resources held by the storage implementation (no-op).
func (s *LocalStorage) Close() error {
return nil
}
pkg/azure/azure.go
+4
View File
@@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
// Package azure is the legacy v0.1 API of go-storage.
//
// Deprecated: use the root package github.com/jkaninda/go-storage
// (NewStorage and the streaming Storage interface) instead.
package azure
import (
pkg/ftp/ftp.go
+4
View File
@@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
// Package ftp is the legacy v0.1 API of go-storage.
//
// Deprecated: use the root package github.com/jkaninda/go-storage
// (NewStorage and the streaming Storage interface) instead.
package ftp
import (
pkg/local/local.go
+4
View File
@@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
// Package local is the legacy v0.1 API of go-storage.
//
// Deprecated: use the root package github.com/jkaninda/go-storage
// (NewStorage and the streaming Storage interface) instead.
package local
import (
pkg/local/local_test.go
+24
View File
@@ -1,3 +1,27 @@
/*
MIT License
Copyright (c) 2024 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package local
import (
pkg/s3/s3.go
+4
View File
@@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
// Package s3 is the legacy v0.1 API of go-storage.
//
// Deprecated: use the root package github.com/jkaninda/go-storage
// (NewStorage and the streaming Storage interface) instead.
package s3
import (
pkg/ssh/ssh.go
+4
View File
@@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
// Package ssh is the legacy v0.1 API of go-storage.
//
// Deprecated: use the root package github.com/jkaninda/go-storage
// (NewStorage and the streaming Storage interface) instead.
package ssh
import (
pkg/ssh/ssh_test.go
+24
View File
@@ -1 +1,25 @@
/*
MIT License
Copyright (c) 2024 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package ssh
pkg/storage.go
+4
View File
@@ -22,6 +22,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
// Package pkg is the legacy v0.1 API of go-storage.
//
// Deprecated: use the root package github.com/jkaninda/go-storage
// (NewStorage and the streaming Storage interface) instead.
package pkg
type Storage interface {
s3.go
+342 -124
View File
@@ -1,25 +1,74 @@
/*
MIT License
Copyright (c) 2024 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package go_storage
import (
"bytes"
"context"
"errors"
"fmt"
"os"
"io"
"net/http"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go-v2/aws"
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
)
// S3Storage implements Storage interface for Amazon S3.
// S3 defaults.
const (
DefaultS3PartSize int64 = 64 << 20
MinS3PartSize int64 = 5 << 20
DefaultS3Concurrency = 2
DefaultS3Region = "us-east-1"
)
// S3Storage implements Storage for Amazon S3 and S3-compatible services
// (MinIO, Ceph, Wasabi, ...).
type S3Storage struct {
session *session.Session
bucket string
region string
client *s3.Client
uploader *manager.Uploader //nolint:staticcheck // see NewS3Storage
bucket string
region string
storageClass types.StorageClass
sse types.ServerSideEncryption
sseKMSKeyID string
}
// NewS3Storage creates a new S3 storage instance.
//
// Credentials: static keys (S3KeyID/S3Secret/S3SessionToken) when set,
// otherwise the default AWS credential chain (environment, shared profile
// S3Profile, web identity/IRSA, ECS/EC2 roles).
func NewS3Storage(config Config) (*S3Storage, error) {
if err := validateConfig(config, map[string]string{
"bucket": config.S3Bucket,
@@ -27,161 +76,330 @@ func NewS3Storage(config Config) (*S3Storage, error) {
return nil, fmt.Errorf("s3: %w", err)
}
awsConfig := aws.NewConfig()
opts := []func(*awsconfig.LoadOptions) error{
awsconfig.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired),
awsconfig.WithResponseChecksumValidation(aws.ResponseChecksumValidationWhenRequired),
}
if config.S3Region != "" {
awsConfig = awsConfig.WithRegion(config.S3Region)
opts = append(opts, awsconfig.WithRegion(config.S3Region))
}
if config.S3KeyID != "" && config.S3Secret != "" {
awsConfig = awsConfig.WithCredentials(
credentials.NewStaticCredentials(config.S3KeyID, config.S3Secret, ""),
)
}
if config.S3Endpoint != "" {
awsConfig = awsConfig.WithEndpoint(config.S3Endpoint)
}
if config.S3ForcePath {
awsConfig = awsConfig.WithS3ForcePathStyle(true)
}
if config.S3DisableTLS {
awsConfig = awsConfig.WithDisableSSL(true)
}
sessionOpts := session.Options{
Config: *awsConfig,
SharedConfigState: session.SharedConfigEnable,
}
if config.S3Profile != "" {
sessionOpts.Profile = config.S3Profile
opts = append(opts, awsconfig.WithSharedConfigProfile(config.S3Profile))
}
if config.S3KeyID != "" && config.S3Secret != "" {
opts = append(opts, awsconfig.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(config.S3KeyID, config.S3Secret, config.S3SessionToken)))
}
sess, err := session.NewSessionWithOptions(sessionOpts)
cfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...)
if err != nil {
return nil, fmt.Errorf("s3: failed to create session: %w", err)
return nil, fmt.Errorf("s3: failed to load AWS configuration: %w", err)
}
if cfg.Region == "" {
cfg.Region = DefaultS3Region
}
return &S3Storage{
session: sess,
bucket: config.S3Bucket,
region: config.S3Region,
endpoint := config.S3Endpoint
if endpoint != "" && !strings.Contains(endpoint, "://") {
if config.S3DisableTLS {
endpoint = "http://" + endpoint
} else {
endpoint = "https://" + endpoint
}
}
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
if endpoint != "" {
o.BaseEndpoint = aws.String(endpoint)
}
o.UsePathStyle = config.S3ForcePath
})
partSize := config.S3PartSize
if partSize <= 0 {
partSize = DefaultS3PartSize
}
if partSize < MinS3PartSize {
partSize = MinS3PartSize
}
concurrency := config.S3Concurrency
if concurrency <= 0 {
concurrency = DefaultS3Concurrency
}
//nolint:staticcheck // feature/s3/manager is deprecated in favour of the (pre-1.0) transfermanager.
uploader := manager.NewUploader(client, func(u *manager.Uploader) {
u.PartSize = partSize
u.Concurrency = concurrency
u.LeavePartsOnError = false
u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
})
s := &S3Storage{
client: client,
uploader: uploader,
bucket: config.S3Bucket,
region: cfg.Region,
storageClass: types.StorageClass(config.S3StorageClass),
}
switch strings.ToLower(config.S3SSE) {
case "":
case "aes256":
s.sse = types.ServerSideEncryptionAes256
case "aws:kms":
s.sse = types.ServerSideEncryptionAwsKms
s.sseKMSKeyID = config.S3SSEKMSKeyID
default:
s.sse = types.ServerSideEncryption(config.S3SSE)
s.sseKMSKeyID = config.S3SSEKMSKeyID
}
return s, nil
}
// Client returns the underlying S3 client.
func (s *S3Storage) Client() *s3.Client { return s.client }
func (s *S3Storage) putInput(key string, body io.Reader, opts *PutOptions) *s3.PutObjectInput {
in := &s3.PutObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
Body: body,
}
if opts != nil {
if opts.ContentType != "" {
in.ContentType = aws.String(opts.ContentType)
}
if len(opts.Metadata) > 0 {
in.Metadata = opts.Metadata
}
}
if s.storageClass != "" {
in.StorageClass = s.storageClass
}
if s.sse != "" {
in.ServerSideEncryption = s.sse
if s.sseKMSKeyID != "" {
in.SSEKMSKeyId = aws.String(s.sseKMSKeyID)
}
}
return in
}
// Put streams r to key using a multipart upload (aborted on error).
func (s *S3Storage) Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error) {
k, err := cleanKey(key)
if err != nil {
return Item{}, fmt.Errorf("s3: %w", err)
}
cr := &countingReader{r: ctxReader{ctx: ctx, r: r}}
//nolint:staticcheck // see NewS3Storage
out, err := s.uploader.Upload(ctx, s.putInput(k, cr, opts))
if err != nil {
return Item{}, fmt.Errorf("s3: failed to upload %s to bucket %s: %w", k, s.bucket, err)
}
return Item{
Key: k,
Size: cr.n,
ModifiedTime: time.Now().UTC(),
ETag: aws.ToString(out.ETag),
VersionID: aws.ToString(out.VersionID),
}, nil
}
// PutIf performs a conditional single-request PutObject (If-Match /
// If-None-Match: *). The body is buffered and limited to 16 MiB.
func (s *S3Storage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error) {
k, err := cleanKey(key)
if err != nil {
return Item{}, fmt.Errorf("s3: %w", err)
}
if err := validateCondition(cond); err != nil {
return Item{}, fmt.Errorf("s3: %w", err)
}
data, err := io.ReadAll(io.LimitReader(ctxReader{ctx: ctx, r: r}, maxConditionalPutSize+1))
if err != nil {
return Item{}, fmt.Errorf("s3: failed to read data: %w", err)
}
if len(data) > maxConditionalPutSize {
return Item{}, fmt.Errorf("s3: conditional put of %s exceeds %d bytes", k, maxConditionalPutSize)
}
in := s.putInput(k, bytes.NewReader(data), opts)
in.ContentLength = aws.Int64(int64(len(data)))
if cond.IfMatch != "" {
in.IfMatch = aws.String(cond.IfMatch)
}
if cond.IfNoneMatch {
in.IfNoneMatch = aws.String("*")
}
out, err := s.client.PutObject(ctx, in)
if err != nil {
// If-Match on a missing key yields 404 NoSuchKey on AWS.
if isS3Precondition(err) || (cond.IfMatch != "" && s3StatusCode(err) == http.StatusNotFound) {
return Item{}, fmt.Errorf("s3: conditional put of %s: %w", k, ErrPreconditionFailed)
}
return Item{}, fmt.Errorf("s3: failed to put %s: %w", k, err)
}
return Item{
Key: k,
Size: int64(len(data)),
ModifiedTime: time.Now().UTC(),
ETag: aws.ToString(out.ETag),
VersionID: aws.ToString(out.VersionId),
}, nil
}
// Get streams the object body.
func (s *S3Storage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
k, err := cleanKey(key)
if err != nil {
return nil, Item{}, fmt.Errorf("s3: %w", err)
}
out, err := s.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(k)})
if err != nil {
return nil, Item{}, s.mapErr("get", k, err)
}
return out.Body, Item{
Key: k,
Size: aws.ToInt64(out.ContentLength),
ModifiedTime: aws.ToTime(out.LastModified),
ETag: aws.ToString(out.ETag),
VersionID: aws.ToString(out.VersionId),
}, nil
}
// Stat returns object metadata (HeadObject).
func (s *S3Storage) Stat(ctx context.Context, key string) (Item, error) {
k, err := cleanKey(key)
if err != nil {
return Item{}, fmt.Errorf("s3: %w", err)
}
out, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(k)})
if err != nil {
return Item{}, s.mapErr("stat", k, err)
}
return Item{
Key: k,
Size: aws.ToInt64(out.ContentLength),
ModifiedTime: aws.ToTime(out.LastModified),
ETag: aws.ToString(out.ETag),
VersionID: aws.ToString(out.VersionId),
}, nil
}
// Upload uploads a file to S3.
func (s *S3Storage) Upload(ctx context.Context, localPath string, remotePath string) error {
file, err := os.Open(localPath)
if err != nil {
return fmt.Errorf("s3: failed to open local file: %w", err)
if err := uploadFile(ctx, s, localPath, remotePath); err != nil {
return fmt.Errorf("s3: %w", err)
}
defer func(file *os.File) {
err = file.Close()
if err != nil {
fmt.Printf("s3: warning: failed to close file: %v\n", err)
}
}(file)
uploader := s3manager.NewUploader(s.session)
_, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{
Bucket: aws.String(s.bucket),
Key: aws.String(normalizePathSeparators(remotePath)),
Body: file,
})
if err != nil {
return fmt.Errorf("s3: failed to upload to bucket %s: %w", s.bucket, err)
}
return nil
}
// Download downloads a file from S3.
func (s *S3Storage) Download(ctx context.Context, remotePath string, localPath string) error {
file, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("s3: failed to create local file: %w", err)
if err := downloadFile(ctx, s, remotePath, localPath); err != nil {
return fmt.Errorf("s3: %w", err)
}
defer func(file *os.File) {
err = file.Close()
if err != nil {
fmt.Printf("s3: warning: failed to close file: %v\n", err)
}
}(file)
downloader := s3manager.NewDownloader(s.session)
_, err = downloader.DownloadWithContext(ctx, file, &s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(normalizePathSeparators(remotePath)),
})
if err != nil {
return fmt.Errorf("s3: failed to download from bucket %s: %w", s.bucket, err)
}
return nil
}
// List lists objects in S3 with the given prefix.
// List lists objects (recursively) with the given prefix.
func (s *S3Storage) List(ctx context.Context, prefix string) ([]Item, error) {
svc := s3.New(s.session)
p, err := cleanPrefix(prefix)
if err != nil {
return nil, fmt.Errorf("s3: %w", err)
}
return s.list(ctx, p, "")
}
// ListDir lists the objects directly under dir.
func (s *S3Storage) ListDir(ctx context.Context, dir string) ([]Item, error) {
d, err := cleanDir(dir)
if err != nil {
return nil, fmt.Errorf("s3: %w", err)
}
return s.list(ctx, dirPrefix(d), "/")
}
func (s *S3Storage) list(ctx context.Context, prefix, delimiter string) ([]Item, error) {
in := &s3.ListObjectsV2Input{Bucket: aws.String(s.bucket), Prefix: aws.String(prefix)}
if delimiter != "" {
in.Delimiter = aws.String(delimiter)
}
items := make([]Item, 0)
var continuationToken *string
for {
input := &s3.ListObjectsV2Input{
Bucket: aws.String(s.bucket),
Prefix: aws.String(normalizePathSeparators(prefix)),
ContinuationToken: continuationToken,
}
resp, err := svc.ListObjectsV2WithContext(ctx, input)
p := s3.NewListObjectsV2Paginator(s.client, in)
for p.HasMorePages() {
page, err := p.NextPage(ctx)
if err != nil {
return nil, fmt.Errorf("s3: failed to list objects in bucket %s: %w", s.bucket, err)
}
for _, obj := range resp.Contents {
for _, obj := range page.Contents {
key := aws.ToString(obj.Key)
if delimiter != "" && strings.HasSuffix(key, "/") {
continue // directory marker
}
items = append(items, Item{
Key: *obj.Key,
ModifiedTime: *obj.LastModified,
Size: *obj.Size,
IsDirectory: false,
Key: key,
ModifiedTime: aws.ToTime(obj.LastModified),
Size: aws.ToInt64(obj.Size),
ETag: aws.ToString(obj.ETag),
})
}
if resp.IsTruncated == nil || !*resp.IsTruncated {
break
}
continuationToken = resp.NextContinuationToken
}
return items, nil
}
// Remove deletes an object from S3.
// Remove deletes an object. A missing key returns ErrNotFound (checked with
// HeadObject because DeleteObject is idempotent on S3).
func (s *S3Storage) Remove(ctx context.Context, remotePath string) error {
svc := s3.New(s.session)
_, err := svc.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(normalizePathSeparators(remotePath)),
})
k, err := cleanKey(remotePath)
if err != nil {
return fmt.Errorf("s3: failed to remove object from bucket %s: %w", s.bucket, err)
return fmt.Errorf("s3: %w", err)
}
if _, err := s.Stat(ctx, k); err != nil {
return err
}
if _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(k)}); err != nil {
return s.mapErr("remove", k, err)
}
return nil
}
// Close releases resources.
// Close releases resources (no-op).
func (s *S3Storage) Close() error {
return nil
}
func (s *S3Storage) mapErr(op, key string, err error) error {
if s3StatusCode(err) == http.StatusNotFound {
return fmt.Errorf("s3: %s %s in bucket %s: %w", op, key, s.bucket, ErrNotFound)
}
var nsk *types.NoSuchKey
var nf *types.NotFound
if errors.As(err, &nsk) || errors.As(err, &nf) {
return fmt.Errorf("s3: %s %s in bucket %s: %w", op, key, s.bucket, ErrNotFound)
}
return fmt.Errorf("s3: %s %s in bucket %s: %w", op, key, s.bucket, err)
}
func s3StatusCode(err error) int {
var re *awshttp.ResponseError
if errors.As(err, &re) {
return re.HTTPStatusCode()
}
return 0
}
func isS3Precondition(err error) bool {
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
switch apiErr.ErrorCode() {
case "PreconditionFailed", "ConditionalRequestConflict":
return true
}
}
switch s3StatusCode(err) {
case http.StatusPreconditionFailed, http.StatusConflict:
return true
}
return false
}
sftp.go
+378 -169
View File
@@ -1,3 +1,27 @@
/*
MIT License
Copyright (c) 2024 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package go_storage
import (
@@ -5,11 +29,14 @@ import (
"errors"
"fmt"
"io"
"io/fs"
"net"
"os"
"os/user"
"path"
"path/filepath"
"strings"
"time"
"github.com/pkg/sftp"
@@ -27,6 +54,9 @@ type SFTPStorage struct {
}
// NewSFTPStorage creates a new SFTP storage instance.
//
// Host keys are verified against SFTPKnownHostsFile (or ~/.ssh/known_hosts
// and /etc/ssh/ssh_known_hosts) unless SFTPIgnoreKnownHosts is true.
func NewSFTPStorage(config Config) (*SFTPStorage, error) {
if err := validateConfig(config, map[string]string{
"host": config.SFTPHost,
@@ -38,6 +68,7 @@ func NewSFTPStorage(config Config) (*SFTPStorage, error) {
if port == "" {
port = "22"
}
hostPort := net.JoinHostPort(config.SFTPHost, port)
username := config.SFTPUsername
if username == "" {
@@ -48,21 +79,15 @@ func NewSFTPStorage(config Config) (*SFTPStorage, error) {
username = currentUser.Username
}
password := config.SFTPPassword
if password == "" {
password = os.Getenv("PGBK_SSH_PASS")
authMethods := make([]ssh.AuthMethod, 0, 2)
if config.SFTPPassword != "" {
authMethods = append(authMethods, ssh.Password(config.SFTPPassword))
}
// Build authentication methods
authMethods := make([]ssh.AuthMethod, 0)
// Password authentication (if no identity file is specified)
if config.SFTPIdentityFile == "" && password != "" {
authMethods = append(authMethods, ssh.Password(password))
passphrase := config.SFTPIdentityPassphrase
if passphrase == "" {
passphrase = config.SFTPPassword // backwards compatibility
}
// Public key authentication
signers, err := getSSHSigners(config.SFTPIdentityFile, password)
signers, err := getSSHSigners(config.SFTPIdentityFile, passphrase)
if err != nil {
return nil, fmt.Errorf("sftp: %w", err)
}
@@ -70,33 +95,36 @@ func NewSFTPStorage(config Config) (*SFTPStorage, error) {
authMethods = append(authMethods, ssh.PublicKeys(signers...))
}
// Build SSH client config
hostKeyCallback, hostKeyAlgos, err := getHostKeyCallback(config, hostPort)
if err != nil {
return nil, fmt.Errorf("sftp: %w", err)
}
sshConfig := &ssh.ClientConfig{
User: username,
Auth: authMethods,
HostKeyCallback: getHostKeyCallback(config.SFTPIgnoreKnownHosts),
User: username,
Auth: authMethods,
HostKeyCallback: hostKeyCallback,
HostKeyAlgorithms: hostKeyAlgos,
Timeout: 30 * time.Second,
}
// Connect to SSH server
hostPort := fmt.Sprintf("%s:%s", config.SFTPHost, port)
sshClient, err := ssh.Dial("tcp", hostPort, sshConfig)
if err != nil {
return nil, fmt.Errorf("sftp: failed to connect to %s: %w", hostPort, err)
}
// Open SFTP session
sftpClient, err := sftp.NewClient(sshClient)
if err != nil {
sshClient.Close()
_ = sshClient.Close()
return nil, fmt.Errorf("sftp: failed to open sftp session: %w", err)
}
baseDir := config.SFTPDirectory
baseDir := normalizePathSeparators(config.SFTPDirectory)
if baseDir == "" {
wd, err := sftpClient.Getwd()
if err != nil {
sftpClient.Close()
sshClient.Close()
_ = sftpClient.Close()
_ = sshClient.Close()
return nil, fmt.Errorf("sftp: failed to get working directory: %w", err)
}
baseDir = wd
@@ -104,169 +132,305 @@ func NewSFTPStorage(config Config) (*SFTPStorage, error) {
return &SFTPStorage{
host: config.SFTPHost,
baseDir: baseDir,
baseDir: path.Clean(baseDir),
sshClient: sshClient,
client: sftpClient,
}, nil
}
// Client returns the underlying sftp client.
func (s *SFTPStorage) Client() *sftp.Client { return s.client }
func (s *SFTPStorage) resolve(key string) (string, string, error) {
k, err := cleanKey(key)
if err != nil {
return "", "", fmt.Errorf("sftp: %w", err)
}
return k, path.Join(s.baseDir, k), nil
}
func (s *SFTPStorage) item(key string, fi os.FileInfo) Item {
it := Item{Key: key, ModifiedTime: fi.ModTime(), IsDirectory: fi.IsDir()}
if !fi.IsDir() {
it.Size = fi.Size()
}
return it
}
func (s *SFTPStorage) mkdirParent(full string) error {
dir := path.Dir(full)
if dir == "." || dir == "/" {
return nil
}
if err := s.client.MkdirAll(dir); err != nil {
return fmt.Errorf("sftp: failed to create directory %s: %w", dir, err)
}
return nil
}
// Put streams r to <key>.<rand>.partial and renames it to key on success.
func (s *SFTPStorage) Put(ctx context.Context, key string, r io.Reader, _ *PutOptions) (Item, error) {
k, full, err := s.resolve(key)
if err != nil {
return Item{}, err
}
if err := s.mkdirParent(full); err != nil {
return Item{}, err
}
tmp := partialName(full)
f, err := s.client.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
if err != nil {
return Item{}, fmt.Errorf("sftp: failed to create %s: %w", tmp, err)
}
if err := copyAndClose(ctx, f, r); err != nil {
_ = s.client.Remove(tmp)
return Item{}, fmt.Errorf("sftp: failed to upload %s: %w", k, err)
}
if err := s.rename(tmp, full); err != nil {
_ = s.client.Remove(tmp)
return Item{}, err
}
return s.Stat(ctx, k)
}
func copyAndClose(ctx context.Context, w io.WriteCloser, r io.Reader) error {
_, err := io.Copy(w, ctxReader{ctx: ctx, r: r})
if cerr := w.Close(); err == nil {
err = cerr
}
if err == nil {
err = ctx.Err()
}
return err
}
func (s *SFTPStorage) rename(from, to string) error {
if _, ok := s.client.HasExtension("posix-rename@openssh.com"); ok {
if err := s.client.PosixRename(from, to); err != nil {
return fmt.Errorf("sftp: failed to rename %s to %s: %w", from, to, err)
}
return nil
}
if err := s.client.Remove(to); err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("sftp: failed to replace %s: %w", to, err)
}
if err := s.client.Rename(from, to); err != nil {
return fmt.Errorf("sftp: failed to rename %s to %s: %w", from, to, err)
}
return nil
}
// PutIf supports only Condition.IfNoneMatch (via O_EXCL on the final path).
// IfMatch returns ErrConditionNotSupported.
func (s *SFTPStorage) PutIf(ctx context.Context, key string, r io.Reader, cond Condition, _ *PutOptions) (Item, error) {
if cond.IfMatch != "" || !cond.IfNoneMatch {
return Item{}, fmt.Errorf("sftp: IfMatch: %w", ErrConditionNotSupported)
}
k, full, err := s.resolve(key)
if err != nil {
return Item{}, err
}
if err := s.mkdirParent(full); err != nil {
return Item{}, err
}
f, err := s.client.OpenFile(full, os.O_WRONLY|os.O_CREATE|os.O_EXCL)
if err != nil {
if _, serr := s.client.Lstat(full); serr == nil {
return Item{}, fmt.Errorf("sftp: %s already exists: %w", k, ErrPreconditionFailed)
}
return Item{}, fmt.Errorf("sftp: failed to create %s: %w", k, err)
}
if err := copyAndClose(ctx, f, r); err != nil {
_ = s.client.Remove(full)
return Item{}, fmt.Errorf("sftp: failed to upload %s: %w", k, err)
}
return s.Stat(ctx, k)
}
// Get opens key for streaming reads.
func (s *SFTPStorage) Get(ctx context.Context, key string) (io.ReadCloser, Item, error) {
if err := ctx.Err(); err != nil {
return nil, Item{}, err
}
k, full, err := s.resolve(key)
if err != nil {
return nil, Item{}, err
}
f, err := s.client.Open(full)
if err != nil {
return nil, Item{}, mapSFTPErr("get", k, err)
}
fi, err := f.Stat()
if err != nil {
_ = f.Close()
return nil, Item{}, mapSFTPErr("get", k, err)
}
if fi.IsDir() {
_ = f.Close()
return nil, Item{}, fmt.Errorf("sftp: %s is a directory", k)
}
return f, s.item(k, fi), nil
}
// Stat returns metadata for key.
func (s *SFTPStorage) Stat(ctx context.Context, key string) (Item, error) {
if err := ctx.Err(); err != nil {
return Item{}, err
}
k, full, err := s.resolve(key)
if err != nil {
return Item{}, err
}
fi, err := s.client.Stat(full)
if err != nil {
return Item{}, mapSFTPErr("stat", k, err)
}
return s.item(k, fi), nil
}
func mapSFTPErr(op, key string, err error) error {
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, sftp.ErrSSHFxNoSuchFile) {
return fmt.Errorf("sftp: %s %s: %w", op, key, ErrNotFound)
}
var se *sftp.StatusError
if errors.As(err, &se) && se.FxCode() == sftp.ErrSSHFxNoSuchFile {
return fmt.Errorf("sftp: %s %s: %w", op, key, ErrNotFound)
}
return fmt.Errorf("sftp: %s %s: %w", op, key, err)
}
// Upload uploads a file via SFTP.
func (s *SFTPStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
src, err := os.Open(localPath)
if err != nil {
return fmt.Errorf("sftp: failed to open local file: %w", err)
}
defer src.Close()
fullRemotePath := filepath.Join(s.baseDir, remotePath)
fullRemotePath = normalizePathSeparators(fullRemotePath)
// Create parent directory if needed
remoteDir := filepath.Dir(fullRemotePath)
remoteDir = normalizePathSeparators(remoteDir)
if remoteDir != "." && remoteDir != "/" {
if err := s.client.MkdirAll(remoteDir); err != nil {
return fmt.Errorf("sftp: failed to create directory %s: %w", remoteDir, err)
}
}
dst, err := s.client.Create(fullRemotePath)
if err != nil {
return fmt.Errorf("sftp: failed to create remote file %s: %w", fullRemotePath, err)
}
defer dst.Close()
if _, err := io.Copy(dst, src); err != nil {
return fmt.Errorf("sftp: failed to transfer data: %w", err)
}
return nil
return uploadFile(ctx, s, localPath, remotePath)
}
// Download downloads a file via SFTP.
func (s *SFTPStorage) Download(ctx context.Context, remotePath string, localPath string) error {
dst, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("sftp: failed to create local file: %w", err)
}
defer dst.Close()
fullRemotePath := filepath.Join(s.baseDir, remotePath)
fullRemotePath = normalizePathSeparators(fullRemotePath)
src, err := s.client.Open(fullRemotePath)
if err != nil {
return fmt.Errorf("sftp: failed to open remote file %s: %w", fullRemotePath, err)
}
defer src.Close()
if _, err := io.Copy(dst, src); err != nil {
return fmt.Errorf("sftp: failed to transfer data: %w", err)
}
return nil
return downloadFile(ctx, s, remotePath, localPath)
}
// List lists files via SFTP with the given prefix.
// List lists files and directories (recursively) whose key starts with prefix.
func (s *SFTPStorage) List(ctx context.Context, prefix string) ([]Item, error) {
items := make([]Item, 0)
baseDir := normalizePathSeparators(s.baseDir)
walker := s.client.Walk(baseDir)
for walker.Step() {
if err := walker.Err(); err != nil {
return nil, fmt.Errorf("sftp: walk error: %w", err)
}
// Get relative path
path := walker.Path()
relPath, err := filepath.Rel(baseDir, path)
if err != nil {
continue
}
// Normalize separators for comparison
relPath = normalizePathSeparators(relPath)
if !strings.HasPrefix(relPath, prefix) {
continue
}
stat := walker.Stat()
items = append(items, Item{
Key: relPath,
ModifiedTime: stat.ModTime(),
Size: stat.Size(),
IsDirectory: stat.IsDir(),
})
p, err := cleanPrefix(prefix)
if err != nil {
return nil, fmt.Errorf("sftp: %w", err)
}
items := make([]Item, 0)
root := s.baseDir
if i := strings.LastIndex(p, "/"); i >= 0 {
root = path.Join(s.baseDir, p[:i])
}
walker := s.client.Walk(root)
for walker.Step() {
if err := ctx.Err(); err != nil {
return nil, err
}
if err := walker.Err(); err != nil {
if walker.Path() == root && errors.Is(err, fs.ErrNotExist) {
return items, nil
}
continue
}
full := walker.Path()
if full == s.baseDir {
continue
}
rel := strings.TrimPrefix(strings.TrimPrefix(full, s.baseDir), "/")
stat := walker.Stat()
if !strings.HasPrefix(rel, p) {
if stat.IsDir() && !strings.HasPrefix(p, rel+"/") {
walker.SkipDir()
}
continue
}
items = append(items, s.item(rel, stat))
}
return items, nil
}
// ListDir lists the files directly inside dir.
func (s *SFTPStorage) ListDir(ctx context.Context, dir string) ([]Item, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
d, err := cleanDir(dir)
if err != nil {
return nil, fmt.Errorf("sftp: %w", err)
}
entries, err := s.client.ReadDir(path.Join(s.baseDir, d))
if errors.Is(err, fs.ErrNotExist) {
return []Item{}, nil
}
if err != nil {
return nil, fmt.Errorf("sftp: failed to read directory %s: %w", dir, err)
}
items := make([]Item, 0, len(entries))
for _, fi := range entries {
if !fi.Mode().IsRegular() {
continue
}
items = append(items, s.item(path.Join(d, fi.Name()), fi))
}
return items, nil
}
// Remove deletes a file via SFTP.
func (s *SFTPStorage) Remove(ctx context.Context, remotePath string) error {
fullRemotePath := filepath.Join(s.baseDir, remotePath)
fullRemotePath = normalizePathSeparators(fullRemotePath)
if err := s.client.Remove(fullRemotePath); err != nil {
return fmt.Errorf("sftp: failed to remove %s: %w", fullRemotePath, err)
if err := ctx.Err(); err != nil {
return err
}
k, full, err := s.resolve(remotePath)
if err != nil {
return err
}
if err := s.client.Remove(full); err != nil {
return mapSFTPErr("remove", k, err)
}
return nil
}
// Close releases resources.
func (s *SFTPStorage) Close() error {
if err := s.client.Close(); err != nil {
s.sshClient.Close()
return err
err := s.client.Close()
if cerr := s.sshClient.Close(); err == nil {
err = cerr
}
return s.sshClient.Close()
return err
}
// getSSHSigners returns SSH signers from identity file and SSH agent.
func getSSHSigners(identityFile string, passphrase string) ([]ssh.Signer, error) {
signers := make([]ssh.Signer, 0)
// Load identity file if provided
if identityFile != "" {
path, err := expandHomeDir(identityFile)
p, err := expandHomeDir(identityFile)
if err != nil {
return nil, fmt.Errorf("failed to expand identity file path: %w", err)
}
keyData, err := os.ReadFile(path)
keyData, err := os.ReadFile(p)
if err != nil {
return nil, fmt.Errorf("failed to read identity file %s: %w", path, err)
return nil, fmt.Errorf("failed to read identity file %s: %w", p, err)
}
signer, err := ssh.ParsePrivateKey(keyData)
if err != nil {
var passErr *ssh.PassphraseMissingError
if errors.As(err, &passErr) {
signer, err = ssh.ParsePrivateKeyWithPassphrase(keyData, []byte(passphrase))
if err != nil {
return nil, fmt.Errorf("failed to decrypt identity file: %w", err)
}
} else {
if !errors.As(err, &passErr) {
return nil, fmt.Errorf("failed to parse identity file: %w", err)
}
signer, err = ssh.ParsePrivateKeyWithPassphrase(keyData, []byte(passphrase))
if err != nil {
return nil, fmt.Errorf("failed to decrypt identity file: %w", err)
}
}
signers = append(signers, signer)
}
// Try to get keys from SSH agent
socket := os.Getenv("SSH_AUTH_SOCK")
if socket != "" {
conn, err := net.Dial("unix", socket)
if err == nil {
agentClient := agent.NewClient(conn)
agentSigners, err := agentClient.Signers()
if err == nil {
// Keys from a running SSH agent, if any.
if socket := os.Getenv("SSH_AUTH_SOCK"); socket != "" {
if conn, err := net.Dial("unix", socket); err == nil {
if agentSigners, err := agent.NewClient(conn).Signers(); err == nil {
signers = append(signers, agentSigners...)
}
}
@@ -275,61 +439,108 @@ func getSSHSigners(identityFile string, passphrase string) ([]ssh.Signer, error)
return signers, nil
}
// getHostKeyCallback returns appropriate host key callback.
func getHostKeyCallback(ignoreHostKey bool) ssh.HostKeyCallback {
if ignoreHostKey {
return ssh.InsecureIgnoreHostKey()
// getHostKeyCallback builds the host key verification callback and the list
// of host key algorithms known for hostPort (so the server negotiates a key
// type present in known_hosts).
func getHostKeyCallback(config Config, hostPort string) (ssh.HostKeyCallback, []string, error) {
if config.SFTPIgnoreKnownHosts {
return ssh.InsecureIgnoreHostKey(), nil, nil //nolint:gosec // explicitly requested
}
knownHostsFiles := make([]string, 0)
for _, p := range []string{"/etc/ssh/ssh_known_hosts", "~/.ssh/known_hosts"} {
path, err := expandHomeDir(p)
files := make([]string, 0, 2)
if config.SFTPKnownHostsFile != "" {
p, err := expandHomeDir(config.SFTPKnownHostsFile)
if err != nil {
continue
return nil, nil, err
}
if _, err := os.Stat(path); err == nil {
knownHostsFiles = append(knownHostsFiles, path)
if _, err := os.Stat(p); err != nil {
return nil, nil, fmt.Errorf("known_hosts file %s: %w", p, err)
}
files = append(files, p)
} else {
for _, candidate := range []string{"~/.ssh/known_hosts", "/etc/ssh/ssh_known_hosts"} {
p, err := expandHomeDir(candidate)
if err != nil {
continue
}
if _, err := os.Stat(p); err == nil {
files = append(files, p)
}
}
}
if len(knownHostsFiles) == 0 {
return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
return fmt.Errorf("no known_hosts files found for host key verification")
}
if len(files) == 0 {
return nil, nil, fmt.Errorf("no known_hosts file available for host key verification " +
"(set SSH_KNOWN_HOSTS or SSH_INSECURE_IGNORE_HOST_KEY=true)")
}
callback, err := knownhosts.New(knownHostsFiles...)
callback, err := knownhosts.New(files...)
if err != nil {
return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
return fmt.Errorf("failed to load known_hosts: %w", err)
}
return nil, nil, fmt.Errorf("failed to load known_hosts: %w", err)
}
return callback
return callback, knownHostAlgorithms(callback, hostPort), nil
}
// knownHostAlgorithms returns the host key algorithms recorded for hostPort,
// using the KeyError returned for a deliberately unknown key.
func knownHostAlgorithms(cb ssh.HostKeyCallback, hostPort string) []string {
addr, err := net.ResolveTCPAddr("tcp", hostPort)
if err != nil {
addr = &net.TCPAddr{}
}
var keyErr *knownhosts.KeyError
if err := cb(hostPort, addr, probeKey{}); !errors.As(err, &keyErr) {
return nil
}
algos := make([]string, 0, len(keyErr.Want))
seen := map[string]bool{}
for _, k := range keyErr.Want {
for _, a := range algorithmsForKeyType(k.Key.Type()) {
if !seen[a] {
seen[a] = true
algos = append(algos, a)
}
}
}
if len(algos) == 0 {
return nil
}
return algos
}
func algorithmsForKeyType(t string) []string {
if t == ssh.KeyAlgoRSA {
return []string{ssh.KeyAlgoRSASHA512, ssh.KeyAlgoRSASHA256, ssh.KeyAlgoRSA}
}
return []string{t}
}
// probeKey is a public key that never matches anything in known_hosts.
type probeKey struct{}
func (probeKey) Type() string { return "probe-key" }
func (probeKey) Marshal() []byte { return []byte("probe-key") }
func (probeKey) Verify([]byte, *ssh.Signature) error { return errors.New("probe key") }
// expandHomeDir expands ~ in file paths to home directory.
func expandHomeDir(path string) (string, error) {
if !strings.HasPrefix(path, "~") {
return filepath.Clean(path), nil
func expandHomeDir(p string) (string, error) {
if !strings.HasPrefix(p, "~") {
return filepath.Clean(p), nil
}
parts := strings.SplitN(path, "/", 2)
parts := strings.SplitN(p, "/", 2)
username := strings.TrimPrefix(parts[0], "~")
var homeDir string
var err error
if username == "" {
homeDir, err = os.UserHomeDir()
if err != nil || homeDir == "" {
h, err := os.UserHomeDir()
if err != nil || h == "" {
currentUser, err := user.Current()
if err != nil {
return "", fmt.Errorf("failed to get home directory: %w", err)
}
homeDir = currentUser.HomeDir
h = currentUser.HomeDir
}
homeDir = h
} else {
userInfo, err := user.Lookup(username)
if err != nil {
@@ -341,10 +552,8 @@ func expandHomeDir(path string) (string, error) {
if homeDir == "" {
return "", fmt.Errorf("empty home directory")
}
if len(parts) == 1 {
return homeDir, nil
}
return filepath.Join(homeDir, parts[1]), nil
}
sftp_test.go
+208
View File
@@ -0,0 +1,208 @@
/*
MIT License
Copyright (c) 2024 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package go_storage
import (
"crypto/ed25519"
"crypto/rand"
"errors"
"io"
"net"
"os"
"path/filepath"
"strings"
"testing"
"github.com/pkg/sftp"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/knownhosts"
)
const (
testSFTPUser = "backup"
testSFTPPass = "s3cret"
)
// startSFTPServer runs an in-process SSH/SFTP server on 127.0.0.1:0 and
// returns its address and a known_hosts file containing its host key.
func startSFTPServer(t *testing.T) (host, port, knownHostsFile string) {
t.Helper()
_, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
signer, err := ssh.NewSignerFromKey(priv)
if err != nil {
t.Fatal(err)
}
cfg := &ssh.ServerConfig{
PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
if c.User() == testSFTPUser && string(pass) == testSFTPPass {
return nil, nil
}
return nil, errors.New("access denied")
},
}
cfg.AddHostKey(signer)
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = ln.Close() })
go func() {
for {
conn, err := ln.Accept()
if err != nil {
return
}
go serveSSHConn(conn, cfg)
}
}()
addr := ln.Addr().String()
knownHostsFile = filepath.Join(t.TempDir(), "known_hosts")
line := knownhosts.Line([]string{knownhosts.Normalize(addr)}, signer.PublicKey())
if err := os.WriteFile(knownHostsFile, []byte(line+"\n"), 0o600); err != nil {
t.Fatal(err)
}
host, port, _ = net.SplitHostPort(addr)
return host, port, knownHostsFile
}
func serveSSHConn(conn net.Conn, cfg *ssh.ServerConfig) {
sconn, chans, reqs, err := ssh.NewServerConn(conn, cfg)
if err != nil {
_ = conn.Close()
return
}
defer func() { _ = sconn.Close() }()
go ssh.DiscardRequests(reqs)
for nc := range chans {
if nc.ChannelType() != "session" {
_ = nc.Reject(ssh.UnknownChannelType, "unsupported")
continue
}
ch, chReqs, err := nc.Accept()
if err != nil {
continue
}
go func() {
for req := range chReqs {
ok := req.Type == "subsystem" && len(req.Payload) > 4 && string(req.Payload[4:]) == "sftp"
_ = req.Reply(ok, nil)
if !ok {
continue
}
srv, err := sftp.NewServer(ch)
if err != nil {
_ = ch.Close()
return
}
if err := srv.Serve(); err != nil && !errors.Is(err, io.EOF) {
_ = srv.Close()
}
_ = ch.Close()
return
}
}()
}
}
func newTestSFTP(t *testing.T) (*SFTPStorage, string) {
t.Helper()
t.Setenv("SSH_AUTH_SOCK", "")
host, port, kh := startSFTPServer(t)
base := t.TempDir()
s, err := NewSFTPStorage(Config{
SFTPHost: host,
SFTPPort: port,
SFTPUsername: testSFTPUser,
SFTPPassword: testSFTPPass,
SFTPDirectory: base,
SFTPKnownHostsFile: kh,
})
if err != nil {
t.Fatalf("NewSFTPStorage: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
return s, base
}
func TestSFTPConformance(t *testing.T) {
s, base := newTestSFTP(t)
runConformance(t, s)
// No temporary files may survive anywhere under the base directory.
_ = filepath.WalkDir(base, func(p string, d os.DirEntry, err error) error {
if err == nil && strings.HasSuffix(p, ".partial") {
t.Errorf("leftover partial file %s", p)
}
return nil
})
}
func TestSFTPHostKeyVerification(t *testing.T) {
t.Setenv("SSH_AUTH_SOCK", "")
host, port, _ := startSFTPServer(t)
base := Config{SFTPHost: host, SFTPPort: port, SFTPUsername: testSFTPUser, SFTPPassword: testSFTPPass, SFTPDirectory: t.TempDir()}
// A known_hosts file with a different key for this host must be rejected.
_, otherPriv, _ := ed25519.GenerateKey(rand.Reader)
otherSigner, _ := ssh.NewSignerFromKey(otherPriv)
wrong := filepath.Join(t.TempDir(), "known_hosts")
line := knownhosts.Line([]string{knownhosts.Normalize(net.JoinHostPort(host, port))}, otherSigner.PublicKey())
if err := os.WriteFile(wrong, []byte(line+"\n"), 0o600); err != nil {
t.Fatal(err)
}
cfg := base
cfg.SFTPKnownHostsFile = wrong
if s, err := NewSFTPStorage(cfg); err == nil {
_ = s.Close()
t.Fatal("connected despite host key mismatch")
}
// Missing known_hosts file is an error with a helpful message.
cfg = base
cfg.SFTPKnownHostsFile = filepath.Join(t.TempDir(), "nope")
if _, err := NewSFTPStorage(cfg); err == nil {
t.Fatal("connected without known_hosts")
}
t.Setenv("HOME", t.TempDir())
cfg.SFTPKnownHostsFile = ""
if _, err := os.Stat("/etc/ssh/ssh_known_hosts"); os.IsNotExist(err) {
if _, err := NewSFTPStorage(cfg); err == nil || !strings.Contains(err.Error(), "SSH_INSECURE_IGNORE_HOST_KEY") {
t.Fatalf("expected helpful known_hosts error, got %v", err)
}
}
// Explicitly ignoring host keys works.
cfg.SFTPIgnoreKnownHosts = true
s, err := NewSFTPStorage(cfg)
if err != nil {
t.Fatalf("SFTPIgnoreKnownHosts: %v", err)
}
_ = s.Close()
}
storage.go
+291 -28
View File
@@ -1,15 +1,64 @@
/*
MIT License
Copyright (c) 2024 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
// Package go_storage provides a uniform, streaming storage abstraction over
// S3 (and S3-compatible services), Azure Blob Storage, Google Cloud Storage,
// SFTP, FTP and the local filesystem.
//
// All backends implement [Storage]. Objects are addressed by keys that are
// relative to the backend base (bucket, container, remote directory or local
// base path) and always use forward slashes.
package go_storage
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path"
"path/filepath"
"strings"
"time"
)
// Sentinel errors. Backends wrap them with %w, so test with errors.Is.
var (
// ErrNotFound is returned (wrapped) by Stat, Get and Remove when the key does not exist.
ErrNotFound = errors.New("storage: object not found")
// ErrPreconditionFailed is returned (wrapped) by PutIf when the condition does not hold.
ErrPreconditionFailed = errors.New("storage: precondition failed")
// ErrConditionNotSupported is returned (wrapped) by PutIf when the backend cannot
// evaluate the requested condition atomically. Callers should fall back to locking.
ErrConditionNotSupported = errors.New("storage: conditional write not supported by backend")
)
// Storage defines the interface for remote file storage operations.
// All implementations must be safe for concurrent use.
// All implementations are safe for concurrent use (see FTPStorage for the
// one restriction regarding open Get bodies).
type Storage interface {
// Upload uploads a local file to the remote storage with the given target name.
Upload(ctx context.Context, localPath string, remotePath string) error
@@ -17,37 +66,91 @@ type Storage interface {
// Download downloads a file from remote storage to a local path.
Download(ctx context.Context, remotePath string, localPath string) error
// List returns all items in the storage that match the given prefix.
// List returns all items (recursively) in the storage that match the given prefix.
List(ctx context.Context, prefix string) ([]Item, error)
// Remove deletes a file from remote storage.
// Remove deletes a file from remote storage. A missing key returns an
// error wrapping ErrNotFound.
Remove(ctx context.Context, remotePath string) error
// Close releases any resources held by the storage implementation.
Close() error
// Put streams r to key. r may be of unknown length; memory use is bounded.
// If reading r fails or ctx is cancelled, Put returns an error and no
// (partial) object is left at key.
Put(ctx context.Context, key string, r io.Reader, opts *PutOptions) (Item, error)
// Get opens key for streaming reads. The caller must Close the body.
// A missing key returns an error wrapping ErrNotFound.
Get(ctx context.Context, key string) (io.ReadCloser, Item, error)
// Stat returns metadata for key. A missing key returns an error wrapping ErrNotFound.
Stat(ctx context.Context, key string) (Item, error)
// PutIf writes r to key only if cond holds, atomically with respect to
// other PutIf calls. It returns an error wrapping ErrPreconditionFailed when
// the condition does not hold, or ErrConditionNotSupported when the backend
// cannot evaluate it. Intended for small objects (index files).
PutIf(ctx context.Context, key string, r io.Reader, cond Condition, opts *PutOptions) (Item, error)
// ListDir lists only the files directly inside dir (dir is treated as a
// directory: "a/b" never matches "a/bc/x"; "" = root). Non-recursive, no
// directories are returned. A missing directory yields an empty list.
ListDir(ctx context.Context, dir string) ([]Item, error)
}
// Item represents a file or directory in remote storage.
type Item struct {
Key string // The path/key of the item
Key string // The path/key of the item, relative to the backend base
ModifiedTime time.Time // Last modification time
Size int64 // Size in bytes (0 for directories)
IsDirectory bool // Whether this item is a directory
// ETag is an opaque version token usable with Condition.IfMatch
// (S3/Azure ETag; GCS generation as decimal string; local:
// "<mtime-unixnano hex>-<size hex>"; SFTP/FTP: "").
ETag string
// VersionID is the S3 version id when bucket versioning is enabled.
VersionID string
}
// PutOptions are optional parameters for Put and PutIf.
type PutOptions struct {
ContentType string
Metadata map[string]string
Size int64 // optional size hint; -1 or 0 = unknown
}
// Condition is a precondition for PutIf. Exactly one field must be set; a
// zero Condition is rejected (it would otherwise silently become an
// unconditional write, e.g. when an empty ETag is passed as IfMatch).
type Condition struct {
IfMatch string // only write if the current ETag equals this
IfNoneMatch bool // only write if the object does not exist
}
// Config holds configuration options for creating storage instances.
type Config struct {
// S3 configuration
S3Region string
S3Bucket string
S3Profile string
S3KeyID string
S3Secret string
S3Endpoint string
S3ForcePath bool
S3DisableTLS bool
// Local configuration
LocalPath string // base directory for the local backend (default "/backup")
// B2 configuration
// S3 configuration
S3Region string
S3Bucket string
S3Profile string
S3KeyID string
S3Secret string
S3SessionToken string
S3Endpoint string
S3ForcePath bool
S3DisableTLS bool
S3StorageClass string // e.g. STANDARD, STANDARD_IA, GLACIER_IR
S3SSE string // "AES256" or "aws:kms"
S3SSEKMSKeyID string // KMS key id when S3SSE is "aws:kms"
S3PartSize int64 // multipart part size in bytes (default 64 MiB, min 5 MiB)
S3Concurrency int // multipart upload concurrency (default 2)
// B2 configuration (reserved, no backend implementation yet)
B2KeyID string
B2AppKey string
B2Bucket string
@@ -58,21 +161,30 @@ type Config struct {
GCSBucket string
GCSEndpoint string
GCSCredentialsFile string
GCSCredentialsJSON string
// Azure configuration
AzureAccount string
AzureKey string
AzureContainer string
AzureEndpoint string
// AzureEndpoint is either a domain suffix (default "blob.core.windows.net",
// the service URL becomes https://<account>.<endpoint>) or a full service
// URL containing a scheme (e.g. "http://127.0.0.1:10000/devstoreaccount1"
// for Azurite), used as is.
AzureEndpoint string
// AzureSASToken, when set, is used instead of the shared key.
AzureSASToken string
// SFTP configuration
SFTPHost string
SFTPPort string
SFTPUsername string
SFTPPassword string
SFTPDirectory string
SFTPIdentityFile string
SFTPIgnoreKnownHosts bool
SFTPHost string
SFTPPort string
SFTPUsername string
SFTPPassword string
SFTPDirectory string
SFTPIdentityFile string
SFTPIdentityPassphrase string
SFTPKnownHostsFile string
SFTPIgnoreKnownHosts bool
// FTP configuration
FTPHost string
@@ -80,11 +192,13 @@ type Config struct {
FTPUsername string
FTPPassword string
FTPDirectory string
FTPTLS bool // explicit FTPS (AUTH TLS)
}
// StorageType represents the type of storage backend.
type StorageType string
// Supported storage types.
const (
StorageTypeS3 StorageType = "s3"
StorageTypeGCS StorageType = "gcs"
@@ -94,6 +208,9 @@ const (
StorageTypeLocal StorageType = "local"
)
// DefaultLocalPath is the base directory of the local backend when Config.LocalPath is empty.
const DefaultLocalPath = "/backup"
// NewStorage creates a new Storage instance based on the specified type and configuration.
func NewStorage(storageType StorageType, config Config) (Storage, error) {
switch storageType {
@@ -108,19 +225,165 @@ func NewStorage(storageType StorageType, config Config) (Storage, error) {
case StorageTypeFTP:
return NewFTPStorage(config)
case StorageTypeLocal:
return NewLocalStorage("/backups")
p := config.LocalPath
if p == "" {
p = DefaultLocalPath
}
return NewLocalStorage(p)
default:
return nil, fmt.Errorf("unsupported storage type: %s", storageType)
}
}
// normalizePathSeparators converts OS-specific path separators to forward slashes.
// This is necessary for cloud storage services which expect forward slashes.
func normalizePathSeparators(path string) string {
if os.PathSeparator == '/' {
return path
func normalizePathSeparators(p string) string {
if os.PathSeparator != '/' {
p = strings.ReplaceAll(p, string(os.PathSeparator), "/")
}
return strings.ReplaceAll(path, string(os.PathSeparator), "/")
return strings.ReplaceAll(p, "\\", "/")
}
// cleanKey normalises an object key: separators become "/", the path is
// cleaned and leading slashes are removed. Keys escaping the base ("..")
// and empty keys are rejected.
func cleanKey(key string) (string, error) {
k, err := cleanDir(key)
if err != nil {
return "", err
}
if k == "" {
return "", fmt.Errorf("storage: empty key")
}
return k, nil
}
// cleanDir is like cleanKey but allows the empty string (root).
func cleanDir(dir string) (string, error) {
d := path.Clean("/" + normalizePathSeparators(dir))
d = strings.TrimPrefix(d, "/")
// path.Clean of an absolute path never yields "..", but check the raw
// input so that escaping attempts are reported rather than silently clamped.
for _, seg := range strings.Split(path.Clean(normalizePathSeparators(dir)), "/") {
if seg == ".." {
return "", fmt.Errorf("storage: key %q escapes the base directory", dir)
}
}
return d, nil
}
// cleanPrefix normalises a List prefix. Unlike keys, a prefix is not
// path-cleaned (a trailing "/" is significant) but must not contain "..".
func cleanPrefix(prefix string) (string, error) {
p := normalizePathSeparators(prefix)
for _, seg := range strings.Split(p, "/") {
if seg == ".." {
return "", fmt.Errorf("storage: prefix %q escapes the base directory", prefix)
}
}
return strings.TrimLeft(p, "/"), nil
}
// dirPrefix returns the object prefix for a directory ("" for root, "a/b/" otherwise).
func dirPrefix(dir string) string {
if dir == "" {
return ""
}
return dir + "/"
}
// partialName returns a unique temporary name next to p.
func partialName(p string) string {
var b [6]byte
_, _ = rand.Read(b[:])
return p + "." + hex.EncodeToString(b[:]) + ".partial"
}
// ctxReader aborts reads once ctx is done.
type ctxReader struct {
ctx context.Context
r io.Reader
}
func (c ctxReader) Read(p []byte) (int, error) {
if err := c.ctx.Err(); err != nil {
return 0, err
}
return c.r.Read(p)
}
// countingReader counts bytes read.
type countingReader struct {
r io.Reader
n int64
}
func (c *countingReader) Read(p []byte) (int, error) {
n, err := c.r.Read(p)
c.n += int64(n)
return n, err
}
// errEmptyCondition is returned by PutIf when neither IfMatch nor IfNoneMatch is set.
var errEmptyCondition = errors.New("storage: PutIf requires Condition.IfMatch or Condition.IfNoneMatch")
func validateCondition(c Condition) error {
if c.IfMatch == "" && !c.IfNoneMatch {
return errEmptyCondition
}
if c.IfMatch != "" && c.IfNoneMatch {
return errors.New("storage: Condition.IfMatch and Condition.IfNoneMatch are mutually exclusive")
}
return nil
}
// maxConditionalPutSize bounds the buffer used by backends whose conditional
// write needs a single request (S3).
const maxConditionalPutSize = 16 << 20
// uploadFile implements Storage.Upload on top of Put.
func uploadFile(ctx context.Context, s Storage, localPath, remotePath string) error {
f, err := os.Open(localPath)
if err != nil {
return fmt.Errorf("failed to open local file: %w", err)
}
defer func() { _ = f.Close() }()
opts := &PutOptions{Size: -1}
if fi, err := f.Stat(); err == nil {
opts.Size = fi.Size()
}
_, err = s.Put(ctx, remotePath, f, opts)
return err
}
// downloadFile implements Storage.Download on top of Get.
func downloadFile(ctx context.Context, s Storage, remotePath, localPath string) (err error) {
body, _, err := s.Get(ctx, remotePath)
if err != nil {
return err
}
defer func() { _ = body.Close() }()
if dir := filepath.Dir(localPath); dir != "" {
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("failed to create local directory: %w", err)
}
}
f, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("failed to create local file: %w", err)
}
defer func() {
if cerr := f.Close(); cerr != nil && err == nil {
err = fmt.Errorf("failed to close local file: %w", cerr)
}
if err != nil {
_ = os.Remove(localPath)
}
}()
if _, err = io.Copy(f, ctxReader{ctx: ctx, r: body}); err != nil {
return fmt.Errorf("failed to download %s: %w", remotePath, err)
}
return nil
}
// validateConfig checks if the required configuration fields are present.
storage_test.go
+191 -20
View File
@@ -1,41 +1,212 @@
/*
MIT License
Copyright (c) 2024 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package go_storage
import (
"context"
"errors"
"io"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
// Test helpers
func TestStorageInterface(t *testing.T) {
var _ Storage = (*S3Storage)(nil)
var _ Storage = (*GCSStorage)(nil)
var _ Storage = (*AzureStorage)(nil)
var _ Storage = (*SFTPStorage)(nil)
var _ Storage = (*FTPStorage)(nil)
var _ Storage = (*LocalStorage)(nil)
}
func TestContextTimeout(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
config := Config{
S3Region: "us-east-1",
S3Bucket: "test-bucket",
func TestCleanKey(t *testing.T) {
ok := map[string]string{
"x/y.txt": "x/y.txt",
"/a/b.txt": "a/b.txt",
"a//b/./c": "a/b/c",
`a\b\c.txt`: "a/b/c.txt",
"a/../b.txt": "b.txt",
"./x": "x",
"backup.sql~": "backup.sql~",
}
s3, err := NewStorage(StorageTypeS3, config)
if err != nil {
t.Fatalf("Failed to create storage: %v", err)
}
defer func(s3 Storage) {
err = s3.Close()
if err != nil {
t.Logf("Failed to close storage: %v", err)
for in, want := range ok {
got, err := cleanKey(in)
if err != nil || got != want {
t.Errorf("cleanKey(%q) = %q, %v; want %q", in, got, err, want)
}
}(s3)
}
for _, in := range []string{"", "/", ".", "..", "../x", "a/../../x", `..\x`} {
if got, err := cleanKey(in); err == nil {
t.Errorf("cleanKey(%q) = %q, want error", in, got)
}
}
if d, err := cleanDir(""); err != nil || d != "" {
t.Errorf("cleanDir(\"\") = %q, %v", d, err)
}
if _, err := cleanPrefix("a/../../b"); err == nil {
t.Error("cleanPrefix with .. accepted")
}
}
err = s3.Upload(ctx, "test.txt", "remote-test.txt")
func TestNewStorageLocalPath(t *testing.T) {
dir := filepath.Join(t.TempDir(), "base")
s, err := NewStorage(StorageTypeLocal, Config{LocalPath: dir})
if err != nil {
t.Logf("Upload failed (expected in test): %v", err)
t.Fatal(err)
}
if got := s.(*LocalStorage).BasePath(); got != dir {
t.Fatalf("base path = %q, want %q", got, dir)
}
}
func TestNewS3StorageOffline(t *testing.T) {
s, err := NewS3Storage(Config{
S3Bucket: "test-bucket",
S3Endpoint: "127.0.0.1:9",
S3DisableTLS: true,
S3KeyID: "k",
S3Secret: "s",
S3PartSize: 1,
})
if err != nil {
t.Fatalf("NewS3Storage: %v", err)
}
if s.region != DefaultS3Region {
t.Errorf("region = %q", s.region)
}
if got := *s.client.Options().BaseEndpoint; got != "http://127.0.0.1:9" {
t.Errorf("endpoint = %q", got)
}
//nolint:staticcheck // manager is deprecated, see s3.go
if s.uploader.PartSize != MinS3PartSize || s.uploader.Concurrency != DefaultS3Concurrency {
t.Errorf("uploader part size/concurrency = %d/%d", s.uploader.PartSize, s.uploader.Concurrency)
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := s.Upload(ctx, filepath.Join(t.TempDir(), "missing.txt"), "remote.txt"); err == nil {
t.Fatal("Upload of a missing local file succeeded")
}
}
func TestLocalConformance(t *testing.T) {
s, err := NewLocalStorage(t.TempDir())
if err != nil {
t.Fatal(err)
}
defer func() { _ = s.Close() }()
runConformance(t, s)
}
func TestLocalNoLeftovers(t *testing.T) {
base := t.TempDir()
s, err := NewLocalStorage(base)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
_, _ = s.Put(ctx, "x/y.bin", &failingReader{r: strings.NewReader(strings.Repeat("a", 100)), n: 10, err: errors.New("boom")}, nil)
if _, err := s.PutIf(ctx, "x/idx.json", strings.NewReader("1"), Condition{IfNoneMatch: true}, nil); err != nil {
t.Fatal(err)
}
_, _ = s.PutIf(ctx, "x/idx.json", strings.NewReader("2"), Condition{IfNoneMatch: true}, nil)
entries, err := os.ReadDir(filepath.Join(base, "x"))
if err != nil {
t.Fatal(err)
}
for _, e := range entries {
if e.Name() != "idx.json" {
t.Errorf("leftover file %s", e.Name())
}
}
}
// TestLocalPutIfConcurrent checks that concurrent read-modify-write cycles
// with IfMatch never lose an update.
func TestLocalPutIfConcurrent(t *testing.T) {
s, err := NewLocalStorage(t.TempDir())
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
if _, err := s.PutIf(ctx, "idx", strings.NewReader(""), Condition{IfNoneMatch: true}, nil); err != nil {
t.Fatal(err)
}
const workers, perWorker = 8, 10
var wg sync.WaitGroup
for w := 0; w < workers; w++ {
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < perWorker; {
body, it, err := s.Get(ctx, "idx")
if err != nil {
t.Error(err)
return
}
b, _ := io.ReadAll(body)
_ = body.Close()
_, err = s.PutIf(ctx, "idx", strings.NewReader(string(b)+"x"), Condition{IfMatch: it.ETag}, nil)
if errors.Is(err, ErrPreconditionFailed) {
continue
}
if err != nil {
t.Error(err)
return
}
i++
}
}()
}
wg.Wait()
body, _, err := s.Get(ctx, "idx")
if err != nil {
t.Fatal(err)
}
defer func() { _ = body.Close() }()
b, _ := io.ReadAll(body)
if len(b) != workers*perWorker {
t.Fatalf("lost updates: got %d, want %d", len(b), workers*perWorker)
}
}
func TestLocalPathTraversal(t *testing.T) {
base := t.TempDir()
s, err := NewLocalStorage(filepath.Join(base, "store"))
if err != nil {
t.Fatal(err)
}
for _, k := range []string{"../outside.txt", "a/../../outside.txt", `..\outside.txt`} {
if _, err := s.Put(context.Background(), k, strings.NewReader("x"), nil); err == nil {
t.Errorf("Put(%q) succeeded", k)
}
}
if _, err := os.Stat(filepath.Join(base, "outside.txt")); !os.IsNotExist(err) {
t.Fatal("file written outside base")
}
}