offen/docker-volume-backup
1
1
Fork 0
mirror of https://github.com/offen/docker-volume-backup.git synced 2024-11-10 08:55:45 +08:00
Code Issues Projects Releases Packages Wiki Activity

Handle configuration errors for Azure storage upfront

Browse source
This commit is contained in:
Frederik Ring 2024-08-11 15:08:06 +02:00
parent 8a64da4b0b
commit d8aa6db3f5
4 changed files with 115 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
cmd/backup/command.go
View file

@ -36,6 +36,9 @@ func (c *command) runAsCommand() error {
}
for _, config := range configurations {
if err := config.validate(); err != nil {
return errwrap.Wrap(err, "error validating config")
}
if err := runScript(config); err != nil {
return errwrap.Wrap(err, "error running script")
}
@ -101,6 +104,12 @@ func (c *command) schedule(strategy configStrategy) error {
}
for _, cfg := range configurations {
if err := cfg.validate(); err != nil {
return errwrap.Wrap(
err,
fmt.Sprintf("error validating config for schedule %s", cfg.BackupCronExpression),
)
}
config := cfg
id, err := c.cr.AddFunc(config.BackupCronExpression, func() {
c.logger.Info(

166
cmd/backup/config.go
View file

@ -12,83 +12,91 @@ import (
"strconv"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/offen/docker-volume-backup/internal/errwrap"
)
// Config holds all configuration values that are expected to be set
// by users.
type Config struct {
AwsS3BucketName string `split_words:"true"`
AwsS3Path string `split_words:"true"`
AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"`
AwsEndpointProto string `split_words:"true" default:"https"`
AwsEndpointInsecure bool `split_words:"true"`
AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"`
AwsStorageClass string `split_words:"true"`
AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"`
AwsSecretAccessKey string `split_words:"true"`
AwsIamRoleEndpoint string `split_words:"true"`
AwsPartSize int64 `split_words:"true"`
BackupCompression CompressionType `split_words:"true" default:"gz"`
GzipParallelism WholeNumber `split_words:"true" default:"1"`
BackupSources string `split_words:"true" default:"/backup"`
BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"`
BackupFilenameExpand bool `split_words:"true"`
BackupLatestSymlink string `split_words:"true"`
BackupArchive string `split_words:"true" default:"/archive"`
BackupCronExpression string `split_words:"true" default:"@daily"`
BackupRetentionDays int32 `split_words:"true" default:"-1"`
BackupPruningLeeway time.Duration `split_words:"true" default:"1m"`
BackupPruningPrefix string `split_words:"true"`
BackupStopContainerLabel string `split_words:"true"`
BackupStopDuringBackupLabel string `split_words:"true" default:"true"`
BackupStopServiceTimeout time.Duration `split_words:"true" default:"5m"`
BackupFromSnapshot bool `split_words:"true"`
BackupExcludeRegexp RegexpDecoder `split_words:"true"`
BackupSkipBackendsFromPrune []string `split_words:"true"`
GpgPassphrase string `split_words:"true"`
GpgPublicKeyRing string `split_words:"true"`
NotificationURLs []string `envconfig:"NOTIFICATION_URLS"`
NotificationLevel string `split_words:"true" default:"error"`
EmailNotificationRecipient string `split_words:"true"`
EmailNotificationSender string `split_words:"true" default:"noreply@nohost"`
EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"`
EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"`
EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"`
EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"`
WebdavUrl string `split_words:"true"`
WebdavUrlInsecure bool `split_words:"true"`
WebdavPath string `split_words:"true" default:"/"`
WebdavUsername string `split_words:"true"`
WebdavPassword string `split_words:"true"`
SSHHostName string `split_words:"true"`
SSHPort string `split_words:"true" default:"22"`
SSHUser string `split_words:"true"`
SSHPassword string `split_words:"true"`
SSHIdentityFile string `split_words:"true" default:"/root/.ssh/id_rsa"`
SSHIdentityPassphrase string `split_words:"true"`
SSHRemotePath string `split_words:"true"`
ExecLabel string `split_words:"true"`
ExecForwardOutput bool `split_words:"true"`
LockTimeout time.Duration `split_words:"true" default:"60m"`
AzureStorageAccountName string `split_words:"true"`
AzureStoragePrimaryAccountKey string `split_words:"true"`
AzureStorageConnectionString string `split_words:"true"`
AzureStorageContainerName string `split_words:"true"`
AzureStoragePath string `split_words:"true"`
AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"`
AzureStorageAccessTier string `split_words:"true"`
DropboxEndpoint string `split_words:"true" default:"https://api.dropbox.com/"`
DropboxOAuth2Endpoint string `envconfig:"DROPBOX_OAUTH2_ENDPOINT" default:"https://api.dropbox.com/"`
DropboxRefreshToken string `split_words:"true"`
DropboxAppKey string `split_words:"true"`
DropboxAppSecret string `split_words:"true"`
DropboxRemotePath string `split_words:"true"`
DropboxConcurrencyLevel NaturalNumber `split_words:"true" default:"6"`
AwsS3BucketName string `split_words:"true"`
AwsS3Path string `split_words:"true"`
AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"`
AwsEndpointProto string `split_words:"true" default:"https"`
AwsEndpointInsecure bool `split_words:"true"`
AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"`
AwsStorageClass string `split_words:"true"`
AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"`
AwsSecretAccessKey string `split_words:"true"`
AwsIamRoleEndpoint string `split_words:"true"`
AwsPartSize int64 `split_words:"true"`
BackupCompression CompressionType `split_words:"true" default:"gz"`
GzipParallelism WholeNumber `split_words:"true" default:"1"`
BackupSources string `split_words:"true" default:"/backup"`
BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"`
BackupFilenameExpand bool `split_words:"true"`
BackupLatestSymlink string `split_words:"true"`
BackupArchive string `split_words:"true" default:"/archive"`
BackupCronExpression string `split_words:"true" default:"@daily"`
BackupRetentionDays int32 `split_words:"true" default:"-1"`
BackupPruningLeeway time.Duration `split_words:"true" default:"1m"`
BackupPruningPrefix string `split_words:"true"`
BackupStopContainerLabel string `split_words:"true"`
BackupStopDuringBackupLabel string `split_words:"true" default:"true"`
BackupStopServiceTimeout time.Duration `split_words:"true" default:"5m"`
BackupFromSnapshot bool `split_words:"true"`
BackupExcludeRegexp RegexpDecoder `split_words:"true"`
BackupSkipBackendsFromPrune []string `split_words:"true"`
GpgPassphrase string `split_words:"true"`
GpgPublicKeyRing string `split_words:"true"`
NotificationURLs []string `envconfig:"NOTIFICATION_URLS"`
NotificationLevel string `split_words:"true" default:"error"`
EmailNotificationRecipient string `split_words:"true"`
EmailNotificationSender string `split_words:"true" default:"noreply@nohost"`
EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"`
EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"`
EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"`
EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"`
WebdavUrl string `split_words:"true"`
WebdavUrlInsecure bool `split_words:"true"`
WebdavPath string `split_words:"true" default:"/"`
WebdavUsername string `split_words:"true"`
WebdavPassword string `split_words:"true"`
SSHHostName string `split_words:"true"`
SSHPort string `split_words:"true" default:"22"`
SSHUser string `split_words:"true"`
SSHPassword string `split_words:"true"`
SSHIdentityFile string `split_words:"true" default:"/root/.ssh/id_rsa"`
SSHIdentityPassphrase string `split_words:"true"`
SSHRemotePath string `split_words:"true"`
ExecLabel string `split_words:"true"`
ExecForwardOutput bool `split_words:"true"`
LockTimeout time.Duration `split_words:"true" default:"60m"`
AzureStorageAccountName string `split_words:"true"`
AzureStoragePrimaryAccountKey string `split_words:"true"`
AzureStorageConnectionString string `split_words:"true"`
AzureStorageContainerName string `split_words:"true"`
AzureStoragePath string `split_words:"true"`
AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"`
AzureStorageAccessTier AzureStorageAccessTier `split_words:"true"`
DropboxEndpoint string `split_words:"true" default:"https://api.dropbox.com/"`
DropboxOAuth2Endpoint string `envconfig:"DROPBOX_OAUTH2_ENDPOINT" default:"https://api.dropbox.com/"`
DropboxRefreshToken string `split_words:"true"`
DropboxAppKey string `split_words:"true"`
DropboxAppSecret string `split_words:"true"`
DropboxRemotePath string `split_words:"true"`
DropboxConcurrencyLevel NaturalNumber `split_words:"true" default:"6"`
source string
additionalEnvVars map[string]string
}
func (c *Config) validate() error {
if c.AzureStoragePrimaryAccountKey != "" && c.AzureStorageConnectionString != "" {
return errwrap.Wrap(nil, "using azure primary account key and connection string are mutually exclusive")
}
return nil
}
type CompressionType string
func (c *CompressionType) Decode(v string) error {
@ -180,6 +188,30 @@ func (n *WholeNumber) Int() int {
return int(*n)
}
type AzureStorageAccessTier string
func (t *AzureStorageAccessTier) Decode(v string) error {
if v == "" {
*t = ""
return nil
}
for _, a := range blob.PossibleAccessTierValues() {
if string(a) == v {
*t = AzureStorageAccessTier(v)
return nil
}
}
return errwrap.Wrap(nil, fmt.Sprintf("%s is not a possible access tier value", v))
}
func (t *AzureStorageAccessTier) AccessTier() *blob.AccessTier {
if *t == "" {
return nil
}
a := blob.AccessTier(*t)
return &a
}
type envVarLookup struct {
ok bool
key string

2
cmd/backup/script.go
View file

@ -199,7 +199,7 @@ func (s *script) init() error {
Endpoint: s.c.AzureStorageEndpoint,
RemotePath: s.c.AzureStoragePath,
ConnectionString: s.c.AzureStorageConnectionString,
AccessTier: s.c.AzureStorageAccessTier,
AccessTier: s.c.AzureStorageAccessTier.AccessTier(),
}
azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc)
if err != nil {

30
internal/storage/azure/azure.go
View file

@ -39,15 +39,11 @@ type Config struct {
ConnectionString string
Endpoint string
RemotePath string
AccessTier string
AccessTier *blob.AccessTier
}
// NewStorageBackend creates and initializes a new Azure Blob Storage backend.
func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
if opts.PrimaryAccountKey != "" && opts.ConnectionString != "" {
return nil, errwrap.Wrap(nil, "using primary account key and connection string are mutually exclusive")
}
endpointTemplate, err := template.New("endpoint").Parse(opts.Endpoint)
if err != nil {
return nil, errwrap.Wrap(err, "error parsing endpoint template")
@ -85,26 +81,12 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error
}
}
var uploadStreamOptions *blockblob.UploadStreamOptions
if opts.AccessTier != "" {
var found bool
for _, t := range blob.PossibleAccessTierValues() {
if string(t) == opts.AccessTier {
found = true
uploadStreamOptions = &blockblob.UploadStreamOptions{
AccessTier: &t,
}
}
}
if !found {
return nil, errwrap.Wrap(nil, fmt.Sprintf("%s is not a possible access tier value", opts.AccessTier))
}
}
storage := azureBlobStorage{
client: client,
uploadStreamOptions: uploadStreamOptions,
containerName: opts.ContainerName,
client: client,
uploadStreamOptions: &blockblob.UploadStreamOptions{
AccessTier: opts.AccessTier,
},
containerName: opts.ContainerName,
StorageBackend: &storage.StorageBackend{
DestinationPath: opts.RemotePath,
Log: logFunc,