diff --git a/cmd/init.go b/cmd/init.go index ff0eeb28..2f7c202f 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -452,7 +452,7 @@ func initPostbackMessengers(m *manager.Manager) []messenger.Messenger { func initMediaStore() media.Store { switch provider := ko.String("upload.provider"); provider { case "s3": - var o s3.Opts + var o s3.Opt ko.Unmarshal("upload.s3", &o) up, err := s3.NewS3Store(o) if err != nil { diff --git a/cmd/settings.go b/cmd/settings.go index ae487058..d2a06179 100644 --- a/cmd/settings.go +++ b/cmd/settings.go @@ -42,6 +42,7 @@ type settings struct { UploadProvider string `json:"upload.provider"` UploadFilesystemUploadPath string `json:"upload.filesystem.upload_path"` UploadFilesystemUploadURI string `json:"upload.filesystem.upload_uri"` + UploadS3URL string `json:"upload.s3.url"` UploadS3AwsAccessKeyID string `json:"upload.s3.aws_access_key_id"` UploadS3AwsDefaultRegion string `json:"upload.s3.aws_default_region"` UploadS3AwsSecretAccessKey string `json:"upload.s3.aws_secret_access_key,omitempty"` diff --git a/frontend/src/views/Settings.vue b/frontend/src/views/Settings.vue index 0d3adfa7..d97bfc15 100644 --- a/frontend/src/views/Settings.vue +++ b/frontend/src/views/Settings.vue @@ -18,7 +18,7 @@
-
+
@@ -213,7 +213,7 @@
- @@ -254,7 +254,7 @@ -
+
+ + + +
@@ -786,6 +796,13 @@ export default Vue.extend({ this.form.messengers.splice(i, 1); }, + onS3URLChange() { + // If a custom non-AWS URL has been entered, don't update it automatically. + if (this.form['upload.s3.url'] !== '' && !this.form['upload.s3.url'].match(/amazonaws\.com/)) { + return; + } + this.form['upload.s3.url'] = `https://s3.${this.form['upload.s3.aws_default_region']}.amazonaws.com`; + }, onSubmit() { const form = JSON.parse(JSON.stringify(this.form)); diff --git a/go.mod b/go.mod index 472c48b5..ee54be08 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/mailru/easyjson v0.7.6 github.com/mitchellh/copystructure v1.1.2 // indirect github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect - github.com/rhnvrm/simples3 v0.5.0 + github.com/rhnvrm/simples3 v0.7.0 github.com/spf13/pflag v1.0.5 github.com/yuin/goldmark v1.3.4 golang.org/x/mod v0.3.0 diff --git a/go.sum b/go.sum index cfc74ecb..2216f9e1 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rhnvrm/simples3 v0.5.0 h1:X+WX0hqoKScdoJAw/G3GArfZ6Ygsn8q+6MdocTMKXOw= github.com/rhnvrm/simples3 v0.5.0/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= +github.com/rhnvrm/simples3 v0.7.0 h1:KSEuKw0eGC5vltLW8ChLvjko+aUr0HbGet+bZHdwfMo= +github.com/rhnvrm/simples3 v0.7.0/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= diff --git a/i18n/de.json b/i18n/de.json index 368577b7..99bef3da 100644 --- a/i18n/de.json +++ b/i18n/de.json @@ -354,6 +354,8 @@ "settings.media.s3.secret": "AWS Access Secret", "settings.media.s3.uploadExpiry": "Upload Ablaufdatum", "settings.media.s3.uploadExpiryHelp": "(Optional) Zeit bis zum Ablauf (in Sekunden) für die generierte URL. Nur für private Buckets. (s, m, h, d für Sekunden, Minuten, Stunden, Tage).", + "settings.media.s3.url": "S3 backend URL", + "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.", "settings.media.title": "Medien Uploads", "settings.media.upload.path": "Upload Pfad", "settings.media.upload.pathHelp": "Pfad zum Upload Verzeichnis.", diff --git a/i18n/en.json b/i18n/en.json index 3fd4494a..aa7005b8 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -354,6 +354,8 @@ "settings.media.s3.secret": "AWS access secret", "settings.media.s3.uploadExpiry": "Upload expiry", "settings.media.s3.uploadExpiryHelp": "(Optional) Specify TTL (in seconds) for the generated presigned URL. Only applicable for private buckets (s, m, h, d for seconds, minutes, hours, days).", + "settings.media.s3.url": "S3 backend URL", + "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.", "settings.media.title": "Media uploads", "settings.media.upload.path": "Upload path", "settings.media.upload.pathHelp": "Path to the directory where media will be uploaded.", diff --git a/i18n/es.json b/i18n/es.json index 32a7dcde..565bb949 100644 --- a/i18n/es.json +++ b/i18n/es.json @@ -354,6 +354,8 @@ "settings.media.s3.secret": "Secrete de acceso a AWS (secret)", "settings.media.s3.uploadExpiry": "Expiración de carga", "settings.media.s3.uploadExpiryHelp": "(Opcional) TTL específico (en segundos) para la URL pre firmada generada. Solo es aplicable para contenedores privados (s, m, h, d para segundos, minutos, horas, días)", + "settings.media.s3.url": "S3 backend URL", + "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.", "settings.media.title": "Cargas de media", "settings.media.upload.path": "Ruta de carga", "settings.media.upload.pathHelp": "Ruta al directorio donde la media será cargada.", diff --git a/i18n/fr.json b/i18n/fr.json index eb9fb45d..65fb08c0 100644 --- a/i18n/fr.json +++ b/i18n/fr.json @@ -354,6 +354,8 @@ "settings.media.s3.secret": "Mot de passe d'accès AWS", "settings.media.s3.uploadExpiry": "Durée de validité", "settings.media.s3.uploadExpiryHelp": "(Facultatif) Spécifiez la durée de validité (en secondes) pour l'URL prédéfinie générée. Uniquement applicable pour les compartiments privés (s, m, h, d pour les secondes, minutes, heures, jours).", + "settings.media.s3.url": "S3 backend URL", + "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.", "settings.media.title": "Mise en ligne de fichiers", "settings.media.upload.path": "Emplacement d'envoi des fichiers", "settings.media.upload.pathHelp": "Chemin vers le répertoire où les médias seront mis en ligne", diff --git a/i18n/it.json b/i18n/it.json index b6de0d52..e2d1a8eb 100644 --- a/i18n/it.json +++ b/i18n/it.json @@ -354,6 +354,8 @@ "settings.media.s3.secret": "Accesso segreto AWS", "settings.media.s3.uploadExpiry": "Caricamento scaduto", "settings.media.s3.uploadExpiryHelp": "(Facoltativo) Specifica il TTL (in secondi) per l'URL predefinito generato. Applicabile solo per i buckets privati (s, m, h, d per i secondi, minuti, ore e giorni).", + "settings.media.s3.url": "S3 backend URL", + "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.", "settings.media.title": "Caricamento dei media", "settings.media.upload.path": "Percorso del caricamento", "settings.media.upload.pathHelp": "Percorso verso il repertorio dove i media saranno caricati.", diff --git a/i18n/ml.json b/i18n/ml.json index 4b250ac2..75a69279 100644 --- a/i18n/ml.json +++ b/i18n/ml.json @@ -354,6 +354,8 @@ "settings.media.s3.secret": "AWS പ്രവേശന രഹസ്യം", "settings.media.s3.uploadExpiry": "അപ്ലോഡിന്റെ കാലാവധി", "settings.media.s3.uploadExpiryHelp": "(ഐച്ഛികം) മുൻകൂട്ടി നിർമ്മിക്കുന്ന യൂ. ആർ. എല്ലിനുള്ള സെക്കന്റിലുള്ള TTL വ്യക്തമാക്കുക . സ്വകാര്യ ബക്കറ്റുകൾക്ക് മാത്രമേ ബാധകമാകൂ (s, m, h, d എന്നിവ യഥാക്രമം സെക്കന്റ്, മിനുട്ട്, മണിക്കൂർ, ദിവസങ്ങൾ എന്നിവയെ സൂചിപ്പിക്കുന്നു).", + "settings.media.s3.url": "S3 backend URL", + "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.", "settings.media.title": "മീഡിയാ അപ്ലോഡുകൾ", "settings.media.upload.path": "അപ്ലോഡ് പാത്ത്", "settings.media.upload.pathHelp": "മീഡിയ അപ്ലോഡ് ചെയ്യുന്നതിനുള്ള ഡയറക്ടറിയിലേക്കുള്ള പാത്ത്.", diff --git a/i18n/pl.json b/i18n/pl.json index c93fd23f..7600a2c7 100644 --- a/i18n/pl.json +++ b/i18n/pl.json @@ -354,6 +354,8 @@ "settings.media.s3.secret": "Sekret dostępu AWS", "settings.media.s3.uploadExpiry": "Wygaśnięcie przesyłania", "settings.media.s3.uploadExpiryHelp": "(Opcjonalne) Zdefiniuj TTL (w sekundach) dla wygenerowanego podpisanego URL. Tylko dla prywatnych komór (bucketów) (s, m, h, d dla sekund, minut, godzin, dni).", + "settings.media.s3.url": "S3 backend URL", + "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.", "settings.media.title": "Wysyłka mediów", "settings.media.upload.path": "Ścieżka do wysyłki", "settings.media.upload.pathHelp": "Ścieżka do folderu do którego media będą wrzucane.", diff --git a/i18n/pt-BR.json b/i18n/pt-BR.json index 23398cf7..f4059590 100644 --- a/i18n/pt-BR.json +++ b/i18n/pt-BR.json @@ -354,6 +354,8 @@ "settings.media.s3.secret": "Segredo de acesso AWS", "settings.media.s3.uploadExpiry": "Expiração do arquivo enviado", "settings.media.s3.uploadExpiryHelp": "(Opcional) Especificar TTL (em segundos) para a URL pré-assinada gerada. Apenas aplicável para buckets privados (s, m, h, d para segundos, minutos, horas e dias).", + "settings.media.s3.url": "S3 backend URL", + "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.", "settings.media.title": "Envios de mídias", "settings.media.upload.path": "Caminho de envio", "settings.media.upload.pathHelp": "Caminho para o diretório onde a mídia será enviado.", diff --git a/i18n/pt.json b/i18n/pt.json index 4db5810d..ea32f36e 100644 --- a/i18n/pt.json +++ b/i18n/pt.json @@ -354,6 +354,8 @@ "settings.media.s3.secret": "Segredo de acesso AWS", "settings.media.s3.uploadExpiry": "Validade do upload", "settings.media.s3.uploadExpiryHelp": "(Opcional) Especifica TTL (em segundos) para o URL pré-assinado gerado. Apenas aplicável a buckets privados (s, m, h, d para segundos, minutos, horas e dias).", + "settings.media.s3.url": "S3 backend URL", + "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.", "settings.media.title": "Upload de mídia", "settings.media.upload.path": "Caminho de upload", "settings.media.upload.pathHelp": "Caminho para a pasta onde será enviada a mídia.", diff --git a/i18n/ru.json b/i18n/ru.json index dcf0edce..66bd615e 100644 --- a/i18n/ru.json +++ b/i18n/ru.json @@ -354,6 +354,8 @@ "settings.media.s3.secret": "Секретаня фраза AWS", "settings.media.s3.uploadExpiry": "Срок жизни выгрузки", "settings.media.s3.uploadExpiryHelp": "(Необязательно) Укажите TTL (в секундах) сгенерированного подписанного URL. Применимо только для приватных bucket (s, m, h, d соответствует секундам, минутам, часам и дням).", + "settings.media.s3.url": "S3 backend URL", + "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.", "settings.media.title": "Выгрузки медиа", "settings.media.upload.path": "Путь для выгрузок", "settings.media.upload.pathHelp": "Путь до каталога, куда будут выгружаться медиа-файлы.", diff --git a/i18n/tr.json b/i18n/tr.json index db42ae9f..2fd05e08 100644 --- a/i18n/tr.json +++ b/i18n/tr.json @@ -354,6 +354,8 @@ "settings.media.s3.secret": "AWS erişim şifresi(secret)", "settings.media.s3.uploadExpiry": "Yükleme sona erme", "settings.media.s3.uploadExpiryHelp": "(İsteğe bağlı) Oluşturulan önceden imzalanmış URL için TTL'yi (saniye cinsinden) belirtin. Yalnızca özel paketler için geçerlidir (saniye, dakika, saat, gün için s, m, h, d).", + "settings.media.s3.url": "S3 backend URL", + "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.", "settings.media.title": "Medya yüklemeleri", "settings.media.upload.path": "Yükleme yolu", "settings.media.upload.pathHelp": "Medyanın yükleneceği dizinin yolu.", diff --git a/internal/media/providers/s3/s3.go b/internal/media/providers/s3/s3.go index eff7c7ae..43a0018e 100644 --- a/internal/media/providers/s3/s3.go +++ b/internal/media/providers/s3/s3.go @@ -2,7 +2,6 @@ package s3 import ( "errors" - "fmt" "io" "strings" "time" @@ -11,16 +10,14 @@ import ( "github.com/rhnvrm/simples3" ) -const amznS3PublicURL = "https://%s.s3.%s.amazonaws.com%s" - -// Opts represents AWS S3 specific params -type Opts struct { +// Opt represents AWS S3 specific params +type Opt struct { + URL string `koanf:"url"` AccessKey string `koanf:"aws_access_key_id"` SecretKey string `koanf:"aws_secret_access_key"` Region string `koanf:"aws_default_region"` Bucket string `koanf:"bucket"` BucketPath string `koanf:"bucket_path"` - BucketURL string `koanf:"bucket_url"` BucketType string `koanf:"bucket_type"` Expiry time.Duration `koanf:"expiry"` } @@ -28,30 +25,36 @@ type Opts struct { // Client implements `media.Store` for S3 provider type Client struct { s3 *simples3.S3 - opts Opts + opts Opt } // NewS3Store initialises store for S3 provider. It takes in the AWS configuration // and sets up the `simples3` client to interact with AWS APIs for all bucket operations. -func NewS3Store(opts Opts) (media.Store, error) { - var s3svc *simples3.S3 - var err error - if opts.Region == "" { - return nil, errors.New("Invalid AWS Region specified. Please check `upload.s3` config") +func NewS3Store(opt Opt) (media.Store, error) { + var ( + cl *simples3.S3 + err error + ) + if opt.URL == "" { + return nil, errors.New("Invalid AWS URL in settings.") } + opt.URL = strings.TrimRight(opt.URL, "/") + // Use Access Key/Secret Key if specified in config. - if opts.AccessKey != "" && opts.SecretKey != "" { - s3svc = simples3.New(opts.Region, opts.AccessKey, opts.SecretKey) + if opt.AccessKey != "" && opt.SecretKey != "" { + cl = simples3.New(opt.Region, opt.AccessKey, opt.SecretKey) } else { // fallback to IAM role if no access key/secret key is provided. - s3svc, err = simples3.NewUsingIAM(opts.Region) + cl, err = simples3.NewUsingIAM(opt.Region) if err != nil { return nil, err } } + cl.SetEndpoint(opt.URL) + return &Client{ - s3: s3svc, - opts: opts, + s3: cl, + opts: opt, }, nil } @@ -65,7 +68,7 @@ func (c *Client) Put(name string, cType string, file io.ReadSeeker) (string, err Body: file, // Paths inside the bucket should not start with /. - ObjectKey: strings.TrimPrefix(makeBucketPath(c.opts.BucketPath, name), "/"), + ObjectKey: c.makeBucketPath(name), } // Perform an upload. if _, err := c.s3.FileUpload(upParams); err != nil { @@ -78,39 +81,42 @@ func (c *Client) Put(name string, cType string, file io.ReadSeeker) (string, err func (c *Client) Get(name string) string { // Generate a private S3 pre-signed URL if it's a private bucket. if c.opts.BucketType == "private" { - url := c.s3.GeneratePresignedURL(simples3.PresignedInput{ + u := c.s3.GeneratePresignedURL(simples3.PresignedInput{ Bucket: c.opts.Bucket, - ObjectKey: makeBucketPath(c.opts.BucketPath, name), + ObjectKey: c.makeBucketPath(name), Method: "GET", Timestamp: time.Now(), ExpirySeconds: int(c.opts.Expiry.Seconds()), }) - return url + return u } // Generate a public S3 URL if it's a public bucket. - url := "" - if c.opts.BucketURL != "" { - url = c.opts.BucketURL + makeBucketPath(c.opts.BucketPath, name) - } else { - url = fmt.Sprintf(amznS3PublicURL, c.opts.Bucket, c.opts.Region, - makeBucketPath(c.opts.BucketPath, name)) - } - return url + return c.makeFileURL(name) } // Delete accepts the filename of the object and deletes from S3. func (c *Client) Delete(name string) error { err := c.s3.FileDelete(simples3.DeleteInput{ Bucket: c.opts.Bucket, - ObjectKey: strings.TrimPrefix(makeBucketPath(c.opts.BucketPath, name), "/"), + ObjectKey: c.makeBucketPath(name), }) return err } -func makeBucketPath(bucketPath string, name string) string { - if bucketPath == "/" { - return "/" + name +// makeBucketPath returns the file path inside the bucket. The path should not +// start with a /. +func (c *Client) makeBucketPath(name string) string { + // If the path is root (/), return the filename without the preceding slash. + p := strings.TrimPrefix(strings.TrimSuffix(c.opts.BucketPath, "/"), "/") + if p == "" { + return name } - return fmt.Sprintf("%s/%s", bucketPath, name) + + // whatever/bucket/path/filename.jpg: No preceding slash. + return p + "/" + name +} + +func (c *Client) makeFileURL(name string) string { + return c.opts.URL + "/" + c.opts.Bucket + "/" + c.makeBucketPath(name) } diff --git a/internal/migrations/v0.7.0.go b/internal/migrations/v0.7.0.go index a0984bd6..c65ade91 100644 --- a/internal/migrations/v0.7.0.go +++ b/internal/migrations/v0.7.0.go @@ -85,7 +85,7 @@ func V0_7_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf) error { ('upload.filesystem.upload_uri', '"/uploads"'), ('upload.s3.aws_access_key_id', '""'), ('upload.s3.aws_secret_access_key', '""'), - ('upload.s3.aws_default_region', '"ap-south-b"'), + ('upload.s3.aws_default_region', '"ap-south-1"'), ('upload.s3.bucket', '""'), ('upload.s3.bucket_domain', '""'), ('upload.s3.bucket_path', '"/"'), diff --git a/internal/migrations/v2.0.0.go b/internal/migrations/v2.0.0.go index 8123f14c..50e2f824 100644 --- a/internal/migrations/v2.0.0.go +++ b/internal/migrations/v2.0.0.go @@ -43,5 +43,17 @@ func V2_0_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf) error { return err } + // S3 URL i snow a settings field. Prepare S3 URL based on region and bucket. + if _, err := db.Exec(` + WITH region AS ( + SELECT value#>>'{}' AS value FROM settings WHERE key='upload.s3.aws_default_region' + ), s3url AS ( + SELECT FORMAT('https://s3.%s.amazonaws.com', (SELECT value FROM region)) AS value + ) + + INSERT INTO settings (key, value) VALUES ('upload.s3.url', TO_JSON((SELECT * FROM s3url))) ON CONFLICT DO NOTHING;`); err != nil { + return err + } + return nil } diff --git a/schema.sql b/schema.sql index e7ff006f..fc389827 100644 --- a/schema.sql +++ b/schema.sql @@ -190,9 +190,10 @@ INSERT INTO settings (key, value) VALUES ('upload.provider', '"filesystem"'), ('upload.filesystem.upload_path', '"uploads"'), ('upload.filesystem.upload_uri', '"/uploads"'), + ('upload.s3.url', '"https://ap-south-1.s3.amazonaws.com"'), ('upload.s3.aws_access_key_id', '""'), ('upload.s3.aws_secret_access_key', '""'), - ('upload.s3.aws_default_region', '"ap-south-b"'), + ('upload.s3.aws_default_region', '"ap-south-1"'), ('upload.s3.bucket', '""'), ('upload.s3.bucket_domain', '""'), ('upload.s3.bucket_path', '"/"'),