package plg_backend_s3 import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" . "github.com/mickael-kerjean/filestash/server/common" "io" "os" "path/filepath" "strings" "fmt" ) var S3Cache AppCache type S3Backend struct { client *s3.S3 config *aws.Config params map[string]string } func init() { Backend.Register("s3", S3Backend{}) S3Cache = NewAppCache(2, 1) } func (s S3Backend) Init(params map[string]string, app *App) (IBackend, error) { if params["encryption_key"] != "" && len(params["encryption_key"]) != 32 { return nil, NewError(fmt.Sprintf("Encryption key needs to be 32 characters (current: %d)", len(params["encryption_key"])), 400) } if params["region"] == "" { params["region"] = "us-east-2" } config := &aws.Config{ Credentials: credentials.NewStaticCredentials(params["access_key_id"], params["secret_access_key"], ""), S3ForcePathStyle: aws.Bool(true), Region: aws.String(params["region"]), } if params["endpoint"] != "" { config.Endpoint = aws.String(params["endpoint"]) } backend := &S3Backend{ config: config, params: params, client: s3.New(session.New(config)), } return backend, nil } func (s S3Backend) LoginForm() Form { return Form{ Elmnts: []FormElement{ FormElement{ Name: "type", Type: "hidden", Value: "s3", }, FormElement{ Name: "access_key_id", Type: "text", Placeholder: "Access Key ID*", }, FormElement{ Name: "secret_access_key", Type: "text", Placeholder: "Secret Access Key*", }, FormElement{ Name: "advanced", Type: "enable", Placeholder: "Advanced", Target: []string{"s3_path", "s3_encryption_key", "s3_region", "s3_endpoint"}, }, FormElement{ Id: "s3_path", Name: "path", Type: "text", Placeholder: "Path", }, FormElement{ Id: "s3_encryption_key", Name: "encryption_key", Type: "text", Placeholder: "Encryption Key", }, FormElement{ Id: "s3_region", Name: "region", Type: "text", Placeholder: "Region", }, FormElement{ Id: "s3_endpoint", Name: "endpoint", Type: "text", Placeholder: "Endpoint", }, }, } } func (s S3Backend) Meta(path string) Metadata { if path == "/" { return Metadata{ CanCreateFile: NewBool(false), CanRename: NewBool(false), CanMove: NewBool(false), CanUpload: NewBool(false), } } return Metadata{} } func (s S3Backend) Ls(path string) ([]os.FileInfo, error) { p := s.path(path) files := make([]os.FileInfo, 0) if p.bucket == "" { b, err := s.client.ListBuckets(&s3.ListBucketsInput{}) if err != nil { return nil, err } for _, bucket := range b.Buckets { files = append(files, &File{ FName: *bucket.Name, FType: "directory", FTime: bucket.CreationDate.Unix(), CanMove: NewBool(false), }) } return files, nil } client := s3.New(s.createSession(p.bucket)) objs, err := client.ListObjects(&s3.ListObjectsInput{ Bucket: aws.String(p.bucket), Prefix: aws.String(p.path), Delimiter: aws.String("/"), }) if err != nil { return nil, err } for i, object := range objs.Contents { if i == 0 && *object.Key == p.path { continue } files = append(files, &File{ FName: filepath.Base(*object.Key), FType: "file", FTime: object.LastModified.Unix(), FSize: *object.Size, }) } for _, object := range objs.CommonPrefixes { files = append(files, &File{ FName: filepath.Base(*object.Prefix), FType: "directory", }) } return files, nil } func (s S3Backend) Cat(path string) (io.ReadCloser, error) { p := s.path(path) client := s3.New(s.createSession(p.bucket)) input := &s3.GetObjectInput{ Bucket: aws.String(p.bucket), Key: aws.String(p.path), } if s.params["encryption_key"] != "" { input.SSECustomerAlgorithm = aws.String("AES256") input.SSECustomerKey = aws.String(s.params["encryption_key"]) } obj, err := client.GetObject(input) if err != nil { awsErr, ok := err.(awserr.Error); if ok == false { return nil, err } if awsErr.Code() == "InvalidRequest" && strings.Contains(awsErr.Message(), "encryption") { input.SSECustomerAlgorithm = nil input.SSECustomerKey = nil obj, err = client.GetObject(input) return obj.Body, err } else if awsErr.Code() == "InvalidArgument" && strings.Contains(awsErr.Message(), "secret key was invalid") { return nil, NewError("This file is encrypted file, you need the correct key!", 400) } else if awsErr.Code() == "AccessDenied" { return nil, ErrNotAllowed } return nil ,err } return obj.Body, nil } func (s S3Backend) Mkdir(path string) error { p := s.path(path) client := s3.New(s.createSession(p.bucket)) if p.path == "" { _, err := client.CreateBucket(&s3.CreateBucketInput{ Bucket: aws.String(path), }) return err } _, err := client.PutObject(&s3.PutObjectInput{ Bucket: aws.String(p.bucket), Key: aws.String(p.path), }) return err } func (s S3Backend) Rm(path string) error { p := s.path(path) client := s3.New(s.createSession(p.bucket)) if p.bucket == "" { return ErrNotFound } objs, err := client.ListObjects(&s3.ListObjectsInput{ Bucket: aws.String(p.bucket), Prefix: aws.String(p.path), Delimiter: aws.String("/"), }) if err != nil { return err } for _, obj := range objs.Contents { // TODO: bug _, err := client.DeleteObject(&s3.DeleteObjectInput{ Bucket: aws.String(p.bucket), Key: obj.Key, }) if err != nil { return err } } for _, pref := range objs.CommonPrefixes { // TODO: bug s.Rm("/" + p.bucket + "/" + *pref.Prefix) _, err := client.DeleteObject(&s3.DeleteObjectInput{ Bucket: aws.String(p.bucket), Key: pref.Prefix, }) if err != nil { return err } } if err != nil { return err } if p.path == "" { _, err := client.DeleteBucket(&s3.DeleteBucketInput{ Bucket: aws.String(p.bucket), }) return err } _, err = client.DeleteObject(&s3.DeleteObjectInput{ Bucket: aws.String(p.bucket), Key: aws.String(p.path), }) return err } func (s S3Backend) Mv(from string, to string) error { f := s.path(from) t := s.path(to) client := s3.New(s.createSession(f.bucket)) if f.path == "" { return ErrNotImplemented } input := &s3.CopyObjectInput{ Bucket: aws.String(t.bucket), CopySource: aws.String(f.bucket + "/" + f.path), Key: aws.String(t.path), } if s.params["encryption_key"] != "" { input.CopySourceSSECustomerAlgorithm = aws.String("AES256") input.CopySourceSSECustomerKey = aws.String(s.params["encryption_key"]) input.SSECustomerAlgorithm = aws.String("AES256") input.SSECustomerKey = aws.String(s.params["encryption_key"]) } _, err := client.CopyObject(input) if err != nil { return err } return s.Rm(from) } func (s S3Backend) Touch(path string) error { p := s.path(path) client := s3.New(s.createSession(p.bucket)) if p.bucket == "" { return ErrNotValid } input := &s3.PutObjectInput{ Body: strings.NewReader(""), ContentLength: aws.Int64(0), Bucket: aws.String(p.bucket), Key: aws.String(p.path), } if s.params["encryption_key"] != "" { input.SSECustomerAlgorithm = aws.String("AES256") input.SSECustomerKey = aws.String(s.params["encryption_key"]) } _, err := client.PutObject(input) return err } func (s S3Backend) Save(path string, file io.Reader) error { p := s.path(path) if p.bucket == "" { return ErrNotValid } uploader := s3manager.NewUploader(s.createSession(path)) input := s3manager.UploadInput{ Body: file, Bucket: aws.String(p.bucket), Key: aws.String(p.path), } if s.params["encryption_key"] != "" { input.SSECustomerAlgorithm = aws.String("AES256") input.SSECustomerKey = aws.String(s.params["encryption_key"]) } _, err := uploader.Upload(&input) return err } func (s S3Backend) createSession(bucket string) *session.Session { params := s.params params["bucket"] = bucket c := S3Cache.Get(params) if c == nil { res, err := s.client.GetBucketLocation(&s3.GetBucketLocationInput{ Bucket: aws.String(bucket), }) if err != nil { s.config.Region = aws.String("us-east-1") } else { if res.LocationConstraint == nil { s.config.Region = aws.String("us-east-1") } else { s.config.Region = res.LocationConstraint } } S3Cache.Set(params, s.config.Region) } else { s.config.Region = c.(*string) } sess := session.New(s.config) return sess } type S3Path struct { bucket string path string } func (s S3Backend) path(p string) S3Path { sp := strings.Split(p, "/") bucket := "" if len(sp) > 1 { bucket = sp[1] } path := "" if len(sp) > 2 { path = strings.Join(sp[2:], "/") } return S3Path{ bucket, path, } }