package plg_backend_backblaze import ( "bytes" "crypto/sha1" "encoding/base64" "encoding/json" "fmt" "io" "net/http" "net/url" "os" "path/filepath" "strconv" "strings" "time" . "github.com/mickael-kerjean/filestash/server/common" ) var backblaze_cache AppCache type Backblaze struct { params map[string]string Buckets map[string]string ApiUrl string `json:"apiUrl"` DownloadUrl string `json:"downloadUrl"` AccountId string `json:"accountId"` Token string `json:"authorizationToken"` Status int `json:"status"` } type BackblazeError struct { Code string `json:"code"` Message string `json:"message"` Status int `json:"status"` } func init() { Backend.Register("backblaze", Backblaze{}) backblaze_cache = NewAppCache() } func (this Backblaze) Init(params map[string]string, app *App) (IBackend, error) { this.params = params // By default backblaze required quite a few API calls to just find the data that's under a given bucket // This would result in a slow application hence we are caching everyting that's in the hot path if obj := backblaze_cache.Get(params); obj != nil { return obj.(*Backblaze), nil } // To perform some query, we need to first know things like where we will have to query, get a token, ... res, err := this.request("GET", "https://api.backblazeb2.com/b2api/v2/b2_authorize_account", nil, nil) if err != nil { return nil, err } body, err := io.ReadAll(res.Body) res.Body.Close() if err != nil { return nil, err } if err := json.Unmarshal(body, &this); err != nil { return nil, err } // Extract bucket related information as backblaze use bucketId as an identifer // BucketId is just some internal ref as people expect to see the bucketName res, err = this.request( "POST", this.ApiUrl+"/b2api/v2/b2_list_buckets", strings.NewReader(fmt.Sprintf( `{"accountId":"%s"}`, this.AccountId, )), nil, ) if err != nil { return nil, err } body, err = io.ReadAll(res.Body) res.Body.Close() if err != nil { return nil, err } var buckets struct { Buckets []struct { BucketId string `json:"bucketId"` BucketName string `json:"bucketName"` } `json:"buckets"` } if err = json.Unmarshal(body, &buckets); err != nil { return nil, err } this.Buckets = make(map[string]string, len(buckets.Buckets)) for i := range buckets.Buckets { this.Buckets[buckets.Buckets[i].BucketName] = buckets.Buckets[i].BucketId } delete(params, "password") backblaze_cache.Set(params, &this) return this, nil } func (this Backblaze) LoginForm() Form { return Form{ Elmnts: []FormElement{ FormElement{ Name: "type", Type: "hidden", Value: "backblaze", }, FormElement{ Name: "username", Type: "text", Placeholder: "KeyID", }, FormElement{ Name: "password", Type: "password", Placeholder: "applicationKey", }, }, } } func (this Backblaze) Ls(path string) ([]os.FileInfo, error) { if path == "/" { files := make([]os.FileInfo, 0, len(this.Buckets)) for key := range this.Buckets { files = append(files, File{ FName: key, FType: "directory", }) } return files, nil } // prepare the query p := this.path(path) reqJSON, _ := json.Marshal(struct { BucketId string `json:"bucketId"` Delimiter string `json:"delimiter"` MaxFileCount int `json:"maxFileCount"` Prefix string `json:"prefix"` }{p.BucketId, "/", 10000, p.Prefix}) res, err := this.request( "POST", this.ApiUrl+"/b2api/v2/b2_list_file_names", bytes.NewReader(reqJSON), nil, ) if err != nil { return nil, err } body, err := io.ReadAll(res.Body) res.Body.Close() if err != nil { return nil, err } var resBody struct { Files []struct { FType string `json:"action"` Size int64 `json:"contentLength"` Name string `json:"fileName"` Time int64 `json:"uploadTimestamp"` } `json:"files"` } if err = json.Unmarshal(body, &resBody); err != nil { return nil, err } files := make([]os.FileInfo, len(resBody.Files)) for i := range resBody.Files { files[i] = File{ FName: strings.TrimSuffix(strings.TrimPrefix(resBody.Files[i].Name, p.Prefix), "/"), FType: func() string { if resBody.Files[i].FType == "folder" { return "directory" } return "file" }(), FSize: resBody.Files[i].Size, FTime: resBody.Files[i].Time / 1000, } } return files, nil } func (this Backblaze) Cat(path string) (io.ReadCloser, error) { res, err := this.request( "GET", this.DownloadUrl+"/file"+path+"?Authorization="+this.Token, nil, nil, ) if err != nil { return nil, err } return res.Body, nil } func (this Backblaze) Mkdir(path string) error { p := this.path(path) if p.BucketId == "" { bucketName := "" if bp := strings.Split(path, "/"); len(bp) > 1 { bucketName = bp[1] } if bucketName == "" { return ErrNotValid } res, err := this.request( "POST", this.ApiUrl+"/b2api/v2/b2_create_bucket", strings.NewReader(fmt.Sprintf( `{"accountId": "%s", "bucketName": "%s", "bucketType": "allPrivate"}`, this.AccountId, bucketName, )), nil, ) if err != nil { return err } body, err := io.ReadAll(res.Body) res.Body.Close() if err != nil { return err } var resError BackblazeError if err := json.Unmarshal(body, &resError); err != nil { return err } if resError.Message != "" { return NewError(resError.Message, resError.Status) } return nil } return this.Touch(path + ".bzEmpty") } func (this Backblaze) Rm(path string) error { p := this.path(path) if p.BucketId == "" { return ErrNotValid } if p.Prefix == "" { backblaze_cache.Del(this.params) // cache invalidation res, err := this.request( "POST", this.ApiUrl+"/b2api/v2/b2_delete_bucket", strings.NewReader(fmt.Sprintf( `{"accountId": "%s", "bucketId": "%s"}`, this.AccountId, p.BucketId, )), nil, ) if err != nil { return err } body, err := io.ReadAll(res.Body) res.Body.Close() if err != nil { return err } var resError BackblazeError if err := json.Unmarshal(body, &resError); err != nil { return err } if resError.Message != "" { return NewError(resError.Message, resError.Status) } return nil } // Backblaze doesn't provide a recursive API to delete => requires multiple steps // Step 1: find every files in a folder: b2_list_file_names res, err := this.request( "POST", this.ApiUrl+"/b2api/v2/b2_list_file_names", strings.NewReader(fmt.Sprintf( `{"bucketId": "%s", "maxFileCount": 10000, "delimiter": "/", "prefix": "%s"}`, p.BucketId, p.Prefix, )), nil, ) if err != nil { return err } body, err := io.ReadAll(res.Body) res.Body.Close() if err != nil { return err } bRes := struct { Files []struct { FileId string `json:"fileId"` FileName string `json:"fileName"` } `json:"files"` }{} if err = json.Unmarshal(body, &bRes); err != nil { return err } // Step 2: delete files 1 by 1: b2_delete_file_version for i := range bRes.Files { res, err := this.request( "POST", this.ApiUrl+"/b2api/v2/b2_delete_file_version", strings.NewReader(fmt.Sprintf( `{"fileName": "%s", "fileId": "%s"}`, bRes.Files[i].FileName, bRes.Files[i].FileId, )), nil, ) if err != nil { return err } if body, err = io.ReadAll(res.Body); err != nil { return err } res.Body.Close() var resError BackblazeError if err := json.Unmarshal(body, &resError); err != nil { return err } if resError.Message != "" { return NewError(resError.Message, resError.Status) } } return nil } func (this Backblaze) Mv(from string, to string) error { return ErrNotSupported } func (this Backblaze) Touch(path string) error { p := this.path(path) // Step 1: get the URL we will proceed to the upload res, err := this.request( "POST", this.ApiUrl+"/b2api/v2/b2_get_upload_url", strings.NewReader(fmt.Sprintf(`{"bucketId": "%s"}`, p.BucketId)), nil, ) if err != nil { return err } body, err := io.ReadAll(res.Body) res.Body.Close() if err != nil { return err } var resBody struct { UploadUrl string `json:"uploadUrl"` Token string `json:"authorizationToken"` } if err := json.Unmarshal(body, &resBody); err != nil { return err } // Step 2: perform the upload of the empty file res, err = this.request( "POST", resBody.UploadUrl, nil, func(r *http.Request) { r.Header.Set("Authorization", resBody.Token) r.Header.Set("X-Bz-File-Name", url.QueryEscape(p.Prefix)) r.Header.Set("Content-Type", "application/octet-stream") r.Header.Set("Content-Length", "0") r.Header.Set("X-Bz-Content-Sha1", "da39a3ee5e6b4b0d3255bfef95601890afd80709") }, ) if err != nil { return err } body, err = io.ReadAll(res.Body) res.Body.Close() if err != nil { return err } var resError BackblazeError if err := json.Unmarshal(body, &resError); err != nil { return err } if resError.Message != "" { return NewError(resError.Message, resError.Status) } return nil } func (this Backblaze) Save(path string, file io.Reader) error { p := this.path(path) // Step 1: get the URL we will proceed to the upload res, err := this.request( "POST", this.ApiUrl+"/b2api/v2/b2_get_upload_url", strings.NewReader(fmt.Sprintf(`{"bucketId": "%s"}`, p.BucketId)), nil, ) if err != nil { return err } body, err := io.ReadAll(res.Body) res.Body.Close() if err != nil { return err } var resBody struct { UploadUrl string `json:"uploadUrl"` Token string `json:"authorizationToken"` } if err := json.Unmarshal(body, &resBody); err != nil { return err } // Step 2: get details backblaze requires to perform the upload backblazeFileDetail := struct { path string ContentLength int64 Sha1 []byte }{} backblazeFileDetail.path = filepath.Join( GetAbsolutePath(TMP_PATH), "data_"+QuickString(20)+".dat", ) f, err := os.OpenFile(backblazeFileDetail.path, os.O_CREATE|os.O_RDWR, os.ModePerm) if err != nil { return err } defer f.Close() defer os.Remove(backblazeFileDetail.path) io.Copy(f, file) if obj, ok := file.(io.Closer); ok { obj.Close() } s, err := f.Stat() if err != nil { return err } backblazeFileDetail.ContentLength = s.Size() f.Seek(0, io.SeekStart) h := sha1.New() if _, err := io.Copy(h, f); err != nil { return err } backblazeFileDetail.Sha1 = h.Sum(nil) // Step 3: perform the upload f.Seek(0, io.SeekStart) res, err = this.request( "POST", resBody.UploadUrl, f, func(r *http.Request) { r.ContentLength = backblazeFileDetail.ContentLength r.Header.Set("Authorization", resBody.Token) r.Header.Set("X-Bz-File-Name", url.QueryEscape(p.Prefix)) r.Header.Set("Content-Type", "application/octet-stream") r.Header.Set("X-Bz-Content-Sha1", fmt.Sprintf("%x", backblazeFileDetail.Sha1)) }, ) if err != nil { return err } body, err = io.ReadAll(res.Body) res.Body.Close() if err != nil { return err } var resError BackblazeError if err := json.Unmarshal(body, &resError); err != nil { return err } if resError.Message != "" { return NewError(resError.Message, resError.Status) } return nil } func (this Backblaze) Meta(path string) Metadata { m := Metadata{ CanRename: NewBool(false), CanMove: NewBool(false), } if path == "/" { m.CanCreateFile = NewBool(false) m.CanUpload = NewBool(false) } return m } func (this Backblaze) request(method string, url string, body io.Reader, fn func(req *http.Request)) (*http.Response, error) { req, err := http.NewRequest(method, url, body) if err != nil { return nil, err } if this.Token == "" { req.Header.Set( "Authorization", fmt.Sprintf( "Basic %s", base64.StdEncoding.EncodeToString( []byte(fmt.Sprintf("%s:%s", this.params["username"], this.params["password"])), ), ), ) } else { req.Header.Set("Authorization", this.Token) } req.Header.Set("User-Agent", "Filestash "+APP_VERSION+"."+BUILD_DATE) req.Header.Set("Accept", "application/json") //req.Header.Set("X-Bz-Test-Mode", "force_cap_exceeded") if fn != nil { fn(req) } if req.Body != nil { defer req.Body.Close() } res, err := HTTPClient.Do(req) if err != nil { return res, err } if res.StatusCode == 401 { res.Body.Close() return res, ErrAuthenticationFailed } else if res.StatusCode == 403 { res.Body.Close() return res, ErrNotAllowed } else if res.StatusCode == 429 { res.Body.Close() retryAfter, err := strconv.Atoi(res.Header.Get("Retry-After")) if err == nil && retryAfter < 10 && retryAfter >= 0 { time.Sleep(time.Duration(retryAfter) * time.Second) return this.request(method, url, body, fn) } return res, ErrCongestion } else if res.StatusCode == 503 { res.Body.Close() retryAfter, err := strconv.Atoi(res.Header.Get("Retry-After")) if err == nil && retryAfter < 10 && retryAfter >= 0 { time.Sleep(time.Duration(retryAfter) * time.Second) return this.request(method, url, body, fn) } return res, ErrCongestion } return res, nil } type BackblazePath struct { BucketId string Prefix string } func (this Backblaze) path(path string) BackblazePath { bp := strings.Split(path, "/") bucket := "" if len(bp) > 1 { bucket = bp[1] } prefix := "" if len(bp) > 2 { prefix = strings.Join(bp[2:], "/") } return BackblazePath{ this.Buckets[bucket], prefix, } }