mirror of
				https://github.com/cloudreve/cloudreve.git
				synced 2025-11-01 00:57:15 +08:00 
			
		
		
		
	Modify: mark as success when deleting a file that does not exist;
Fix: minio is not usable in S3 policy Modify: use batch request to delete S3 files
This commit is contained in:
		
							
								
								
									
										2
									
								
								assets
									
									
									
									
									
								
							
							
								
								
								
								
								
							
						
						
									
										2
									
								
								assets
									
									
									
									
									
								
							 Submodule assets updated: 92f6981cb3...d626a4d9cc
									
								
							| @ -130,12 +130,15 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err | |||||||
| 	var retErr error | 	var retErr error | ||||||
|  |  | ||||||
| 	for _, value := range files { | 	for _, value := range files { | ||||||
| 		err := os.Remove(util.RelativePath(filepath.FromSlash(value))) | 		filePath := util.RelativePath(filepath.FromSlash(value)) | ||||||
|  | 		if util.Exists(filePath) { | ||||||
|  | 			err := os.Remove(filePath) | ||||||
| 			if err != nil { | 			if err != nil { | ||||||
| 				util.Log().Warning("无法删除文件,%s", err) | 				util.Log().Warning("无法删除文件,%s", err) | ||||||
| 				retErr = err | 				retErr = err | ||||||
| 				deleteFailed = append(deleteFailed, value) | 				deleteFailed = append(deleteFailed, value) | ||||||
| 			} | 			} | ||||||
|  | 		} | ||||||
|  |  | ||||||
| 		// 尝试删除文件的缩略图(如果有) | 		// 尝试删除文件的缩略图(如果有) | ||||||
| 		_ = os.Remove(util.RelativePath(value + conf.ThumbConfig.FileSuffix)) | 		_ = os.Remove(util.RelativePath(value + conf.ThumbConfig.FileSuffix)) | ||||||
|  | |||||||
| @ -370,7 +370,7 @@ func (client *Client) Delete(ctx context.Context, dst []string) ([]string, error | |||||||
| func getDeleteFailed(res *BatchResponses) []string { | func getDeleteFailed(res *BatchResponses) []string { | ||||||
| 	var failed = make([]string, 0, len(res.Responses)) | 	var failed = make([]string, 0, len(res.Responses)) | ||||||
| 	for _, v := range res.Responses { | 	for _, v := range res.Responses { | ||||||
| 		if v.Status != 204 { | 		if v.Status != 204 && v.Status != 404 { | ||||||
| 			failed = append(failed, v.ID) | 			failed = append(failed, v.ID) | ||||||
| 		} | 		} | ||||||
| 	} | 	} | ||||||
|  | |||||||
| @ -206,7 +206,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err | |||||||
| 	if err != nil { | 	if err != nil { | ||||||
| 		failed := make([]string, 0, len(rets)) | 		failed := make([]string, 0, len(rets)) | ||||||
| 		for k, ret := range rets { | 		for k, ret := range rets { | ||||||
| 			if ret.Code != 200 { | 			if ret.Code != 200 && ret.Code != 612 { | ||||||
| 				failed = append(failed, files[k]) | 				failed = append(failed, files[k]) | ||||||
| 			} | 			} | ||||||
| 		} | 		} | ||||||
|  | |||||||
| @ -8,13 +8,13 @@ import ( | |||||||
| 	"encoding/hex" | 	"encoding/hex" | ||||||
| 	"encoding/json" | 	"encoding/json" | ||||||
| 	"errors" | 	"errors" | ||||||
|  | 	"github.com/cloudreve/Cloudreve/v3/pkg/util" | ||||||
| 	"io" | 	"io" | ||||||
| 	"net/http" | 	"net/http" | ||||||
| 	"net/url" | 	"net/url" | ||||||
| 	"path" | 	"path" | ||||||
| 	"path/filepath" | 	"path/filepath" | ||||||
| 	"strings" | 	"strings" | ||||||
| 	"sync" |  | ||||||
| 	"time" | 	"time" | ||||||
|  |  | ||||||
| 	"github.com/aws/aws-sdk-go/aws" | 	"github.com/aws/aws-sdk-go/aws" | ||||||
| @ -60,7 +60,7 @@ func (handler *Driver) InitS3Client() error { | |||||||
| 			Credentials:      credentials.NewStaticCredentials(handler.Policy.AccessKey, handler.Policy.SecretKey, ""), | 			Credentials:      credentials.NewStaticCredentials(handler.Policy.AccessKey, handler.Policy.SecretKey, ""), | ||||||
| 			Endpoint:         &handler.Policy.Server, | 			Endpoint:         &handler.Policy.Server, | ||||||
| 			Region:           &handler.Policy.OptionsSerialized.Region, | 			Region:           &handler.Policy.OptionsSerialized.Region, | ||||||
| 			S3ForcePathStyle: aws.Bool(false), | 			S3ForcePathStyle: aws.Bool(true), | ||||||
| 		}) | 		}) | ||||||
|  |  | ||||||
| 		if err != nil { | 		if err != nil { | ||||||
| @ -229,53 +229,35 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err | |||||||
| 		return files, err | 		return files, err | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	var ( | 	failed := make([]string, 0, len(files)) | ||||||
| 		failed       = make([]string, 0, len(files)) | 	deleted := make([]string, 0, len(files)) | ||||||
| 		lastErr      error |  | ||||||
| 		currentIndex = 0 |  | ||||||
| 		indexLock    sync.Mutex |  | ||||||
| 		failedLock   sync.Mutex |  | ||||||
| 		wg           sync.WaitGroup |  | ||||||
| 		routineNum   = 4 |  | ||||||
| 	) |  | ||||||
| 	wg.Add(routineNum) |  | ||||||
|  |  | ||||||
| 	// S3不支持批量操作,这里开四个协程并行操作 | 	keys := make([]*s3.ObjectIdentifier, 0, len(files)) | ||||||
| 	for i := 0; i < routineNum; i++ { | 	for _, file := range files { | ||||||
| 		go func() { | 		filePath := file | ||||||
| 			for { | 		keys = append(keys, &s3.ObjectIdentifier{Key: &filePath}) | ||||||
| 				// 取得待删除文件 |  | ||||||
| 				indexLock.Lock() |  | ||||||
| 				if currentIndex >= len(files) { |  | ||||||
| 					// 所有文件处理完成 |  | ||||||
| 					wg.Done() |  | ||||||
| 					indexLock.Unlock() |  | ||||||
| 					return |  | ||||||
| 	} | 	} | ||||||
| 				path := files[currentIndex] |  | ||||||
| 				currentIndex++ |  | ||||||
| 				indexLock.Unlock() |  | ||||||
|  |  | ||||||
| 	// 发送异步删除请求 | 	// 发送异步删除请求 | ||||||
| 				_, err := handler.svc.DeleteObject( | 	res, err := handler.svc.DeleteObjects( | ||||||
| 					&s3.DeleteObjectInput{ | 		&s3.DeleteObjectsInput{ | ||||||
| 			Bucket: &handler.Policy.BucketName, | 			Bucket: &handler.Policy.BucketName, | ||||||
| 						Key:    &path, | 			Delete: &s3.Delete{ | ||||||
|  | 				Objects: keys, | ||||||
|  | 			}, | ||||||
| 		}) | 		}) | ||||||
|  |  | ||||||
| 				// 处理错误 |  | ||||||
| 	if err != nil { | 	if err != nil { | ||||||
| 					failedLock.Lock() | 		return files, err | ||||||
| 					lastErr = err |  | ||||||
| 					failed = append(failed, path) |  | ||||||
| 					failedLock.Unlock() |  | ||||||
| 				} |  | ||||||
| 			} |  | ||||||
| 		}() |  | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	wg.Wait() | 	// 统计未删除的文件 | ||||||
| 	return failed, lastErr | 	for _, deleteRes := range res.Deleted { | ||||||
|  | 		deleted = append(deleted, *deleteRes.Key) | ||||||
|  | 	} | ||||||
|  | 	failed = util.SliceDifference(failed, deleted) | ||||||
|  |  | ||||||
|  | 	return failed, nil | ||||||
|  |  | ||||||
| } | } | ||||||
|  |  | ||||||
|  | |||||||
		Reference in New Issue
	
	Block a user
	 HFO4
					HFO4