mirror of
https://gitcode.com/gitea/gitea.git
synced 2025-06-25 06:21:26 +08:00
Add a storage layer for attachments (#11387)
* Add a storage layer for attachments * Fix some bug * fix test * Fix copyright head and lint * Fix bug * Add setting for minio and flags for migrate-storage * Add documents * fix lint * Add test for minio store type on attachments * fix test * fix test * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Add warning when storage migrated successfully * Fix drone * fix test * rebase * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * remove log on xorm * Fi download bug * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * Add URL function to serve attachments directly from S3/Minio * Add ability to enable/disable redirection in attachment configuration * Fix typo * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * don't change unrelated files * Fix lint * Fix build * update go.mod and go.sum * Use github.com/minio/minio-go/v6 * Remove unused function * Upgrade minio to v7 and some other improvements * fix lint * Fix go mod Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Tyler <tystuyfzand@gmail.com>
This commit is contained in:
.drone.yml
cmd
custom/conf
docs/content/doc/advanced
go.modgo.sumintegrations
main.gomodels
modules
routers
services/release
vendor
github.com
json-iterator/go
.codecov.yml.gitignore.travis.ymlGopkg.lockGopkg.tomlLICENSEREADME.mdadapter.goany.goany_array.goany_bool.goany_float.goany_int32.goany_int64.goany_invalid.goany_nil.goany_number.goany_object.goany_str.goany_uint32.goany_uint64.gobuild.shconfig.gofuzzy_mode_convert_table.mdgo.modgo.sumiter.goiter_array.goiter_float.goiter_int.goiter_object.goiter_skip.goiter_skip_sloppy.goiter_skip_strict.goiter_str.gojsoniter.gopool.goreflect.goreflect_array.goreflect_dynamic.goreflect_extension.goreflect_json_number.goreflect_json_raw_message.goreflect_map.goreflect_marshaler.goreflect_native.goreflect_optional.goreflect_slice.goreflect_struct_decoder.goreflect_struct_encoder.gostream.gostream_float.gostream_int.gostream_str.gotest.sh
klauspost/cpuid
.gitignore.travis.ymlCONTRIBUTING.txtLICENSEREADME.mdcpuid.gocpuid_386.scpuid_amd64.scpuid_arm64.sdetect_arm64.godetect_intel.godetect_ref.gogo.mod
minio
md5-simd
LICENSEREADME.mdblock-generic.goblock16_amd64.sblock8_amd64.sblock_amd64.gogo.modgo.summd5-digest_amd64.gomd5-server_amd64.gomd5-server_fallback.gomd5-util_amd64.gomd5.go
minio-go/v7
.gitignore.golangci.ymlCNAMECONTRIBUTING.mdLICENSEMAINTAINERS.mdMakefileNOTICEREADME.mdREADME_zh_CN.mdapi-bucket-encryption.goapi-bucket-lifecycle.goapi-bucket-notification.goapi-bucket-policy.goapi-bucket-replication.goapi-bucket-tagging.goapi-bucket-versioning.goapi-compose-object.goapi-datatypes.goapi-error-response.goapi-get-object-acl.goapi-get-object-file.goapi-get-object.goapi-get-options.goapi-list.goapi-object-legal-hold.goapi-object-lock.goapi-object-retention.goapi-object-tagging.goapi-presigned.goapi-put-bucket.goapi-put-object-common.goapi-put-object-copy.goapi-put-object-file-context.goapi-put-object-multipart.goapi-put-object-streaming.goapi-put-object.goapi-remove.goapi-s3-datatypes.goapi-select.goapi-stat.goapi.gobucket-cache.gocode_of_conduct.mdconstants.gocore.gogo.modgo.sumhook-reader.gopost-policy.goretry-continous.goretry.gos3-endpoints.gos3-error.gostaticcheck.conftransport.goutils.go
pkg
credentials
assume_role.gochain.goconfig.json.samplecredentials.gocredentials.sampledoc.goenv_aws.goenv_minio.gofile_aws_credentials.gofile_minio_client.goiam_aws.gosignature-type.gostatic.gosts_client_grants.gosts_ldap_identity.gosts_web_identity.go
encrypt
lifecycle
notification
replication
s3utils
set
signer
sse
tags
sha256-simd
.gitignore.travis.ymlLICENSEREADME.mdappveyor.ymlcpuid.gocpuid_386.gocpuid_386.scpuid_amd64.gocpuid_amd64.scpuid_arm.gocpuid_linux_arm64.gocpuid_other.gogo.modsha256.gosha256blockAvx2_amd64.gosha256blockAvx2_amd64.ssha256blockAvx512_amd64.asmsha256blockAvx512_amd64.gosha256blockAvx512_amd64.ssha256blockAvx_amd64.gosha256blockAvx_amd64.ssha256blockSha_amd64.gosha256blockSha_amd64.ssha256blockSsse_amd64.gosha256blockSsse_amd64.ssha256block_amd64.gosha256block_arm64.gosha256block_arm64.ssha256block_other.gotest-architectures.sh
modern-go
concurrent
.gitignore.travis.ymlLICENSEREADME.mdexecutor.gogo_above_19.gogo_below_19.golog.gotest.shunbounded_executor.go
reflect2
.gitignore.travis.ymlGopkg.lockGopkg.tomlLICENSEREADME.mdgo_above_17.gogo_above_19.gogo_below_17.gogo_below_19.goreflect2.goreflect2_amd64.sreflect2_kind.gorelfect2_386.srelfect2_amd64p32.srelfect2_arm.srelfect2_arm64.srelfect2_mips64x.srelfect2_mipsx.srelfect2_ppc64x.srelfect2_s390x.ssafe_field.gosafe_map.gosafe_slice.gosafe_struct.gosafe_type.gotest.shtype_map.gounsafe_array.gounsafe_eface.gounsafe_field.gounsafe_iface.gounsafe_link.gounsafe_map.gounsafe_ptr.gounsafe_slice.gounsafe_struct.gounsafe_type.go
rs/xid
golang.org/x
net
sys/unix
mkerrors.shsyscall_linux.gozerrors_linux_386.gozerrors_linux_amd64.gozerrors_linux_arm.gozerrors_linux_arm64.gozerrors_linux_mips.gozerrors_linux_mips64.gozerrors_linux_mips64le.gozerrors_linux_mipsle.gozerrors_linux_ppc64.gozerrors_linux_ppc64le.gozerrors_linux_riscv64.gozerrors_linux_s390x.gozerrors_linux_sparc64.goztypes_freebsd_arm.go
text
encoding/unicode
transform
unicode
width
gopkg.in/ini.v1
.travis.ymlMakefileREADME.mdcodecov.ymldata_source.gofile.goini.gokey.goparser.gosection.gostruct.go
modules.txt
119
vendor/github.com/modern-go/concurrent/unbounded_executor.go
generated
vendored
Normal file
119
vendor/github.com/modern-go/concurrent/unbounded_executor.go
generated
vendored
Normal file
@ -0,0 +1,119 @@
|
||||
package concurrent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
"time"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
// HandlePanic logs goroutine panic by default
|
||||
var HandlePanic = func(recovered interface{}, funcName string) {
|
||||
ErrorLogger.Println(fmt.Sprintf("%s panic: %v", funcName, recovered))
|
||||
ErrorLogger.Println(string(debug.Stack()))
|
||||
}
|
||||
|
||||
// UnboundedExecutor is a executor without limits on counts of alive goroutines
|
||||
// it tracks the goroutine started by it, and can cancel them when shutdown
|
||||
type UnboundedExecutor struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
activeGoroutinesMutex *sync.Mutex
|
||||
activeGoroutines map[string]int
|
||||
HandlePanic func(recovered interface{}, funcName string)
|
||||
}
|
||||
|
||||
// GlobalUnboundedExecutor has the life cycle of the program itself
|
||||
// any goroutine want to be shutdown before main exit can be started from this executor
|
||||
// GlobalUnboundedExecutor expects the main function to call stop
|
||||
// it does not magically knows the main function exits
|
||||
var GlobalUnboundedExecutor = NewUnboundedExecutor()
|
||||
|
||||
// NewUnboundedExecutor creates a new UnboundedExecutor,
|
||||
// UnboundedExecutor can not be created by &UnboundedExecutor{}
|
||||
// HandlePanic can be set with a callback to override global HandlePanic
|
||||
func NewUnboundedExecutor() *UnboundedExecutor {
|
||||
ctx, cancel := context.WithCancel(context.TODO())
|
||||
return &UnboundedExecutor{
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
activeGoroutinesMutex: &sync.Mutex{},
|
||||
activeGoroutines: map[string]int{},
|
||||
}
|
||||
}
|
||||
|
||||
// Go starts a new goroutine and tracks its lifecycle.
|
||||
// Panic will be recovered and logged automatically, except for StopSignal
|
||||
func (executor *UnboundedExecutor) Go(handler func(ctx context.Context)) {
|
||||
pc := reflect.ValueOf(handler).Pointer()
|
||||
f := runtime.FuncForPC(pc)
|
||||
funcName := f.Name()
|
||||
file, line := f.FileLine(pc)
|
||||
executor.activeGoroutinesMutex.Lock()
|
||||
defer executor.activeGoroutinesMutex.Unlock()
|
||||
startFrom := fmt.Sprintf("%s:%d", file, line)
|
||||
executor.activeGoroutines[startFrom] += 1
|
||||
go func() {
|
||||
defer func() {
|
||||
recovered := recover()
|
||||
// if you want to quit a goroutine without trigger HandlePanic
|
||||
// use runtime.Goexit() to quit
|
||||
if recovered != nil {
|
||||
if executor.HandlePanic == nil {
|
||||
HandlePanic(recovered, funcName)
|
||||
} else {
|
||||
executor.HandlePanic(recovered, funcName)
|
||||
}
|
||||
}
|
||||
executor.activeGoroutinesMutex.Lock()
|
||||
executor.activeGoroutines[startFrom] -= 1
|
||||
executor.activeGoroutinesMutex.Unlock()
|
||||
}()
|
||||
handler(executor.ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
// Stop cancel all goroutines started by this executor without wait
|
||||
func (executor *UnboundedExecutor) Stop() {
|
||||
executor.cancel()
|
||||
}
|
||||
|
||||
// StopAndWaitForever cancel all goroutines started by this executor and
|
||||
// wait until all goroutines exited
|
||||
func (executor *UnboundedExecutor) StopAndWaitForever() {
|
||||
executor.StopAndWait(context.Background())
|
||||
}
|
||||
|
||||
// StopAndWait cancel all goroutines started by this executor and wait.
|
||||
// Wait can be cancelled by the context passed in.
|
||||
func (executor *UnboundedExecutor) StopAndWait(ctx context.Context) {
|
||||
executor.cancel()
|
||||
for {
|
||||
oneHundredMilliseconds := time.NewTimer(time.Millisecond * 100)
|
||||
select {
|
||||
case <-oneHundredMilliseconds.C:
|
||||
if executor.checkNoActiveGoroutines() {
|
||||
return
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (executor *UnboundedExecutor) checkNoActiveGoroutines() bool {
|
||||
executor.activeGoroutinesMutex.Lock()
|
||||
defer executor.activeGoroutinesMutex.Unlock()
|
||||
for startFrom, count := range executor.activeGoroutines {
|
||||
if count > 0 {
|
||||
InfoLogger.Println("UnboundedExecutor is still waiting goroutines to quit",
|
||||
"startFrom", startFrom,
|
||||
"count", count)
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
Reference in New Issue
Block a user