opensource/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2025-06-24 14:08:13 +08:00
Code Issues Projects Releases Wiki Activity

Merge pull request #3801 from hoenirvili/refactor-code

Browse Source 
Safe check rlimit syscall and refactoring
This commit is contained in:
Whyrusleeping
2017-12-05 18:34:51 +01:00
committed by GitHub
parent a234c56bec f16ada6159
commit 715bd16e0e
9 changed files with 243 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/ipfs/daemon.go
View File

@ -11,6 +11,7 @@ import (
"sort"
"sync"
utilmain "github.com/ipfs/go-ipfs/cmd/ipfs/util"
"github.com/ipfs/go-ipfs/core"
commands "github.com/ipfs/go-ipfs/core/commands"
corehttp "github.com/ipfs/go-ipfs/core/corehttp"
@ -180,8 +181,6 @@ func defaultMux(path string) corehttp.ServeOption {
}
}
var fileDescriptorCheck = func() error { return nil }
func daemonFunc(req cmds.Request, re cmds.ResponseEmitter) {
// Inject metrics before we do anything
@ -193,9 +192,8 @@ func daemonFunc(req cmds.Request, re cmds.ResponseEmitter) {
// let the user know we're going.
fmt.Printf("Initializing daemon...\n")
managefd, _, _ := req.Option(adjustFDLimitKwd).Bool()
if managefd {
if err := fileDescriptorCheck(); err != nil {
if managed, _, _ := req.Option(adjustFDLimitKwd).Bool(); managed {
if err := utilmain.ManageFdLimit(); err != nil {
log.Errorf("setting file descriptor limit: %s", err)
}
}

19
cmd/ipfs/ulimit.go
View File

@ -1,19 +0,0 @@
package main
import (
"os"
"strconv"
)
var ipfsFileDescNum = uint64(2048)
func init() {
if val := os.Getenv("IPFS_FD_MAX"); val != "" {
n, err := strconv.Atoi(val)
if err != nil {
log.Errorf("bad value for IPFS_FD_MAX: %s", err)
} else {
ipfsFileDescNum = uint64(n)
}
}
}

45
cmd/ipfs/ulimit_freebsd.go
View File

@ -1,45 +0,0 @@
// +build freebsd
package main
import (
"fmt"
unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix"
)
func init() {
fileDescriptorCheck = checkAndSetUlimit
}
func checkAndSetUlimit() error {
var rLimit unix.Rlimit
err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rLimit)
if err != nil {
return fmt.Errorf("error getting rlimit: %s", err)
}
ipfsFileDescNum := int64(ipfsFileDescNum)
var setting bool
if rLimit.Cur < ipfsFileDescNum {
if rLimit.Max < ipfsFileDescNum {
log.Error("adjusting max")
rLimit.Max = ipfsFileDescNum
}
fmt.Printf("Adjusting current ulimit to %d...\n", ipfsFileDescNum)
rLimit.Cur = ipfsFileDescNum
setting = true
}
err = unix.Setrlimit(unix.RLIMIT_NOFILE, &rLimit)
if err != nil {
return fmt.Errorf("error setting ulimit: %s", err)
}
if setting {
fmt.Printf("Successfully raised file descriptor limit to %d.\n", ipfsFileDescNum)
}
return nil
}

43
cmd/ipfs/ulimit_unix.go
View File

@ -1,43 +0,0 @@
// +build darwin linux netbsd openbsd
package main
import (
"fmt"
unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix"
)
func init() {
fileDescriptorCheck = checkAndSetUlimit
}
func checkAndSetUlimit() error {
var rLimit unix.Rlimit
err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rLimit)
if err != nil {
return fmt.Errorf("error getting rlimit: %s", err)
}
var setting bool
if rLimit.Cur < ipfsFileDescNum {
if rLimit.Max < ipfsFileDescNum {
log.Error("adjusting max")
rLimit.Max = ipfsFileDescNum
}
fmt.Printf("Adjusting current ulimit to %d...\n", ipfsFileDescNum)
rLimit.Cur = ipfsFileDescNum
setting = true
}
err = unix.Setrlimit(unix.RLIMIT_NOFILE, &rLimit)
if err != nil {
return fmt.Errorf("error setting ulimit: %s", err)
}
if setting {
fmt.Printf("Successfully raised file descriptor limit to %d.\n", ipfsFileDescNum)
}
return nil
}

91
cmd/ipfs/util/ulimit.go Normal file
View File

@ -0,0 +1,91 @@
package util
import (
"errors"
"fmt"
"os"
"strconv"
"syscall"
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
)
var log = logging.Logger("ulimit")
var (
supportsFDManagement = false
// getlimit returns the soft and hard limits of file descriptors counts
getLimit func() (int64, int64, error)
// set limit sets the soft and hard limits of file descriptors counts
setLimit func(int64, int64) error
)
// maxFds is the maximum number of file descriptors that go-ipfs
// can use. The default value is 1024. This can be overwritten by the
// IPFS_FD_MAX env variable
var maxFds = uint64(2048)
// setMaxFds sets the maxFds value from IPFS_FD_MAX
// env variable if it's present on the system
func setMaxFds() {
// check if the IPFS_FD_MAX is set up and if it does
// not have a valid fds number notify the user
if val := os.Getenv("IPFS_FD_MAX"); val != "" {
fds, err := strconv.ParseUint(val, 10, 64)
if err != nil {
log.Errorf("bad value for IPFS_FD_MAX: %s", err)
return
}
maxFds = fds
}
}
// ManageFdLimit raise the current max file descriptor count
// of the process based on the IPFS_FD_MAX value
func ManageFdLimit() error {
if !supportsFDManagement {
return nil
}
setMaxFds()
soft, hard, err := getLimit()
if err != nil {
return err
}
max := int64(maxFds)
if max <= soft {
return nil
}
// the soft limit is the value that the kernel enforces for the
// corresponding resource
// the hard limit acts as a ceiling for the soft limit
// an unprivileged process may only set it's soft limit to a
// alue in the range from 0 up to the hard limit
if err = setLimit(max, max); err != nil {
if err != syscall.EPERM {
return fmt.Errorf("error setting: ulimit: %s", err)
}
// the process does not have permission so we should only
// set the soft value
if max > hard {
return errors.New(
"cannot set rlimit, IPFS_FD_MAX is larger than the hard limit",
)
}
if err = setLimit(max, hard); err != nil {
return fmt.Errorf("error setting ulimit wihout hard limit: %s", err)
}
}
fmt.Printf("Successfully raised file descriptor limit to %d.\n", max)
return nil
}

27
cmd/ipfs/util/ulimit_freebsd.go Normal file
View File

@ -0,0 +1,27 @@
// +build freebsd
package util
import (
unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix"
)
func init() {
supportsFDManagement = true
getLimit = freebsdGetLimit
setLimit = freebdsSetLimit
}
func freebsdGetLimit() (int64, int64, error) {
rlimit := unix.Rlimit{}
err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit)
return rlimit.Cur, rlimit.Max, err
}
func freebdsSetLimit(soft int64, max int64) error {
rlimit := unix.Rlimit{
Cur: soft,
Max: max,
}
return unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit)
}

88
cmd/ipfs/util/ulimit_test.go Normal file
View File

@ -0,0 +1,88 @@
package util
import (
"fmt"
"os"
"strings"
"syscall"
"testing"
)
func TestManageFdLimit(t *testing.T) {
t.Log("Testing file descriptor count")
if err := ManageFdLimit(); err != nil {
t.Errorf("Cannot manage file descriptors")
}
if maxFds != uint64(2048) {
t.Errorf("Maximum file descriptors default value changed")
}
}
func TestManageInvalidNFds(t *testing.T) {
t.Logf("Testing file descriptor invalidity")
var err error
if err = os.Unsetenv("IPFS_FD_MAX"); err != nil {
t.Fatal("Cannot unset the IPFS_FD_MAX env variable")
}
rlimit := syscall.Rlimit{}
if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil {
t.Fatal("Cannot get the file descriptor count")
}
value := rlimit.Max + rlimit.Cur
if err = os.Setenv("IPFS_FD_MAX", fmt.Sprintf("%d", value)); err != nil {
t.Fatal("Cannot set the IPFS_FD_MAX env variable")
}
// call to check and set the maximum file descriptor from the env
setMaxFds()
if err = ManageFdLimit(); err == nil {
t.Errorf("ManageFdLimit should return an error")
} else if err != nil {
flag := strings.Contains(err.Error(),
"cannot set rlimit, IPFS_FD_MAX is larger than the hard limit")
if !flag {
t.Errorf("ManageFdLimit returned unexpected error")
}
}
// unset all previous operations
if err = os.Unsetenv("IPFS_FD_MAX"); err != nil {
t.Fatal("Cannot unset the IPFS_FD_MAX env variable")
}
}
func TestManageFdLimitWithEnvSet(t *testing.T) {
t.Logf("Testing file descriptor manager with IPFS_FD_MAX set")
var err error
if err = os.Unsetenv("IPFS_FD_MAX"); err != nil {
t.Fatal("Cannot unset the IPFS_FD_MAX env variable")
}
rlimit := syscall.Rlimit{}
if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil {
t.Fatal("Cannot get the file descriptor count")
}
value := rlimit.Max - rlimit.Cur + 1
if err = os.Setenv("IPFS_FD_MAX", fmt.Sprintf("%d", value)); err != nil {
t.Fatal("Cannot set the IPFS_FD_MAX env variable")
}
setMaxFds()
if maxFds != value {
t.Errorf("The maxfds is not set from IPFS_FD_MAX")
}
if err = ManageFdLimit(); err != nil {
t.Errorf("Cannot manage file descriptor count")
}
// unset all previous operations
if err = os.Unsetenv("IPFS_FD_MAX"); err != nil {
t.Fatal("Cannot unset the IPFS_FD_MAX env variable")
}
}

27
cmd/ipfs/util/ulimit_unix.go Normal file
View File

@ -0,0 +1,27 @@
// +build darwin linux netbsd openbsd
package util
import (
unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix"
)
func init() {
supportsFDManagement = true
getLimit = unixGetLimit
setLimit = unixSetLimit
}
func unixGetLimit() (int64, int64, error) {
rlimit := unix.Rlimit{}
err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit)
return int64(rlimit.Cur), int64(rlimit.Max), err
}
func unixSetLimit(soft int64, max int64) error {
rlimit := unix.Rlimit{
Cur: uint64(soft),
Max: uint64(max),
}
return unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit)
}

7
cmd/ipfs/util/ulimit_windows.go Normal file
View File

@ -0,0 +1,7 @@
// +build windows
package util
func init() {
supportsFDManagement = false
}