From f16ada6159f12532b291fac7e47ab78ca6f684ed Mon Sep 17 00:00:00 2001 From: hoenirvili Date: Tue, 21 Mar 2017 23:52:09 +0200 Subject: [PATCH] This patch contains more safe checking in syscall rlimit and unit tests The soft limit is the value that the kernel enforces for the corresponding resource The hard limit acts as a ceiling for the soft limit an unprivileged process may only set its soft limit to a value in the range from 0 up to the hard limit. So in order to make the change in fds count without any error we should inform the user to make the process have CAP_SYS_RESOURCE capability in order to set the hard limit. License: MIT Signed-off-by: hoenirvili --- cmd/ipfs/daemon.go | 8 ++- cmd/ipfs/ulimit.go | 19 ------- cmd/ipfs/ulimit_freebsd.go | 45 ---------------- cmd/ipfs/ulimit_unix.go | 43 ---------------- cmd/ipfs/util/ulimit.go | 91 +++++++++++++++++++++++++++++++++ cmd/ipfs/util/ulimit_freebsd.go | 27 ++++++++++ cmd/ipfs/util/ulimit_test.go | 88 +++++++++++++++++++++++++++++++ cmd/ipfs/util/ulimit_unix.go | 27 ++++++++++ cmd/ipfs/util/ulimit_windows.go | 7 +++ 9 files changed, 243 insertions(+), 112 deletions(-) delete mode 100644 cmd/ipfs/ulimit.go delete mode 100644 cmd/ipfs/ulimit_freebsd.go delete mode 100644 cmd/ipfs/ulimit_unix.go create mode 100644 cmd/ipfs/util/ulimit.go create mode 100644 cmd/ipfs/util/ulimit_freebsd.go create mode 100644 cmd/ipfs/util/ulimit_test.go create mode 100644 cmd/ipfs/util/ulimit_unix.go create mode 100644 cmd/ipfs/util/ulimit_windows.go diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 0d93e3252..93632856c 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -11,6 +11,7 @@ import ( "sort" "sync" + utilmain "github.com/ipfs/go-ipfs/cmd/ipfs/util" "github.com/ipfs/go-ipfs/core" commands "github.com/ipfs/go-ipfs/core/commands" corehttp "github.com/ipfs/go-ipfs/core/corehttp" @@ -180,8 +181,6 @@ func defaultMux(path string) corehttp.ServeOption { } } -var fileDescriptorCheck = func() error { return nil } - func daemonFunc(req cmds.Request, re cmds.ResponseEmitter) { // Inject metrics before we do anything @@ -193,9 +192,8 @@ func daemonFunc(req cmds.Request, re cmds.ResponseEmitter) { // let the user know we're going. fmt.Printf("Initializing daemon...\n") - managefd, _, _ := req.Option(adjustFDLimitKwd).Bool() - if managefd { - if err := fileDescriptorCheck(); err != nil { + if managed, _, _ := req.Option(adjustFDLimitKwd).Bool(); managed { + if err := utilmain.ManageFdLimit(); err != nil { log.Errorf("setting file descriptor limit: %s", err) } } diff --git a/cmd/ipfs/ulimit.go b/cmd/ipfs/ulimit.go deleted file mode 100644 index bfbd04df1..000000000 --- a/cmd/ipfs/ulimit.go +++ /dev/null @@ -1,19 +0,0 @@ -package main - -import ( - "os" - "strconv" -) - -var ipfsFileDescNum = uint64(2048) - -func init() { - if val := os.Getenv("IPFS_FD_MAX"); val != "" { - n, err := strconv.Atoi(val) - if err != nil { - log.Errorf("bad value for IPFS_FD_MAX: %s", err) - } else { - ipfsFileDescNum = uint64(n) - } - } -} diff --git a/cmd/ipfs/ulimit_freebsd.go b/cmd/ipfs/ulimit_freebsd.go deleted file mode 100644 index d97d05c9b..000000000 --- a/cmd/ipfs/ulimit_freebsd.go +++ /dev/null @@ -1,45 +0,0 @@ -// +build freebsd - -package main - -import ( - "fmt" - - unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix" -) - -func init() { - fileDescriptorCheck = checkAndSetUlimit -} - -func checkAndSetUlimit() error { - var rLimit unix.Rlimit - err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rLimit) - if err != nil { - return fmt.Errorf("error getting rlimit: %s", err) - } - - ipfsFileDescNum := int64(ipfsFileDescNum) - - var setting bool - if rLimit.Cur < ipfsFileDescNum { - if rLimit.Max < ipfsFileDescNum { - log.Error("adjusting max") - rLimit.Max = ipfsFileDescNum - } - fmt.Printf("Adjusting current ulimit to %d...\n", ipfsFileDescNum) - rLimit.Cur = ipfsFileDescNum - setting = true - } - - err = unix.Setrlimit(unix.RLIMIT_NOFILE, &rLimit) - if err != nil { - return fmt.Errorf("error setting ulimit: %s", err) - } - - if setting { - fmt.Printf("Successfully raised file descriptor limit to %d.\n", ipfsFileDescNum) - } - - return nil -} diff --git a/cmd/ipfs/ulimit_unix.go b/cmd/ipfs/ulimit_unix.go deleted file mode 100644 index 0f541a648..000000000 --- a/cmd/ipfs/ulimit_unix.go +++ /dev/null @@ -1,43 +0,0 @@ -// +build darwin linux netbsd openbsd - -package main - -import ( - "fmt" - - unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix" -) - -func init() { - fileDescriptorCheck = checkAndSetUlimit -} - -func checkAndSetUlimit() error { - var rLimit unix.Rlimit - err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rLimit) - if err != nil { - return fmt.Errorf("error getting rlimit: %s", err) - } - - var setting bool - if rLimit.Cur < ipfsFileDescNum { - if rLimit.Max < ipfsFileDescNum { - log.Error("adjusting max") - rLimit.Max = ipfsFileDescNum - } - fmt.Printf("Adjusting current ulimit to %d...\n", ipfsFileDescNum) - rLimit.Cur = ipfsFileDescNum - setting = true - } - - err = unix.Setrlimit(unix.RLIMIT_NOFILE, &rLimit) - if err != nil { - return fmt.Errorf("error setting ulimit: %s", err) - } - - if setting { - fmt.Printf("Successfully raised file descriptor limit to %d.\n", ipfsFileDescNum) - } - - return nil -} diff --git a/cmd/ipfs/util/ulimit.go b/cmd/ipfs/util/ulimit.go new file mode 100644 index 000000000..742a080a3 --- /dev/null +++ b/cmd/ipfs/util/ulimit.go @@ -0,0 +1,91 @@ +package util + +import ( + "errors" + "fmt" + "os" + "strconv" + "syscall" + + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" +) + +var log = logging.Logger("ulimit") + +var ( + supportsFDManagement = false + + // getlimit returns the soft and hard limits of file descriptors counts + getLimit func() (int64, int64, error) + // set limit sets the soft and hard limits of file descriptors counts + setLimit func(int64, int64) error +) + +// maxFds is the maximum number of file descriptors that go-ipfs +// can use. The default value is 1024. This can be overwritten by the +// IPFS_FD_MAX env variable +var maxFds = uint64(2048) + +// setMaxFds sets the maxFds value from IPFS_FD_MAX +// env variable if it's present on the system +func setMaxFds() { + // check if the IPFS_FD_MAX is set up and if it does + // not have a valid fds number notify the user + if val := os.Getenv("IPFS_FD_MAX"); val != "" { + + fds, err := strconv.ParseUint(val, 10, 64) + if err != nil { + log.Errorf("bad value for IPFS_FD_MAX: %s", err) + return + } + + maxFds = fds + } +} + +// ManageFdLimit raise the current max file descriptor count +// of the process based on the IPFS_FD_MAX value +func ManageFdLimit() error { + if !supportsFDManagement { + return nil + } + + setMaxFds() + soft, hard, err := getLimit() + if err != nil { + return err + } + + max := int64(maxFds) + + if max <= soft { + return nil + } + + // the soft limit is the value that the kernel enforces for the + // corresponding resource + // the hard limit acts as a ceiling for the soft limit + // an unprivileged process may only set it's soft limit to a + // alue in the range from 0 up to the hard limit + if err = setLimit(max, max); err != nil { + if err != syscall.EPERM { + return fmt.Errorf("error setting: ulimit: %s", err) + } + + // the process does not have permission so we should only + // set the soft value + if max > hard { + return errors.New( + "cannot set rlimit, IPFS_FD_MAX is larger than the hard limit", + ) + } + + if err = setLimit(max, hard); err != nil { + return fmt.Errorf("error setting ulimit wihout hard limit: %s", err) + } + } + + fmt.Printf("Successfully raised file descriptor limit to %d.\n", max) + + return nil +} diff --git a/cmd/ipfs/util/ulimit_freebsd.go b/cmd/ipfs/util/ulimit_freebsd.go new file mode 100644 index 000000000..d8b1d5582 --- /dev/null +++ b/cmd/ipfs/util/ulimit_freebsd.go @@ -0,0 +1,27 @@ +// +build freebsd + +package util + +import ( + unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix" +) + +func init() { + supportsFDManagement = true + getLimit = freebsdGetLimit + setLimit = freebdsSetLimit +} + +func freebsdGetLimit() (int64, int64, error) { + rlimit := unix.Rlimit{} + err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit) + return rlimit.Cur, rlimit.Max, err +} + +func freebdsSetLimit(soft int64, max int64) error { + rlimit := unix.Rlimit{ + Cur: soft, + Max: max, + } + return unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit) +} diff --git a/cmd/ipfs/util/ulimit_test.go b/cmd/ipfs/util/ulimit_test.go new file mode 100644 index 000000000..ffba72ea8 --- /dev/null +++ b/cmd/ipfs/util/ulimit_test.go @@ -0,0 +1,88 @@ +package util + +import ( + "fmt" + "os" + "strings" + "syscall" + "testing" +) + +func TestManageFdLimit(t *testing.T) { + t.Log("Testing file descriptor count") + if err := ManageFdLimit(); err != nil { + t.Errorf("Cannot manage file descriptors") + } + + if maxFds != uint64(2048) { + t.Errorf("Maximum file descriptors default value changed") + } +} + +func TestManageInvalidNFds(t *testing.T) { + t.Logf("Testing file descriptor invalidity") + var err error + if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { + t.Fatal("Cannot unset the IPFS_FD_MAX env variable") + } + + rlimit := syscall.Rlimit{} + if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil { + t.Fatal("Cannot get the file descriptor count") + } + + value := rlimit.Max + rlimit.Cur + if err = os.Setenv("IPFS_FD_MAX", fmt.Sprintf("%d", value)); err != nil { + t.Fatal("Cannot set the IPFS_FD_MAX env variable") + } + + // call to check and set the maximum file descriptor from the env + setMaxFds() + + if err = ManageFdLimit(); err == nil { + t.Errorf("ManageFdLimit should return an error") + } else if err != nil { + flag := strings.Contains(err.Error(), + "cannot set rlimit, IPFS_FD_MAX is larger than the hard limit") + if !flag { + t.Errorf("ManageFdLimit returned unexpected error") + } + } + + // unset all previous operations + if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { + t.Fatal("Cannot unset the IPFS_FD_MAX env variable") + } +} + +func TestManageFdLimitWithEnvSet(t *testing.T) { + t.Logf("Testing file descriptor manager with IPFS_FD_MAX set") + var err error + if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { + t.Fatal("Cannot unset the IPFS_FD_MAX env variable") + } + + rlimit := syscall.Rlimit{} + if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil { + t.Fatal("Cannot get the file descriptor count") + } + + value := rlimit.Max - rlimit.Cur + 1 + if err = os.Setenv("IPFS_FD_MAX", fmt.Sprintf("%d", value)); err != nil { + t.Fatal("Cannot set the IPFS_FD_MAX env variable") + } + + setMaxFds() + if maxFds != value { + t.Errorf("The maxfds is not set from IPFS_FD_MAX") + } + + if err = ManageFdLimit(); err != nil { + t.Errorf("Cannot manage file descriptor count") + } + + // unset all previous operations + if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { + t.Fatal("Cannot unset the IPFS_FD_MAX env variable") + } +} diff --git a/cmd/ipfs/util/ulimit_unix.go b/cmd/ipfs/util/ulimit_unix.go new file mode 100644 index 000000000..f931034ae --- /dev/null +++ b/cmd/ipfs/util/ulimit_unix.go @@ -0,0 +1,27 @@ +// +build darwin linux netbsd openbsd + +package util + +import ( + unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix" +) + +func init() { + supportsFDManagement = true + getLimit = unixGetLimit + setLimit = unixSetLimit +} + +func unixGetLimit() (int64, int64, error) { + rlimit := unix.Rlimit{} + err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit) + return int64(rlimit.Cur), int64(rlimit.Max), err +} + +func unixSetLimit(soft int64, max int64) error { + rlimit := unix.Rlimit{ + Cur: uint64(soft), + Max: uint64(max), + } + return unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit) +} diff --git a/cmd/ipfs/util/ulimit_windows.go b/cmd/ipfs/util/ulimit_windows.go new file mode 100644 index 000000000..3cd9908c3 --- /dev/null +++ b/cmd/ipfs/util/ulimit_windows.go @@ -0,0 +1,7 @@ +// +build windows + +package util + +func init() { + supportsFDManagement = false +}