Merge pull request #3801 from hoenirvili/refactor-code

Safe check rlimit syscall and refactoring

cmd/ipfs/daemon.go

@@ -11,6 +11,7 @@ import (
 	"sort"
 	"sync"
 
+	utilmain "github.com/ipfs/go-ipfs/cmd/ipfs/util"
 	"github.com/ipfs/go-ipfs/core"
 	commands "github.com/ipfs/go-ipfs/core/commands"
 	corehttp "github.com/ipfs/go-ipfs/core/corehttp"
@@ -180,8 +181,6 @@ func defaultMux(path string) corehttp.ServeOption {
 	}
 }
 
-var fileDescriptorCheck = func() error { return nil }
-
 func daemonFunc(req cmds.Request, re cmds.ResponseEmitter) {
 	// Inject metrics before we do anything
 
@@ -193,9 +192,8 @@ func daemonFunc(req cmds.Request, re cmds.ResponseEmitter) {
 	// let the user know we're going.
 	fmt.Printf("Initializing daemon...\n")
 
-	managefd, _, _ := req.Option(adjustFDLimitKwd).Bool()
+	if managed, _, _ := req.Option(adjustFDLimitKwd).Bool(); managed {
-	if managefd {
+		if err := utilmain.ManageFdLimit(); err != nil {
-		if err := fileDescriptorCheck(); err != nil {
 			log.Errorf("setting file descriptor limit: %s", err)
 		}
 	}

cmd/ipfs/ulimit.go

@@ -1,19 +0,0 @@
-package main
-
-import (
-	"os"
-	"strconv"
-)
-
-var ipfsFileDescNum = uint64(2048)
-
-func init() {
-	if val := os.Getenv("IPFS_FD_MAX"); val != "" {
-		n, err := strconv.Atoi(val)
-		if err != nil {
-			log.Errorf("bad value for IPFS_FD_MAX: %s", err)
-		} else {
-			ipfsFileDescNum = uint64(n)
-		}
-	}
-}

cmd/ipfs/ulimit_freebsd.go

@@ -1,45 +0,0 @@
-// +build freebsd
-
-package main
-
-import (
-	"fmt"
-
-	unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix"
-)
-
-func init() {
-	fileDescriptorCheck = checkAndSetUlimit
-}
-
-func checkAndSetUlimit() error {
-	var rLimit unix.Rlimit
-	err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rLimit)
-	if err != nil {
-		return fmt.Errorf("error getting rlimit: %s", err)
-	}
-
-	ipfsFileDescNum := int64(ipfsFileDescNum)
-
-	var setting bool
-	if rLimit.Cur < ipfsFileDescNum {
-		if rLimit.Max < ipfsFileDescNum {
-			log.Error("adjusting max")
-			rLimit.Max = ipfsFileDescNum
-		}
-		fmt.Printf("Adjusting current ulimit to %d...\n", ipfsFileDescNum)
-		rLimit.Cur = ipfsFileDescNum
-		setting = true
-	}
-
-	err = unix.Setrlimit(unix.RLIMIT_NOFILE, &rLimit)
-	if err != nil {
-		return fmt.Errorf("error setting ulimit: %s", err)
-	}
-
-	if setting {
-		fmt.Printf("Successfully raised file descriptor limit to %d.\n", ipfsFileDescNum)
-	}
-
-	return nil
-}

cmd/ipfs/ulimit_unix.go

@@ -1,43 +0,0 @@
-// +build darwin linux netbsd openbsd
-
-package main
-
-import (
-	"fmt"
-
-	unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix"
-)
-
-func init() {
-	fileDescriptorCheck = checkAndSetUlimit
-}
-
-func checkAndSetUlimit() error {
-	var rLimit unix.Rlimit
-	err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rLimit)
-	if err != nil {
-		return fmt.Errorf("error getting rlimit: %s", err)
-	}
-
-	var setting bool
-	if rLimit.Cur < ipfsFileDescNum {
-		if rLimit.Max < ipfsFileDescNum {
-			log.Error("adjusting max")
-			rLimit.Max = ipfsFileDescNum
-		}
-		fmt.Printf("Adjusting current ulimit to %d...\n", ipfsFileDescNum)
-		rLimit.Cur = ipfsFileDescNum
-		setting = true
-	}
-
-	err = unix.Setrlimit(unix.RLIMIT_NOFILE, &rLimit)
-	if err != nil {
-		return fmt.Errorf("error setting ulimit: %s", err)
-	}
-
-	if setting {
-		fmt.Printf("Successfully raised file descriptor limit to %d.\n", ipfsFileDescNum)
-	}
-
-	return nil
-}

cmd/ipfs/util/ulimit.go

@@ -0,0 +1,91 @@
+package util
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"syscall"
+
+	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
+)
+
+var log = logging.Logger("ulimit")
+
+var (
+	supportsFDManagement = false
+
+	// getlimit returns the soft and hard limits of file descriptors counts
+	getLimit func() (int64, int64, error)
+	// set limit sets the soft and hard limits of file descriptors counts
+	setLimit func(int64, int64) error
+)
+
+// maxFds is the maximum number of file descriptors that go-ipfs
+// can use. The default value is 1024. This can be overwritten by the
+// IPFS_FD_MAX env variable
+var maxFds = uint64(2048)
+
+// setMaxFds sets the maxFds value from IPFS_FD_MAX
+// env variable if it's present on the system
+func setMaxFds() {
+	// check if the IPFS_FD_MAX is set up and if it does
+	// not have a valid fds number notify the user
+	if val := os.Getenv("IPFS_FD_MAX"); val != "" {
+
+		fds, err := strconv.ParseUint(val, 10, 64)
+		if err != nil {
+			log.Errorf("bad value for IPFS_FD_MAX: %s", err)
+			return
+		}
+
+		maxFds = fds
+	}
+}
+
+// ManageFdLimit raise the current max file descriptor count
+// of the process based on the IPFS_FD_MAX value
+func ManageFdLimit() error {
+	if !supportsFDManagement {
+		return nil
+	}
+
+	setMaxFds()
+	soft, hard, err := getLimit()
+	if err != nil {
+		return err
+	}
+
+	max := int64(maxFds)
+
+	if max <= soft {
+		return nil
+	}
+
+	// the soft limit is the value that the kernel enforces for the
+	// corresponding resource
+	// the hard limit acts as a ceiling for the soft limit
+	// an unprivileged process may only set it's soft limit to a
+	// alue in the range from 0 up to the hard limit
+	if err = setLimit(max, max); err != nil {
+		if err != syscall.EPERM {
+			return fmt.Errorf("error setting: ulimit: %s", err)
+		}
+
+		// the process does not have permission so we should only
+		// set the soft value
+		if max > hard {
+			return errors.New(
+				"cannot set rlimit, IPFS_FD_MAX is larger than the hard limit",
+			)
+		}
+
+		if err = setLimit(max, hard); err != nil {
+			return fmt.Errorf("error setting ulimit wihout hard limit: %s", err)
+		}
+	}
+
+	fmt.Printf("Successfully raised file descriptor limit to %d.\n", max)
+
+	return nil
+}

cmd/ipfs/util/ulimit_freebsd.go

@@ -0,0 +1,27 @@
+// +build freebsd
+
+package util
+
+import (
+	unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix"
+)
+
+func init() {
+	supportsFDManagement = true
+	getLimit = freebsdGetLimit
+	setLimit = freebdsSetLimit
+}
+
+func freebsdGetLimit() (int64, int64, error) {
+	rlimit := unix.Rlimit{}
+	err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit)
+	return rlimit.Cur, rlimit.Max, err
+}
+
+func freebdsSetLimit(soft int64, max int64) error {
+	rlimit := unix.Rlimit{
+		Cur: soft,
+		Max: max,
+	}
+	return unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit)
+}

cmd/ipfs/util/ulimit_test.go

@@ -0,0 +1,88 @@
+package util
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"syscall"
+	"testing"
+)
+
+func TestManageFdLimit(t *testing.T) {
+	t.Log("Testing file descriptor count")
+	if err := ManageFdLimit(); err != nil {
+		t.Errorf("Cannot manage file descriptors")
+	}
+
+	if maxFds != uint64(2048) {
+		t.Errorf("Maximum file descriptors default value changed")
+	}
+}
+
+func TestManageInvalidNFds(t *testing.T) {
+	t.Logf("Testing file descriptor invalidity")
+	var err error
+	if err = os.Unsetenv("IPFS_FD_MAX"); err != nil {
+		t.Fatal("Cannot unset the IPFS_FD_MAX env variable")
+	}
+
+	rlimit := syscall.Rlimit{}
+	if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil {
+		t.Fatal("Cannot get the file descriptor count")
+	}
+
+	value := rlimit.Max + rlimit.Cur
+	if err = os.Setenv("IPFS_FD_MAX", fmt.Sprintf("%d", value)); err != nil {
+		t.Fatal("Cannot set the IPFS_FD_MAX env variable")
+	}
+
+	// call to check and set the maximum file descriptor from the env
+	setMaxFds()
+
+	if err = ManageFdLimit(); err == nil {
+		t.Errorf("ManageFdLimit should return an error")
+	} else if err != nil {
+		flag := strings.Contains(err.Error(),
+			"cannot set rlimit, IPFS_FD_MAX is larger than the hard limit")
+		if !flag {
+			t.Errorf("ManageFdLimit returned unexpected error")
+		}
+	}
+
+	// unset all previous operations
+	if err = os.Unsetenv("IPFS_FD_MAX"); err != nil {
+		t.Fatal("Cannot unset the IPFS_FD_MAX env variable")
+	}
+}
+
+func TestManageFdLimitWithEnvSet(t *testing.T) {
+	t.Logf("Testing file descriptor manager with IPFS_FD_MAX set")
+	var err error
+	if err = os.Unsetenv("IPFS_FD_MAX"); err != nil {
+		t.Fatal("Cannot unset the IPFS_FD_MAX env variable")
+	}
+
+	rlimit := syscall.Rlimit{}
+	if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil {
+		t.Fatal("Cannot get the file descriptor count")
+	}
+
+	value := rlimit.Max - rlimit.Cur + 1
+	if err = os.Setenv("IPFS_FD_MAX", fmt.Sprintf("%d", value)); err != nil {
+		t.Fatal("Cannot set the IPFS_FD_MAX env variable")
+	}
+
+	setMaxFds()
+	if maxFds != value {
+		t.Errorf("The maxfds is not set from IPFS_FD_MAX")
+	}
+
+	if err = ManageFdLimit(); err != nil {
+		t.Errorf("Cannot manage file descriptor count")
+	}
+
+	// unset all previous operations
+	if err = os.Unsetenv("IPFS_FD_MAX"); err != nil {
+		t.Fatal("Cannot unset the IPFS_FD_MAX env variable")
+	}
+}

cmd/ipfs/util/ulimit_unix.go

@@ -0,0 +1,27 @@
+// +build darwin linux netbsd openbsd
+
+package util
+
+import (
+	unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix"
+)
+
+func init() {
+	supportsFDManagement = true
+	getLimit = unixGetLimit
+	setLimit = unixSetLimit
+}
+
+func unixGetLimit() (int64, int64, error) {
+	rlimit := unix.Rlimit{}
+	err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit)
+	return int64(rlimit.Cur), int64(rlimit.Max), err
+}
+
+func unixSetLimit(soft int64, max int64) error {
+	rlimit := unix.Rlimit{
+		Cur: uint64(soft),
+		Max: uint64(max),
+	}
+	return unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit)
+}

cmd/ipfs/util/ulimit_windows.go

@@ -0,0 +1,7 @@
+// +build windows
+
+package util
+
+func init() {
+	supportsFDManagement = false
+}