integrate memory watchdog; impose limits on badger caches.

system/resources.go

@@ -0,0 +1,66 @@
+package system
+
+import (
+	"os"
+
+	"github.com/dustin/go-humanize"
+	"github.com/elastic/gosigar"
+	logging "github.com/ipfs/go-log/v2"
+)
+
+var (
+	logSystem = logging.Logger("system")
+)
+
+// EnvMaximumHeap is name of the environment variable with which the user can
+// specify a maximum heap size to abide by. The value of the env variable should
+// be in bytes, or in SI bytes (e.g. 32GiB).
+const EnvMaximumHeap = "LOTUS_MAX_HEAP"
+
+// ResourceConstraints represents resource constraints that Lotus and the go
+// runtime should abide by.
+var ResourceConstraints struct {
+	// MaxHeapMem is the maximum heap memory that has been set by the user
+	// through the LOTUS_MAX_HEAP env variable. If zero, there is no max heap
+	// limit set.
+	MaxHeapMem uint64
+
+	// TotalSystemMem is the total system memory as reported by go-sigar. If
+	// zero, it was impossible to determine the total system memory.
+	TotalSystemMem uint64
+
+	// EffectiveMemLimit is the memory limit in effect, in bytes.
+	//
+	// In order of precedence:
+	//  1. MaxHeapMem if non-zero.
+	//  2. TotalSystemMem if non-zero.
+	//  3. Zero (no known limit).
+	EffectiveMemLimit uint64
+}
+
+// init populates the global resource constraints for this process.
+func init() {
+	_ = logging.SetLogLevel("system", "INFO")
+	var mem gosigar.Mem
+	if err := mem.Get(); err != nil {
+		logSystem.Warnf("failed to acquire total system memory: %s", err)
+	} else {
+		ResourceConstraints.TotalSystemMem = mem.Total
+		ResourceConstraints.EffectiveMemLimit = mem.Total
+	}
+
+	if v := os.Getenv(EnvMaximumHeap); v != "" {
+		bytes, err := humanize.ParseBytes(v)
+		if err != nil {
+			logSystem.Warnf("failed to parse %s env variable with value %s: %s; ignoring max heap limit", EnvMaximumHeap, v, err)
+		} else {
+			ResourceConstraints.MaxHeapMem = bytes
+			ResourceConstraints.EffectiveMemLimit = bytes
+		}
+	}
+
+	logSystem.Infow("memory limits initialized",
+		"max_mem_heap", ResourceConstraints.MaxHeapMem,
+		"total_system_mem", ResourceConstraints.TotalSystemMem,
+		"effective_mem_limit", ResourceConstraints.EffectiveMemLimit)
+}