Update containerd/cgroups repo fix perf issue

Signed-off-by: Matthew Heon <matthew.heon@gmail.com>
This commit is contained in:
Matthew Heon
2018-02-01 12:55:28 -05:00
parent 03cfe5ebbe
commit ae89dc28d0
15 changed files with 4074 additions and 221 deletions

View File

@ -3,12 +3,12 @@ package cgroups
import (
"bufio"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"syscall"
specs "github.com/opencontainers/runtime-spec/specs-go"
)
@ -56,8 +56,8 @@ func (b *blkioController) Update(path string, resources *specs.LinuxResources) e
return b.Create(path, resources)
}
func (b *blkioController) Stat(path string, stats *Stats) error {
stats.Blkio = &BlkioStat{}
func (b *blkioController) Stat(path string, stats *Metrics) error {
stats.Blkio = &BlkIOStat{}
settings := []blkioStatSettings{
{
name: "throttle.io_serviced",
@ -105,8 +105,13 @@ func (b *blkioController) Stat(path string, stats *Stats) error {
},
)
}
f, err := os.Open("/proc/diskstats")
if err != nil {
return err
}
defer f.Close()
devices, err := getDevices("/dev")
devices, err := getDevices(f)
if err != nil {
return err
}
@ -119,7 +124,7 @@ func (b *blkioController) Stat(path string, stats *Stats) error {
return nil
}
func (b *blkioController) readEntry(devices map[deviceKey]string, path, name string, entry *[]BlkioEntry) error {
func (b *blkioController) readEntry(devices map[deviceKey]string, path, name string, entry *[]*BlkIOEntry) error {
f, err := os.Open(filepath.Join(b.Path(path), fmt.Sprintf("blkio.%s", name)))
if err != nil {
return err
@ -131,7 +136,7 @@ func (b *blkioController) readEntry(devices map[deviceKey]string, path, name str
return err
}
// format: dev type amount
fields := strings.FieldsFunc(sc.Text(), splitBlkioStatLine)
fields := strings.FieldsFunc(sc.Text(), splitBlkIOStatLine)
if len(fields) < 3 {
if len(fields) == 2 && fields[0] == "Total" {
// skip total line
@ -158,7 +163,7 @@ func (b *blkioController) readEntry(devices map[deviceKey]string, path, name str
if err != nil {
return err
}
*entry = append(*entry, BlkioEntry{
*entry = append(*entry, &BlkIOEntry{
Device: devices[deviceKey{major, minor}],
Major: major,
Minor: minor,
@ -235,7 +240,7 @@ type blkioSettings struct {
type blkioStatSettings struct {
name string
entry *[]BlkioEntry
entry *[]*BlkIOEntry
}
func uintf(v interface{}) []byte {
@ -257,7 +262,7 @@ func throttleddev(v interface{}) []byte {
return []byte(fmt.Sprintf("%d:%d %d", td.Major, td.Minor, td.Rate))
}
func splitBlkioStatLine(r rune) bool {
func splitBlkIOStatLine(r rune) bool {
return r == ' ' || r == ':'
}
@ -268,50 +273,32 @@ type deviceKey struct {
// getDevices makes a best effort attempt to read all the devices into a map
// keyed by major and minor number. Since devices may be mapped multiple times,
// we err on taking the first occurrence.
func getDevices(path string) (map[deviceKey]string, error) {
// TODO(stevvooe): We are ignoring lots of errors. It might be kind of
// challenging to debug this if we aren't mapping devices correctly.
// Consider logging these errors.
devices := map[deviceKey]string{}
if err := filepath.Walk(path, func(p string, fi os.FileInfo, err error) error {
func getDevices(r io.Reader) (map[deviceKey]string, error) {
var (
s = bufio.NewScanner(r)
devices = make(map[deviceKey]string)
)
for s.Scan() {
fields := strings.Fields(s.Text())
major, err := strconv.Atoi(fields[0])
if err != nil {
return err
return nil, err
}
switch {
case fi.IsDir():
switch fi.Name() {
case "pts", "shm", "fd", "mqueue", ".lxc", ".lxd-mounts":
return filepath.SkipDir
default:
return nil
}
case fi.Name() == "console":
return nil
default:
if fi.Mode()&os.ModeDevice == 0 {
// skip non-devices
return nil
}
st, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
return fmt.Errorf("%s: unable to convert to system stat", p)
}
key := deviceKey{major(st.Rdev), minor(st.Rdev)}
if _, ok := devices[key]; ok {
return nil // skip it if we have already populated the path.
}
devices[key] = p
minor, err := strconv.Atoi(fields[1])
if err != nil {
return nil, err
}
return nil
}); err != nil {
return nil, err
key := deviceKey{
major: uint64(major),
minor: uint64(minor),
}
if _, ok := devices[key]; ok {
continue
}
devices[key] = filepath.Join("/dev", fields[2])
}
return devices, nil
return devices, s.Err()
}
func major(devNumber uint64) uint64 {

View File

@ -10,6 +10,7 @@ import (
"sync"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/errors"
)
// New returns a new control via the cgroup cgroups interface
@ -39,6 +40,9 @@ func Load(hierarchy Hierarchy, path Path) (Cgroup, error) {
for _, s := range pathers(subsystems) {
p, err := path(s.Name())
if err != nil {
if os.IsNotExist(errors.Cause(err)) {
return nil, ErrCgroupDeleted
}
return nil, err
}
if _, err := os.Lstat(s.Path(p)); err != nil {
@ -154,8 +158,8 @@ func (c *cgroup) Delete() error {
return nil
}
// Stat returns the current stats for the cgroup
func (c *cgroup) Stat(handlers ...ErrorHandler) (*Stats, error) {
// Stat returns the current metrics for the cgroup
func (c *cgroup) Stat(handlers ...ErrorHandler) (*Metrics, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.err != nil {
@ -165,9 +169,14 @@ func (c *cgroup) Stat(handlers ...ErrorHandler) (*Stats, error) {
handlers = append(handlers, errPassthrough)
}
var (
stats = &Stats{}
wg = &sync.WaitGroup{}
errs = make(chan error, len(c.subsystems))
stats = &Metrics{
CPU: &CPUStat{
Throttling: &Throttle{},
Usage: &CPUUsage{},
},
}
wg = &sync.WaitGroup{}
errs = make(chan error, len(c.subsystems))
)
for _, s := range c.subsystems {
if ss, ok := s.(stater); ok {
@ -301,7 +310,8 @@ func (c *cgroup) Thaw() error {
}
// OOMEventFD returns the memory cgroup's out of memory event fd that triggers
// when processes inside the cgroup receive an oom event
// when processes inside the cgroup receive an oom event. Returns
// ErrMemoryNotSupported if memory cgroups is not supported.
func (c *cgroup) OOMEventFD() (uintptr, error) {
c.mu.Lock()
defer c.mu.Unlock()

View File

@ -40,7 +40,7 @@ type Cgroup interface {
// subsystems are moved one at a time
MoveTo(Cgroup) error
// Stat returns the stats for all subsystems in the cgroup
Stat(...ErrorHandler) (*Stats, error)
Stat(...ErrorHandler) (*Metrics, error)
// Update updates all the subsystems with the provided resource changes
Update(resources *specs.LinuxResources) error
// Processes returns all the processes in a select subsystem for the cgroup

View File

@ -84,20 +84,13 @@ func (c *cpuController) Update(path string, resources *specs.LinuxResources) err
return c.Create(path, resources)
}
func (c *cpuController) Stat(path string, stats *Stats) error {
func (c *cpuController) Stat(path string, stats *Metrics) error {
f, err := os.Open(filepath.Join(c.Path(path), "cpu.stat"))
if err != nil {
return err
}
defer f.Close()
// get or create the cpu field because cpuacct can also set values on this struct
stats.cpuMu.Lock()
cpu := stats.Cpu
if cpu == nil {
cpu = &CpuStat{}
stats.Cpu = cpu
}
stats.cpuMu.Unlock()
sc := bufio.NewScanner(f)
for sc.Scan() {
if err := sc.Err(); err != nil {
@ -109,11 +102,11 @@ func (c *cpuController) Stat(path string, stats *Stats) error {
}
switch key {
case "nr_periods":
cpu.Throttling.Periods = v
stats.CPU.Throttling.Periods = v
case "nr_throttled":
cpu.Throttling.ThrottledPeriods = v
stats.CPU.Throttling.ThrottledPeriods = v
case "throttled_time":
cpu.Throttling.ThrottledTime = v
stats.CPU.Throttling.ThrottledTime = v
}
}
return nil

View File

@ -30,7 +30,7 @@ func (c *cpuacctController) Path(path string) string {
return filepath.Join(c.root, path)
}
func (c *cpuacctController) Stat(path string, stats *Stats) error {
func (c *cpuacctController) Stat(path string, stats *Metrics) error {
user, kernel, err := c.getUsage(path)
if err != nil {
return err
@ -43,17 +43,10 @@ func (c *cpuacctController) Stat(path string, stats *Stats) error {
if err != nil {
return err
}
stats.cpuMu.Lock()
cpu := stats.Cpu
if cpu == nil {
cpu = &CpuStat{}
stats.Cpu = cpu
}
stats.cpuMu.Unlock()
cpu.Usage.Total = total
cpu.Usage.User = user
cpu.Usage.Kernel = kernel
cpu.Usage.PerCpu = percpu
stats.CPU.Usage.Total = total
stats.CPU.Usage.User = user
stats.CPU.Usage.Kernel = kernel
stats.CPU.Usage.PerCPU = percpu
return nil
}

View File

@ -12,7 +12,7 @@ var (
ErrFreezerNotSupported = errors.New("cgroups: freezer cgroup not supported on this system")
ErrMemoryNotSupported = errors.New("cgroups: memory cgroup not supported on this system")
ErrCgroupDeleted = errors.New("cgroups: cgroup deleted")
ErrNoCgroupMountDestination = errors.New("cgroups: cannot found cgroup mount destination")
ErrNoCgroupMountDestination = errors.New("cgroups: cannot find cgroup mount destination")
)
// ErrorHandler is a function that handles and acts on errors

View File

@ -26,16 +26,10 @@ func (f *freezerController) Path(path string) string {
}
func (f *freezerController) Freeze(path string) error {
if err := f.changeState(path, Frozen); err != nil {
return err
}
return f.waitState(path, Frozen)
}
func (f *freezerController) Thaw(path string) error {
if err := f.changeState(path, Thawed); err != nil {
return err
}
return f.waitState(path, Thawed)
}
@ -57,6 +51,9 @@ func (f *freezerController) state(path string) (State, error) {
func (f *freezerController) waitState(path string, state State) error {
for {
if err := f.changeState(path, state); err != nil {
return err
}
current, err := f.state(path)
if err != nil {
return err

View File

@ -51,20 +51,21 @@ func (h *hugetlbController) Create(path string, resources *specs.LinuxResources)
return nil
}
func (h *hugetlbController) Stat(path string, stats *Stats) error {
stats.Hugetlb = make(map[string]HugetlbStat)
func (h *hugetlbController) Stat(path string, stats *Metrics) error {
for _, size := range h.sizes {
s, err := h.readSizeStat(path, size)
if err != nil {
return err
}
stats.Hugetlb[size] = s
stats.Hugetlb = append(stats.Hugetlb, s)
}
return nil
}
func (h *hugetlbController) readSizeStat(path, size string) (HugetlbStat, error) {
var s HugetlbStat
func (h *hugetlbController) readSizeStat(path, size string) (*HugetlbStat, error) {
s := HugetlbStat{
Pagesize: size,
}
for _, t := range []struct {
name string
value *uint64
@ -84,9 +85,9 @@ func (h *hugetlbController) readSizeStat(path, size string) (HugetlbStat, error)
} {
v, err := readUint(filepath.Join(h.Path(path), strings.Join([]string{"hugetlb", size, t.name}, ".")))
if err != nil {
return s, err
return nil, err
}
*t.value = v
}
return s, nil
return &s, nil
}

View File

@ -81,13 +81,18 @@ func (m *memoryController) Update(path string, resources *specs.LinuxResources)
return m.set(path, settings)
}
func (m *memoryController) Stat(path string, stats *Stats) error {
func (m *memoryController) Stat(path string, stats *Metrics) error {
f, err := os.Open(filepath.Join(m.Path(path), "memory.stat"))
if err != nil {
return err
}
defer f.Close()
stats.Memory = &MemoryStat{}
stats.Memory = &MemoryStat{
Usage: &MemoryEntry{},
Swap: &MemoryEntry{},
Kernel: &MemoryEntry{},
KernelTCP: &MemoryEntry{},
}
if err := m.parseStats(f, stats.Memory); err != nil {
return err
}
@ -97,19 +102,19 @@ func (m *memoryController) Stat(path string, stats *Stats) error {
}{
{
module: "",
entry: &stats.Memory.Usage,
entry: stats.Memory.Usage,
},
{
module: "memsw",
entry: &stats.Memory.Swap,
entry: stats.Memory.Swap,
},
{
module: "kmem",
entry: &stats.Memory.Kernel,
entry: stats.Memory.Kernel,
},
{
module: "kmem.tcp",
entry: &stats.Memory.KernelTCP,
entry: stats.Memory.KernelTCP,
},
} {
for _, tt := range []struct {
@ -155,7 +160,7 @@ func (m *memoryController) OOMEventFD(path string) (uintptr, error) {
return 0, err
}
defer f.Close()
fd, _, serr := unix.RawSyscall(unix.SYS_EVENTFD2, 0, unix.FD_CLOEXEC, 0)
fd, _, serr := unix.RawSyscall(unix.SYS_EVENTFD2, 0, unix.EFD_CLOEXEC, 0)
if serr != 0 {
return 0, serr
}

3851
vendor/github.com/containerd/cgroups/metrics.pb.go generated vendored Normal file

File diff suppressed because it is too large Load Diff

111
vendor/github.com/containerd/cgroups/metrics.proto generated vendored Normal file
View File

@ -0,0 +1,111 @@
syntax = "proto3";
package io.containerd.cgroups.v1;
import "gogoproto/gogo.proto";
message Metrics {
repeated HugetlbStat hugetlb = 1;
PidsStat pids = 2;
CPUStat cpu = 3 [(gogoproto.customname) = "CPU"];
MemoryStat memory = 4;
BlkIOStat blkio = 5;
}
message HugetlbStat {
uint64 usage = 1;
uint64 max = 2;
uint64 failcnt = 3;
string pagesize = 4;
}
message PidsStat {
uint64 current = 1;
uint64 limit = 2;
}
message CPUStat {
CPUUsage usage = 1;
Throttle throttling = 2;
}
message CPUUsage {
// values in nanoseconds
uint64 total = 1;
uint64 kernel = 2;
uint64 user = 3;
repeated uint64 per_cpu = 4 [(gogoproto.customname) = "PerCPU"];
}
message Throttle {
uint64 periods = 1;
uint64 throttled_periods = 2;
uint64 throttled_time = 3;
}
message MemoryStat {
uint64 cache = 1;
uint64 rss = 2 [(gogoproto.customname) = "RSS"];
uint64 rss_huge = 3 [(gogoproto.customname) = "RSSHuge"];
uint64 mapped_file = 4;
uint64 dirty = 5;
uint64 writeback = 6;
uint64 pg_pg_in = 7;
uint64 pg_pg_out = 8;
uint64 pg_fault = 9;
uint64 pg_maj_fault = 10;
uint64 inactive_anon = 11;
uint64 active_anon = 12;
uint64 inactive_file = 13;
uint64 active_file = 14;
uint64 unevictable = 15;
uint64 hierarchical_memory_limit = 16;
uint64 hierarchical_swap_limit = 17;
uint64 total_cache = 18;
uint64 total_rss = 19 [(gogoproto.customname) = "TotalRSS"];
uint64 total_rss_huge = 20 [(gogoproto.customname) = "TotalRSSHuge"];
uint64 total_mapped_file = 21;
uint64 total_dirty = 22;
uint64 total_writeback = 23;
uint64 total_pg_pg_in = 24;
uint64 total_pg_pg_out = 25;
uint64 total_pg_fault = 26;
uint64 total_pg_maj_fault = 27;
uint64 total_inactive_anon = 28;
uint64 total_active_anon = 29;
uint64 total_inactive_file = 30;
uint64 total_active_file = 31;
uint64 total_unevictable = 32;
MemoryEntry usage = 33;
MemoryEntry swap = 34;
MemoryEntry kernel = 35;
MemoryEntry kernel_tcp = 36 [(gogoproto.customname) = "KernelTCP"];
}
message MemoryEntry {
uint64 limit = 1;
uint64 usage = 2;
uint64 max = 3;
uint64 failcnt = 4;
}
message BlkIOStat {
repeated BlkIOEntry io_service_bytes_recursive = 1;
repeated BlkIOEntry io_serviced_recursive = 2;
repeated BlkIOEntry io_queued_recursive = 3;
repeated BlkIOEntry io_service_time_recursive = 4;
repeated BlkIOEntry io_wait_time_recursive = 5;
repeated BlkIOEntry io_merged_recursive = 6;
repeated BlkIOEntry io_time_recursive = 7;
repeated BlkIOEntry sectors_recursive = 8;
}
message BlkIOEntry {
string op = 1;
string device = 2;
uint64 major = 3;
uint64 minor = 4;
uint64 value = 5;
}

View File

@ -46,7 +46,7 @@ func (p *pidsController) Update(path string, resources *specs.LinuxResources) er
return p.Create(path, resources)
}
func (p *pidsController) Stat(path string, stats *Stats) error {
func (p *pidsController) Stat(path string, stats *Metrics) error {
current, err := readUint(filepath.Join(p.Path(path), "pids.current"))
if err != nil {
return err

View File

@ -1,109 +0,0 @@
package cgroups
import "sync"
type Stats struct {
cpuMu sync.Mutex
Hugetlb map[string]HugetlbStat
Pids *PidsStat
Cpu *CpuStat
Memory *MemoryStat
Blkio *BlkioStat
}
type HugetlbStat struct {
Usage uint64
Max uint64
Failcnt uint64
}
type PidsStat struct {
Current uint64
Limit uint64
}
type CpuStat struct {
Usage CpuUsage
Throttling Throttle
}
type CpuUsage struct {
// Units: nanoseconds.
Total uint64
PerCpu []uint64
Kernel uint64
User uint64
}
type Throttle struct {
Periods uint64
ThrottledPeriods uint64
ThrottledTime uint64
}
type MemoryStat struct {
Cache uint64
RSS uint64
RSSHuge uint64
MappedFile uint64
Dirty uint64
Writeback uint64
PgPgIn uint64
PgPgOut uint64
PgFault uint64
PgMajFault uint64
InactiveAnon uint64
ActiveAnon uint64
InactiveFile uint64
ActiveFile uint64
Unevictable uint64
HierarchicalMemoryLimit uint64
HierarchicalSwapLimit uint64
TotalCache uint64
TotalRSS uint64
TotalRSSHuge uint64
TotalMappedFile uint64
TotalDirty uint64
TotalWriteback uint64
TotalPgPgIn uint64
TotalPgPgOut uint64
TotalPgFault uint64
TotalPgMajFault uint64
TotalInactiveAnon uint64
TotalActiveAnon uint64
TotalInactiveFile uint64
TotalActiveFile uint64
TotalUnevictable uint64
Usage MemoryEntry
Swap MemoryEntry
Kernel MemoryEntry
KernelTCP MemoryEntry
}
type MemoryEntry struct {
Limit uint64
Usage uint64
Max uint64
Failcnt uint64
}
type BlkioStat struct {
IoServiceBytesRecursive []BlkioEntry
IoServicedRecursive []BlkioEntry
IoQueuedRecursive []BlkioEntry
IoServiceTimeRecursive []BlkioEntry
IoWaitTimeRecursive []BlkioEntry
IoMergedRecursive []BlkioEntry
IoTimeRecursive []BlkioEntry
SectorsRecursive []BlkioEntry
}
type BlkioEntry struct {
Op string
Device string
Major uint64
Minor uint64
Value uint64
}

View File

@ -67,7 +67,7 @@ type deleter interface {
type stater interface {
Subsystem
Stat(path string, stats *Stats) error
Stat(path string, stats *Metrics) error
}
type updater interface {

View File

@ -43,19 +43,13 @@ func Slice(slice, name string) Path {
}
func NewSystemd(root string) (*SystemdController, error) {
conn, err := systemdDbus.New()
if err != nil {
return nil, err
}
return &SystemdController{
root: root,
conn: conn,
}, nil
}
type SystemdController struct {
mu sync.Mutex
conn *systemdDbus.Conn
root string
}
@ -64,6 +58,11 @@ func (s *SystemdController) Name() Name {
}
func (s *SystemdController) Create(path string, resources *specs.LinuxResources) error {
conn, err := systemdDbus.New()
if err != nil {
return err
}
defer conn.Close()
slice, name := splitName(path)
properties := []systemdDbus.Property{
systemdDbus.PropDescription(fmt.Sprintf("cgroup %s", name)),
@ -74,14 +73,29 @@ func (s *SystemdController) Create(path string, resources *specs.LinuxResources)
newProperty("CPUAccounting", true),
newProperty("BlockIOAccounting", true),
}
_, err := s.conn.StartTransientUnit(name, "replace", properties, nil)
return err
ch := make(chan string)
_, err = conn.StartTransientUnit(name, "replace", properties, ch)
if err != nil {
return err
}
<-ch
return nil
}
func (s *SystemdController) Delete(path string) error {
conn, err := systemdDbus.New()
if err != nil {
return err
}
defer conn.Close()
_, name := splitName(path)
_, err := s.conn.StopUnit(name, "replace", nil)
return err
ch := make(chan string)
_, err = conn.StopUnit(name, "replace", ch)
if err != nil {
return err
}
<-ch
return nil
}
func newProperty(name string, units interface{}) systemdDbus.Property {