Add ability to update container status from runc

Wire this in to all state-bound container operations to ensure
syncronization of container state.

Also exposes PID of running containers via API.

Signed-off-by: Matthew Heon <matthew.heon@gmail.com>

Closes: #56
Approved by: rhatdan
This commit is contained in:
Matthew Heon
2017-11-21 13:44:22 -05:00
committed by Atomic Bot
parent 7b736e3333
commit 8e76ebcf6e
6 changed files with 197 additions and 60 deletions

View File

@ -76,6 +76,11 @@ type containerRuntimeInfo struct {
FinishedTime time.Time `json:"finishedTime,omitempty"`
// ExitCode is the exit code returned when the container stopped
ExitCode int32 `json:"exitCode,omitempty"`
// OOMKilled indicates that the container was killed as it ran out of
// memory
OOMKilled bool `json:"oomKilled,omitempty"`
// PID is the PID of a running container
PID int `json:"pid,omitempty"`
// TODO: Save information about image used in container if one is used
}
@ -129,10 +134,10 @@ func (c *Container) Name() string {
// The spec returned is the one used to create the container. The running
// spec may differ slightly as mounts are added based on the image
func (c *Container) Spec() *spec.Spec {
spec := new(spec.Spec)
deepcopier.Copy(c.config.Spec).To(spec)
returnSpec := new(spec.Spec)
deepcopier.Copy(c.config.Spec).To(returnSpec)
return spec
return returnSpec
}
// Labels returns the container's labels
@ -150,13 +155,26 @@ func (c *Container) State() (ContainerState, error) {
c.lock.Lock()
defer c.lock.Unlock()
if err := c.runtime.state.UpdateContainer(c); err != nil {
return ContainerStateUnknown, errors.Wrapf(err, "error updating container %s state", c.ID())
if err := c.syncContainer(); err != nil {
return ContainerStateUnknown, err
}
return c.state.State, nil
}
// PID returns the PID of the container
// An error is returned if the container is not running
func (c *Container) PID() (int, error) {
c.lock.Lock()
defer c.lock.Unlock()
if err := c.syncContainer(); err != nil {
return -1, err
}
return c.state.PID, nil
}
// The path to the container's root filesystem - where the OCI spec will be
// placed, amongst other things
func (c *Container) bundlePath() string {
@ -168,6 +186,32 @@ func (c *Container) attachSocketPath() string {
return filepath.Join(c.runtime.ociRuntime.socketsDir, c.ID(), "attach")
}
// Sync this container with on-disk state and runc status
// Should only be called with container lock held
func (c *Container) syncContainer() error {
if err := c.runtime.state.UpdateContainer(c); err != nil {
return err
}
// If runc knows about the container, update its status in runc
// And then save back to disk
if (c.state.State != ContainerStateUnknown) &&
(c.state.State != ContainerStateConfigured) {
if err := c.runtime.ociRuntime.updateContainerStatus(c); err != nil {
return err
}
if err := c.runtime.state.SaveContainer(c); err != nil {
return err
}
}
if !c.valid {
return errors.Wrapf(ErrCtrRemoved, "container %s is not valid", c.ID())
}
return nil
}
// Make a new container
func newContainer(rspec *spec.Spec) (*Container, error) {
if rspec == nil {
@ -191,9 +235,6 @@ func newContainer(rspec *spec.Spec) (*Container, error) {
// Create container root filesystem for use
func (c *Container) setupStorage() error {
c.lock.Lock()
defer c.lock.Unlock()
if !c.valid {
return errors.Wrapf(ErrCtrRemoved, "container %s is not valid", c.ID())
}
@ -220,9 +261,6 @@ func (c *Container) setupStorage() error {
// Tear down a container's storage prior to removal
func (c *Container) teardownStorage() error {
c.lock.Lock()
defer c.lock.Unlock()
if !c.valid {
return errors.Wrapf(ErrCtrRemoved, "container %s is not valid", c.ID())
}
@ -251,12 +289,8 @@ func (c *Container) Init() (err error) {
c.lock.Lock()
defer c.lock.Unlock()
if err := c.runtime.state.UpdateContainer(c); err != nil {
return errors.Wrapf(err, "error updating container %s state", c.ID())
}
if !c.valid {
return errors.Wrapf(ErrCtrRemoved, "container %s is not valid", c.ID())
if err := c.syncContainer(); err != nil {
return err
}
if c.state.State != ContainerStateConfigured {
@ -325,12 +359,8 @@ func (c *Container) Start() error {
c.lock.Lock()
defer c.lock.Unlock()
if err := c.runtime.state.UpdateContainer(c); err != nil {
return errors.Wrapf(err, "error updating container %s state", c.ID())
}
if !c.valid {
return ErrCtrRemoved
if err := c.syncContainer(); err != nil {
return err
}
// Container must be created or stopped to be started
@ -344,8 +374,10 @@ func (c *Container) Start() error {
logrus.Debugf("Started container %s", c.ID())
c.state.StartedTime = time.Now()
c.state.State = ContainerStateRunning
// Update container's state as it should be ContainerStateRunning now
if err := c.runtime.ociRuntime.updateContainerStatus(c); err != nil {
return err
}
if err := c.runtime.state.SaveContainer(c); err != nil {
return errors.Wrapf(err, "error saving container %s state", c.ID())
@ -373,12 +405,8 @@ func (c *Container) Exec(cmd []string, tty bool, stdin bool) (string, error) {
// Attach attaches to a container
// Returns fully qualified URL of streaming server for the container
func (c *Container) Attach(noStdin bool, keys string, attached chan<- bool) error {
if err := c.runtime.state.UpdateContainer(c); err != nil {
return errors.Wrapf(err, "error updating container %s state", c.ID())
}
if !c.valid {
return errors.Wrapf(ErrCtrRemoved, "container %s is not valid", c.ID())
if err := c.syncContainer(); err != nil {
return err
}
if c.state.State == ContainerStateRunning || c.state.State == ContainerStatePaused {

16
libpod/finished_amd64.go Normal file
View File

@ -0,0 +1,16 @@
// +build !arm,!386
package libpod
import (
"os"
"syscall"
"time"
)
// Get the created time of a file
// Only works on 64-bit OSes
func getFinishedTime(fi os.FileInfo) time.Time {
st := fi.Sys().(*syscall.Stat_t)
return time.Unix(st.Ctim.Sec, st.Ctim.Nsec)
}

View File

@ -4,9 +4,11 @@ import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strconv"
"syscall"
"time"
@ -15,6 +17,7 @@ import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
kwait "k8s.io/apimachinery/pkg/util/wait"
// TODO import these functions into libpod and remove the import
// Trying to keep libpod from depending on CRI-O code
@ -255,19 +258,90 @@ func (r *OCIRuntime) createContainer(ctr *Container, cgroupParent string) error
}
// updateContainerStatus retrieves the current status of the container from the
// runtime
// remove nolint when implemented
func (r *OCIRuntime) updateContainerStatus(ctr *Container) error { //nolint
return ErrNotImplemented
// runtime. It updates the container's state but does not save it.
func (r *OCIRuntime) updateContainerStatus(ctr *Container) error {
state := new(spec.State)
out, err := exec.Command(r.path, "state", ctr.ID()).CombinedOutput()
if err != nil {
return errors.Wrapf(err, "error getting container %s state. stderr/out: %s", ctr.ID(), out)
}
if err := json.NewDecoder(bytes.NewBuffer(out)).Decode(state); err != nil {
return errors.Wrapf(err, "error decoding container status for container %s", ctr.ID())
}
ctr.state.PID = state.Pid
switch state.Status {
case "created":
ctr.state.State = ContainerStateCreated
case "paused":
ctr.state.State = ContainerStatePaused
case "running":
ctr.state.State = ContainerStateRunning
case "stopped":
ctr.state.State = ContainerStateStopped
default:
return errors.Wrapf(ErrInternal, "unrecognized status returned by runc for container %s: %s",
ctr.ID(), state.Status)
}
if ctr.state.State == ContainerStateStopped {
exitFile := filepath.Join(r.exitsDir, ctr.ID())
var fi os.FileInfo
err = kwait.ExponentialBackoff(
kwait.Backoff{
Duration: 500 * time.Millisecond,
Factor: 1.2,
Steps: 6,
},
func() (bool, error) {
var err error
fi, err = os.Stat(exitFile)
if err != nil {
// wait longer
return false, nil
}
return true, nil
})
if err != nil {
ctr.state.ExitCode = -1
ctr.state.FinishedTime = time.Now()
return errors.Wrapf(err, "no exit file for container %s found", ctr.ID())
}
ctr.state.FinishedTime = getFinishedTime(fi)
statusCodeStr, err := ioutil.ReadFile(exitFile)
if err != nil {
return errors.Wrapf(err, "failed to read exit file for container %s", ctr.ID())
}
statusCode, err := strconv.Atoi(string(statusCodeStr))
if err != nil {
return errors.Wrapf(err, "error convertaing exit status code for container %s to int",
ctr.ID())
}
ctr.state.ExitCode = int32(statusCode)
oomFilePath := filepath.Join(ctr.bundlePath(), "oom")
if _, err = os.Stat(oomFilePath); err == nil {
ctr.state.OOMKilled = true
}
}
return nil
}
// startContainer starts the given container
// remove nolint when function is complete
func (r *OCIRuntime) startContainer(ctr *Container) error { //nolint
// Sets time the container was started, but does not save it.
func (r *OCIRuntime) startContainer(ctr *Container) error {
// TODO: streams should probably *not* be our STDIN/OUT/ERR - redirect to buffers?
err := utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, r.path, "start", ctr.ID())
if err := utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, r.path, "start", ctr.ID()); err != nil {
return err
}
// TODO record start time in container struct
ctr.state.StartedTime = time.Now()
return err
return nil
}

View File

@ -91,12 +91,8 @@ func (r *Runtime) RemoveContainer(c *Container, force bool) error {
return ErrRuntimeStopped
}
if !c.valid {
return ErrCtrRemoved
}
// Update the container to get current state
if err := r.state.UpdateContainer(c); err != nil {
if err := c.syncContainer(); err != nil {
return err
}
@ -225,9 +221,3 @@ func (r *Runtime) removeMultipleContainers(containers []storage.Container) error
}
return nil
}
// ContainerConfigToDisk saves a container's nonvolatile configuration to disk
// remove nolint when implemented
func (r *Runtime) containerConfigToDisk(ctr *Container) error { //nolint
return ErrNotImplemented
}

View File

@ -103,7 +103,9 @@ func (s *SQLState) Container(id string) (*Container, error) {
containerState.MountPoint,
containerState.StartedTime,
containerState.FinishedTime,
containerState.ExitCode
containerState.ExitCode,
containerState.OomKilled,
containerState.Pid
FROM containers
INNER JOIN
containerState ON containers.Id = containerState.Id
@ -136,7 +138,9 @@ func (s *SQLState) LookupContainer(idOrName string) (*Container, error) {
containerState.MountPoint,
containerState.StartedTime,
containerState.FinishedTime,
containerState.ExitCode
containerState.ExitCode,
containerState.OomKilled,
containerState.Pid
FROM containers
INNER JOIN
containerState ON containers.Id = containerState.Id
@ -220,7 +224,7 @@ func (s *SQLState) AddContainer(ctr *Container) (err error) {
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
);`
addCtrState = `INSERT INTO containerState VALUES (
?, ?, ?, ?, ?, ?, ?, ?
?, ?, ?, ?, ?, ?, ?, ?, ?, ?
);`
)
@ -278,7 +282,9 @@ func (s *SQLState) AddContainer(ctr *Container) (err error) {
ctr.state.Mountpoint,
timeToSQL(ctr.state.StartedTime),
timeToSQL(ctr.state.FinishedTime),
ctr.state.ExitCode)
ctr.state.ExitCode,
boolToSQL(ctr.state.OOMKilled),
ctr.state.PID)
if err != nil {
return errors.Wrapf(err, "error adding container %s state to database", ctr.ID())
}
@ -315,7 +321,9 @@ func (s *SQLState) UpdateContainer(ctr *Container) error {
Mountpoint,
StartedTime,
FinishedTime,
ExitCode
ExitCode,
OomKilled,
Pid
FROM containerState WHERE ID=?;`
var (
@ -326,6 +334,8 @@ func (s *SQLState) UpdateContainer(ctr *Container) error {
startedTimeString string
finishedTimeString string
exitCode int32
oomKilled int
pid int
)
if !s.valid {
@ -344,7 +354,9 @@ func (s *SQLState) UpdateContainer(ctr *Container) error {
&mountpoint,
&startedTimeString,
&finishedTimeString,
&exitCode)
&exitCode,
&oomKilled,
&pid)
if err != nil {
// The container may not exist in the database
if err == sql.ErrNoRows {
@ -364,6 +376,8 @@ func (s *SQLState) UpdateContainer(ctr *Container) error {
newState.RunDir = runDir
newState.Mountpoint = mountpoint
newState.ExitCode = exitCode
newState.OOMKilled = boolFromSQL(oomKilled)
newState.PID = pid
if newState.Mountpoint != "" {
newState.Mounted = true
@ -396,7 +410,9 @@ func (s *SQLState) SaveContainer(ctr *Container) error {
Mountpoint=?,
StartedTime=?,
FinishedTime=?,
ExitCode=?
ExitCode=?,
OomKilled=?,
Pid=?
WHERE Id=?;`
s.lock.Lock()
@ -431,6 +447,8 @@ func (s *SQLState) SaveContainer(ctr *Container) error {
timeToSQL(ctr.state.StartedTime),
timeToSQL(ctr.state.FinishedTime),
ctr.state.ExitCode,
boolToSQL(ctr.state.OOMKilled),
ctr.state.PID,
ctr.ID())
if err != nil {
return errors.Wrapf(err, "error updating container %s state in database", ctr.ID())
@ -521,7 +539,9 @@ func (s *SQLState) AllContainers() ([]*Container, error) {
containerState.MountPoint,
containerState.StartedTime,
containerState.FinishedTime,
containerState.ExitCode
containerState.ExitCode,
containerState.OomKilled,
containerState.Pid
FROM containers
INNER JOIN
containerState ON containers.Id = containerState.Id;`

View File

@ -61,7 +61,10 @@ func prepareDB(db *sql.DB) (err error) {
StartedTime TEXT NUT NULL,
FinishedTime TEXT NOT NULL,
ExitCode INTEGER NOT NULL,
OomKilled INTEGER NOT NULL,
Pid INTEGER NOT NULL,
CHECK (State>0),
CHECK (OomKilled IN (0, 1)),
FOREIGN KEY (Id) REFERENCES containers(Id) DEFERRABLE INITIALLY DEFERRED
);
`
@ -149,6 +152,8 @@ func ctrFromScannable(row scannable, runtime *Runtime, specsDir string) (*Contai
startedTimeString string
finishedTimeString string
exitCode int32
oomKilled int
pid int
)
err := row.Scan(
@ -169,7 +174,9 @@ func ctrFromScannable(row scannable, runtime *Runtime, specsDir string) (*Contai
&mountpoint,
&startedTimeString,
&finishedTimeString,
&exitCode)
&exitCode,
&oomKilled,
&pid)
if err != nil {
if err == sql.ErrNoRows {
return nil, ErrNoSuchCtr
@ -197,6 +204,8 @@ func ctrFromScannable(row scannable, runtime *Runtime, specsDir string) (*Contai
ctr.state.RunDir = runDir
ctr.state.Mountpoint = mountpoint
ctr.state.ExitCode = exitCode
ctr.state.OOMKilled = boolFromSQL(oomKilled)
ctr.state.PID = pid
// TODO should we store this in the database separately instead?
if ctr.state.Mountpoint != "" {