Implement Secrets

Implement podman secret create, inspect, ls, rm
Implement podman run/create --secret
Secrets are blobs of data that are sensitive.
Currently, the only secret driver supported is filedriver, which means creating a secret stores it in base64 unencrypted in a file.
After creating a secret, a user can use the --secret flag to expose the secret inside the container at /run/secrets/[secretname]
This secret will not be commited to an image on a podman commit

Signed-off-by: Ashley Cui <acui@redhat.com>
This commit is contained in:
Ashley Cui
2021-01-15 01:27:23 -05:00
parent 2aaf631586
commit 832a69b0be
58 changed files with 2962 additions and 7 deletions

View File

@ -0,0 +1,158 @@
package filedriver
import (
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"sort"
"github.com/containers/storage/pkg/lockfile"
"github.com/pkg/errors"
)
// secretsDataFile is the file where secrets data/payload will be stored
var secretsDataFile = "secretsdata.json"
// errNoSecretData indicates that there is not data associated with an id
var errNoSecretData = errors.New("no secret data with ID")
// errNoSecretData indicates that there is secret data already associated with an id
var errSecretIDExists = errors.New("secret data with ID already exists")
// Driver is the filedriver object
type Driver struct {
// secretsDataFilePath is the path to the secretsfile
secretsDataFilePath string
// lockfile is the filedriver lockfile
lockfile lockfile.Locker
}
// NewDriver creates a new file driver.
// rootPath is the directory where the secrets data file resides.
func NewDriver(rootPath string) (*Driver, error) {
fileDriver := new(Driver)
fileDriver.secretsDataFilePath = filepath.Join(rootPath, secretsDataFile)
// the lockfile functions requre that the rootPath dir is executable
if err := os.MkdirAll(rootPath, 0700); err != nil {
return nil, err
}
lock, err := lockfile.GetLockfile(filepath.Join(rootPath, "secretsdata.lock"))
if err != nil {
return nil, err
}
fileDriver.lockfile = lock
return fileDriver, nil
}
// List returns all secret IDs
func (d *Driver) List() ([]string, error) {
d.lockfile.Lock()
defer d.lockfile.Unlock()
secretData, err := d.getAllData()
if err != nil {
return nil, err
}
var allID []string
for k := range secretData {
allID = append(allID, k)
}
sort.Strings(allID)
return allID, err
}
// Lookup returns the bytes associated with a secret ID
func (d *Driver) Lookup(id string) ([]byte, error) {
d.lockfile.Lock()
defer d.lockfile.Unlock()
secretData, err := d.getAllData()
if err != nil {
return nil, err
}
if data, ok := secretData[id]; ok {
return data, nil
}
return nil, errors.Wrapf(errNoSecretData, "%s", id)
}
// Store stores the bytes associated with an ID. An error is returned if the ID arleady exists
func (d *Driver) Store(id string, data []byte) error {
d.lockfile.Lock()
defer d.lockfile.Unlock()
secretData, err := d.getAllData()
if err != nil {
return err
}
if _, ok := secretData[id]; ok {
return errors.Wrapf(errSecretIDExists, "%s", id)
}
secretData[id] = data
marshalled, err := json.MarshalIndent(secretData, "", " ")
if err != nil {
return err
}
err = ioutil.WriteFile(d.secretsDataFilePath, marshalled, 0600)
if err != nil {
return err
}
return nil
}
// Delete deletes the secret associated with the specified ID. An error is returned if no matching secret is found.
func (d *Driver) Delete(id string) error {
d.lockfile.Lock()
defer d.lockfile.Unlock()
secretData, err := d.getAllData()
if err != nil {
return err
}
if _, ok := secretData[id]; ok {
delete(secretData, id)
} else {
return errors.Wrap(errNoSecretData, id)
}
marshalled, err := json.MarshalIndent(secretData, "", " ")
if err != nil {
return err
}
err = ioutil.WriteFile(d.secretsDataFilePath, marshalled, 0600)
if err != nil {
return err
}
return nil
}
// getAllData reads the data file and returns all data
func (d *Driver) getAllData() (map[string][]byte, error) {
// check if the db file exists
_, err := os.Stat(d.secretsDataFilePath)
if err != nil {
if os.IsNotExist(err) {
// the file will be created later on a store()
return make(map[string][]byte), nil
} else {
return nil, err
}
}
file, err := os.Open(d.secretsDataFilePath)
if err != nil {
return nil, err
}
defer file.Close()
byteValue, err := ioutil.ReadAll(file)
if err != nil {
return nil, err
}
secretData := new(map[string][]byte)
err = json.Unmarshal([]byte(byteValue), secretData)
if err != nil {
return nil, err
}
return *secretData, nil
}

View File

@ -0,0 +1,282 @@
package secrets
import (
"os"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/containers/common/pkg/secrets/filedriver"
"github.com/containers/storage/pkg/lockfile"
"github.com/containers/storage/pkg/stringid"
"github.com/pkg/errors"
)
// maxSecretSize is the max size for secret data - 512kB
const maxSecretSize = 512000
// secretIDLength is the character length of a secret ID - 25
const secretIDLength = 25
// errInvalidPath indicates that the secrets path is invalid
var errInvalidPath = errors.New("invalid secrets path")
// errNoSuchSecret indicates that the secret does not exist
var errNoSuchSecret = errors.New("no such secret")
// errSecretNameInUse indicates that the secret name is already in use
var errSecretNameInUse = errors.New("secret name in use")
// errInvalidSecretName indicates that the secret name is invalid
var errInvalidSecretName = errors.New("invalid secret name")
// errInvalidDriver indicates that the driver type is invalid
var errInvalidDriver = errors.New("invalid driver")
// errInvalidDriverOpt indicates that a driver option is invalid
var errInvalidDriverOpt = errors.New("invalid driver option")
// errAmbiguous indicates that a secret is ambiguous
var errAmbiguous = errors.New("secret is ambiguous")
// errDataSize indicates that the secret data is too large or too small
var errDataSize = errors.New("secret data must be larger than 0 and less than 512000 bytes")
// secretsFile is the name of the file that the secrets database will be stored in
var secretsFile = "secrets.json"
// secretNameRegexp matches valid secret names
// Allowed: 64 [a-zA-Z0-9-_.] characters, and the start and end character must be [a-zA-Z0-9]
var secretNameRegexp = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_.-]*$`)
// SecretsManager holds information on handling secrets
type SecretsManager struct {
// secretsPath is the path to the db file where secrets are stored
secretsDBPath string
// lockfile is the locker for the secrets file
lockfile lockfile.Locker
// db is an in-memory cache of the database of secrets
db *db
}
// Secret defines a secret
type Secret struct {
// Name is the name of the secret
Name string `json:"name"`
// ID is the unique secret ID
ID string `json:"id"`
// Metadata stores other metadata on the secret
Metadata map[string]string `json:"metadata,omitempty"`
// CreatedAt is when the secret was created
CreatedAt time.Time `json:"createdAt"`
// Driver is the driver used to store secret data
Driver string `json:"driver"`
// DriverOptions is other metadata needed to use the driver
DriverOptions map[string]string `json:"driverOptions"`
}
// SecretsDriver interfaces with the secrets data store.
// The driver stores the actual bytes of secret data, as opposed to
// the secret metadata.
// Currently only the unencrypted filedriver is implemented.
type SecretsDriver interface {
// List lists all secret ids in the secrets data store
List() ([]string, error)
// Lookup gets the secret's data bytes
Lookup(id string) ([]byte, error)
// Store stores the secret's data bytes
Store(id string, data []byte) error
// Delete deletes a secret's data from the driver
Delete(id string) error
}
// NewManager creates a new secrets manager
// rootPath is the directory where the secrets data file resides
func NewManager(rootPath string) (*SecretsManager, error) {
manager := new(SecretsManager)
if !filepath.IsAbs(rootPath) {
return nil, errors.Wrapf(errInvalidPath, "path must be absolute: %s", rootPath)
}
// the lockfile functions requre that the rootPath dir is executable
if err := os.MkdirAll(rootPath, 0700); err != nil {
return nil, err
}
lock, err := lockfile.GetLockfile(filepath.Join(rootPath, "secrets.lock"))
if err != nil {
return nil, err
}
manager.lockfile = lock
manager.secretsDBPath = filepath.Join(rootPath, secretsFile)
manager.db = new(db)
manager.db.Secrets = make(map[string]Secret)
manager.db.NameToID = make(map[string]string)
manager.db.IDToName = make(map[string]string)
return manager, nil
}
// Store takes a name, creates a secret and stores the secret metadata and the secret payload.
// It returns a generated ID that is associated with the secret.
// The max size for secret data is 512kB.
func (s *SecretsManager) Store(name string, data []byte, driverType string, driverOpts map[string]string) (string, error) {
err := validateSecretName(name)
if err != nil {
return "", err
}
if !(len(data) > 0 && len(data) < maxSecretSize) {
return "", errDataSize
}
s.lockfile.Lock()
defer s.lockfile.Unlock()
exist, err := s.exactSecretExists(name)
if err != nil {
return "", err
}
if exist {
return "", errors.Wrapf(errSecretNameInUse, name)
}
secr := new(Secret)
secr.Name = name
for {
newID := stringid.GenerateNonCryptoID()
// GenerateNonCryptoID() gives 64 characters, so we truncate to correct length
newID = newID[0:secretIDLength]
_, err := s.lookupSecret(newID)
if err != nil {
if errors.Cause(err) == errNoSuchSecret {
secr.ID = newID
break
} else {
return "", err
}
}
}
secr.Driver = driverType
secr.Metadata = make(map[string]string)
secr.CreatedAt = time.Now()
secr.DriverOptions = driverOpts
driver, err := getDriver(driverType, driverOpts)
if err != nil {
return "", err
}
err = driver.Store(secr.ID, data)
if err != nil {
return "", errors.Wrapf(err, "error creating secret %s", name)
}
err = s.store(secr)
if err != nil {
return "", errors.Wrapf(err, "error creating secret %s", name)
}
return secr.ID, nil
}
// Delete removes all secret metadata and secret data associated with the specified secret.
// Delete takes a name, ID, or partial ID.
func (s *SecretsManager) Delete(nameOrID string) (string, error) {
err := validateSecretName(nameOrID)
if err != nil {
return "", err
}
s.lockfile.Lock()
defer s.lockfile.Unlock()
secret, err := s.lookupSecret(nameOrID)
if err != nil {
return "", err
}
secretID := secret.ID
driver, err := getDriver(secret.Driver, secret.DriverOptions)
if err != nil {
return "", err
}
err = driver.Delete(secretID)
if err != nil {
return "", errors.Wrapf(err, "error deleting secret %s", nameOrID)
}
err = s.delete(secretID)
if err != nil {
return "", errors.Wrapf(err, "error deleting secret %s", nameOrID)
}
return secretID, nil
}
// Lookup gives a secret's metadata given its name, ID, or partial ID.
func (s *SecretsManager) Lookup(nameOrID string) (*Secret, error) {
s.lockfile.Lock()
defer s.lockfile.Unlock()
return s.lookupSecret(nameOrID)
}
// List lists all secrets.
func (s *SecretsManager) List() ([]Secret, error) {
s.lockfile.Lock()
defer s.lockfile.Unlock()
secrets, err := s.lookupAll()
if err != nil {
return nil, err
}
var ls []Secret
for _, v := range secrets {
ls = append(ls, v)
}
return ls, nil
}
// LookupSecretData returns secret metadata as well as secret data in bytes.
// The secret data can be looked up using its name, ID, or partial ID.
func (s *SecretsManager) LookupSecretData(nameOrID string) (*Secret, []byte, error) {
s.lockfile.Lock()
defer s.lockfile.Unlock()
secret, err := s.lookupSecret(nameOrID)
if err != nil {
return nil, nil, err
}
driver, err := getDriver(secret.Driver, secret.DriverOptions)
if err != nil {
return nil, nil, err
}
data, err := driver.Lookup(secret.ID)
if err != nil {
return nil, nil, err
}
return secret, data, nil
}
// validateSecretName checks if the secret name is valid.
func validateSecretName(name string) error {
if !secretNameRegexp.MatchString(name) || len(name) > 64 || strings.HasSuffix(name, "-") || strings.HasSuffix(name, ".") {
return errors.Wrapf(errInvalidSecretName, "only 64 [a-zA-Z0-9-_.] characters allowed, and the start and end character must be [a-zA-Z0-9]: %s", name)
}
return nil
}
// getDriver creates a new driver.
func getDriver(name string, opts map[string]string) (SecretsDriver, error) {
if name == "file" {
if path, ok := opts["path"]; ok {
return filedriver.NewDriver(path)
} else {
return nil, errors.Wrap(errInvalidDriverOpt, "need path for filedriver")
}
}
return nil, errInvalidDriver
}

View File

@ -0,0 +1,211 @@
package secrets
import (
"encoding/json"
"io/ioutil"
"os"
"strings"
"time"
"github.com/pkg/errors"
)
type db struct {
// Secrets maps a secret id to secret metadata
Secrets map[string]Secret `json:"secrets"`
// NameToID maps a secret name to a secret id
NameToID map[string]string `json:"nameToID"`
// IDToName maps a secret id to a secret name
IDToName map[string]string `json:"idToName"`
// lastModified is the time when the database was last modified on the file system
lastModified time.Time
}
// loadDB loads database data into the in-memory cache if it has been modified
func (s *SecretsManager) loadDB() error {
// check if the db file exists
fileInfo, err := os.Stat(s.secretsDBPath)
if err != nil {
if !os.IsExist(err) {
// If the file doesn't exist, then there's no reason to update the db cache,
// the db cache will show no entries anyway.
// The file will be created later on a store()
return nil
} else {
return err
}
}
// We check if the file has been modified after the last time it was loaded into the cache.
// If the file has been modified, then we know that our cache is not up-to-date, so we load
// the db into the cache.
if s.db.lastModified.Equal(fileInfo.ModTime()) {
return nil
}
file, err := os.Open(s.secretsDBPath)
if err != nil {
return err
}
defer file.Close()
if err != nil {
return err
}
byteValue, err := ioutil.ReadAll(file)
if err != nil {
return err
}
unmarshalled := new(db)
if err := json.Unmarshal(byteValue, unmarshalled); err != nil {
return err
}
s.db = unmarshalled
s.db.lastModified = fileInfo.ModTime()
return nil
}
// getNameAndID takes a secret's name, ID, or partial ID, and returns both its name and full ID.
func (s *SecretsManager) getNameAndID(nameOrID string) (name, id string, err error) {
name, id, err = s.getExactNameAndID(nameOrID)
if err == nil {
return name, id, nil
} else if errors.Cause(err) != errNoSuchSecret {
return "", "", err
}
// ID prefix may have been given, iterate through all IDs.
// ID and partial ID has a max lenth of 25, so we return if its greater than that.
if len(nameOrID) > secretIDLength {
return "", "", errors.Wrapf(errNoSuchSecret, "no secret with name or id %q", nameOrID)
}
exists := false
var foundID, foundName string
for id, name := range s.db.IDToName {
if strings.HasPrefix(id, nameOrID) {
if exists {
return "", "", errors.Wrapf(errAmbiguous, "more than one result secret with prefix %s", nameOrID)
}
exists = true
foundID = id
foundName = name
}
}
if exists {
return foundName, foundID, nil
}
return "", "", errors.Wrapf(errNoSuchSecret, "no secret with name or id %q", nameOrID)
}
// getExactNameAndID takes a secret's name or ID and returns both its name and full ID.
func (s *SecretsManager) getExactNameAndID(nameOrID string) (name, id string, err error) {
err = s.loadDB()
if err != nil {
return "", "", err
}
if name, ok := s.db.IDToName[nameOrID]; ok {
id := nameOrID
return name, id, nil
}
if id, ok := s.db.NameToID[nameOrID]; ok {
name := nameOrID
return name, id, nil
}
return "", "", errors.Wrapf(errNoSuchSecret, "no secret with name or id %q", nameOrID)
}
// exactSecretExists checks if the secret exists, given a name or ID
// Does not match partial name or IDs
func (s *SecretsManager) exactSecretExists(nameOrID string) (bool, error) {
_, _, err := s.getExactNameAndID(nameOrID)
if err != nil {
if errors.Cause(err) == errNoSuchSecret {
return false, nil
}
return false, err
}
return true, nil
}
// lookupAll gets all secrets stored.
func (s *SecretsManager) lookupAll() (map[string]Secret, error) {
err := s.loadDB()
if err != nil {
return nil, err
}
return s.db.Secrets, nil
}
// lookupSecret returns a secret with the given name, ID, or partial ID.
func (s *SecretsManager) lookupSecret(nameOrID string) (*Secret, error) {
err := s.loadDB()
if err != nil {
return nil, err
}
_, id, err := s.getNameAndID(nameOrID)
if err != nil {
return nil, err
}
allSecrets, err := s.lookupAll()
if err != nil {
return nil, err
}
if secret, ok := allSecrets[id]; ok {
return &secret, nil
}
return nil, errors.Wrapf(errNoSuchSecret, "no secret with name or id %q", nameOrID)
}
// Store creates a new secret in the secrets database.
// It deals with only storing metadata, not data payload.
func (s *SecretsManager) store(entry *Secret) error {
err := s.loadDB()
if err != nil {
return err
}
s.db.Secrets[entry.ID] = *entry
s.db.NameToID[entry.Name] = entry.ID
s.db.IDToName[entry.ID] = entry.Name
marshalled, err := json.MarshalIndent(s.db, "", " ")
if err != nil {
return err
}
err = ioutil.WriteFile(s.secretsDBPath, marshalled, 0600)
if err != nil {
return err
}
return nil
}
// delete deletes a secret from the secrets database, given a name, ID, or partial ID.
// It deals with only deleting metadata, not data payload.
func (s *SecretsManager) delete(nameOrID string) error {
name, id, err := s.getNameAndID(nameOrID)
if err != nil {
return err
}
err = s.loadDB()
if err != nil {
return err
}
delete(s.db.Secrets, id)
delete(s.db.NameToID, name)
delete(s.db.IDToName, id)
marshalled, err := json.MarshalIndent(s.db, "", " ")
if err != nil {
return err
}
err = ioutil.WriteFile(s.secretsDBPath, marshalled, 0600)
if err != nil {
return err
}
return nil
}