Merge pull request #19096 from BlackHole1/support-new-line

feat(env): support multiline in env-file

docs/source/markdown/options/env-file.md

@@ -4,4 +4,8 @@
 ####> are applicable to all of those.
 #### **--env-file**=*file*
 
-Read in a line-delimited file of environment variables.
+Read the environment variables from the file, supporting prefix matching: `KEY*`, as well as multiline values in double quotes and single quotes, but not multiline values in backticks.
+The env-file will ignore comments and empty lines. And spaces or tabs before and after the KEY.
+If an invalid value is encountered, such as only an `=` sign, it will be skipped. If it is a prefix match (`KEY*`), all environment variables starting with KEY on the host machine will be loaded.
+If it is only KEY (`KEY`), the KEY environment variable on the host machine will be loaded.
+Compatible with the `export` syntax in **dotenv**, such as: `export KEY=bar`.

pkg/env/env.go

@@ -1,13 +1,29 @@
-// Package for processing environment variables.
 package env
 
-// TODO: we need to add tests for this package.
-
 import (
-	"bufio"
 	"fmt"
+	"io"
 	"os"
 	"strings"
+
+	"github.com/containers/storage/pkg/regexp"
+)
+
+var (
+	// Form: https://github.com/motdotla/dotenv/blob/aa03dcad1002027390dac1e8d96ac236274de354/lib/main.js#L9C76-L9C76
+	// (?:export\s+)?([\w.-]+) match key
+	// ([\w.%-]+)(\s*[=|*]\s*?|:\s+?) match separator
+	// Remaining match value
+	// e.g. KEY=VALUE => KEY, =, VALUE
+	//
+	//	KEY= => KEY, =, ""
+	//	KEY* => KEY, *, ""
+	//	KEY*=1 => KEY, *, =1
+	lineRegexp = regexp.Delayed(
+		`(?m)(?:^|^)\s*(?:export\s+)?([\w.%-]+)(\s*[=|*]\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*` +
+			"`(?:\\`|[^`])*`" + `|[^#\r\n]+)?\s*(?:#.*)?(?:$|$)`,
+	)
+	onlyKeyRegexp = regexp.Delayed(`^[\w.-]+$`)
 )
 
 const whiteSpaces = " \t"
@@ -80,47 +96,103 @@ func ParseFile(path string) (_ map[string]string, err error) {
 	}
 	defer fh.Close()
 
-	scanner := bufio.NewScanner(fh)
+	content, err := io.ReadAll(fh)
-	for scanner.Scan() {
+	if err != nil {
-		// trim the line from all leading whitespace first
-		line := strings.TrimLeft(scanner.Text(), whiteSpaces)
-		// line is not empty, and not starting with '#'
-		if len(line) > 0 && !strings.HasPrefix(line, "#") {
-			if err := parseEnv(env, line); err != nil {
 		return nil, err
 	}
+
+	// replace all \r\n and \r with \n
+	text := strings.NewReplacer("\r\n", "\n", "\r", "\n").Replace(string(content))
+	if err := parseEnv(env, text, false); err != nil {
+		return nil, err
 	}
-	}
+
-	return env, scanner.Err()
+	return env, nil
 }
 
-func parseEnv(env map[string]string, line string) error {
+// parseEnv parse the given content into env format
-	data := strings.SplitN(line, "=", 2)
+// @param enforceMatch bool	"it throws an error if there is no match"
+//
+// @example: parseEnv(env, "#comment", true) => error("invalid variable: #comment")
+// @example: parseEnv(env, "#comment", false) => nil
+// @example: parseEnv(env, "", false) => nil
+// @example: parseEnv(env, "KEY=FOO", true) => nil
+// @example: parseEnv(env, "KEY", true) => nil
+func parseEnv(env map[string]string, content string, enforceMatch bool) error {
+	m := envMatch(content)
 
-	// catch invalid variables such as "=" or "=A"
+	if len(m) == 0 && enforceMatch {
-	if data[0] == "" {
+		return fmt.Errorf("invalid variable: %q", content)
-		return fmt.Errorf("invalid variable: %q", line)
+	}
+
+	for _, match := range m {
+		key := match[1]
+		separator := strings.Trim(match[2], whiteSpaces)
+		value := match[3]
+
+		if strings.Contains(value, "\n") {
+			if strings.HasPrefix(value, "`") {
+				return fmt.Errorf("only support multi-line environment variables surrounded by "+
+					"double quotation marks or single quotation marks. invalid variable: %q", match[0])
+			}
+
+			// In the case of multi-line values, we need to remove the surrounding " '
+			value = strings.Trim(value, "\"'")
+		}
+
+		// KEY*=1 => KEY, *, =1 => KEY*, =, 1
+		if separator == "*" && strings.HasPrefix(value, "=") {
+			key += "*"
+			separator = "="
+			value = strings.TrimPrefix(value, "=")
+		}
+
+		switch separator {
+		case "=":
+			// KEY=
+			if value == "" {
+				if val, ok := os.LookupEnv(key); ok {
+					env[key] = val
 				}
-	// trim the front of a variable, but nothing else
-	name := strings.TrimLeft(data[0], whiteSpaces)
-	if len(data) > 1 {
-		env[name] = data[1]
 			} else {
-		if strings.HasSuffix(name, "*") {
+				env[key] = value
-			name = strings.TrimSuffix(name, "*")
+			}
+		case "*":
 			for _, e := range os.Environ() {
 				part := strings.SplitN(e, "=", 2)
 				if len(part) < 2 {
 					continue
 				}
-				if strings.HasPrefix(part[0], name) {
+				if strings.HasPrefix(part[0], key) {
 					env[part[0]] = part[1]
 				}
 			}
-		} else if val, ok := os.LookupEnv(name); ok {
-			// if only a pass-through variable is given, clean it up.
-			env[name] = val
 		}
 	}
 	return nil
 }
+
+func envMatch(content string) [][]string {
+	m := lineRegexp.FindAllStringSubmatch(content, -1)
+
+	// KEY => KEY, =, ""
+	// Due to the above regex pattern, it will skip cases where only KEY is present (e.g., foo).
+	// However, in our requirement, this situation is equivalent to foo=(i.e., "foo" == "foo=").
+	// Therefore, we need to perform additional processing.
+	// The reason for needing to support this scenario is that we need to consider: `podman run -e CI -e USERNAME`.
+	{
+		noMatched := lineRegexp.ReplaceAllString(content, "")
+		nl := strings.Split(noMatched, "\n")
+		for _, key := range nl {
+			key := strings.Trim(key, whiteSpaces)
+			if key == "" {
+				continue
+			}
+			if onlyKeyRegexp.MatchString(key) {
+				m = append(m, []string{key, key, "=", ""})
+			}
+		}
+	}
+
+	return m
+}

pkg/env/env_test.go

@@ -1,6 +1,8 @@
 package env
 
 import (
+	"fmt"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -99,6 +101,228 @@ func TestJoin(t *testing.T) {
 	}
 }
 
+func createTmpFile(content string) (string, error) {
+	tmpfile, err := os.CreateTemp(os.TempDir(), "podman-test-parse-env-")
+	if err != nil {
+		return "", err
+	}
+
+	if _, err := tmpfile.WriteString(content); err != nil {
+		return "", err
+	}
+	if err := tmpfile.Close(); err != nil {
+		return "", err
+	}
+	return tmpfile.Name(), nil
+}
+
+func Test_ParseFile(t *testing.T) {
+	tests := []struct {
+		name      string
+		key       string
+		separator string // = or *
+		value     string
+
+		// environment variable
+		envKey   string
+		envValue string
+
+		expectedKey   string
+		expectedValue string
+		success       bool
+	}{
+		{
+			name:      "Good",
+			key:       "Key",
+			separator: "=",
+			value:     "Value1",
+			success:   true,
+		},
+		{
+			name:      "HasDoubleQuotesWithSingleLine",
+			key:       "Key2",
+			separator: "=",
+			value:     `"Value2"`,
+			success:   true,
+		},
+		{
+			name:      "HasSingleQuotesWithSingleLine",
+			key:       "Key3",
+			separator: "=",
+			value:     `'Value3'`,
+			success:   true,
+		},
+		{
+			name:      "KeepValueSpace",
+			key:       "Key4",
+			separator: "=",
+			value:     "  Value4  ",
+			success:   true,
+		},
+		{
+			name:        "RemoveKeySpace",
+			key:         "  Key5  ",
+			separator:   "=",
+			expectedKey: "Key5",
+			value:       "Value5",
+			success:     true,
+		},
+		{
+			name:          "NoValue",
+			key:           "Key6",
+			separator:     "=",
+			value:         "",
+			envValue:      "Value6",
+			expectedValue: "Value6",
+			success:       true,
+		},
+		{
+			name:          "FromEnv",
+			key:           "Key7",
+			separator:     "=",
+			value:         "",
+			envValue:      "Value7",
+			expectedValue: "Value7",
+			success:       true,
+		},
+		{
+			name:          "OnlyKey",
+			key:           "Key8",
+			separator:     "",
+			value:         "",
+			envValue:      "Value8",
+			expectedValue: "Value8",
+			success:       true,
+		},
+		{
+			name:          "GlobKey",
+			key:           "Key9",
+			separator:     "*",
+			value:         "",
+			envKey:        "Key9999",
+			envValue:      "Value9",
+			expectedKey:   "Key9999",
+			expectedValue: "Value9",
+			success:       true,
+		},
+		{
+			name:          "InvalidGlobKey",
+			key:           "Key10*",
+			separator:     "=",
+			value:         "1",
+			envKey:        "Key1010",
+			envValue:      "Value10",
+			expectedKey:   "Key10*",
+			expectedValue: "1",
+			success:       true,
+		},
+		{
+			name:          "MultilineWithDoubleQuotes",
+			key:           "Key11",
+			separator:     "=",
+			value:         "\"First line1\nlast line1\"",
+			expectedValue: "First line1\nlast line1",
+			success:       true,
+		},
+		{
+			name:          "MultilineWithSingleQuotes",
+			key:           "Key12",
+			separator:     "=",
+			value:         "'First line2\nlast line2'",
+			expectedValue: "First line2\nlast line2",
+			success:       true,
+		},
+		{
+			name:      "Has%",
+			key:       "BASH_FUNC__fmt_ctx%%",
+			separator: "=",
+			value:     "() { echo 1; }",
+			success:   true,
+		},
+		{
+			name:          "Export syntax",
+			key:           "export Key13",
+			separator:     "=",
+			value:         "Value13",
+			expectedKey:   "Key13",
+			expectedValue: "Value13",
+			success:       true,
+		},
+		{
+			name:      "NoValueAndNoEnv",
+			key:       "Key14",
+			separator: "=",
+			value:     "",
+			success:   false,
+		},
+		{
+			name:      "OnlyValue",
+			key:       "",
+			separator: "=",
+			value:     "Value",
+			success:   false,
+		},
+		{
+			name:      "OnlyDelim",
+			key:       "",
+			separator: "=",
+			value:     "",
+			success:   false,
+		},
+		{
+			name:      "Comment",
+			key:       "#aaaa",
+			separator: "",
+			value:     "",
+			success:   false,
+		},
+	}
+
+	content := ""
+	for _, tt := range tests {
+		content += fmt.Sprintf("%s%s%s\n", tt.key, tt.separator, tt.value)
+
+		if tt.envValue != "" {
+			key := tt.key
+			if tt.envKey != "" {
+				key = tt.envKey
+			}
+			t.Setenv(key, tt.envValue)
+		}
+	}
+	tFile, err := createTmpFile(content)
+	defer os.Remove(tFile)
+	assert.NoError(t, err)
+
+	env, err := ParseFile(tFile)
+	assert.NoError(t, err)
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			key := tt.key
+			if tt.expectedKey != "" {
+				key = tt.expectedKey
+			}
+			val, ok := env[key]
+			if ok && !tt.success {
+				t.Errorf("not should set key:%s ", tt.key)
+				return
+			} else if !ok && tt.success {
+				t.Errorf("should set key:%s ", tt.key)
+				return
+			}
+
+			if tt.success {
+				value := tt.value
+				if tt.expectedValue != "" {
+					value = tt.expectedValue
+				}
+				assert.Equal(t, value, val, "value should be equal")
+			}
+		})
+	}
+}
+
 func Test_parseEnv(t *testing.T) {
 	good := make(map[string]string)
 
@@ -135,6 +359,14 @@ func Test_parseEnv(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "GoodOnlyKey",
+			args: args{
+				env:  good,
+				line: "apple",
+			},
+			wantErr: false,
+		},
 		{
 			name: "BadNoKey",
 			args: args{
@@ -151,10 +383,18 @@ func Test_parseEnv(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "MultilineWithBackticksQuotes",
+			args: args{
+				env:  good,
+				line: "apple=`foo\nbar`",
+			},
+			wantErr: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if err := parseEnv(tt.args.env, tt.args.line); (err != nil) != tt.wantErr {
+			if err := parseEnv(tt.args.env, tt.args.line, true); (err != nil) != tt.wantErr {
 				t.Errorf("parseEnv() error = %v, wantErr %v", err, tt.wantErr)
 			}
 		})

pkg/env/env_unix.go

@@ -8,7 +8,7 @@ package env
 func ParseSlice(s []string) (map[string]string, error) {
 	env := make(map[string]string, len(s))
 	for _, e := range s {
-		if err := parseEnv(env, e); err != nil {
+		if err := parseEnv(env, e, true); err != nil {
 			return nil, err
 		}
 	}

pkg/env/env_windows.go

@@ -17,7 +17,7 @@ func ParseSlice(s []string) (map[string]string, error) {
 			continue
 		}
 
-		if err := parseEnv(env, e); err != nil {
+		if err := parseEnv(env, e, true); err != nil {
 			return nil, err
 		}
 	}