feat: Use loadfile that allows reading from local file or uri (#558)

* feat: Use loadfile that allows reading from local file or uri

close #553

bootstrap/config/config.go

@@ -20,14 +20,15 @@ import (
 	"errors"
 	"fmt"
 	"math"
-	"os"
+	"net/url"
 	"path/filepath"
 	"reflect"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/file"
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/utils"
 	"github.com/edgexfoundry/go-mod-core-contracts/v3/common"
 	"github.com/mitchellh/copystructure"
@@ -637,8 +638,10 @@ func CreateProviderClient(
 
 // loadConfigYamlFromFile attempts to read the specified configuration yaml file
 func (cp *Processor) loadConfigYamlFromFile(yamlFile string) (map[string]any, error) {
+	secretProvider := container.SecretProviderExtFrom(cp.dic.Get)
+
 	cp.lc.Infof("Loading configuration file from %s", yamlFile)
-	contents, err := os.ReadFile(yamlFile)
+	contents, err := file.Load(yamlFile, secretProvider, cp.lc)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read configuration file %s: %s", yamlFile, err.Error())
 	}
@@ -654,10 +657,21 @@ func (cp *Processor) loadConfigYamlFromFile(yamlFile string) (map[string]any, er
 
 // GetConfigFileLocation uses the environment variables and flags to determine the location of the configuration
 func GetConfigFileLocation(lc logger.LoggingClient, flags flags.Common) string {
-	configDir := environment.GetConfigDir(lc, flags.ConfigDirectory())
-	profileDir := environment.GetProfileDir(lc, flags.Profile())
 	configFileName := environment.GetConfigFileName(lc, flags.ConfigFileName())
 
+	// Check for uri path
+	parsedUrl, err := url.Parse(configFileName)
+	if err != nil {
+		lc.Errorf("Could not parse file path: %v", err)
+		return ""
+	}
+
+	if parsedUrl.Scheme == "http" || parsedUrl.Scheme == "https" {
+		return configFileName
+	}
+
+	configDir := environment.GetConfigDir(lc, flags.ConfigDirectory())
+	profileDir := environment.GetProfileDir(lc, flags.Profile())
 	return filepath.Join(configDir, profileDir, configFileName)
 }
 

bootstrap/config/config_test.go

@@ -378,21 +378,65 @@ func TestFindChangedKey(t *testing.T) {
 }
 
 func TestGetConfigFileLocation(t *testing.T) {
-	dir := "myRes"
-	profile := "myProfile"
-	file := "myFile.yaml"
-	expected := filepath.Join(dir, profile, file)
-	defer os.Clearenv()
+	tests := []struct {
+		name       string
+		dir        string
+		profile    string
+		path       string
+		secretName string
+		expected   string
+	}{
+		{
+			name:     "valid - file",
+			dir:      "myRes",
+			profile:  "myProfile",
+			path:     "myFile.yaml",
+			expected: filepath.Join("myRes", "myProfile", "myFile.yaml"),
+		},
+		{
+			name:     "valid - file absolute path",
+			dir:      "/myRes",
+			profile:  "myProfile",
+			path:     "myFile.yaml",
+			expected: "/myRes/myProfile/myFile.yaml",
+		},
+		{
+			name:     "valid - file relative path",
+			dir:      "../../myRes",
+			profile:  "myProfile",
+			path:     "myFile.yaml",
+			expected: "../../myRes/myProfile/myFile.yaml",
+		},
+		{
+			name:     "valid - url",
+			dir:      "",
+			profile:  "",
+			path:     "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml",
+			expected: "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml",
+		},
+		{
+			name:     "invalid - url",
+			dir:      "",
+			profile:  "",
+			path:     "{test:\"test\"}",
+			expected: "",
+		},
+	}
 
-	lc := logger.NewMockClient()
-	flags := flags.New()
+	for _, test := range tests {
+		t.Run(test.secretName, func(t *testing.T) {
+			lc := logger.NewMockClient()
+			flags := flags.New()
 
-	os.Setenv("EDGEX_CONFIG_DIR", dir)
-	os.Setenv("EDGEX_PROFILE", profile)
-	os.Setenv("EDGEX_CONFIG_FILE", file)
+			defer os.Clearenv()
+			os.Setenv("EDGEX_CONFIG_DIR", test.dir)
+			os.Setenv("EDGEX_PROFILE", test.profile)
+			os.Setenv("EDGEX_CONFIG_FILE", test.path)
 
-	actual := GetConfigFileLocation(lc, flags)
-	assert.Equal(t, expected, actual)
+			actual := GetConfigFileLocation(lc, flags)
+			assert.Equal(t, test.expected, actual)
+		})
+	}
 }
 
 func TestGetInsecureSecretNameFullPath(t *testing.T) {

bootstrap/environment/variables.go

@@ -23,6 +23,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/utils"
 	"github.com/edgexfoundry/go-mod-core-contracts/v3/clients/logger"
@@ -44,6 +45,7 @@ const (
 	envKeyConfigDir       = "EDGEX_CONFIG_DIR"
 	envKeyProfile         = "EDGEX_PROFILE"
 	envKeyConfigFile      = "EDGEX_CONFIG_FILE"
+	envKeyFileURITimeout  = "EDGEX_FILE_URI_TIMEOUT"
 
 	noConfigProviderValue = "none"
 
@@ -59,6 +61,8 @@ const (
 	insecureSecretsRegexStr = "^Writable\\.InsecureSecrets\\.[^.]+\\.Secrets\\..+$" //#nosec G101 -- This is a false positive
 	// redactedStr is the value to print for redacted variable values
 	redactedStr = "<redacted>"
+
+	defaultFileURITimeout = 15 * time.Second
 )
 
 var (
@@ -456,3 +460,21 @@ func logEnvironmentOverride(lc logger.LoggingClient, name string, key string, va
 	}
 	lc.Infof("Variables override of '%s' by environment variable: %s=%s", name, key, valueStr)
 }
+
+// GetURIRequestTimeout gets the configuration request timeout value from an environment variable (if it exists)
+// or uses default value.
+func GetURIRequestTimeout(lc logger.LoggingClient) time.Duration {
+	envValue := os.Getenv(envKeyFileURITimeout)
+	if len(envValue) <= 0 {
+		return defaultFileURITimeout
+	}
+
+	requestTimeout, err := time.ParseDuration(envValue)
+	if err != nil {
+		lc.Warnf("Could not parse value for %s = %s: %v. Using default of %s", envKeyFileURITimeout, envValue, err, defaultFileURITimeout)
+		return defaultFileURITimeout
+	}
+
+	lc.Infof("Variables override of 'URI Request Timeout' by environment variable: %s=%s", envKeyFileURITimeout, envValue)
+	return requestTimeout
+}

bootstrap/environment/variables_test.go

@@ -21,6 +21,7 @@ import (
 	"strconv"
 	"strings"
 	"testing"
+	"time"
 
 	loggerMocks "github.com/edgexfoundry/go-mod-core-contracts/v3/clients/logger/mocks"
 	"github.com/stretchr/testify/mock"
@@ -586,3 +587,30 @@ func TestOverrideConfigMapValues(t *testing.T) {
 		})
 	}
 }
+
+func TestGetRequestTimeout(t *testing.T) {
+	_, lc := initializeTest()
+
+	testCases := []struct {
+		TestName        string
+		EnvTimeout      string
+		ExpectedTimeout time.Duration
+	}{
+		{"With Env Var", envKeyFileURITimeout, 15 * time.Second},
+		{"With No Env Var", "", 15 * time.Second},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.TestName, func(t *testing.T) {
+			os.Clearenv()
+
+			if len(test.EnvTimeout) > 0 {
+				err := os.Setenv(test.EnvTimeout, test.ExpectedTimeout.String())
+				require.NoError(t, err)
+			}
+
+			actual := GetURIRequestTimeout(lc)
+			assert.Equal(t, test.ExpectedTimeout, actual)
+		})
+	}
+}

bootstrap/file/file.go

@@ -6,12 +6,13 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"time"
 
+	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/environment"
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/interfaces"
+	"github.com/edgexfoundry/go-mod-core-contracts/v3/clients/logger"
 )
 
-func Load(path string, timeout time.Duration, provider interfaces.SecretProvider) ([]byte, error) {
+func Load(path string, provider interfaces.SecretProvider, lc logger.LoggingClient) ([]byte, error) {
 	var fileBytes []byte
 	var err error
 
@@ -22,7 +23,7 @@ func Load(path string, timeout time.Duration, provider interfaces.SecretProvider
 
 	if parsedUrl.Scheme == "http" || parsedUrl.Scheme == "https" {
 		client := &http.Client{
-			Timeout: timeout,
+			Timeout: environment.GetURIRequestTimeout(lc),
 		}
 		req, err := http.NewRequest("GET", path, nil)
 		if err != nil {

bootstrap/file/file_test.go

@@ -3,17 +3,18 @@ package file
 import (
 	"path"
 	"testing"
-	"time"
 
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/container"
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/interfaces/mocks"
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/di"
+	"github.com/edgexfoundry/go-mod-core-contracts/v3/clients/logger"
 	"github.com/stretchr/testify/assert"
 )
 
 var dic *di.Container
 
 func TestLoadFile(t *testing.T) {
+	lc := logger.NewMockClient()
 	tests := []struct {
 		Name               string
 		Path               string
@@ -25,11 +26,13 @@ func TestLoadFile(t *testing.T) {
 		{"Valid - load from JSON file", path.Join(".", "testdata", "configuration.json"), 142, "", nil},
 		{"Valid - load from HTTP", "http://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml", 4533, "", nil},
 		{"Valid - load from HTTPS", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml", 4533, "", nil},
+		{"Valid - load from HTTPS with secret", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml?edgexSecretName=mySecretName", 4533, "", map[string]string{"type": "httpheader", "headername": "Authorization", "headercontents": "Basic 1234567890"}},
 		{"Invalid - File not found", "bogus", 0, "Could not read file", nil},
+		{"Invalid - parse uri fail", "{test:\"test\"}", 0, "Could not parse file path", nil},
 		{"Invalid - load from invalid HTTP", "http://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/configuration.yaml", 1, "Invalid status code", nil},
 		{"Invalid - load from invalid HTTPS", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/configuration.yaml", 1, "Invalid status code", nil},
-		{"Valid - load from HTTPS with secret", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml?edgexSecretName=mySecretName", 4533, "", map[string]string{"type": "httpheader", "headername": "Authorization", "headercontents": "Basic 1234567890"}},
 		{"Invalid - load from HTTPS with invalid secret", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml?edgexSecretName=mySecretName", 4533, "Secret type is not httpheader", map[string]string{"type": "invalidheader", "headername": "Authorization", "headercontents": "Basic 1234567890"}},
+		{"Invalid - load from HTTPS with empty secret", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml?edgexSecretName=mySecretName", 0, "Secret headername and headercontents can not be empty", map[string]string{"type": "httpheader", "headername": "", "headercontents": ""}},
 	}
 
 	for _, tc := range tests {
@@ -45,7 +48,7 @@ func TestLoadFile(t *testing.T) {
 		})
 
 		t.Run(tc.Name, func(t *testing.T) {
-			bytesOut, err := Load(tc.Path, 10*time.Second, mockSecretProvider)
+			bytesOut, err := Load(tc.Path, mockSecretProvider, lc)
 			if tc.ExpectedErr != "" {
 				assert.Contains(t, err.Error(), tc.ExpectedErr)
 				return