feat: Use loadfile that allows reading Config from local file or uri

bootstrap/config/config.go

@@ -20,14 +20,15 @@ import (
 	"errors"
 	"fmt"
 	"math"
-	"os"
+	"net/url"
 	"path/filepath"
 	"reflect"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/file"
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/utils"
 	"github.com/edgexfoundry/go-mod-core-contracts/v3/common"
 	"github.com/mitchellh/copystructure"
@@ -62,6 +63,8 @@ const (
 // UpdatedStream defines the stream type that is notified by ListenForChanges when a configuration update is received.
 type UpdatedStream chan struct{}
 
+var defaultTimeout = 15 * time.Second
+
 type Processor struct {
 	lc                 logger.LoggingClient
 	flags              flags.Common
@@ -197,7 +200,7 @@ func (cp *Processor) Process(
 		// NOTE: Some security services don't use any common configuration and don't use the configuration provider.
 		commonConfigLocation := environment.GetCommonConfigFileName(cp.lc, cp.flags.CommonConfig())
 		if commonConfigLocation != "" {
-			err := cp.loadCommonConfigFromFile(commonConfigLocation, serviceConfig, serviceType)
+			err := cp.loadCommonConfigFromFile(commonConfigLocation, serviceConfig, serviceType, secretProvider)
 			if err != nil {
 				return err
 			}
@@ -212,8 +215,14 @@ func (cp *Processor) Process(
 
 	// Now load the private config from a local file if any of these conditions are true
 	if !useProvider || !cp.providerHasConfig || cp.overwriteConfig {
+		requestTimeout, err := time.ParseDuration(environment.GetRequestTimeout(cp.lc, cp.flags.RequestTimeout()))
+		if err != nil {
+			cp.lc.Debug("Failed to parse Service.RequestTimeout configuration value: %v, using default value", err)
+			requestTimeout = 15 * time.Second
+		}
+
 		filePath := GetConfigFileLocation(cp.lc, cp.flags)
-		configMap, err := cp.loadConfigYamlFromFile(filePath)
+		configMap, err := cp.loadConfigYamlFromFile(filePath, requestTimeout, secretProvider)
 		if err != nil {
 			return err
 		}
@@ -393,11 +402,12 @@ func (cp *Processor) loadCommonConfig(
 func (cp *Processor) loadCommonConfigFromFile(
 	configFile string,
 	serviceConfig interfaces.Configuration,
-	serviceType string) error {
+	serviceType string,
+	secretProvider interfaces.SecretProviderExt) error {
 
 	var err error
 
-	commonConfig, err := cp.loadConfigYamlFromFile(configFile)
+	commonConfig, err := cp.loadConfigYamlFromFile(configFile, defaultTimeout, secretProvider)
 	if err != nil {
 		return err
 	}
@@ -465,7 +475,7 @@ func (cp *Processor) getAccessTokenCallback(serviceKey string, secretProvider in
 // LoadCustomConfigSection loads the specified custom configuration section from file or Configuration provider.
 // Section will be seed if Configuration provider does yet have it. This is used for structures custom configuration
 // in App and Device services
-func (cp *Processor) LoadCustomConfigSection(updatableConfig interfaces.UpdatableConfig, sectionName string) error {
+func (cp *Processor) LoadCustomConfigSection(updatableConfig interfaces.UpdatableConfig, sectionName string, secretProvider interfaces.SecretProviderExt) error {
 	if cp.envVars == nil {
 		cp.envVars = environment.NewVariables(cp.lc)
 	}
@@ -474,7 +484,7 @@ func (cp *Processor) LoadCustomConfigSection(updatableConfig interfaces.Updatabl
 	if configClient == nil {
 		cp.lc.Info("Skipping use of Configuration Provider for custom configuration: Provider not available")
 		filePath := GetConfigFileLocation(cp.lc, cp.flags)
-		configMap, err := cp.loadConfigYamlFromFile(filePath)
+		configMap, err := cp.loadConfigYamlFromFile(filePath, defaultTimeout, secretProvider)
 		if err != nil {
 			return err
 		}
@@ -508,7 +518,7 @@ func (cp *Processor) LoadCustomConfigSection(updatableConfig interfaces.Updatabl
 			cp.lc.Info("Loaded custom configuration from Configuration Provider, no overrides applied")
 		} else {
 			filePath := GetConfigFileLocation(cp.lc, cp.flags)
-			configMap, err := cp.loadConfigYamlFromFile(filePath)
+			configMap, err := cp.loadConfigYamlFromFile(filePath, defaultTimeout, secretProvider)
 			if err != nil {
 				return err
 			}
@@ -636,9 +646,9 @@ func CreateProviderClient(
 }
 
 // loadConfigYamlFromFile attempts to read the specified configuration yaml file
-func (cp *Processor) loadConfigYamlFromFile(yamlFile string) (map[string]any, error) {
+func (cp *Processor) loadConfigYamlFromFile(yamlFile string, timeout time.Duration, provider interfaces.SecretProvider) (map[string]any, error) {
 	cp.lc.Infof("Loading configuration file from %s", yamlFile)
-	contents, err := os.ReadFile(yamlFile)
+	contents, err := file.Load(yamlFile, timeout, provider)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read configuration file %s: %s", yamlFile, err.Error())
 	}
@@ -654,10 +664,21 @@ func (cp *Processor) loadConfigYamlFromFile(yamlFile string) (map[string]any, er
 
 // GetConfigFileLocation uses the environment variables and flags to determine the location of the configuration
 func GetConfigFileLocation(lc logger.LoggingClient, flags flags.Common) string {
-	configDir := environment.GetConfigDir(lc, flags.ConfigDirectory())
-	profileDir := environment.GetProfileDir(lc, flags.Profile())
 	configFileName := environment.GetConfigFileName(lc, flags.ConfigFileName())
 
+	// Check for uri path
+	parsedUrl, err := url.Parse(configFileName)
+	if err != nil {
+		lc.Errorf("Could not parse file path: %v", err)
+		return ""
+	}
+
+	if parsedUrl.Scheme == "http" || parsedUrl.Scheme == "https" {
+		return configFileName
+	}
+
+	configDir := environment.GetConfigDir(lc, flags.ConfigDirectory())
+	profileDir := environment.GetProfileDir(lc, flags.Profile())
 	return filepath.Join(configDir, profileDir, configFileName)
 }
 

bootstrap/config/config_test.go

@@ -299,7 +299,7 @@ func TestLoadCommonConfigFromFile(t *testing.T) {
 			proc := NewProcessor(f, env, timer, ctx, &wg, nil, dic)
 
 			// call load common config
-			err := proc.loadCommonConfigFromFile(tc.config, tc.serviceConfig, tc.serviceType)
+			err := proc.loadCommonConfigFromFile(tc.config, tc.serviceConfig, tc.serviceType, nil)
 			// make assertions
 			require.NotNil(t, cancel)
 			if tc.expectedErr == "" {
@@ -378,21 +378,51 @@ func TestFindChangedKey(t *testing.T) {
 }
 
 func TestGetConfigFileLocation(t *testing.T) {
-	dir := "myRes"
-	profile := "myProfile"
-	file := "myFile.yaml"
-	expected := filepath.Join(dir, profile, file)
-	defer os.Clearenv()
+	tests := []struct {
+		name       string
+		dir        string
+		profile    string
+		path       string
+		secretName string
+		expected   string
+	}{
+		{
+			name:     "valid - file",
+			dir:      "myRes",
+			profile:  "myProfile",
+			path:     "myFile.yaml",
+			expected: filepath.Join("myRes", "myProfile", "myFile.yaml"),
+		},
+		{
+			name:     "valid - url",
+			dir:      "",
+			profile:  "",
+			path:     "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml",
+			expected: "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml",
+		},
+		{
+			name:     "invalid - url",
+			dir:      "",
+			profile:  "",
+			path:     "{test:\"test\"}",
+			expected: "",
+		},
+	}
 
-	lc := logger.NewMockClient()
-	flags := flags.New()
+	for _, test := range tests {
+		t.Run(test.secretName, func(t *testing.T) {
+			lc := logger.NewMockClient()
+			flags := flags.New()
 
-	os.Setenv("EDGEX_CONFIG_DIR", dir)
-	os.Setenv("EDGEX_PROFILE", profile)
-	os.Setenv("EDGEX_CONFIG_FILE", file)
+			defer os.Clearenv()
+			os.Setenv("EDGEX_CONFIG_DIR", test.dir)
+			os.Setenv("EDGEX_PROFILE", test.profile)
+			os.Setenv("EDGEX_CONFIG_FILE", test.path)
 
-	actual := GetConfigFileLocation(lc, flags)
-	assert.Equal(t, expected, actual)
+			actual := GetConfigFileLocation(lc, flags)
+			assert.Equal(t, test.expected, actual)
+		})
+	}
 }
 
 func TestGetInsecureSecretNameFullPath(t *testing.T) {

bootstrap/environment/variables.go

@@ -44,6 +44,7 @@ const (
 	envKeyConfigDir       = "EDGEX_CONFIG_DIR"
 	envKeyProfile         = "EDGEX_PROFILE"
 	envKeyConfigFile      = "EDGEX_CONFIG_FILE"
+	envKeyRequestTimeout  = "EDGEX_SERVICE_REQUESTTIMEOUT"
 
 	noConfigProviderValue = "none"
 
@@ -456,3 +457,15 @@ func logEnvironmentOverride(lc logger.LoggingClient, name string, key string, va
 	}
 	lc.Infof("Variables override of '%s' by environment variable: %s=%s", name, key, valueStr)
 }
+
+// GetRequestTimeout gets the configuration request timeout value from a Variables variable value (if it exists)
+// or uses passed in value.
+func GetRequestTimeout(lc logger.LoggingClient, requestTimeout string) string {
+	envValue := os.Getenv(envKeyRequestTimeout)
+	if len(envValue) > 0 {
+		requestTimeout = envValue
+		logEnvironmentOverride(lc, "-rt/--requestTimeout", envKeyRequestTimeout, envValue)
+	}
+
+	return requestTimeout
+}

bootstrap/environment/variables_test.go

@@ -586,3 +586,32 @@ func TestOverrideConfigMapValues(t *testing.T) {
 		})
 	}
 }
+
+func TestGetRequestTimeout(t *testing.T) {
+	_, lc := initializeTest()
+
+	testCases := []struct {
+		TestName        string
+		EnvTimeout      string
+		PassedInTimeout string
+		ExpectedTimeout string
+	}{
+		{"With Env Var", envKeyRequestTimeout, "15", "14"},
+		{"With No Env Var", "", "15", "15"},
+		{"With No Env Var and no passed in", "", "", ""},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.TestName, func(t *testing.T) {
+			os.Clearenv()
+
+			if len(test.EnvTimeout) > 0 {
+				err := os.Setenv(test.EnvTimeout, test.ExpectedTimeout)
+				require.NoError(t, err)
+			}
+
+			actual := GetRequestTimeout(lc, test.PassedInTimeout)
+			assert.Equal(t, test.ExpectedTimeout, actual)
+		})
+	}
+}

bootstrap/file/file.go

@@ -11,6 +11,8 @@ import (
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/interfaces"
 )
 
+const DefaultTimeout = 15 * time.Second
+
 func Load(path string, timeout time.Duration, provider interfaces.SecretProvider) ([]byte, error) {
 	var fileBytes []byte
 	var err error

bootstrap/file/file_test.go

@@ -3,7 +3,6 @@ package file
 import (
 	"path"
 	"testing"
-	"time"
 
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/container"
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/interfaces/mocks"
@@ -25,11 +24,13 @@ func TestLoadFile(t *testing.T) {
 		{"Valid - load from JSON file", path.Join(".", "testdata", "configuration.json"), 142, "", nil},
 		{"Valid - load from HTTP", "http://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml", 4533, "", nil},
 		{"Valid - load from HTTPS", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml", 4533, "", nil},
+		{"Valid - load from HTTPS with secret", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml?edgexSecretName=mySecretName", 4533, "", map[string]string{"type": "httpheader", "headername": "Authorization", "headercontents": "Basic 1234567890"}},
 		{"Invalid - File not found", "bogus", 0, "Could not read file", nil},
+		{"Invalid - parse uri fail", "{test:\"test\"}", 0, "Could not parse file path", nil},
 		{"Invalid - load from invalid HTTP", "http://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/configuration.yaml", 1, "Invalid status code", nil},
 		{"Invalid - load from invalid HTTPS", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/configuration.yaml", 1, "Invalid status code", nil},
-		{"Valid - load from HTTPS with secret", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml?edgexSecretName=mySecretName", 4533, "", map[string]string{"type": "httpheader", "headername": "Authorization", "headercontents": "Basic 1234567890"}},
 		{"Invalid - load from HTTPS with invalid secret", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml?edgexSecretName=mySecretName", 4533, "Secret type is not httpheader", map[string]string{"type": "invalidheader", "headername": "Authorization", "headercontents": "Basic 1234567890"}},
+		{"Invalid - load from HTTPS with empty secret", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml?edgexSecretName=mySecretName", 0, "Secret headername and headercontents can not be empty", map[string]string{"type": "httpheader", "headername": "", "headercontents": ""}},
 	}
 
 	for _, tc := range tests {
@@ -45,7 +46,7 @@ func TestLoadFile(t *testing.T) {
 		})
 
 		t.Run(tc.Name, func(t *testing.T) {
-			bytesOut, err := Load(tc.Path, 10*time.Second, mockSecretProvider)
+			bytesOut, err := Load(tc.Path, DefaultTimeout, mockSecretProvider)
 			if tc.ExpectedErr != "" {
 				assert.Contains(t, err.Error(), tc.ExpectedErr)
 				return

bootstrap/flags/flags.go

@@ -24,6 +24,7 @@ import (
 const (
 	DefaultConfigProvider = "consul.http://localhost:8500"
 	DefaultConfigFile     = "configuration.yaml"
+	DefaultRequestTimeout = "15"
 )
 
 // Common is an interface that defines AP for the common command-line flags used by most EdgeX services
@@ -35,6 +36,7 @@ type Common interface {
 	Profile() string
 	ConfigDirectory() string
 	ConfigFileName() string
+	RequestTimeout() string
 	CommonConfig() string
 	Parse([]string)
 	Help()
@@ -52,6 +54,7 @@ type Default struct {
 	profile           string
 	configDir         string
 	configFileName    string
+	requestTimeout    string
 }
 
 // NewWithUsage returns a Default struct.
@@ -93,6 +96,8 @@ func (d *Default) Parse(arguments []string) {
 	d.FlagSet.BoolVar(&d.overwriteConfig, "o", false, "")
 	d.FlagSet.StringVar(&d.configFileName, "cf", DefaultConfigFile, "")
 	d.FlagSet.StringVar(&d.configFileName, "configFile", DefaultConfigFile, "")
+	d.FlagSet.StringVar(&d.requestTimeout, "rt", DefaultRequestTimeout, "")
+	d.FlagSet.StringVar(&d.requestTimeout, "requestTimeout", DefaultRequestTimeout, "")
 	d.FlagSet.StringVar(&d.profile, "profile", "", "")
 	d.FlagSet.StringVar(&d.profile, "p", "", ".")
 	d.FlagSet.StringVar(&d.configDir, "configDir", "", "")
@@ -151,6 +156,11 @@ func (d *Default) CommonConfig() string {
 	return d.commonConfig
 }
 
+// CommonConfig returns the location for the common configuration
+func (d *Default) RequestTimeout() string {
+	return d.requestTimeout
+}
+
 // Help displays the usage help message and exit.
 func (d *Default) Help() {
 	d.helpCallback()
@@ -169,6 +179,7 @@ func (d *Default) helpCallback() {
 			"                                    *** Use with cation *** Use will clobber existing settings in provider,\n"+
 			"                                    problematic if those settings were edited by hand intentionally\n"+
 			"    -cf, --configFile <name>        Indicates name of the local configuration file. Defaults to configuration.toml\n"+
+			"    -rt, --requestTimeout <seconds> Set the uri file load request timeout\n"+
 			"    -p, --profile <name>            Indicate configuration profile other than default\n"+
 			"    -cd, --configDir                Specify local configuration directory\n"+
 			"    -r, --registry                  Indicates service should use Registry.\n"+