feat: Implement loading from URI for Profile, Device, & Provision Wat…

…cher files (#1471)

Closes #1467

Signed-off-by: Elizabeth J Lee <[email protected]>

example/cmd/device-simple/res/configuration.yaml

@@ -19,8 +19,8 @@ MessageBus:
 Device:
   AsyncBufferSize: 1
   # These have common values (currently), but must be here for service local env overrides to apply when customized
-  ProfilesDir: "./res/profiles"
-  DevicesDir: "./res/devices"
+  ProfilesDir: ./res/profiles
+  DevicesDir: ./res/devices
   # Only needed if device service implements auto provisioning
   ProvisionWatchersDir: ./res/provisionwatchers
 # Example structured custom configuration

internal/provision/common.go

@@ -0,0 +1,50 @@
+// -*- Mode: Go; indent-tabs-mode: t -*-
+//
+// # Copyright (C) 2023 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+package provision
+
+import (
+	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/utils"
+	"github.com/edgexfoundry/go-mod-core-contracts/v3/clients/logger"
+	"net/url"
+	"path"
+	"strings"
+)
+
+type FileType int
+
+const (
+	YAML FileType = iota
+	JSON
+	OTHER
+)
+
+func GetFileType(fullPath string) FileType {
+	if strings.HasSuffix(fullPath, yamlExt) || strings.HasSuffix(fullPath, ymlExt) {
+		return YAML
+	} else if strings.HasSuffix(fullPath, ".json") {
+		return JSON
+	} else {
+		return OTHER
+	}
+}
+
+func GetFullAndRedactedURI(baseURI *url.URL, file, description string, lc logger.LoggingClient) (string, string) {
+	basePath, _ := path.Split(baseURI.Path)
+	newPath, err := url.JoinPath(basePath, file)
+	if err != nil {
+		lc.Error("could not join URI path for %s %s/%s: %v", description, basePath, file, err)
+		return "", ""
+	}
+	var fullURI url.URL
+	err = utils.DeepCopy(baseURI, &fullURI)
+	if err != nil {
+		lc.Error("could not copy URI for %s %s: %v", description, newPath, err)
+		return "", ""
+	}
+	fullURI.User = baseURI.User
+	fullURI.Path = newPath
+	return fullURI.String(), fullURI.Redacted()
+}

internal/provision/common_test.go

@@ -0,0 +1,57 @@
+// -*- Mode: Go; indent-tabs-mode: t -*-
+//
+// # Copyright (C) 2023 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+package provision
+
+import (
+	"github.com/edgexfoundry/go-mod-core-contracts/v3/clients/logger"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"net/url"
+	"path"
+	"testing"
+)
+
+func Test_GetFileType(t *testing.T) {
+	tests := []struct {
+		name             string
+		path             string
+		expectedFileType FileType
+	}{
+		{"valid get Yaml file type", path.Join("..", "..", "example", "cmd", "device-simple", "res", "devices", "simple-device.yml"), YAML},
+		{"valid get Json file type", path.Join("..", "..", "example", "cmd", "device-simple", "res", "devices", "simple-device.json"), JSON},
+		{"valid get other file type", path.Join("..", "..", "example", "cmd", "device-simple", "res", "devices", "simple-device.bogus"), OTHER},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actualType := GetFileType(tt.path)
+			assert.Equal(t, tt.expectedFileType, actualType)
+		})
+	}
+}
+
+func Test_GetFullAndRedactedURI(t *testing.T) {
+	tests := []struct {
+		name             string
+		baseURI          string
+		file             string
+		expectedURI      string
+		expectedRedacted string
+	}{
+		{"valid no secret uri", "https://raw.githubusercontent.com/edgexfoundry/device-virtual-go/main/cmd/res/devices/devices.yaml", "device-simple.yaml", "https://raw.githubusercontent.com/edgexfoundry/device-virtual-go/main/cmd/res/devices/device-simple.yaml", "https://raw.githubusercontent.com/edgexfoundry/device-virtual-go/main/cmd/res/devices/device-simple.yaml"},
+		{"valid query secret uri", "https://raw.githubusercontent.com/edgexfoundry/device-simple/main/devices/index.json?edgexSecretName=githubCredentials", "device-simple.yaml", "https://raw.githubusercontent.com/edgexfoundry/device-simple/main/devices/device-simple.yaml?edgexSecretName=githubCredentials", "https://raw.githubusercontent.com/edgexfoundry/device-simple/main/devices/device-simple.yaml?edgexSecretName=githubCredentials"},
+		{"valid query secret uri", "https://myuser:[email protected]/edgexfoundry/device-simple/main/devices/index.json", "device-simple.yaml", "https://myuser:[email protected]/edgexfoundry/device-simple/main/devices/device-simple.yaml", "https://myuser:[email protected]/edgexfoundry/device-simple/main/devices/device-simple.yaml"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			testURI, err := url.Parse(tt.baseURI)
+			require.NoError(t, err)
+			lc := logger.MockLogger{}
+			actualURI, actualRedacted := GetFullAndRedactedURI(testURI, tt.file, "test", lc)
+			assert.Equal(t, tt.expectedURI, actualURI)
+			assert.Equal(t, tt.expectedRedacted, actualRedacted)
+		})
+	}
+}

internal/provision/devices.go

@@ -2,6 +2,7 @@
 //
 // Copyright (C) 2017-2018 Canonical Ltd
 // Copyright (C) 2018-2023 IOTech Ltd
+// Copyright (C) 2023 Intel Corporation
 //
 // SPDX-License-Identifier: Apache-2.0
 
@@ -11,13 +12,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-
 	bootstrapContainer "github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/container"
+	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/file"
+	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/interfaces"
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/di"
+	"github.com/edgexfoundry/go-mod-core-contracts/v3/clients/logger"
 	"github.com/edgexfoundry/go-mod-core-contracts/v3/common"
 	"github.com/edgexfoundry/go-mod-core-contracts/v3/dtos"
 	"github.com/edgexfoundry/go-mod-core-contracts/v3/dtos/requests"
@@ -26,79 +25,38 @@ import (
 	"github.com/google/uuid"
 	"github.com/hashicorp/go-multierror"
 	"gopkg.in/yaml.v3"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
 
 	"github.com/edgexfoundry/device-sdk-go/v3/internal/cache"
 	"github.com/edgexfoundry/device-sdk-go/v3/internal/container"
 )
 
 func LoadDevices(path string, dic *di.Container) errors.EdgeX {
+	var addDevicesReq []requests.AddDeviceRequest
+	var edgexErr errors.EdgeX
 	if path == "" {
 		return nil
 	}
 
-	absPath, err := filepath.Abs(path)
-	if err != nil {
-		return errors.NewCommonEdgeX(errors.KindServerError, "failed to create absolute path", err)
-	}
-
-	files, err := os.ReadDir(absPath)
-	if err != nil {
-		return errors.NewCommonEdgeX(errors.KindServerError, "failed to read directory", err)
-	}
-
-	if len(files) == 0 {
-		return nil
-	}
-
 	lc := bootstrapContainer.LoggingClientFrom(dic.Get)
-	lc.Infof("Loading pre-defined devices from %s(%d files found)", absPath, len(files))
-
-	var addDevicesReq []requests.AddDeviceRequest
 	serviceName := container.DeviceServiceFrom(dic.Get).Name
-	for _, file := range files {
-		var devices []dtos.Device
-		fullPath := filepath.Join(absPath, file.Name())
-		if strings.HasSuffix(fullPath, yamlExt) || strings.HasSuffix(fullPath, ymlExt) {
-			content, err := os.ReadFile(fullPath)
-			if err != nil {
-				lc.Errorf("Failed to read %s: %v", fullPath, err)
-				continue
-			}
-			d := struct {
-				DeviceList []dtos.Device `yaml:"deviceList"`
-			}{}
-			err = yaml.Unmarshal(content, &d)
-			if err != nil {
-				lc.Errorf("Failed to YAML decode %s: %v", fullPath, err)
-				continue
-			}
-			devices = d.DeviceList
-		} else if strings.HasSuffix(fullPath, ".json") {
-			content, err := os.ReadFile(fullPath)
-			if err != nil {
-				lc.Errorf("Failed to read %s: %v", fullPath, err)
-				continue
-			}
-			err = json.Unmarshal(content, &devices)
-			if err != nil {
-				lc.Errorf("Failed to JSON decode %s: %v", fullPath, err)
-				continue
-			}
-		} else {
-			continue
+	parsedUrl, err := url.Parse(path)
+	if err != nil {
+		return errors.NewCommonEdgeX(errors.KindServerError, "failed to parse Devices path as a URI", err)
+	}
+	if parsedUrl.Scheme == "http" || parsedUrl.Scheme == "https" {
+		secretProvider := bootstrapContainer.SecretProviderFrom(dic.Get)
+		addDevicesReq, edgexErr = loadDevicesFromURI(path, parsedUrl, serviceName, secretProvider, lc)
+		if edgexErr != nil {
+			return edgexErr
 		}
-
-		for _, device := range devices {
-			if _, ok := cache.Devices().ForName(device.Name); ok {
-				lc.Infof("Device %s exists, using the existing one", device.Name)
-			} else {
-				lc.Infof("Device %s not found in Metadata, adding it ...", device.Name)
-				device.ServiceName = serviceName
-				device.AdminState = models.Unlocked
-				device.OperatingState = models.Up
-				req := requests.NewAddDeviceRequest(device)
-				addDevicesReq = append(addDevicesReq, req)
-			}
+	} else {
+		addDevicesReq, edgexErr = loadDevicesFromFile(path, serviceName, lc)
+		if edgexErr != nil {
+			return edgexErr
 		}
 	}
 
@@ -116,11 +74,11 @@ func LoadDevices(path string, dic *di.Container) errors.EdgeX {
 	for _, response := range responses {
 		if response.StatusCode != http.StatusCreated {
 			if response.StatusCode == http.StatusConflict {
-				lc.Warnf("%s. Device may be owned by other device service instance.", response.Message)
+				lc.Warnf("%s. Device may be owned by other Device service instance.", response.Message)
 				continue
 			}
 
-			err = multierror.Append(err, fmt.Errorf("add device failed: %s", response.Message))
+			err = multierror.Append(err, fmt.Errorf("add Device failed: %s", response.Message))
 		}
 	}
 
@@ -130,3 +88,111 @@ func LoadDevices(path string, dic *di.Container) errors.EdgeX {
 
 	return nil
 }
+
+func loadDevicesFromFile(path, serviceName string, lc logger.LoggingClient) ([]requests.AddDeviceRequest, errors.EdgeX) {
+	absPath, err := filepath.Abs(path)
+	if err != nil {
+		return nil, errors.NewCommonEdgeX(errors.KindServerError, "failed to create absolute path for Devices", err)
+	}
+
+	files, err := os.ReadDir(absPath)
+	if err != nil {
+		return nil, errors.NewCommonEdgeX(errors.KindServerError, "failed to read directory for Devices", err)
+	}
+
+	if len(files) == 0 {
+		return nil, nil
+	}
+
+	lc.Infof("Loading pre-defined Devices from %s(%d files found)", absPath, len(files))
+	var addDevicesReq, processedDevicesReq []requests.AddDeviceRequest
+	for _, file := range files {
+		fullPath := filepath.Join(absPath, file.Name())
+		processedDevicesReq = processDevices(fullPath, fullPath, serviceName, nil, lc)
+		if len(processedDevicesReq) > 0 {
+			addDevicesReq = append(addDevicesReq, processedDevicesReq...)
+		}
+	}
+	return addDevicesReq, nil
+}
+
+func loadDevicesFromURI(inputURI string, parsedURI *url.URL, serviceName string, secretProvider interfaces.SecretProvider, lc logger.LoggingClient) ([]requests.AddDeviceRequest, errors.EdgeX) {
+	// the input URI contains the index file containing the Device list to be loaded
+	bytes, err := file.Load(inputURI, secretProvider, lc)
+	if err != nil {
+		return nil, errors.NewCommonEdgeX(errors.KindServerError, fmt.Sprintf("failed to load Devices list from URI %s", parsedURI.Redacted()), err)
+	}
+
+	var files []string
+	err = json.Unmarshal(bytes, &files)
+	if err != nil {
+		return nil, errors.NewCommonEdgeX(errors.KindServerError, "could not unmarshal Devices list contents", err)
+	}
+
+	if len(files) == 0 {
+		lc.Infof("Index file %s for Devices list is empty", parsedURI.Redacted())
+		return nil, nil
+	}
+
+	lc.Infof("Loading pre-defined devices from %s(%d files found)", parsedURI.Redacted(), len(files))
+	var addDevicesReq, processedDevicesReq []requests.AddDeviceRequest
+	for _, file := range files {
+		fullPath, redactedPath := GetFullAndRedactedURI(parsedURI, file, "Device", lc)
+		processedDevicesReq = processDevices(fullPath, redactedPath, serviceName, secretProvider, lc)
+		if len(processedDevicesReq) > 0 {
+			addDevicesReq = append(addDevicesReq, processedDevicesReq...)
+		}
+	}
+	return addDevicesReq, nil
+}
+
+func processDevices(fullPath, displayPath, serviceName string, secretProvider interfaces.SecretProvider, lc logger.LoggingClient) []requests.AddDeviceRequest {
+	var devices []dtos.Device
+	var addDevicesReq []requests.AddDeviceRequest
+
+	fileType := GetFileType(fullPath)
+
+	// if the file type is not yaml or json, it cannot be parsed - just return to not break the loop for other devices
+	if fileType == OTHER {
+		return nil
+	}
+
+	content, err := file.Load(fullPath, secretProvider, lc)
+	if err != nil {
+		lc.Errorf("Failed to read Devices from %s: %v", displayPath, err)
+		return nil
+	}
+
+	switch fileType {
+	case YAML:
+		d := struct {
+			DeviceList []dtos.Device `yaml:"deviceList"`
+		}{}
+		err = yaml.Unmarshal(content, &d)
+		if err != nil {
+			lc.Errorf("Failed to YAML decode Devices from %s: %v", displayPath, err)
+			return nil
+		}
+		devices = d.DeviceList
+	case JSON:
+		err = json.Unmarshal(content, &devices)
+		if err != nil {
+			lc.Errorf("Failed to JSON decode Devices from %s: %v", displayPath, err)
+			return nil
+		}
+	}
+
+	for _, device := range devices {
+		if _, ok := cache.Devices().ForName(device.Name); ok {
+			lc.Infof("Device %s exists, using the existing one", device.Name)
+		} else {
+			lc.Infof("Device %s not found in Metadata, adding it ...", device.Name)
+			device.ServiceName = serviceName
+			device.AdminState = models.Unlocked
+			device.OperatingState = models.Up
+			req := requests.NewAddDeviceRequest(device)
+			addDevicesReq = append(addDevicesReq, req)
+		}
+	}
+	return addDevicesReq
+}