Skip to content
This repository has been archived by the owner on Jun 21, 2022. It is now read-only.

Commit

Permalink
PMM-4879 Adding support for defaults-file in new mysql service. (#1068)
Browse files Browse the repository at this point in the history
* PMM-4879 Adding support for defaults-file in new mysql service.

* PMM-4879 defaults-file adjustments.

* PMM-4879 Linter & tests fix.

* PMM-4879 Adding support for configuration entry: socket.

* PMM-4879 Code review adjustments.

* PMM-4879 Code review adjustments: replacing API dependencies.

* PMM-4879 Code review adjustments: replacing API dependencies.

* PMM-4879 Code review adjustments: fixing code for CI.

* PMM-4879 Added coverage tests.

* PMM-4879 Added coverage tests.

* PMM-4879 Code review adjustments.

* PMM-4879 Updating api-tests.

Co-authored-by: Alexey Mukas <[email protected]>
  • Loading branch information
pkadej and ritbl authored Jun 11, 2022
1 parent 05124a1 commit 1011fab
Show file tree
Hide file tree
Showing 14 changed files with 482 additions and 10 deletions.
10 changes: 8 additions & 2 deletions api-tests/management/mysql_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -596,6 +596,7 @@ func TestAddMySQL(t *testing.T) {
assert.Nil(t, addMySQLOK)
})

// according to defaultFiles parameter, passing empty username is allowed
t.Run("Empty username", func(t *testing.T) {
nodeName := pmmapitests.TestString(t, "node-name")
nodeID, pmmAgentID := RegisterGenericNode(t, node.RegisterNodeBody{
Expand All @@ -614,11 +615,16 @@ func TestAddMySQL(t *testing.T) {
Address: "10.10.10.10",
Port: 3306,
PMMAgentID: pmmAgentID,

SkipConnectionCheck: true,
},
}
addMySQLOK, err := client.Default.MySQL.AddMySQL(params)
pmmapitests.AssertAPIErrorf(t, err, 400, codes.InvalidArgument, "invalid field Username: value '' must not be an empty string")
assert.Nil(t, addMySQLOK)
require.NoError(t, err)
require.NotNil(t, addMySQLOK)
require.NotNil(t, addMySQLOK.Payload.Service)
serviceID := addMySQLOK.Payload.Service.ServiceID
defer pmmapitests.RemoveServices(t, serviceID)
})

t.Run("With MetricsModePush", func(t *testing.T) {
Expand Down
5 changes: 3 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.18

// Use for local development, but do not commit:
// replace github.com/percona/pmm => ../pmm
// replace github.com/percona/pmm => ../pmm
// replace github.com/percona-platform/saas => ../saas
// replace github.com/percona-platform/dbaas-api => ../dbaas-api

Expand Down Expand Up @@ -36,7 +37,7 @@ require (
github.com/minio/minio-go/v7 v7.0.27
github.com/percona-platform/dbaas-api v0.0.0-20220110092915-5aacd784d472
github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16
github.com/percona/pmm v0.0.0-20220607154345-cf9e6085e661
github.com/percona/pmm v0.0.0-20220609163826-96fb5593d69b
github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54
github.com/pkg/errors v0.9.1
github.com/pmezard/go-difflib v1.0.0
Expand Down Expand Up @@ -138,7 +139,7 @@ require (
golang.org/x/tools v0.1.9 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect
gopkg.in/ini.v1 v1.66.4 // indirect
gopkg.in/ini.v1 v1.66.6 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -467,8 +467,8 @@ github.com/percona-platform/dbaas-api v0.0.0-20220110092915-5aacd784d472 h1:Henk
github.com/percona-platform/dbaas-api v0.0.0-20220110092915-5aacd784d472/go.mod h1:WZZ3Hi+lAWCaGWmsrfkkvRQPkIa8n1OZ0s8Su+vbgus=
github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16 h1:0fx16uGtl4MwrBwm9/VSoNEhjL0cXYxS0quEhLthGcc=
github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16/go.mod h1:gFUwaFp6Ugu5qsBwiOVJYbDlzgZ77tmXdXGO7tG5xVI=
github.com/percona/pmm v0.0.0-20220607154345-cf9e6085e661 h1:xgAObd0x67UJdtEwnN5JeZFyCYQ9BCiGCnaBy4gB4F4=
github.com/percona/pmm v0.0.0-20220607154345-cf9e6085e661/go.mod h1:ix7lLnoQysi2ki1TxfWB3GSckkvER4nVRKVHo+zkagg=
github.com/percona/pmm v0.0.0-20220609163826-96fb5593d69b h1:b3MUqjYU1L85FExBgewV9xFluJEFRO129pRHskgxW+0=
github.com/percona/pmm v0.0.0-20220609163826-96fb5593d69b/go.mod h1:c22C8QvyFlcxr61TbqPlLGVC2u4xDptqYuYAoV0Umbs=
github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54 h1:aI1emmycDTGWKsBdxFPKZqohfBbK4y2ta9G4+RX7gVg=
github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54/go.mod h1:Y2uXi5QNk71+ceJHuI9poank+0S1kjxd3K105fXKVkg=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
Expand Down Expand Up @@ -938,8 +938,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4=
gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI=
gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/reform.v1 v1.5.1 h1:7vhDFW1n1xAPC6oDSvIvVvpRkaRpXlxgJ4QB4s3aDdo=
gopkg.in/reform.v1 v1.5.1/go.mod h1:AIv0CbDRJ0ljQwptGeaIXfpDRo02uJwTq92aMFELEeU=
gopkg.in/telebot.v3 v3.0.0/go.mod h1:7rExV8/0mDDNu9epSrDm/8j22KLaActH1Tbee6YjzWg=
Expand Down
5 changes: 4 additions & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ type gRPCServerDeps struct {
actions *agents.ActionsService
agentsStateUpdater *agents.StateUpdater
connectionCheck *agents.ConnectionChecker
defaultsFileParser *agents.DefaultsFileParser
grafanaClient *grafana.Client
checksService *checks.Service
dbaasClient *dbaas.Client
Expand Down Expand Up @@ -190,7 +191,7 @@ func runGRPCServer(ctx context.Context, deps *gRPCServerDeps) {

nodeSvc := management.NewNodeService(deps.db)
serviceSvc := management.NewServiceService(deps.db, deps.agentsStateUpdater, deps.vmdb)
mysqlSvc := management.NewMySQLService(deps.db, deps.agentsStateUpdater, deps.connectionCheck, deps.versionCache)
mysqlSvc := management.NewMySQLService(deps.db, deps.agentsStateUpdater, deps.connectionCheck, deps.versionCache, deps.defaultsFileParser)
mongodbSvc := management.NewMongoDBService(deps.db, deps.agentsStateUpdater, deps.connectionCheck)
postgresqlSvc := management.NewPostgreSQLService(deps.db, deps.agentsStateUpdater, deps.connectionCheck)
proxysqlSvc := management.NewProxySQLService(deps.db, deps.agentsStateUpdater, deps.connectionCheck)
Expand Down Expand Up @@ -737,6 +738,7 @@ func main() {
schedulerService := scheduler.New(db, backupService)
versionCache := versioncache.New(db, versioner)
emailer := alertmanager.NewEmailer(logrus.WithField("component", "alertmanager-emailer").Logger)
defaultsFileParser := agents.NewDefaultsFileParser(agentsRegistry)

serverParams := &server.Params{
DB: db,
Expand Down Expand Up @@ -937,6 +939,7 @@ func main() {
versionCache: versionCache,
supervisord: supervisord,
config: &cfg.Config,
defaultsFileParser: defaultsFileParser,
})
}()

Expand Down
26 changes: 26 additions & 0 deletions models/defaults_file.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// pmm-managed
// Copyright (C) 2017 Percona LLC
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.

package models

// ParseDefaultsFileResult contains result of parsing defaults file.
type ParseDefaultsFileResult struct {
Username string
Password string
Host string
Port uint32
Socket string
}
2 changes: 2 additions & 0 deletions services/agents/channel/channel.go
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,8 @@ func (c *Channel) runReceiver() {
c.publish(msg.Id, msg.Status, p.GetVersions)
case *agentpb.AgentMessage_PbmSwitchPitr:
c.publish(msg.Id, msg.Status, p.PbmSwitchPitr)
case *agentpb.AgentMessage_ParseDefaultsFile:
c.publish(msg.Id, msg.Status, p.ParseDefaultsFile)

case nil:
c.cancel(msg.Id, errors.Errorf("unimplemented: failed to handle received message %s", msg))
Expand Down
47 changes: 47 additions & 0 deletions services/agents/channel/channel_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -356,3 +356,50 @@ func TestUnexpectedResponsePayloadFromAgent(t *testing.T) {
close(stopServer)
<-stop
}

func TestChannelForDefaultsFileParser(t *testing.T) {
const count = 50
require.True(t, count > agentRequestsCap)

connect := func(ch *Channel) error {
testValue := "test"
testPort := uint32(123123)
for i := uint32(1); i <= count; i++ {
resp, err := ch.SendAndWaitResponse(&agentpb.ParseDefaultsFileRequest{})
assert.NotNil(t, resp)
parserResponse := resp.(*agentpb.ParseDefaultsFileResponse)
assert.Equal(t, parserResponse.Username, testValue)
assert.Equal(t, parserResponse.Password, testValue)
assert.Equal(t, parserResponse.Socket, testValue)
assert.Equal(t, parserResponse.Port, testPort)
assert.NoError(t, err)
}

assert.Nil(t, <-ch.Requests())
return nil
}

stream, _, teardown := setup(t, connect, io.EOF) // EOF = server exits from handler
defer teardown(t)

for i := uint32(1); i <= count; i++ {
msg, err := stream.Recv()
assert.NoError(t, err)
assert.Equal(t, i, msg.Id)
assert.NotNil(t, msg.GetParseDefaultsFile())

err = stream.Send(&agentpb.AgentMessage{
Id: i,
Payload: (&agentpb.ParseDefaultsFileResponse{
Username: "test",
Password: "test",
Port: 123123,
Socket: "test",
}).AgentMessageResponsePayload(),
})
assert.NoError(t, err)
}

err := stream.CloseSend()
assert.NoError(t, err)
}
98 changes: 98 additions & 0 deletions services/agents/parse_defaults_file.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
// pmm-managed
// Copyright (C) 2017 Percona LLC
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.

package agents

import (
"context"
"time"

"github.com/percona/pmm/api/agentpb"
"github.com/percona/pmm/api/inventorypb"
"github.com/pkg/errors"

"github.com/percona/pmm-managed/models"
"github.com/percona/pmm-managed/utils/logger"
)

// DefaultsFileParser requests from agent to parse defaultsFile.
type DefaultsFileParser struct {
r *Registry
}

// NewDefaultsFileParser creates new ParseDefaultsFile request.
func NewDefaultsFileParser(r *Registry) *DefaultsFileParser {
return &DefaultsFileParser{
r: r,
}
}

// ParseDefaultsFile sends request (with file path) to pmm-agent to parse defaults file.
func (p *DefaultsFileParser) ParseDefaultsFile(ctx context.Context, pmmAgentID, filePath string, serviceType models.ServiceType) (*models.ParseDefaultsFileResult, error) {
l := logger.Get(ctx)

pmmAgent, err := p.r.get(pmmAgentID)
if err != nil {
return nil, err
}

start := time.Now()
defer func() {
if dur := time.Since(start); dur > 5*time.Second {
l.Warnf("ParseDefaultsFile took %s.", dur)
}
}()

request, err := createRequest(filePath, serviceType)
if err != nil {
l.Debugf("can't create ParseDefaultsFileRequest %s", err)
return nil, err
}

resp, err := pmmAgent.channel.SendAndWaitResponse(request)
if err != nil {
return nil, err
}

l.Infof("ParseDefaultsFile response from agent: %+v.", resp)
parserResponse, ok := resp.(*agentpb.ParseDefaultsFileResponse)
if !ok {
return nil, errors.New("wrong response from agent (not ParseDefaultsFileResponse model)")
}
if parserResponse.Error != "" {
return nil, errors.New(parserResponse.Error)
}

return &models.ParseDefaultsFileResult{
Username: parserResponse.Username,
Password: parserResponse.Password,
Host: parserResponse.Host,
Port: parserResponse.Port,
Socket: parserResponse.Socket,
}, nil
}

func createRequest(configPath string, serviceType models.ServiceType) (*agentpb.ParseDefaultsFileRequest, error) {
if serviceType == models.MySQLServiceType {
request := &agentpb.ParseDefaultsFileRequest{
ServiceType: inventorypb.ServiceType_MYSQL_SERVICE,
ConfigPath: configPath,
}
return request, nil
} else {
return nil, errors.Errorf("unhandled service type %s", serviceType)
}
}
41 changes: 41 additions & 0 deletions services/agents/parse_defaults_file_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// pmm-managed
// Copyright (C) 2017 Percona LLC
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.

package agents

import (
"testing"

"github.com/stretchr/testify/require"

"github.com/percona/pmm-managed/models"
)

func TestCreateRequest(t *testing.T) {
t.Parallel()
response, err := createRequest("/path/to/file", models.MySQLServiceType)

require.NoError(t, err)
require.NotNil(t, response, "ParseDefaultsFileRequest is nil")
}

func TestCreateRequestNotSupported(t *testing.T) {
t.Parallel()
response, err := createRequest("/path/to/file", models.PostgreSQLServiceType)

require.Error(t, err)
require.Nil(t, response, "ParseDefaultsFileRequest is not nil")
}
8 changes: 8 additions & 0 deletions services/management/deps.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ import (
//go:generate mockery -name=grafanaClient -case=snake -inpkg -testonly
//go:generate mockery -name=jobsService -case=snake -inpkg -testonly
//go:generate mockery -name=connectionChecker -case=snake -inpkg -testonly
//go:generate mockery -name=defaultsFileParser -case=snake -inpkg -testonly
//go:generate mockery -name=versionCache -case=snake -inpkg -testonly

// agentsRegistry is a subset of methods of agents.Registry used by this package.
// We use it instead of real type for testing and to avoid dependency cycle.
Expand Down Expand Up @@ -93,3 +95,9 @@ type connectionChecker interface {
type versionCache interface {
RequestSoftwareVersionsUpdate()
}

// defaultsFileParser is a subset of methods of agents.ParseDefaultsFile.
// We use it instead of real type for testing and to avoid dependency cycle.
type defaultsFileParser interface {
ParseDefaultsFile(ctx context.Context, pmmAgentID, filePath string, serviceType models.ServiceType) (*models.ParseDefaultsFileResult, error)
}
39 changes: 39 additions & 0 deletions services/management/mock_defaults_file_parser_test.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 1011fab

Please sign in to comment.