From 06ab353f5d7e9a6450e45dd472ed94573326bcfe Mon Sep 17 00:00:00 2001 From: fizzers123 Date: Sat, 4 Nov 2023 16:21:42 +0100 Subject: [PATCH] implementation of ipfixlookupprocessor --- processor/ipfixlookupprocessor/Makefile | 1 + processor/ipfixlookupprocessor/README.md | 149 + processor/ipfixlookupprocessor/config.go | 100 + processor/ipfixlookupprocessor/doc.go | 4 + processor/ipfixlookupprocessor/factory.go | 81 + .../ipfixlookupprocessor/factory_test.go | 54 + processor/ipfixlookupprocessor/go.mod | 52 + processor/ipfixlookupprocessor/go.sum | 141 + .../internal/metadata/generated_status.go | 12 + processor/ipfixlookupprocessor/metadata.yaml | 9 + processor/ipfixlookupprocessor/processor.go | 579 ++++ .../ipfixlookupprocessor/processor_test.go | 156 + .../processor_test_sort.go | 73 + ...0.10.10:41662-10.0.20.20:3001.elastic.json | 226 ++ ...0.10.10:54321-10.0.20.20:3001.elastic.json | 226 ++ ...0.10.10:54322-10.0.20.20:3001.elastic.json | 640 +++++ ...0.20.20:3001-10.0.10.10:41662.elastic.json | 226 ++ ...0.20.20:3001-10.0.10.10:54322.elastic.json | 640 +++++ .../testdata/traces/NoResult-int.after.yaml | 281 ++ .../testdata/traces/NoResult-int.before.yaml | 40 + .../testdata/traces/NoResult-int.output.yaml | 281 ++ .../testdata/traces/NoResult.after.yaml | 281 ++ .../testdata/traces/NoResult.before.yaml | 40 + .../testdata/traces/NoResult.output.yaml | 281 ++ .../traces/OneSpanOneResult.after.yaml | 655 +++++ .../traces/OneSpanOneResult.before.yaml | 40 + .../traces/OneSpanOneResult.output.yaml | 655 +++++ .../traces/OneSpanWithAnswer.after.yaml | 1021 +++++++ .../traces/OneSpanWithAnswer.before.yaml | 40 + .../traces/OneSpanWithAnswer.output.yaml | 1021 +++++++ .../traces/OneSpanWithTwoFirewalls.after.yaml | 2545 +++++++++++++++++ .../OneSpanWithTwoFirewalls.before.yaml | 40 + .../OneSpanWithTwoFirewalls.output.yaml | 2545 +++++++++++++++++ .../traces/TwoSpanOneResult.after.yaml | 672 +++++ .../traces/TwoSpanOneResult.before.yaml | 57 + .../traces/TwoSpanOneResult.output.yaml | 672 +++++ .../testdata/traces/empty.elastic.json | 18 + versions.yaml | 1 + 38 files changed, 14555 insertions(+) create mode 100644 processor/ipfixlookupprocessor/Makefile create mode 100644 processor/ipfixlookupprocessor/README.md create mode 100644 processor/ipfixlookupprocessor/config.go create mode 100644 processor/ipfixlookupprocessor/doc.go create mode 100644 processor/ipfixlookupprocessor/factory.go create mode 100644 processor/ipfixlookupprocessor/factory_test.go create mode 100644 processor/ipfixlookupprocessor/go.mod create mode 100644 processor/ipfixlookupprocessor/go.sum create mode 100644 processor/ipfixlookupprocessor/internal/metadata/generated_status.go create mode 100644 processor/ipfixlookupprocessor/metadata.yaml create mode 100644 processor/ipfixlookupprocessor/processor.go create mode 100644 processor/ipfixlookupprocessor/processor_test.go create mode 100644 processor/ipfixlookupprocessor/processor_test_sort.go create mode 100644 processor/ipfixlookupprocessor/testdata/traces/10.0.10.10:41662-10.0.20.20:3001.elastic.json create mode 100644 processor/ipfixlookupprocessor/testdata/traces/10.0.10.10:54321-10.0.20.20:3001.elastic.json create mode 100644 processor/ipfixlookupprocessor/testdata/traces/10.0.10.10:54322-10.0.20.20:3001.elastic.json create mode 100644 processor/ipfixlookupprocessor/testdata/traces/10.0.20.20:3001-10.0.10.10:41662.elastic.json create mode 100644 processor/ipfixlookupprocessor/testdata/traces/10.0.20.20:3001-10.0.10.10:54322.elastic.json create mode 100644 processor/ipfixlookupprocessor/testdata/traces/NoResult-int.after.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/NoResult-int.before.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/NoResult-int.output.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/NoResult.after.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/NoResult.before.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/NoResult.output.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/OneSpanOneResult.after.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/OneSpanOneResult.before.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/OneSpanOneResult.output.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/OneSpanWithAnswer.after.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/OneSpanWithAnswer.before.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/OneSpanWithAnswer.output.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/OneSpanWithTwoFirewalls.after.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/OneSpanWithTwoFirewalls.before.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/OneSpanWithTwoFirewalls.output.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/TwoSpanOneResult.after.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/TwoSpanOneResult.before.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/TwoSpanOneResult.output.yaml create mode 100644 processor/ipfixlookupprocessor/testdata/traces/empty.elastic.json diff --git a/processor/ipfixlookupprocessor/Makefile b/processor/ipfixlookupprocessor/Makefile new file mode 100644 index 000000000000..ded7a36092dc --- /dev/null +++ b/processor/ipfixlookupprocessor/Makefile @@ -0,0 +1 @@ +include ../../Makefile.Common diff --git a/processor/ipfixlookupprocessor/README.md b/processor/ipfixlookupprocessor/README.md new file mode 100644 index 000000000000..82afd6f5293e --- /dev/null +++ b/processor/ipfixlookupprocessor/README.md @@ -0,0 +1,149 @@ +# IPFIX Lookup Processor + +| Status | | +| ------------- |-----------| +| Distributions | [contrib]| +| Issues |[![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Aconnector%2Fipfix%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Aconnector%2Fipfix) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Aconnector%2Fipfix%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Aconnector%2Fipfix) | +| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner) | [@fizzers123](https://www.github.com/fizzers123), [@SuniAve](https://www.github.com/SuniAve) | + +[development]: https://github.com/open-telemetry/opentelemetry-collector#development +[contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib + +## Supported Pipeline Types + +| [Exporter Pipeline Type] | [Receiver Pipeline Type] | [Stability Level] | +| ------------------------ | ------------------------ | ----------------- | +| traces | traces | [development] | + +[Exporter Pipeline Type]: https://github.com/open-telemetry/opentelemetry-collector/blob/main/connector/README.md#exporter-pipeline-type +[Receiver Pipeline Type]: https://github.com/open-telemetry/opentelemetry-collector/blob/main/connector/README.md#receiver-pipeline-type +[Stability Level]: https://github.com/open-telemetry/opentelemetry-collector#stability-levels + + +[ElasticSearch]: https://www.elastic.co/elasticsearch/ + +The `ipfixlookup` processor can be used to inject IPFIX spans into existing traces. + +## Prerequisits + +You need to store your Netflow/IPFIX logs in [ElasticSearch] to use this processor. Please check the [Netflow Integration](https://docs.elastic.co/en/integrations/netflow) for more information. + +### Default Configuration + +The `ipfixlookup` processorr will require the following minimum configuration. + + +For example, in the following configuration, the processor will connect to the specified [ElasticSearch] instance and search for IPFIX/Netflow events in the specified time window. +```yaml +processors: + groupbytrace: + wait_duration: 100s + num_traces: 1000 + num_workers: 2 + ipfix_lookup: + elastic_search: + connection: + addresses: + - https://:9200/ + username: elastic + password: + certificate_fingerprint: + timing: + lookup_window: 25 + +service: + pipelines: + traces: + receivers: [otlp] + processors: [groupbytrace, ipfix_lookup] + exporters: [otlp/jaeger, debug] +``` + +### Custom lookup fields + + + +Optionally, you can specify the fields the processor will look up in [ElasticSearch] and match within the spans + +| Full Path Configuration | Description | +| -------------------------------------------- | --------------------------------------------------------------------------------------------- | +| `ipfix_lookup.query_parameters` | Parameters used for querying the IPFIX lookup processor. | +| `ipfix_lookup.base_query.field_name` | The name of the field used in the base query. | +| `ipfix_lookup.base_query.field_value` | The value of the field used in the base query. | +| `ipfix_lookup.device_identifier` | The field used to identify the device in IPFIX records. | +| `ipfix_lookup.lookup_fields.source_ip` | Field representing the source IP address. | +| `ipfix_lookup.lookup_fields.source_port` | Field representing the source port. | +| `ipfix_lookup.lookup_fields.destination_ip` | Field representing the destination IP address. | +| `ipfix_lookup.lookup_fields.destination_port`| Field representing the destination port. | +| `ipfix_lookup.span_attribute_fields` | Fields to be added as attributes to the new span. In [gjson](https://gjson.dev/) format. | +| `ipfix_lookup.spans.source_ips` | Fields representing the source IP address of the span. | +| `ipfix_lookup.spans.source_ports` | Fields representing the source port of the span. | +| `ipfix_lookup.spans.destination_ip_and_port` | Fields representing the destination IP and port of the span. Like: `192.168.10.10:443` | +| `ipfix_lookup.spans.destination_ips` | Fields representing the destination IP address of the span. | +| `ipfix_lookup.spans.destination_ports` | Fields representing the destination port of the span. | + +The configuration below shows the default values: + + +```yaml +processors: + groupbytrace: + wait_duration: 10s + num_traces: 1000 + num_workers: 2 + ipfix_lookup: + query_parameters: + base_query: + field_name: input.type + field_value: netflow + device_identifier: "fields.observer\\.ip.0" + lookup_fields: + source_ip: source.ip + source_port: source.port + destination_ip: destination.ip + destination_port: destination.port + span_attribute_fields: + - "@this" + - "fields.event\\.duration.0" + - "fields.observer\\.ip.0" + - "fields.source\\.ip.0" + - "fields.source\\.port.0" + - "fields.destination\\.ip.0" + - "fields.destination\\.port.0" + - "fields.netflow\\.ip_next_hop_ipv4_address" + spans: + span_fields: + source_ips: + - net.peer.ip + - net.peer.name + - src.ip + source_ports: + - net.peer.port + - src.port + destination_ip_and_port: + - http.host + destination_ips: + - dst.ip + - net.peer.name + destination_ports: + - dst.port +``` + +### Timings: + +This processor is responsible for looking through the spans in each trace. If the IP and port quartet (`source.ip, source.port, destination.ip, destination.port`) are found in a span, the corresponding flow is looked up in ElasticSearch. When flows are found, a new span is added to the trace, and the trace is exported. + +![CorrelationUnitv3 drawio](https://github.com/open-telemetry/opentelemetry-collector-contrib/assets/15678530/5bb8e7de-c254-4991-871d-05c9c6d6f3f6) + + +The timing configuration is needed because there is an ingest delay in any large distributed search engine. Because of this, the processor must wait a bit before the search can be started. This delay can be defined in the `processors.groupbytrace.wait_duration` value. Afterwards, the search can be started. The time window that will be searched can be configured in the` processors.ipfix_lookup.timing.lookup_window`. To keep the processor simple, the lookup_window is added before the start timestamp and after the end timestamp. This way, the chance that the Netflow/IPFIX records leading or being caused by this span is found is maximized. + + +# Example screenshot +Example of a working implementation: +![finnal-implementation](https://github.com/open-telemetry/opentelemetry-collector-contrib/assets/15678530/37036d33-07f1-4c9e-bdea-7834a5e01015) +(The network was intentionally slowed down for this screenshot) + + + + diff --git a/processor/ipfixlookupprocessor/config.go b/processor/ipfixlookupprocessor/config.go new file mode 100644 index 000000000000..288a2afe7da2 --- /dev/null +++ b/processor/ipfixlookupprocessor/config.go @@ -0,0 +1,100 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ipfixlookupprocessor + +import "fmt" + +type ElasticsearchConnection struct { + Addresses []string `mapstructure:"addresses"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + CertificateFingerprint string `mapstructure:"certificate_fingerprint"` +} + +type ElasticsearchConfig struct { + Connection ElasticsearchConnection `mapstructure:"connection"` +} + +type LookupFields struct { + SourceIP string `mapstructure:"source_ip"` + SourcePort string `mapstructure:"source_port"` + DestinationIP string `mapstructure:"destination_ip"` + DestinationPort string `mapstructure:"destination_port"` +} + +type BaseQuery struct { + FieldName string `mapstructure:"field_name"` + FieldValue string `mapstructure:"field_value"` + LookupFields LookupFields `mapstructure:"lookup_fields"` +} + +type QueryParameters struct { + BaseQuery BaseQuery `mapstructure:"base_query"` + DeviceIdentifier string `mapstructure:"device_identifier"` + LookupFields LookupFields `mapstructure:"lookup_fields"` +} + +type TimingConfig struct { + LookupWindow int `mapstructure:"lookup_window"` +} + +type SpanFields struct { + SourceIPs []string `mapstructure:"source_ips"` + SourcePorts []string `mapstructure:"source_ports"` + DestinationIPandPort []string `mapstructure:"destination_ip_and_port"` + DestinationIPs []string `mapstructure:"destination_ips"` + DestinationPorts []string `mapstructure:"destination_ports"` +} + +type Spans struct { + SpanFields SpanFields `mapstructure:"span_fields"` +} + +type Config struct { + Elasticsearch ElasticsearchConfig `mapstructure:"elastic_search"` + QueryParameters QueryParameters `mapstructure:"query_parameters"` + SpanAttributeFields []string `mapstructure:"span_attribute_fields"` + Timing TimingConfig `mapstructure:"timing"` + Spans Spans `mapstructure:"spans"` +} + +func (c *Config) Validate() error { + // Validate Elasticsearch fields + if len(c.Elasticsearch.Connection.Addresses) == 0 { + return fmt.Errorf("elasticsearch addresses must not be empty") + } + if c.Elasticsearch.Connection.Username == "" { + return fmt.Errorf("elasticsearch username must not be empty") + } + if c.Elasticsearch.Connection.Password == "" { + return fmt.Errorf("elasticsearch password must not be empty") + } + if c.Elasticsearch.Connection.CertificateFingerprint == "" { + return fmt.Errorf("elasticsearch certificateFingerprint must not be empty") + } + + // Validate QueryParameters fields + if c.QueryParameters.DeviceIdentifier == "" { + return fmt.Errorf("queryParameters deviceIdentifier must not be empty") + } + if c.QueryParameters.BaseQuery.FieldName == "" || c.QueryParameters.BaseQuery.FieldValue == "" { + return fmt.Errorf("queryParameters baseQuery fieldName and fieldValue must not be empty") + } + + // Validate SpanAttributeFields + if len(c.SpanAttributeFields) == 0 { + return fmt.Errorf("spanAttributeFields must not be empty") + } + + // Validate Spans fields + if len(c.Spans.SpanFields.SourceIPs) == 0 || len(c.Spans.SpanFields.DestinationIPs) == 0 { + return fmt.Errorf("spans sourceIPs and destinationIPs must not be empty") + } + + // Validate timing fields + if c.Timing.LookupWindow < 0 { + return fmt.Errorf("lookupWindow must be greater than 0") + } + return nil +} diff --git a/processor/ipfixlookupprocessor/doc.go b/processor/ipfixlookupprocessor/doc.go new file mode 100644 index 000000000000..de37a27d310e --- /dev/null +++ b/processor/ipfixlookupprocessor/doc.go @@ -0,0 +1,4 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ipfixlookupprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/ipfixlookupprocessor" diff --git a/processor/ipfixlookupprocessor/factory.go b/processor/ipfixlookupprocessor/factory.go new file mode 100644 index 000000000000..b0240503998d --- /dev/null +++ b/processor/ipfixlookupprocessor/factory.go @@ -0,0 +1,81 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ipfixlookupprocessor + +import ( + "context" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/processor" +) + +const ( + // this is the name used to refer to the processor in the config.yaml + typeStr = "ipfix_lookup" +) + +func NewFactory() processor.Factory { + + return processor.NewFactory( + typeStr, + createDefaultConfig, + processor.WithTraces(createTracesToTracesProcessor, component.StabilityLevelAlpha)) +} + +func createDefaultConfig() component.Config { + return &Config{ + QueryParameters: QueryParameters{ + BaseQuery: BaseQuery{ + FieldName: "input.type", + FieldValue: "netflow", + }, + DeviceIdentifier: "fields.observer\\.ip.0", + LookupFields: LookupFields{ + SourceIP: "source.ip", + SourcePort: "source.port", + DestinationIP: "destination.ip", + DestinationPort: "destination.port", + }, + }, + SpanAttributeFields: []string{ + "@this", + "fields.event\\.duration.0", + "fields.observer\\.ip.0", + "fields.source\\.ip.0", + "fields.source\\.port.0", + "fields.destination\\.ip.0", + "fields.destination\\.port.0", + "fields.netflow\\.ip_next_hop_ipv4_address", + }, + + Spans: Spans{ + SpanFields: SpanFields{ + SourceIPs: []string{ + "net.peer.ip", + "src.ip", + }, + SourcePorts: []string{ + "net.peer.port", + "src.port", + }, + DestinationIPandPort: []string{ + "http.host", + }, + DestinationIPs: []string{ + "dst.ip", + }, + DestinationPorts: []string{ + "dst.port", + }, + }, + }, + } +} + +func createTracesToTracesProcessor(_ context.Context, params processor.CreateSettings, cfg component.Config, nextConsumer consumer.Traces) (processor.Traces, error) { + c := newIPFIXLookupProcessor(params.Logger, cfg) + c.tracesConsumer = nextConsumer + return c, nil +} diff --git a/processor/ipfixlookupprocessor/factory_test.go b/processor/ipfixlookupprocessor/factory_test.go new file mode 100644 index 000000000000..f7dc7a0d73e5 --- /dev/null +++ b/processor/ipfixlookupprocessor/factory_test.go @@ -0,0 +1,54 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ipfixlookupprocessor + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/processor/processortest" +) + +func TestFactory_Type(t *testing.T) { + factory := NewFactory() + assert.Equal(t, factory.Type(), component.Type(typeStr)) +} + +func TestFactory_CreateDefaultConfig(t *testing.T) { + factory := NewFactory() + cfg := factory.CreateDefaultConfig() + assert.NotEqual(t, cfg, &Config{}) + assert.NoError(t, componenttest.CheckConfigStruct(cfg)) +} + +func TestFactory_ValidateConfig(t *testing.T) { + factory := NewFactory() + cfg := factory.CreateDefaultConfig() + assert.EqualError(t, component.ValidateConfig(cfg), "elasticsearch addresses must not be empty") + cfg.(*Config).Elasticsearch.Connection.Addresses = []string{"http://localhost:9200"} + assert.EqualError(t, component.ValidateConfig(cfg), "elasticsearch username must not be empty") + cfg.(*Config).Elasticsearch.Connection.Username = "elastic" + assert.EqualError(t, component.ValidateConfig(cfg), "elasticsearch password must not be empty") + cfg.(*Config).Elasticsearch.Connection.Password = "changeme" + assert.EqualError(t, component.ValidateConfig(cfg), "elasticsearch certificateFingerprint must not be empty") + cfg.(*Config).Elasticsearch.Connection.CertificateFingerprint = "xxxx" + assert.NoError(t, component.ValidateConfig(cfg), "elasticsearch addresses must not be empty") +} + +func TestNewFactory(t *testing.T) { + factory := NewFactory() + conn, err := factory.CreateTracesProcessor( + context.Background(), + processortest.NewNopCreateSettings(), + factory.CreateDefaultConfig(), + consumertest.NewNop(), + ) + + assert.NoError(t, err) + assert.NotNil(t, conn) +} diff --git a/processor/ipfixlookupprocessor/go.mod b/processor/ipfixlookupprocessor/go.mod new file mode 100644 index 000000000000..72eaabd5cc28 --- /dev/null +++ b/processor/ipfixlookupprocessor/go.mod @@ -0,0 +1,52 @@ +module github.com/open-telemetry/opentelemetry-collector-contrib/processor/ipfixlookupprocessor + +go 1.21.3 + +require ( + github.com/elastic/go-elasticsearch/v8 v8.11.1 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.91.0 + github.com/stretchr/testify v1.8.4 + github.com/tidwall/gjson v1.17.0 + go.opentelemetry.io/collector/component v0.91.0 + go.opentelemetry.io/collector/consumer v0.91.0 + go.opentelemetry.io/collector/pdata v1.0.0 + go.opentelemetry.io/collector/processor v0.91.0 + go.uber.org/zap v1.26.0 + google.golang.org/grpc v1.60.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/elastic/elastic-transport-go/v8 v8.3.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.0.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.91.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + go.opentelemetry.io/collector v0.91.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.91.0 // indirect + go.opentelemetry.io/collector/confmap v0.91.0 // indirect + go.opentelemetry.io/collector/featuregate v1.0.0 // indirect + go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.21.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/processor/ipfixlookupprocessor/go.sum b/processor/ipfixlookupprocessor/go.sum new file mode 100644 index 000000000000..ad2a380cfc15 --- /dev/null +++ b/processor/ipfixlookupprocessor/go.sum @@ -0,0 +1,141 @@ +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWION0Bh6d9eCFBriiHo= +github.com/elastic/elastic-transport-go/v8 v8.3.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= +github.com/elastic/go-elasticsearch/v8 v8.11.1 h1:1VgTgUTbpqQZ4uE+cPjkOvy/8aw1ZvKcU0ZUE5Cn1mc= +github.com/elastic/go-elasticsearch/v8 v8.11.1/go.mod h1:GU1BJHO7WeamP7UhuElYwzzHtvf9SDmeVpSSy9+o6Qg= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.0.1 h1:1dYGITt1I23x8cfx8ZnldtezdyaZtfAuRtIFOiRzK7g= +github.com/knadh/koanf/v2 v2.0.1/go.mod h1:ZeiIlIDXTE7w1lMT6UVcNiRAS2/rCeLn/GdLNvY1Dus= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 h1:BpfhmLKZf+SjVanKKhCgf3bg+511DmU9eDQTen7LLbY= +github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.91.0 h1:Mx/Xyxfeqr0dm8M9XL6lM5jZN3sp+5PUGZBR6QUJbyA= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.91.0/go.mod h1:JuAibPtRgW2nNPJ3kTBsZIKtriAfa8Dh3yrlPHVvuMM= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.91.0 h1:a4XbucJve0K8g7kCO25EpNinBsXRGBJ8IhoLKNM0kdQ= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.91.0/go.mod h1:LrsgmhaNo+f3xb4loclG8+gLTWgyzmiS1bplK1CVRu0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= +github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector v0.91.0 h1:C7sGUJDJ5nwm+CkWpAaVP3lNsuYpwSRbkmLncFjkmO8= +go.opentelemetry.io/collector v0.91.0/go.mod h1:YhQpIDZsn+bICAAqgBwXk9wqK8GKZDv+aogfG52zUuE= +go.opentelemetry.io/collector/component v0.91.0 h1:aBT1i2zGyfh9PalYJLfXVvQp+osHyalwyDFselI1CtA= +go.opentelemetry.io/collector/component v0.91.0/go.mod h1:2KBHvjNFdU7oOjsObQeC4Ta2Ef607OISU5obznW00fw= +go.opentelemetry.io/collector/config/configtelemetry v0.91.0 h1:mEwvqrYfwUJ7LwYfpcF9M8z7LHFoYaKhEPhnERD/88E= +go.opentelemetry.io/collector/config/configtelemetry v0.91.0/go.mod h1:+LAXM5WFMW/UbTlAuSs6L/W72WC+q8TBJt/6z39FPOU= +go.opentelemetry.io/collector/confmap v0.91.0 h1:7U2MT+u74oEzq/WWrpXSLKB7nX5jPNC4drwtQdYfwKk= +go.opentelemetry.io/collector/confmap v0.91.0/go.mod h1:uxV+fZ85kG31oovL6Cl3fAMQ3RRPwUvfAbbA9WT1Yhk= +go.opentelemetry.io/collector/consumer v0.91.0 h1:0nU1lUe2S0b8iOmF3w3R/9Dt24n413thRTbXz/nJgrM= +go.opentelemetry.io/collector/consumer v0.91.0/go.mod h1:phTUQmr7hpYfwXyDXo4mFHVjYrlSbZE+nZYlKlbVxGs= +go.opentelemetry.io/collector/featuregate v1.0.0 h1:5MGqe2v5zxaoo73BUOvUTunftX5J8RGrbFsC2Ha7N3g= +go.opentelemetry.io/collector/featuregate v1.0.0/go.mod h1:xGbRuw+GbutRtVVSEy3YR2yuOlEyiUMhN2M9DJljgqY= +go.opentelemetry.io/collector/pdata v1.0.0 h1:ECP2jnLztewsHmL1opL8BeMtWVc7/oSlKNhfY9jP8ec= +go.opentelemetry.io/collector/pdata v1.0.0/go.mod h1:TsDFgs4JLNG7t6x9D8kGswXUz4mme+MyNChHx8zSF6k= +go.opentelemetry.io/collector/processor v0.91.0 h1:Xi52gYMXTG4zYmNhsqJ8ly/9f7b0n0crMhKxVVI9HpY= +go.opentelemetry.io/collector/processor v0.91.0/go.mod h1:naTuusZNfzM5MSqoTVzkKbR1MaJ8oD8v5ginR5JreDE= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= +google.golang.org/grpc v1.60.0 h1:6FQAR0kM31P6MRdeluor2w2gPaS4SVNrD/DNTxrQ15k= +google.golang.org/grpc v1.60.0/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/processor/ipfixlookupprocessor/internal/metadata/generated_status.go b/processor/ipfixlookupprocessor/internal/metadata/generated_status.go new file mode 100644 index 000000000000..3c892de7e170 --- /dev/null +++ b/processor/ipfixlookupprocessor/internal/metadata/generated_status.go @@ -0,0 +1,12 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" +) + +const ( + Type = "ipfixlookup" + TracesToTracesStability = component.StabilityLevelDevelopment +) diff --git a/processor/ipfixlookupprocessor/metadata.yaml b/processor/ipfixlookupprocessor/metadata.yaml new file mode 100644 index 000000000000..466cee2986e5 --- /dev/null +++ b/processor/ipfixlookupprocessor/metadata.yaml @@ -0,0 +1,9 @@ +type: ipfixlookup + +status: + class: connector + stability: + development: [traces_to_traces] + distributions: [contrib] + codeowners: + active: [tbd] diff --git a/processor/ipfixlookupprocessor/processor.go b/processor/ipfixlookupprocessor/processor.go new file mode 100644 index 000000000000..c8b98a144644 --- /dev/null +++ b/processor/ipfixlookupprocessor/processor.go @@ -0,0 +1,579 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ipfixlookupprocessor + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "io" + "net" + "strconv" + "strings" + "time" + + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/esapi" + "github.com/tidwall/gjson" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.uber.org/zap" +) + +// Question: Is there a better way of doing this ? +// Needed for testing. Monkey patching +var randRead = rand.Read // Todo: potentially fix by controlling time +var searchElasticFunc = searchElastic + +// schema for processor +type processorImp struct { + config Config + tracesConsumer consumer.Traces + logger *zap.Logger + es *elasticsearch.Client +} + +// Network Quartet +type NetworkQuartet struct { + SourceIP net.IP + SourcePort uint16 + DestinationIP net.IP + DestinationPort uint16 + elasticQuery string +} + +// Check if all required information is available +func (nq *NetworkQuartet) isValid() bool { + return !nq.SourceIP.Equal(net.IP{}) && nq.SourceIP != nil && + nq.SourcePort > 0 && + !nq.DestinationIP.Equal(net.IP{}) && nq.DestinationIP != nil && + nq.DestinationPort > 0 +} + +// isEqual compares two NetworkQuartet instances for equality. +func (nq *NetworkQuartet) isEqual(other *NetworkQuartet) bool { + return nq.SourceIP.Equal(other.SourceIP) && + nq.SourcePort == other.SourcePort && + nq.DestinationIP.Equal(other.DestinationIP) && + nq.DestinationPort == other.DestinationPort +} + +func (nq *NetworkQuartet) String() string { + return fmt.Sprintf("%s:%d-%s:%d", + nq.SourceIP.String(), nq.SourcePort, nq.DestinationIP.String(), nq.DestinationPort) +} + +// newConnector is a function to create a new connector +func newIPFIXLookupProcessor(logger *zap.Logger, config component.Config) *processorImp { + logger.Info("Building IPFIXLookupProcessor") + cfg := config.(*Config) + + return &processorImp{ + config: *cfg, + logger: logger, + } +} + +// Capabilities implements the consumer interface. +func (p *processorImp) Capabilities() consumer.Capabilities { + return consumer.Capabilities{MutatesData: true} +} + +func (p *processorImp) Start(_ context.Context, _ component.Host) error { + p.logger.Info("Starting spanmetrics processor") + p.es = connectToElasticSearch(p.logger, &p.config) + return nil +} + +func (p *processorImp) Shutdown(context.Context) error { + p.logger.Info("Shutting down spanmetrics processor") + return nil +} + +func connectToElasticSearch(logger *zap.Logger, config *Config) *elasticsearch.Client { + elasticConfig := elasticsearch.Config{ + Addresses: config.Elasticsearch.Connection.Addresses, + Username: config.Elasticsearch.Connection.Username, + Password: config.Elasticsearch.Connection.Password, + CertificateFingerprint: config.Elasticsearch.Connection.CertificateFingerprint, + } + es, err := elasticsearch.NewClient(elasticConfig) + if err != nil { + logger.Error("Failed to create Elasticsearch client", zap.String("Error", err.Error())) + } + + response, err := es.Info() + if err != nil { + logger.Debug(fmt.Sprintf("Elasticsearch Error: %v", err)) + } else { + logger.Debug(fmt.Sprintf("Elasticsearch Response: %+v", response)) + } + return es +} + +// ConsumeTraces method is called for each instance of a trace sent to the processor +func (p *processorImp) ConsumeTraces(ctx context.Context, td ptrace.Traces) error { + p.logger.Info("ConsumeTraces: Checking Trace now") + // loop through the levels of spans of the one trace consumed + ipfixresourceSpan := td.ResourceSpans().AppendEmpty() + ipfixresourceSpan.Resource().Attributes().PutStr("service.name", "IPFIX") // TODO: add config parameter + ipfixscopedSpan := ipfixresourceSpan.ScopeSpans().AppendEmpty() + ipfixscopedSpan.Scope().SetName("ipfix") + + for i := 0; i < td.ResourceSpans().Len(); i++ { + resourceSpan := td.ResourceSpans().At(i) + + for j := 0; j < resourceSpan.ScopeSpans().Len(); j++ { + scopeSpan := resourceSpan.ScopeSpans().At(j) + + for k := 0; k < scopeSpan.Spans().Len(); k++ { + span := scopeSpan.Spans().At(k) + err := p.findAndHandleSpan(span, ipfixscopedSpan, td) // TODO: better method name + if err != nil { + p.logger.Error("Error finding and handling span", zap.Error(err)) + } + + } + } + } + p.ipfixLookup(ipfixscopedSpan) + return p.tracesConsumer.ConsumeTraces(ctx, td) +} + +func (p *processorImp) findAndHandleSpan(span ptrace.Span, ipfixscopedSpan ptrace.ScopeSpans, td ptrace.Traces) error { + p.logger.Info("ConsumeTraces: Checking Span ...", zap.String("spanid", span.SpanID().String())) + + validNetworkQuartet, networkQuartet := p.extractNetworkQuartet(span) + + // check parent for the same IP and Port Quartet + if validNetworkQuartet { + parentFound, parent := findSpanByID(span.ParentSpanID(), td) + if parentFound { + validParentNetworkQuartet, parentNetworkQuartet := p.extractNetworkQuartet(parent) + if validParentNetworkQuartet && parentNetworkQuartet.isEqual(networkQuartet) { + return p.findAndHandleSpan(parent, ipfixscopedSpan, td) + } + } + createSummarySpan(span, networkQuartet, ipfixscopedSpan) + } else { + p.logger.Info("Unable to find all required information to lookup IPFIX information") + } + return nil +} + +func findSpanByID(spanID pcommon.SpanID, td ptrace.Traces) (bool, ptrace.Span) { + for i := 0; i < td.ResourceSpans().Len(); i++ { + resourceSpan := td.ResourceSpans().At(i) + + for j := 0; j < resourceSpan.ScopeSpans().Len(); j++ { + scopeSpan := resourceSpan.ScopeSpans().At(j) + + for k := 0; k < scopeSpan.Spans().Len(); k++ { + span := scopeSpan.Spans().At(k) + if span.SpanID() == spanID { + return true, span + } + + } + } + } + return false, ptrace.Span{} +} + +// ipfixLookup performs IPFIX lookup for the given network quartet. +func (p *processorImp) ipfixLookup(ipfixscopedSpan ptrace.ScopeSpans) { + var requestNetworkQuartet *NetworkQuartet + for i := 0; i < ipfixscopedSpan.Spans().Len(); i++ { + summarySpan := ipfixscopedSpan.Spans().At(i) + var valid bool + valid, requestNetworkQuartet = p.extractNetworkQuartet(summarySpan) + if !valid { + p.logger.Error("Unable to find all required information to lookup IPFIX information") + return + } + + requestSpans, requestSpanCount := p.performIPFIXLookup(summarySpan, requestNetworkQuartet, ipfixscopedSpan) + // Create response network quartet + responseNetworkQuartet := &NetworkQuartet{ + SourceIP: requestNetworkQuartet.DestinationIP, + SourcePort: requestNetworkQuartet.DestinationPort, + DestinationIP: requestNetworkQuartet.SourceIP, + DestinationPort: requestNetworkQuartet.SourcePort, + } + responseSpans, responeSpanCount := p.performIPFIXLookup(summarySpan, responseNetworkQuartet, ipfixscopedSpan) + summarySpan.Attributes().PutInt("ipfix.request.flows", requestSpanCount) + summarySpan.Attributes().PutInt("ipfix.response.flows", responeSpanCount) + + p.setNameOfIPFIXSpan(requestSpans, "Request - ") + p.setNameOfIPFIXSpan(responseSpans, "Response - ") + + earliest, latest := findMinMaxTimestamp(append(requestSpans, responseSpans...)) + summarySpan.SetStartTimestamp(earliest) + summarySpan.SetEndTimestamp(latest) + + requestQueryField := summarySpan.Attributes().PutEmptyMap("z.elasticQuery.request") + if err := requestQueryField.FromRaw(gjson.Parse(requestNetworkQuartet.elasticQuery).Value().(map[string]any)); err != nil { + p.logger.Error("Error parsing request query field", zap.Error(err)) + } + responseQueryField := summarySpan.Attributes().PutEmptyMap("z.elasticQuery.response") + if err := responseQueryField.FromRaw(gjson.Parse(responseNetworkQuartet.elasticQuery).Value().(map[string]any)); err != nil { + p.logger.Error("Error parsing response query field", zap.Error(err)) + } + } + +} + +func findMinMaxTimestamp(spans []ptrace.Span) (pcommon.Timestamp, pcommon.Timestamp) { + var earliestTimestamp pcommon.Timestamp + var latestTimestamp pcommon.Timestamp + for _, span := range spans { + startTimestamp := span.StartTimestamp() + if earliestTimestamp == 0 || startTimestamp < earliestTimestamp { + earliestTimestamp = startTimestamp + } + endTimestamp := span.EndTimestamp() + if latestTimestamp == 0 || endTimestamp > latestTimestamp { + latestTimestamp = endTimestamp + } + } + return earliestTimestamp, latestTimestamp +} + +func (p *processorImp) setNameOfIPFIXSpan(spans []ptrace.Span, prefix string) { + for _, span := range spans { + name, found := span.Attributes().Get(p.config.QueryParameters.DeviceIdentifier) + if found { + span.SetName(prefix + name.AsString()) + } else { + span.SetName(prefix + "no IPFIX logs found") + } + } +} + +// createSummarySpan creates a summary span for the given network quartet. +func createSummarySpan(span ptrace.Span, requestNetworkQuartet *NetworkQuartet, ipfixscopedSpan ptrace.ScopeSpans) ptrace.Span { + summarySpan := findSummarySpan(ipfixscopedSpan, requestNetworkQuartet.String()) + if summarySpan != (ptrace.Span{}) { + return summarySpan + } + summarySpan = createNetworkSpan(ipfixscopedSpan, span) + summarySpan.SetName(requestNetworkQuartet.String()) + summarySpan.SetParentSpanID(span.ParentSpanID()) + + summarySpan.Attributes().PutStr("src.ip", requestNetworkQuartet.SourceIP.String()) + summarySpan.Attributes().PutStr("src.port", strconv.Itoa(int(requestNetworkQuartet.SourcePort))) + summarySpan.Attributes().PutStr("dst.ip", requestNetworkQuartet.DestinationIP.String()) + summarySpan.Attributes().PutStr("dst.port", strconv.Itoa(int(requestNetworkQuartet.DestinationPort))) + summarySpan.Attributes().PutInt("ipfix.request.flows", 0) + summarySpan.Attributes().PutInt("ipfix.response.flows", 0) + return summarySpan +} + +// findSummarySpan finds the summary span for the given network quartet in the scope spans. +func findSummarySpan(ipfixscopedSpan ptrace.ScopeSpans, networkQuartetName string) ptrace.Span { + for i := 0; i < ipfixscopedSpan.Spans().Len(); i++ { + span := ipfixscopedSpan.Spans().At(i) + if span.Name() == networkQuartetName { + return span + } + } + return ptrace.Span{} +} + +// performIPFIXLookup performs IPFIX lookup for the given network quartet. +func (p *processorImp) performIPFIXLookup(summarySpan ptrace.Span, networkQuartet *NetworkQuartet, ipfixscopedSpan ptrace.ScopeSpans) ([]ptrace.Span, int64) { + // Search IPFIX events for the network quartet + elasticResponse := p.lookupSpanInElasticSearch(summarySpan, networkQuartet) + var networkSpans []ptrace.Span + hits := gjson.GetBytes(elasticResponse, "hits.hits") + if hits.IsArray() { + hitsArray := hits.Array() + if len(hitsArray) == 0 { + networkSpan := createNetworkSpan(ipfixscopedSpan, summarySpan) + networkSpan.SetParentSpanID(summarySpan.SpanID()) + summarySpan.Attributes().PutStr("NoHits-Warning", "No hits were found when searching!\nThis could be due to:\n- Sampling rate\n- Bad time settings\n- Others") + networkSpans = append(networkSpans, networkSpan) + return networkSpans, 0 + + } + for i := 0; i < len(hitsArray); i++ { + // Create IPFIX span + networkSpan := createNetworkSpan(ipfixscopedSpan, summarySpan) + networkSpan.SetParentSpanID(summarySpan.SpanID()) + if addAttributesErr := p.addAttributesToNetworkSpan(networkSpan, hitsArray[i]); addAttributesErr != nil { + p.logger.Error("Error adding attributes to span", zap.Error(addAttributesErr)) + } + networkSpans = append(networkSpans, networkSpan) + + } + + } + + return networkSpans, int64(len((networkSpans))) +} + +func (p *processorImp) extractNetworkQuartet(span ptrace.Span) (bool, *NetworkQuartet) { + // NetworkQuartet represents a quartet of network information. + attrs := span.Attributes() + mapping := attrs.AsRaw() + + // Extracting needed information + sourceIPValue, foundSourceIP := getStringValue(mapping, p.config.Spans.SpanFields.SourceIPs...) + sourcePortValue, foundSourcePort := getStringValue(mapping, p.config.Spans.SpanFields.SourcePorts...) + destinationIPandPortValue, foundDestinationIPandPort := getStringValue(mapping, p.config.Spans.SpanFields.DestinationIPandPort...) + destinationIPValue, foundDestinationIP := getStringValue(mapping, p.config.Spans.SpanFields.DestinationIPs...) + destinationPortValue, foundDestinationPort := getStringValue(mapping, p.config.Spans.SpanFields.DestinationPorts...) + + // Converting concatenated IP and port, e.g: 192.168.1.0:443 + if !foundDestinationIP && !foundDestinationPort && foundDestinationIPandPort { + destinationIPandPortArray := strings.Split(destinationIPandPortValue, ":") + destinationIPValue = destinationIPandPortArray[0] + foundDestinationIP = true + destinationPortValue = destinationIPandPortArray[1] + foundDestinationPort = true + } + + networkQuartet := NetworkQuartet{} + if foundSourceIP { + networkQuartet.SourceIP = p.convertToIP(sourceIPValue) + } + if foundSourcePort { + networkQuartet.SourcePort = p.convertToPort(sourcePortValue) + } + if foundDestinationIP { + networkQuartet.DestinationIP = p.convertToIP(destinationIPValue) + } + if foundDestinationPort { + networkQuartet.DestinationPort = p.convertToPort(destinationPortValue) + } + return networkQuartet.isValid(), &networkQuartet +} + +func getStringValue(mapping map[string]any, keys ...string) (string, bool) { + for _, key := range keys { + value := mapping[key] + switch v := value.(type) { + case string: + return v, true + case int64: + return strconv.FormatInt(v, 10), true + } + } + return "", false +} + +func (p *processorImp) convertToIP(ipString string) net.IP { + ip := net.ParseIP(ipString) + if ip == nil { + p.logger.Debug(fmt.Sprintf("failed to Parse IP from : %v", ipString)) + } + return ip +} + +func (p *processorImp) convertToPort(sourcePortValue string) uint16 { + sourcePort, err := strconv.ParseUint(sourcePortValue, 10, 16) + if err == nil { + p.logger.Debug(fmt.Sprintf("sourcePort: %v", err)) + } + return uint16(sourcePort) +} + +func (p *processorImp) addAttributesToNetworkSpan(networkSpan ptrace.Span, hit gjson.Result) error { + errHitsAttribute := AddAttributeByJSONPath(networkSpan, hit, "hits.total.value") + if errHitsAttribute != nil { + p.logger.Error("Error adding attribute to span", zap.Error(errHitsAttribute)) + } + + // TODO: make configurable in config + attributePaths := p.config.SpanAttributeFields + + for _, path := range attributePaths { + err := AddAttributeByJSONPath(networkSpan, hit, path) + if err != nil { + p.logger.Error("Error adding attribute to span", zap.Error(err)) + } + } + + renameAttribute(networkSpan, "@this", "z.elasticResponse") + + // Set time + eventStart, err := extractTimeStamp(hit, `fields.event\.start.0`) + if err != nil { + return err + } + eventEnd, err := extractTimeStamp(hit, `fields.event\.end.0`) + if err != nil { + return err + } + networkSpan.SetStartTimestamp(pcommon.NewTimestampFromTime(eventStart)) + networkSpan.SetEndTimestamp(pcommon.NewTimestampFromTime(eventEnd)) + return nil +} + +func renameAttribute(networkSpan ptrace.Span, oldAttributeName string, newAttributeName string) { + attribute, _ := networkSpan.Attributes().Get(oldAttributeName) + attribute.Map().CopyTo(networkSpan.Attributes().PutEmptyMap(newAttributeName)) + networkSpan.Attributes().Remove(oldAttributeName) +} + +func createNetworkSpan(ipfixscopedSpan ptrace.ScopeSpans, span ptrace.Span) ptrace.Span { + networkSpan := ipfixscopedSpan.Spans().AppendEmpty() + networkSpan.SetTraceID(span.TraceID()) + networkSpan.SetSpanID(generateRandomSpanID()) + networkSpan.SetName("injected") + + networkSpan.SetStartTimestamp(span.EndTimestamp()) + networkSpan.SetEndTimestamp(span.EndTimestamp()) + + return networkSpan +} + +func (p *processorImp) lookupSpanInElasticSearch(span ptrace.Span, networkQuartet *NetworkQuartet) []byte { + p.logger.Info("ConsumeTraces: attributes found in Span", zap.String("spanid", span.SpanID().String())) + + elasticQuery := p.generateElasticsearchQuery( + networkQuartet.SourceIP, + networkQuartet.SourcePort, + networkQuartet.DestinationIP, + networkQuartet.DestinationPort, + span.EndTimestamp().AsTime(), + time.Second*time.Duration(p.config.Timing.LookupWindow), + ) + networkQuartet.elasticQuery = elasticQuery + p.logger.Debug("elasticsearch query", zap.String("query", elasticQuery)) + + elasticResponse := p.lookupIpfixInElastic(elasticQuery) + + return elasticResponse +} + +func (p *processorImp) lookupIpfixInElastic(query string) []byte { + + searchResults, err := searchElasticFunc(query, p.es) + if err != nil { + p.logger.Error("Elasticsearch Result Error", zap.Error(err)) + return nil + } + + responseBody, err := io.ReadAll(searchResults.Body) + if err != nil { + p.logger.Error("Error reading elastic response body", zap.Error(err)) + } + + p.logger.Debug("Elasticsearch Response", zap.ByteString("elasticsearch.Body", responseBody)) + return responseBody +} + +func (p *processorImp) generateElasticsearchQuery(sourceIP net.IP, sourcePort uint16, destinationIP net.IP, destinationPort uint16, startTime time.Time, duration time.Duration) string { + queryTemplate := ` + { + "query": { + "bool": { + "must": [ + { "match": { "%s": "%s" }}, + { "match": { "%s": "%s" }}, + { "match": { "%s": "%d" }}, + { "match": { "%s": "%s" }}, + { "match": { "%s": "%d" }}, + { "range": { "@timestamp": { "gte": "%s", "lte": "%s" }}} + ] + } + }, + "_source": false, + "fields": [ + { + "field": "*", + "include_unmapped": "true" + } + ] + }` + + endTime := startTime.Add(duration) + startTime = startTime.Add(-duration) + + query := fmt.Sprintf( + queryTemplate, + p.config.QueryParameters.BaseQuery.FieldName, + p.config.QueryParameters.BaseQuery.FieldValue, + p.config.QueryParameters.LookupFields.SourceIP, + sourceIP.String(), + p.config.QueryParameters.LookupFields.SourcePort, + sourcePort, + p.config.QueryParameters.LookupFields.DestinationIP, + destinationIP.String(), + p.config.QueryParameters.LookupFields.DestinationPort, + destinationPort, + startTime.Format(time.RFC3339), + endTime.Format(time.RFC3339), + ) + + return query +} + +func searchElastic(query string, es *elasticsearch.Client) (*esapi.Response, error) { + searchResults, err := es.Search( + es.Search.WithIndex("logs-*"), + es.Search.WithBody(strings.NewReader(query)), + ) + return searchResults, err +} + +func generateRandomSpanID() pcommon.SpanID { + var spanID pcommon.SpanID + _, err := randRead(spanID[:]) + if err != nil { + return pcommon.SpanID{} + } + return spanID +} + +func AddAttributeByJSONPath(span ptrace.Span, json gjson.Result, jsonPath string) error { + result := json.Get(jsonPath) + + switch result.Type { + case gjson.Number: + span.Attributes().PutInt(jsonPath, result.Int()) + case gjson.String: + span.Attributes().PutStr(jsonPath, result.String()) + case gjson.True, gjson.False: + span.Attributes().PutBool(jsonPath, result.Bool()) + case gjson.JSON: + switch { + case result.IsArray(): + array, ok := result.Value().([]any) + if !ok { + return nil + } + attributeArray := span.Attributes().PutEmptySlice(jsonPath) + if err := attributeArray.FromRaw(array); err != nil { + return err + } + case result.IsObject(): + m, ok := result.Value().(map[string]any) + if !ok { + return nil + } + attributeMap := span.Attributes().PutEmptyMap(jsonPath) + if err := attributeMap.FromRaw(m); err != nil { + return err + } + default: + return errors.New("unsupported JSON type") + } + case gjson.Null: + span.Attributes().PutStr(jsonPath, "null") + default: + return errors.New("unsupported JSON type") + } + + return nil +} + +func extractTimeStamp(elasticResponse gjson.Result, jsonPath string) (time.Time, error) { + timeStamp, err := time.Parse(time.RFC3339, elasticResponse.Get(jsonPath).String()) + return timeStamp, err +} diff --git a/processor/ipfixlookupprocessor/processor_test.go b/processor/ipfixlookupprocessor/processor_test.go new file mode 100644 index 000000000000..5fb6f97602dd --- /dev/null +++ b/processor/ipfixlookupprocessor/processor_test.go @@ -0,0 +1,156 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ipfixlookupprocessor + +import ( + "context" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/esapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.uber.org/zap/zaptest" + "google.golang.org/grpc/metadata" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden" +) + +// --------------- +// Rand Mock +// --------------- + +var counter int + +func randReadMock(b []byte) (n int, err error) { + for i := range b { + b[i] = byte(counter) + + } + counter++ + return len(b), nil +} + +// --------------- +// Elastic Mock +// --------------- + +func searchElasticMock(query string, _ *elasticsearch.Client) (*esapi.Response, error) { + // find file by IP and port + sourceIP := gjson.Get(query, "query.bool.must.#.match.source\\.ip").Array()[0].String() + sourcePort := gjson.Get(query, "query.bool.must.#.match.source\\.port").Array()[0].String() + destinationIP := gjson.Get(query, "query.bool.must.#.match.destination\\.ip").Array()[0].String() + destinationPort := gjson.Get(query, "query.bool.must.#.match.destination\\.port").Array()[0].String() + + fileName := sourceIP + ":" + sourcePort + "-" + destinationIP + ":" + destinationPort + ".elastic.json" + var body []byte + var err error + + if fileExists(filepath.Join("testdata", "traces", fileName)) { + body, err = readJSONFile(filepath.Join("testdata", "traces", fileName)) + } else { + body, err = readJSONFile(filepath.Join("testdata", "traces", "empty.elastic.json")) + } + if err != nil { + return &esapi.Response{ + StatusCode: 500, + Body: io.NopCloser(strings.NewReader("Elastic Search failure")), + }, err + } + + return &esapi.Response{ + StatusCode: 200, + Body: io.NopCloser(strings.NewReader(string(body))), + }, nil +} + +func fileExists(filePath string) bool { + _, err := os.Stat(filePath) + return err == nil +} + +// readJSONFile reads the contents of a JSON file +func readJSONFile(filePath string) ([]byte, error) { + data, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + return data, nil +} + +// --------------- +// Tests +// --------------- + +func TestProcessor(t *testing.T) { + testCases := []struct { + name string + }{ + {"NoResult"}, + {"NoResult-int"}, + {"OneSpanOneResult"}, + {"TwoSpanOneResult"}, + {"OneSpanWithAnswer"}, + {"OneSpanWithTwoFirewalls"}, + // Add more test cases here + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Create Processor + factory := NewFactory() + cfg := factory.CreateDefaultConfig().(*Config) + p := newIPFIXLookupProcessor(zaptest.NewLogger(t), cfg) + + // Fake consumer + sink := &consumertest.TracesSink{} + p.tracesConsumer = sink + + // Create Context + ctx := metadata.NewIncomingContext(context.Background(), nil) + require.NoError(t, p.Start(ctx, componenttest.NewNopHost())) + defer func() { require.NoError(t, p.Shutdown(ctx)) }() + + // Mock rand.read Is there a better way of doing this ? + counter = 1 + randRead = randReadMock + + // Mock elastic search lookup + searchElasticFunc = searchElasticMock + + // traces + tracesBefore, err := golden.ReadTraces(filepath.Join("testdata", "traces", tc.name+".before.yaml")) + require.NoError(t, err) + tracesAfter, err := golden.ReadTraces(filepath.Join("testdata", "traces", tc.name+".after.yaml")) + assert.NoError(t, err) + errConsumeTraces := p.ConsumeTraces(ctx, tracesBefore) + assert.NoError(t, errConsumeTraces) + sortTraceAttributes(tracesBefore) // The Traces need to be sorted as the Attribute maps can variy in order + + // golden.WriteTraces(t, filepath.Join("testdata", "traces", tc.name+".output.yaml"), sink.AllTraces()[0]) + + require.Equal(t, tracesAfter, sink.AllTraces()[0]) + }) + } +} + +func TestCapabilitiesMutatesData(t *testing.T) { + // Create an instance of processorImp + c := &processorImp{} // Assuming processorImp is the type you provided + + // Call the Capabilities method + capabilities := c.Capabilities() + + // Check if MutatesData is true + if !capabilities.MutatesData { + t.Errorf("Expected MutatesData to be true, but got false") + } +} diff --git a/processor/ipfixlookupprocessor/processor_test_sort.go b/processor/ipfixlookupprocessor/processor_test_sort.go new file mode 100644 index 000000000000..5eb6ea548379 --- /dev/null +++ b/processor/ipfixlookupprocessor/processor_test_sort.go @@ -0,0 +1,73 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ipfixlookupprocessor + +import ( + "sort" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/ptrace" +) + +// Question: Is there a better way of doing this ? +func sortTraceAttributes(traces ptrace.Traces) { + for i := 0; i < traces.ResourceSpans().Len(); i++ { + resourceSpan := traces.ResourceSpans().At(i) + for j := 0; j < resourceSpan.ScopeSpans().Len(); j++ { + scopeSpan := resourceSpan.ScopeSpans().At(j) + for k := 0; k < scopeSpan.Spans().Len(); k++ { + span := scopeSpan.Spans().At(k) + attributes := span.Attributes() + sortedAttributes := sortAttributeMap(attributes) + sortedAttributes.CopyTo(attributes) + } + } + } +} + +// derviced from https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/pkg/golden/v0.89.0/pkg/golden/sort_metrics.go +// sortAttributeMap sorts the attributes of a pcommon.Map according to the alphanumeric ordering of the keys +func sortAttributeMap(mp pcommon.Map) pcommon.Map { + tempMap := pcommon.NewMap() + keys := []string{} + mp.Range(func(key string, _ pcommon.Value) bool { + keys = append(keys, key) + return true + }) + sort.Strings(keys) + for _, k := range keys { + value, exists := mp.Get(k) + if exists { + switch value.Type() { + case pcommon.ValueTypeMap: + sortedMap := sortAttributeMap(value.Map()) + sortedMap.CopyTo(tempMap.PutEmptyMap(k)) + case pcommon.ValueTypeSlice: + sortedSlice := sortAttributeSlice(value.Slice()) + sortedSlice.CopyTo(tempMap.PutEmptySlice(k)) + default: + value.CopyTo(tempMap.PutEmpty(k)) + } + } + } + return tempMap +} + +func sortAttributeSlice(slice pcommon.Slice) pcommon.Slice { + tempSlice := pcommon.NewSlice() + for i := 0; i < slice.Len(); i++ { + value := slice.At(i) + switch value.Type() { + case pcommon.ValueTypeMap: + sortedMap := sortAttributeMap(value.Map()) + sortedMap.CopyTo(tempSlice.AppendEmpty().SetEmptyMap()) + case pcommon.ValueTypeSlice: + sortedSlice := sortAttributeSlice(value.Slice()) + sortedSlice.CopyTo(tempSlice.AppendEmpty().SetEmptySlice()) + default: + value.CopyTo(tempSlice.AppendEmpty()) + } + } + return tempSlice +} diff --git a/processor/ipfixlookupprocessor/testdata/traces/10.0.10.10:41662-10.0.20.20:3001.elastic.json b/processor/ipfixlookupprocessor/testdata/traces/10.0.10.10:41662-10.0.20.20:3001.elastic.json new file mode 100644 index 000000000000..bfbdaaa159ca --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/10.0.10.10:41662-10.0.20.20:3001.elastic.json @@ -0,0 +1,226 @@ +{ + "took": 46, + "timed_out": false, + "_shards": { + "total": 8, + "successful": 8, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 5.000001, + "hits": [ + { + "_index": ".ds-logs-netflow.log-default-2023.10.30-000002", + "_id": "auVm7YsB9hE7__H6VGKl", + "_score": 5.000001, + "fields": { + "flow.id": [ + "9of6On_LptU" + ], + "elastic_agent.version": [ + "8.10.2" + ], + "event.category": [ + "network" + ], + "netflow.ip_class_of_service": [ + 0 + ], + "netflow.source_transport_port": [ + 41662 + ], + "netflow.tcp_control_bits": [ + 27 + ], + "netflow.exporter.version": [ + 9 + ], + "netflow.exporter.address": [ + "192.168.200.100:12377" + ], + "netflow.ip_version": [ + 4 + ], + "source.ip": [ + "10.0.10.10" + ], + "agent.name": [ + "quickstart-agent-n5j6z" + ], + "network.community_id": [ + "1:v6MCwUb4XYI1KPRAFQ5jX/W7FaY=" + ], + "event.agent_id_status": [ + "auth_metadata_missing" + ], + "event.kind": [ + "event" + ], + "source.packets": [ + 7 + ], + "network.packets": [ + 7 + ], + "netflow.destination_ipv4_address": [ + "10.0.20.20" + ], + "netflow.flow_start_sys_up_time": [ + 175329483 + ], + "flow.locality": [ + "internal" + ], + "input.type": [ + "netflow" + ], + "data_stream.type": [ + "logs" + ], + "tags": [ + "netflow", + "forwarded" + ], + "agent.id": [ + "bdf672da-9aa2-4205-b423-fc105b6b14e1" + ], + "source.port": [ + 41662 + ], + "ecs.version": [ + "8.10.0" + ], + "event.created": [ + "2023-11-20T15:43:07.448Z" + ], + "network.iana_number": [ + "6" + ], + "agent.version": [ + "8.10.2" + ], + "event.start": [ + "2023-11-20T15:43:00.963Z" + ], + "observer.ip": [ + "192.168.200.100" + ], + "netflow.source_ipv4_address": [ + "10.0.10.10" + ], + "netflow.type": [ + "netflow_flow" + ], + "destination.port": [ + 3001 + ], + "netflow.flow_end_sys_up_time": [ + 175329931 + ], + "event.end": [ + "2023-11-20T15:43:01.411Z" + ], + "netflow.octet_delta_count": [ + 834 + ], + "agent.type": [ + "filebeat" + ], + "netflow.exporter.source_id": [ + 0 + ], + "event.module": [ + "netflow" + ], + "related.ip": [ + "10.0.10.10", + "10.0.20.20" + ], + "netflow.ingress_interface": [ + 1 + ], + "netflow.packet_delta_count": [ + 7 + ], + "network.bytes": [ + 834 + ], + "elastic_agent.snapshot": [ + false + ], + "network.direction": [ + "internal" + ], + "network.type": [ + "ipv4" + ], + "netflow.exporter.uptime_millis": [ + 175335520 + ], + "source.bytes": [ + 834 + ], + "destination.locality": [ + "internal" + ], + "elastic_agent.id": [ + "bdf672da-9aa2-4205-b423-fc105b6b14e1" + ], + "data_stream.namespace": [ + "default" + ], + "netflow.destination_transport_port": [ + 3001 + ], + "netflow.exporter.timestamp": [ + "2023-11-20T15:43:07.000Z" + ], + "destination.ip": [ + "10.0.20.20" + ], + "source.locality": [ + "internal" + ], + "network.transport": [ + "tcp" + ], + "event.duration": [ + 448000000 + ], + "netflow.protocol_identifier": [ + 6 + ], + "event.action": [ + "netflow_flow" + ], + "event.ingested": [ + "2023-11-20T15:43:08Z" + ], + "@timestamp": [ + "2023-11-20T15:43:07.000Z" + ], + "data_stream.dataset": [ + "netflow.log" + ], + "event.type": [ + "connection" + ], + "agent.ephemeral_id": [ + "b65c4bd4-dec2-46dc-b2a7-2d8a5ded7d25" + ], + "event.dataset": [ + "netflow.log" + ], + "netflow.egress_interface": [ + 1 + ] + } + } + ] + } + } \ No newline at end of file diff --git a/processor/ipfixlookupprocessor/testdata/traces/10.0.10.10:54321-10.0.20.20:3001.elastic.json b/processor/ipfixlookupprocessor/testdata/traces/10.0.10.10:54321-10.0.20.20:3001.elastic.json new file mode 100644 index 000000000000..339379fd4fc0 --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/10.0.10.10:54321-10.0.20.20:3001.elastic.json @@ -0,0 +1,226 @@ +{ + "took": 46, + "timed_out": false, + "_shards": { + "total": 8, + "successful": 8, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 5.000001, + "hits": [ + { + "_index": ".ds-logs-netflow.log-default-2023.10.30-000002", + "_id": "auVm7YsB9hE7__H6VGKl", + "_score": 5.000001, + "fields": { + "flow.id": [ + "9of6On_LptU" + ], + "elastic_agent.version": [ + "8.10.2" + ], + "event.category": [ + "network" + ], + "netflow.ip_class_of_service": [ + 0 + ], + "netflow.source_transport_port": [ + 54321 + ], + "netflow.tcp_control_bits": [ + 27 + ], + "netflow.exporter.version": [ + 9 + ], + "netflow.exporter.address": [ + "192.168.200.100:12377" + ], + "netflow.ip_version": [ + 4 + ], + "source.ip": [ + "10.0.10.10" + ], + "agent.name": [ + "quickstart-agent-n5j6z" + ], + "network.community_id": [ + "1:v6MCwUb4XYI1KPRAFQ5jX/W7FaY=" + ], + "event.agent_id_status": [ + "auth_metadata_missing" + ], + "event.kind": [ + "event" + ], + "source.packets": [ + 7 + ], + "network.packets": [ + 7 + ], + "netflow.destination_ipv4_address": [ + "10.0.20.20" + ], + "netflow.flow_start_sys_up_time": [ + 175329483 + ], + "flow.locality": [ + "internal" + ], + "input.type": [ + "netflow" + ], + "data_stream.type": [ + "logs" + ], + "tags": [ + "netflow", + "forwarded" + ], + "agent.id": [ + "bdf672da-9aa2-4205-b423-fc105b6b14e1" + ], + "source.port": [ + 54321 + ], + "ecs.version": [ + "8.10.0" + ], + "event.created": [ + "2023-11-20T15:43:07.448Z" + ], + "network.iana_number": [ + "6" + ], + "agent.version": [ + "8.10.2" + ], + "event.start": [ + "2023-11-20T15:43:00.963Z" + ], + "observer.ip": [ + "192.168.200.100" + ], + "netflow.source_ipv4_address": [ + "10.0.10.10" + ], + "netflow.type": [ + "netflow_flow" + ], + "destination.port": [ + 3001 + ], + "netflow.flow_end_sys_up_time": [ + 175329931 + ], + "event.end": [ + "2023-11-20T15:43:01.411Z" + ], + "netflow.octet_delta_count": [ + 834 + ], + "agent.type": [ + "filebeat" + ], + "netflow.exporter.source_id": [ + 0 + ], + "event.module": [ + "netflow" + ], + "related.ip": [ + "10.0.10.10", + "10.0.20.20" + ], + "netflow.ingress_interface": [ + 1 + ], + "netflow.packet_delta_count": [ + 7 + ], + "network.bytes": [ + 834 + ], + "elastic_agent.snapshot": [ + false + ], + "network.direction": [ + "internal" + ], + "network.type": [ + "ipv4" + ], + "netflow.exporter.uptime_millis": [ + 175335520 + ], + "source.bytes": [ + 834 + ], + "destination.locality": [ + "internal" + ], + "elastic_agent.id": [ + "bdf672da-9aa2-4205-b423-fc105b6b14e1" + ], + "data_stream.namespace": [ + "default" + ], + "netflow.destination_transport_port": [ + 3001 + ], + "netflow.exporter.timestamp": [ + "2023-11-20T15:43:07.000Z" + ], + "destination.ip": [ + "10.0.20.20" + ], + "source.locality": [ + "internal" + ], + "network.transport": [ + "tcp" + ], + "event.duration": [ + 448000000 + ], + "netflow.protocol_identifier": [ + 6 + ], + "event.action": [ + "netflow_flow" + ], + "event.ingested": [ + "2023-11-20T15:43:08Z" + ], + "@timestamp": [ + "2023-11-20T15:43:07.000Z" + ], + "data_stream.dataset": [ + "netflow.log" + ], + "event.type": [ + "connection" + ], + "agent.ephemeral_id": [ + "b65c4bd4-dec2-46dc-b2a7-2d8a5ded7d25" + ], + "event.dataset": [ + "netflow.log" + ], + "netflow.egress_interface": [ + 1 + ] + } + } + ] + } + } \ No newline at end of file diff --git a/processor/ipfixlookupprocessor/testdata/traces/10.0.10.10:54322-10.0.20.20:3001.elastic.json b/processor/ipfixlookupprocessor/testdata/traces/10.0.10.10:54322-10.0.20.20:3001.elastic.json new file mode 100644 index 000000000000..3616c13c1b67 --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/10.0.10.10:54322-10.0.20.20:3001.elastic.json @@ -0,0 +1,640 @@ +{ + "took": 25, + "timed_out": false, + "_shards": { + "total": 12, + "successful": 12, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 3, + "relation": "eq" + }, + "max_score": 5.0000377, + "hits": [ + { + "_index": ".ds-logs-netflow.log-default-2023.11.29-000003", + "_id": "PVRtJIwB9hE7__H6RRK1", + "_score": 5.0000377, + "fields": { + "flow.id": [ + "kmkdh7501nI" + ], + "elastic_agent.version": [ + "8.10.2" + ], + "event.category": [ + "network" + ], + "netflow.ip_class_of_service": [ + 0 + ], + "netflow.source_transport_port": [ + 3001 + ], + "netflow.tcp_control_bits": [ + 27 + ], + "netflow.exporter.version": [ + 9 + ], + "netflow.exporter.address": [ + "192.168.200.110:10029" + ], + "netflow.ip_version": [ + 4 + ], + "source.ip": [ + "10.0.20.20" + ], + "agent.name": [ + "ipfix-agent-ch4cq" + ], + "network.community_id": [ + "1:WKydzbXVu9IwFFuVVrA/KwJFFNA=" + ], + "event.agent_id_status": [ + "auth_metadata_missing" + ], + "event.kind": [ + "event" + ], + "source.packets": [ + 6 + ], + "network.packets": [ + 6 + ], + "netflow.destination_ipv4_address": [ + "10.0.10.10" + ], + "netflow.flow_start_sys_up_time": [ + 321294019 + ], + "flow.locality": [ + "internal" + ], + "input.type": [ + "netflow" + ], + "data_stream.type": [ + "logs" + ], + "tags": [ + "netflow", + "forwarded" + ], + "agent.id": [ + "4267406d-d5d3-489b-8dd7-76cc7e1d336e" + ], + "source.port": [ + 3001 + ], + "ecs.version": [ + "8.10.0" + ], + "event.created": [ + "2023-12-01T08:09:49.256Z" + ], + "network.iana_number": [ + "6" + ], + "agent.version": [ + "8.10.2" + ], + "event.start": [ + "2023-12-01T08:09:43.307Z" + ], + "observer.ip": [ + "192.168.200.110" + ], + "netflow.source_ipv4_address": [ + "10.0.20.20" + ], + "netflow.type": [ + "netflow_flow" + ], + "destination.port": [ + 54322 + ], + "netflow.flow_end_sys_up_time": [ + 321294473 + ], + "event.end": [ + "2023-12-01T08:09:43.761Z" + ], + "netflow.octet_delta_count": [ + 1014 + ], + "agent.type": [ + "filebeat" + ], + "netflow.exporter.source_id": [ + 0 + ], + "event.module": [ + "netflow" + ], + "related.ip": [ + "10.0.10.10", + "10.0.20.20" + ], + "netflow.ingress_interface": [ + 1 + ], + "netflow.packet_delta_count": [ + 6 + ], + "network.bytes": [ + 1014 + ], + "elastic_agent.snapshot": [ + false + ], + "network.direction": [ + "internal" + ], + "network.type": [ + "ipv4" + ], + "netflow.exporter.uptime_millis": [ + 321299712 + ], + "source.bytes": [ + 1014 + ], + "destination.locality": [ + "internal" + ], + "elastic_agent.id": [ + "4267406d-d5d3-489b-8dd7-76cc7e1d336e" + ], + "data_stream.namespace": [ + "default" + ], + "netflow.destination_transport_port": [ + 54322 + ], + "netflow.exporter.timestamp": [ + "2023-12-01T08:09:49.000Z" + ], + "destination.ip": [ + "10.0.10.10" + ], + "source.locality": [ + "internal" + ], + "network.transport": [ + "tcp" + ], + "event.duration": [ + 454000000 + ], + "netflow.protocol_identifier": [ + 6 + ], + "event.action": [ + "netflow_flow" + ], + "event.ingested": [ + "2023-12-01T08:09:50Z" + ], + "@timestamp": [ + "2023-12-01T08:09:49.000Z" + ], + "data_stream.dataset": [ + "netflow.log" + ], + "event.type": [ + "connection" + ], + "agent.ephemeral_id": [ + "2c589e2c-2889-415c-a1d3-abc7aeb27f41" + ], + "event.dataset": [ + "netflow.log" + ], + "netflow.egress_interface": [ + 1 + ] + } + }, + { + "_index": ".ds-logs-netflow.log-default-2023.11.29-000003", + "_id": "v1RuJIwB9hE7__H68RdM", + "_score": 5.0000377, + "fields": { + "flow.id": [ + "kmkdh7501nI" + ], + "elastic_agent.version": [ + "8.10.2" + ], + "event.category": [ + "network" + ], + "netflow.ip_class_of_service": [ + 0 + ], + "netflow.source_transport_port": [ + 3001 + ], + "netflow.tcp_control_bits": [ + 27 + ], + "netflow.exporter.version": [ + 9 + ], + "netflow.exporter.address": [ + "192.168.200.110:4556" + ], + "netflow.ip_version": [ + 4 + ], + "source.ip": [ + "10.0.20.20" + ], + "agent.name": [ + "ipfix-agent-ch4cq" + ], + "network.community_id": [ + "1:WKydzbXVu9IwFFuVVrA/KwJFFNA=" + ], + "event.agent_id_status": [ + "auth_metadata_missing" + ], + "event.kind": [ + "event" + ], + "source.packets": [ + 6 + ], + "network.packets": [ + 6 + ], + "netflow.destination_ipv4_address": [ + "10.0.10.10" + ], + "netflow.flow_start_sys_up_time": [ + 321294158 + ], + "flow.locality": [ + "internal" + ], + "input.type": [ + "netflow" + ], + "data_stream.type": [ + "logs" + ], + "tags": [ + "netflow", + "forwarded" + ], + "agent.id": [ + "4267406d-d5d3-489b-8dd7-76cc7e1d336e" + ], + "source.port": [ + 3001 + ], + "ecs.version": [ + "8.10.0" + ], + "event.created": [ + "2023-12-01T08:11:38.867Z" + ], + "network.iana_number": [ + "6" + ], + "agent.version": [ + "8.10.2" + ], + "event.start": [ + "2023-12-01T08:09:43.101Z" + ], + "observer.ip": [ + "192.168.200.110" + ], + "netflow.source_ipv4_address": [ + "10.0.20.20" + ], + "netflow.type": [ + "netflow_flow" + ], + "destination.port": [ + 54322 + ], + "netflow.flow_end_sys_up_time": [ + 321294470 + ], + "event.end": [ + "2023-12-01T08:09:43.413Z" + ], + "netflow.octet_delta_count": [ + 1014 + ], + "agent.type": [ + "filebeat" + ], + "netflow.exporter.source_id": [ + 0 + ], + "event.module": [ + "netflow" + ], + "related.ip": [ + "10.0.10.10", + "10.0.20.20" + ], + "netflow.ingress_interface": [ + 2 + ], + "netflow.packet_delta_count": [ + 6 + ], + "network.bytes": [ + 1014 + ], + "elastic_agent.snapshot": [ + false + ], + "network.direction": [ + "internal" + ], + "network.type": [ + "ipv4" + ], + "netflow.exporter.uptime_millis": [ + 321409057 + ], + "source.bytes": [ + 1014 + ], + "destination.locality": [ + "internal" + ], + "elastic_agent.id": [ + "4267406d-d5d3-489b-8dd7-76cc7e1d336e" + ], + "data_stream.namespace": [ + "default" + ], + "netflow.destination_transport_port": [ + 54322 + ], + "netflow.exporter.timestamp": [ + "2023-12-01T08:11:38.000Z" + ], + "destination.ip": [ + "10.0.10.10" + ], + "source.locality": [ + "internal" + ], + "network.transport": [ + "tcp" + ], + "event.duration": [ + 312000000 + ], + "netflow.protocol_identifier": [ + 6 + ], + "event.action": [ + "netflow_flow" + ], + "event.ingested": [ + "2023-12-01T08:11:39Z" + ], + "@timestamp": [ + "2023-12-01T08:11:38.000Z" + ], + "data_stream.dataset": [ + "netflow.log" + ], + "event.type": [ + "connection" + ], + "agent.ephemeral_id": [ + "2c589e2c-2889-415c-a1d3-abc7aeb27f41" + ], + "event.dataset": [ + "netflow.log" + ], + "netflow.egress_interface": [ + 2 + ] + } + }, + { + "_index": ".ds-logs-netflow.log-default-2023.11.29-000003", + "_id": "0VRuJIwB9hE7__H6-xhC", + "_score": 5.0000377, + "fields": { + "flow.id": [ + "kmkdh7501nI" + ], + "elastic_agent.version": [ + "8.10.2" + ], + "event.category": [ + "network" + ], + "netflow.ip_class_of_service": [ + 0 + ], + "netflow.source_transport_port": [ + 3001 + ], + "netflow.tcp_control_bits": [ + 27 + ], + "netflow.exporter.version": [ + 9 + ], + "netflow.exporter.address": [ + "192.168.200.120:3455" + ], + "netflow.ip_version": [ + 4 + ], + "source.ip": [ + "10.0.20.20" + ], + "agent.name": [ + "ipfix-agent-ch4cq" + ], + "network.community_id": [ + "1:WKydzbXVu9IwFFuVVrA/KwJFFNA=" + ], + "event.agent_id_status": [ + "auth_metadata_missing" + ], + "event.kind": [ + "event" + ], + "source.packets": [ + 6 + ], + "network.packets": [ + 6 + ], + "netflow.destination_ipv4_address": [ + "10.0.10.10" + ], + "netflow.flow_start_sys_up_time": [ + 321292160 + ], + "flow.locality": [ + "internal" + ], + "input.type": [ + "netflow" + ], + "data_stream.type": [ + "logs" + ], + "tags": [ + "netflow", + "forwarded" + ], + "agent.id": [ + "4267406d-d5d3-489b-8dd7-76cc7e1d336e" + ], + "source.port": [ + 3001 + ], + "ecs.version": [ + "8.10.0" + ], + "event.created": [ + "2023-12-01T08:11:41.275Z" + ], + "network.iana_number": [ + "6" + ], + "agent.version": [ + "8.10.2" + ], + "event.start": [ + "2023-12-01T08:09:43.543Z" + ], + "observer.ip": [ + "192.168.200.120" + ], + "netflow.source_ipv4_address": [ + "10.0.20.20" + ], + "netflow.type": [ + "netflow_flow" + ], + "destination.port": [ + 54322 + ], + "netflow.flow_end_sys_up_time": [ + 321292471 + ], + "event.end": [ + "2023-12-01T08:09:43.854Z" + ], + "netflow.octet_delta_count": [ + 1014 + ], + "agent.type": [ + "filebeat" + ], + "netflow.exporter.source_id": [ + 0 + ], + "event.module": [ + "netflow" + ], + "related.ip": [ + "10.0.10.10", + "10.0.20.20" + ], + "netflow.ingress_interface": [ + 2 + ], + "netflow.packet_delta_count": [ + 6 + ], + "network.bytes": [ + 1014 + ], + "elastic_agent.snapshot": [ + false + ], + "network.direction": [ + "internal" + ], + "network.type": [ + "ipv4" + ], + "netflow.exporter.uptime_millis": [ + 321409617 + ], + "source.bytes": [ + 1014 + ], + "destination.locality": [ + "internal" + ], + "elastic_agent.id": [ + "4267406d-d5d3-489b-8dd7-76cc7e1d336e" + ], + "data_stream.namespace": [ + "default" + ], + "netflow.destination_transport_port": [ + 54322 + ], + "netflow.exporter.timestamp": [ + "2023-12-01T08:11:41.000Z" + ], + "destination.ip": [ + "10.0.10.10" + ], + "source.locality": [ + "internal" + ], + "network.transport": [ + "tcp" + ], + "event.duration": [ + 311000000 + ], + "netflow.protocol_identifier": [ + 6 + ], + "event.action": [ + "netflow_flow" + ], + "event.ingested": [ + "2023-12-01T08:11:42Z" + ], + "@timestamp": [ + "2023-12-01T08:11:41.000Z" + ], + "data_stream.dataset": [ + "netflow.log" + ], + "event.type": [ + "connection" + ], + "agent.ephemeral_id": [ + "2c589e2c-2889-415c-a1d3-abc7aeb27f41" + ], + "event.dataset": [ + "netflow.log" + ], + "netflow.egress_interface": [ + 2 + ] + } + } + ] + } +} \ No newline at end of file diff --git a/processor/ipfixlookupprocessor/testdata/traces/10.0.20.20:3001-10.0.10.10:41662.elastic.json b/processor/ipfixlookupprocessor/testdata/traces/10.0.20.20:3001-10.0.10.10:41662.elastic.json new file mode 100644 index 000000000000..e44ba47e1827 --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/10.0.20.20:3001-10.0.10.10:41662.elastic.json @@ -0,0 +1,226 @@ +{ + "took": 64, + "timed_out": false, + "_shards": { + "total": 8, + "successful": 8, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 5.000001, + "hits": [ + { + "_index": ".ds-logs-netflow.log-default-2023.10.30-000002", + "_id": "a-Vm7YsB9hE7__H6VGKl", + "_score": 5.000001, + "fields": { + "flow.id": [ + "9of6On_LptU" + ], + "elastic_agent.version": [ + "8.10.2" + ], + "event.category": [ + "network" + ], + "netflow.ip_class_of_service": [ + 0 + ], + "netflow.source_transport_port": [ + 3001 + ], + "netflow.tcp_control_bits": [ + 27 + ], + "netflow.exporter.version": [ + 9 + ], + "netflow.exporter.address": [ + "192.168.200.100:12377" + ], + "netflow.ip_version": [ + 4 + ], + "source.ip": [ + "10.0.20.20" + ], + "agent.name": [ + "quickstart-agent-n5j6z" + ], + "network.community_id": [ + "1:v6MCwUb4XYI1KPRAFQ5jX/W7FaY=" + ], + "event.agent_id_status": [ + "auth_metadata_missing" + ], + "event.kind": [ + "event" + ], + "source.packets": [ + 6 + ], + "network.packets": [ + 6 + ], + "netflow.destination_ipv4_address": [ + "10.0.10.10" + ], + "netflow.flow_start_sys_up_time": [ + 175329483 + ], + "flow.locality": [ + "internal" + ], + "input.type": [ + "netflow" + ], + "data_stream.type": [ + "logs" + ], + "tags": [ + "netflow", + "forwarded" + ], + "agent.id": [ + "bdf672da-9aa2-4205-b423-fc105b6b14e1" + ], + "source.port": [ + 3001 + ], + "ecs.version": [ + "8.10.0" + ], + "event.created": [ + "2023-11-20T15:43:07.448Z" + ], + "network.iana_number": [ + "6" + ], + "agent.version": [ + "8.10.2" + ], + "event.start": [ + "2023-11-20T15:43:00.963Z" + ], + "observer.ip": [ + "192.168.200.100" + ], + "netflow.source_ipv4_address": [ + "10.0.20.20" + ], + "netflow.type": [ + "netflow_flow" + ], + "destination.port": [ + 41662 + ], + "netflow.flow_end_sys_up_time": [ + 175329931 + ], + "event.end": [ + "2023-11-20T15:43:01.411Z" + ], + "netflow.octet_delta_count": [ + 1014 + ], + "agent.type": [ + "filebeat" + ], + "netflow.exporter.source_id": [ + 0 + ], + "event.module": [ + "netflow" + ], + "related.ip": [ + "10.0.10.10", + "10.0.20.20" + ], + "netflow.ingress_interface": [ + 1 + ], + "netflow.packet_delta_count": [ + 6 + ], + "network.bytes": [ + 1014 + ], + "elastic_agent.snapshot": [ + false + ], + "network.direction": [ + "internal" + ], + "network.type": [ + "ipv4" + ], + "netflow.exporter.uptime_millis": [ + 175335520 + ], + "source.bytes": [ + 1014 + ], + "destination.locality": [ + "internal" + ], + "elastic_agent.id": [ + "bdf672da-9aa2-4205-b423-fc105b6b14e1" + ], + "data_stream.namespace": [ + "default" + ], + "netflow.destination_transport_port": [ + 41662 + ], + "netflow.exporter.timestamp": [ + "2023-11-20T15:43:07.000Z" + ], + "destination.ip": [ + "10.0.10.10" + ], + "source.locality": [ + "internal" + ], + "network.transport": [ + "tcp" + ], + "event.duration": [ + 448000000 + ], + "netflow.protocol_identifier": [ + 6 + ], + "event.action": [ + "netflow_flow" + ], + "event.ingested": [ + "2023-11-20T15:43:08Z" + ], + "@timestamp": [ + "2023-11-20T15:43:07.000Z" + ], + "data_stream.dataset": [ + "netflow.log" + ], + "event.type": [ + "connection" + ], + "agent.ephemeral_id": [ + "b65c4bd4-dec2-46dc-b2a7-2d8a5ded7d25" + ], + "event.dataset": [ + "netflow.log" + ], + "netflow.egress_interface": [ + 1 + ] + } + } + ] + } +} \ No newline at end of file diff --git a/processor/ipfixlookupprocessor/testdata/traces/10.0.20.20:3001-10.0.10.10:54322.elastic.json b/processor/ipfixlookupprocessor/testdata/traces/10.0.20.20:3001-10.0.10.10:54322.elastic.json new file mode 100644 index 000000000000..e6801adc71d2 --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/10.0.20.20:3001-10.0.10.10:54322.elastic.json @@ -0,0 +1,640 @@ +{ + "took": 19, + "timed_out": false, + "_shards": { + "total": 12, + "successful": 12, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 3, + "relation": "eq" + }, + "max_score": 5.000033, + "hits": [ + { + "_index": ".ds-logs-netflow.log-default-2023.11.29-000003", + "_id": "PFRtJIwB9hE7__H6RRK1", + "_score": 5.000033, + "fields": { + "flow.id": [ + "kmkdh7501nI" + ], + "elastic_agent.version": [ + "8.10.2" + ], + "event.category": [ + "network" + ], + "netflow.ip_class_of_service": [ + 0 + ], + "netflow.source_transport_port": [ + 35340 + ], + "netflow.tcp_control_bits": [ + 27 + ], + "netflow.exporter.version": [ + 9 + ], + "netflow.exporter.address": [ + "192.168.200.110:10029" + ], + "netflow.ip_version": [ + 4 + ], + "source.ip": [ + "10.0.10.10" + ], + "agent.name": [ + "ipfix-agent-ch4cq" + ], + "network.community_id": [ + "1:WKydzbXVu9IwFFuVVrA/KwJFFNA=" + ], + "event.agent_id_status": [ + "auth_metadata_missing" + ], + "event.kind": [ + "event" + ], + "source.packets": [ + 7 + ], + "network.packets": [ + 7 + ], + "netflow.destination_ipv4_address": [ + "10.0.20.20" + ], + "netflow.flow_start_sys_up_time": [ + 321294019 + ], + "flow.locality": [ + "internal" + ], + "input.type": [ + "netflow" + ], + "data_stream.type": [ + "logs" + ], + "tags": [ + "netflow", + "forwarded" + ], + "agent.id": [ + "4267406d-d5d3-489b-8dd7-76cc7e1d336e" + ], + "source.port": [ + 35340 + ], + "ecs.version": [ + "8.10.0" + ], + "event.created": [ + "2023-12-01T08:09:49.256Z" + ], + "network.iana_number": [ + "6" + ], + "agent.version": [ + "8.10.2" + ], + "event.start": [ + "2023-12-01T08:09:43.307Z" + ], + "observer.ip": [ + "192.168.200.110" + ], + "netflow.source_ipv4_address": [ + "10.0.10.10" + ], + "netflow.type": [ + "netflow_flow" + ], + "destination.port": [ + 3001 + ], + "netflow.flow_end_sys_up_time": [ + 321294473 + ], + "event.end": [ + "2023-12-01T08:09:43.761Z" + ], + "netflow.octet_delta_count": [ + 834 + ], + "agent.type": [ + "filebeat" + ], + "netflow.exporter.source_id": [ + 0 + ], + "event.module": [ + "netflow" + ], + "related.ip": [ + "10.0.10.10", + "10.0.20.20" + ], + "netflow.ingress_interface": [ + 1 + ], + "netflow.packet_delta_count": [ + 7 + ], + "network.bytes": [ + 834 + ], + "elastic_agent.snapshot": [ + false + ], + "network.direction": [ + "internal" + ], + "network.type": [ + "ipv4" + ], + "netflow.exporter.uptime_millis": [ + 321299712 + ], + "source.bytes": [ + 834 + ], + "destination.locality": [ + "internal" + ], + "elastic_agent.id": [ + "4267406d-d5d3-489b-8dd7-76cc7e1d336e" + ], + "data_stream.namespace": [ + "default" + ], + "netflow.destination_transport_port": [ + 3001 + ], + "netflow.exporter.timestamp": [ + "2023-12-01T08:09:49.000Z" + ], + "destination.ip": [ + "10.0.20.20" + ], + "source.locality": [ + "internal" + ], + "network.transport": [ + "tcp" + ], + "event.duration": [ + 454000000 + ], + "netflow.protocol_identifier": [ + 6 + ], + "event.action": [ + "netflow_flow" + ], + "event.ingested": [ + "2023-12-01T08:09:50Z" + ], + "@timestamp": [ + "2023-12-01T08:09:49.000Z" + ], + "data_stream.dataset": [ + "netflow.log" + ], + "event.type": [ + "connection" + ], + "agent.ephemeral_id": [ + "2c589e2c-2889-415c-a1d3-abc7aeb27f41" + ], + "event.dataset": [ + "netflow.log" + ], + "netflow.egress_interface": [ + 1 + ] + } + }, + { + "_index": ".ds-logs-netflow.log-default-2023.11.29-000003", + "_id": "vlRuJIwB9hE7__H68RdM", + "_score": 5.000033, + "fields": { + "flow.id": [ + "kmkdh7501nI" + ], + "elastic_agent.version": [ + "8.10.2" + ], + "event.category": [ + "network" + ], + "netflow.ip_class_of_service": [ + 0 + ], + "netflow.source_transport_port": [ + 35340 + ], + "netflow.tcp_control_bits": [ + 27 + ], + "netflow.exporter.version": [ + 9 + ], + "netflow.exporter.address": [ + "192.168.200.110:4556" + ], + "netflow.ip_version": [ + 4 + ], + "source.ip": [ + "10.0.10.10" + ], + "agent.name": [ + "ipfix-agent-ch4cq" + ], + "network.community_id": [ + "1:WKydzbXVu9IwFFuVVrA/KwJFFNA=" + ], + "event.agent_id_status": [ + "auth_metadata_missing" + ], + "event.kind": [ + "event" + ], + "source.packets": [ + 7 + ], + "network.packets": [ + 7 + ], + "netflow.destination_ipv4_address": [ + "10.0.20.20" + ], + "netflow.flow_start_sys_up_time": [ + 321294158 + ], + "flow.locality": [ + "internal" + ], + "input.type": [ + "netflow" + ], + "data_stream.type": [ + "logs" + ], + "tags": [ + "netflow", + "forwarded" + ], + "agent.id": [ + "4267406d-d5d3-489b-8dd7-76cc7e1d336e" + ], + "source.port": [ + 35340 + ], + "ecs.version": [ + "8.10.0" + ], + "event.created": [ + "2023-12-01T08:11:38.867Z" + ], + "network.iana_number": [ + "6" + ], + "agent.version": [ + "8.10.2" + ], + "event.start": [ + "2023-12-01T08:09:43.101Z" + ], + "observer.ip": [ + "192.168.200.110" + ], + "netflow.source_ipv4_address": [ + "10.0.10.10" + ], + "netflow.type": [ + "netflow_flow" + ], + "destination.port": [ + 3001 + ], + "netflow.flow_end_sys_up_time": [ + 321294470 + ], + "event.end": [ + "2023-12-01T08:09:43.413Z" + ], + "netflow.octet_delta_count": [ + 834 + ], + "agent.type": [ + "filebeat" + ], + "netflow.exporter.source_id": [ + 0 + ], + "event.module": [ + "netflow" + ], + "related.ip": [ + "10.0.10.10", + "10.0.20.20" + ], + "netflow.ingress_interface": [ + 2 + ], + "netflow.packet_delta_count": [ + 7 + ], + "network.bytes": [ + 834 + ], + "elastic_agent.snapshot": [ + false + ], + "network.direction": [ + "internal" + ], + "network.type": [ + "ipv4" + ], + "netflow.exporter.uptime_millis": [ + 321409057 + ], + "source.bytes": [ + 834 + ], + "destination.locality": [ + "internal" + ], + "elastic_agent.id": [ + "4267406d-d5d3-489b-8dd7-76cc7e1d336e" + ], + "data_stream.namespace": [ + "default" + ], + "netflow.destination_transport_port": [ + 3001 + ], + "netflow.exporter.timestamp": [ + "2023-12-01T08:11:38.000Z" + ], + "destination.ip": [ + "10.0.20.20" + ], + "source.locality": [ + "internal" + ], + "network.transport": [ + "tcp" + ], + "event.duration": [ + 312000000 + ], + "netflow.protocol_identifier": [ + 6 + ], + "event.action": [ + "netflow_flow" + ], + "event.ingested": [ + "2023-12-01T08:11:39Z" + ], + "@timestamp": [ + "2023-12-01T08:11:38.000Z" + ], + "data_stream.dataset": [ + "netflow.log" + ], + "event.type": [ + "connection" + ], + "agent.ephemeral_id": [ + "2c589e2c-2889-415c-a1d3-abc7aeb27f41" + ], + "event.dataset": [ + "netflow.log" + ], + "netflow.egress_interface": [ + 2 + ] + } + }, + { + "_index": ".ds-logs-netflow.log-default-2023.11.29-000003", + "_id": "0FRuJIwB9hE7__H6-xhC", + "_score": 5.000033, + "fields": { + "flow.id": [ + "kmkdh7501nI" + ], + "elastic_agent.version": [ + "8.10.2" + ], + "event.category": [ + "network" + ], + "netflow.ip_class_of_service": [ + 0 + ], + "netflow.source_transport_port": [ + 35340 + ], + "netflow.tcp_control_bits": [ + 27 + ], + "netflow.exporter.version": [ + 9 + ], + "netflow.exporter.address": [ + "192.168.200.120:3455" + ], + "netflow.ip_version": [ + 4 + ], + "source.ip": [ + "10.0.10.10" + ], + "agent.name": [ + "ipfix-agent-ch4cq" + ], + "network.community_id": [ + "1:WKydzbXVu9IwFFuVVrA/KwJFFNA=" + ], + "event.agent_id_status": [ + "auth_metadata_missing" + ], + "event.kind": [ + "event" + ], + "source.packets": [ + 7 + ], + "network.packets": [ + 7 + ], + "netflow.destination_ipv4_address": [ + "10.0.20.20" + ], + "netflow.flow_start_sys_up_time": [ + 321292160 + ], + "flow.locality": [ + "internal" + ], + "input.type": [ + "netflow" + ], + "data_stream.type": [ + "logs" + ], + "tags": [ + "netflow", + "forwarded" + ], + "agent.id": [ + "4267406d-d5d3-489b-8dd7-76cc7e1d336e" + ], + "source.port": [ + 35340 + ], + "ecs.version": [ + "8.10.0" + ], + "event.created": [ + "2023-12-01T08:11:41.275Z" + ], + "network.iana_number": [ + "6" + ], + "agent.version": [ + "8.10.2" + ], + "event.start": [ + "2023-12-01T08:09:43.543Z" + ], + "observer.ip": [ + "192.168.200.120" + ], + "netflow.source_ipv4_address": [ + "10.0.10.10" + ], + "netflow.type": [ + "netflow_flow" + ], + "destination.port": [ + 3001 + ], + "netflow.flow_end_sys_up_time": [ + 321292471 + ], + "event.end": [ + "2023-12-01T08:09:43.854Z" + ], + "netflow.octet_delta_count": [ + 834 + ], + "agent.type": [ + "filebeat" + ], + "netflow.exporter.source_id": [ + 0 + ], + "event.module": [ + "netflow" + ], + "related.ip": [ + "10.0.10.10", + "10.0.20.20" + ], + "netflow.ingress_interface": [ + 2 + ], + "netflow.packet_delta_count": [ + 7 + ], + "network.bytes": [ + 834 + ], + "elastic_agent.snapshot": [ + false + ], + "network.direction": [ + "internal" + ], + "network.type": [ + "ipv4" + ], + "netflow.exporter.uptime_millis": [ + 321409617 + ], + "source.bytes": [ + 834 + ], + "destination.locality": [ + "internal" + ], + "elastic_agent.id": [ + "4267406d-d5d3-489b-8dd7-76cc7e1d336e" + ], + "data_stream.namespace": [ + "default" + ], + "netflow.destination_transport_port": [ + 3001 + ], + "netflow.exporter.timestamp": [ + "2023-12-01T08:11:41.000Z" + ], + "destination.ip": [ + "10.0.20.20" + ], + "source.locality": [ + "internal" + ], + "network.transport": [ + "tcp" + ], + "event.duration": [ + 311000000 + ], + "netflow.protocol_identifier": [ + 6 + ], + "event.action": [ + "netflow_flow" + ], + "event.ingested": [ + "2023-12-01T08:11:42Z" + ], + "@timestamp": [ + "2023-12-01T08:11:41.000Z" + ], + "data_stream.dataset": [ + "netflow.log" + ], + "event.type": [ + "connection" + ], + "agent.ephemeral_id": [ + "2c589e2c-2889-415c-a1d3-abc7aeb27f41" + ], + "event.dataset": [ + "netflow.log" + ], + "netflow.egress_interface": [ + 2 + ] + } + } + ] + } +} \ No newline at end of file diff --git a/processor/ipfixlookupprocessor/testdata/traces/NoResult-int.after.yaml b/processor/ipfixlookupprocessor/testdata/traces/NoResult-int.after.yaml new file mode 100644 index 000000000000..c0c2a8ebbd7a --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/NoResult-int.after.yaml @@ -0,0 +1,281 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + intValue: "65535" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" + - resource: + attributes: + - key: service.name + value: + stringValue: IPFIX + scopeSpans: + - scope: + name: ipfix + spans: + - attributes: + - key: NoHits-Warning + value: + stringValue: |- + No hits were found when searching! + This could be due to: + - Sampling rate + - Bad time settings + - Others + - key: dst.ip + value: + stringValue: 10.0.20.20 + - key: dst.port + value: + stringValue: "3001" + - key: ipfix.request.flows + value: + intValue: "0" + - key: ipfix.response.flows + value: + intValue: "0" + - key: src.ip + value: + stringValue: 10.0.10.10 + - key: src.port + value: + stringValue: "65535" + - key: z.elasticQuery.request + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "65535" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: z.elasticQuery.response + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "65535" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + endTimeUnixNano: "1672527604000000000" + name: 10.0.10.10:65535-10.0.20.20:3001 + parentSpanId: "1000000000000000" + spanId: "0101010101010101" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" + - endTimeUnixNano: "1672527604000000000" + name: Request - no IPFIX logs found + parentSpanId: "0101010101010101" + spanId: "0202020202020202" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" + - endTimeUnixNano: "1672527604000000000" + name: Response - no IPFIX logs found + parentSpanId: "0101010101010101" + spanId: "0303030303030303" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" diff --git a/processor/ipfixlookupprocessor/testdata/traces/NoResult-int.before.yaml b/processor/ipfixlookupprocessor/testdata/traces/NoResult-int.before.yaml new file mode 100644 index 000000000000..a04a27022759 --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/NoResult-int.before.yaml @@ -0,0 +1,40 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + intValue: 65535 + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" \ No newline at end of file diff --git a/processor/ipfixlookupprocessor/testdata/traces/NoResult-int.output.yaml b/processor/ipfixlookupprocessor/testdata/traces/NoResult-int.output.yaml new file mode 100644 index 000000000000..c0c2a8ebbd7a --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/NoResult-int.output.yaml @@ -0,0 +1,281 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + intValue: "65535" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" + - resource: + attributes: + - key: service.name + value: + stringValue: IPFIX + scopeSpans: + - scope: + name: ipfix + spans: + - attributes: + - key: NoHits-Warning + value: + stringValue: |- + No hits were found when searching! + This could be due to: + - Sampling rate + - Bad time settings + - Others + - key: dst.ip + value: + stringValue: 10.0.20.20 + - key: dst.port + value: + stringValue: "3001" + - key: ipfix.request.flows + value: + intValue: "0" + - key: ipfix.response.flows + value: + intValue: "0" + - key: src.ip + value: + stringValue: 10.0.10.10 + - key: src.port + value: + stringValue: "65535" + - key: z.elasticQuery.request + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "65535" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: z.elasticQuery.response + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "65535" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + endTimeUnixNano: "1672527604000000000" + name: 10.0.10.10:65535-10.0.20.20:3001 + parentSpanId: "1000000000000000" + spanId: "0101010101010101" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" + - endTimeUnixNano: "1672527604000000000" + name: Request - no IPFIX logs found + parentSpanId: "0101010101010101" + spanId: "0202020202020202" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" + - endTimeUnixNano: "1672527604000000000" + name: Response - no IPFIX logs found + parentSpanId: "0101010101010101" + spanId: "0303030303030303" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" diff --git a/processor/ipfixlookupprocessor/testdata/traces/NoResult.after.yaml b/processor/ipfixlookupprocessor/testdata/traces/NoResult.after.yaml new file mode 100644 index 000000000000..6da7bac31899 --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/NoResult.after.yaml @@ -0,0 +1,281 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "65535" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" + - resource: + attributes: + - key: service.name + value: + stringValue: IPFIX + scopeSpans: + - scope: + name: ipfix + spans: + - attributes: + - key: NoHits-Warning + value: + stringValue: |- + No hits were found when searching! + This could be due to: + - Sampling rate + - Bad time settings + - Others + - key: dst.ip + value: + stringValue: 10.0.20.20 + - key: dst.port + value: + stringValue: "3001" + - key: ipfix.request.flows + value: + intValue: "0" + - key: ipfix.response.flows + value: + intValue: "0" + - key: src.ip + value: + stringValue: 10.0.10.10 + - key: src.port + value: + stringValue: "65535" + - key: z.elasticQuery.request + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "65535" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: z.elasticQuery.response + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "65535" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + endTimeUnixNano: "1672527604000000000" + name: 10.0.10.10:65535-10.0.20.20:3001 + parentSpanId: "1000000000000000" + spanId: "0101010101010101" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" + - endTimeUnixNano: "1672527604000000000" + name: Request - no IPFIX logs found + parentSpanId: "0101010101010101" + spanId: "0202020202020202" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" + - endTimeUnixNano: "1672527604000000000" + name: Response - no IPFIX logs found + parentSpanId: "0101010101010101" + spanId: "0303030303030303" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" diff --git a/processor/ipfixlookupprocessor/testdata/traces/NoResult.before.yaml b/processor/ipfixlookupprocessor/testdata/traces/NoResult.before.yaml new file mode 100644 index 000000000000..f77e2b054d3c --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/NoResult.before.yaml @@ -0,0 +1,40 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "65535" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" \ No newline at end of file diff --git a/processor/ipfixlookupprocessor/testdata/traces/NoResult.output.yaml b/processor/ipfixlookupprocessor/testdata/traces/NoResult.output.yaml new file mode 100644 index 000000000000..6da7bac31899 --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/NoResult.output.yaml @@ -0,0 +1,281 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "65535" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" + - resource: + attributes: + - key: service.name + value: + stringValue: IPFIX + scopeSpans: + - scope: + name: ipfix + spans: + - attributes: + - key: NoHits-Warning + value: + stringValue: |- + No hits were found when searching! + This could be due to: + - Sampling rate + - Bad time settings + - Others + - key: dst.ip + value: + stringValue: 10.0.20.20 + - key: dst.port + value: + stringValue: "3001" + - key: ipfix.request.flows + value: + intValue: "0" + - key: ipfix.response.flows + value: + intValue: "0" + - key: src.ip + value: + stringValue: 10.0.10.10 + - key: src.port + value: + stringValue: "65535" + - key: z.elasticQuery.request + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "65535" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: z.elasticQuery.response + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "65535" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + endTimeUnixNano: "1672527604000000000" + name: 10.0.10.10:65535-10.0.20.20:3001 + parentSpanId: "1000000000000000" + spanId: "0101010101010101" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" + - endTimeUnixNano: "1672527604000000000" + name: Request - no IPFIX logs found + parentSpanId: "0101010101010101" + spanId: "0202020202020202" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" + - endTimeUnixNano: "1672527604000000000" + name: Response - no IPFIX logs found + parentSpanId: "0101010101010101" + spanId: "0303030303030303" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" diff --git a/processor/ipfixlookupprocessor/testdata/traces/OneSpanOneResult.after.yaml b/processor/ipfixlookupprocessor/testdata/traces/OneSpanOneResult.after.yaml new file mode 100644 index 000000000000..c7ad6b05b11c --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/OneSpanOneResult.after.yaml @@ -0,0 +1,655 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "54321" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" + - resource: + attributes: + - key: service.name + value: + stringValue: IPFIX + scopeSpans: + - scope: + name: ipfix + spans: + - attributes: + - key: NoHits-Warning + value: + stringValue: |- + No hits were found when searching! + This could be due to: + - Sampling rate + - Bad time settings + - Others + - key: dst.ip + value: + stringValue: 10.0.20.20 + - key: dst.port + value: + stringValue: "3001" + - key: ipfix.request.flows + value: + intValue: "1" + - key: ipfix.response.flows + value: + intValue: "0" + - key: src.ip + value: + stringValue: 10.0.10.10 + - key: src.port + value: + stringValue: "54321" + - key: z.elasticQuery.request + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "54321" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: z.elasticQuery.response + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "54321" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + endTimeUnixNano: "1700494981411000000" + name: 10.0.10.10:54321-10.0.20.20:3001 + parentSpanId: "1000000000000000" + spanId: "0101010101010101" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.destination\.port.0 + value: + intValue: "3001" + - key: fields.event\.duration.0 + value: + intValue: "448000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.100 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.source\.port.0 + value: + intValue: "54321" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: auVm7YsB9hE7__H6VGKl + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.10.30-000002 + - key: _score + value: + doubleValue: 5.000001 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: b65c4bd4-dec2-46dc-b2a7-2d8a5ded7d25 + - key: agent.id + value: + arrayValue: + values: + - stringValue: bdf672da-9aa2-4205-b423-fc105b6b14e1 + - key: agent.name + value: + arrayValue: + values: + - stringValue: quickstart-agent-n5j6z + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: bdf672da-9aa2-4205-b423-fc105b6b14e1 + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.448Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 4.48e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:01.411Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:08Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:00.963Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: 9of6On_LptU + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.100:12377 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 1.7533552e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 1.75329931e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 1.75329483e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 834 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 7 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 54321 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:v6MCwUb4XYI1KPRAFQ5jX/W7FaY= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.100 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 54321 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1700494981411000000" + name: Request - 192.168.200.100 + parentSpanId: "0101010101010101" + spanId: "0202020202020202" + startTimeUnixNano: "1700494980963000000" + status: {} + traceId: "10000000000000000000000000000000" + - endTimeUnixNano: "1672527604000000000" + name: Response - no IPFIX logs found + parentSpanId: "0101010101010101" + spanId: "0303030303030303" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" diff --git a/processor/ipfixlookupprocessor/testdata/traces/OneSpanOneResult.before.yaml b/processor/ipfixlookupprocessor/testdata/traces/OneSpanOneResult.before.yaml new file mode 100644 index 000000000000..899bbe72a8df --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/OneSpanOneResult.before.yaml @@ -0,0 +1,40 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "54321" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" \ No newline at end of file diff --git a/processor/ipfixlookupprocessor/testdata/traces/OneSpanOneResult.output.yaml b/processor/ipfixlookupprocessor/testdata/traces/OneSpanOneResult.output.yaml new file mode 100644 index 000000000000..c7ad6b05b11c --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/OneSpanOneResult.output.yaml @@ -0,0 +1,655 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "54321" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" + - resource: + attributes: + - key: service.name + value: + stringValue: IPFIX + scopeSpans: + - scope: + name: ipfix + spans: + - attributes: + - key: NoHits-Warning + value: + stringValue: |- + No hits were found when searching! + This could be due to: + - Sampling rate + - Bad time settings + - Others + - key: dst.ip + value: + stringValue: 10.0.20.20 + - key: dst.port + value: + stringValue: "3001" + - key: ipfix.request.flows + value: + intValue: "1" + - key: ipfix.response.flows + value: + intValue: "0" + - key: src.ip + value: + stringValue: 10.0.10.10 + - key: src.port + value: + stringValue: "54321" + - key: z.elasticQuery.request + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "54321" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: z.elasticQuery.response + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "54321" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + endTimeUnixNano: "1700494981411000000" + name: 10.0.10.10:54321-10.0.20.20:3001 + parentSpanId: "1000000000000000" + spanId: "0101010101010101" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.destination\.port.0 + value: + intValue: "3001" + - key: fields.event\.duration.0 + value: + intValue: "448000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.100 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.source\.port.0 + value: + intValue: "54321" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: auVm7YsB9hE7__H6VGKl + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.10.30-000002 + - key: _score + value: + doubleValue: 5.000001 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: b65c4bd4-dec2-46dc-b2a7-2d8a5ded7d25 + - key: agent.id + value: + arrayValue: + values: + - stringValue: bdf672da-9aa2-4205-b423-fc105b6b14e1 + - key: agent.name + value: + arrayValue: + values: + - stringValue: quickstart-agent-n5j6z + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: bdf672da-9aa2-4205-b423-fc105b6b14e1 + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.448Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 4.48e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:01.411Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:08Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:00.963Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: 9of6On_LptU + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.100:12377 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 1.7533552e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 1.75329931e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 1.75329483e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 834 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 7 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 54321 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:v6MCwUb4XYI1KPRAFQ5jX/W7FaY= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.100 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 54321 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1700494981411000000" + name: Request - 192.168.200.100 + parentSpanId: "0101010101010101" + spanId: "0202020202020202" + startTimeUnixNano: "1700494980963000000" + status: {} + traceId: "10000000000000000000000000000000" + - endTimeUnixNano: "1672527604000000000" + name: Response - no IPFIX logs found + parentSpanId: "0101010101010101" + spanId: "0303030303030303" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" diff --git a/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithAnswer.after.yaml b/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithAnswer.after.yaml new file mode 100644 index 000000000000..599c7a66544b --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithAnswer.after.yaml @@ -0,0 +1,1021 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "41662" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" + - resource: + attributes: + - key: service.name + value: + stringValue: IPFIX + scopeSpans: + - scope: + name: ipfix + spans: + - attributes: + - key: dst.ip + value: + stringValue: 10.0.20.20 + - key: dst.port + value: + stringValue: "3001" + - key: ipfix.request.flows + value: + intValue: "1" + - key: ipfix.response.flows + value: + intValue: "1" + - key: src.ip + value: + stringValue: 10.0.10.10 + - key: src.port + value: + stringValue: "41662" + - key: z.elasticQuery.request + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "41662" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: z.elasticQuery.response + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "41662" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + endTimeUnixNano: "1700494981411000000" + name: 10.0.10.10:41662-10.0.20.20:3001 + parentSpanId: "1000000000000000" + spanId: "0101010101010101" + startTimeUnixNano: "1700494980963000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.destination\.port.0 + value: + intValue: "3001" + - key: fields.event\.duration.0 + value: + intValue: "448000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.100 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.source\.port.0 + value: + intValue: "41662" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: auVm7YsB9hE7__H6VGKl + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.10.30-000002 + - key: _score + value: + doubleValue: 5.000001 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: b65c4bd4-dec2-46dc-b2a7-2d8a5ded7d25 + - key: agent.id + value: + arrayValue: + values: + - stringValue: bdf672da-9aa2-4205-b423-fc105b6b14e1 + - key: agent.name + value: + arrayValue: + values: + - stringValue: quickstart-agent-n5j6z + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: bdf672da-9aa2-4205-b423-fc105b6b14e1 + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.448Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 4.48e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:01.411Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:08Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:00.963Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: 9of6On_LptU + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.100:12377 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 1.7533552e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 1.75329931e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 1.75329483e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 834 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 7 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 41662 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:v6MCwUb4XYI1KPRAFQ5jX/W7FaY= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.100 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 41662 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1700494981411000000" + name: Request - 192.168.200.100 + parentSpanId: "0101010101010101" + spanId: "0202020202020202" + startTimeUnixNano: "1700494980963000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.destination\.port.0 + value: + intValue: "41662" + - key: fields.event\.duration.0 + value: + intValue: "448000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.100 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.source\.port.0 + value: + intValue: "3001" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: a-Vm7YsB9hE7__H6VGKl + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.10.30-000002 + - key: _score + value: + doubleValue: 5.000001 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: b65c4bd4-dec2-46dc-b2a7-2d8a5ded7d25 + - key: agent.id + value: + arrayValue: + values: + - stringValue: bdf672da-9aa2-4205-b423-fc105b6b14e1 + - key: agent.name + value: + arrayValue: + values: + - stringValue: quickstart-agent-n5j6z + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 41662 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: bdf672da-9aa2-4205-b423-fc105b6b14e1 + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.448Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 4.48e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:01.411Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:08Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:00.963Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: 9of6On_LptU + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 41662 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.100:12377 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 1.7533552e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 1.75329931e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 1.75329483e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 1014 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 1014 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:v6MCwUb4XYI1KPRAFQ5jX/W7FaY= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 6 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.100 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 1014 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 6 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1700494981411000000" + name: Response - 192.168.200.100 + parentSpanId: "0101010101010101" + spanId: "0303030303030303" + startTimeUnixNano: "1700494980963000000" + status: {} + traceId: "10000000000000000000000000000000" diff --git a/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithAnswer.before.yaml b/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithAnswer.before.yaml new file mode 100644 index 000000000000..3565ccbba22a --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithAnswer.before.yaml @@ -0,0 +1,40 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "41662" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" \ No newline at end of file diff --git a/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithAnswer.output.yaml b/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithAnswer.output.yaml new file mode 100644 index 000000000000..599c7a66544b --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithAnswer.output.yaml @@ -0,0 +1,1021 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "41662" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" + - resource: + attributes: + - key: service.name + value: + stringValue: IPFIX + scopeSpans: + - scope: + name: ipfix + spans: + - attributes: + - key: dst.ip + value: + stringValue: 10.0.20.20 + - key: dst.port + value: + stringValue: "3001" + - key: ipfix.request.flows + value: + intValue: "1" + - key: ipfix.response.flows + value: + intValue: "1" + - key: src.ip + value: + stringValue: 10.0.10.10 + - key: src.port + value: + stringValue: "41662" + - key: z.elasticQuery.request + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "41662" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: z.elasticQuery.response + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "41662" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + endTimeUnixNano: "1700494981411000000" + name: 10.0.10.10:41662-10.0.20.20:3001 + parentSpanId: "1000000000000000" + spanId: "0101010101010101" + startTimeUnixNano: "1700494980963000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.destination\.port.0 + value: + intValue: "3001" + - key: fields.event\.duration.0 + value: + intValue: "448000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.100 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.source\.port.0 + value: + intValue: "41662" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: auVm7YsB9hE7__H6VGKl + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.10.30-000002 + - key: _score + value: + doubleValue: 5.000001 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: b65c4bd4-dec2-46dc-b2a7-2d8a5ded7d25 + - key: agent.id + value: + arrayValue: + values: + - stringValue: bdf672da-9aa2-4205-b423-fc105b6b14e1 + - key: agent.name + value: + arrayValue: + values: + - stringValue: quickstart-agent-n5j6z + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: bdf672da-9aa2-4205-b423-fc105b6b14e1 + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.448Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 4.48e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:01.411Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:08Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:00.963Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: 9of6On_LptU + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.100:12377 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 1.7533552e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 1.75329931e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 1.75329483e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 834 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 7 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 41662 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:v6MCwUb4XYI1KPRAFQ5jX/W7FaY= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.100 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 41662 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1700494981411000000" + name: Request - 192.168.200.100 + parentSpanId: "0101010101010101" + spanId: "0202020202020202" + startTimeUnixNano: "1700494980963000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.destination\.port.0 + value: + intValue: "41662" + - key: fields.event\.duration.0 + value: + intValue: "448000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.100 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.source\.port.0 + value: + intValue: "3001" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: a-Vm7YsB9hE7__H6VGKl + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.10.30-000002 + - key: _score + value: + doubleValue: 5.000001 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: b65c4bd4-dec2-46dc-b2a7-2d8a5ded7d25 + - key: agent.id + value: + arrayValue: + values: + - stringValue: bdf672da-9aa2-4205-b423-fc105b6b14e1 + - key: agent.name + value: + arrayValue: + values: + - stringValue: quickstart-agent-n5j6z + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 41662 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: bdf672da-9aa2-4205-b423-fc105b6b14e1 + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.448Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 4.48e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:01.411Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:08Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:00.963Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: 9of6On_LptU + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 41662 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.100:12377 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 1.7533552e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 1.75329931e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 1.75329483e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 1014 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 1014 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:v6MCwUb4XYI1KPRAFQ5jX/W7FaY= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 6 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.100 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 1014 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 6 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1700494981411000000" + name: Response - 192.168.200.100 + parentSpanId: "0101010101010101" + spanId: "0303030303030303" + startTimeUnixNano: "1700494980963000000" + status: {} + traceId: "10000000000000000000000000000000" diff --git a/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithTwoFirewalls.after.yaml b/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithTwoFirewalls.after.yaml new file mode 100644 index 000000000000..6a0b5bc8b2fc --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithTwoFirewalls.after.yaml @@ -0,0 +1,2545 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "54322" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" + - resource: + attributes: + - key: service.name + value: + stringValue: IPFIX + scopeSpans: + - scope: + name: ipfix + spans: + - attributes: + - key: dst.ip + value: + stringValue: 10.0.20.20 + - key: dst.port + value: + stringValue: "3001" + - key: ipfix.request.flows + value: + intValue: "3" + - key: ipfix.response.flows + value: + intValue: "3" + - key: src.ip + value: + stringValue: 10.0.10.10 + - key: src.port + value: + stringValue: "54322" + - key: z.elasticQuery.request + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "54322" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: z.elasticQuery.response + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "54322" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + endTimeUnixNano: "1701418183854000000" + name: 10.0.10.10:54322-10.0.20.20:3001 + parentSpanId: "1000000000000000" + spanId: "0101010101010101" + startTimeUnixNano: "1701418183101000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.destination\.port.0 + value: + intValue: "54322" + - key: fields.event\.duration.0 + value: + intValue: "454000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.110 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.source\.port.0 + value: + intValue: "3001" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: PVRtJIwB9hE7__H6RRK1 + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.11.29-000003 + - key: _score + value: + doubleValue: 5.0000377 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:49.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: 2c589e2c-2889-415c-a1d3-abc7aeb27f41 + - key: agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: agent.name + value: + arrayValue: + values: + - stringValue: ipfix-agent-ch4cq + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 54322 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:49.256Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 4.54e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.761Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:50Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.307Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: kmkdh7501nI + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 54322 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.110:10029 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:49.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 3.21299712e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.21294473e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.21294019e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 1014 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 1014 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:WKydzbXVu9IwFFuVVrA/KwJFFNA= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 6 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.110 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 1014 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 6 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1701418183761000000" + name: Request - 192.168.200.110 + parentSpanId: "0101010101010101" + spanId: "0202020202020202" + startTimeUnixNano: "1701418183307000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.destination\.port.0 + value: + intValue: "54322" + - key: fields.event\.duration.0 + value: + intValue: "312000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.110 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.source\.port.0 + value: + intValue: "3001" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: v1RuJIwB9hE7__H68RdM + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.11.29-000003 + - key: _score + value: + doubleValue: 5.0000377 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:38.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: 2c589e2c-2889-415c-a1d3-abc7aeb27f41 + - key: agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: agent.name + value: + arrayValue: + values: + - stringValue: ipfix-agent-ch4cq + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 54322 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:38.867Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 3.12e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.413Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:39Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.101Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: kmkdh7501nI + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 54322 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 2 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.110:4556 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:38.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 3.21409057e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.2129447e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.21294158e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 2 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 1014 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 1014 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:WKydzbXVu9IwFFuVVrA/KwJFFNA= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 6 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.110 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 1014 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 6 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1701418183413000000" + name: Request - 192.168.200.110 + parentSpanId: "0101010101010101" + spanId: "0303030303030303" + startTimeUnixNano: "1701418183101000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.destination\.port.0 + value: + intValue: "54322" + - key: fields.event\.duration.0 + value: + intValue: "311000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.120 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.source\.port.0 + value: + intValue: "3001" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: 0VRuJIwB9hE7__H6-xhC + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.11.29-000003 + - key: _score + value: + doubleValue: 5.0000377 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:41.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: 2c589e2c-2889-415c-a1d3-abc7aeb27f41 + - key: agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: agent.name + value: + arrayValue: + values: + - stringValue: ipfix-agent-ch4cq + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 54322 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:41.275Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 3.11e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.854Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:42Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.543Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: kmkdh7501nI + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 54322 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 2 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.120:3455 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:41.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 3.21409617e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.21292471e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.2129216e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 2 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 1014 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 1014 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:WKydzbXVu9IwFFuVVrA/KwJFFNA= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 6 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.120 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 1014 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 6 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1701418183854000000" + name: Request - 192.168.200.120 + parentSpanId: "0101010101010101" + spanId: "0404040404040404" + startTimeUnixNano: "1701418183543000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.destination\.port.0 + value: + intValue: "3001" + - key: fields.event\.duration.0 + value: + intValue: "454000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.110 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.source\.port.0 + value: + intValue: "35340" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: PFRtJIwB9hE7__H6RRK1 + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.11.29-000003 + - key: _score + value: + doubleValue: 5.000033 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:49.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: 2c589e2c-2889-415c-a1d3-abc7aeb27f41 + - key: agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: agent.name + value: + arrayValue: + values: + - stringValue: ipfix-agent-ch4cq + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:49.256Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 4.54e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.761Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:50Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.307Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: kmkdh7501nI + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.110:10029 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:49.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 3.21299712e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.21294473e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.21294019e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 834 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 7 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 35340 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:WKydzbXVu9IwFFuVVrA/KwJFFNA= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.110 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 35340 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1701418183761000000" + name: Response - 192.168.200.110 + parentSpanId: "0101010101010101" + spanId: "0505050505050505" + startTimeUnixNano: "1701418183307000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.destination\.port.0 + value: + intValue: "3001" + - key: fields.event\.duration.0 + value: + intValue: "312000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.110 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.source\.port.0 + value: + intValue: "35340" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: vlRuJIwB9hE7__H68RdM + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.11.29-000003 + - key: _score + value: + doubleValue: 5.000033 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:38.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: 2c589e2c-2889-415c-a1d3-abc7aeb27f41 + - key: agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: agent.name + value: + arrayValue: + values: + - stringValue: ipfix-agent-ch4cq + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:38.867Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 3.12e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.413Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:39Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.101Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: kmkdh7501nI + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 2 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.110:4556 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:38.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 3.21409057e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.2129447e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.21294158e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 2 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 834 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 7 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 35340 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:WKydzbXVu9IwFFuVVrA/KwJFFNA= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.110 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 35340 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1701418183413000000" + name: Response - 192.168.200.110 + parentSpanId: "0101010101010101" + spanId: "0606060606060606" + startTimeUnixNano: "1701418183101000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.destination\.port.0 + value: + intValue: "3001" + - key: fields.event\.duration.0 + value: + intValue: "311000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.120 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.source\.port.0 + value: + intValue: "35340" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: 0FRuJIwB9hE7__H6-xhC + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.11.29-000003 + - key: _score + value: + doubleValue: 5.000033 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:41.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: 2c589e2c-2889-415c-a1d3-abc7aeb27f41 + - key: agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: agent.name + value: + arrayValue: + values: + - stringValue: ipfix-agent-ch4cq + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:41.275Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 3.11e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.854Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:42Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.543Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: kmkdh7501nI + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 2 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.120:3455 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:41.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 3.21409617e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.21292471e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.2129216e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 2 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 834 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 7 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 35340 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:WKydzbXVu9IwFFuVVrA/KwJFFNA= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.120 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 35340 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1701418183854000000" + name: Response - 192.168.200.120 + parentSpanId: "0101010101010101" + spanId: "0707070707070707" + startTimeUnixNano: "1701418183543000000" + status: {} + traceId: "10000000000000000000000000000000" diff --git a/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithTwoFirewalls.before.yaml b/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithTwoFirewalls.before.yaml new file mode 100644 index 000000000000..850ceefa0536 --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithTwoFirewalls.before.yaml @@ -0,0 +1,40 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "54322" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" \ No newline at end of file diff --git a/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithTwoFirewalls.output.yaml b/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithTwoFirewalls.output.yaml new file mode 100644 index 000000000000..6a0b5bc8b2fc --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/OneSpanWithTwoFirewalls.output.yaml @@ -0,0 +1,2545 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "54322" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" + - resource: + attributes: + - key: service.name + value: + stringValue: IPFIX + scopeSpans: + - scope: + name: ipfix + spans: + - attributes: + - key: dst.ip + value: + stringValue: 10.0.20.20 + - key: dst.port + value: + stringValue: "3001" + - key: ipfix.request.flows + value: + intValue: "3" + - key: ipfix.response.flows + value: + intValue: "3" + - key: src.ip + value: + stringValue: 10.0.10.10 + - key: src.port + value: + stringValue: "54322" + - key: z.elasticQuery.request + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "54322" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: z.elasticQuery.response + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "54322" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + endTimeUnixNano: "1701418183854000000" + name: 10.0.10.10:54322-10.0.20.20:3001 + parentSpanId: "1000000000000000" + spanId: "0101010101010101" + startTimeUnixNano: "1701418183101000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.destination\.port.0 + value: + intValue: "54322" + - key: fields.event\.duration.0 + value: + intValue: "454000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.110 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.source\.port.0 + value: + intValue: "3001" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: PVRtJIwB9hE7__H6RRK1 + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.11.29-000003 + - key: _score + value: + doubleValue: 5.0000377 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:49.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: 2c589e2c-2889-415c-a1d3-abc7aeb27f41 + - key: agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: agent.name + value: + arrayValue: + values: + - stringValue: ipfix-agent-ch4cq + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 54322 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:49.256Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 4.54e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.761Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:50Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.307Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: kmkdh7501nI + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 54322 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.110:10029 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:49.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 3.21299712e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.21294473e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.21294019e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 1014 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 1014 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:WKydzbXVu9IwFFuVVrA/KwJFFNA= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 6 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.110 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 1014 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 6 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1701418183761000000" + name: Request - 192.168.200.110 + parentSpanId: "0101010101010101" + spanId: "0202020202020202" + startTimeUnixNano: "1701418183307000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.destination\.port.0 + value: + intValue: "54322" + - key: fields.event\.duration.0 + value: + intValue: "312000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.110 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.source\.port.0 + value: + intValue: "3001" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: v1RuJIwB9hE7__H68RdM + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.11.29-000003 + - key: _score + value: + doubleValue: 5.0000377 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:38.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: 2c589e2c-2889-415c-a1d3-abc7aeb27f41 + - key: agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: agent.name + value: + arrayValue: + values: + - stringValue: ipfix-agent-ch4cq + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 54322 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:38.867Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 3.12e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.413Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:39Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.101Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: kmkdh7501nI + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 54322 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 2 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.110:4556 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:38.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 3.21409057e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.2129447e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.21294158e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 2 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 1014 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 1014 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:WKydzbXVu9IwFFuVVrA/KwJFFNA= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 6 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.110 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 1014 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 6 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1701418183413000000" + name: Request - 192.168.200.110 + parentSpanId: "0101010101010101" + spanId: "0303030303030303" + startTimeUnixNano: "1701418183101000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.destination\.port.0 + value: + intValue: "54322" + - key: fields.event\.duration.0 + value: + intValue: "311000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.120 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.source\.port.0 + value: + intValue: "3001" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: 0VRuJIwB9hE7__H6-xhC + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.11.29-000003 + - key: _score + value: + doubleValue: 5.0000377 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:41.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: 2c589e2c-2889-415c-a1d3-abc7aeb27f41 + - key: agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: agent.name + value: + arrayValue: + values: + - stringValue: ipfix-agent-ch4cq + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 54322 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:41.275Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 3.11e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.854Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:42Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.543Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: kmkdh7501nI + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 54322 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 2 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.120:3455 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:41.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 3.21409617e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.21292471e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.2129216e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 2 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 1014 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 1014 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:WKydzbXVu9IwFFuVVrA/KwJFFNA= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 6 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.120 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 1014 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 6 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1701418183854000000" + name: Request - 192.168.200.120 + parentSpanId: "0101010101010101" + spanId: "0404040404040404" + startTimeUnixNano: "1701418183543000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.destination\.port.0 + value: + intValue: "3001" + - key: fields.event\.duration.0 + value: + intValue: "454000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.110 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.source\.port.0 + value: + intValue: "35340" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: PFRtJIwB9hE7__H6RRK1 + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.11.29-000003 + - key: _score + value: + doubleValue: 5.000033 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:49.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: 2c589e2c-2889-415c-a1d3-abc7aeb27f41 + - key: agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: agent.name + value: + arrayValue: + values: + - stringValue: ipfix-agent-ch4cq + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:49.256Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 4.54e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.761Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:50Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.307Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: kmkdh7501nI + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.110:10029 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:49.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 3.21299712e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.21294473e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.21294019e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 834 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 7 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 35340 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:WKydzbXVu9IwFFuVVrA/KwJFFNA= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.110 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 35340 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1701418183761000000" + name: Response - 192.168.200.110 + parentSpanId: "0101010101010101" + spanId: "0505050505050505" + startTimeUnixNano: "1701418183307000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.destination\.port.0 + value: + intValue: "3001" + - key: fields.event\.duration.0 + value: + intValue: "312000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.110 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.source\.port.0 + value: + intValue: "35340" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: vlRuJIwB9hE7__H68RdM + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.11.29-000003 + - key: _score + value: + doubleValue: 5.000033 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:38.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: 2c589e2c-2889-415c-a1d3-abc7aeb27f41 + - key: agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: agent.name + value: + arrayValue: + values: + - stringValue: ipfix-agent-ch4cq + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:38.867Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 3.12e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.413Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:39Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.101Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: kmkdh7501nI + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 2 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.110:4556 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:38.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 3.21409057e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.2129447e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.21294158e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 2 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 834 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 7 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 35340 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:WKydzbXVu9IwFFuVVrA/KwJFFNA= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.110 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 35340 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1701418183413000000" + name: Response - 192.168.200.110 + parentSpanId: "0101010101010101" + spanId: "0606060606060606" + startTimeUnixNano: "1701418183101000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.destination\.port.0 + value: + intValue: "3001" + - key: fields.event\.duration.0 + value: + intValue: "311000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.120 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.source\.port.0 + value: + intValue: "35340" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: 0FRuJIwB9hE7__H6-xhC + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.11.29-000003 + - key: _score + value: + doubleValue: 5.000033 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:41.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: 2c589e2c-2889-415c-a1d3-abc7aeb27f41 + - key: agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: agent.name + value: + arrayValue: + values: + - stringValue: ipfix-agent-ch4cq + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: 4267406d-d5d3-489b-8dd7-76cc7e1d336e + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:41.275Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 3.11e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.854Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:42Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:09:43.543Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: kmkdh7501nI + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 2 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.120:3455 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-12-01T08:11:41.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 3.21409617e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.21292471e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 3.2129216e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 2 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 834 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 7 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 35340 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:WKydzbXVu9IwFFuVVrA/KwJFFNA= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.120 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 35340 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1701418183854000000" + name: Response - 192.168.200.120 + parentSpanId: "0101010101010101" + spanId: "0707070707070707" + startTimeUnixNano: "1701418183543000000" + status: {} + traceId: "10000000000000000000000000000000" diff --git a/processor/ipfixlookupprocessor/testdata/traces/TwoSpanOneResult.after.yaml b/processor/ipfixlookupprocessor/testdata/traces/TwoSpanOneResult.after.yaml new file mode 100644 index 000000000000..e861e795afcc --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/TwoSpanOneResult.after.yaml @@ -0,0 +1,672 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "54321" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "54321" + endTimeUnixNano: "1672527605000000000" + name: server + parentSpanId: "3000000000000000" + spanId: "4000000000000000" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" + - resource: + attributes: + - key: service.name + value: + stringValue: IPFIX + scopeSpans: + - scope: + name: ipfix + spans: + - attributes: + - key: NoHits-Warning + value: + stringValue: |- + No hits were found when searching! + This could be due to: + - Sampling rate + - Bad time settings + - Others + - key: dst.ip + value: + stringValue: 10.0.20.20 + - key: dst.port + value: + stringValue: "3001" + - key: ipfix.request.flows + value: + intValue: "1" + - key: ipfix.response.flows + value: + intValue: "0" + - key: src.ip + value: + stringValue: 10.0.10.10 + - key: src.port + value: + stringValue: "54321" + - key: z.elasticQuery.request + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "54321" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: z.elasticQuery.response + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "54321" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + endTimeUnixNano: "1700494981411000000" + name: 10.0.10.10:54321-10.0.20.20:3001 + parentSpanId: "1000000000000000" + spanId: "0101010101010101" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.destination\.port.0 + value: + intValue: "3001" + - key: fields.event\.duration.0 + value: + intValue: "448000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.100 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.source\.port.0 + value: + intValue: "54321" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: auVm7YsB9hE7__H6VGKl + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.10.30-000002 + - key: _score + value: + doubleValue: 5.000001 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: b65c4bd4-dec2-46dc-b2a7-2d8a5ded7d25 + - key: agent.id + value: + arrayValue: + values: + - stringValue: bdf672da-9aa2-4205-b423-fc105b6b14e1 + - key: agent.name + value: + arrayValue: + values: + - stringValue: quickstart-agent-n5j6z + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: bdf672da-9aa2-4205-b423-fc105b6b14e1 + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.448Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 4.48e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:01.411Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:08Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:00.963Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: 9of6On_LptU + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.100:12377 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 1.7533552e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 1.75329931e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 1.75329483e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 834 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 7 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 54321 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:v6MCwUb4XYI1KPRAFQ5jX/W7FaY= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.100 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 54321 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1700494981411000000" + name: Request - 192.168.200.100 + parentSpanId: "0101010101010101" + spanId: "0202020202020202" + startTimeUnixNano: "1700494980963000000" + status: {} + traceId: "10000000000000000000000000000000" + - endTimeUnixNano: "1672527604000000000" + name: Response - no IPFIX logs found + parentSpanId: "0101010101010101" + spanId: "0303030303030303" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" diff --git a/processor/ipfixlookupprocessor/testdata/traces/TwoSpanOneResult.before.yaml b/processor/ipfixlookupprocessor/testdata/traces/TwoSpanOneResult.before.yaml new file mode 100644 index 000000000000..3301211d21f9 --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/TwoSpanOneResult.before.yaml @@ -0,0 +1,57 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "54321" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "54321" + endTimeUnixNano: "1672527605000000000" + name: server + parentSpanId: "3000000000000000" + spanId: "4000000000000000" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" \ No newline at end of file diff --git a/processor/ipfixlookupprocessor/testdata/traces/TwoSpanOneResult.output.yaml b/processor/ipfixlookupprocessor/testdata/traces/TwoSpanOneResult.output.yaml new file mode 100644 index 000000000000..e861e795afcc --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/TwoSpanOneResult.output.yaml @@ -0,0 +1,672 @@ +resourceSpans: + - resource: + attributes: + - key: resource.required + value: + stringValue: foo + - key: resource.optional + value: + stringValue: bar + scopeSpans: + - scope: {} + spans: + - attributes: + - key: span.optional + value: + stringValue: bar + endTimeUnixNano: "1672527601000000000" + name: proxy + parentSpanId: "" + spanId: "1000000000000000" + startTimeUnixNano: "1672527600000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "54321" + endTimeUnixNano: "1672527604000000000" + name: server + parentSpanId: "1000000000000000" + spanId: "3000000000000000" + startTimeUnixNano: "1672527603000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: http.host + value: + stringValue: 10.0.20.20:3001 + - key: net.peer.ip + value: + stringValue: 10.0.10.10 + - key: net.peer.port + value: + stringValue: "54321" + endTimeUnixNano: "1672527605000000000" + name: server + parentSpanId: "3000000000000000" + spanId: "4000000000000000" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" + - resource: + attributes: + - key: service.name + value: + stringValue: IPFIX + scopeSpans: + - scope: + name: ipfix + spans: + - attributes: + - key: NoHits-Warning + value: + stringValue: |- + No hits were found when searching! + This could be due to: + - Sampling rate + - Bad time settings + - Others + - key: dst.ip + value: + stringValue: 10.0.20.20 + - key: dst.port + value: + stringValue: "3001" + - key: ipfix.request.flows + value: + intValue: "1" + - key: ipfix.response.flows + value: + intValue: "0" + - key: src.ip + value: + stringValue: 10.0.10.10 + - key: src.port + value: + stringValue: "54321" + - key: z.elasticQuery.request + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "54321" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: z.elasticQuery.response + value: + kvlistValue: + values: + - key: _source + value: + boolValue: false + - key: fields + value: + arrayValue: + values: + - kvlistValue: + values: + - key: field + value: + stringValue: '*' + - key: include_unmapped + value: + stringValue: "true" + - key: query + value: + kvlistValue: + values: + - key: bool + value: + kvlistValue: + values: + - key: must + value: + arrayValue: + values: + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: input.type + value: + stringValue: netflow + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.ip + value: + stringValue: 10.0.20.20 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: source.port + value: + stringValue: "3001" + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.ip + value: + stringValue: 10.0.10.10 + - kvlistValue: + values: + - key: match + value: + kvlistValue: + values: + - key: destination.port + value: + stringValue: "54321" + - kvlistValue: + values: + - key: range + value: + kvlistValue: + values: + - key: '@timestamp' + value: + kvlistValue: + values: + - key: gte + value: + stringValue: "2022-12-31T23:00:04Z" + - key: lte + value: + stringValue: "2022-12-31T23:00:04Z" + endTimeUnixNano: "1700494981411000000" + name: 10.0.10.10:54321-10.0.20.20:3001 + parentSpanId: "1000000000000000" + spanId: "0101010101010101" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" + - attributes: + - key: fields.destination\.ip.0 + value: + stringValue: 10.0.20.20 + - key: fields.destination\.port.0 + value: + intValue: "3001" + - key: fields.event\.duration.0 + value: + intValue: "448000000" + - key: fields.netflow\.ip_next_hop_ipv4_address + value: + stringValue: "null" + - key: fields.observer\.ip.0 + value: + stringValue: 192.168.200.100 + - key: fields.source\.ip.0 + value: + stringValue: 10.0.10.10 + - key: fields.source\.port.0 + value: + intValue: "54321" + - key: hits.total.value + value: + stringValue: "null" + - key: z.elasticResponse + value: + kvlistValue: + values: + - key: _id + value: + stringValue: auVm7YsB9hE7__H6VGKl + - key: _index + value: + stringValue: .ds-logs-netflow.log-default-2023.10.30-000002 + - key: _score + value: + doubleValue: 5.000001 + - key: fields + value: + kvlistValue: + values: + - key: '@timestamp' + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.000Z" + - key: agent.ephemeral_id + value: + arrayValue: + values: + - stringValue: b65c4bd4-dec2-46dc-b2a7-2d8a5ded7d25 + - key: agent.id + value: + arrayValue: + values: + - stringValue: bdf672da-9aa2-4205-b423-fc105b6b14e1 + - key: agent.name + value: + arrayValue: + values: + - stringValue: quickstart-agent-n5j6z + - key: agent.type + value: + arrayValue: + values: + - stringValue: filebeat + - key: agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: data_stream.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: data_stream.namespace + value: + arrayValue: + values: + - stringValue: default + - key: data_stream.type + value: + arrayValue: + values: + - stringValue: logs + - key: destination.ip + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: destination.locality + value: + arrayValue: + values: + - stringValue: internal + - key: destination.port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: ecs.version + value: + arrayValue: + values: + - stringValue: 8.10.0 + - key: elastic_agent.id + value: + arrayValue: + values: + - stringValue: bdf672da-9aa2-4205-b423-fc105b6b14e1 + - key: elastic_agent.snapshot + value: + arrayValue: + values: + - boolValue: false + - key: elastic_agent.version + value: + arrayValue: + values: + - stringValue: 8.10.2 + - key: event.action + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: event.agent_id_status + value: + arrayValue: + values: + - stringValue: auth_metadata_missing + - key: event.category + value: + arrayValue: + values: + - stringValue: network + - key: event.created + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.448Z" + - key: event.dataset + value: + arrayValue: + values: + - stringValue: netflow.log + - key: event.duration + value: + arrayValue: + values: + - doubleValue: 4.48e+08 + - key: event.end + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:01.411Z" + - key: event.ingested + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:08Z" + - key: event.kind + value: + arrayValue: + values: + - stringValue: event + - key: event.module + value: + arrayValue: + values: + - stringValue: netflow + - key: event.start + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:00.963Z" + - key: event.type + value: + arrayValue: + values: + - stringValue: connection + - key: flow.id + value: + arrayValue: + values: + - stringValue: 9of6On_LptU + - key: flow.locality + value: + arrayValue: + values: + - stringValue: internal + - key: input.type + value: + arrayValue: + values: + - stringValue: netflow + - key: netflow.destination_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.20.20 + - key: netflow.destination_transport_port + value: + arrayValue: + values: + - doubleValue: 3001 + - key: netflow.egress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.exporter.address + value: + arrayValue: + values: + - stringValue: 192.168.200.100:12377 + - key: netflow.exporter.source_id + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.exporter.timestamp + value: + arrayValue: + values: + - stringValue: "2023-11-20T15:43:07.000Z" + - key: netflow.exporter.uptime_millis + value: + arrayValue: + values: + - doubleValue: 1.7533552e+08 + - key: netflow.exporter.version + value: + arrayValue: + values: + - doubleValue: 9 + - key: netflow.flow_end_sys_up_time + value: + arrayValue: + values: + - doubleValue: 1.75329931e+08 + - key: netflow.flow_start_sys_up_time + value: + arrayValue: + values: + - doubleValue: 1.75329483e+08 + - key: netflow.ingress_interface + value: + arrayValue: + values: + - doubleValue: 1 + - key: netflow.ip_class_of_service + value: + arrayValue: + values: + - doubleValue: 0 + - key: netflow.ip_version + value: + arrayValue: + values: + - doubleValue: 4 + - key: netflow.octet_delta_count + value: + arrayValue: + values: + - doubleValue: 834 + - key: netflow.packet_delta_count + value: + arrayValue: + values: + - doubleValue: 7 + - key: netflow.protocol_identifier + value: + arrayValue: + values: + - doubleValue: 6 + - key: netflow.source_ipv4_address + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: netflow.source_transport_port + value: + arrayValue: + values: + - doubleValue: 54321 + - key: netflow.tcp_control_bits + value: + arrayValue: + values: + - doubleValue: 27 + - key: netflow.type + value: + arrayValue: + values: + - stringValue: netflow_flow + - key: network.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: network.community_id + value: + arrayValue: + values: + - stringValue: 1:v6MCwUb4XYI1KPRAFQ5jX/W7FaY= + - key: network.direction + value: + arrayValue: + values: + - stringValue: internal + - key: network.iana_number + value: + arrayValue: + values: + - stringValue: "6" + - key: network.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: network.transport + value: + arrayValue: + values: + - stringValue: tcp + - key: network.type + value: + arrayValue: + values: + - stringValue: ipv4 + - key: observer.ip + value: + arrayValue: + values: + - stringValue: 192.168.200.100 + - key: related.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - stringValue: 10.0.20.20 + - key: source.bytes + value: + arrayValue: + values: + - doubleValue: 834 + - key: source.ip + value: + arrayValue: + values: + - stringValue: 10.0.10.10 + - key: source.locality + value: + arrayValue: + values: + - stringValue: internal + - key: source.packets + value: + arrayValue: + values: + - doubleValue: 7 + - key: source.port + value: + arrayValue: + values: + - doubleValue: 54321 + - key: tags + value: + arrayValue: + values: + - stringValue: netflow + - stringValue: forwarded + endTimeUnixNano: "1700494981411000000" + name: Request - 192.168.200.100 + parentSpanId: "0101010101010101" + spanId: "0202020202020202" + startTimeUnixNano: "1700494980963000000" + status: {} + traceId: "10000000000000000000000000000000" + - endTimeUnixNano: "1672527604000000000" + name: Response - no IPFIX logs found + parentSpanId: "0101010101010101" + spanId: "0303030303030303" + startTimeUnixNano: "1672527604000000000" + status: {} + traceId: "10000000000000000000000000000000" diff --git a/processor/ipfixlookupprocessor/testdata/traces/empty.elastic.json b/processor/ipfixlookupprocessor/testdata/traces/empty.elastic.json new file mode 100644 index 000000000000..595df3f4e84b --- /dev/null +++ b/processor/ipfixlookupprocessor/testdata/traces/empty.elastic.json @@ -0,0 +1,18 @@ +{ + "took": 15, + "timed_out": false, + "_shards": { + "total": 8, + "successful": 8, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 0, + "relation": "eq" + }, + "max_score": null, + "hits": [] + } +} \ No newline at end of file diff --git a/versions.yaml b/versions.yaml index 1acae5e3f58e..bef97fe4f3ac 100644 --- a/versions.yaml +++ b/versions.yaml @@ -144,6 +144,7 @@ module-sets: - github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor - github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor - github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbytraceprocessor + - github.com/open-telemetry/opentelemetry-collector-contrib/processor/ipfixlookupprocessor - github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor - github.com/open-telemetry/opentelemetry-collector-contrib/processor/logstransformprocessor - github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricsgenerationprocessor