Merge pull request #44 from bobrik/program-tags

Add ebpf_exporter_ebpf_programs with program tags

README.md

@@ -772,6 +772,41 @@ name: <decoder name>
 # ... decoder specific configuration
 ```
 
+## Built-in metrics
+
+### `ebpf_exporter_enabled_programs`
+
+This gauge reports a timeseries for every loaded logical program:
+
+```
+# HELP ebpf_exporter_enabled_programs The set of enabled programs
+# TYPE ebpf_exporter_enabled_programs gauge
+ebpf_exporter_enabled_programs{name="xfs_reclaim"} 1
+```
+
+### `ebpf_exporter_ebpf_programs`
+
+This gauge reports information available for every ebpf program:
+
+```
+# HELP ebpf_exporter_ebpf_programs Info about ebpf programs
+# TYPE ebpf_exporter_ebpf_programs gauge
+ebpf_exporter_ebpf_programs{function="xfs_fs_free_cached_objects_end",program="xfs_reclaim",tag="d5e845dc27b372e4"} 1
+ebpf_exporter_ebpf_programs{function="xfs_fs_free_cached_objects_start",program="xfs_reclaim",tag="c2439d02dd0ba000"} 1
+ebpf_exporter_ebpf_programs{function="xfs_fs_nr_cached_objects_end",program="xfs_reclaim",tag="598375893f34ef39"} 1
+ebpf_exporter_ebpf_programs{function="xfs_fs_nr_cached_objects_start",program="xfs_reclaim",tag="cf30348184f983dd"} 1
+```
+
+Here `tag` can be used for tracing and performance analysis with two conditions:
+
+* `net.core.bpf_jit_kallsyms=1` sysctl is set
+* `--kallsyms=/proc/kallsyms` is passed to `perf record`
+
+Newer kernels allow `--kallsyms` to `perf top` as well,
+in the future it may not be required at all:
+
+* https://www.spinics.net/lists/linux-perf-users/msg07216.html
+
 ## License
 
 MIT

benchmark/README.md

@@ -80,3 +80,8 @@ Tracing 1 functions for "sys_getpid"... Hit Ctrl-C to end.
 These measurements already include some overhead from `funclatency` itself.
 
 You can see frequency of calls in the output as well.
+
+## Performance analysis with perf
+
+See [`ebpf_exporter_ebpf_programs`](../README.md#ebpf_exporter_ebpf_programs)
+for more information on runtime analysis of performance impact.

exporter/attach.go

@@ -6,91 +6,95 @@ import (
 	"github.com/iovisor/gobpf/bcc"
 )
 
-// attach attaches functions to tracing points in provided module
-func attach(module *bcc.Module, kprobes, kretprobes, tracepoints, rawTracepoints map[string]string) error {
-	if err := attachKprobes(module, kprobes); err != nil {
-		return fmt.Errorf("failed to attach kprobes: %s", err)
-	}
+// attacher attaches some sort of tracepoints or probes
+type attacher func(*bcc.Module, map[string]string) (map[string]uint64, error)
 
-	if err := attachKretprobes(module, kretprobes); err != nil {
-		return fmt.Errorf("failed to attach kretprobes: %s", err)
-	}
+// mergeTags runs attacher and merges produced tags
+func mergedTags(dst map[string]uint64, attach attacher, module *bcc.Module, attachments map[string]string) error {
+	src, err := attach(module, attachments)
 
-	if err := attachTracepoints(module, tracepoints); err != nil {
-		return fmt.Errorf("failed to attach tracepoints: %s", err)
+	if err != nil {
+		return err
 	}
 
-	if err := attachRawTracepoints(module, rawTracepoints); err != nil {
-		return fmt.Errorf("failed to attach raw tracepoints: %s", err)
+	for name, tag := range src {
+		dst[name] = tag
 	}
 
 	return nil
 }
 
-// attachKprobes attaches functions to their kprobles in provided module
-func attachKprobes(module *bcc.Module, kprobes map[string]string) error {
-	for kprobeName, targetName := range kprobes {
-		target, err := module.LoadKprobe(targetName)
-		if err != nil {
-			return fmt.Errorf("failed to load target %q: %s", targetName, err)
-		}
+// attach attaches functions to tracing points in provided module
+func attach(module *bcc.Module, kprobes, kretprobes, tracepoints, rawTracepoints map[string]string) (map[string]uint64, error) {
+	tags := map[string]uint64{}
 
-		err = module.AttachKprobe(kprobeName, target)
-		if err != nil {
-			return fmt.Errorf("failed to attach kprobe %q to %q: %s", kprobeName, targetName, err)
-		}
+	if err := mergedTags(tags, attachKprobes, module, kprobes); err != nil {
+		return nil, fmt.Errorf("failed to attach kprobes: %s", err)
 	}
 
-	return nil
-}
+	if err := mergedTags(tags, attachKretprobes, module, kretprobes); err != nil {
+		return nil, fmt.Errorf("failed to attach kretprobes: %s", err)
+	}
 
-// attachKretprobes attaches functions to their kretprobles in provided module
-func attachKretprobes(module *bcc.Module, kretprobes map[string]string) error {
-	for kretprobeName, targetName := range kretprobes {
-		target, err := module.LoadKprobe(targetName)
-		if err != nil {
-			return fmt.Errorf("failed to load target %q: %s", targetName, err)
-		}
+	if err := mergedTags(tags, attachTracepoints, module, tracepoints); err != nil {
+		return nil, fmt.Errorf("failed to attach tracepoints: %s", err)
+	}
 
-		err = module.AttachKretprobe(kretprobeName, target)
-		if err != nil {
-			return fmt.Errorf("failed to attach kretprobe %q to %q: %s", kretprobeName, targetName, err)
-		}
+	if err := mergedTags(tags, attachRawTracepoints, module, rawTracepoints); err != nil {
+		return nil, fmt.Errorf("failed to attach raw tracepoints: %s", err)
 	}
 
-	return nil
+	return tags, nil
 }
 
-// attachTracepoints attaches functions to their tracepoints in provided module
-func attachTracepoints(module *bcc.Module, tracepoints map[string]string) error {
-	for tracepointName, targetName := range tracepoints {
-		target, err := module.LoadTracepoint(targetName)
-		if err != nil {
-			return fmt.Errorf("failed to load target %q: %s", targetName, err)
-		}
+// probeLoader attaches some sort of probe
+type probeLoader func(string) (int, error)
 
-		err = module.AttachTracepoint(tracepointName, target)
+// probeAttacher attaches loaded some sort of probe to some sort of tracepoint
+type probeAttacher func(string, int) error
+
+// attachSomething attaches some kind of probes and returns program tags
+func attachSomething(module *bcc.Module, loader probeLoader, attacher probeAttacher, probes map[string]string) (map[string]uint64, error) {
+	tags := map[string]uint64{}
+
+	for probe, targetName := range probes {
+		target, err := loader(targetName)
 		if err != nil {
-			return fmt.Errorf("failed to attach tracepoint %q to %q: %s", tracepointName, targetName, err)
+			return nil, fmt.Errorf("failed to load probe %q: %s", targetName, err)
 		}
-	}
-
-	return nil
-}
 
-// attachRawTracepoints attaches functions to their tracepoints in provided module
-func attachRawTracepoints(module *bcc.Module, tracepoints map[string]string) error {
-	for tracepointName, targetName := range tracepoints {
-		target, err := module.LoadRawTracepoint(targetName)
+		tag, err := module.GetProgramTag(target)
 		if err != nil {
-			return fmt.Errorf("failed to load target %q: %s", targetName, err)
+			return nil, fmt.Errorf("failed to get program tag for %q (fd=%d): %s", targetName, target, err)
 		}
 
-		err = module.AttachRawTracepoint(tracepointName, target)
+		tags[targetName] = tag
+
+		err = attacher(probe, target)
 		if err != nil {
-			return fmt.Errorf("failed to attach raw tracepoint %q to %q: %s", tracepointName, targetName, err)
+			return nil, fmt.Errorf("failed to attach probe %q to %q: %s", probe, targetName, err)
 		}
 	}
 
-	return nil
+	return tags, nil
+}
+
+// attachKprobes attaches functions to their kprobles in provided module
+func attachKprobes(module *bcc.Module, kprobes map[string]string) (map[string]uint64, error) {
+	return attachSomething(module, module.LoadKprobe, module.AttachKprobe, kprobes)
+}
+
+// attachKretprobes attaches functions to their kretprobles in provided module
+func attachKretprobes(module *bcc.Module, kretprobes map[string]string) (map[string]uint64, error) {
+	return attachSomething(module, module.LoadKprobe, module.AttachKretprobe, kretprobes)
+}
+
+// attachTracepoints attaches functions to their tracepoints in provided module
+func attachTracepoints(module *bcc.Module, tracepoints map[string]string) (map[string]uint64, error) {
+	return attachSomething(module, module.LoadTracepoint, module.AttachTracepoint, tracepoints)
+}
+
+// attachRawTracepoints attaches functions to their tracepoints in provided module
+func attachRawTracepoints(module *bcc.Module, tracepoints map[string]string) (map[string]uint64, error) {
+	return attachSomething(module, module.LoadRawTracepoint, module.AttachRawTracepoint, tracepoints)
 }

exporter/exporter.go

@@ -21,6 +21,8 @@ type Exporter struct {
 	modules             map[string]*bcc.Module
 	ksyms               map[uint64]string
 	enabledProgramsDesc *prometheus.Desc
+	programInfoDesc     *prometheus.Desc
+	programTags         map[string]map[string]uint64
 	descs               map[string]map[string]*prometheus.Desc
 	decoders            *decoder.Set
 }
@@ -34,11 +36,20 @@ func New(config config.Config) *Exporter {
 		nil,
 	)
 
+	programInfoDesc := prometheus.NewDesc(
+		prometheus.BuildFQName(prometheusNamespace, "", "ebpf_programs"),
+		"Info about ebpf programs",
+		[]string{"program", "function", "tag"},
+		nil,
+	)
+
 	return &Exporter{
 		config:              config,
 		modules:             map[string]*bcc.Module{},
 		ksyms:               map[uint64]string{},
 		enabledProgramsDesc: enabledProgramsDesc,
+		programInfoDesc:     programInfoDesc,
+		programTags:         map[string]map[string]uint64{},
 		descs:               map[string]map[string]*prometheus.Desc{},
 		decoders:            decoder.NewSet(),
 	}
@@ -56,10 +67,14 @@ func (e *Exporter) Attach() error {
 			return fmt.Errorf("error compiling module for program %q", program.Name)
 		}
 
-		if err := attach(module, program.Kprobes, program.Kretprobes, program.Tracepoints, program.RawTracepoints); err != nil {
+		tags, err := attach(module, program.Kprobes, program.Kretprobes, program.Tracepoints, program.RawTracepoints)
+
+		if err != nil {
 			return fmt.Errorf("failed to attach to program %q: %s", program.Name, err)
 		}
 
+		e.programTags[program.Name] = tags
+
 		for _, perfEventConfig := range program.PerfEvents {
 			target, err := module.LoadPerfEvent(perfEventConfig.Target)
 			if err != nil {
@@ -96,6 +111,7 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
 	}
 
 	ch <- e.enabledProgramsDesc
+	ch <- e.programInfoDesc
 
 	for _, program := range e.config.Programs {
 		if _, ok := e.descs[program.Name]; !ok {
@@ -118,6 +134,12 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
 		ch <- prometheus.MustNewConstMetric(e.enabledProgramsDesc, prometheus.GaugeValue, 1, program.Name)
 	}
 
+	for program, tags := range e.programTags {
+		for function, tag := range tags {
+			ch <- prometheus.MustNewConstMetric(e.programInfoDesc, prometheus.GaugeValue, 1, program, function, fmt.Sprintf("%x", tag))
+		}
+	}
+
 	e.collectCounters(ch)
 	e.collectHistograms(ch)
 }

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang/protobuf v1.2.0 // indirect
-	github.com/iovisor/gobpf v0.0.0-20181115102816-c5918016dd31
+	github.com/iovisor/gobpf v0.0.0-20190101180936-b0976764b280
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

go.sum

@@ -10,6 +10,8 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/iovisor/gobpf v0.0.0-20181115102816-c5918016dd31 h1:1aM9orF3Ap9PACtNbi+PLEFpyowpy0Xz9U/mLh28m+E=
 github.com/iovisor/gobpf v0.0.0-20181115102816-c5918016dd31/go.mod h1:+5U5qu5UOu8YJ5oHVLvWKH7/Dr5QNHU7mZ2RfPEeXg8=
+github.com/iovisor/gobpf v0.0.0-20190101180936-b0976764b280 h1:/s/gVPSoThcIBrunQtZNgTZJzSPWuI2ASMDdiRQio5g=
+github.com/iovisor/gobpf v0.0.0-20190101180936-b0976764b280/go.mod h1:+5U5qu5UOu8YJ5oHVLvWKH7/Dr5QNHU7mZ2RfPEeXg8=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=

vendor/modules.txt

@@ -7,7 +7,7 @@ github.com/alecthomas/units
 github.com/beorn7/perks/quantile
 # github.com/golang/protobuf v1.2.0
 github.com/golang/protobuf/proto
-# github.com/iovisor/gobpf v0.0.0-20181115102816-c5918016dd31
+# github.com/iovisor/gobpf v0.0.0-20190101180936-b0976764b280
 github.com/iovisor/gobpf/bcc
 github.com/iovisor/gobpf/pkg/ksym
 github.com/iovisor/gobpf/pkg/cpuonline