diff --git a/config/config.go b/config/config.go index b7174bc8..897e745a 100644 --- a/config/config.go +++ b/config/config.go @@ -35,10 +35,11 @@ type Metrics struct { // Counter is a metric defining prometheus counter type Counter struct { - Name string `yaml:"name"` - Help string `yaml:"help"` - Table string `yaml:"table"` - Labels []Label `yaml:"labels"` + Name string `yaml:"name"` + Help string `yaml:"help"` + Table string `yaml:"table"` + PerfMap string `yaml:"perf_map"` + Labels []Label `yaml:"labels"` } // Histogram is a metric defining prometheus histogram diff --git a/decoder/cgroup.go b/decoder/cgroup.go index 8469130b..e2e0760e 100644 --- a/decoder/cgroup.go +++ b/decoder/cgroup.go @@ -1,15 +1,15 @@ - package decoder +package decoder import ( - "os" - "fmt" - "log" - "strconv" - "path/filepath" + "fmt" + "log" + "os" + "path/filepath" + "strconv" - "golang.org/x/sys/unix" "github.com/cloudflare/ebpf_exporter/config" - "github.com/iovisor/gobpf/bcc" + "github.com/iovisor/gobpf/bcc" + "golang.org/x/sys/unix" ) // CGroup is a decoder that transforms cgroup id to path in cgroupfs @@ -18,55 +18,52 @@ type CGroup struct { } // Decode transforms cgroup id to path in cgroupfs -func (c *CGroup) Decode(in []byte, conf config.Decoder) ([]byte, error) { - if c.cache == nil { - c.cache = map[uint64][]byte{} - } +func (c *CGroup) Decode(in []byte, _ config.Decoder) ([]byte, error) { + if c.cache == nil { + c.cache = map[uint64][]byte{} + } - cgroupID, err := strconv.Atoi(string(in)) - if err != nil { - return nil, err - } + cgroupID, err := strconv.Atoi(string(in)) + if err != nil { + return nil, err + } - if path, ok := c.cache[uint64(cgroupID)]; ok { - return path, nil - } + if path, ok := c.cache[uint64(cgroupID)]; ok { + return path, nil + } - if err = c.refreshCache(); err != nil { - log.Printf("Error refreshing cgroup id to path map: %s", err) - } + if err = c.refreshCache(); err != nil { + log.Printf("Error refreshing cgroup id to path map: %s", err) + } - if path, ok := c.cache[uint64(cgroupID)]; ok { - return path, nil - } + if path, ok := c.cache[uint64(cgroupID)]; ok { + return path, nil + } - return []byte(fmt.Sprintf("unknown_cgroup_id:%d", cgroupID)), nil + return []byte(fmt.Sprintf("unknown_cgroup_id:%d", cgroupID)), nil } func (c *CGroup) refreshCache() error { - byteOrder := bcc.GetHostByteOrder() + byteOrder := bcc.GetHostByteOrder() - cgroupPath := "/sys/fs/cgroup/unified" - if _, err := os.Stat(cgroupPath); os.IsNotExist(err) { - cgroupPath = "/sys/fs/cgroup" - } + cgroupPath := "/sys/fs/cgroup" - return filepath.Walk(cgroupPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } + return filepath.Walk(cgroupPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } - if !info.IsDir() { - return nil - } + if !info.IsDir() { + return nil + } - handle, _, err := unix.NameToHandleAt(unix.AT_FDCWD, path, 0) - if err != nil { - log.Printf("Error resolving handle of %s: %s", path, err) - } + handle, _, err := unix.NameToHandleAt(unix.AT_FDCWD, path, 0) + if err != nil { + log.Printf("Error resolving handle of %s: %s", path, err) + } - c.cache[byteOrder.Uint64(handle.Bytes())] = []byte(path) + c.cache[byteOrder.Uint64(handle.Bytes())] = []byte(path) - return nil - }) + return nil + }) } diff --git a/examples/oomkill.yaml b/examples/oomkill.yaml new file mode 100644 index 00000000..e84c1de6 --- /dev/null +++ b/examples/oomkill.yaml @@ -0,0 +1,47 @@ +programs: + # See: + # * https://github.com/iovisor/bcc/blob/master/tools/oomkill.py + # * https://github.com/iovisor/bcc/blob/master/tools/oomkill_example.txt + - name: oomkill + metrics: + counters: + - name: oom_kills + help: Count global and cgroup level OOMs + perf_map: events + labels: + - name: cgroup_path + size: 8 + decoders: + - name: uint + - name: cgroup + - name: global_oom + size: 1 + decoders: + - name: uint + kprobes: + oom_kill_process: count_ooms + code: | + #include + #include + #include + + BPF_PERF_OUTPUT(events); + + struct data_t { + u64 cgroup_id; + u8 global_oom; + }; + + void count_ooms(struct pt_regs *ctx, struct oom_control *oc, const char *message) { + struct data_t data = {}; + + struct mem_cgroup *mcg = oc->memcg; + if (!mcg) { + data.global_oom = 1; + events.perf_submit(ctx, &data, sizeof(data)); + return; + } + + data.cgroup_id = mcg->css.cgroup->kn->id.id; + events.perf_submit(ctx, &data, sizeof(data)); + } diff --git a/exporter/exporter.go b/exporter/exporter.go index b5808927..99496d3f 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -19,6 +19,7 @@ const prometheusNamespace = "ebpf_exporter" type Exporter struct { config config.Config modules map[string]*bcc.Module + perfMapCollectors []*PerfMapSink ksyms map[uint64]string enabledProgramsDesc *prometheus.Desc programInfoDesc *prometheus.Desc @@ -119,6 +120,11 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { } for _, counter := range program.Metrics.Counters { + if len(counter.PerfMap) != 0 { + perfSink := NewPerfMapSink(e.decoders, e.modules[program.Name], counter) + e.perfMapCollectors = append(e.perfMapCollectors, perfSink) + } + addDescs(program.Name, counter.Name, counter.Help, counter.Labels) } @@ -140,6 +146,10 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { } } + for _, perfMapCollector := range e.perfMapCollectors { + perfMapCollector.Collect(ch) + } + e.collectCounters(ch) e.collectHistograms(ch) } @@ -148,6 +158,10 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { func (e *Exporter) collectCounters(ch chan<- prometheus.Metric) { for _, program := range e.config.Programs { for _, counter := range program.Metrics.Counters { + if len(counter.PerfMap) != 0 { + continue + } + tableValues, err := e.tableValues(e.modules[program.Name], counter.Table, counter.Labels) if err != nil { log.Printf("Error getting table %q values for metric %q of program %q: %s", counter.Table, counter.Name, program.Name, err) diff --git a/exporter/perf_map.go b/exporter/perf_map.go new file mode 100644 index 00000000..7f9b5c15 --- /dev/null +++ b/exporter/perf_map.go @@ -0,0 +1,99 @@ +package exporter + +import ( + "log" + "time" + + "github.com/cloudflare/ebpf_exporter/config" + "github.com/cloudflare/ebpf_exporter/decoder" + "github.com/iovisor/gobpf/bcc" + "github.com/prometheus/client_golang/prometheus" +) + +type PerfMapSink struct { + counterConfig config.Counter + counterVec *prometheus.CounterVec +} + +func NewPerfMapSink(decoders *decoder.Set, module *bcc.Module, counterConfig config.Counter) *PerfMapSink { + var ( + receiveCh = make(chan []byte) + lostCh = make(chan uint64) + ) + + sink := &PerfMapSink{ + counterConfig: counterConfig, + } + sink.resetCounterVec() + + table := bcc.NewTable(module.TableId(counterConfig.PerfMap), module) + + perfMap, err := bcc.InitPerfMap(table, receiveCh, lostCh) + if err != nil { + log.Panicf("Can't init PerfMap: %s", err) + } + + go func(sink *PerfMapSink, counterConfig config.Counter, receiveCh <-chan []byte) { + for rawBytes := range receiveCh { + // https://github.com/cilium/ebpf/pull/94#discussion_r425823371 + // https://lore.kernel.org/patchwork/patch/1244339/ + var validDataSize uint + for _, labelConfig := range counterConfig.Labels { + validDataSize += labelConfig.Size + } + + labelValues, err := decoders.DecodeLabels(rawBytes[:validDataSize], counterConfig.Labels) + if err != nil { + if err == decoder.ErrSkipLabelSet { + continue + } + + log.Printf("failed to decode labels: %s", err) + } + + sink.counterVec.WithLabelValues(labelValues...).Inc() + + } + }(sink, counterConfig, receiveCh) + + go func(sink *PerfMapSink) { + ticker := time.NewTicker(time.Hour) + + for { + <-ticker.C + sink.resetCounterVec() + } + }(sink) + + perfMap.Start() + + return sink +} + +func (s *PerfMapSink) resetCounterVec() { + s.counterVec = createCounterVecForPerfMap(s.counterConfig, labelNamesFromCounterConfig(s.counterConfig)) +} + +func (s *PerfMapSink) Collect(ch chan<- prometheus.Metric) { + s.counterVec.Collect(ch) +} + +func (s *PerfMapSink) Describe(ch chan<- *prometheus.Desc) { + s.counterVec.Describe(ch) +} + +func createCounterVecForPerfMap(counterConfig config.Counter, labelNames []string) *prometheus.CounterVec { + return prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: prometheusNamespace, + Name: counterConfig.Name, + Help: counterConfig.Help, + }, labelNames) +} + +func labelNamesFromCounterConfig(counterConfig config.Counter) (labelNames []string) { + for _, label := range counterConfig.Labels { + labelNames = append(labelNames, label.Name) + } + + return +}