fix: only export scrape_errors_total for related job filters (#915)

Author: burningalchemist

cmd/sql_exporter/promhttp.go

@@ -36,7 +36,11 @@ func ExporterHandlerFor(exporter sql_exporter.Exporter) http.Handler {
 
 		// Parse the query params and set the job filters if any
 		jobFilters := req.URL.Query()["jobs[]"]
-		exporter.SetJobFilters(jobFilters)
+		if err := exporter.SetJobFilters(jobFilters); err != nil {
+			slog.Warn("Error setting job filters, ignoring", "error", err)
+			http.Error(w, "Error setting job filters: "+err.Error(), http.StatusBadRequest)
+			return
+		}
 
 		// Go through prometheus.Gatherers to sanitize and sort metrics.
 		gatherer := prometheus.Gatherers{exporter.WithContext(ctx), sql_exporter.SvcRegistry}
@@ -61,6 +65,9 @@ func ExporterHandlerFor(exporter sql_exporter.Exporter) http.Handler {
 			}
 		}
 
+		// Filter the scrape_errors_total metric family to only include metrics for the jobs in the jobFilters list.
+		mfs = exporter.FilterScrapeErrorsTotal(mfs)
+
 		contentType := expfmt.Negotiate(req.Header)
 		buf := getBuf()
 		defer giveBuf(buf)
@@ -115,7 +122,8 @@ func contextFor(req *http.Request, exporter sql_exporter.Exporter) (context.Cont
 			// Subtract the timeout offset, unless the result would be negative or zero.
 			timeoutOffset := time.Duration(exporter.Config().Globals.TimeoutOffset)
 			if timeoutOffset > timeout {
-				slog.Error("global.scrape_timeout_offset is greater than Prometheus' scraping timeout, ignoring", "timeout", timeout, "timeoutOffset", timeoutOffset)
+				slog.Error("global.scrape_timeout_offset is greater than Prometheus' scraping timeout, ignoring",
+					"timeout", timeout, "timeoutOffset", timeoutOffset)
 			} else {
 				timeout -= timeoutOffset
 			}

collector.go

@@ -121,7 +121,8 @@ func (cc *cachingCollector) Collect(ctx context.Context, conn *sql.DB, ch chan<-
 		// Have the lock.
 		if age := collTime.Sub(cacheTime); age > cc.minInterval || len(cc.cache) == 0 {
 			// Cache contents are older than minInterval, collect fresh metrics, cache them and pipe them through.
-			slog.Debug("Collecting fresh metrics", "logContext", cc.rawColl.logContext, "min_interval", cc.minInterval.Seconds(), "cache_age", age.Seconds())
+			slog.Debug("Collecting fresh metrics", "logContext", cc.rawColl.logContext, "min_interval",
+				cc.minInterval.Seconds(), "cache_age", age.Seconds())
 			cacheChan := make(chan Metric, capMetricChan)
 			cc.cache = make([]Metric, 0, len(cc.cache))
 			go func() {
@@ -131,7 +132,8 @@ func (cc *cachingCollector) Collect(ctx context.Context, conn *sql.DB, ch chan<-
 			for metric := range cacheChan {
 				// catch invalid metrics and return them immediately, don't cache them
 				if ctx.Err() != nil {
-					slog.Debug("Context closed, returning invalid metric", "logContext", cc.rawColl.logContext)
+					slog.Debug("Context closed, returning invalid metric", "logContext",
+						cc.rawColl.logContext)
 					ch <- NewInvalidMetric(errors.Wrap(cc.rawColl.logContext, ctx.Err()))
 					continue
 				}
@@ -141,7 +143,8 @@ func (cc *cachingCollector) Collect(ctx context.Context, conn *sql.DB, ch chan<-
 			}
 			cacheTime = collTime
 		} else {
-			slog.Debug("Returning cached metrics", "logContext", cc.rawColl.logContext, "min_interval", cc.minInterval.Seconds(), "cache_age", age.Seconds())
+			slog.Debug("Returning cached metrics", "logContext", cc.rawColl.logContext, "min_interval",
+				cc.minInterval.Seconds(), "cache_age", age.Seconds())
 			for _, metric := range cc.cache {
 				ch <- metric
 			}

exporter.go

@@ -37,9 +37,12 @@ type Exporter interface {
 	// UpdateTarget updates the targets field
 	UpdateTarget([]Target)
 	// SetJobFilters sets the jobFilters field
-	SetJobFilters([]string)
+	SetJobFilters([]string) error
 	// DropErrorMetrics resets the scrape_errors_total metric
 	DropErrorMetrics()
+	// FilterScrapeErrorsTotal filters the scrape_errors_total metric family to only include metrics for the jobs in
+	// the jobFilters list.
+	FilterScrapeErrorsTotal([]*dto.MetricFamily) []*dto.MetricFamily
 }
 
 type exporter struct {
@@ -67,7 +70,8 @@ func NewExporter(configFile string) (Exporter, error) {
 
 	var targets []Target
 	if c.Target != nil {
-		target, err := NewTarget("", c.Target.Name, "", string(c.Target.DSN), c.Target.Collectors(), nil, c.Globals, c.Target.EnablePing)
+		target, err := NewTarget("", c.Target.Name, "", string(c.Target.DSN),
+			c.Target.Collectors(), nil, c.Globals, c.Target.EnablePing)
 		if err != nil {
 			return nil, err
 		}
@@ -91,7 +95,7 @@ func NewExporter(configFile string) (Exporter, error) {
 	return &exporter{
 		config:     c,
 		targets:    targets,
-		jobFilters: []string{},
+		jobFilters: nil,
 		ctx:        context.Background(),
 	}, nil
 }
@@ -181,6 +185,7 @@ func (e *exporter) Gather() ([]*dto.MetricFamily, error) {
 	for _, mf := range dtoMetricFamilies {
 		result = append(result, mf)
 	}
+
 	return result, errs
 }
 
@@ -192,13 +197,49 @@ func (e *exporter) filterTargets(jf []string) {
 				filteredTargets = append(filteredTargets, target)
 			}
 		}
-		if len(filteredTargets) == 0 {
-			slog.Error("No targets found for job filters. Nothing to scrape.")
-		}
 		e.targets = filteredTargets
 	}
 }
 
+// FilterScrapeErrorsTotal filters the scrape_errors_total metric family to only include metrics for the jobs in the
+// jobFilters list. If jobFilters is empty it returns the original metric families unmodified.
+func (e *exporter) FilterScrapeErrorsTotal(mfs []*dto.MetricFamily) []*dto.MetricFamily {
+	if len(e.jobFilters) == 0 {
+		return mfs
+	}
+
+	result := make([]*dto.MetricFamily, 0, len(mfs))
+	for _, mf := range mfs {
+		if mf.GetName() != "scrape_errors_total" {
+			result = append(result, mf)
+			continue
+		}
+
+		// Filter metrics in this family to only include those with a job label in the jobFilters list.
+		filtered := make([]*dto.Metric, 0, len(mf.Metric))
+		for _, metric := range mf.Metric {
+			for _, label := range metric.Label {
+				if label.GetName() == "job" {
+					if slices.Contains(e.jobFilters, label.GetValue()) {
+						filtered = append(filtered, metric)
+					}
+					break
+				}
+			}
+		}
+
+		if len(filtered) > 0 {
+			result = append(result, &dto.MetricFamily{
+				Name:   mf.Name,
+				Help:   mf.Help,
+				Type:   mf.Type,
+				Metric: filtered,
+			})
+		}
+	}
+	return result
+}
+
 // Config implements Exporter.
 func (e *exporter) Config() *config.Config {
 	return e.config
@@ -210,8 +251,31 @@ func (e *exporter) UpdateTarget(target []Target) {
 }
 
 // SetJobFilters implements Exporter.
-func (e *exporter) SetJobFilters(filters []string) {
+func (e *exporter) SetJobFilters(filters []string) error {
+	// If the filters list contains a single empty string, treat it as no filters.
+	if len(filters) == 0 || (len(filters) == 1 && filters[0] == "") {
+		slog.Debug("Received empty job filter, treating as no filters")
+		e.jobFilters = nil
+		return nil
+	}
+
+	// Single target mode has no jobs - filters are not applicable
+	if len(e.config.Jobs) == 0 {
+		slog.Warn("Job filters are not applicable in single target mode, ignoring", "filters", filters)
+		e.jobFilters = nil
+		return nil
+	}
+
+	for _, name := range filters {
+		if !slices.ContainsFunc(e.config.Jobs, func(j *config.JobConfig) bool {
+			return j.Name == name
+		}) {
+			return fmt.Errorf("invalid job name: %s", name)
+		}
+	}
+
 	e.jobFilters = filters
+	return nil
 }
 
 // DropErrorMetrics implements Exporter.
@@ -233,8 +297,12 @@ func registerScrapeErrorMetric() *prometheus.CounterVec {
 // split comma separated list of key=value pairs and return a map of key value pairs
 func parseContextLog(list string) map[string]string {
 	m := make(map[string]string)
-	for _, item := range strings.Split(list, ",") {
+	for item := range strings.SplitSeq(list, ",") {
 		parts := strings.SplitN(item, "=", 2)
+		if len(parts) != 2 {
+			slog.Warn("Invalid context log item, ignoring", "item", item)
+			continue
+		}
 		m[parts[0]] = parts[1]
 	}
 	return m
@@ -243,8 +311,5 @@ func parseContextLog(list string) map[string]string {
 // TrimMissingCtx trims the leading comma and space from the log context string.
 // Leading comma appears when previous parameter is undefined, which is a side-effect of running in single target mode.
 func TrimMissingCtx(logContext string) string {
-	if strings.HasPrefix(logContext, ",") {
-		logContext = strings.TrimLeft(logContext, ", ")
-	}
-	return logContext
+	return strings.TrimPrefix(logContext, ", ")
 }

job.go

@@ -45,7 +45,8 @@ func NewJob(jc *config.JobConfig, gc *config.GlobalConfig) (Job, errors.WithCont
 				}
 				constLabels[name] = value
 			}
-			t, err := NewTarget(j.logContext, tname, jc.Name, string(dsn), jc.Collectors(), constLabels, gc, jc.EnablePing)
+			t, err := NewTarget(j.logContext, tname, jc.Name, string(dsn), jc.Collectors(),
+				constLabels, gc, jc.EnablePing)
 			if err != nil {
 				return nil, err
 			}