Wrap errors returned by Queryable to custom wrapper.

This allows us to distinguish between those errors and errors returned by PromQL engine, and react appropriately.

Signed-off-by: Peter Štibraný <[email protected]>

Author: pstibrany

pkg/ruler/compat.go

@@ -149,16 +149,30 @@ func MetricsQueryFunc(qf rules.QueryFunc, queries, failedQueries prometheus.Coun
 	return func(ctx context.Context, qs string, t time.Time) (promql.Vector, error) {
 		queries.Inc()
 		result, err := qf(ctx, qs, t)
-		// We rely on TranslateToPromqlApiError to do its job here... it returns nil, if err is nil.
-		// It returns promql.ErrStorage, if error should be reported back as 500.
-		// Other errors it returns are either for canceled or timed-out queriers (we're not reporting those as failures),
-		// or various user-errors (limits, duplicate samples, etc. ... also not failures).
-		//
-		// All errors will still be counted towards "evaluation failures" metrics and logged by Prometheus Ruler,
-		// but we only want internal errors here.
-		if _, ok := querier.TranslateToPromqlAPIError(err).(promql.ErrStorage); ok {
-			failedQueries.Inc()
+
+		// We only care about errors returned by underlying Queryable. Errors returned by PromQL engine are "user-errors",
+		// and not interesting here.
+		qerr := QueryableError{}
+		if err != nil && errors.As(err, &qerr) {
+			origErr := qerr.Unwrap()
+
+			// Not all errors returned by Queryable are interesting, only those that would result in 500 status code.
+			//
+			// We rely on TranslateToPromqlApiError to do its job here... it returns nil, if err is nil.
+			// It returns promql.ErrStorage, if error should be reported back as 500.
+			// Other errors it returns are either for canceled or timed-out queriers (we're not reporting those as failures),
+			// or various user-errors (limits, duplicate samples, etc. ... also not failures).
+			//
+			// All errors will still be counted towards "evaluation failures" metrics and logged by Prometheus Ruler,
+			// but we only want internal errors here.
+			if _, ok := querier.TranslateToPromqlAPIError(origErr).(promql.ErrStorage); ok {
+				failedQueries.Inc()
+			}
+
+			// Return unwrapped error.
+			return result, origErr
 		}
+
 		return result, err
 	}
 }
@@ -234,10 +248,10 @@ func DefaultTenantManagerFactory(cfg Config, p Pusher, q storage.Queryable, engi
 		}, []string{"user"})
 	}
 
-	// Make sure that queryable reports errors as expected by our MetricsQueryFunc.
-	// Note that if queryable is already wrapped into ErrorTranslateQueryable, wrapping it again
-	// is fine (see TestApiStatusCodesDoubleWrapper in querier package).
-	q = querier.NewErrorTranslateQueryable(q)
+	// Wrap errors returned by Queryable to our wrapper, so that we can distinguish between those errors
+	// and errors returned by PromQL engine. Errors from Queryable can be either caused by user (limits) or internal errors.
+	// Errors from PromQL are always "user" errors.
+	q = querier.NewErrorTranslateQueryableWithFn(q, WrapQueryableErrors)
 
 	return func(ctx context.Context, userID string, notifier *notifier.Manager, logger log.Logger, reg prometheus.Registerer) RulesManager {
 		var queryTime prometheus.Counter = nil
@@ -260,3 +274,23 @@ func DefaultTenantManagerFactory(cfg Config, p Pusher, q storage.Queryable, engi
 		})
 	}
 }
+
+type QueryableError struct {
+	err error
+}
+
+func (q QueryableError) Unwrap() error {
+	return q.err
+}
+
+func (q QueryableError) Error() string {
+	return q.err.Error()
+}
+
+func WrapQueryableErrors(err error) error {
+	if err == nil {
+		return err
+	}
+
+	return QueryableError{err: err}
+}

pkg/ruler/compat_test.go

@@ -229,7 +229,7 @@ func TestMetricsQueryFuncErrors(t *testing.T) {
 			failures := prometheus.NewCounter(prometheus.CounterOpts{})
 
 			mockFunc := func(ctx context.Context, q string, t time.Time) (promql.Vector, error) {
-				return promql.Vector{}, tc.returnedError
+				return promql.Vector{}, WrapQueryableErrors(tc.returnedError)
 			}
 			qf := MetricsQueryFunc(mockFunc, queries, failures)