| package compare |
| |
| import ( |
| "encoding/json" |
| "math" |
| "slices" |
| "sort" |
| "testing" |
| |
| "github.com/stretchr/testify/assert" |
| "github.com/stretchr/testify/require" |
| |
| "go.skia.org/infra/pinpoint/go/compare/stats" |
| "go.skia.org/infra/pinpoint/go/compare/thresholds" |
| ) |
| |
| func convertNormalizedToRawMagnitude(a, b []float64, normMagnitude float64) float64 { |
| allValues := slices.Concat(a, b) |
| sort.Float64s(allValues) |
| iqr := allValues[len(allValues)*3/4] - allValues[len(allValues)/4] |
| return normMagnitude * iqr |
| } |
| |
| func TestCompareFunctional_GivenNoData_ReturnsError(t *testing.T) { |
| x := []float64{} |
| y := []float64{0, 0, 0, 0, 0} |
| result, err := CompareFunctional(x, y, DefaultFunctionalErrRate) |
| require.Error(t, err) |
| assert.Equal(t, ErrorVerdict, result.Verdict) |
| } |
| |
| func TestCompareFunctional_GivenValidInputs_ReturnsCorrectResult(t *testing.T) { |
| test := func(name string, x, y []float64, expectedErrRate float64, expected Verdict) { |
| t.Run(name, func(t *testing.T) { |
| result, err := CompareFunctional(x, y, expectedErrRate) |
| require.NoError(t, err) |
| assert.Equal(t, expected, result.Verdict) |
| switch result.Verdict { |
| case Unknown: |
| assert.LessOrEqual(t, result.PValue, result.HighThreshold) |
| assert.Greater(t, result.PValue, result.LowThreshold) |
| case Same: |
| assert.Greater(t, result.PValue, result.HighThreshold) |
| case Different: |
| assert.LessOrEqual(t, result.PValue, result.LowThreshold) |
| default: |
| t.Errorf("Obtained non-existent verdict %s", result.Verdict) |
| } |
| }) |
| } |
| x := []float64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0} |
| y := []float64{0, 0, 0, 0, 0, 0, 0, 0, 0, 1} |
| test("arrays are slightly different, return unknown", x, y, 0.5, Unknown) |
| |
| x = []float64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0} |
| y = []float64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0} |
| test("arrays are the same, return same", x, y, 1.0, Same) |
| |
| x = []float64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0} |
| y = []float64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1} |
| test("arrays are significantly different, return different", x, y, 1.0, Different) |
| } |
| |
| func TestComparePerformance_GivenNoData_ReturnsNil(t *testing.T) { |
| x := []float64{0, 0, 0, 0, 0} |
| y := []float64{} |
| const magnitude = 1.0 |
| result, err := ComparePerformance(x, y, magnitude, UnknownDir) |
| require.NoError(t, err) |
| assert.Equal(t, NilVerdict, result.Verdict) |
| } |
| |
| func TestComparePerformance_GivenValidInputs_ReturnsCorrectResult(t *testing.T) { |
| test := func(name string, x, y []float64, magnitude float64, expected Verdict) { |
| t.Run(name, func(t *testing.T) { |
| result, err := ComparePerformance(x, y, magnitude, UnknownDir) |
| require.NoError(t, err) |
| assert.Equal(t, expected, result.Verdict) |
| switch result.Verdict { |
| case Unknown: |
| assert.LessOrEqual(t, result.PValue, result.HighThreshold) |
| assert.Greater(t, result.PValue, result.LowThreshold) |
| case Same: |
| assert.Greater(t, result.PValue, result.HighThreshold) |
| case Different: |
| assert.LessOrEqual(t, result.PValue, result.LowThreshold) |
| default: |
| assert.Fail(t, "Unsupported verdict found %s", result.Verdict) |
| } |
| }) |
| } |
| x := []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} |
| y := []float64{3, 4, 5, 6, 7, 8, 9, 10, 11, 12} |
| mag := convertNormalizedToRawMagnitude(x, y, 0.5) |
| test("arrays are slightly different, return unknown", x, y, mag, Unknown) |
| |
| x = []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} |
| y = []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} |
| test("arrays are the same, return same", x, y, 0.0, Same) |
| |
| x = []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} |
| y = []float64{7, 8, 9, 10, 11, 12, 13, 14, 15, 16} |
| test("arrays are significantly different, return different", x, y, 0.0, Different) |
| } |
| |
| func TestComparePerformance_GivenSmallDifference_ReturnsSame(t *testing.T) { |
| x := []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} |
| y := []float64{7, 8, 9, 10, 11, 12, 13, 14, 15, 16} |
| result, err := ComparePerformance(x, y, 1e6, UnknownDir) |
| require.NoError(t, err) |
| assert.Equal(t, Same, result.Verdict) |
| assert.Zero(t, result.PValue) |
| assert.True(t, result.IsTooSmall) |
| } |
| |
| func TestCompare_GivenImprovement_ReturnsSameAndNoPValue(t *testing.T) { |
| test := func(name string, x, y []float64, dir ImprovementDir) { |
| t.Run(name, func(t *testing.T) { |
| result, err := compare(x, y, 0.0, 0.0, dir) |
| require.NoError(t, err) |
| assert.Equal(t, Same, result.Verdict) |
| assert.Zero(t, result.PValue) |
| switch dir { |
| case Up: |
| assert.Positive(t, result.MeanDiff) |
| case Down: |
| assert.Negative(t, result.MeanDiff) |
| } |
| }) |
| } |
| x := []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} |
| y := []float64{7, 8, 9, 10, 11, 12, 13, 14, 15, 16} |
| test("x < y, ImprovementDir = Up, return same", x, y, Up) |
| |
| x = []float64{7, 8, 9, 10, 11, 12, 13, 14, 15, 16} |
| y = []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} |
| test("x > y, ImprovementDir = Down, return same", x, y, Down) |
| } |
| |
| func TestCompare_GivenRegression_ReturnsPValue(t *testing.T) { |
| test := func(name string, x, y []float64, dir ImprovementDir) { |
| t.Run(name, func(t *testing.T) { |
| result, err := compare(x, y, 0.0, 0.0, dir) |
| require.NoError(t, err) |
| assert.NotZero(t, result.PValue) |
| switch dir { |
| case Up: |
| assert.Negative(t, result.MeanDiff) |
| case Down: |
| assert.Positive(t, result.MeanDiff) |
| } |
| }) |
| } |
| x := []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} |
| y := []float64{7, 8, 9, 10, 11, 12, 13, 14, 15, 16} |
| test("x < y, ImprovementDir = Down", x, y, Down) |
| |
| x = []float64{7, 8, 9, 10, 11, 12, 13, 14, 15, 16} |
| y = []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} |
| test("x > y, ImprovementDir = Up", x, y, Up) |
| } |
| |
| func TestComparePairwise_GivenImprovement_ReturnsSame(t *testing.T) { |
| test := func(name string, x, y []float64, dir ImprovementDir) { |
| t.Run(name, func(t *testing.T) { |
| result, err := ComparePairwise(x, y, dir) |
| require.NoError(t, err) |
| require.LessOrEqual(t, result.PairwiseWilcoxonSignedRankedTestResult.PValue, thresholds.LowThreshold) |
| assert.Equal(t, Same, result.Verdict) |
| switch dir { |
| case Up: |
| assert.Positive(t, result.PairwiseWilcoxonSignedRankedTestResult.UpperCi) |
| case Down: |
| assert.Negative(t, result.PairwiseWilcoxonSignedRankedTestResult.LowerCi) |
| } |
| }) |
| } |
| x := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9} |
| y := []float64{7, 8, 9, 10, 11, 12, 13, 14, 15} |
| test("x < y, ImprovementDir = Up", x, y, Up) |
| |
| x = []float64{7, 8, 9, 10, 11, 12, 13, 14, 15} |
| y = []float64{1, 2, 3, 4, 5, 6, 7, 8, 9} |
| test("x > y, ImprovementDir = Down", x, y, Down) |
| } |
| |
| func TestComparePairwise_GivenRegression_ReturnsDifferent(t *testing.T) { |
| test := func(name string, x, y []float64, dir ImprovementDir) { |
| t.Run(name, func(t *testing.T) { |
| result, err := ComparePairwise(x, y, dir) |
| require.NoError(t, err) |
| require.LessOrEqual(t, result.PairwiseWilcoxonSignedRankedTestResult.PValue, thresholds.LowThreshold) |
| assert.Equal(t, Different, result.Verdict) |
| assert.Positive(t, result.PairwiseWilcoxonSignedRankedTestResult.LowerCi*result.PairwiseWilcoxonSignedRankedTestResult.UpperCi) |
| switch dir { |
| case Up: |
| assert.Negative(t, result.PairwiseWilcoxonSignedRankedTestResult.UpperCi) |
| case Down: |
| assert.Positive(t, result.PairwiseWilcoxonSignedRankedTestResult.LowerCi) |
| } |
| }) |
| } |
| x := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9} |
| y := []float64{7, 8, 9, 10, 11, 12, 13, 14, 15} |
| test("x < y, ImprovementDir = Down", x, y, Down) |
| test("x < y, ImprovementDir = Unknown", x, y, UnknownDir) |
| |
| x = []float64{7, 8, 9, 10, 11, 12, 13, 14, 15} |
| y = []float64{1, 2, 3, 4, 5, 6, 7, 8, 9} |
| test("x > y, ImprovementDir = Up", x, y, Up) |
| } |
| |
| func TestComparePairwise_GivenNoRegression_ReturnsSame(t *testing.T) { |
| test := func(name string, x, y []float64, dir ImprovementDir) { |
| t.Run(name, func(t *testing.T) { |
| result, err := ComparePairwise(x, y, dir) |
| require.NoError(t, err) |
| require.GreaterOrEqual(t, result.PairwiseWilcoxonSignedRankedTestResult.PValue, thresholds.LowThreshold) |
| assert.Equal(t, Same, result.Verdict) |
| assert.Negative(t, result.PairwiseWilcoxonSignedRankedTestResult.LowerCi*result.PairwiseWilcoxonSignedRankedTestResult.UpperCi) |
| }) |
| } |
| x := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9} |
| y := []float64{1.1, 1.9, 3.1, 3.9, 4.1, 4.9, 6.1, 7.9, 9.1} |
| test("ImprovementDir = Down", x, y, Down) |
| test("ImprovementDir = Unknown", x, y, UnknownDir) |
| } |
| |
| func TestComparePairwise_GivenZeros_UsesNormalizedResult(t *testing.T) { |
| x := []float64{1, 2, 0, 4, 5, 6, 7, 8, 9} |
| y := []float64{1.1, 0, 3.1, 3.9, 4.1, 4.9, 6.1, 7.9, 9.1} |
| |
| result, err := ComparePairwise(x, y, Up) |
| require.NoError(t, err) |
| normalizedResult, err := stats.PairwiseWilcoxonSignedRankedTest(y, x, stats.TwoSided, stats.NormalizeResult) |
| require.NoError(t, err) |
| assert.Equal(t, normalizedResult, result.PairwiseWilcoxonSignedRankedTestResult) |
| } |
| |
| func TestComparePairwise_GivenNonZeros_UsesLogTransform(t *testing.T) { |
| x := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9} |
| y := []float64{1.1, 2.1, 3.1, 3.9, 4.1, 4.9, 6.1, 7.9, 9.1} |
| |
| result, err := ComparePairwise(x, y, Up) |
| require.NoError(t, err) |
| logTransformResult, err := stats.PairwiseWilcoxonSignedRankedTest(y, x, stats.TwoSided, stats.LogTransform) |
| require.NoError(t, err) |
| assert.Equal(t, logTransformResult, result.PairwiseWilcoxonSignedRankedTestResult) |
| } |
| |
| func TestComparePairwise_GivenNegativeNumbers_ThrowsError(t *testing.T) { |
| x := []float64{1, 2, 3, -4, 5, 6, 7, 8, 9} |
| y := []float64{1.1, 2.1, 3.1, 3.9, 4.1, 4.9, 6.1, 7.9, 9.1} |
| |
| result, err := ComparePairwise(x, y, Up) |
| require.Error(t, err) |
| assert.Nil(t, result) |
| } |
| |
| func TestComparePairwise_AEqualsB_ReturnsSame(t *testing.T) { |
| x := []float64{1, 2, 3, 4} |
| y := []float64{1, 2, 3, 4} |
| |
| expected := &ComparePairwiseResult{ |
| Verdict: Same, |
| PairwiseWilcoxonSignedRankedTestResult: stats.PairwiseWilcoxonSignedRankedTestResult{ |
| Estimate: 0.0, |
| LowerCi: 0.0, |
| UpperCi: 0.0, |
| PValue: 1, |
| XMedian: median(x), |
| YMedian: median(y), |
| }, |
| } |
| |
| result, err := ComparePairwise(x, y, UnknownDir) |
| require.NoError(t, err) |
| assert.Equal(t, expected, result) |
| } |
| |
| func TestComparePairwise_AllSameValues_ReturnsSame(t *testing.T) { |
| x := []float64{2.2, 2.2, 2.2, 2.2} |
| y := []float64{2.2, 2.2, 2.2, 2.2} |
| |
| expected := &ComparePairwiseResult{ |
| Verdict: Same, |
| PairwiseWilcoxonSignedRankedTestResult: stats.PairwiseWilcoxonSignedRankedTestResult{ |
| Estimate: 0.0, |
| LowerCi: 0.0, |
| UpperCi: 0.0, |
| PValue: 1, |
| XMedian: median(x), |
| YMedian: median(y), |
| }, |
| } |
| |
| result, err := ComparePairwise(x, y, UnknownDir) |
| require.NoError(t, err) |
| assert.Equal(t, expected, result) |
| } |
| |
| func TestComparePairwise_SameValues_ReturnsEstimate(t *testing.T) { |
| valuesA := []float64{3, 3, 3, 3, 3, 3, 3, 3} |
| valuesB := []float64{8, 8, 8, 8, 8, 8, 8, 8} |
| |
| // demonstrate edge case |
| transform := stats.LogTransform |
| edgeResult, err := stats.PairwiseWilcoxonSignedRankedTest(valuesB, valuesA, stats.TwoSided, transform) |
| require.NoError(t, err) |
| require.True(t, math.IsNaN(edgeResult.LowerCi)) |
| require.True(t, math.IsNaN(edgeResult.UpperCi)) |
| |
| valuesB = handlePairwiseEdgeCase(valuesA, valuesB) |
| expected, err := stats.PairwiseWilcoxonSignedRankedTest(valuesB, valuesA, stats.TwoSided, transform) |
| require.NoError(t, err) |
| |
| result, err := ComparePairwise(valuesA, valuesB, UnknownDir) |
| require.NoError(t, err) |
| assert.Equal(t, expected, result.PairwiseWilcoxonSignedRankedTestResult) |
| } |
| |
| func TestComparePairwise_NaNAndInfInputs_SuccessfullySanitizes(t *testing.T) { |
| t.Run("Verify identical arrays do not return NaN or Inf and are JSON serializable", func(t *testing.T) { |
| // identical values inside each array usually produce NaN inside stats.PairwiseWilcoxonSignedRankedTest |
| valuesA := []float64{3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0} |
| valuesB := []float64{8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0} |
| |
| result, err := ComparePairwise(valuesA, valuesB, UnknownDir) |
| require.NoError(t, err) |
| |
| // assert returned floats are completely valid and finite (no NaN/Inf) |
| assert.False(t, math.IsNaN(result.PValue)) |
| assert.False(t, math.IsNaN(result.Estimate)) |
| assert.False(t, math.IsNaN(result.LowerCi)) |
| assert.False(t, math.IsNaN(result.UpperCi)) |
| |
| assert.False(t, math.IsInf(result.PValue, 0)) |
| assert.False(t, math.IsInf(result.Estimate, 0)) |
| assert.False(t, math.IsInf(result.LowerCi, 0)) |
| assert.False(t, math.IsInf(result.UpperCi, 0)) |
| |
| // Verify Go JSON marshaller succeeds with zero errors |
| _, jsonErr := json.Marshal(result) |
| assert.NoError(t, jsonErr) |
| }) |
| } |
| |
| func TestJSONMarshal_WithNaNAndInfValues_CrashesWithoutSanitization(t *testing.T) { |
| t.Run("Prove that raw NaN confidence intervals return JSON marshalling errors", func(t *testing.T) { |
| valuesA := []float64{3, 3, 3, 3, 3, 3, 3, 3} |
| valuesB := []float64{8, 8, 8, 8, 8, 8, 8, 8} |
| |
| transform := stats.LogTransform |
| // Calling Wilcoxon directly on un-nudged identical arrays produces NaN LowerCi and UpperCi |
| edgeResult, err := stats.PairwiseWilcoxonSignedRankedTest(valuesB, valuesA, stats.TwoSided, transform) |
| require.NoError(t, err) |
| require.True(t, math.IsNaN(edgeResult.LowerCi)) |
| require.True(t, math.IsNaN(edgeResult.UpperCi)) |
| |
| res := &ComparePairwiseResult{ |
| Verdict: Same, |
| PairwiseWilcoxonSignedRankedTestResult: edgeResult, |
| } |
| |
| // Verify that marshalling this un-sanitized result crashes with JSON error |
| _, jsonErr := json.Marshal(res) |
| require.Error(t, jsonErr) |
| assert.Contains(t, jsonErr.Error(), "json: unsupported value: NaN") |
| }) |
| } |
| |
| func TestSanitizeFloat_WithVariousInputs_CorrectlySanitizes(t *testing.T) { |
| t.Run("NaN is converted to 0.0", func(t *testing.T) { |
| assert.InDelta(t, 0.0, sanitizeFloat(math.NaN()), 0.0001) |
| }) |
| |
| t.Run("+Inf is converted to 0.0", func(t *testing.T) { |
| assert.InDelta(t, 0.0, sanitizeFloat(math.Inf(1)), 0.0001) |
| }) |
| |
| t.Run("-Inf is converted to 0.0", func(t *testing.T) { |
| assert.InDelta(t, 0.0, sanitizeFloat(math.Inf(-1)), 0.0001) |
| }) |
| |
| t.Run("Normal finite float is preserved unchanged", func(t *testing.T) { |
| assert.InDelta(t, 0.05, sanitizeFloat(0.05), 0.0001) |
| assert.InDelta(t, -1.23, sanitizeFloat(-1.23), 0.0001) |
| }) |
| } |
| |
| func TestComparePairwise_TiedNormalizedDifferences_FailsFastWithoutSanitization(t *testing.T) { |
| t.Run("Verify tied normalized differences produce NaN and fail-fast when sanitization is disabled", func(t *testing.T) { |
| // These arrays contain 0.0 (forces NormalizeResult) and are not internally identical |
| // but their differences have ties that natively produce NaN LowerCi and UpperCi inside Wilcoxon |
| valuesA := []float64{0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0} |
| valuesB := []float64{0.0, 2.0, 0.0, 2.0, 0.0, 2.0, 0.0, 2.0} |
| |
| result, err := ComparePairwise(valuesA, valuesB, UnknownDir) |
| require.NoError(t, err) |
| |
| // If sanitization is enabled, these MUST be safely mapped to 0.0 |
| assert.InDelta(t, 0.0, result.LowerCi, 0.0001) |
| assert.InDelta(t, 0.0, result.UpperCi, 0.0001) |
| }) |
| } |
