Add precision-recall curves

Project.toml

@@ -12,6 +12,7 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
+REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 ScientificTypesBase = "30f210dd-8aff-4c5f-94ba-8e64358c1161"
 StatisticalMeasuresBase = "c062fc1d-0d66-479b-b6ac-8b44719de4cc"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
@@ -34,7 +35,7 @@ LossFunctions = "0.10, 0.11, 1"
 MacroTools = "0.5"
 OrderedCollections = "1"
 PrecompileTools = "1.1"
-ScientificTypes = "3"
+REPL = "1"
 ScientificTypesBase = "3"
 StatisticalMeasuresBase = "0.1"
 Statistics = "1"

docs/make.jl

@@ -28,6 +28,7 @@ makedocs(
         "The Measures" => "auto_generated_list_of_measures.md",
         "Confusion Matrices" => "confusion_matrices.md",
         "Receiver Operator Characteristics" => "roc.md",
+        "Precision-Recall Curves" => "precision_recall.md",
         "Tools" => "tools.md",
         "Reference" => "reference.md",
     ],
@@ -41,4 +42,3 @@ deploydocs(
     devbranch="dev",
     push_preview=false,
 )
-

docs/src/precision_recall.md

@@ -0,0 +1,40 @@
+# Precision-Recall Curves
+
+In binary classification problems, precision-recall curves (or PR curves) are a popular
+alternative to [Receiver Operator Characteristics](@ref) when the target values are highly
+imbalanced.
+
+## Example
+
+```@example 70
+using StatisticalMeasures
+using CategoricalArrays
+using CategoricalDistributions
+
+# ground truth:
+y = categorical(["X", "O", "X", "X", "O", "X", "X", "O", "O", "X"], ordered=true)
+
+# probabilistic predictions:
+X_probs = [0.3, 0.2, 0.4, 0.9, 0.1, 0.4, 0.5, 0.2, 0.8, 0.7]
+ŷ = UnivariateFinite(["O", "X"], X_probs, augment=true, pool=y)
+ŷ[1]
+```
+
+```julia
+using Plots
+recalls, precisions, thresholds = precision_recall_curve(ŷ, y)
+plt = plot(recalls, precisions, legend=false)
+plot!(plt, xlab="recall", ylab="precision")
+
+# proportion of observations that are positive:
+p = precisions[end] # threshold=0
+plot!([0, 1], [p, p], linewidth=2, linestyle=:dash, color=:black)
+```
+
+![](assets/precision_recall_curve.png)
+
+## Reference
+
+```@docs
+precision_recall_curve
+```

docs/src/reference.md

@@ -1,5 +1,9 @@
 # Reference
 
+```@index
+Pages = ["reference.md",]
+```
+
 ```@docs
 StatisticalMeasuresBase.unwrap
 StatisticalMeasuresBase.is_measure

docs/src/roc.md

@@ -18,8 +18,8 @@ ŷ[1]
 
 ```julia
 using Plots
-curve = roc_curve(ŷ, y)
-plt = plot(curve, legend=false)
+false_positive_rates, true_positive_rates, thresholds = roc_curve(ŷ, y)
+plt = plot(false_positive_rates, true_positive_rates; legend=false)
 plot!(plt, xlab="false positive rate", ylab="true positive rate")
 plot!([0, 1], [0, 1], linewidth=2, linestyle=:dash, color=:black)
 ```

src/StatisticalMeasures.jl

@@ -14,6 +14,7 @@ using LinearAlgebra
 using StatsBase
 import Distributions
 using PrecompileTools
+using REPL # needed for `Base.Docs.doc`
 
 const SM = "StatisticalMeasures"
 const CatArrOrSub{T, N} =
@@ -34,6 +35,7 @@ include("tools.jl")
 include("functions.jl")
 include("confusion_matrices.jl")
 include("roc.jl")
+include("precision_recall.jl")
 include("docstrings.jl")
 include("registry.jl")
 include("continuous.jl")
@@ -71,22 +73,7 @@ export measures,
     supports_missings_measure,
     fussy_measure
 
-export Functions, ConfusionMatrices, NoAvg, MacroAvg, MicroAvg, roc_curve
-
-#tod look out for MLJBase.aggregate called on scalars, which is not supported here.
-#todo in mljbase, single(measure, array1, array2)
-
-#todo need a show(::Measure)
-#todo `is_measure_type` in MLJBase is not provided here
-
-#todo: following needs adding to section on continuous measures
-# _scale(x, w::Arr, i) = x*w[i]
-# _scale(x, ::Nothing, i::Any) = x
-
-#todo: _skipinvalid from MLJBase/src/data/data.jl is needed for balanced accuracy, barring
-# a refactor of that measure to use `skipinvalid` as provided in this package.
-
-#todo: look for uses of aggregation of dictionaries in MLJBase, which is no longer
-# supported, or add support.
+export Functions, ConfusionMatrices, NoAvg, MacroAvg, MicroAvg
+export roc_curve, precision_recall_curve
 
 end