Incremental improvements. (#34)

## Breaking changes

* Change ltd covariance name to its full description, increase test coverage.

## Bug fixes

* Fix various typing bugs.

* Fix bug where provided weights are not passed to argument in entropy pooling.

* Fix `group_to_val!` to enable its use in nested clustered optimisations.

* Add missing factory functions for dimensionality reduction regression.

* Fix when using weights in dimensionality reduction regression.

## Maintenance

* Remove dead code.

* Make some functions private.

* Add missing convenience function, increase coverage.

* Optimise code.

* Replace throws with argcheck.

* Make all argument checks more explicit.

Author: dcelisgarza

Project.toml

@@ -1,6 +1,6 @@
 name = "PortfolioOptimisers"
 uuid = "e0036ec9-05e5-505d-a6a9-07af41c94861"
-version = "0.9.1"
+version = "0.10.0"
 authors = ["Daniel Celis Garza <[email protected]>"]
 
 [deps]

docs/src/api/00_Introduction.md

@@ -47,7 +47,7 @@ This philosophy has three primary goals:
 ## Contents
 
 ```@contents
-Pages = ["01_Base.md", "02_Tools.md", "03_PosdefMatrix.md", "04_Denoise.md", "05_Detone.md", "06_MatrixProcessing.md", "09_JuMPModelOptimisation.md", "10_OWA.md", "07_Moments/01_Base_Moments.md", "07_Moments/02_SimpleExpectedReturns.md", "07_Moments/03_Covariance.md", "07_Moments/04_SimpleVariance.md", "07_Moments/05_GerberCovariances.md", "07_Moments/06_SmythBrobyCovariance.md", "07_Moments/07_DistanceCovariance.md", "07_Moments/08_LTDCovariance.md", "07_Moments/09_RankCovariance.md", "07_Moments/10_Histogram.md", "07_Moments/11_MutualInfoCovariance.md", "07_Moments/12_PortfolioOptimisersCovariance.md", "07_Moments/13_ShrunkExpectedReturns.md", "07_Moments/14_EquilibriumExpectedReturns.md", "07_Moments/15_ExcessExpectedReturns.md", "07_Moments/16_Coskewness.md", "07_Moments/17_Cokurtosis.md", "07_Moments/18_Base_Regression.md", "07_Moments/19_StepwiseRegression.md", "07_Moments/20_DimensionReductionRegression.md", "07_Moments/21_ImpliedVolatility.md", "08_Distance/1_Base_Distance.md", "08_Distance/2_Distance.md", "08_Distance/3_DistanceDistance.md", "08_Distance/4_GeneralDistance.md", "08_Distance/5_GeneralDistanceDistance.md", "11_Phylogeny/1_Base_Phylogeny.md", "11_Phylogeny/2_Clustering.md", "11_Phylogeny/3_Hierarchical.md", "11_Phylogeny/4_DBHT.md", "11_Phylogeny/5_Phylogeny.md", "12_ConstraintGeneration/1_Base_ConstraintGeneration.md", "12_ConstraintGeneration/2_LinearConstraintGeneration.md", "12_ConstraintGeneration/3_PhylogenyConstraintGeneration.md", "12_ConstraintGeneration/4_WeightBoundsConstraintGeneration.md", "12_ConstraintGeneration/5_ThresholdConstraintGeneration.md", "13_Prior/10_EntropyPoolingPrior.md", "14_UncertaintySets.md/1_Base_UncertaintySets.md", "18_RiskMeasures/1_Base_RiskMeasures.md", "19_Optimisation/15_NestedClustering.md"]
+Pages = ["01_Base.md", "02_Tools.md", "03_Preprocessing.md", "04_PosdefMatrix.md", "05_Denoise.md", "06_Detone.md", "07_MatrixProcessing.md", "08_Moments/01_Base_Moments.md", "08_Moments/02_SimpleExpectedReturns.md", "08_Moments/03_Covariance.md", "08_Moments/04_SimpleVariance.md", "08_Moments/05_GerberCovariances.md", "08_Moments/06_SmythBrobyCovariance.md", "08_Moments/07_DistanceCovariance.md", "08_Moments/08_LowerTailDependenceCovariance.md", "08_Moments/09_RankCovariance.md", "08_Moments/10_Histogram.md", "08_Moments/11_MutualInfoCovariance.md", "08_Moments/12_PortfolioOptimisersCovariance.md", "08_Moments/13_ShrunkExpectedReturns.md", "08_Moments/14_EquilibriumExpectedReturns.md", "08_Moments/15_ExcessExpectedReturns.md", "08_Moments/16_Coskewness.md", "08_Moments/17_Cokurtosis.md", "08_Moments/18_Base_Regression.md", "08_Moments/19_StepwiseRegression.md", "08_Moments/20_DimensionReductionRegression.md", "08_Moments/21_ImpliedVolatility.md", "09_Distance/01_Base_Distance.md", "09_Distance/02_Distance.md", "09_Distance/03_DistanceDistance.md", "10_JuMPModelOptimisation.md", "11_OWA.md", "12_Phylogeny/01_Base_Phylogeny.md", "12_Phylogeny/02_Clustering.md", "12_Phylogeny/03_Hierarchical.md", "12_Phylogeny/04_DBHT.md", "12_Phylogeny/05_Phylogeny.md", "13_ConstraintGeneration/01_Base_ConstraintGeneration.md", "13_ConstraintGeneration/02_LinearConstraintGeneration.md", "13_ConstraintGeneration/03_PhylogenyConstraintGeneration.md", "13_ConstraintGeneration/04_WeightBoundsConstraintGeneration.md", "13_ConstraintGeneration/05_ThresholdConstraintGeneration.md", "14_Prior/01_Base_Prior.md", "14_Prior/02_EmpiricalPrior.md", "14_Prior/03_FactorPrior.md", "14_Prior/04_HighOrderPrior.md", "14_Prior/05_BlackLittermanViewsGeneration.md", "14_Prior/06_BlackLittermanPrior.md", "14_Prior/07_BayesianBlackLittermanPrior.md", "14_Prior/08_FactorBlackLittermanPrior.md", "14_Prior/09_AugmentedBlackLittermanPrior.md", "14_Prior/10_EntropyPoolingPrior.md", "14_Prior/11_OpinionPoolingPrior.md", "15_UncertaintySets/01_Base_UncertaintySets.md", "15_UncertaintySets/02_DeltaUncertaintySets.md", "15_UncertaintySets/03_NormalUncertaintySets.md", "15_UncertaintySets/04_BootstrapUncertaintySets.md", "16_Turnover.md", "17_Fees.md", "18_Tracking.md", "19_RiskMeasures/01_Base_RiskMeasures.md", "19_RiskMeasures/02_Variance.md", "19_RiskMeasures/03_MomentRiskMeasures.md", "19_RiskMeasures/04_Kurtosis.md", "19_RiskMeasures/05_NegativeSkewness.md", "19_RiskMeasures/06_XatRisk.md", "19_RiskMeasures/07_ConditionalXatRisk.md", "19_RiskMeasures/08_EntropicXatRisk.md", "19_RiskMeasures/09_RelativisticXatRisk.md", "19_RiskMeasures/10_OWARiskMeasures.md", "19_RiskMeasures/11_AverageDrawdown.md", "19_RiskMeasures/12_UlcerIndex.md", "19_RiskMeasures/13_MaximumDrawdown.md", "19_RiskMeasures/14_BrownianDistanceVariance.md", "19_RiskMeasures/15_WorstRealisation.md", "19_RiskMeasures/16_Range.md", "19_RiskMeasures/17_TurnoverRiskMeasure.md", "19_RiskMeasures/18_TrackingRiskMeasure.md", "19_RiskMeasures/19_RatioRiskMeasure.md", "19_RiskMeasures/20_EqualRiskMeasure.md", "19_RiskMeasures/21_MedianAbsoluteDeviationRisk.md", "19_RiskMeasures/22_NoOptimisationRiskMeasures.md", "19_RiskMeasures/23_AdjustRiskContributions.md", "19_RiskMeasures/24_ExpectedRisk.md", "19_RiskMeasures/25_RiskMeasureTools.md", "20_Optimisation/01_Base_Optimisation.md", "20_Optimisation/02_NaiveOptimisation.md", "20_Optimisation/03_Base_ClusteringOptimisation.md", "20_Optimisation/04_HierarchicalOptimiser.md", "20_Optimisation/05_HierarchicalRiskParity.md", "20_Optimisation/06_SchurComplementHierarchicalRiskParity.md", "20_Optimisation/07_HierarchicalEqualRiskContribution.md", "20_Optimisation/08_Base_JuMPOptimisation.md", "20_Optimisation/09_JuMPConstraints/01_Returns_and_ObjectiveFunctions.md", "20_Optimisation/09_JuMPConstraints/02_BudgetConstraints.md", "20_Optimisation/09_JuMPConstraints/03_WeightConstraints.md", "20_Optimisation/09_JuMPConstraints/04_SDPConstraints.md", "20_Optimisation/09_JuMPConstraints/05_MIPConstraints.md", "20_Optimisation/09_JuMPConstraints/06_TurnoverConstraints.md", "20_Optimisation/09_JuMPConstraints/07_FeesConstraints.md", "20_Optimisation/09_JuMPConstraints/08_TrackingErrorConstraints.md", "20_Optimisation/09_JuMPConstraints/09_EffectiveNumberAssetsConstraints.md", "20_Optimisation/09_JuMPConstraints/10_RegularisationConstraints.md", "20_Optimisation/10_JuMPOptimiser.md", "20_Optimisation/11_MeanRisk.md", "20_Optimisation/12_FactorRiskContribution.md", "20_Optimisation/13_NearOptimalCentering.md", "20_Optimisation/14_RiskBudgeting.md", "20_Optimisation/15_RelaxedRiskBudgeting.md", "20_Optimisation/16_NestedClustered.md", "20_Optimisation/17_Stacking.md", "20_Optimisation/18_RiskMeasureConstraints/01_BaseRiskConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/02_VarianceConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/03_MomentRiskMeasureConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/04_KurtosisConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/05_NegativeSkewnessConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/06_XatRiskConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/07_ConditionalXatRiskConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/08_EntropicXatRiskConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/09_RelativisticXatRiskConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/10_OWARiskMeasuresConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/11_AverageDrawdownConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/12_UlcerIndexConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/13_MaximumDrawdownConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/14_BrownianDistanceVarianceConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/15_WorstRealisationConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/16_RangeConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/17_TurnoverRiskMeasureConstraints.md", "20_Optimisation/18_RiskMeasureConstraints/18_TrackingRiskMeasureConstraints.md", "20_Optimisation/19_Base_FiniteAllocation.md", "20_Optimisation/20_DiscreteFiniteAllocation.md", "20_Optimisation/21_GreedyFiniteAllocation.md", "21_Expected_Returns.md", "22_Plotting.md", "23_Interfaces.md", "24_Precompilation.md"]
 ```
 
 [^1]: Except for a few cases, most of which are convenience function overloads. This means some links do not go to the exact method definition. Other than hard-coding links to specific lines of code, which is fragile, I haven't found an easy solution.

docs/src/api/08_Moments/08_LTDCovariance.md

@@ -0,0 +1,8 @@
+# Lower Tail Dependence Covariance
+
+```@docs
+LowerTailDependenceCovariance
+cov(ce::LowerTailDependenceCovariance, X::AbstractMatrix; dims::Int = 1, kwargs...)
+cor(ce::LowerTailDependenceCovariance, X::AbstractMatrix; dims::Int = 1, kwargs...)
+PortfolioOptimisers.lower_tail_dependence
+```

docs/src/api/08_Moments/08_LowerTailDependenceCovariance.md

@@ -5,14 +5,14 @@ PValue
 Forward
 Backward
 StepwiseRegression
-regression(re::StepwiseRegression{<:PValue, <:Forward}, x::AbstractVector,
+regression(re::StepwiseRegression, X::AbstractMatrix, F::AbstractMatrix)
+PortfolioOptimisers._regression(re::StepwiseRegression{<:PValue, <:Forward}, x::AbstractVector,
                     F::AbstractMatrix)
-regression(re::StepwiseRegression{<:Union{<:PortfolioOptimisers.AbstractMinValStepwiseRegressionCriterion, <:PortfolioOptimisers.AbstractMaxValStepwiseRegressionCriteria}, <:Forward}, x::AbstractVector, F::AbstractMatrix)
-regression(re::StepwiseRegression{<:PValue, <:Backward}, x::AbstractVector,
+PortfolioOptimisers._regression(re::StepwiseRegression{<:Union{<:PortfolioOptimisers.AbstractMinValStepwiseRegressionCriterion, <:PortfolioOptimisers.AbstractMaxValStepwiseRegressionCriteria}, <:Forward}, x::AbstractVector, F::AbstractMatrix)
+PortfolioOptimisers._regression(re::StepwiseRegression{<:PValue, <:Backward}, x::AbstractVector,
                     F::AbstractMatrix)
-regression(re::StepwiseRegression{<:Union{<:PortfolioOptimisers.AbstractMinValStepwiseRegressionCriterion, <:PortfolioOptimisers.AbstractMaxValStepwiseRegressionCriteria}, <:Backward}, x::AbstractVector,
+PortfolioOptimisers._regression(re::StepwiseRegression{<:Union{<:PortfolioOptimisers.AbstractMinValStepwiseRegressionCriterion, <:PortfolioOptimisers.AbstractMaxValStepwiseRegressionCriteria}, <:Backward}, x::AbstractVector,
                     F::AbstractMatrix)
-regression(re::StepwiseRegression, X::AbstractMatrix, F::AbstractMatrix)
 PortfolioOptimisers.add_best_feature_after_pval_failure!
 PortfolioOptimisers.get_forward_reg_incl_excl!(::PortfolioOptimisers.AbstractMinValStepwiseRegressionCriterion, value::AbstractVector, excluded::AbstractVector, included::AbstractVector, threshold::Real)
 PortfolioOptimisers.get_forward_reg_incl_excl!(::PortfolioOptimisers.AbstractMaxValStepwiseRegressionCriteria, value::AbstractVector, excluded::AbstractVector, included::AbstractVector, threshold::Real)

docs/src/api/08_Moments/19_StepwiseRegression.md

@@ -6,9 +6,9 @@ PortfolioOptimisers.fit(drtgt::PCA, X::AbstractMatrix)
 PPCA
 PortfolioOptimisers.fit(drtgt::PPCA, X::AbstractMatrix)
 DimensionReductionRegression
-regression(retgt::PortfolioOptimisers.AbstractRegressionTarget, y::AbstractVector, mu::AbstractVector,
-                    sigma::AbstractVector, x1::AbstractMatrix, Vp::AbstractMatrix)
 regression(re::DimensionReductionRegression, X::AbstractMatrix, F::AbstractMatrix)
 PortfolioOptimisers.DimensionReductionTarget
+PortfolioOptimisers._regression(re::DimensionReductionRegression, y::AbstractVector, mu::AbstractVector,
+                    sigma::AbstractVector, x1::AbstractMatrix, Vp::AbstractMatrix)
 PortfolioOptimisers.prep_dim_red_reg
 ```

docs/src/api/08_Moments/20_DimensionReductionRegression.md

@@ -5,8 +5,8 @@ Distance
 distance(::Distance{Nothing, <:SimpleDistance}, ce::StatsBase.CovarianceEstimator,
                   X::AbstractMatrix; dims::Int = 1, kwargs...)
 distance(::Distance{Nothing, <:LogDistance},
-                  ce::Union{<:LTDCovariance,
-                            <:PortfolioOptimisersCovariance{<:LTDCovariance, <:Any}},
+                  ce::Union{<:LowerTailDependenceCovariance,
+                            <:PortfolioOptimisersCovariance{<:LowerTailDependenceCovariance, <:Any}},
                   X::AbstractMatrix; dims::Int = 1, kwargs...)
 distance(de::Distance{Nothing, <:VariationInfoDistance}, ::Any, X::AbstractMatrix;
                   dims::Int = 1, kwargs...)

docs/src/api/09_Distance/2_Distance.md

@@ -299,5 +299,9 @@ function Base.iterate(obj::Union{<:AbstractEstimator, <:AbstractAlgorithm,
     return state > 1 ? nothing : (obj, state + 1)
 end
 Base.length(::Union{<:AbstractEstimator, <:AbstractAlgorithm, <:AbstractResult}) = 1
+function Base.getindex(obj::Union{<:AbstractEstimator, <:AbstractAlgorithm,
+                                  <:AbstractResult}, i::Int)
+    return i == 1 ? obj : throw(BoundsError())
+end
 
 export IsEmptyError, IsNothingError, IsNothingEmptyError, IsNonFiniteError

src/01_Base.jl

@@ -374,7 +374,7 @@ Utility for safely viewing or indexing into possibly `nothing`, scalar, or array
       + `nothing`: returns `nothing`.
       + `Real`: returns `x`.
       + `AbstractVector{<:Real}`: returns `view(x, i)`.
-      + `AbstractVector{<:AbstractVector}`: returns `[view(_x, i) for _x in x]`.
+      + `AbstractVector{<:AbstractVector}`: returns `[view(xi, i) for xi in x]`.
       + `AbstractArray`: returns `view(x, i, i)`.
 
 # Arguments
@@ -419,7 +419,7 @@ function nothing_scalar_array_view(x::VecScalar, i)
 end
 function nothing_scalar_array_view(x::AbstractVector{<:Union{<:AbstractVector, <:VecScalar}},
                                    i)
-    return [view(_x, i) for _x in x]
+    return [view(xi, i) for xi in x]
 end
 function nothing_scalar_array_view(x::AbstractArray, i)
     return view(x, i, i)
@@ -573,7 +573,7 @@ julia> struct MyConcrete1 <: MyAbstract end
 
 julia> struct MyConcrete2 <: MyAbstract end
 
-julia> display(traverse_concrete_subtypes(MyAbstract))
+julia> traverse_concrete_subtypes(MyAbstract)
 2-element Vector{Any}:
  MyConcrete1
  MyConcrete2

src/02_Tools.jl

@@ -233,16 +233,14 @@ These methods are called internally by [`denoise!`](@ref) and [`denoise`](@ref)
 """
 function _denoise!(::SpectralDenoise, X::AbstractMatrix, vals::AbstractVector,
                    vecs::AbstractMatrix, num_factors::Integer)
-    _vals = copy(vals)
-    _vals[1:num_factors] .= zero(eltype(X))
-    X .= cov2cor(vecs * Diagonal(_vals) * transpose(vecs))
+    vals[1:num_factors] .= zero(eltype(X))
+    X .= cov2cor(vecs * Diagonal(vals) * transpose(vecs))
     return nothing
 end
 function _denoise!(::FixedDenoise, X::AbstractMatrix, vals::AbstractVector,
                    vecs::AbstractMatrix, num_factors::Integer)
-    _vals = copy(vals)
-    _vals[1:num_factors] .= sum(_vals[1:num_factors]) / num_factors
-    X .= cov2cor(vecs * Diagonal(_vals) * transpose(vecs))
+    vals[1:num_factors] .= sum(vals[1:num_factors]) / num_factors
+    X .= cov2cor(vecs * Diagonal(vals) * transpose(vecs))
     return nothing
 end
 function _denoise!(de::ShrunkDenoise, X::AbstractMatrix, vals::AbstractVector,
@@ -415,7 +413,7 @@ function denoise!(de::Denoise, X::AbstractMatrix, q::Real,
     vals, vecs = eigen(X)
     max_val = find_max_eval(vals, q; kernel = de.kernel, m = de.m, n = de.n, args = de.args,
                             kwargs = de.kwargs)[1]
-    num_factors = findlast(vals .< max_val)
+    num_factors = searchsortedlast(vals, max_val)
     _denoise!(de.alg, X, vals, vecs, num_factors)
     posdef!(pdm, X)
     if iscov