diff --git a/examples/pcovc/KPCovC_Comparison.py b/examples/pcovc/KPCovC_Comparison.py index 5028a9b5c..0811224a2 100644 --- a/examples/pcovc/KPCovC_Comparison.py +++ b/examples/pcovc/KPCovC_Comparison.py @@ -85,7 +85,7 @@ # Both PCA and PCovC fail to produce linearly separable latent space # maps. We will need a kernel method to effectively separate the moon classes. -mixing = 0.10 +mixing = 0.5 alpha_d = 0.5 alpha_p = 0.4 diff --git a/examples/pcovc/KPCovC_Hyperparameters.py b/examples/pcovc/KPCovC_Hyperparameters.py index ce3948e25..3b02cab62 100644 --- a/examples/pcovc/KPCovC_Hyperparameters.py +++ b/examples/pcovc/KPCovC_Hyperparameters.py @@ -65,7 +65,7 @@ fig, axs = plt.subplots(2, len(kernels), figsize=(len(kernels) * 4, 8)) center = True -mixing = 0.10 +mixing = 0.5 for i, kernel in enumerate(kernels): kpca = KernelPCA( diff --git a/src/skmatter/decomposition/_kernel_pcovc.py b/src/skmatter/decomposition/_kernel_pcovc.py index e8965a223..976946dd7 100644 --- a/src/skmatter/decomposition/_kernel_pcovc.py +++ b/src/skmatter/decomposition/_kernel_pcovc.py @@ -16,7 +16,7 @@ from sklearn.linear_model._base import LinearClassifierMixin from sklearn.utils.multiclass import check_classification_targets, type_of_target -from skmatter.preprocessing import KernelNormalizer +from skmatter.preprocessing import KernelNormalizer, StandardFlexibleScaler from skmatter.utils import check_cl_fit from skmatter.decomposition import _BaseKPCov @@ -86,6 +86,9 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): If None, ``sklearn.linear_model.LogisticRegression()`` is used as the classifier. + scale_z: bool, default=True + Whether to scale Z prior to eigendecomposition. + kernel : {"linear", "poly", "rbf", "sigmoid", "precomputed"} or callable, default="linear" Kernel. @@ -174,7 +177,7 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): >>> from sklearn.preprocessing import StandardScaler >>> X = np.array([[-2, 3, -1, 0], [2, 0, -3, 1], [3, 0, -1, 3], [2, -2, 1, 0]]) >>> X = StandardScaler().fit_transform(X) - >>> Y = np.array([[2], [0], [1], [2]]) + >>> Y = np.array([2, 0, 1, 2]) >>> kpcovc = KernelPCovC( ... mixing=0.1, ... n_components=2, @@ -184,10 +187,10 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): >>> kpcovc.fit(X, Y) KernelPCovC(gamma=1, kernel='rbf', mixing=0.1, n_components=2) >>> kpcovc.transform(X) - array([[-4.45970689e-01, 8.95327566e-06], - [ 4.52745933e-01, 5.54810948e-01], - [ 4.52881359e-01, -5.54708315e-01], - [-4.45921092e-01, -7.32157649e-05]]) + array([[-4.41692911e-01, 6.87831803e-06], + [ 4.47719340e-01, 5.47456981e-01], + [ 4.47850288e-01, -5.47360522e-01], + [-4.41645711e-01, -7.05197801e-05]]) >>> kpcovc.predict(X) array([2, 0, 1, 2]) >>> kpcovc.score(X, Y) @@ -200,6 +203,7 @@ def __init__( n_components=None, svd_solver="auto", classifier=None, + scale_z=True, kernel="linear", gamma=None, degree=3, @@ -229,6 +233,7 @@ def __init__( fit_inverse_transform=fit_inverse_transform, ) self.classifier = classifier + self.scale_z = scale_z def fit(self, X, Y, W=None): r"""Fit the model with X and Y. @@ -323,6 +328,8 @@ def fit(self, X, Y, W=None): W = LogisticRegression().fit(K, Y).coef_.T Z = K @ W + if self.scale_z: + Z = StandardFlexibleScaler().fit_transform(Z) self._fit(K, Z, W) diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index e0cee034e..e4f6698be 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -16,6 +16,7 @@ from sklearn.utils.validation import check_is_fitted, validate_data from skmatter.decomposition import _BasePCov from skmatter.utils import check_cl_fit +from skmatter.preprocessing import StandardFlexibleScaler class PCovC(LinearClassifierMixin, _BasePCov): @@ -123,6 +124,9 @@ class PCovC(LinearClassifierMixin, _BasePCov): If None, ``sklearn.linear_model.LogisticRegression()`` is used as the classifier. + scale_z: bool, default=True + Whether to scale Z prior to eigendecomposition. + iterated_power : int or 'auto', default='auto' Number of iterations for the power method computed by svd_solver == 'randomized'. @@ -194,10 +198,10 @@ class PCovC(LinearClassifierMixin, _BasePCov): >>> pcovc.fit(X, Y) PCovC(mixing=0.1, n_components=2) >>> pcovc.transform(X) - array([[-0.4794854 , -0.46228114], - [ 1.9416966 , 0.2532831 ], - [-1.08744947, 0.89117784], - [-0.37476173, -0.6821798 ]]) + array([[-0.38989065, -0.21368409], + [ 1.55313271, 0.20273297], + [-0.87105559, 0.68233882], + [-0.29218647, -0.6713877 ]]) >>> pcovc.predict(X) array([0, 1, 2, 0]) """ # NoQa: E501 @@ -210,6 +214,7 @@ def __init__( tol=1e-12, space="auto", classifier=None, + scale_z=True, iterated_power="auto", random_state=None, whiten=False, @@ -225,6 +230,7 @@ def __init__( whiten=whiten, ) self.classifier = classifier + self.scale_z = scale_z def fit(self, X, Y, W=None): r"""Fit the model with X and Y. @@ -291,7 +297,7 @@ def fit(self, X, Y, W=None): classifier = self.classifier self.z_classifier_ = check_cl_fit(classifier, X, Y) - W = self.z_classifier_.coef_.T + W = self.z_classifier_.coef_.T.copy() else: # If precomputed, use default classifier to predict Y from T @@ -301,6 +307,11 @@ def fit(self, X, Y, W=None): Z = X @ W + if self.scale_z: + z_scaler = StandardFlexibleScaler().fit(Z) + Z = z_scaler.transform(Z) + W /= z_scaler.scale_.reshape(1, -1) + if self.space_ == "feature": self._fit_feature_space(X, Y, Z) else: diff --git a/tests/test_kernel_pcovc.py b/tests/test_kernel_pcovc.py index 9b29b8437..bbfe8fbb3 100644 --- a/tests/test_kernel_pcovc.py +++ b/tests/test_kernel_pcovc.py @@ -327,6 +327,15 @@ def test_precomputed_classification(self): self.assertTrue(np.linalg.norm(t3 - t2) < self.error_tol) self.assertTrue(np.linalg.norm(t3 - t1) < self.error_tol) + def test_scale_z_parameter(self): + """Check that changing scale_z changes the eigendecomposition.""" + kpcovc_scaled = self.model(scale_z=True) + kpcovc_scaled.fit(self.X, self.Y) + + kpcovc_unscaled = self.model(scale_z=False) + kpcovc_unscaled.fit(self.X, self.Y) + assert not np.allclose(kpcovc_scaled.pkt_, kpcovc_unscaled.pkt_) + class KernelTests(KernelPCovCBaseTest): def test_kernel_types(self): diff --git a/tests/test_pcovc.py b/tests/test_pcovc.py index 8607a2e2a..883279b59 100644 --- a/tests/test_pcovc.py +++ b/tests/test_pcovc.py @@ -464,6 +464,9 @@ def test_default_ncomponents(self): self.assertEqual(pcovc.n_components_, min(self.X.shape)) def test_prefit_classifier(self): + """Check that a passed prefit classifier is not modified in + PCovC's `fit` call. + """ classifier = LinearSVC() classifier.fit(self.X, self.Y) pcovc = self.model(mixing=0.5, classifier=classifier) @@ -575,6 +578,17 @@ def test_incompatible_coef_shape(self): % (len(pcovc_multi.classes_), self.X.shape[1], cl_binary.coef_.shape), ) + def test_scale_z_parameter(self): + """Check that changing scale_z changes the eigendecomposition.""" + pcovc_scaled = self.model(scale_z=True) + pcovc_scaled.fit(self.X, self.Y) + + pcovc_unscaled = self.model(scale_z=False) + pcovc_unscaled.fit(self.X, self.Y) + assert not np.allclose( + pcovc_scaled.singular_values_, pcovc_unscaled.singular_values_ + ) + if __name__ == "__main__": unittest.main(verbosity=2)