diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3b5a8690..22c44f6e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,11 +4,11 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-## [0.6.13]
+## [0.6.14]
### Added
- `decomposition/lda.rs`: `LDA`, linear discriminant analysis for supervised dimensionality reduction (#136). It projects the data onto the directions that best separate the classes, keeping `min(n_classes - 1, n_features)` components by default, and implements the `Transformer` interface next to `PCA`. Directions match scikit-learn's `LinearDiscriminantAnalysis(solver="eigen")` up to sign.
-## [0.6.12]
+## [0.6.13]
### Fixed
- `xgboost/xgb_regressor.rs`: `XGRegressor::fit` no longer panics when `subsample` is less than 1.0 on a small dataset (#444). The sample for each tree now keeps a minimum of one row, as scikit-learn does for its own `subsample` parameter. Sample sizes of one row or more are unchanged.
diff --git a/Cargo.toml b/Cargo.toml
index 12ca418c..4fe4ded9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -2,7 +2,7 @@
name = "smartcore"
description = "Machine Learning in Rust."
homepage = "https://smartcorelib.github.io/"
-version = "0.6.13"
+version = "0.6.14"
authors = ["smartcore Developers"]
edition = "2024"
rust-version = "1.85"
diff --git a/src/decomposition/lda.rs b/src/decomposition/lda.rs
index 4c07f1b1..d4e2755f 100644
--- a/src/decomposition/lda.rs
+++ b/src/decomposition/lda.rs
@@ -43,7 +43,6 @@
//! * ["An Introduction to Statistical Learning", James G., Witten D., Hastie T., Tibshirani R., 4.4 Linear Discriminant Analysis](http://faculty.marshall.usc.edu/gareth-james/ISL/)
//! * ["Pattern Classification", Duda R.O., Hart P.E., Stork D.G., 2nd ed., 3.8.3 Multiple Discriminant Analysis](https://www.wiley.com/en-us/Pattern+Classification%2C+2nd+Edition-p-9780471056690)
//!
-//!
//!
use std::cmp::Ordering;
use std::fmt::Debug;
@@ -63,7 +62,7 @@ use crate::numbers::realnum::RealNumber;
#[derive(Debug)]
pub struct LDA + EVDDecomposable> {
// Projection matrix, one column per kept discriminant direction (n_features x n_components).
- scalings: X,
+ projection_matrix: X,
// Generalized eigenvalue of every kept direction, largest first.
eigenvalues: Vec,
// Number of features expected by `transform`.
@@ -72,18 +71,19 @@ pub struct LDA + EVDDecomposable> {
impl + EVDDecomposable> PartialEq for LDA {
fn eq(&self, other: &Self) -> bool {
+ let tol = T::from(1e-6).unwrap();
if self.n_features != other.n_features
|| self.eigenvalues.len() != other.eigenvalues.len()
|| self
- .scalings
+ .projection_matrix
.iterator(0)
- .zip(other.scalings.iterator(0))
- .any(|(&a, &b)| (a - b).abs() > T::epsilon())
+ .zip(other.projection_matrix.iterator(0))
+ .any(|(&a, &b)| (a - b).abs() > tol)
{
return false;
}
for i in 0..self.eigenvalues.len() {
- if (self.eigenvalues[i] - other.eigenvalues[i]).abs() > T::epsilon() {
+ if (self.eigenvalues[i] - other.eigenvalues[i]).abs() > tol {
return false;
}
}
@@ -276,8 +276,13 @@ impl + EVDDecomposable> LDA {
let sw_evd = sw.evd(true)?;
let u = sw_evd.V;
let l = sw_evd.d;
+ let l_max = l
+ .iter()
+ .cloned()
+ .fold(T::zero(), |a, b| if b > a { b } else { a });
+ let tol = T::from(1e-10).unwrap().max(T::from(1e-4).unwrap() * l_max);
for &li in &l {
- if li <= T::epsilon() {
+ if li <= tol {
return Err(Failed::fit(
"Within-class scatter matrix is singular, provide more samples per class or fewer features",
));
@@ -310,17 +315,17 @@ impl + EVDDecomposable> LDA {
}
});
- let mut scalings = X::zeros(m, n_components);
+ let mut projection_matrix = X::zeros(m, n_components);
let mut eigenvalues = vec![T::zero(); n_components];
for (col, &src) in order.iter().take(n_components).enumerate() {
eigenvalues[col] = g[src];
for row in 0..m {
- scalings.set((row, col), *directions.get((row, src)));
+ projection_matrix.set((row, col), *directions.get((row, src)));
}
}
Ok(LDA {
- scalings,
+ projection_matrix,
eigenvalues,
n_features: m,
})
@@ -336,12 +341,12 @@ impl + EVDDecomposable> LDA {
ncols, self.n_features
)));
}
- Ok(x.matmul(&self.scalings))
+ Ok(x.matmul(&self.projection_matrix))
}
/// Get the projection matrix, one column per discriminant direction.
pub fn scalings(&self) -> &X {
- &self.scalings
+ &self.projection_matrix
}
}
@@ -499,6 +504,27 @@ mod tests {
assert!(result.is_err());
}
+ #[test]
+ fn mismatched_x_y_is_rejected() {
+ let (x, _) = three_class_data();
+ let y_short = vec![0i32; x.shape().0 - 1];
+ assert!(LDA::fit(&x, &y_short, LDAParameters::default()).is_err());
+ }
+
+ #[test]
+ fn zero_n_components_is_rejected() {
+ let (x, y) = three_class_data();
+ assert!(LDA::fit(&x, &y, LDAParameters::default().with_n_components(0)).is_err());
+ }
+
+ #[test]
+ fn transform_wrong_features_is_rejected() {
+ let (x, y) = three_class_data();
+ let lda = LDA::fit(&x, &y, LDAParameters::default()).unwrap();
+ let bad = DenseMatrix::::zeros(3, 2);
+ assert!(lda.transform(&bad).is_err());
+ }
+
#[cfg_attr(
all(target_arch = "wasm32", not(target_os = "wasi")),
wasm_bindgen_test::wasm_bindgen_test
diff --git a/src/decomposition/mod.rs b/src/decomposition/mod.rs
index 0e444902..bdac6812 100644
--- a/src/decomposition/mod.rs
+++ b/src/decomposition/mod.rs
@@ -13,6 +13,7 @@
/// LDA is a supervised approach that projects the data onto the directions that best separate the classes.
pub mod lda;
+
/// PCA is a popular approach for deriving a low-dimensional set of features from a large set of variables.
pub mod pca;
pub mod svd;