diff --git a/docs/api_doc/imputation/MeanImputer.rst b/docs/api_doc/imputation/MeanImputer.rst new file mode 100644 index 000000000..22fada8a8 --- /dev/null +++ b/docs/api_doc/imputation/MeanImputer.rst @@ -0,0 +1,6 @@ +MeanImputer +=========== + +.. autoclass:: feature_engine.imputation.MeanImputer + :members: + diff --git a/docs/api_doc/imputation/MeanMedianImputer.rst b/docs/api_doc/imputation/MeanMedianImputer.rst deleted file mode 100644 index cb1713ada..000000000 --- a/docs/api_doc/imputation/MeanMedianImputer.rst +++ /dev/null @@ -1,6 +0,0 @@ -MeanMedianImputer -================= - -.. autoclass:: feature_engine.imputation.MeanMedianImputer - :members: - diff --git a/docs/api_doc/imputation/index.rst b/docs/api_doc/imputation/index.rst index 7900832d4..b06c6cc2b 100644 --- a/docs/api_doc/imputation/index.rst +++ b/docs/api_doc/imputation/index.rst @@ -12,7 +12,7 @@ The following table summarises each imputer's functionality: ================================== ===================== ======================= ==================================================================================== Transformer Numerical variables Categorical variables Description ================================== ===================== ======================= ==================================================================================== -:class:`MeanMedianImputer()` √ × Replaces missing values with the mean or median +:class:`MeanImputer()` √ × Replaces missing values with the mean or median :class:`ArbitraryNumberImputer()` √ × Replaces missing values with an arbitrary value :class:`EndTailImputer()` √ × Replaces missing values with a value at the end of the distribution :class:`CategoricalImputer()` × √ Replaces missing values with the most frequent category or an arbitrary string @@ -27,7 +27,7 @@ Imputers .. toctree:: :maxdepth: 1 - MeanMedianImputer + MeanImputer ArbitraryNumberImputer EndTailImputer CategoricalImputer diff --git a/docs/index.rst b/docs/index.rst index ffea4ee20..9533fd98c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -179,7 +179,7 @@ Missing Data Imputation: Imputers Missing data imputation consists in replacing missing values in categorical data and numerical variables with estimates of those nan values or arbitrary data points. Feature-engine supports the following missing data imputation methods: -- :doc:`api_doc/imputation/MeanMedianImputer`: replaces missing data in numerical variables with the mean or median +- :doc:`api_doc/imputation/MeanImputer`: replaces missing data in numerical variables with the mean or median - :doc:`api_doc/imputation/ArbitraryNumberImputer`: replaces missing data in numerical variables with an arbitrary number - :doc:`api_doc/imputation/EndTailImputer`: replaces missing data in numerical variables with numbers at the distribution tails - :doc:`api_doc/imputation/CategoricalImputer`: replaces missing data with an arbitrary string or with the most frequent category diff --git a/docs/quickstart/index.rst b/docs/quickstart/index.rst index fb31f765e..87ee64c55 100644 --- a/docs/quickstart/index.rst +++ b/docs/quickstart/index.rst @@ -57,7 +57,7 @@ imputation. from sklearn.datasets import fetch_openml from sklearn.model_selection import train_test_split - from feature_engine.imputation import MeanMedianImputer + from feature_engine.imputation import MeanImputer # Load dataset X, y = fetch_openml( @@ -76,7 +76,7 @@ imputation. ) # set up the imputer - median_imputer = MeanMedianImputer( + median_imputer = MeanImputer( imputation_method='median', variables=['LotFrontage', 'MasVnrArea'] ) @@ -127,7 +127,7 @@ pickle (.pkl). Here is an example of how to do it: from feature_engine.discretisation import DecisionTreeDiscretiser from feature_engine.imputation import ( AddMissingIndicator, - MeanMedianImputer, + MeanImputer, CategoricalImputer, ) @@ -172,7 +172,7 @@ pickle (.pkl). Here is an example of how to do it: ('continuous_var_imputer', AddMissingIndicator(variables=['LotFrontage'])), # Replace NA with the median in 2 variables - ('continuous_var_median_imputer', MeanMedianImputer( + ('continuous_var_median_imputer', MeanImputer( imputation_method='median', variables=['LotFrontage', 'MasVnrArea'] )), diff --git a/docs/user_guide/imputation/MeanMedianImputer.rst b/docs/user_guide/imputation/MeanImputer.rst similarity index 93% rename from docs/user_guide/imputation/MeanMedianImputer.rst rename to docs/user_guide/imputation/MeanImputer.rst index f94460f1a..964ea169d 100644 --- a/docs/user_guide/imputation/MeanMedianImputer.rst +++ b/docs/user_guide/imputation/MeanImputer.rst @@ -1,9 +1,9 @@ -.. _mean_median_imputer: +.. _mean_imputer: .. currentmodule:: feature_engine.imputation -MeanMedianImputer -================= +MeanImputer +=========== Mean imputation and median imputation consist of replacing missing data in numerical variables with the variable's mean or median. These simple univariate missing data @@ -74,18 +74,18 @@ used with decision tree-based algorithms. When the relationship among the variab crucial, you might want to consider better ways to estimate the missing data, such as multiple imputation (aka, multivariate imputation). -MeanMedianImputer ------------------ +MeanImputer +----------- -Feature-engine's :class:`MeanMedianImputer()` replaces missing data with the variable's +Feature-engine's :class:`MeanImputer()` replaces missing data with the variable's mean or median value, determined over the observed values. Hence, it can only impute numerical variables. You can pass the -list of variables you want to impute, or alternatively, :class:`MeanMedianImputer()` +list of variables you want to impute, or alternatively, :class:`MeanImputer()` will automatically impute all numerical variables in the training set. .. attention:: - **New in version 2.0:** When `variables` is `None`, :class:`MeanMedianImputer()` used to + **New in version 2.0:** When `variables` is `None`, :class:`MeanImputer()` used to raise an error if the dataframe contained no numerical variables. You can now set the new parameter `return_empty` to `True` to make the transformer return an empty list of variables and skip the imputation instead, leaving the dataframe @@ -97,7 +97,7 @@ will automatically impute all numerical variables in the training set. Python implementation --------------------- -In this section, we will explore :class:`MeanMedianImputer()`'s functionality. Let's start by +In this section, we will explore :class:`MeanImputer()`'s functionality. Let's start by importing the required libraries: .. code:: python @@ -107,7 +107,7 @@ importing the required libraries: from sklearn.datasets import fetch_openml from sklearn.pipeline import make_pipeline from sklearn.model_selection import train_test_split - from feature_engine.imputation import MeanMedianImputer + from feature_engine.imputation import MeanImputer from feature_engine.imputation import AddMissingIndicator @@ -156,13 +156,13 @@ either **LotFrontage** or **MasVnrArea**: 1018 Gilbert NaN 76.0 -Let's now set up and fit :class:`MeanMedianImputer()` with the strategy set to mean, +Let's now set up and fit :class:`MeanImputer()` with the strategy set to mean, so we can impute the variables `LotFrontage` and `MasVnrArea`: .. code:: python # Set up the imputer - mmi = MeanMedianImputer( + mmi = MeanImputer( imputation_method='mean', variables=['LotFrontage', 'MasVnrArea'] ) @@ -173,10 +173,10 @@ so we can impute the variables `LotFrontage` and `MasVnrArea`: .. note:: It's worth noting that we have the flexibility to omit the `variables` parameter, - in which case, :class:`MeanMedianImputer()` will automatically find and impute all + in which case, :class:`MeanImputer()` will automatically find and impute all numeric features. -After fitting :class:`MeanMedianImputer()`, we can check out the statistics +After fitting :class:`MeanImputer()`, we can check out the statistics (either mean or median; mean in this case) for each of the variables to impute: .. code:: python @@ -217,7 +217,7 @@ Imputing missing values alongside missing indicators Mean or median imputation are commonly done alongside adding missing indicators. We can add missing indicators with :class:`AddMissingIndicator()` from feature-engine. -We can chain :class:`AddMissingIndicator()` with :class:`MeanMedianImputer()` using a +We can chain :class:`AddMissingIndicator()` with :class:`MeanImputer()` using a `scikit-learn pipeline `_. For example, let's create an imputation pipeline to add missing indicators and then @@ -228,7 +228,7 @@ impute the missing values: # Create imputation pipeline imputer = make_pipeline( AddMissingIndicator(), - MeanMedianImputer() + MeanImputer() ) # Fit the pipeline diff --git a/docs/user_guide/imputation/index.rst b/docs/user_guide/imputation/index.rst index 4df142f06..030b2ec26 100644 --- a/docs/user_guide/imputation/index.rst +++ b/docs/user_guide/imputation/index.rst @@ -101,7 +101,7 @@ Feature-engine's imputers' main characteristics ================================== ===================== ======================= ==================================================================================== Transformer Numerical variables Categorical variables Description ================================== ===================== ======================= ==================================================================================== -:class:`MeanMedianImputer()` √ × Replaces missing values with the mean or median +:class:`MeanImputer()` √ × Replaces missing values with the mean or median :class:`ArbitraryNumberImputer()` √ x Replaces missing values with an arbitrary value :class:`EndTailImputer()` √ × Replaces missing values with a value at the end of the distribution :class:`CategoricalImputer()` × √ Replaces missing values with the most frequent category or an arbitrary string @@ -132,7 +132,7 @@ assume that the missing values are close in value to the majority, that is, to t Data imputed with the mean or median is commonly used to train linear regression and logistic regression models. -The :class:`MeanMedianImputer()` implements mean-median imputation. +The :class:`MeanImputer()` implements mean-median imputation. Arbitrary Number Imputation ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -340,7 +340,7 @@ Imputers .. toctree:: :maxdepth: 1 - MeanMedianImputer + MeanImputer ArbitraryNumberImputer EndTailImputer CategoricalImputer diff --git a/docs/user_guide/outliers/Winsoriser.rst b/docs/user_guide/outliers/Winsoriser.rst index cffafdf0f..31babf233 100644 --- a/docs/user_guide/outliers/Winsoriser.rst +++ b/docs/user_guide/outliers/Winsoriser.rst @@ -207,7 +207,7 @@ using 1.5 of the IQR to find those limits (param `fold`). .. code:: python - from feature_engine.imputation import MeanMedianImputer + from feature_engine.imputation import MeanImputer from feature_engine.pipeline import Pipeline from feature_engine.outliers import Winsoriser @@ -218,7 +218,7 @@ using 1.5 of the IQR to find those limits (param `fold`). ) pipe = Pipeline([ - ("imputer", MeanMedianImputer()), + ("imputer", MeanImputer()), ("outlier", w), ]) @@ -286,7 +286,7 @@ As an alternative, let's cap the variables at their 10th percentile on the right ) pipe = Pipeline([ - ("imputer", MeanMedianImputer()), + ("imputer", MeanImputer()), ("outlier", w), ]) diff --git a/docs/user_guide/timeseries/forecasting/ExpandingWindowFeatures.rst b/docs/user_guide/timeseries/forecasting/ExpandingWindowFeatures.rst index 3ee374a82..3d8714a67 100644 --- a/docs/user_guide/timeseries/forecasting/ExpandingWindowFeatures.rst +++ b/docs/user_guide/timeseries/forecasting/ExpandingWindowFeatures.rst @@ -289,20 +289,20 @@ Imputing rows with nan If instead of removing the row with nan in the expanding window features, we want to impute those values, we can do so with any of feature-engine's imputers. Here, we will replace nan with -the median value of the resulting window features, using the `MeanMedianImputer` within +the median value of the resulting window features, using the `MeanImputer` within a pipeline: .. code:: python - from feature_engine.imputation import MeanMedianImputer + from feature_engine.imputation import MeanImputer from feature_engine.pipeline import Pipeline win_f = ExpandingWindowFeatures(functions=["mean", "std"]) pipe = Pipeline([ ("windows", win_f), - ("imputer", MeanMedianImputer(imputation_method="median")) + ("imputer", MeanImputer(imputation_method="median")) ]) X_tr = pipe.fit_transform(X, y) diff --git a/docs/user_guide/wrappers/Wrapper.rst b/docs/user_guide/wrappers/Wrapper.rst index dffd91936..dac6a8d0d 100644 --- a/docs/user_guide/wrappers/Wrapper.rst +++ b/docs/user_guide/wrappers/Wrapper.rst @@ -268,7 +268,7 @@ scikit-learn's PolynomialFeatures. We start with the imports: from sklearn.preprocessing import PolynomialFeatures from sklearn.pipeline import Pipeline from feature_engine.datasets import load_titanic - from feature_engine.imputation import CategoricalImputer, MeanMedianImputer + from feature_engine.imputation import CategoricalImputer, MeanImputer from feature_engine.encoding import OrdinalEncoder from feature_engine.wrappers import SklearnTransformerWrapper @@ -292,7 +292,7 @@ Now, we assemble the pipeline. The last step wraps scikit-learn's PolynomialFeat pipeline = Pipeline(steps = [ ('ci', CategoricalImputer(imputation_method='frequent')), - ('mmi', MeanMedianImputer(imputation_method='mean')), + ('mmi', MeanImputer(imputation_method='mean')), ('od', OrdinalEncoder(encoding_method='arbitrary')), ('pl', SklearnTransformerWrapper( PolynomialFeatures(interaction_only = True, include_bias=False), diff --git a/feature_engine/datasets/titanic.py b/feature_engine/datasets/titanic.py index 5a7c3c3f9..e46214aa7 100644 --- a/feature_engine/datasets/titanic.py +++ b/feature_engine/datasets/titanic.py @@ -2,7 +2,7 @@ import pandas as pd from sklearn.pipeline import Pipeline -from feature_engine.imputation import CategoricalImputer, MeanMedianImputer +from feature_engine.imputation import CategoricalImputer, MeanImputer # TODO: loading the dataset from the internet is not the best, we need to store it @@ -95,7 +95,7 @@ def load_titanic( "categorical_imputer", CategoricalImputer(imputation_method="missing"), ), - ("mean_median_imputer", MeanMedianImputer(imputation_method="mean")), + ("mean_imputer", MeanImputer(imputation_method="mean")), ] ) diff --git a/feature_engine/imputation/__init__.py b/feature_engine/imputation/__init__.py index beb4c41f8..1158c3172 100644 --- a/feature_engine/imputation/__init__.py +++ b/feature_engine/imputation/__init__.py @@ -6,11 +6,12 @@ from .categorical import CategoricalImputer from .drop_missing_data import DropMissingData from .end_tail import EndTailImputer -from .mean_median import MeanMedianImputer +from .mean_median import MeanImputer, MeanMedianImputer from .missing_indicator import AddMissingIndicator from .random_sample import RandomSampleImputer __all__ = [ + "MeanImputer", "MeanMedianImputer", "ArbitraryNumberImputer", "CategoricalImputer", diff --git a/feature_engine/imputation/mean_median.py b/feature_engine/imputation/mean_median.py index 997ec2813..7812c1217 100644 --- a/feature_engine/imputation/mean_median.py +++ b/feature_engine/imputation/mean_median.py @@ -1,6 +1,7 @@ # Authors: Soledad Galli # License: BSD 3 clause +import warnings from typing import List, Optional, Union import pandas as pd @@ -43,16 +44,16 @@ transform=_transform_imputers_docstring, fit_transform=_fit_transform_docstring, ) -class MeanMedianImputer(BaseImputer): +class MeanImputer(BaseImputer): """ - The MeanMedianImputer() replaces missing data by the mean or median value of the + The MeanImputer() replaces missing data by the mean or median value of the variable. It works only with numerical variables. You can pass a list of variables to impute. Alternatively, the - MeanMedianImputer() will automatically select all variables of type numeric in the + MeanImputer() will automatically select all variables of type numeric in the training set. - More details in the :ref:`User Guide `. + More details in the :ref:`User Guide `. Parameters ---------- @@ -87,12 +88,12 @@ class MeanMedianImputer(BaseImputer): >>> import pandas as pd >>> import numpy as np - >>> from feature_engine.imputation import MeanMedianImputer + >>> from feature_engine.imputation import MeanImputer >>> X = pd.DataFrame(dict( >>> x1 = [np.nan,1,1,0,np.nan], >>> x2 = ["a", np.nan, "b", np.nan, "a"], >>> )) - >>> mmi = MeanMedianImputer(imputation_method='median') + >>> mmi = MeanImputer(imputation_method='median') >>> mmi.fit(X) >>> mmi.transform(X) x1 x2 @@ -151,3 +152,25 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): self._get_feature_names_in(X) return self + + +# TODO: remove in version 2.1.0 +class MeanMedianImputer(MeanImputer): + def __init__( + self, + imputation_method: str = "median", + variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, + ) -> None: + warnings.warn( + "MeanMedianImputer was deprecated in favour of MeanImputer in version " + "2.0.0 and will be removed in version 2.1.0. To silence this warning, " + "use MeanImputer instead.", + FutureWarning, + stacklevel=2, + ) + super().__init__( + imputation_method=imputation_method, + variables=variables, + return_empty=return_empty, + ) diff --git a/tests/check_estimators_with_parametrize_tests.py b/tests/check_estimators_with_parametrize_tests.py index 4962ded00..2f22e5150 100644 --- a/tests/check_estimators_with_parametrize_tests.py +++ b/tests/check_estimators_with_parametrize_tests.py @@ -30,7 +30,7 @@ CategoricalImputer, DropMissingData, EndTailImputer, - MeanMedianImputer, + MeanImputer, RandomSampleImputer, ) from feature_engine.outliers import ArbitraryOutlierCapper, OutlierTrimmer, Winsoriser @@ -87,7 +87,7 @@ def test_sklearn_compatible_creator(estimator, check): # imputation @parametrize_with_checks( [ - MeanMedianImputer(), + MeanImputer(), ArbitraryNumberImputer(), CategoricalImputer(fill_value=0, ignore_format=True), EndTailImputer(), diff --git a/tests/test_imputation/test_check_estimator_imputers.py b/tests/test_imputation/test_check_estimator_imputers.py index 0091c7bf7..71d3fef51 100644 --- a/tests/test_imputation/test_check_estimator_imputers.py +++ b/tests/test_imputation/test_check_estimator_imputers.py @@ -11,13 +11,13 @@ CategoricalImputer, DropMissingData, EndTailImputer, - MeanMedianImputer, + MeanImputer, RandomSampleImputer, ) from tests.estimator_checks.estimator_checks import check_feature_engine_estimator _estimators = [ - MeanMedianImputer(), + MeanImputer(), ArbitraryNumberImputer(), CategoricalImputer(fill_value=0, ignore_format=True), EndTailImputer(), diff --git a/tests/test_imputation/test_mean_median_imputer.py b/tests/test_imputation/test_mean_median_imputer.py index b065c2190..c3603ecf7 100644 --- a/tests/test_imputation/test_mean_median_imputer.py +++ b/tests/test_imputation/test_mean_median_imputer.py @@ -1,12 +1,40 @@ +import re + import pandas as pd import pytest -from feature_engine.imputation import MeanMedianImputer +from feature_engine.imputation import MeanImputer, MeanMedianImputer + +DEPRECATION_WARNING = ( + "MeanMedianImputer was deprecated in favour of MeanImputer in version " + "2.0.0 and will be removed in version 2.1.0. To silence this warning, " + "use MeanImputer instead." +) + + +@pytest.fixture( + params=[MeanImputer, MeanMedianImputer], + ids=["MeanImputer", "MeanMedianImputer"], +) +def imputer_class(request): + return request.param + + +def make_imputer(imputer_class, **kwargs): + if imputer_class is MeanMedianImputer: + with pytest.warns(FutureWarning, match=re.escape(DEPRECATION_WARNING)): + return imputer_class(**kwargs) + return imputer_class(**kwargs) + + +def test_mean_median_imputer_raises_future_warning(): + with pytest.warns(FutureWarning, match=re.escape(DEPRECATION_WARNING)): + MeanMedianImputer() -def test_mean_imputation_and_automatically_select_variables(df_na): +def test_mean_imputation_and_automatically_select_variables(df_na, imputer_class): # set up transformer - imputer = MeanMedianImputer(imputation_method="mean", variables=None) + imputer = make_imputer(imputer_class, imputation_method="mean", variables=None) X_transformed = imputer.fit_transform(df_na) # set up reference result @@ -37,9 +65,9 @@ def test_mean_imputation_and_automatically_select_variables(df_na): pd.testing.assert_frame_equal(X_transformed, X_reference) -def test_median_imputation_when_user_enters_single_variables(df_na): +def test_median_imputation_when_user_enters_single_variables(df_na, imputer_class): # set up trasnformer - imputer = MeanMedianImputer(imputation_method="median", variables=["Age"]) + imputer = make_imputer(imputer_class, imputation_method="median", variables=["Age"]) X_transformed = imputer.fit_transform(df_na) # set up reference output @@ -59,6 +87,6 @@ def test_median_imputation_when_user_enters_single_variables(df_na): pd.testing.assert_frame_equal(X_transformed, X_reference) -def test_error_with_wrong_imputation_method(): +def test_error_with_wrong_imputation_method(imputer_class): with pytest.raises(ValueError): - MeanMedianImputer(imputation_method="arbitrary") + make_imputer(imputer_class, imputation_method="arbitrary")