forked from 170010011/fr
1745 lines
70 KiB
Python
1745 lines
70 KiB
Python
"""
|
|
This module gathers tree-based methods, including decision, regression and
|
|
randomized trees. Single and multi-output problems are both handled.
|
|
"""
|
|
|
|
# Authors: Gilles Louppe <g.louppe@gmail.com>
|
|
# Peter Prettenhofer <peter.prettenhofer@gmail.com>
|
|
# Brian Holt <bdholt1@gmail.com>
|
|
# Noel Dawe <noel@dawe.me>
|
|
# Satrajit Gosh <satrajit.ghosh@gmail.com>
|
|
# Joly Arnaud <arnaud.v.joly@gmail.com>
|
|
# Fares Hedayati <fares.hedayati@gmail.com>
|
|
# Nelson Liu <nelson@nelsonliu.me>
|
|
#
|
|
# License: BSD 3 clause
|
|
|
|
import numbers
|
|
import warnings
|
|
from abc import ABCMeta
|
|
from abc import abstractmethod
|
|
from math import ceil
|
|
|
|
import numpy as np
|
|
from scipy.sparse import issparse
|
|
|
|
from ..base import BaseEstimator
|
|
from ..base import ClassifierMixin
|
|
from ..base import clone
|
|
from ..base import RegressorMixin
|
|
from ..base import is_classifier
|
|
from ..base import MultiOutputMixin
|
|
from ..utils import Bunch
|
|
from ..utils import check_random_state
|
|
from ..utils.validation import _check_sample_weight
|
|
from ..utils import compute_sample_weight
|
|
from ..utils.multiclass import check_classification_targets
|
|
from ..utils.validation import check_is_fitted
|
|
from ..utils.validation import _deprecate_positional_args
|
|
|
|
from ._criterion import Criterion
|
|
from ._splitter import Splitter
|
|
from ._tree import DepthFirstTreeBuilder
|
|
from ._tree import BestFirstTreeBuilder
|
|
from ._tree import Tree
|
|
from ._tree import _build_pruned_tree_ccp
|
|
from ._tree import ccp_pruning_path
|
|
from . import _tree, _splitter, _criterion
|
|
|
|
__all__ = ["DecisionTreeClassifier",
|
|
"DecisionTreeRegressor",
|
|
"ExtraTreeClassifier",
|
|
"ExtraTreeRegressor"]
|
|
|
|
|
|
# =============================================================================
|
|
# Types and constants
|
|
# =============================================================================
|
|
|
|
DTYPE = _tree.DTYPE
|
|
DOUBLE = _tree.DOUBLE
|
|
|
|
CRITERIA_CLF = {"gini": _criterion.Gini,
|
|
"entropy": _criterion.Entropy}
|
|
CRITERIA_REG = {"mse": _criterion.MSE,
|
|
"friedman_mse": _criterion.FriedmanMSE,
|
|
"mae": _criterion.MAE,
|
|
"poisson": _criterion.Poisson}
|
|
|
|
DENSE_SPLITTERS = {"best": _splitter.BestSplitter,
|
|
"random": _splitter.RandomSplitter}
|
|
|
|
SPARSE_SPLITTERS = {"best": _splitter.BestSparseSplitter,
|
|
"random": _splitter.RandomSparseSplitter}
|
|
|
|
# =============================================================================
|
|
# Base decision tree
|
|
# =============================================================================
|
|
|
|
|
|
class BaseDecisionTree(MultiOutputMixin, BaseEstimator, metaclass=ABCMeta):
|
|
"""Base class for decision trees.
|
|
|
|
Warning: This class should not be used directly.
|
|
Use derived classes instead.
|
|
"""
|
|
|
|
@abstractmethod
|
|
@_deprecate_positional_args
|
|
def __init__(self, *,
|
|
criterion,
|
|
splitter,
|
|
max_depth,
|
|
min_samples_split,
|
|
min_samples_leaf,
|
|
min_weight_fraction_leaf,
|
|
max_features,
|
|
max_leaf_nodes,
|
|
random_state,
|
|
min_impurity_decrease,
|
|
min_impurity_split,
|
|
class_weight=None,
|
|
ccp_alpha=0.0):
|
|
self.criterion = criterion
|
|
self.splitter = splitter
|
|
self.max_depth = max_depth
|
|
self.min_samples_split = min_samples_split
|
|
self.min_samples_leaf = min_samples_leaf
|
|
self.min_weight_fraction_leaf = min_weight_fraction_leaf
|
|
self.max_features = max_features
|
|
self.max_leaf_nodes = max_leaf_nodes
|
|
self.random_state = random_state
|
|
self.min_impurity_decrease = min_impurity_decrease
|
|
self.min_impurity_split = min_impurity_split
|
|
self.class_weight = class_weight
|
|
self.ccp_alpha = ccp_alpha
|
|
|
|
def get_depth(self):
|
|
"""Return the depth of the decision tree.
|
|
|
|
The depth of a tree is the maximum distance between the root
|
|
and any leaf.
|
|
|
|
Returns
|
|
-------
|
|
self.tree_.max_depth : int
|
|
The maximum depth of the tree.
|
|
"""
|
|
check_is_fitted(self)
|
|
return self.tree_.max_depth
|
|
|
|
def get_n_leaves(self):
|
|
"""Return the number of leaves of the decision tree.
|
|
|
|
Returns
|
|
-------
|
|
self.tree_.n_leaves : int
|
|
Number of leaves.
|
|
"""
|
|
check_is_fitted(self)
|
|
return self.tree_.n_leaves
|
|
|
|
def fit(self, X, y, sample_weight=None, check_input=True,
|
|
X_idx_sorted="deprecated"):
|
|
|
|
random_state = check_random_state(self.random_state)
|
|
|
|
if self.ccp_alpha < 0.0:
|
|
raise ValueError("ccp_alpha must be greater than or equal to 0")
|
|
|
|
if check_input:
|
|
# Need to validate separately here.
|
|
# We can't pass multi_ouput=True because that would allow y to be
|
|
# csr.
|
|
check_X_params = dict(dtype=DTYPE, accept_sparse="csc")
|
|
check_y_params = dict(ensure_2d=False, dtype=None)
|
|
X, y = self._validate_data(X, y,
|
|
validate_separately=(check_X_params,
|
|
check_y_params))
|
|
if issparse(X):
|
|
X.sort_indices()
|
|
|
|
if X.indices.dtype != np.intc or X.indptr.dtype != np.intc:
|
|
raise ValueError("No support for np.int64 index based "
|
|
"sparse matrices")
|
|
|
|
if self.criterion == "poisson":
|
|
if np.any(y < 0):
|
|
raise ValueError("Some value(s) of y are negative which is"
|
|
" not allowed for Poisson regression.")
|
|
if np.sum(y) <= 0:
|
|
raise ValueError("Sum of y is not positive which is "
|
|
"necessary for Poisson regression.")
|
|
|
|
# Determine output settings
|
|
n_samples, self.n_features_ = X.shape
|
|
self.n_features_in_ = self.n_features_
|
|
is_classification = is_classifier(self)
|
|
|
|
y = np.atleast_1d(y)
|
|
expanded_class_weight = None
|
|
|
|
if y.ndim == 1:
|
|
# reshape is necessary to preserve the data contiguity against vs
|
|
# [:, np.newaxis] that does not.
|
|
y = np.reshape(y, (-1, 1))
|
|
|
|
self.n_outputs_ = y.shape[1]
|
|
|
|
if is_classification:
|
|
check_classification_targets(y)
|
|
y = np.copy(y)
|
|
|
|
self.classes_ = []
|
|
self.n_classes_ = []
|
|
|
|
if self.class_weight is not None:
|
|
y_original = np.copy(y)
|
|
|
|
y_encoded = np.zeros(y.shape, dtype=int)
|
|
for k in range(self.n_outputs_):
|
|
classes_k, y_encoded[:, k] = np.unique(y[:, k],
|
|
return_inverse=True)
|
|
self.classes_.append(classes_k)
|
|
self.n_classes_.append(classes_k.shape[0])
|
|
y = y_encoded
|
|
|
|
if self.class_weight is not None:
|
|
expanded_class_weight = compute_sample_weight(
|
|
self.class_weight, y_original)
|
|
|
|
self.n_classes_ = np.array(self.n_classes_, dtype=np.intp)
|
|
|
|
if getattr(y, "dtype", None) != DOUBLE or not y.flags.contiguous:
|
|
y = np.ascontiguousarray(y, dtype=DOUBLE)
|
|
|
|
# Check parameters
|
|
max_depth = (np.iinfo(np.int32).max if self.max_depth is None
|
|
else self.max_depth)
|
|
max_leaf_nodes = (-1 if self.max_leaf_nodes is None
|
|
else self.max_leaf_nodes)
|
|
|
|
if isinstance(self.min_samples_leaf, numbers.Integral):
|
|
if not 1 <= self.min_samples_leaf:
|
|
raise ValueError("min_samples_leaf must be at least 1 "
|
|
"or in (0, 0.5], got %s"
|
|
% self.min_samples_leaf)
|
|
min_samples_leaf = self.min_samples_leaf
|
|
else: # float
|
|
if not 0. < self.min_samples_leaf <= 0.5:
|
|
raise ValueError("min_samples_leaf must be at least 1 "
|
|
"or in (0, 0.5], got %s"
|
|
% self.min_samples_leaf)
|
|
min_samples_leaf = int(ceil(self.min_samples_leaf * n_samples))
|
|
|
|
if isinstance(self.min_samples_split, numbers.Integral):
|
|
if not 2 <= self.min_samples_split:
|
|
raise ValueError("min_samples_split must be an integer "
|
|
"greater than 1 or a float in (0.0, 1.0]; "
|
|
"got the integer %s"
|
|
% self.min_samples_split)
|
|
min_samples_split = self.min_samples_split
|
|
else: # float
|
|
if not 0. < self.min_samples_split <= 1.:
|
|
raise ValueError("min_samples_split must be an integer "
|
|
"greater than 1 or a float in (0.0, 1.0]; "
|
|
"got the float %s"
|
|
% self.min_samples_split)
|
|
min_samples_split = int(ceil(self.min_samples_split * n_samples))
|
|
min_samples_split = max(2, min_samples_split)
|
|
|
|
min_samples_split = max(min_samples_split, 2 * min_samples_leaf)
|
|
|
|
if isinstance(self.max_features, str):
|
|
if self.max_features == "auto":
|
|
if is_classification:
|
|
max_features = max(1, int(np.sqrt(self.n_features_)))
|
|
else:
|
|
max_features = self.n_features_
|
|
elif self.max_features == "sqrt":
|
|
max_features = max(1, int(np.sqrt(self.n_features_)))
|
|
elif self.max_features == "log2":
|
|
max_features = max(1, int(np.log2(self.n_features_)))
|
|
else:
|
|
raise ValueError("Invalid value for max_features. "
|
|
"Allowed string values are 'auto', "
|
|
"'sqrt' or 'log2'.")
|
|
elif self.max_features is None:
|
|
max_features = self.n_features_
|
|
elif isinstance(self.max_features, numbers.Integral):
|
|
max_features = self.max_features
|
|
else: # float
|
|
if self.max_features > 0.0:
|
|
max_features = max(1,
|
|
int(self.max_features * self.n_features_))
|
|
else:
|
|
max_features = 0
|
|
|
|
self.max_features_ = max_features
|
|
|
|
if len(y) != n_samples:
|
|
raise ValueError("Number of labels=%d does not match "
|
|
"number of samples=%d" % (len(y), n_samples))
|
|
if not 0 <= self.min_weight_fraction_leaf <= 0.5:
|
|
raise ValueError("min_weight_fraction_leaf must in [0, 0.5]")
|
|
if max_depth <= 0:
|
|
raise ValueError("max_depth must be greater than zero. ")
|
|
if not (0 < max_features <= self.n_features_):
|
|
raise ValueError("max_features must be in (0, n_features]")
|
|
if not isinstance(max_leaf_nodes, numbers.Integral):
|
|
raise ValueError("max_leaf_nodes must be integral number but was "
|
|
"%r" % max_leaf_nodes)
|
|
if -1 < max_leaf_nodes < 2:
|
|
raise ValueError(("max_leaf_nodes {0} must be either None "
|
|
"or larger than 1").format(max_leaf_nodes))
|
|
|
|
if sample_weight is not None:
|
|
sample_weight = _check_sample_weight(sample_weight, X, DOUBLE)
|
|
|
|
if expanded_class_weight is not None:
|
|
if sample_weight is not None:
|
|
sample_weight = sample_weight * expanded_class_weight
|
|
else:
|
|
sample_weight = expanded_class_weight
|
|
|
|
# Set min_weight_leaf from min_weight_fraction_leaf
|
|
if sample_weight is None:
|
|
min_weight_leaf = (self.min_weight_fraction_leaf *
|
|
n_samples)
|
|
else:
|
|
min_weight_leaf = (self.min_weight_fraction_leaf *
|
|
np.sum(sample_weight))
|
|
|
|
min_impurity_split = self.min_impurity_split
|
|
if min_impurity_split is not None:
|
|
warnings.warn(
|
|
"The min_impurity_split parameter is deprecated. Its default "
|
|
"value has changed from 1e-7 to 0 in version 0.23, and it "
|
|
"will be removed in 1.0 (renaming of 0.25). Use the "
|
|
"min_impurity_decrease parameter instead.",
|
|
FutureWarning
|
|
)
|
|
|
|
if min_impurity_split < 0.:
|
|
raise ValueError("min_impurity_split must be greater than "
|
|
"or equal to 0")
|
|
else:
|
|
min_impurity_split = 0
|
|
|
|
if self.min_impurity_decrease < 0.:
|
|
raise ValueError("min_impurity_decrease must be greater than "
|
|
"or equal to 0")
|
|
|
|
# TODO: Remove in 1.1
|
|
if X_idx_sorted != "deprecated":
|
|
warnings.warn(
|
|
"The parameter 'X_idx_sorted' is deprecated and has no "
|
|
"effect. It will be removed in 1.1 (renaming of 0.26). You "
|
|
"can suppress this warning by not passing any value to the "
|
|
"'X_idx_sorted' parameter.",
|
|
FutureWarning
|
|
)
|
|
|
|
# Build tree
|
|
criterion = self.criterion
|
|
if not isinstance(criterion, Criterion):
|
|
if is_classification:
|
|
criterion = CRITERIA_CLF[self.criterion](self.n_outputs_,
|
|
self.n_classes_)
|
|
else:
|
|
criterion = CRITERIA_REG[self.criterion](self.n_outputs_,
|
|
n_samples)
|
|
|
|
SPLITTERS = SPARSE_SPLITTERS if issparse(X) else DENSE_SPLITTERS
|
|
|
|
splitter = self.splitter
|
|
if not isinstance(self.splitter, Splitter):
|
|
splitter = SPLITTERS[self.splitter](criterion,
|
|
self.max_features_,
|
|
min_samples_leaf,
|
|
min_weight_leaf,
|
|
random_state)
|
|
|
|
if is_classifier(self):
|
|
self.tree_ = Tree(self.n_features_,
|
|
self.n_classes_, self.n_outputs_)
|
|
else:
|
|
self.tree_ = Tree(self.n_features_,
|
|
# TODO: tree should't need this in this case
|
|
np.array([1] * self.n_outputs_, dtype=np.intp),
|
|
self.n_outputs_)
|
|
|
|
# Use BestFirst if max_leaf_nodes given; use DepthFirst otherwise
|
|
if max_leaf_nodes < 0:
|
|
builder = DepthFirstTreeBuilder(splitter, min_samples_split,
|
|
min_samples_leaf,
|
|
min_weight_leaf,
|
|
max_depth,
|
|
self.min_impurity_decrease,
|
|
min_impurity_split)
|
|
else:
|
|
builder = BestFirstTreeBuilder(splitter, min_samples_split,
|
|
min_samples_leaf,
|
|
min_weight_leaf,
|
|
max_depth,
|
|
max_leaf_nodes,
|
|
self.min_impurity_decrease,
|
|
min_impurity_split)
|
|
|
|
builder.build(self.tree_, X, y, sample_weight)
|
|
|
|
if self.n_outputs_ == 1 and is_classifier(self):
|
|
self.n_classes_ = self.n_classes_[0]
|
|
self.classes_ = self.classes_[0]
|
|
|
|
self._prune_tree()
|
|
|
|
return self
|
|
|
|
def _validate_X_predict(self, X, check_input):
|
|
"""Validate the training data on predict (probabilities)."""
|
|
if check_input:
|
|
X = self._validate_data(X, dtype=DTYPE, accept_sparse="csr",
|
|
reset=False)
|
|
if issparse(X) and (X.indices.dtype != np.intc or
|
|
X.indptr.dtype != np.intc):
|
|
raise ValueError("No support for np.int64 index based "
|
|
"sparse matrices")
|
|
else:
|
|
# The number of features is checked regardless of `check_input`
|
|
self._check_n_features(X, reset=False)
|
|
return X
|
|
|
|
def predict(self, X, check_input=True):
|
|
"""Predict class or regression value for X.
|
|
|
|
For a classification model, the predicted class for each sample in X is
|
|
returned. For a regression model, the predicted value based on X is
|
|
returned.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
|
The input samples. Internally, it will be converted to
|
|
``dtype=np.float32`` and if a sparse matrix is provided
|
|
to a sparse ``csr_matrix``.
|
|
|
|
check_input : bool, default=True
|
|
Allow to bypass several input checking.
|
|
Don't use this parameter unless you know what you do.
|
|
|
|
Returns
|
|
-------
|
|
y : array-like of shape (n_samples,) or (n_samples, n_outputs)
|
|
The predicted classes, or the predict values.
|
|
"""
|
|
check_is_fitted(self)
|
|
X = self._validate_X_predict(X, check_input)
|
|
proba = self.tree_.predict(X)
|
|
n_samples = X.shape[0]
|
|
|
|
# Classification
|
|
if is_classifier(self):
|
|
if self.n_outputs_ == 1:
|
|
return self.classes_.take(np.argmax(proba, axis=1), axis=0)
|
|
|
|
else:
|
|
class_type = self.classes_[0].dtype
|
|
predictions = np.zeros((n_samples, self.n_outputs_),
|
|
dtype=class_type)
|
|
for k in range(self.n_outputs_):
|
|
predictions[:, k] = self.classes_[k].take(
|
|
np.argmax(proba[:, k], axis=1),
|
|
axis=0)
|
|
|
|
return predictions
|
|
|
|
# Regression
|
|
else:
|
|
if self.n_outputs_ == 1:
|
|
return proba[:, 0]
|
|
|
|
else:
|
|
return proba[:, :, 0]
|
|
|
|
def apply(self, X, check_input=True):
|
|
"""Return the index of the leaf that each sample is predicted as.
|
|
|
|
.. versionadded:: 0.17
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
|
The input samples. Internally, it will be converted to
|
|
``dtype=np.float32`` and if a sparse matrix is provided
|
|
to a sparse ``csr_matrix``.
|
|
|
|
check_input : bool, default=True
|
|
Allow to bypass several input checking.
|
|
Don't use this parameter unless you know what you do.
|
|
|
|
Returns
|
|
-------
|
|
X_leaves : array-like of shape (n_samples,)
|
|
For each datapoint x in X, return the index of the leaf x
|
|
ends up in. Leaves are numbered within
|
|
``[0; self.tree_.node_count)``, possibly with gaps in the
|
|
numbering.
|
|
"""
|
|
check_is_fitted(self)
|
|
X = self._validate_X_predict(X, check_input)
|
|
return self.tree_.apply(X)
|
|
|
|
def decision_path(self, X, check_input=True):
|
|
"""Return the decision path in the tree.
|
|
|
|
.. versionadded:: 0.18
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
|
The input samples. Internally, it will be converted to
|
|
``dtype=np.float32`` and if a sparse matrix is provided
|
|
to a sparse ``csr_matrix``.
|
|
|
|
check_input : bool, default=True
|
|
Allow to bypass several input checking.
|
|
Don't use this parameter unless you know what you do.
|
|
|
|
Returns
|
|
-------
|
|
indicator : sparse matrix of shape (n_samples, n_nodes)
|
|
Return a node indicator CSR matrix where non zero elements
|
|
indicates that the samples goes through the nodes.
|
|
"""
|
|
X = self._validate_X_predict(X, check_input)
|
|
return self.tree_.decision_path(X)
|
|
|
|
def _prune_tree(self):
|
|
"""Prune tree using Minimal Cost-Complexity Pruning."""
|
|
check_is_fitted(self)
|
|
|
|
if self.ccp_alpha < 0.0:
|
|
raise ValueError("ccp_alpha must be greater than or equal to 0")
|
|
|
|
if self.ccp_alpha == 0.0:
|
|
return
|
|
|
|
# build pruned tree
|
|
if is_classifier(self):
|
|
n_classes = np.atleast_1d(self.n_classes_)
|
|
pruned_tree = Tree(self.n_features_, n_classes, self.n_outputs_)
|
|
else:
|
|
pruned_tree = Tree(self.n_features_,
|
|
# TODO: the tree shouldn't need this param
|
|
np.array([1] * self.n_outputs_, dtype=np.intp),
|
|
self.n_outputs_)
|
|
_build_pruned_tree_ccp(pruned_tree, self.tree_, self.ccp_alpha)
|
|
|
|
self.tree_ = pruned_tree
|
|
|
|
def cost_complexity_pruning_path(self, X, y, sample_weight=None):
|
|
"""Compute the pruning path during Minimal Cost-Complexity Pruning.
|
|
|
|
See :ref:`minimal_cost_complexity_pruning` for details on the pruning
|
|
process.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
|
The training input samples. Internally, it will be converted to
|
|
``dtype=np.float32`` and if a sparse matrix is provided
|
|
to a sparse ``csc_matrix``.
|
|
|
|
y : array-like of shape (n_samples,) or (n_samples, n_outputs)
|
|
The target values (class labels) as integers or strings.
|
|
|
|
sample_weight : array-like of shape (n_samples,), default=None
|
|
Sample weights. If None, then samples are equally weighted. Splits
|
|
that would create child nodes with net zero or negative weight are
|
|
ignored while searching for a split in each node. Splits are also
|
|
ignored if they would result in any single class carrying a
|
|
negative weight in either child node.
|
|
|
|
Returns
|
|
-------
|
|
ccp_path : :class:`~sklearn.utils.Bunch`
|
|
Dictionary-like object, with the following attributes.
|
|
|
|
ccp_alphas : ndarray
|
|
Effective alphas of subtree during pruning.
|
|
|
|
impurities : ndarray
|
|
Sum of the impurities of the subtree leaves for the
|
|
corresponding alpha value in ``ccp_alphas``.
|
|
"""
|
|
est = clone(self).set_params(ccp_alpha=0.0)
|
|
est.fit(X, y, sample_weight=sample_weight)
|
|
return Bunch(**ccp_pruning_path(est.tree_))
|
|
|
|
@property
|
|
def feature_importances_(self):
|
|
"""Return the feature importances.
|
|
|
|
The importance of a feature is computed as the (normalized) total
|
|
reduction of the criterion brought by that feature.
|
|
It is also known as the Gini importance.
|
|
|
|
Warning: impurity-based feature importances can be misleading for
|
|
high cardinality features (many unique values). See
|
|
:func:`sklearn.inspection.permutation_importance` as an alternative.
|
|
|
|
Returns
|
|
-------
|
|
feature_importances_ : ndarray of shape (n_features,)
|
|
Normalized total reduction of criteria by feature
|
|
(Gini importance).
|
|
"""
|
|
check_is_fitted(self)
|
|
|
|
return self.tree_.compute_feature_importances()
|
|
|
|
|
|
# =============================================================================
|
|
# Public estimators
|
|
# =============================================================================
|
|
|
|
class DecisionTreeClassifier(ClassifierMixin, BaseDecisionTree):
|
|
"""A decision tree classifier.
|
|
|
|
Read more in the :ref:`User Guide <tree>`.
|
|
|
|
Parameters
|
|
----------
|
|
criterion : {"gini", "entropy"}, default="gini"
|
|
The function to measure the quality of a split. Supported criteria are
|
|
"gini" for the Gini impurity and "entropy" for the information gain.
|
|
|
|
splitter : {"best", "random"}, default="best"
|
|
The strategy used to choose the split at each node. Supported
|
|
strategies are "best" to choose the best split and "random" to choose
|
|
the best random split.
|
|
|
|
max_depth : int, default=None
|
|
The maximum depth of the tree. If None, then nodes are expanded until
|
|
all leaves are pure or until all leaves contain less than
|
|
min_samples_split samples.
|
|
|
|
min_samples_split : int or float, default=2
|
|
The minimum number of samples required to split an internal node:
|
|
|
|
- If int, then consider `min_samples_split` as the minimum number.
|
|
- If float, then `min_samples_split` is a fraction and
|
|
`ceil(min_samples_split * n_samples)` are the minimum
|
|
number of samples for each split.
|
|
|
|
.. versionchanged:: 0.18
|
|
Added float values for fractions.
|
|
|
|
min_samples_leaf : int or float, default=1
|
|
The minimum number of samples required to be at a leaf node.
|
|
A split point at any depth will only be considered if it leaves at
|
|
least ``min_samples_leaf`` training samples in each of the left and
|
|
right branches. This may have the effect of smoothing the model,
|
|
especially in regression.
|
|
|
|
- If int, then consider `min_samples_leaf` as the minimum number.
|
|
- If float, then `min_samples_leaf` is a fraction and
|
|
`ceil(min_samples_leaf * n_samples)` are the minimum
|
|
number of samples for each node.
|
|
|
|
.. versionchanged:: 0.18
|
|
Added float values for fractions.
|
|
|
|
min_weight_fraction_leaf : float, default=0.0
|
|
The minimum weighted fraction of the sum total of weights (of all
|
|
the input samples) required to be at a leaf node. Samples have
|
|
equal weight when sample_weight is not provided.
|
|
|
|
max_features : int, float or {"auto", "sqrt", "log2"}, default=None
|
|
The number of features to consider when looking for the best split:
|
|
|
|
- If int, then consider `max_features` features at each split.
|
|
- If float, then `max_features` is a fraction and
|
|
`int(max_features * n_features)` features are considered at each
|
|
split.
|
|
- If "auto", then `max_features=sqrt(n_features)`.
|
|
- If "sqrt", then `max_features=sqrt(n_features)`.
|
|
- If "log2", then `max_features=log2(n_features)`.
|
|
- If None, then `max_features=n_features`.
|
|
|
|
Note: the search for a split does not stop until at least one
|
|
valid partition of the node samples is found, even if it requires to
|
|
effectively inspect more than ``max_features`` features.
|
|
|
|
random_state : int, RandomState instance or None, default=None
|
|
Controls the randomness of the estimator. The features are always
|
|
randomly permuted at each split, even if ``splitter`` is set to
|
|
``"best"``. When ``max_features < n_features``, the algorithm will
|
|
select ``max_features`` at random at each split before finding the best
|
|
split among them. But the best found split may vary across different
|
|
runs, even if ``max_features=n_features``. That is the case, if the
|
|
improvement of the criterion is identical for several splits and one
|
|
split has to be selected at random. To obtain a deterministic behaviour
|
|
during fitting, ``random_state`` has to be fixed to an integer.
|
|
See :term:`Glossary <random_state>` for details.
|
|
|
|
max_leaf_nodes : int, default=None
|
|
Grow a tree with ``max_leaf_nodes`` in best-first fashion.
|
|
Best nodes are defined as relative reduction in impurity.
|
|
If None then unlimited number of leaf nodes.
|
|
|
|
min_impurity_decrease : float, default=0.0
|
|
A node will be split if this split induces a decrease of the impurity
|
|
greater than or equal to this value.
|
|
|
|
The weighted impurity decrease equation is the following::
|
|
|
|
N_t / N * (impurity - N_t_R / N_t * right_impurity
|
|
- N_t_L / N_t * left_impurity)
|
|
|
|
where ``N`` is the total number of samples, ``N_t`` is the number of
|
|
samples at the current node, ``N_t_L`` is the number of samples in the
|
|
left child, and ``N_t_R`` is the number of samples in the right child.
|
|
|
|
``N``, ``N_t``, ``N_t_R`` and ``N_t_L`` all refer to the weighted sum,
|
|
if ``sample_weight`` is passed.
|
|
|
|
.. versionadded:: 0.19
|
|
|
|
min_impurity_split : float, default=0
|
|
Threshold for early stopping in tree growth. A node will split
|
|
if its impurity is above the threshold, otherwise it is a leaf.
|
|
|
|
.. deprecated:: 0.19
|
|
``min_impurity_split`` has been deprecated in favor of
|
|
``min_impurity_decrease`` in 0.19. The default value of
|
|
``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it
|
|
will be removed in 1.0 (renaming of 0.25).
|
|
Use ``min_impurity_decrease`` instead.
|
|
|
|
class_weight : dict, list of dict or "balanced", default=None
|
|
Weights associated with classes in the form ``{class_label: weight}``.
|
|
If None, all classes are supposed to have weight one. For
|
|
multi-output problems, a list of dicts can be provided in the same
|
|
order as the columns of y.
|
|
|
|
Note that for multioutput (including multilabel) weights should be
|
|
defined for each class of every column in its own dict. For example,
|
|
for four-class multilabel classification weights should be
|
|
[{0: 1, 1: 1}, {0: 1, 1: 5}, {0: 1, 1: 1}, {0: 1, 1: 1}] instead of
|
|
[{1:1}, {2:5}, {3:1}, {4:1}].
|
|
|
|
The "balanced" mode uses the values of y to automatically adjust
|
|
weights inversely proportional to class frequencies in the input data
|
|
as ``n_samples / (n_classes * np.bincount(y))``
|
|
|
|
For multi-output, the weights of each column of y will be multiplied.
|
|
|
|
Note that these weights will be multiplied with sample_weight (passed
|
|
through the fit method) if sample_weight is specified.
|
|
|
|
ccp_alpha : non-negative float, default=0.0
|
|
Complexity parameter used for Minimal Cost-Complexity Pruning. The
|
|
subtree with the largest cost complexity that is smaller than
|
|
``ccp_alpha`` will be chosen. By default, no pruning is performed. See
|
|
:ref:`minimal_cost_complexity_pruning` for details.
|
|
|
|
.. versionadded:: 0.22
|
|
|
|
Attributes
|
|
----------
|
|
classes_ : ndarray of shape (n_classes,) or list of ndarray
|
|
The classes labels (single output problem),
|
|
or a list of arrays of class labels (multi-output problem).
|
|
|
|
feature_importances_ : ndarray of shape (n_features,)
|
|
The impurity-based feature importances.
|
|
The higher, the more important the feature.
|
|
The importance of a feature is computed as the (normalized)
|
|
total reduction of the criterion brought by that feature. It is also
|
|
known as the Gini importance [4]_.
|
|
|
|
Warning: impurity-based feature importances can be misleading for
|
|
high cardinality features (many unique values). See
|
|
:func:`sklearn.inspection.permutation_importance` as an alternative.
|
|
|
|
max_features_ : int
|
|
The inferred value of max_features.
|
|
|
|
n_classes_ : int or list of int
|
|
The number of classes (for single output problems),
|
|
or a list containing the number of classes for each
|
|
output (for multi-output problems).
|
|
|
|
n_features_ : int
|
|
The number of features when ``fit`` is performed.
|
|
|
|
n_outputs_ : int
|
|
The number of outputs when ``fit`` is performed.
|
|
|
|
tree_ : Tree instance
|
|
The underlying Tree object. Please refer to
|
|
``help(sklearn.tree._tree.Tree)`` for attributes of Tree object and
|
|
:ref:`sphx_glr_auto_examples_tree_plot_unveil_tree_structure.py`
|
|
for basic usage of these attributes.
|
|
|
|
See Also
|
|
--------
|
|
DecisionTreeRegressor : A decision tree regressor.
|
|
|
|
Notes
|
|
-----
|
|
The default values for the parameters controlling the size of the trees
|
|
(e.g. ``max_depth``, ``min_samples_leaf``, etc.) lead to fully grown and
|
|
unpruned trees which can potentially be very large on some data sets. To
|
|
reduce memory consumption, the complexity and size of the trees should be
|
|
controlled by setting those parameter values.
|
|
|
|
The :meth:`predict` method operates using the :func:`numpy.argmax`
|
|
function on the outputs of :meth:`predict_proba`. This means that in
|
|
case the highest predicted probabilities are tied, the classifier will
|
|
predict the tied class with the lowest index in :term:`classes_`.
|
|
|
|
References
|
|
----------
|
|
|
|
.. [1] https://en.wikipedia.org/wiki/Decision_tree_learning
|
|
|
|
.. [2] L. Breiman, J. Friedman, R. Olshen, and C. Stone, "Classification
|
|
and Regression Trees", Wadsworth, Belmont, CA, 1984.
|
|
|
|
.. [3] T. Hastie, R. Tibshirani and J. Friedman. "Elements of Statistical
|
|
Learning", Springer, 2009.
|
|
|
|
.. [4] L. Breiman, and A. Cutler, "Random Forests",
|
|
https://www.stat.berkeley.edu/~breiman/RandomForests/cc_home.htm
|
|
|
|
Examples
|
|
--------
|
|
>>> from sklearn.datasets import load_iris
|
|
>>> from sklearn.model_selection import cross_val_score
|
|
>>> from sklearn.tree import DecisionTreeClassifier
|
|
>>> clf = DecisionTreeClassifier(random_state=0)
|
|
>>> iris = load_iris()
|
|
>>> cross_val_score(clf, iris.data, iris.target, cv=10)
|
|
... # doctest: +SKIP
|
|
...
|
|
array([ 1. , 0.93..., 0.86..., 0.93..., 0.93...,
|
|
0.93..., 0.93..., 1. , 0.93..., 1. ])
|
|
"""
|
|
@_deprecate_positional_args
|
|
def __init__(self, *,
|
|
criterion="gini",
|
|
splitter="best",
|
|
max_depth=None,
|
|
min_samples_split=2,
|
|
min_samples_leaf=1,
|
|
min_weight_fraction_leaf=0.,
|
|
max_features=None,
|
|
random_state=None,
|
|
max_leaf_nodes=None,
|
|
min_impurity_decrease=0.,
|
|
min_impurity_split=None,
|
|
class_weight=None,
|
|
ccp_alpha=0.0):
|
|
super().__init__(
|
|
criterion=criterion,
|
|
splitter=splitter,
|
|
max_depth=max_depth,
|
|
min_samples_split=min_samples_split,
|
|
min_samples_leaf=min_samples_leaf,
|
|
min_weight_fraction_leaf=min_weight_fraction_leaf,
|
|
max_features=max_features,
|
|
max_leaf_nodes=max_leaf_nodes,
|
|
class_weight=class_weight,
|
|
random_state=random_state,
|
|
min_impurity_decrease=min_impurity_decrease,
|
|
min_impurity_split=min_impurity_split,
|
|
ccp_alpha=ccp_alpha)
|
|
|
|
def fit(self, X, y, sample_weight=None, check_input=True,
|
|
X_idx_sorted="deprecated"):
|
|
"""Build a decision tree classifier from the training set (X, y).
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
|
The training input samples. Internally, it will be converted to
|
|
``dtype=np.float32`` and if a sparse matrix is provided
|
|
to a sparse ``csc_matrix``.
|
|
|
|
y : array-like of shape (n_samples,) or (n_samples, n_outputs)
|
|
The target values (class labels) as integers or strings.
|
|
|
|
sample_weight : array-like of shape (n_samples,), default=None
|
|
Sample weights. If None, then samples are equally weighted. Splits
|
|
that would create child nodes with net zero or negative weight are
|
|
ignored while searching for a split in each node. Splits are also
|
|
ignored if they would result in any single class carrying a
|
|
negative weight in either child node.
|
|
|
|
check_input : bool, default=True
|
|
Allow to bypass several input checking.
|
|
Don't use this parameter unless you know what you do.
|
|
|
|
X_idx_sorted : deprecated, default="deprecated"
|
|
This parameter is deprecated and has no effect.
|
|
It will be removed in 1.1 (renaming of 0.26).
|
|
|
|
.. deprecated :: 0.24
|
|
|
|
Returns
|
|
-------
|
|
self : DecisionTreeClassifier
|
|
Fitted estimator.
|
|
"""
|
|
|
|
super().fit(
|
|
X, y,
|
|
sample_weight=sample_weight,
|
|
check_input=check_input,
|
|
X_idx_sorted=X_idx_sorted)
|
|
return self
|
|
|
|
def predict_proba(self, X, check_input=True):
|
|
"""Predict class probabilities of the input samples X.
|
|
|
|
The predicted class probability is the fraction of samples of the same
|
|
class in a leaf.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
|
The input samples. Internally, it will be converted to
|
|
``dtype=np.float32`` and if a sparse matrix is provided
|
|
to a sparse ``csr_matrix``.
|
|
|
|
check_input : bool, default=True
|
|
Allow to bypass several input checking.
|
|
Don't use this parameter unless you know what you do.
|
|
|
|
Returns
|
|
-------
|
|
proba : ndarray of shape (n_samples, n_classes) or list of n_outputs \
|
|
such arrays if n_outputs > 1
|
|
The class probabilities of the input samples. The order of the
|
|
classes corresponds to that in the attribute :term:`classes_`.
|
|
"""
|
|
check_is_fitted(self)
|
|
X = self._validate_X_predict(X, check_input)
|
|
proba = self.tree_.predict(X)
|
|
|
|
if self.n_outputs_ == 1:
|
|
proba = proba[:, :self.n_classes_]
|
|
normalizer = proba.sum(axis=1)[:, np.newaxis]
|
|
normalizer[normalizer == 0.0] = 1.0
|
|
proba /= normalizer
|
|
|
|
return proba
|
|
|
|
else:
|
|
all_proba = []
|
|
|
|
for k in range(self.n_outputs_):
|
|
proba_k = proba[:, k, :self.n_classes_[k]]
|
|
normalizer = proba_k.sum(axis=1)[:, np.newaxis]
|
|
normalizer[normalizer == 0.0] = 1.0
|
|
proba_k /= normalizer
|
|
all_proba.append(proba_k)
|
|
|
|
return all_proba
|
|
|
|
def predict_log_proba(self, X):
|
|
"""Predict class log-probabilities of the input samples X.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
|
The input samples. Internally, it will be converted to
|
|
``dtype=np.float32`` and if a sparse matrix is provided
|
|
to a sparse ``csr_matrix``.
|
|
|
|
Returns
|
|
-------
|
|
proba : ndarray of shape (n_samples, n_classes) or list of n_outputs \
|
|
such arrays if n_outputs > 1
|
|
The class log-probabilities of the input samples. The order of the
|
|
classes corresponds to that in the attribute :term:`classes_`.
|
|
"""
|
|
proba = self.predict_proba(X)
|
|
|
|
if self.n_outputs_ == 1:
|
|
return np.log(proba)
|
|
|
|
else:
|
|
for k in range(self.n_outputs_):
|
|
proba[k] = np.log(proba[k])
|
|
|
|
return proba
|
|
|
|
|
|
class DecisionTreeRegressor(RegressorMixin, BaseDecisionTree):
|
|
"""A decision tree regressor.
|
|
|
|
Read more in the :ref:`User Guide <tree>`.
|
|
|
|
Parameters
|
|
----------
|
|
criterion : {"mse", "friedman_mse", "mae", "poisson"}, default="mse"
|
|
The function to measure the quality of a split. Supported criteria
|
|
are "mse" for the mean squared error, which is equal to variance
|
|
reduction as feature selection criterion and minimizes the L2 loss
|
|
using the mean of each terminal node, "friedman_mse", which uses mean
|
|
squared error with Friedman's improvement score for potential splits,
|
|
"mae" for the mean absolute error, which minimizes the L1 loss using
|
|
the median of each terminal node, and "poisson" which uses reduction in
|
|
Poisson deviance to find splits.
|
|
|
|
.. versionadded:: 0.18
|
|
Mean Absolute Error (MAE) criterion.
|
|
|
|
.. versionadded:: 0.24
|
|
Poisson deviance criterion.
|
|
|
|
splitter : {"best", "random"}, default="best"
|
|
The strategy used to choose the split at each node. Supported
|
|
strategies are "best" to choose the best split and "random" to choose
|
|
the best random split.
|
|
|
|
max_depth : int, default=None
|
|
The maximum depth of the tree. If None, then nodes are expanded until
|
|
all leaves are pure or until all leaves contain less than
|
|
min_samples_split samples.
|
|
|
|
min_samples_split : int or float, default=2
|
|
The minimum number of samples required to split an internal node:
|
|
|
|
- If int, then consider `min_samples_split` as the minimum number.
|
|
- If float, then `min_samples_split` is a fraction and
|
|
`ceil(min_samples_split * n_samples)` are the minimum
|
|
number of samples for each split.
|
|
|
|
.. versionchanged:: 0.18
|
|
Added float values for fractions.
|
|
|
|
min_samples_leaf : int or float, default=1
|
|
The minimum number of samples required to be at a leaf node.
|
|
A split point at any depth will only be considered if it leaves at
|
|
least ``min_samples_leaf`` training samples in each of the left and
|
|
right branches. This may have the effect of smoothing the model,
|
|
especially in regression.
|
|
|
|
- If int, then consider `min_samples_leaf` as the minimum number.
|
|
- If float, then `min_samples_leaf` is a fraction and
|
|
`ceil(min_samples_leaf * n_samples)` are the minimum
|
|
number of samples for each node.
|
|
|
|
.. versionchanged:: 0.18
|
|
Added float values for fractions.
|
|
|
|
min_weight_fraction_leaf : float, default=0.0
|
|
The minimum weighted fraction of the sum total of weights (of all
|
|
the input samples) required to be at a leaf node. Samples have
|
|
equal weight when sample_weight is not provided.
|
|
|
|
max_features : int, float or {"auto", "sqrt", "log2"}, default=None
|
|
The number of features to consider when looking for the best split:
|
|
|
|
- If int, then consider `max_features` features at each split.
|
|
- If float, then `max_features` is a fraction and
|
|
`int(max_features * n_features)` features are considered at each
|
|
split.
|
|
- If "auto", then `max_features=n_features`.
|
|
- If "sqrt", then `max_features=sqrt(n_features)`.
|
|
- If "log2", then `max_features=log2(n_features)`.
|
|
- If None, then `max_features=n_features`.
|
|
|
|
Note: the search for a split does not stop until at least one
|
|
valid partition of the node samples is found, even if it requires to
|
|
effectively inspect more than ``max_features`` features.
|
|
|
|
random_state : int, RandomState instance or None, default=None
|
|
Controls the randomness of the estimator. The features are always
|
|
randomly permuted at each split, even if ``splitter`` is set to
|
|
``"best"``. When ``max_features < n_features``, the algorithm will
|
|
select ``max_features`` at random at each split before finding the best
|
|
split among them. But the best found split may vary across different
|
|
runs, even if ``max_features=n_features``. That is the case, if the
|
|
improvement of the criterion is identical for several splits and one
|
|
split has to be selected at random. To obtain a deterministic behaviour
|
|
during fitting, ``random_state`` has to be fixed to an integer.
|
|
See :term:`Glossary <random_state>` for details.
|
|
|
|
max_leaf_nodes : int, default=None
|
|
Grow a tree with ``max_leaf_nodes`` in best-first fashion.
|
|
Best nodes are defined as relative reduction in impurity.
|
|
If None then unlimited number of leaf nodes.
|
|
|
|
min_impurity_decrease : float, default=0.0
|
|
A node will be split if this split induces a decrease of the impurity
|
|
greater than or equal to this value.
|
|
|
|
The weighted impurity decrease equation is the following::
|
|
|
|
N_t / N * (impurity - N_t_R / N_t * right_impurity
|
|
- N_t_L / N_t * left_impurity)
|
|
|
|
where ``N`` is the total number of samples, ``N_t`` is the number of
|
|
samples at the current node, ``N_t_L`` is the number of samples in the
|
|
left child, and ``N_t_R`` is the number of samples in the right child.
|
|
|
|
``N``, ``N_t``, ``N_t_R`` and ``N_t_L`` all refer to the weighted sum,
|
|
if ``sample_weight`` is passed.
|
|
|
|
.. versionadded:: 0.19
|
|
|
|
min_impurity_split : float, default=0
|
|
Threshold for early stopping in tree growth. A node will split
|
|
if its impurity is above the threshold, otherwise it is a leaf.
|
|
|
|
.. deprecated:: 0.19
|
|
``min_impurity_split`` has been deprecated in favor of
|
|
``min_impurity_decrease`` in 0.19. The default value of
|
|
``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it
|
|
will be removed in 1.0 (renaming of 0.25).
|
|
Use ``min_impurity_decrease`` instead.
|
|
|
|
ccp_alpha : non-negative float, default=0.0
|
|
Complexity parameter used for Minimal Cost-Complexity Pruning. The
|
|
subtree with the largest cost complexity that is smaller than
|
|
``ccp_alpha`` will be chosen. By default, no pruning is performed. See
|
|
:ref:`minimal_cost_complexity_pruning` for details.
|
|
|
|
.. versionadded:: 0.22
|
|
|
|
Attributes
|
|
----------
|
|
feature_importances_ : ndarray of shape (n_features,)
|
|
The feature importances.
|
|
The higher, the more important the feature.
|
|
The importance of a feature is computed as the
|
|
(normalized) total reduction of the criterion brought
|
|
by that feature. It is also known as the Gini importance [4]_.
|
|
|
|
Warning: impurity-based feature importances can be misleading for
|
|
high cardinality features (many unique values). See
|
|
:func:`sklearn.inspection.permutation_importance` as an alternative.
|
|
|
|
max_features_ : int
|
|
The inferred value of max_features.
|
|
|
|
n_features_ : int
|
|
The number of features when ``fit`` is performed.
|
|
|
|
n_outputs_ : int
|
|
The number of outputs when ``fit`` is performed.
|
|
|
|
tree_ : Tree instance
|
|
The underlying Tree object. Please refer to
|
|
``help(sklearn.tree._tree.Tree)`` for attributes of Tree object and
|
|
:ref:`sphx_glr_auto_examples_tree_plot_unveil_tree_structure.py`
|
|
for basic usage of these attributes.
|
|
|
|
See Also
|
|
--------
|
|
DecisionTreeClassifier : A decision tree classifier.
|
|
|
|
Notes
|
|
-----
|
|
The default values for the parameters controlling the size of the trees
|
|
(e.g. ``max_depth``, ``min_samples_leaf``, etc.) lead to fully grown and
|
|
unpruned trees which can potentially be very large on some data sets. To
|
|
reduce memory consumption, the complexity and size of the trees should be
|
|
controlled by setting those parameter values.
|
|
|
|
References
|
|
----------
|
|
|
|
.. [1] https://en.wikipedia.org/wiki/Decision_tree_learning
|
|
|
|
.. [2] L. Breiman, J. Friedman, R. Olshen, and C. Stone, "Classification
|
|
and Regression Trees", Wadsworth, Belmont, CA, 1984.
|
|
|
|
.. [3] T. Hastie, R. Tibshirani and J. Friedman. "Elements of Statistical
|
|
Learning", Springer, 2009.
|
|
|
|
.. [4] L. Breiman, and A. Cutler, "Random Forests",
|
|
https://www.stat.berkeley.edu/~breiman/RandomForests/cc_home.htm
|
|
|
|
Examples
|
|
--------
|
|
>>> from sklearn.datasets import load_diabetes
|
|
>>> from sklearn.model_selection import cross_val_score
|
|
>>> from sklearn.tree import DecisionTreeRegressor
|
|
>>> X, y = load_diabetes(return_X_y=True)
|
|
>>> regressor = DecisionTreeRegressor(random_state=0)
|
|
>>> cross_val_score(regressor, X, y, cv=10)
|
|
... # doctest: +SKIP
|
|
...
|
|
array([-0.39..., -0.46..., 0.02..., 0.06..., -0.50...,
|
|
0.16..., 0.11..., -0.73..., -0.30..., -0.00...])
|
|
"""
|
|
@_deprecate_positional_args
|
|
def __init__(self, *,
|
|
criterion="mse",
|
|
splitter="best",
|
|
max_depth=None,
|
|
min_samples_split=2,
|
|
min_samples_leaf=1,
|
|
min_weight_fraction_leaf=0.,
|
|
max_features=None,
|
|
random_state=None,
|
|
max_leaf_nodes=None,
|
|
min_impurity_decrease=0.,
|
|
min_impurity_split=None,
|
|
ccp_alpha=0.0):
|
|
super().__init__(
|
|
criterion=criterion,
|
|
splitter=splitter,
|
|
max_depth=max_depth,
|
|
min_samples_split=min_samples_split,
|
|
min_samples_leaf=min_samples_leaf,
|
|
min_weight_fraction_leaf=min_weight_fraction_leaf,
|
|
max_features=max_features,
|
|
max_leaf_nodes=max_leaf_nodes,
|
|
random_state=random_state,
|
|
min_impurity_decrease=min_impurity_decrease,
|
|
min_impurity_split=min_impurity_split,
|
|
ccp_alpha=ccp_alpha)
|
|
|
|
def fit(self, X, y, sample_weight=None, check_input=True,
|
|
X_idx_sorted="deprecated"):
|
|
"""Build a decision tree regressor from the training set (X, y).
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
|
The training input samples. Internally, it will be converted to
|
|
``dtype=np.float32`` and if a sparse matrix is provided
|
|
to a sparse ``csc_matrix``.
|
|
|
|
y : array-like of shape (n_samples,) or (n_samples, n_outputs)
|
|
The target values (real numbers). Use ``dtype=np.float64`` and
|
|
``order='C'`` for maximum efficiency.
|
|
|
|
sample_weight : array-like of shape (n_samples,), default=None
|
|
Sample weights. If None, then samples are equally weighted. Splits
|
|
that would create child nodes with net zero or negative weight are
|
|
ignored while searching for a split in each node.
|
|
|
|
check_input : bool, default=True
|
|
Allow to bypass several input checking.
|
|
Don't use this parameter unless you know what you do.
|
|
|
|
X_idx_sorted : deprecated, default="deprecated"
|
|
This parameter is deprecated and has no effect.
|
|
It will be removed in 1.1 (renaming of 0.26).
|
|
|
|
.. deprecated :: 0.24
|
|
|
|
Returns
|
|
-------
|
|
self : DecisionTreeRegressor
|
|
Fitted estimator.
|
|
"""
|
|
|
|
super().fit(
|
|
X, y,
|
|
sample_weight=sample_weight,
|
|
check_input=check_input,
|
|
X_idx_sorted=X_idx_sorted)
|
|
return self
|
|
|
|
def _compute_partial_dependence_recursion(self, grid, target_features):
|
|
"""Fast partial dependence computation.
|
|
|
|
Parameters
|
|
----------
|
|
grid : ndarray of shape (n_samples, n_target_features)
|
|
The grid points on which the partial dependence should be
|
|
evaluated.
|
|
target_features : ndarray of shape (n_target_features)
|
|
The set of target features for which the partial dependence
|
|
should be evaluated.
|
|
|
|
Returns
|
|
-------
|
|
averaged_predictions : ndarray of shape (n_samples,)
|
|
The value of the partial dependence function on each grid point.
|
|
"""
|
|
grid = np.asarray(grid, dtype=DTYPE, order='C')
|
|
averaged_predictions = np.zeros(shape=grid.shape[0],
|
|
dtype=np.float64, order='C')
|
|
|
|
self.tree_.compute_partial_dependence(
|
|
grid, target_features, averaged_predictions)
|
|
return averaged_predictions
|
|
|
|
|
|
class ExtraTreeClassifier(DecisionTreeClassifier):
|
|
"""An extremely randomized tree classifier.
|
|
|
|
Extra-trees differ from classic decision trees in the way they are built.
|
|
When looking for the best split to separate the samples of a node into two
|
|
groups, random splits are drawn for each of the `max_features` randomly
|
|
selected features and the best split among those is chosen. When
|
|
`max_features` is set 1, this amounts to building a totally random
|
|
decision tree.
|
|
|
|
Warning: Extra-trees should only be used within ensemble methods.
|
|
|
|
Read more in the :ref:`User Guide <tree>`.
|
|
|
|
Parameters
|
|
----------
|
|
criterion : {"gini", "entropy"}, default="gini"
|
|
The function to measure the quality of a split. Supported criteria are
|
|
"gini" for the Gini impurity and "entropy" for the information gain.
|
|
|
|
splitter : {"random", "best"}, default="random"
|
|
The strategy used to choose the split at each node. Supported
|
|
strategies are "best" to choose the best split and "random" to choose
|
|
the best random split.
|
|
|
|
max_depth : int, default=None
|
|
The maximum depth of the tree. If None, then nodes are expanded until
|
|
all leaves are pure or until all leaves contain less than
|
|
min_samples_split samples.
|
|
|
|
min_samples_split : int or float, default=2
|
|
The minimum number of samples required to split an internal node:
|
|
|
|
- If int, then consider `min_samples_split` as the minimum number.
|
|
- If float, then `min_samples_split` is a fraction and
|
|
`ceil(min_samples_split * n_samples)` are the minimum
|
|
number of samples for each split.
|
|
|
|
.. versionchanged:: 0.18
|
|
Added float values for fractions.
|
|
|
|
min_samples_leaf : int or float, default=1
|
|
The minimum number of samples required to be at a leaf node.
|
|
A split point at any depth will only be considered if it leaves at
|
|
least ``min_samples_leaf`` training samples in each of the left and
|
|
right branches. This may have the effect of smoothing the model,
|
|
especially in regression.
|
|
|
|
- If int, then consider `min_samples_leaf` as the minimum number.
|
|
- If float, then `min_samples_leaf` is a fraction and
|
|
`ceil(min_samples_leaf * n_samples)` are the minimum
|
|
number of samples for each node.
|
|
|
|
.. versionchanged:: 0.18
|
|
Added float values for fractions.
|
|
|
|
min_weight_fraction_leaf : float, default=0.0
|
|
The minimum weighted fraction of the sum total of weights (of all
|
|
the input samples) required to be at a leaf node. Samples have
|
|
equal weight when sample_weight is not provided.
|
|
|
|
max_features : int, float, {"auto", "sqrt", "log2"} or None, default="auto"
|
|
The number of features to consider when looking for the best split:
|
|
|
|
- If int, then consider `max_features` features at each split.
|
|
- If float, then `max_features` is a fraction and
|
|
`int(max_features * n_features)` features are considered at each
|
|
split.
|
|
- If "auto", then `max_features=sqrt(n_features)`.
|
|
- If "sqrt", then `max_features=sqrt(n_features)`.
|
|
- If "log2", then `max_features=log2(n_features)`.
|
|
- If None, then `max_features=n_features`.
|
|
|
|
Note: the search for a split does not stop until at least one
|
|
valid partition of the node samples is found, even if it requires to
|
|
effectively inspect more than ``max_features`` features.
|
|
|
|
random_state : int, RandomState instance or None, default=None
|
|
Used to pick randomly the `max_features` used at each split.
|
|
See :term:`Glossary <random_state>` for details.
|
|
|
|
max_leaf_nodes : int, default=None
|
|
Grow a tree with ``max_leaf_nodes`` in best-first fashion.
|
|
Best nodes are defined as relative reduction in impurity.
|
|
If None then unlimited number of leaf nodes.
|
|
|
|
min_impurity_decrease : float, default=0.0
|
|
A node will be split if this split induces a decrease of the impurity
|
|
greater than or equal to this value.
|
|
|
|
The weighted impurity decrease equation is the following::
|
|
|
|
N_t / N * (impurity - N_t_R / N_t * right_impurity
|
|
- N_t_L / N_t * left_impurity)
|
|
|
|
where ``N`` is the total number of samples, ``N_t`` is the number of
|
|
samples at the current node, ``N_t_L`` is the number of samples in the
|
|
left child, and ``N_t_R`` is the number of samples in the right child.
|
|
|
|
``N``, ``N_t``, ``N_t_R`` and ``N_t_L`` all refer to the weighted sum,
|
|
if ``sample_weight`` is passed.
|
|
|
|
.. versionadded:: 0.19
|
|
|
|
min_impurity_split : float, default=None
|
|
Threshold for early stopping in tree growth. A node will split
|
|
if its impurity is above the threshold, otherwise it is a leaf.
|
|
|
|
.. deprecated:: 0.19
|
|
``min_impurity_split`` has been deprecated in favor of
|
|
``min_impurity_decrease`` in 0.19. The default value of
|
|
``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it
|
|
will be removed in 1.0 (renaming of 0.25).
|
|
Use ``min_impurity_decrease`` instead.
|
|
|
|
class_weight : dict, list of dict or "balanced", default=None
|
|
Weights associated with classes in the form ``{class_label: weight}``.
|
|
If None, all classes are supposed to have weight one. For
|
|
multi-output problems, a list of dicts can be provided in the same
|
|
order as the columns of y.
|
|
|
|
Note that for multioutput (including multilabel) weights should be
|
|
defined for each class of every column in its own dict. For example,
|
|
for four-class multilabel classification weights should be
|
|
[{0: 1, 1: 1}, {0: 1, 1: 5}, {0: 1, 1: 1}, {0: 1, 1: 1}] instead of
|
|
[{1:1}, {2:5}, {3:1}, {4:1}].
|
|
|
|
The "balanced" mode uses the values of y to automatically adjust
|
|
weights inversely proportional to class frequencies in the input data
|
|
as ``n_samples / (n_classes * np.bincount(y))``
|
|
|
|
For multi-output, the weights of each column of y will be multiplied.
|
|
|
|
Note that these weights will be multiplied with sample_weight (passed
|
|
through the fit method) if sample_weight is specified.
|
|
|
|
ccp_alpha : non-negative float, default=0.0
|
|
Complexity parameter used for Minimal Cost-Complexity Pruning. The
|
|
subtree with the largest cost complexity that is smaller than
|
|
``ccp_alpha`` will be chosen. By default, no pruning is performed. See
|
|
:ref:`minimal_cost_complexity_pruning` for details.
|
|
|
|
.. versionadded:: 0.22
|
|
|
|
Attributes
|
|
----------
|
|
classes_ : ndarray of shape (n_classes,) or list of ndarray
|
|
The classes labels (single output problem),
|
|
or a list of arrays of class labels (multi-output problem).
|
|
|
|
max_features_ : int
|
|
The inferred value of max_features.
|
|
|
|
n_classes_ : int or list of int
|
|
The number of classes (for single output problems),
|
|
or a list containing the number of classes for each
|
|
output (for multi-output problems).
|
|
|
|
feature_importances_ : ndarray of shape (n_features,)
|
|
The impurity-based feature importances.
|
|
The higher, the more important the feature.
|
|
The importance of a feature is computed as the (normalized)
|
|
total reduction of the criterion brought by that feature. It is also
|
|
known as the Gini importance.
|
|
|
|
Warning: impurity-based feature importances can be misleading for
|
|
high cardinality features (many unique values). See
|
|
:func:`sklearn.inspection.permutation_importance` as an alternative.
|
|
|
|
n_features_ : int
|
|
The number of features when ``fit`` is performed.
|
|
|
|
n_outputs_ : int
|
|
The number of outputs when ``fit`` is performed.
|
|
|
|
tree_ : Tree instance
|
|
The underlying Tree object. Please refer to
|
|
``help(sklearn.tree._tree.Tree)`` for attributes of Tree object and
|
|
:ref:`sphx_glr_auto_examples_tree_plot_unveil_tree_structure.py`
|
|
for basic usage of these attributes.
|
|
|
|
See Also
|
|
--------
|
|
ExtraTreeRegressor : An extremely randomized tree regressor.
|
|
sklearn.ensemble.ExtraTreesClassifier : An extra-trees classifier.
|
|
sklearn.ensemble.ExtraTreesRegressor : An extra-trees regressor.
|
|
|
|
Notes
|
|
-----
|
|
The default values for the parameters controlling the size of the trees
|
|
(e.g. ``max_depth``, ``min_samples_leaf``, etc.) lead to fully grown and
|
|
unpruned trees which can potentially be very large on some data sets. To
|
|
reduce memory consumption, the complexity and size of the trees should be
|
|
controlled by setting those parameter values.
|
|
|
|
References
|
|
----------
|
|
|
|
.. [1] P. Geurts, D. Ernst., and L. Wehenkel, "Extremely randomized trees",
|
|
Machine Learning, 63(1), 3-42, 2006.
|
|
|
|
Examples
|
|
--------
|
|
>>> from sklearn.datasets import load_iris
|
|
>>> from sklearn.model_selection import train_test_split
|
|
>>> from sklearn.ensemble import BaggingClassifier
|
|
>>> from sklearn.tree import ExtraTreeClassifier
|
|
>>> X, y = load_iris(return_X_y=True)
|
|
>>> X_train, X_test, y_train, y_test = train_test_split(
|
|
... X, y, random_state=0)
|
|
>>> extra_tree = ExtraTreeClassifier(random_state=0)
|
|
>>> cls = BaggingClassifier(extra_tree, random_state=0).fit(
|
|
... X_train, y_train)
|
|
>>> cls.score(X_test, y_test)
|
|
0.8947...
|
|
"""
|
|
@_deprecate_positional_args
|
|
def __init__(self, *,
|
|
criterion="gini",
|
|
splitter="random",
|
|
max_depth=None,
|
|
min_samples_split=2,
|
|
min_samples_leaf=1,
|
|
min_weight_fraction_leaf=0.,
|
|
max_features="auto",
|
|
random_state=None,
|
|
max_leaf_nodes=None,
|
|
min_impurity_decrease=0.,
|
|
min_impurity_split=None,
|
|
class_weight=None,
|
|
ccp_alpha=0.0):
|
|
super().__init__(
|
|
criterion=criterion,
|
|
splitter=splitter,
|
|
max_depth=max_depth,
|
|
min_samples_split=min_samples_split,
|
|
min_samples_leaf=min_samples_leaf,
|
|
min_weight_fraction_leaf=min_weight_fraction_leaf,
|
|
max_features=max_features,
|
|
max_leaf_nodes=max_leaf_nodes,
|
|
class_weight=class_weight,
|
|
min_impurity_decrease=min_impurity_decrease,
|
|
min_impurity_split=min_impurity_split,
|
|
random_state=random_state,
|
|
ccp_alpha=ccp_alpha)
|
|
|
|
|
|
class ExtraTreeRegressor(DecisionTreeRegressor):
|
|
"""An extremely randomized tree regressor.
|
|
|
|
Extra-trees differ from classic decision trees in the way they are built.
|
|
When looking for the best split to separate the samples of a node into two
|
|
groups, random splits are drawn for each of the `max_features` randomly
|
|
selected features and the best split among those is chosen. When
|
|
`max_features` is set 1, this amounts to building a totally random
|
|
decision tree.
|
|
|
|
Warning: Extra-trees should only be used within ensemble methods.
|
|
|
|
Read more in the :ref:`User Guide <tree>`.
|
|
|
|
Parameters
|
|
----------
|
|
criterion : {"mse", "friedman_mse", "mae"}, default="mse"
|
|
The function to measure the quality of a split. Supported criteria
|
|
are "mse" for the mean squared error, which is equal to variance
|
|
reduction as feature selection criterion and "mae" for the mean
|
|
absolute error.
|
|
|
|
.. versionadded:: 0.18
|
|
Mean Absolute Error (MAE) criterion.
|
|
|
|
.. versionadded:: 0.24
|
|
Poisson deviance criterion.
|
|
|
|
splitter : {"random", "best"}, default="random"
|
|
The strategy used to choose the split at each node. Supported
|
|
strategies are "best" to choose the best split and "random" to choose
|
|
the best random split.
|
|
|
|
max_depth : int, default=None
|
|
The maximum depth of the tree. If None, then nodes are expanded until
|
|
all leaves are pure or until all leaves contain less than
|
|
min_samples_split samples.
|
|
|
|
min_samples_split : int or float, default=2
|
|
The minimum number of samples required to split an internal node:
|
|
|
|
- If int, then consider `min_samples_split` as the minimum number.
|
|
- If float, then `min_samples_split` is a fraction and
|
|
`ceil(min_samples_split * n_samples)` are the minimum
|
|
number of samples for each split.
|
|
|
|
.. versionchanged:: 0.18
|
|
Added float values for fractions.
|
|
|
|
min_samples_leaf : int or float, default=1
|
|
The minimum number of samples required to be at a leaf node.
|
|
A split point at any depth will only be considered if it leaves at
|
|
least ``min_samples_leaf`` training samples in each of the left and
|
|
right branches. This may have the effect of smoothing the model,
|
|
especially in regression.
|
|
|
|
- If int, then consider `min_samples_leaf` as the minimum number.
|
|
- If float, then `min_samples_leaf` is a fraction and
|
|
`ceil(min_samples_leaf * n_samples)` are the minimum
|
|
number of samples for each node.
|
|
|
|
.. versionchanged:: 0.18
|
|
Added float values for fractions.
|
|
|
|
min_weight_fraction_leaf : float, default=0.0
|
|
The minimum weighted fraction of the sum total of weights (of all
|
|
the input samples) required to be at a leaf node. Samples have
|
|
equal weight when sample_weight is not provided.
|
|
|
|
max_features : int, float, {"auto", "sqrt", "log2"} or None, default="auto"
|
|
The number of features to consider when looking for the best split:
|
|
|
|
- If int, then consider `max_features` features at each split.
|
|
- If float, then `max_features` is a fraction and
|
|
`int(max_features * n_features)` features are considered at each
|
|
split.
|
|
- If "auto", then `max_features=n_features`.
|
|
- If "sqrt", then `max_features=sqrt(n_features)`.
|
|
- If "log2", then `max_features=log2(n_features)`.
|
|
- If None, then `max_features=n_features`.
|
|
|
|
Note: the search for a split does not stop until at least one
|
|
valid partition of the node samples is found, even if it requires to
|
|
effectively inspect more than ``max_features`` features.
|
|
|
|
random_state : int, RandomState instance or None, default=None
|
|
Used to pick randomly the `max_features` used at each split.
|
|
See :term:`Glossary <random_state>` for details.
|
|
|
|
min_impurity_decrease : float, default=0.0
|
|
A node will be split if this split induces a decrease of the impurity
|
|
greater than or equal to this value.
|
|
|
|
The weighted impurity decrease equation is the following::
|
|
|
|
N_t / N * (impurity - N_t_R / N_t * right_impurity
|
|
- N_t_L / N_t * left_impurity)
|
|
|
|
where ``N`` is the total number of samples, ``N_t`` is the number of
|
|
samples at the current node, ``N_t_L`` is the number of samples in the
|
|
left child, and ``N_t_R`` is the number of samples in the right child.
|
|
|
|
``N``, ``N_t``, ``N_t_R`` and ``N_t_L`` all refer to the weighted sum,
|
|
if ``sample_weight`` is passed.
|
|
|
|
.. versionadded:: 0.19
|
|
|
|
min_impurity_split : float, default=None
|
|
Threshold for early stopping in tree growth. A node will split
|
|
if its impurity is above the threshold, otherwise it is a leaf.
|
|
|
|
.. deprecated:: 0.19
|
|
``min_impurity_split`` has been deprecated in favor of
|
|
``min_impurity_decrease`` in 0.19. The default value of
|
|
``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it
|
|
will be removed in 1.0 (renaming of 0.25).
|
|
Use ``min_impurity_decrease`` instead.
|
|
|
|
max_leaf_nodes : int, default=None
|
|
Grow a tree with ``max_leaf_nodes`` in best-first fashion.
|
|
Best nodes are defined as relative reduction in impurity.
|
|
If None then unlimited number of leaf nodes.
|
|
|
|
ccp_alpha : non-negative float, default=0.0
|
|
Complexity parameter used for Minimal Cost-Complexity Pruning. The
|
|
subtree with the largest cost complexity that is smaller than
|
|
``ccp_alpha`` will be chosen. By default, no pruning is performed. See
|
|
:ref:`minimal_cost_complexity_pruning` for details.
|
|
|
|
.. versionadded:: 0.22
|
|
|
|
Attributes
|
|
----------
|
|
max_features_ : int
|
|
The inferred value of max_features.
|
|
|
|
n_features_ : int
|
|
The number of features when ``fit`` is performed.
|
|
|
|
feature_importances_ : ndarray of shape (n_features,)
|
|
Return impurity-based feature importances (the higher, the more
|
|
important the feature).
|
|
|
|
Warning: impurity-based feature importances can be misleading for
|
|
high cardinality features (many unique values). See
|
|
:func:`sklearn.inspection.permutation_importance` as an alternative.
|
|
|
|
n_outputs_ : int
|
|
The number of outputs when ``fit`` is performed.
|
|
|
|
tree_ : Tree instance
|
|
The underlying Tree object. Please refer to
|
|
``help(sklearn.tree._tree.Tree)`` for attributes of Tree object and
|
|
:ref:`sphx_glr_auto_examples_tree_plot_unveil_tree_structure.py`
|
|
for basic usage of these attributes.
|
|
|
|
See Also
|
|
--------
|
|
ExtraTreeClassifier : An extremely randomized tree classifier.
|
|
sklearn.ensemble.ExtraTreesClassifier : An extra-trees classifier.
|
|
sklearn.ensemble.ExtraTreesRegressor : An extra-trees regressor.
|
|
|
|
Notes
|
|
-----
|
|
The default values for the parameters controlling the size of the trees
|
|
(e.g. ``max_depth``, ``min_samples_leaf``, etc.) lead to fully grown and
|
|
unpruned trees which can potentially be very large on some data sets. To
|
|
reduce memory consumption, the complexity and size of the trees should be
|
|
controlled by setting those parameter values.
|
|
|
|
References
|
|
----------
|
|
|
|
.. [1] P. Geurts, D. Ernst., and L. Wehenkel, "Extremely randomized trees",
|
|
Machine Learning, 63(1), 3-42, 2006.
|
|
|
|
Examples
|
|
--------
|
|
>>> from sklearn.datasets import load_diabetes
|
|
>>> from sklearn.model_selection import train_test_split
|
|
>>> from sklearn.ensemble import BaggingRegressor
|
|
>>> from sklearn.tree import ExtraTreeRegressor
|
|
>>> X, y = load_diabetes(return_X_y=True)
|
|
>>> X_train, X_test, y_train, y_test = train_test_split(
|
|
... X, y, random_state=0)
|
|
>>> extra_tree = ExtraTreeRegressor(random_state=0)
|
|
>>> reg = BaggingRegressor(extra_tree, random_state=0).fit(
|
|
... X_train, y_train)
|
|
>>> reg.score(X_test, y_test)
|
|
0.33...
|
|
"""
|
|
@_deprecate_positional_args
|
|
def __init__(self, *,
|
|
criterion="mse",
|
|
splitter="random",
|
|
max_depth=None,
|
|
min_samples_split=2,
|
|
min_samples_leaf=1,
|
|
min_weight_fraction_leaf=0.,
|
|
max_features="auto",
|
|
random_state=None,
|
|
min_impurity_decrease=0.,
|
|
min_impurity_split=None,
|
|
max_leaf_nodes=None,
|
|
ccp_alpha=0.0):
|
|
super().__init__(
|
|
criterion=criterion,
|
|
splitter=splitter,
|
|
max_depth=max_depth,
|
|
min_samples_split=min_samples_split,
|
|
min_samples_leaf=min_samples_leaf,
|
|
min_weight_fraction_leaf=min_weight_fraction_leaf,
|
|
max_features=max_features,
|
|
max_leaf_nodes=max_leaf_nodes,
|
|
min_impurity_decrease=min_impurity_decrease,
|
|
min_impurity_split=min_impurity_split,
|
|
random_state=random_state,
|
|
ccp_alpha=ccp_alpha)
|