Source code for mlens.index.subsemble

"""ML-ENSEMBLE

:author: Sebastian Flennerhag
:copyright: 2017
:license: MIT

Partitioning estimators
"""
from __future__ import division

import numpy as np

from ._checks import check_subsample_index
from .base import BaseIndex, partition, make_tuple, prune_train


[docs]class SubsetIndex(BaseIndex): r"""Subsample index generator. Generates cross-validation folds according used to create ``J`` partitions of the data and ``v`` folds on each partition according to as per [#]_: 1. Split ``X`` into ``J`` partitions 2. For each partition: (a) For each fold ``v``, create train index of all idx not in ``v`` (b) Concatenate all the fold ``v`` indices into a test index for fold ``v`` that spans all partitions Setting ``J = 1`` is equivalent to the :class:`FullIndexer`, which returns standard K-Fold train and test set indices. See Also -------- :class:`FoldIndex`, :class:`BlendIndex`, :class:`Subsemble` References ---------- .. [#] Sapp, S., van der Laan, M. J., & Canny, J. (2014). Subsemble: an ensemble method for combining subset-specific algorithm fits. Journal of Applied Statistics, 41(6), 1247-1259. http://doi.org/10.1080/02664763.2013.864263 Parameters ---------- partitions : int, list (default = 2) Number of partitions to split data in. If ``partitions=1``, :class:`SubsetIndex` reduces to standard K-Fold. folds : int (default = 2) Number of splits to create in each partition. ``folds`` can not be 1 if ``n_partition > 1``. Note that if ``folds = 1``, both the train and test set will index the full data. X : array-like of shape [n_samples,] , optional the training set to partition. The training label array is also, accepted, as only the first dimension is used. If ``X`` is not passed at instantiation, the ``fit`` method must be called before ``generate``, or ``X`` must be passed as an argument of ``generate``. raise_on_exception : bool (default = True) whether to warn on suspicious slices or raise an error. Examples -------- >>> import numpy as np >>> from mlens.index import SubsetIndex >>> X = np.arange(10) >>> idx = SubsetIndex(3, X=X) >>> >>> print('Expected partitions of X:') >>> print('J = 1: {!r}'.format(X[0:4])) >>> print('J = 2: {!r}'.format(X[4:7])) >>> print('J = 3: {!r}'.format(X[7:10])) >>> print('SubsetIndexer partitions:') >>> for i, part in enumerate(idx.partition(as_array=True)): ... print('J = {}: {!r}'.format(i + 1, part)) >>> print('SubsetIndexer folds on partitions:') >>> for i, (tri, tei) in enumerate(idx.generate()): ... fold = i % 2 + 1 ... part = i // 2 + 1 ... train = np.hstack([np.arange(t0, t1) for t0, t1 in tri]) ... test = np.hstack([np.arange(t0, t1) for t0, t1 in tei]) >>> print("J = %i | f = %i | " ... "train: %15r | test: %r" % (part, fold, train, test)) Expected partitions of X: J = 1: array([0, 1, 2, 3]) J = 2: array([4, 5, 6]) J = 3: array([7, 8, 9]) SubsetIndexer partitions: J = 1: array([0, 1, 2, 3]) J = 2: array([4, 5, 6]) J = 3: array([7, 8, 9]) SubsetIndexer folds on partitions: J = 1 | f = 1 | train: array([2, 3]) | test: array([0, 1, 4, 5, 7, 8]) J = 1 | f = 2 | train: array([0, 1]) | test: array([2, 3, 6, 9]) J = 2 | f = 1 | train: array([6]) | test: array([0, 1, 4, 5, 7, 8]) J = 2 | f = 2 | train: array([4, 5]) | test: array([2, 3, 6, 9]) J = 3 | f = 1 | train: array([9]) | test: array([0, 1, 4, 5, 7, 8]) J = 3 | f = 2 | train: array([7, 8]) | test: array([2, 3, 6, 9]) """ def __init__(self, partitions=2, folds=2, X=None, raise_on_exception=True): super(SubsetIndex, self).__init__() self.partitions = partitions self.folds = folds self.raise_on_exception = raise_on_exception if X is not None: self.fit(X)
[docs] def fit(self, X, y=None, job=None): """Method for storing array data. Parameters ---------- X : array-like of shape [n_samples, optional] array to _collect dimension data from. y : None for compatibility job : None for compatibility Returns ------- instance : indexer with stores sample size data. """ n = X.shape[0] check_subsample_index(n, self.partitions, self.folds, self.raise_on_exception) self.n_samples = self.n_test_samples = n self.__fitted__ = True return self
[docs] def partition(self, X=None, as_array=False): """Get partition indices for training full subset estimators. Returns the index range for each partition of X. Parameters ---------- X : array-like of shape [n_samples,] , optional the training set to partition. The training label array is also, accepted, as only the first dimension is used. If ``X`` is not passed at instantiation, the ``fit`` method must be called before ``generate``, or ``X`` must be passed as an argument of ``generate``. as_array : bool (default = False) whether to return partition as an index array. Otherwise tuples of ``(start, stop)`` indices are returned. """ if not self.__fitted__: if X is None: raise AttributeError("No array provided to indexer. Either " "pass an array to the 'generate' method, " "or call the 'fit' method first or " "initiate the instance with an array X " "as argument.") else: # Need to call fit to continue self.fit(X) # Return the partition indices. parts = partition(self.n_samples, self.partitions) last = 0 for size in parts: idx = last, last + size if as_array: idx = self._build_range(idx) yield idx last += size
def _build_test_sets(self): """Build global test folds for each split of every partition. This method runs through each partition and fold to register all the test set indices across partitions. For each test fold ``i``, the test set indices are thus the union of fold ``i`` indices across all ``J`` partitions. """ partitions = self.partitions n_samples = self.n_samples folds = self.folds # --- Create global test set folds --- # In each partition, the test set spans all partitions # Hence we must run through all partitions once first to # register # the global test set fold for each split of the folds # Partition sizes p_len = partition(n_samples, partitions) # Since the splitting is sequential and deterministic, # we build a # list of global test set indices. Hence, for any split, the # test index will be a tuple of indexes of test folds from each # partition. By concatenating these slices, the full test fold # is constructed. tei = [[] for _ in range(folds)] p_last = 0 for p_size in p_len: p_start, p_stop = p_last, p_last + p_size t_len = partition(p_stop - p_start, folds) # Append the partition's test fold indices to the # global directory for that fold number. t_last = p_start for i, t_size in enumerate(t_len): t_start, t_stop = t_last, t_last + t_size tei[i] += [(t_start, t_stop)] t_last += t_size p_last += p_size return tei def _gen_indices(self): """Create generator for subsample. Generate indices of training set and test set for - each partition - each fold in the partition Note that the test index return is *global*, i.e. it contains the test indices of that fold across partitions. See Examples for further details. """ partitions = self.partitions n_samples = self.n_samples folds = self.folds T = self._build_test_sets() # For each partition, for each fold, get the global test fold # from T and index the partition samples not in T as train set p_len = partition(n_samples, partitions) p_last = 0 for p_size in p_len: p_start, p_stop = p_last, p_last + p_size t_len = partition(p_stop - p_start, folds) t_last = p_start for i, t_size in enumerate(t_len): t_start, t_stop = t_last, t_last + t_size # Get global test set indices tei = T[i] # Construct train set tri_start_below, tri_stop_below = p_start, t_start tri_start_above, tri_stop_above = t_stop, p_stop tri = prune_train(tri_start_below, tri_stop_below, tri_start_above, tri_stop_above) yield tri, tei t_last += t_size p_last += p_size
[docs]class ClusteredSubsetIndex(BaseIndex): """Clustered Subsample index generator. Generates cross-validation folds according used to create ``J`` partitions of the data and ``v`` folds on each partition according to as per [#]_: 1. Split ``X`` into ``J`` partitions 2. For each partition: (a) For each fold ``v``, create train index of all idx not in ``v`` (b) Concatenate all the fold ``v`` indices into a test index for fold ``v`` that spans all partitions Setting ``J = 1`` is equivalent to the :class:`FullIndexer`, which returns standard K-Fold train and test set indices. :class:`ClusteredSubsetIndex` uses a user-provided estimator to partition the data, in contrast to the :class:`SubsetIndex` generator, which partitions data into randomly into equal sizes. See Also -------- :class:`FoldIndex`, :class:`BlendIndex`, :class:`SubsetIndex` References ---------- .. [#] Sapp, S., van der Laan, M. J., & Canny, J. (2014). Subsemble: an ensemble method for combining subset-specific algorithm fits. Journal of Applied Statistics, 41(6), 1247-1259. http://doi.org/10.1080/02664763.2013.864263 Parameters ---------- partition_estimator : instance Estimator to use for clustering. partitions : int Number of partitions the estimator will create. folds : int (default = 2) Number of folds to create in each partition. ``folds`` can not be 1 if ``n_partition > 1``. Note that if ``folds = 1``, both the train and test set will index the full data. fit_estimator : bool (default = True) whether to fit the estimator separately before generating labels. attr : str (default = 'predict') the attribute to use for generating cluster membership labels. X : array-like of shape [n_samples,] , optional the training set to partition. The training label array is also, accepted, as only the first dimension is used. If ``X`` is not passed at instantiation, the ``fit`` method must be called before ``generate``, or ``X`` must be passed as an argument of ``generate``. raise_on_exception : bool (default = True) whether to warn on suspicious slices or raise an error. Examples -------- >>> import numpy as np >>> from sklearn.cluster import KMeans >>> from mlens.index import ClusteredSubsetIndex >>> >>> km = KMeans(3, random_state=0) >>> X = np.arange(12).reshape(-1, 1); np.random.shuffle(X) >>> print("Data: {}".format(X.ravel())) >>> >>> s = ClusteredSubsetIndex(km) >>> s.fit(X) >>> >>> P = s.partition_estimator.predict(X) >>> print("cluster labels: {}".format(P)) >>> >>> for j, i in enumerate(s.partition(as_array=True)): ... print("partition ({}) index: {}, cluster labels: {}".format(i, j + 1, P[i])) >>> >>> for i in s.generate(as_array=True): ... print("train fold index: {}, cluster labels: {}".format(i[0], P[i[0]])) Data: [ 8 7 5 2 4 10 11 1 3 6 9 0] cluster labels: [0 2 2 1 2 0 0 1 1 2 0 1] partition (1) index: [ 0 5 6 10], cluster labels: [0 0 0 0] partition (2) index: [ 3 7 8 11], cluster labels: [1 1 1 1] partition (3) index: [1 2 4 9], cluster labels: [2 2 2 2] train fold index: [0 3 5], cluster labels: [0 0 0] train fold index: [ 6 10], cluster labels: [0 0] train fold index: [2 7], cluster labels: [1 1] train fold index: [ 9 11], cluster labels: [1 1] train fold index: [1 4], cluster labels: [2 2] train fold index: [8], cluster labels: [2] """ def __init__(self, partition_estimator, partitions=2, folds=2, X=None, y=None, fit_estimator=True, attr='predict', partition_on='X', raise_on_exception=True): super(ClusteredSubsetIndex, self).__init__() self.partition_estimator = partition_estimator self.fit_estimator = fit_estimator self.attr = attr self.partition_on = partition_on self.partitions = partitions self.folds = folds self.raise_on_exception = raise_on_exception self._clusters_ = None if X is not None: self.fit(X, y)
[docs] def fit(self, X, y=None, job='fit'): """Method for storing array data. Parameters ---------- X : array-like of shape [n_samples, n_features] input array. y : array-like of shape [n_samples, ] labels. job : str, ['fit', 'predict'] (default='fit') type of estimation job. If 'fit', the indexer will be fitted, which involves fitting the estimator. Otherwise, the indexer will not be fitted (since it is not used for prediction). Returns ------- instance : indexer with stores sample size data. """ n = X.shape[0] self.n_samples = self.n_test_samples = n if 'fit' in job: # Only generate new clusters if fitting an ensemble if self.fit_estimator: try: self.partition_estimator.fit(X, y) except TypeError: # Safeguard against estimators that do not accept y. self.partition_estimator.fit(X) # Indexers are assumed to need fitting once, so we need to # generate cluster predictions during the fit call. To minimize # memory consumption, store cluster indexes as list of tuples self._clusters_ = self._get_partitions(X, y) self.__fitted__ = True return self
[docs] def partition(self, X=None, y=None, as_array=False): """Get partition indices for training full subset estimators. Returns the index range for each partition of X. Parameters ---------- X : array-like of shape [n_samples, n_features] , optional the set to partition. The training label array is also, accepted, as only the first dimension is used. If ``X`` is not passed at instantiation, the ``fit`` method must be called before ``generate``, or ``X`` must be passed as an argument of ``generate``. y : array-like of shape [n_samples,], optional the labels of the set to partition. as_array : bool (default = False) whether to return partition as an index array. Otherwise tuples of ``(start, stop)`` indices are returned. """ if X is not None: self.fit(X, y) return self._partition_generator(as_array)
def _partition_generator(self, as_array): """Generator for partitions. Parameters ---------- as_array : bool: whether to return partition indexes as a list of index tuples, or as an array. """ for cluster_index in self._clusters_: if as_array: yield self._build_range(cluster_index) else: yield cluster_index def _get_partitions(self, X, y=None): """Get clustered partition indices from estimator. Returns the index range for each partition of X. See :func:`partition` for further details. """ n_samples = X.shape[0] f = getattr(self.partition_estimator, self.attr) if self.partition_on == 'X': cluster_ids = f(X) elif self.partition_on == 'y': cluster_ids = f(y) else: cluster_ids = f(X, y) clusters = np.unique(cluster_ids) self.partitions = len(clusters) # Condense the cluster index array into a list of tuples out = list() # list of cluster indexes index = np.arange(n_samples) for c in clusters: cluster_index = index[cluster_ids == c] cluster_index_tup = make_tuple(cluster_index) out.append(cluster_index_tup) return out def _gen_indices(self): """Generator for clustered subsample. Generate indices of training set and test set for - each partition - each fold in the partition Note that the test index return is *global*, i.e. it contains the test indices of that fold across partitions. """ n_samples = self.n_samples folds = self.folds I = np.arange(n_samples) for prt in self._partition_generator(as_array=True): t_len = partition(prt.shape[0], folds) t_last = 0 for t_size in t_len: t_start, t_stop = t_last, t_last + t_size tri = prt[t_start:t_stop] # Create test set by iterating over the index range tei = np.asarray([i for i in I if i not in tri]) # Condense indexes to list of tuples tri = make_tuple(tri) tei = make_tuple(tei) yield tri, tei t_last += t_size