From 6ff9f986bbbd0e635e0e6c1f938ca6af1d4d93f7 Mon Sep 17 00:00:00 2001 From: shu-kun-zhang Date: Thu, 3 Mar 2022 15:54:24 +0800 Subject: [PATCH] Add API Examples for Reliability Module --- .../concept_drift/concept_drift_check_images.py | 48 +++++++++++++++++----- .../concept_drift_check_time_series.py | 2 + .../model_fault_injection/fault_injection.py | 35 +++++++++++++--- 3 files changed, 70 insertions(+), 15 deletions(-) diff --git a/mindarmour/reliability/concept_drift/concept_drift_check_images.py b/mindarmour/reliability/concept_drift/concept_drift_check_images.py index ec0046b..9aebc99 100644 --- a/mindarmour/reliability/concept_drift/concept_drift_check_images.py +++ b/mindarmour/reliability/concept_drift/concept_drift_check_images.py @@ -13,18 +13,16 @@ # limitations under the License. # ============================================================================ +""" +Out-of-Distribution detection module for images. +""" import heapq import numpy as np -from mindspore import Tensor from sklearn.cluster import KMeans -from mindarmour.utils._check_param import check_param_type, check_param_in_range +from mindspore import Tensor from mindspore.train.summary.summary_record import _get_summary_tensor_data - - -""" -Out-of-Distribution detection for images. -""" +from mindarmour.utils._check_param import check_param_type, check_param_in_range class OodDetector: @@ -67,7 +65,7 @@ class OodDetector: Returns: - float, the optimal threshold. """ - pass + def ood_predict(self, threshold, ds_test): """ @@ -81,7 +79,6 @@ class OodDetector: Returns: - numpy.ndarray, the detection result. 0 means the data is not ood, 1 means the data is ood. """ - pass class OodDetectorFeatureCluster(OodDetector): @@ -90,6 +87,8 @@ class OodDetectorFeatureCluster(OodDetector): the testing data features and the clustering centers determines whether an image is an out-of-distribution(OOD) image or not. + For details, please check `Tutorial `_ + Args: model (Model):The training model. ds_train (numpy.ndarray): The training dataset. @@ -100,9 +99,38 @@ class OodDetectorFeatureCluster(OodDetector): layer (str): The name of the feature layer. layer (str) is represented by 'name[:Tensor]', where 'name' is given by users when training the model. Please see more details about how to name the model layer in 'README.md'. + + Examples: + >>> from mindspore import Model + >>> from mindspore.ops import TensorSummary + >>> import mindspore.ops.operations as P + >>> from mindarmour.reliability.concept_drift.concept_drift_check_images import OodDetectorFeatureCluster + >>> class Net(nn.Cell): + ... def __init__(self): + ... super(Net, self).__init__() + ... self._softmax = P.Softmax() + ... self._Dense = nn.Dense(10,10) + ... self._squeeze = P.Squeeze(1) + ... self._summary = TensorSummary() + ... def construct(self, inputs): + ... out = self._softmax(inputs) + ... out = self._Dense(out) + ... self._summary('output', out) + ... return self._squeeze(out) + >>> net = Net() + >>> model = Model(net) + >>> batch_size = 16 + >>> batches = 1 + >>> ds_train = np.random.randn(batches * batch_size, 1, 10).astype(np.float32) + >>> ds_eval = np.random.randn(batches * batch_size, 1, 10).astype(np.float32) + >>> detector = OodDetectorFeatureCluster(model, ds_train, n_cluster=10, layer='output[:Tensor]') + >>> num = int(len(ds_eval) / 2) + >>> ood_label = np.concatenate((np.zeros(num), np.ones(num)), axis=0) + >>> optimal_threshold = detector.get_optimal_threshold(ood_label, ds_eval) """ def __init__(self, model, ds_train, n_cluster, layer): + super(OodDetectorFeatureCluster, self).__init__(model, ds_train) self.model = model self.ds_train = check_param_type('ds_train', ds_train, np.ndarray) self.n_cluster = check_param_type('n_cluster', n_cluster, int) @@ -173,7 +201,7 @@ class OodDetectorFeatureCluster(OodDetector): threshold.append(threshold_change) acc = np.array(acc) threshold = np.array(threshold) - optimal_threshold = threshold[np.where(acc==np.max(acc))[0]][0] + optimal_threshold = threshold[np.where(acc == np.max(acc))[0]][0] return optimal_threshold def ood_predict(self, threshold, ds_test): diff --git a/mindarmour/reliability/concept_drift/concept_drift_check_time_series.py b/mindarmour/reliability/concept_drift/concept_drift_check_time_series.py index 61e553f..0b02e18 100644 --- a/mindarmour/reliability/concept_drift/concept_drift_check_time_series.py +++ b/mindarmour/reliability/concept_drift/concept_drift_check_time_series.py @@ -25,6 +25,7 @@ from mindarmour.utils._check_param import check_param_type, check_param_in_range class ConceptDriftCheckTimeSeries: r""" ConceptDriftCheckTimeSeries is used for example series distribution change detection. + For details, please check `Tutorial `_ Args: window_size(int): Size of a concept window, no less than 10. If given the input data, @@ -38,6 +39,7 @@ class ConceptDriftCheckTimeSeries: Default: False. Examples: + >>> from mindarmour import ConceptDriftCheckTimeSeries >>> concept = ConceptDriftCheckTimeSeries(window_size=100, rolling_window=10, ... step=10, threshold_index=1.5, need_label=False) >>> data_example = 5*np.random.rand(1000) diff --git a/mindarmour/reliability/model_fault_injection/fault_injection.py b/mindarmour/reliability/model_fault_injection/fault_injection.py index 9f7d4ec..d9a3b21 100644 --- a/mindarmour/reliability/model_fault_injection/fault_injection.py +++ b/mindarmour/reliability/model_fault_injection/fault_injection.py @@ -31,6 +31,7 @@ TAG = 'FaultInjector' class FaultInjector: """ Fault injection for deep neural networks and evaluate performance. + For details, please check `Tutorial `_ Args: model (Model): The model need to be evaluated. @@ -40,14 +41,38 @@ class FaultInjector: fi_size (list): The number of fault injection.It mean that how many values need to be injected. Examples: + >>> from mindspore import Model + >>> import mindspore.dataset as ds + >>> import mindspore.ops.operations as P + >>> from mindarmour.reliability.model_fault_injection.fault_injection import FaultInjector + >>> class Net(nn.Cell): + ... def __init__(self): + ... super(Net, self).__init__() + ... self._softmax = P.Softmax() + ... self._Dense = nn.Dense(10,10) + ... self._squeeze = P.Squeeze(1) + ... def construct(self, inputs): + ... out = self._softmax(inputs) + ... out = self._Dense(out) + ... return self._squeeze(out) + >>> def dataset_generator(): + ... batch_size = 16 + ... batches = 1 + ... data = np.random.randn(batches * batch_size,1,10).astype(np.float32) + ... label = np.random.randint(0,10, batches * batch_size).astype(np.int32) + ... for i in range(batches): + ... yield data[i*batch_size:(i+1)*batch_size],\ + ... label[i*batch_size:(i+1)*batch_size] >>> net = Net() >>> model = Model(net) - >>> ds_data, ds_label = create_data() - >>> fi_type = ['bitflips_random', 'zeros'] + >>> ds_eval = ds.GeneratorDataset(dataset_generator, ['image', 'label']) + >>> fi_type = ['bitflips_random', 'bitflips_designated', 'random', 'zeros', + 'nan', 'inf', 'anti_activation', 'precision_loss'] >>> fi_mode = ['single_layer', 'all_layer'] - >>> fi_size = [1, 2] - >>> fi = FaultInjector(model, fi_type=fi_type, fi_mode=fi_mode, fi_size=fi_size) - >>> fi.kick_off(ds_data, ds_label) + >>> fi_size = [1] + >>> fi = FaultInjector(model, ds_eval, fi_type, fi_mode, fi_size) + >>> fi.kick_off() + >>> fi.metrics() """ def __init__(self, model, fi_type=None, fi_mode=None, fi_size=None):