[Day27]玉宇江才千古愁-鸡蛋里挑钢筋水泥，异常排除

今天我们要来模拟基於柴比雪夫不等式的异常值检测，首先我们先用NumPy产生一条随机乱数

import pandas as pd
import numpy as np

if __name__ == "__main__":
    # 随机产生数列
    data = np.random.rand(25) * 25
    data = data.astype("int32")    
    print(data)
    # [23 19 13 11  6  9 11 13 14 23 10 21 24  2 11 20  6 16 12 22 12  1  3  6  6]

我们来看一下图

import pandas as pd
import numpy as np
from matplotlib import pyplot as plt

if __name__ == "__main__":
    # 随机产生数列
    data = np.random.rand(25) * 25
    data = data.astype("int32")    
    print(data)
    # [23 19 13 11  6  9 11 13 14 23 10 21 24  2 11 20  6 16 12 22 12  1  3  6  6]
    
    plt.plot(data)
    plt.show()

定义基於柴比雪夫不等式的异常值检测副程序，参数有两个，一个是Pandas的Series，另一个是高於数列平均值的标准差倍数

import pandas as pd
import numpy as np
from matplotlib import pyplot as plt

def chebyshev(data, n):
    avg = data.mean()
    std = n * data.std()
    #将平均值以下n倍标准差的资料去除
    minmask = data < avg - std
    data[minmask] = np.nan
    #将平均值以上n倍标准差的资料去除
    maxmask = data > avg + std
    data[maxmask] = np.nan
    return data


if __name__ == "__main__":
    # 随机产生数列
    data = np.random.rand(25) * 25
    data = data.astype("int32")    
    print(data)
    # [23 19 13 11  6  9 11 13 14 23 10 21 24  2 11 20  6 16 12 22 12  1  3  6  6]
    
    #建立Series
    data = pd.Series(data)

    #将平均值以上及以下1倍标准差的资料去除
    data = chebyshev(data, 1)

    print(data)
    """
    0      NaN
    1     13.0
    2      7.0
    3     11.0
    4     12.0
    5     11.0
    6     17.0
    7      5.0
    8     14.0
    9     15.0
    10     NaN
    11     NaN
    12     NaN
    13    12.0
    14     NaN
    15    15.0
    16    15.0
    17     NaN
    18     8.0
    19    17.0
    20     NaN
    21     NaN
    22    17.0
    23    16.0
    24     NaN
    dtype: float64
    """