python如何去除异常值和缺失值的插值

python如何去除异常值和缺失值的插值

1.使用箱型法去除异常值:

import numpy as np import pandas as pd import matplotlib as plt import os data = pd.read_excel('try.xls', header=0) # print(data.shape) # print(data.head(10)) # print(data.describe()) neg_list = ['位移'] print("(1)数据的行数为:") R = data.shape[0] print(R) print("(2)小于或大于阈值的数据提取:") for item in neg_list: neg_item = data[item]<2000 print(item + '小于2000的有' + str(neg_item.sum()) + '个') print("(3)异常值的个数:") for item in neg_list: iqr = data[item].quantile(0.75) - data[item].quantile(0.25) q_abnormal_L = data[item] < data[item].quantile(0.25) - 1.5 * iqr q_abnormal_U = data[item] > data[item].quantile(0.75) + 1.5 * iqr print(item + '中有' + str(q_abnormal_L.sum() + q_abnormal_U.sum()) + '个异常值') print("(4)箱型图确定上下限:") for item in neg_list: iqr = data[item].quantile(0.75) - data[item].quantile(0.25) Too_small = data[item].quantile(0.25) - 1.5 * iqr Too_big = data[item].quantile(0.25) + 1.5 * iqr print("下限是", Too_small) print("上限是", Too_big ) print("(5)所有数据为:") a = [] for i in neg_list: a.append(data[i]) print(a) print("(6)所有正常数据:") b = [] j = 0 while j < R: if (a[0][j] > Too_small): if (a[0][j] < Too_big): b.append(a[0][j]) j += 1 print(b) print("(7)所有异常数据:") c = [] i = 0 while i < R: if (a[0][i] < Too_small or a[0][i] > Too_big): c.append(a[0][i]) a[0][i] = None i +=1 print(c) print("(8)把所有异常数据删除后:") print(a) print("(9)所有数据处理后输出:") d = [] k = 0 while k < R: d.append(a[0][k]) k +=1 print(d) df = pd.DataFrame(d,columns= ['位移']) df.to_excel("try_result.xls")

2.拉格朗日插值:

import os import pandas as pd import numpy as np from scipy.interpolate import lagrange import matplotlib.pyplot as plt plt.rcParams['font.sans-serif']=['SimHei'] #用来正常显示中文标签 plt.rcParams['axes.unicode_minus']=False #用来正常显示负号 # 数据的读取 data = pd.read_excel('try.xls', header=0) neg_list = ['位移'] # 数据的行数 R = data.shape[0] # 异常数据的个数 for item in neg_list: iqr = data[item].quantile(0.75) - data[item].quantile(0.25) q_abnormal_L = data[item] < data[item].quantile(0.25) - 1.5 * iqr q_abnormal_U = data[item] > data[item].quantile(0.75) + 1.5 * iqr # print(item + '中有' + str(q_abnormal_L.sum() + q_abnormal_U.sum()) + '个异常值') # 确定数据上限和下限 for item in neg_list: iqr = data[item].quantile(0.75) - data[item].quantile(0.25) Too_small = data[item].quantile(0.25) - 1.5 * iqr Too_big = data[item].quantile(0.25) + 1.5 * iqr data[u'位移'][(data[u'位移']<Too_small) | (data[u'位移']>Too_big)] = None #过滤异常值,将其变为空值 #s为列向量,n为被插值位置,k为取前后的数据个数 def ployinter(s,n,k=5): y = s[list(range(n-k,n)) + list(range(n+1,n+1+k))] y = y[y.notnull()] #剔除空值 return lagrange(y.index,list(y))(n) #逐个元素判断是否需要插值 for i in data.columns: for j in range(len(data)): if(data[i].isnull())[j]: data[i][j] = ployinter(data[i],j) # print(data[u'位移']) # 输出拉格朗日插值后的数据 data.to_excel("try_result.xls") # 把表格列数据调整为arr,arr为修改后的数据 print("拉格朗日插值后的数据:") d = [] k = 0 while k < R: d.append(data[u'位移'][k]) k +=1 # print(d) arr = np.array(d) print(arr) # 输出图像 x = np.arange(len(d)) plt.plot(x,d,'b-',label="one", marker='*',markersize=4,linewidth=1) # b代表blue颜色 -代表直线 plt.title('位移曲线') plt.legend(loc='upper left',bbox_to_anchor=(1.0,1.0)) # 直接更改X轴坐标数 # plt.xticks((0,1,2,3,4,5,6,7,8),('0', '1', '2', '3', '4', '5', '6', '7', '8')) plt.xlabel('时间/h') plt.ylabel('位移/mm') #plt.grid(x1) plt.show()

3.数据拟合:

import numpy as np import pandas as pd import matplotlib.pyplot as plt from scipy.optimize import leastsq def Fun(p, x): # 定义拟合函数形式 a1, a2, a3 , a4 = p return a1 * x ** 3 + a2 * x ** 2 + a3 * x + a4 def error(p, x, y): # 拟合残差 return Fun(p, x) - y def main(): x = np.linspace(1, 31, 31) # 创建时间序列 data = pd.read_excel('try.xls', header=0) y = data[u'位移'] p0 = [0.1, -0.01, 100, 1000] # 拟合的初始参数设置 para = leastsq(error, p0, args=(x, y)) # 进行拟合 y_fitted = Fun(para[0], x) # 画出拟合后的曲线 plt.figure plt.plot(x, y, 'r', label='Original curve') plt.plot(x, y_fitted, '-b', label='Fitted curve') plt.legend() plt.show() print(para[0]) if __name__ == '__main__': main()

4.输出图像:

import pandas as pd import numpy as np import matplotlib.pyplot as plt plt.rcParams['font.sans-serif']=['SimHei'] #用来正常显示中文标签 plt.rcParams['axes.unicode_minus']=False #用来正常显示负号 jiaodu = ['0', '15', '30', '15', '60', '75', '90', '105', '120'] x = range(len(jiaodu)) y = [85.6801, 7.64586, 86.0956, 159.229, 179.534, 163.238, 96.4436, 10.1619, 90.9262,] #plt.figure(figsize=(10, 6)) plt.plot(x,y,'b-',label="1", marker='*',markersize=7,linewidth=3) # b代表blue颜色 -代表直线 plt.title('各个区域亮度变化') plt.legend(loc='upper left',bbox_to_anchor=(1.0,1.0)) plt.xticks((0,1,2,3,4,5,6,7,8),('0', '15', '30', '15', '60', '75', '90', '105', '120')) plt.xlabel('角度') plt.ylabel('亮度') #plt.grid(x1) plt.show()

到此这篇关于python如何去除异常值和缺失值的插值的文章就介绍到这了,更多相关python去除异常值和缺失值内容请搜索易知道(ezd.cc)以前的文章或继续浏览下面的相关文章希望大家以后多多支持易知道(ezd.cc)!

推荐阅读