数据预处理
数据集
本数据集来自AI Challenger2018农作物检测比赛所用数据集，官方前后总共发布了两次比赛数据集．第一次数据集有很多图像存在标签交叉的情况(即相同的图像存在不同的标签，特别是病害程度一般和严重存在交叉情况)，训练图像总数为32768张，验证图像总数为4992张，测试集A图像总数为4959张；因为比赛社区很多人反馈数据交叉的现象，官方在比赛的又发布了第二次比赛数据，第二次发布的数据是对存在标签交叉部分的进行更新，训练图像总数为31718张，验证图像总数为4540张。
用于训练的数据集为第二个版本。
数据可视化
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29import os
import pandas as pd
import matplotlib.pyplot as plt
train_path = 'data/data17627/AgriculturalDisease_trainingset/images'
valid_path = "data/data17627/AgriculturalDisease_validationset/images"
train_list = os.listdir(train_path)
valid_list = os.listdir(valid_path)
print('AgriculturalDisease_trainingset:',len(train_list))
print('AgriculturalDisease_validationset:',len(valid_list))
train_json_path = 'data/data17627/AgriculturalDisease_trainingset/AgriculturalDisease_train_annotations.json'
valid_json_path = 'data/data17627/AgriculturalDisease_validationset/AgriculturalDisease_validation_annotations.json'
# train_json_path = 'work/data/train_no_44_45_9_24.json'
# valid_json_path = 'work/data/valid_no_44_45_9_24.json'
with open(train_json_path,encoding='utf-8') as datafile1:
train_label_df = pd.read_json(datafile1,orient='records')
with open(valid_json_path,encoding='utf-8') as datafile2:
valid_label_df = pd.read_json(datafile2,orient='records')
# 查看有没有缺失值
print(train_label_df.isnull().sum())
print(valid_label_df.isnull().sum())
plt.figure(figsize=(12, 8))
plt.subplot(2, 1, 1)
train_label_df['disease_class'].value_counts().plot(kind='bar',grid=True,rot = 45,alpha = 0.8,title = 'trainset_label_distribute',legend = True )
plt.subplot(2, 1, 2)
valid_label_df['disease_class'].value_counts().plot(kind='bar',grid=True,rot = 45,alpha = 0.8,title = 'validationset_label_distribute')
# 查看各类样本总数
train_label_df['disease_class'].value_counts().sort_index() 进行可视化后，得到结果
其中横坐标为作物的病种类，纵坐标为数量。
数据剔除
通过可视化我们发现，44和45类型的图片数量很少，接近于没有，为了让结果准确率更高，我们可以将这两类剔除。
原理很简单，44之前的disease_label不用动，45之后的disease_label减2即可。
1
2
3
4
5
6with codecs.open(train_file) as flist:
lines = json.load(flist)
new_list = [i for i in lines if i['disease_class'] not in [44,45]]
for i in range(len(new_list)):
if(new_list[i]['disease_class'] >43 ):
new_list[i]['disease_class'] -= 2对图片的操作
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122def resize_img(img, target_size):
"""
强制缩放图片
:param img:
:param target_size:
:return:
"""
target_size = input_size
img = img.resize((target_size[1], target_size[2]), Image.BILINEAR)
return img
def random_crop(img, scale=[0.08, 1.0], ratio=[3. / 4., 4. / 3.]):
aspect_ratio = math.sqrt(np.random.uniform(*ratio))
w = 1. * aspect_ratio
h = 1. / aspect_ratio
bound = min((float(img.size[0]) / img.size[1]) / (w**2),
(float(img.size[1]) / img.size[0]) / (h**2))
scale_max = min(scale[1], bound)
scale_min = min(scale[0], bound)
target_area = img.size[0] * img.size[1] * np.random.uniform(scale_min,
scale_max)
target_size = math.sqrt(target_area)
w = int(target_size * w)
h = int(target_size * h)
i = np.random.randint(0, img.size[0] - w + 1)
j = np.random.randint(0, img.size[1] - h + 1)
img = img.crop((i, j, i + w, j + h))
img = img.resize((train_parameters['input_size'][1], train_parameters['input_size'][2]), Image.BILINEAR)
return img
def rotate_image(img):
"""
图像增强，增加随机旋转角度
"""
angle = np.random.randint(-14, 15)
img = img.rotate(angle)
return img
def random_brightness(img):
"""
图像增强，亮度调整
:param img:
:return:
"""
prob = np.random.uniform(0, 1)
if prob < train_parameters['image_enhance_strategy']['brightness_prob']:
brightness_delta = train_parameters['image_enhance_strategy']['brightness_delta']
delta = np.random.uniform(-brightness_delta, brightness_delta) + 1
img = ImageEnhance.Brightness(img).enhance(delta)
return img
def random_contrast(img):
"""
图像增强，对比度调整
:param img:
:return:
"""
prob = np.random.uniform(0, 1)
if prob < train_parameters['image_enhance_strategy']['contrast_prob']:
contrast_delta = train_parameters['image_enhance_strategy']['contrast_delta']
delta = np.random.uniform(-contrast_delta, contrast_delta) + 1
img = ImageEnhance.Contrast(img).enhance(delta)
return img
def random_saturation(img):
"""
图像增强，饱和度调整
:param img:
:return:
"""
prob = np.random.uniform(0, 1)
if prob < train_parameters['image_enhance_strategy']['saturation_prob']:
saturation_delta = train_parameters['image_enhance_strategy']['saturation_delta']
delta = np.random.uniform(-saturation_delta, saturation_delta) + 1
img = ImageEnhance.Color(img).enhance(delta)
return img
def random_hue(img):
"""
图像增强，色度调整
:param img:
:return:
"""
prob = np.random.uniform(0, 1)
if prob < train_parameters['image_enhance_strategy']['hue_prob']:
hue_delta = train_parameters['image_enhance_strategy']['hue_delta']
delta = np.random.uniform(-hue_delta, hue_delta)
img_hsv = np.array(img.convert('HSV'))
img_hsv[:, :, 0] = img_hsv[:, :, 0] + delta
img = Image.fromarray(img_hsv, mode='HSV').convert('RGB')
return img
def distort_color(img):
"""
概率的图像增强
:param img:
:return:
"""
prob = np.random.uniform(0, 1)
# Apply different distort order
if prob < 0.35:
img = random_brightness(img)
img = random_contrast(img)
img = random_saturation(img)
img = random_hue(img)
elif prob < 0.7:
img = random_brightness(img)
img = random_saturation(img)
img = random_hue(img)
img = random_contrast(img)
return img 一些常规的对图象进行的预处理，为了之后的训练得到较好的效果。