├── .DS_Store
├── MalariaCellImages_FlyAI_baseline
├── .flyai_darwin
├── .flyai_linux
├── README.md
├── app.yaml
├── flyai
├── flyai.exe
├── main.py
├── model.py
├── net.py
├── path.py
├── predict.py
├── processor.py
├── transformation.py
├── 使用jupyter调试.ipynb
├── 常见问题.html
└── 第一次使用请读我.html
├── MalariaCellImages_FlyAI_resnet18
├── .flyai_darwin
├── .flyai_linux
├── .idea
│ ├── MalariaCellImages_FlyAI.iml
│ ├── inspectionProfiles
│ │ └── profiles_settings.xml
│ ├── libraries
│ │ └── Google_App_Engine_SDK.xml
│ ├── misc.xml
│ ├── modules.xml
│ └── workspace.xml
├── README.md
├── app.yaml
├── flyai
├── flyai.exe
├── main.py
├── model.py
├── net.py
├── path.py
├── predict.py
├── processor.py
├── transformation.py
├── 使用jupyter调试.ipynb
├── 常见问题.html
└── 第一次使用请读我.html
├── MalariaCellImages_FlyAI_resnet18_2
├── README.md
├── app.yaml
├── flyai
├── flyai.exe
├── main.py
├── model.py
├── net.py
├── path.py
├── predict.py
├── processor.py
├── transformation.py
├── 使用jupyter调试.ipynb
├── 常见问题.html
└── 第一次使用请读我.html
├── MalariaCellImages_FlyAI_resnet50
├── README.md
├── app.yaml
├── flyai
├── flyai.exe
├── main.py
├── model.py
├── net.py
├── path.py
├── predict.py
├── processor.py
├── transformation.py
├── 使用jupyter调试.ipynb
├── 常见问题.html
└── 第一次使用请读我.html
├── PIC
├── baseline.png
├── change.png
├── original.png
├── resnet18-94.56.png
└── resnet50_87.83.png
├── pic
└── resnet18-94.56
├── README.md
├── app.yaml
├── flyai
├── flyai.exe
├── main.py
├── model.py
├── net.py
├── path.py
├── predict.py
├── processor.py
├── transformation.py
├── 使用jupyter调试.ipynb
├── 常见问题.html
└── 第一次使用请读我.html
/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zwkkk/pytorch-project-exercise/7f3a8930d82e7143afbea022f778ba52a8bfa662/.DS_Store
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_baseline/.flyai_darwin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zwkkk/pytorch-project-exercise/7f3a8930d82e7143afbea022f778ba52a8bfa662/MalariaCellImages_FlyAI_baseline/.flyai_darwin
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_baseline/.flyai_linux:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zwkkk/pytorch-project-exercise/7f3a8930d82e7143afbea022f778ba52a8bfa662/MalariaCellImages_FlyAI_baseline/.flyai_linux
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_baseline/README.md:
--------------------------------------------------------------------------------
1 | 此文件为官方所给baseline
2 | 评分34.22分
3 |
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_baseline/app.yaml:
--------------------------------------------------------------------------------
1 | ---
2 | name: malaria Cell Images
3 | author: M
4 | description: malaria Cell Images Classification
5 | topic: malaria Cell Images Classification
6 | algorithm: CNN
7 | language: python3
8 | virtualenv: flyai_env/bin/
9 |
10 | framework: PyTorch
11 |
12 | data:
13 | id: MalariaCellImages
14 | name: malaria Cell Images
15 | category: image
16 |
17 |
18 | model:
19 | processor: Processor
20 | input_x: input_x
21 | input_y: input_y
22 | output_y: output_y
23 | input:
24 | columns:
25 | - name: image_path
26 | type: string
27 | to_type: float
28 | to_shape: [-1, 1]
29 | output:
30 | columns:
31 | - name: label
32 | type: int
33 | to_type: float
34 | to_shape: [-1,7]
35 |
36 |
37 | evaluate:
38 | score: torch_accuracy
39 |
40 | servers:
41 | - id: flyai
42 | url: https://flyai.com
43 | ...
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_baseline/flyai:
--------------------------------------------------------------------------------
1 | unameOut="$(uname -s)"
2 | case "${unameOut}" in
3 | Linux*) machine=Linux;;
4 | Darwin*) machine=Mac;;
5 | CYGWIN*) machine=Cygwin;;
6 | MINGW*) machine=MinGw;;
7 | *) machine="UNKNOWN:${unameOut}"
8 | esac
9 |
10 |
11 | if [ $machine = "Mac" ]; then
12 | chmod +x ./.flyai_darwin
13 | ./.flyai_darwin "$@"
14 | fi
15 |
16 | if [ $machine = "Linux" ]; then
17 | chmod +x ./.flyai_linux
18 | ./.flyai_linux "$@"
19 | fi
20 |
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_baseline/flyai.exe:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zwkkk/pytorch-project-exercise/7f3a8930d82e7143afbea022f778ba52a8bfa662/MalariaCellImages_FlyAI_baseline/flyai.exe
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_baseline/main.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import argparse
3 | import torch
4 | import torch.nn as nn
5 | from flyai.dataset import Dataset
6 | from torch.optim import Adam
7 |
8 | from model import Model
9 | from net import Net
10 | from path import MODEL_PATH
11 |
12 | # 数据获取辅助类
13 | dataset = Dataset()
14 |
15 | # 模型操作辅助类
16 | model = Model(dataset)
17 |
18 | # 超参
19 | parser = argparse.ArgumentParser()
20 | parser.add_argument("-e", "--EPOCHS", default=100, type=int, help="train epochs")
21 | parser.add_argument("-b", "--BATCH", default=24, type=int, help="batch size")
22 | args = parser.parse_args()
23 |
24 | # 判断gpu是否可用
25 | if torch.cuda.is_available():
26 | device = 'cuda'
27 | else:
28 | device = 'cpu'
29 | # device = 'cpu'
30 | device = torch.device(device)
31 |
32 |
33 | def eval(model, x_test, y_test):
34 | cnn.eval()
35 | batch_eval = model.batch_iter(x_test, y_test)
36 | total_acc = 0.0
37 | data_len = len(x_test)
38 | for x_batch, y_batch in batch_eval:
39 | batch_len = len(x_batch)
40 | outputs = cnn(x_batch)
41 | _, prediction = torch.max(outputs.data, 1)
42 | correct = (prediction == y_batch).sum().item()
43 | acc = correct / batch_len
44 | total_acc += acc * batch_len
45 | return total_acc / data_len
46 |
47 |
48 | cnn = Net().to(device)
49 | optimizer = Adam(cnn.parameters(), lr=0.001, betas=(0.9, 0.999)) # 选用AdamOptimizer
50 | loss_fn = nn.CrossEntropyLoss() # 定义损失函数
51 |
52 | # 训练并评估模型
53 |
54 | data = Dataset()
55 | model = Model(data)
56 |
57 | best_accuracy = 0
58 | for i in range(args.EPOCHS):
59 | cnn.train()
60 | x_train, y_train, x_test, y_test = data.next_batch(args.BATCH) # 读取数据
61 |
62 | x_train = torch.from_numpy(x_train)
63 | y_train = torch.from_numpy(y_train)
64 | x_train = x_train.float().to(device)
65 | y_train = y_train.long().to(device)
66 |
67 | x_test = torch.from_numpy(x_test)
68 | y_test = torch.from_numpy(y_test)
69 | x_test = x_test.float().to(device)
70 | y_test = y_test.long().to(device)
71 |
72 | outputs = cnn(x_train)
73 | _, prediction = torch.max(outputs.data, 1)
74 |
75 | optimizer.zero_grad()
76 | # print(x_train.shape,outputs.shape,y_train.shape)
77 | loss = loss_fn(outputs, y_train)
78 | loss.backward()
79 | optimizer.step()
80 | print(loss.detach())
81 | # 若测试准确率高于当前最高准确率，则保存模型
82 | train_accuracy = eval(model, x_test, y_test)
83 | if train_accuracy > best_accuracy:
84 | best_accuracy = train_accuracy
85 | model.save_model(cnn, MODEL_PATH, overwrite=True)
86 | print("step %d, best accuracy %g" % (i, best_accuracy))
87 |
88 | print(str(i) + "/" + str(args.EPOCHS))
89 |
90 | print(best_accuracy)
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_baseline/model.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import numpy
3 | import os
4 | import torch
5 | from flyai.model.base import Base
6 | from torch.autograd import Variable
7 |
8 | from path import MODEL_PATH
9 |
10 | __import__('net', fromlist=["Net"])
11 |
12 | Torch_MODEL_NAME = "model.pkl"
13 |
14 | cuda_avail = torch.cuda.is_available()
15 |
16 |
17 | class Model(Base):
18 | def __init__(self, data):
19 | self.data = data
20 |
21 | def predict(self, **data):
22 | cnn = torch.load(os.path.join(MODEL_PATH, Torch_MODEL_NAME))
23 | if cuda_avail:
24 | cnn.cuda()
25 | x_data = self.data.predict_data(**data)
26 | x_data = torch.from_numpy(x_data)
27 | x_data = x_data.float()
28 | if cuda_avail:
29 | x_data = Variable(x_data.cuda())
30 | outputs = cnn(x_data)
31 | outputs = outputs.cpu()
32 | prediction = outputs.data.numpy()
33 | prediction = self.data.to_categorys(prediction)
34 | return prediction
35 |
36 | def predict_all(self, datas):
37 | print(os.path.join(MODEL_PATH, Torch_MODEL_NAME))
38 | cnn = torch.load(os.path.join(MODEL_PATH, Torch_MODEL_NAME))
39 | if cuda_avail:
40 | cnn.cuda()
41 | labels = []
42 | for data in datas:
43 | x_data = self.data.predict_data(**data)
44 | x_data = torch.from_numpy(x_data)
45 | x_data = x_data.float()
46 | if cuda_avail:
47 | x_data = Variable(x_data.cuda())
48 | outputs = cnn(x_data)
49 | outputs = outputs.cpu()
50 | prediction = outputs.data.numpy()
51 | prediction = self.data.to_categorys(prediction)
52 | labels.append(prediction)
53 | return labels
54 |
55 | def batch_iter(self, x, y, batch_size=128):
56 | """生成批次数据"""
57 | data_len = len(x)
58 | num_batch = int((data_len - 1) / batch_size) + 1
59 |
60 | indices = numpy.random.permutation(numpy.arange(data_len))
61 | x_shuffle = x[indices]
62 | y_shuffle = y[indices]
63 |
64 | for i in range(num_batch):
65 | start_id = i * batch_size
66 | end_id = min((i + 1) * batch_size, data_len)
67 | yield x_shuffle[start_id:end_id], y_shuffle[start_id:end_id]
68 |
69 | def save_model(self, network, path, name=Torch_MODEL_NAME, overwrite=False):
70 | super().save_model(network, path, name, overwrite)
71 | torch.save(network, os.path.join(path, name))
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_baseline/net.py:
--------------------------------------------------------------------------------
1 | ## build CNN
2 | from torch import nn
3 |
4 |
5 | ## build CNN
6 | class Net(nn.Module):
7 | # def __init__(self,num_classes=10):
8 | def __init__(self):
9 | super(Net, self).__init__()
10 | self.conv1 = nn.Conv2d(1, 32, 5, stride=1, padding=2)
11 | self.relu1 = nn.ReLU(True)
12 | self.bn1 = nn.BatchNorm2d(32)
13 | self.pool1 = nn.MaxPool2d(2, 2)
14 | self.conv2 = nn.Conv2d(32, 64, 3, stride=1, padding=1)
15 | self.relu2 = nn.ReLU(True)
16 | self.bn2 = nn.BatchNorm2d(64)
17 | self.pool2 = nn.MaxPool2d(2, 2)
18 | self.conv3 = nn.Conv2d(64, 128, 3, stride=1, padding=1)
19 | self.relu3 = nn.ReLU(True)
20 | self.bn3 = nn.BatchNorm2d(128)
21 | self.pool3 = nn.MaxPool2d(2, 2)
22 | self.fc1 = nn.Linear(128 * 16 * 16, 1024)
23 | self.relu4 = nn.ReLU(True)
24 | self.fc2 = nn.Linear(1024, 7)
25 |
26 | def forward(self, input):
27 | output = self.conv1(input)
28 | output = self.relu1(output)
29 | output = self.bn1(output)
30 | output = self.pool1(output)
31 |
32 | output = self.conv2(output)
33 | output = self.relu2(output)
34 | output = self.bn2(output)
35 | output = self.pool2(output)
36 |
37 | output = self.conv3(output)
38 | output = self.relu3(output)
39 | output = self.bn3(output)
40 | output = self.pool3(output)
41 |
42 | output = output.view(-1, 128 * 16 * 16)
43 | output = self.fc1(output)
44 | output = self.relu4(output)
45 | output = self.fc2(output)
46 |
47 | return output
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_baseline/path.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import sys
3 |
4 | import os
5 |
6 | DATA_PATH = os.path.join(sys.path[0], 'data', 'input')
7 | MODEL_PATH = os.path.join(sys.path[0], 'data', 'output', 'model')
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_baseline/predict.py:
--------------------------------------------------------------------------------
1 | from flyai.dataset import Dataset
2 |
3 | from model import Model
4 |
5 | data = Dataset()
6 | model = Model(data)
7 |
8 | p = model.predict(image_path="images/C39P4thinF_original_IMG_20150622_105253_cell_111.png")
9 |
10 | print(p)
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_baseline/processor.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import numpy
3 | from PIL import Image
4 | from flyai.processor.base import Base
5 | from flyai.processor.download import check_download
6 | import torch
7 |
8 | from path import DATA_PATH
9 |
10 | # 判断gpu是否可用
11 | if torch.cuda.is_available():
12 | device = 'cuda'
13 | else:
14 | device = 'cpu'
15 | device = torch.device(device)
16 | #device = torch.device('cpu')
17 |
18 | class Processor(Base):
19 | def input_x(self, image_path):
20 | '''
21 | 参数为csv中作为输入x的一条数据，该方法会被Dataset多次调用
22 | '''
23 | path = check_download(image_path, DATA_PATH)
24 | path = path.replace('\\','/')
25 | image = Image.open(path).convert('L')
26 | image = image.crop((32, 32, 223, 223))
27 | image = image.resize((128, 128))
28 | x_data = numpy.array(image)
29 | x_data = x_data.astype(numpy.float32)
30 | x_data = x_data.reshape([128, 128, 1])
31 | x_data = numpy.multiply(x_data, 1.0 / 255.0) ## scale to [0,1] from [0,255]
32 | x_data = numpy.transpose(x_data, (2, 0, 1)) ## reshape
33 | return x_data
34 |
35 | # 该参数需要与app.yaml的Model的output-->columns->name 一一对应
36 | def input_y(self, label):
37 | '''
38 | 参数为csv中作为输入y的一条数据，该方法会被Dataset多次调用
39 | '''
40 | return label
41 |
42 | def output_y(self, data):
43 | '''
44 | 验证时使用，把模型输出的y转为对应的结果
45 | '''
46 | return numpy.argmax(data)
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_baseline/transformation.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | class Transformation:
3 | '''
4 | 处理训练数据的类，某些情况下需要对训练的数据再一次的处理。
5 | 如无需处理的话，不用实现该方法。
6 | '''
7 |
8 | def transformation_data(self, x_train=None, y_train=None, x_test=None, y_test=None):
9 | return x_train, y_train, x_test, y_test
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_baseline/使用jupyter调试.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "### 一、本地运行调试代码"
8 | ]
9 | },
10 | {
11 | "cell_type": "markdown",
12 | "metadata": {},
13 | "source": [
14 | "网络结构在main.py之后，执行`run main.py`可以本地调试。\n",
15 | "\n",
16 | "执行下面代码可使用快捷键`shift+enter`"
17 | ]
18 | },
19 | {
20 | "cell_type": "code",
21 | "execution_count": null,
22 | "metadata": {},
23 | "outputs": [],
24 | "source": [
25 | "run main.py"
26 | ]
27 | },
28 | {
29 | "cell_type": "markdown",
30 | "metadata": {},
31 | "source": [
32 | "### 二、提交到GPU训练"
33 | ]
34 | },
35 | {
36 | "cell_type": "markdown",
37 | "metadata": {},
38 | "source": [
39 | "#### Windows环境执行"
40 | ]
41 | },
42 | {
43 | "cell_type": "markdown",
44 | "metadata": {},
45 | "source": [
46 | "通过执行训练命令，`-e=`是整个数据集循环10次，`-b=`是每次训练读取的数据量为32 "
47 | ]
48 | },
49 | {
50 | "cell_type": "code",
51 | "execution_count": null,
52 | "metadata": {},
53 | "outputs": [],
54 | "source": [
55 | "! flyai.exe train -e=10 -b=32"
56 | ]
57 | },
58 | {
59 | "cell_type": "markdown",
60 | "metadata": {},
61 | "source": [
62 | "#### Mac和Linux执行"
63 | ]
64 | },
65 | {
66 | "cell_type": "markdown",
67 | "metadata": {},
68 | "source": [
69 | "给执行脚本授权"
70 | ]
71 | },
72 | {
73 | "cell_type": "code",
74 | "execution_count": 6,
75 | "metadata": {},
76 | "outputs": [],
77 | "source": [
78 | "! chmod +x ./flyai"
79 | ]
80 | },
81 | {
82 | "cell_type": "markdown",
83 | "metadata": {},
84 | "source": [
85 | "通过执行训练命令，整个数据集循环10次，每次训练读取的数据量为 32 "
86 | ]
87 | },
88 | {
89 | "cell_type": "code",
90 | "execution_count": null,
91 | "metadata": {},
92 | "outputs": [],
93 | "source": [
94 | "! ./flyai train -e=10 -b=32"
95 | ]
96 | },
97 | {
98 | "cell_type": "markdown",
99 | "metadata": {},
100 | "source": [
101 | "### 三、FlyAI的Python环境"
102 | ]
103 | },
104 | {
105 | "cell_type": "markdown",
106 | "metadata": {},
107 | "source": [
108 | "#### Windows用户，Python路径在："
109 | ]
110 | },
111 | {
112 | "cell_type": "code",
113 | "execution_count": null,
114 | "metadata": {},
115 | "outputs": [],
116 | "source": [
117 | "! %USERPROFILE%\\.flyai\\env\\python.exe"
118 | ]
119 | },
120 | {
121 | "cell_type": "markdown",
122 | "metadata": {},
123 | "source": [
124 | "#### pip路径："
125 | ]
126 | },
127 | {
128 | "cell_type": "code",
129 | "execution_count": null,
130 | "metadata": {},
131 | "outputs": [],
132 | "source": [
133 | "! %USERPROFILE%\\.flyai\\env\\Scripts\\pip.exe list"
134 | ]
135 | },
136 | {
137 | "cell_type": "markdown",
138 | "metadata": {},
139 | "source": [
140 | "#### 如本地遇到No module name 'xxx'问题时可以执行下面代码安装"
141 | ]
142 | },
143 | {
144 | "cell_type": "code",
145 | "execution_count": null,
146 | "metadata": {},
147 | "outputs": [],
148 | "source": [
149 | "! %USERPROFILE%\\.flyai\\env\\Scripts\\pip.exe install xxx -i https://pypi.flyai.com/simple "
150 | ]
151 | },
152 | {
153 | "cell_type": "markdown",
154 | "metadata": {},
155 | "source": [
156 | "#### Mac和Linux用户，Python路径在："
157 | ]
158 | },
159 | {
160 | "cell_type": "code",
161 | "execution_count": null,
162 | "metadata": {},
163 | "outputs": [],
164 | "source": [
165 | "! ~/.flyai/env/bin/python3"
166 | ]
167 | },
168 | {
169 | "cell_type": "markdown",
170 | "metadata": {},
171 | "source": [
172 | "#### pip路径："
173 | ]
174 | },
175 | {
176 | "cell_type": "code",
177 | "execution_count": null,
178 | "metadata": {},
179 | "outputs": [],
180 | "source": [
181 | "! ~/.flyai/env/bin/pip list"
182 | ]
183 | },
184 | {
185 | "cell_type": "markdown",
186 | "metadata": {},
187 | "source": [
188 | "#### 如本地遇到No module name 'xxx'问题时可以执行下面代码安装"
189 | ]
190 | },
191 | {
192 | "cell_type": "code",
193 | "execution_count": null,
194 | "metadata": {},
195 | "outputs": [],
196 | "source": [
197 | "! ~/.flyai/env/bin/pip install xxx -i https://pypi.flyai.com/simple"
198 | ]
199 | },
200 | {
201 | "cell_type": "markdown",
202 | "metadata": {},
203 | "source": [
204 | "### 四、样例代码说明\n",
205 | "\n",
206 | "##### `app.yaml`\n",
207 | "\n",
208 | "> 是项目的配置文件，项目目录下**必须**存在这个文件，是项目运行的依赖。\n",
209 | "\n",
210 | "##### `processor.py`\n",
211 | "\n",
212 | "> **样例代码中已做简单实现，可供查考。**\n",
213 | ">\n",
214 | "> 处理数据的输入输出文件，把通过csv文件返回的数据，处理成能让程序识别、训练的矩阵。\n",
215 | ">\n",
216 | "> 可以自己定义输入输出的方法名，在`app.yaml`中声明即可。\n",
217 | ">\n",
218 | "> ```python\n",
219 | "> def input_x(self, image_path):\n",
220 | "> '''\n",
221 | "> \t参数为csv中作为输入x的一条数据，该方法会被dataset.next_train_batch()\n",
222 | "> \t和dataset.next_validation_batch()多次调用。可在该方法中做数据增强\n",
223 | "> \t该方法字段与app.yaml中的input:->columns:对应\n",
224 | "> \t'''\n",
225 | "> pass\n",
226 | ">\n",
227 | "> def output_x(self, image_path):\n",
228 | "> '''\n",
229 | "> \t参数为csv中作为输入x的一条数据，该方法会被dataset.next_train_batch()\n",
230 | "> \t和dataset.next_validation_batch()多次调用。\n",
231 | "> \t该方法字段与app.yaml中的input:->columns:对应\n",
232 | "> \t'''\n",
233 | "> pass\n",
234 | ">\n",
235 | "> def input_y(self, labels):\n",
236 | "> '''\n",
237 | "> 参数为csv中作为输入y的一条数据，该方法会被dataset.next_train_batch()\n",
238 | "> \t和dataset.next_validation_batch()多次调用。\n",
239 | "> \t该方法字段与app.yaml中的output:->columns:对应\n",
240 | "> '''\n",
241 | "> pass\n",
242 | ">\n",
243 | "> def output_y(self, data):\n",
244 | "> '''\n",
245 | "> 输出的结果，会被dataset.to_categorys(data)调用\n",
246 | "> :param data: 预测返回的数据\n",
247 | "> :return: 返回预测的标签\n",
248 | "> '''\n",
249 | "> pass\n",
250 | ">\n",
251 | "> ```\n",
252 | "\n",
253 | "##### `main.py`\n",
254 | "\n",
255 | "> **样例代码中已做简单实现，可供查考。**\n",
256 | ">\n",
257 | "> 程序入口，编写算法，训练模型的文件。在该文件中实现自己的算法。\n",
258 | ">\n",
259 | "> 通过`dataset.py`中的`next_batch`方法获取训练和测试数据。\n",
260 | ">\n",
261 | "> ```python\n",
262 | "> '''\n",
263 | "> Flyai库中的提供的数据处理方法\n",
264 | "> 传入整个数据训练多少轮，每批次批大小\n",
265 | "> '''\n",
266 | "> dataset = Dataset(epochs=args.EPOCHS, batch=args.BATCH)\n",
267 | "> #获取训练数据\n",
268 | "> x_train, y_train = dataset.next_train_batch()\n",
269 | "> #获取验证数据\n",
270 | "> x_val, y_val = dataset.next_validation_batch()\n",
271 | "> ```\n",
272 | ">\n",
273 | "> 通过`model.py`中的`save_model`方法保存模型\n",
274 | ">\n",
275 | "> ```python\n",
276 | "> # 模型操作辅助类\n",
277 | "> model = Model(dataset)\n",
278 | "> model.save_model(YOU_NET)\n",
279 | "> ```\n",
280 | ">\n",
281 | "> **如果使用`PyTorch`框架，需要在`net.py`文件中实现网络。其它用法同上。**\n",
282 | "\n",
283 | "##### `model.py`\n",
284 | "\n",
285 | "> **样例代码中已做简单实现，可供查考。**\n",
286 | ">\n",
287 | "> 训练好模型之后可以继承`flyai.model.base`包中的`base`重写下面三个方法实现模型的保存、验证和使用。\n",
288 | ">\n",
289 | "> ```python\n",
290 | "> def predict(self, **data):\n",
291 | "> '''\n",
292 | "> 使用模型\n",
293 | "> \t:param data: 模型的输入的一个或多个参数\n",
294 | "> :return:\n",
295 | "> '''\n",
296 | "> pass\n",
297 | ">\n",
298 | "> def predict_all(self, datas):\n",
299 | "> '''\n",
300 | "> （必须实现的方法）评估模型，对训练的好的模型进行打分\n",
301 | "> \t:param datas: 验证集上的随机数据，类型为list\n",
302 | "> :return outputs: 返回调用模型评估之后的list数据\n",
303 | "> '''\n",
304 | "> pass\n",
305 | ">\n",
306 | "> def save_model(self, network, path=MODEL_PATH, name=MODEL_NAME, overwrite=False):\n",
307 | "> '''\n",
308 | "> 保存模型\n",
309 | "> :param network: 训练模型的网络\n",
310 | "> :param path: 要保存模型的路径\n",
311 | "> :param name: 要保存模型的名字\n",
312 | "> :param overwrite: 是否覆盖当前模型\n",
313 | "> :return:\n",
314 | "> '''\n",
315 | "> self.check(path, overwrite)\n",
316 | ">\n",
317 | "> ```\n",
318 | "\n",
319 | "##### `predict.py`\n",
320 | "\n",
321 | ">**样例代码中已做简单实现，可供查考。**\n",
322 | ">\n",
323 | ">对训练完成的模型使用和预测。\n",
324 | "\n",
325 | "##### `path.py`\n",
326 | "\n",
327 | "> 可以设置数据文件、模型文件的存放路径。\n",
328 | "\n",
329 | "##### `dataset.py`\n",
330 | "\n",
331 | "> 该文件在**FlyAI开源库**的`flyai.dataset`包中，通过`next_train_batch()`和`next_validation_batch()`方法获得`x_train` `y_train` `x_val` `y_val`数据。\n",
332 | ">\n",
333 | "> FlyAI开源库可以通过`pip3 install -i https://pypi.flyai.com/simple flyai` 安装。\n",
334 | "\n",
335 | "***\n",
336 | "\n",
337 | "### [FlyAI全球人工智能专业开发平台，一站式服务平台](https://flyai.com)\n",
338 | "\n",
339 | "**扫描下方二维码，及时获取FlyAI最新消息，抢先体验最新功能。**\n",
340 | "\n",
341 | "\n",
342 | "\n",
343 | "[](https://flyai.com)\n",
344 | "\n",
345 | "\n",
346 | "\n"
347 | ]
348 | },
349 | {
350 | "cell_type": "code",
351 | "execution_count": null,
352 | "metadata": {},
353 | "outputs": [],
354 | "source": []
355 | }
356 | ],
357 | "metadata": {
358 | "kernelspec": {
359 | "display_name": "Python 3",
360 | "language": "python",
361 | "name": "python3"
362 | },
363 | "language_info": {
364 | "codemirror_mode": {
365 | "name": "ipython",
366 | "version": 3
367 | },
368 | "file_extension": ".py",
369 | "mimetype": "text/x-python",
370 | "name": "python",
371 | "nbconvert_exporter": "python",
372 | "pygments_lexer": "ipython3",
373 | "version": "3.6.4"
374 | }
375 | },
376 | "nbformat": 4,
377 | "nbformat_minor": 4
378 | }
379 |
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_baseline/常见问题.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | 常见问题
521 |
522 |
523 | FlyAI竞赛常见问题
如何获得奖金？
超过项目设置的最低分，根据公式计算，就可以获得奖金。
比赛使用什么框架？
比赛支持常用的机器学习和深度学习框架，比如TensorFlow，PyTorch，Keras，Scikit-learn等。
怎么参加比赛，需不需要提交csv文件？
FlyAI竞赛平台无需提交csv文件，在网页上点击报名，下载项目，使用你熟练的框架，修改main.py中的网络结构，和processor.py中的数据处理。使用FlyAI提供的命令提交，就可以参加比赛了。
比赛排行榜分数怎么得到的？
参加项目竞赛必须实现 model.py 中的predict_all方法。系统通过该方法，调用模型得出评分。
平台机器什么配置？
目前每个训练独占一块P40显卡，显存24G。
本地数据集在哪？
运行 flyai.exe test or ./flyai test 命令之后会下载100条数据到项目的data目录下
也可以本地使用ide运行 main.py 也会下载数据
FlyAI自带的Python环境在哪,会不会覆盖本地环境？
FlyAI不会覆盖本地现有的Python环境。
windows用户:
C:\Users{你计算机用户名}.flyai\env\python.exe
C:\Users{你计算机用户名}.flyai\env\Scripts\pip.exe
mac和linux用户:
/Users/{你计算机用户名}/.flyai/env/bin/python3.6
/Users/{你计算机用户名}/.flyai/env/bin/pip
FAI训练积分不够用怎么办？
目前GPU免费使用，可以进入到：我的积分，通过签到和分享等途径获得大量积分。
离线训练代码不符合规范问题
main.py中必须使用args.EPOCHS和args.BATCH。
项目什么时候开始，持续多长时间？
网站上能看见的项目就是已经开始的，项目会一直存在，随时可以提交。
排行榜和获奖问题
目前可能获得奖金的项目是审核之后才能上榜的，每天会审核一次。
通过审核之后，奖金才能发送到账户。
全量数据集怎么查看，数据集支不支持下载？
暂时不支持查看和下载，如果需要，可以进入数据集来源链接查看。
运行 flyai.exe train -e=xx -b=xx or ./flyai train -e=xx -b=xx 命令之后会提交代码到服务器上使用全量数据集训练。
from flyai.dataset import Dataset 报错
No module name 'flyai'
先找到ide中使用的Python对应的pip的位置。
windows用户：pip所在路径\pip.exe install -i https://pypi.flyai.com/simple flyai
mac和linux用户：pip所在路径/pip install -i https://pypi.flyai.com/simple flyai
其他 No module name 'xxxx'问题 也可以参考上面。
如果遇到其他问题还可以添加24小时客服微信：flyaixzs
扫描下方二维码，及时获取FlyAI最新消息，抢先体验最新功能。
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 | 1574561204986
53 |
54 |
55 | 1574561204986
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18/README.md:
--------------------------------------------------------------------------------
1 | 本次采用resnet18网络进行分类任务，
2 |
3 | 相较baseline改动:
4 | 1. 将net.py改为resnet18，
5 | 2. processor.py图像尺寸改为（224，224）以适应resnet尺寸。
6 | epoch：10 batch_size:32
7 | 评分：34.22
8 |
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18/app.yaml:
--------------------------------------------------------------------------------
1 | ---
2 | name: malaria Cell Images
3 | author: M
4 | description: malaria Cell Images Classification
5 | topic: malaria Cell Images Classification
6 | algorithm: CNN
7 | language: python3
8 | virtualenv: flyai_env/bin/
9 |
10 | framework: PyTorch
11 |
12 | data:
13 | id: MalariaCellImages
14 | name: malaria Cell Images
15 | category: image
16 |
17 |
18 | model:
19 | processor: Processor
20 | input_x: input_x
21 | input_y: input_y
22 | output_y: output_y
23 | input:
24 | columns:
25 | - name: image_path
26 | type: string
27 | to_type: float
28 | to_shape: [-1, 1]
29 | output:
30 | columns:
31 | - name: label
32 | type: int
33 | to_type: float
34 | to_shape: [-1,7]
35 |
36 |
37 | evaluate:
38 | score: torch_accuracy
39 |
40 | servers:
41 | - id: flyai
42 | url: https://flyai.com
43 | ...
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18/flyai:
--------------------------------------------------------------------------------
1 | unameOut="$(uname -s)"
2 | case "${unameOut}" in
3 | Linux*) machine=Linux;;
4 | Darwin*) machine=Mac;;
5 | CYGWIN*) machine=Cygwin;;
6 | MINGW*) machine=MinGw;;
7 | *) machine="UNKNOWN:${unameOut}"
8 | esac
9 |
10 |
11 | if [ $machine = "Mac" ]; then
12 | chmod +x ./.flyai_darwin
13 | ./.flyai_darwin "$@"
14 | fi
15 |
16 | if [ $machine = "Linux" ]; then
17 | chmod +x ./.flyai_linux
18 | ./.flyai_linux "$@"
19 | fi
20 |
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18/flyai.exe:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zwkkk/pytorch-project-exercise/7f3a8930d82e7143afbea022f778ba52a8bfa662/MalariaCellImages_FlyAI_resnet18/flyai.exe
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18/main.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import argparse
3 | import torch
4 | import torch.nn as nn
5 | from flyai.dataset import Dataset
6 | from torch.optim import Adam
7 |
8 | from model import Model
9 | from net import resnet18
10 | from path import MODEL_PATH
11 | from transformation import src
12 | # 数据获取辅助类
13 | dataset = Dataset()
14 |
15 | # 模型操作辅助类
16 | model = Model(dataset)
17 |
18 | # 超参
19 | parser = argparse.ArgumentParser()
20 | parser.add_argument("-e", "--EPOCHS", default=100, type=int, help="train epochs")
21 | parser.add_argument("-b", "--BATCH", default=24, type=int, help="batch size")
22 | args = parser.parse_args()
23 |
24 | # 判断gpu是否可用
25 | if torch.cuda.is_available():
26 | device = 'cuda'
27 | else:
28 | device = 'cpu'
29 | # device = 'cpu'
30 | device = torch.device(device)
31 |
32 |
33 | def eval(model, x_test, y_test):
34 | cnn.eval()
35 | batch_eval = model.batch_iter(x_test, y_test)
36 | total_acc = 0.0
37 | data_len = len(x_test)
38 | for x_batch, y_batch in batch_eval:
39 | batch_len = len(x_batch)
40 | outputs = cnn(x_batch)
41 | _, prediction = torch.max(outputs.data, 1)
42 | correct = (prediction == y_batch).sum().item()
43 | acc = correct / batch_len
44 | total_acc += acc * batch_len
45 | return total_acc / data_len
46 |
47 |
48 | cnn = resnet18().to(device)
49 | optimizer = Adam(cnn.parameters(), lr=0.001, betas=(0.9, 0.999)) # 选用AdamOptimizer
50 | loss_fn = nn.CrossEntropyLoss() # 定义损失函数
51 |
52 | # 训练并评估模型
53 |
54 | data = Dataset()
55 | model = Model(data)
56 |
57 | best_accuracy = 0
58 | for i in range(args.EPOCHS):
59 | cnn.train()
60 | x_train, y_train, x_test, y_test = data.next_batch(args.BATCH) # 读取数据
61 |
62 | x_train = torch.from_numpy(x_train)
63 | y_train = torch.from_numpy(y_train)
64 | x_train = x_train.float().to(device)
65 | y_train = y_train.long().to(device)
66 |
67 | x_test = torch.from_numpy(x_test)
68 | y_test = torch.from_numpy(y_test)
69 | x_test = x_test.float().to(device)
70 | y_test = y_test.long().to(device)
71 |
72 | outputs = cnn(x_train)
73 | _, prediction = torch.max(outputs.data, 1)
74 |
75 | optimizer.zero_grad()
76 | # print(x_train.shape,outputs.shape,y_train.shape)
77 | loss = loss_fn(outputs, y_train)
78 | loss.backward()
79 | optimizer.step()
80 | print(loss.detach())
81 | # 若测试准确率高于当前最高准确率，则保存模型
82 | train_accuracy = eval(model, x_test, y_test)
83 | if train_accuracy > best_accuracy:
84 | best_accuracy = train_accuracy
85 | model.save_model(cnn, MODEL_PATH, overwrite=True)
86 | print("step %d, best accuracy %g" % (i, best_accuracy))
87 |
88 | print(str(i) + "/" + str(args.EPOCHS))
89 |
90 | print(best_accuracy)
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18/model.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import numpy
3 | import os
4 | import torch
5 | from flyai.model.base import Base
6 | from torch.autograd import Variable
7 |
8 | from path import MODEL_PATH
9 |
10 | __import__('net', fromlist=["Net"])
11 |
12 | Torch_MODEL_NAME = "model.pkl"
13 |
14 | cuda_avail = torch.cuda.is_available()
15 |
16 |
17 | class Model(Base):
18 | def __init__(self, data):
19 | self.data = data
20 |
21 | def predict(self, **data):
22 | cnn = torch.load(os.path.join(MODEL_PATH, Torch_MODEL_NAME))
23 | if cuda_avail:
24 | cnn.cuda()
25 | x_data = self.data.predict_data(**data)
26 | x_data = torch.from_numpy(x_data)
27 | x_data = x_data.float()
28 | if cuda_avail:
29 | x_data = Variable(x_data.cuda())
30 | outputs = cnn(x_data)
31 | outputs = outputs.cpu()
32 | prediction = outputs.data.numpy()
33 | prediction = self.data.to_categorys(prediction)
34 | return prediction
35 |
36 | def predict_all(self, datas):
37 | print(os.path.join(MODEL_PATH, Torch_MODEL_NAME))
38 | cnn = torch.load(os.path.join(MODEL_PATH, Torch_MODEL_NAME))
39 | if cuda_avail:
40 | cnn.cuda()
41 | labels = []
42 | for data in datas:
43 | x_data = self.data.predict_data(**data)
44 | x_data = torch.from_numpy(x_data)
45 | x_data = x_data.float()
46 | if cuda_avail:
47 | x_data = Variable(x_data.cuda())
48 | outputs = cnn(x_data)
49 | outputs = outputs.cpu()
50 | prediction = outputs.data.numpy()
51 | prediction = self.data.to_categorys(prediction)
52 | labels.append(prediction)
53 | return labels
54 |
55 | def batch_iter(self, x, y, batch_size=128):
56 | """生成批次数据"""
57 | data_len = len(x)
58 | num_batch = int((data_len - 1) / batch_size) + 1
59 |
60 | indices = numpy.random.permutation(numpy.arange(data_len))
61 | x_shuffle = x[indices]
62 | y_shuffle = y[indices]
63 |
64 | for i in range(num_batch):
65 | start_id = i * batch_size
66 | end_id = min((i + 1) * batch_size, data_len)
67 | yield x_shuffle[start_id:end_id], y_shuffle[start_id:end_id]
68 |
69 | def save_model(self, network, path, name=Torch_MODEL_NAME, overwrite=False):
70 | super().save_model(network, path, name, overwrite)
71 | torch.save(network, os.path.join(path, name))
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18/net.py:
--------------------------------------------------------------------------------
1 | ## build CNN
2 | from torch import nn
3 | import math
4 | '''
5 |
6 | ## build CNN
7 | class Net(nn.Module):
8 | # def __init__(self,num_classes=10):
9 | def __init__(self):
10 | super(Net, self).__init__()
11 | self.conv1 = nn.Conv2d(1, 32, 5, stride=1, padding=2)
12 | self.relu1 = nn.ReLU(True)
13 | self.bn1 = nn.BatchNorm2d(32)
14 | self.pool1 = nn.MaxPool2d(2, 2)
15 | self.conv2 = nn.Conv2d(32, 64, 3, stride=1, padding=1)
16 | self.relu2 = nn.ReLU(True)
17 | self.bn2 = nn.BatchNorm2d(64)
18 | self.pool2 = nn.MaxPool2d(2, 2)
19 | self.conv3 = nn.Conv2d(64, 128, 3, stride=1, padding=1)
20 | self.relu3 = nn.ReLU(True)
21 | self.bn3 = nn.BatchNorm2d(128)
22 | self.pool3 = nn.MaxPool2d(2, 2)
23 | self.fc1 = nn.Linear(128 * 16 * 16, 1024)
24 | self.relu4 = nn.ReLU(True)
25 | self.fc2 = nn.Linear(1024, 7)
26 |
27 | def forward(self, input):
28 | output = self.conv1(input)
29 | output = self.relu1(output)
30 | output = self.bn1(output)
31 | output = self.pool1(output)
32 |
33 | output = self.conv2(output)
34 | output = self.relu2(output)
35 | output = self.bn2(output)
36 | output = self.pool2(output)
37 |
38 | output = self.conv3(output)
39 | output = self.relu3(output)
40 | output = self.bn3(output)
41 | output = self.pool3(output)
42 |
43 | output = output.view(-1, 128 * 16 * 16)
44 | output = self.fc1(output)
45 | output = self.relu4(output)
46 | output = self.fc2(output)
47 |
48 | return output
49 | '''
50 |
51 | '''
52 | 定义resnet18
53 | '''
54 |
55 |
56 | def conv3x3(in_planes, out_planes, stride=1):
57 | "3x3 convolution with padding"
58 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
59 | padding=1, bias=False)
60 |
61 |
62 | class BasicBlock(nn.Module):
63 | expansion = 1
64 |
65 | # inplanes其实就是channel,叫法不同
66 | def __init__(self, inplanes, planes, stride=1, downsample=None):
67 | super(BasicBlock, self).__init__()
68 | self.conv1 = conv3x3(inplanes, planes, stride)
69 | self.bn1 = nn.BatchNorm2d(planes)
70 | self.relu = nn.ReLU(inplace=True)
71 | self.conv2 = conv3x3(planes, planes)
72 | self.bn2 = nn.BatchNorm2d(planes)
73 | self.downsample = downsample
74 | self.stride = stride
75 |
76 | def forward(self, x):
77 | residual = x
78 |
79 | out = self.conv1(x)
80 | out = self.bn1(out)
81 | out = self.relu(out)
82 |
83 | out = self.conv2(out)
84 | out = self.bn2(out)
85 | # 把shortcut那的channel的维度统一
86 | if self.downsample is not None:
87 | residual = self.downsample(x)
88 |
89 | out += residual
90 | out = self.relu(out)
91 |
92 | return out
93 |
94 |
95 | class ResNet(nn.Module):
96 | def __init__(self, block, layers, num_classes=2):
97 | self.inplanes = 64
98 | super(ResNet, self).__init__()
99 | self.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, # 因为mnist为（1，28，28）灰度图，因此输入通道数为1
100 | bias=False)
101 | self.bn1 = nn.BatchNorm2d(64)
102 | self.relu = nn.ReLU(inplace=True)
103 | self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
104 | self.layer1 = self._make_layer(block, 64, layers[0])
105 | self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
106 | self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
107 | self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
108 | self.avgpool = nn.AvgPool2d(7, stride=1)
109 | self.fc = nn.Linear(512 * block.expansion, num_classes)
110 |
111 | for m in self.modules():
112 | if isinstance(m, nn.Conv2d):
113 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
114 | m.weight.data.normal_(0, math.sqrt(2. / n))
115 | elif isinstance(m, nn.BatchNorm2d):
116 | m.weight.data.fill_(1)
117 | m.bias.data.zero_()
118 |
119 | def _make_layer(self, block, planes, blocks, stride=1):
120 | # downsample 主要用来处理H(x)=F(x)+x中F(x)和xchannel维度不匹配问题
121 | downsample = None
122 | # self.inplanes为上个box_block的输出channel,planes为当前box_block块的输入channel
123 | if stride != 1 or self.inplanes != planes * block.expansion:
124 | downsample = nn.Sequential(
125 | nn.Conv2d(self.inplanes, planes * block.expansion,
126 | kernel_size=1, stride=stride, bias=False),
127 | nn.BatchNorm2d(planes * block.expansion),
128 | )
129 |
130 | layers = []
131 | layers.append(block(self.inplanes, planes, stride, downsample))
132 | self.inplanes = planes * block.expansion
133 | for i in range(1, blocks):
134 | layers.append(block(self.inplanes, planes))
135 |
136 | return nn.Sequential(*layers)
137 |
138 | def forward(self, x):
139 | x = self.conv1(x)
140 | x = self.bn1(x)
141 | x = self.relu(x)
142 | x = self.maxpool(x)
143 |
144 | x = self.layer1(x)
145 | x = self.layer2(x)
146 | x = self.layer3(x)
147 | x = self.layer4(x)
148 |
149 | x = self.avgpool(x)
150 | x = x.view(x.size(0), -1)
151 | x = self.fc(x)
152 |
153 | return x
154 |
155 |
156 | def resnet18(pretrained=False, **kwargs):
157 | """Constructs a ResNet-18 model.
158 | Args:
159 | pretrained (bool): If True, returns a model pre-trained on ImageNet
160 | """
161 | # [2, 2, 2, 2]和结构图[]X2是对应的
162 | model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs)
163 | if pretrained: # 加载模型权重
164 | model.load_state_dict(model_zoo.load_url(model_urls['resnet18']))
165 | return model
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18/path.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import sys
3 |
4 | import os
5 |
6 | DATA_PATH = os.path.join(sys.path[0], 'data', 'input')
7 | MODEL_PATH = os.path.join(sys.path[0], 'data', 'output', 'model')
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18/predict.py:
--------------------------------------------------------------------------------
1 | from flyai.dataset import Dataset
2 |
3 | from model import Model
4 |
5 | data = Dataset()
6 | model = Model(data)
7 |
8 | p = model.predict(image_path="images/C39P4thinF_original_IMG_20150622_105253_cell_111.png")
9 |
10 | print(p)
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18/processor.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import numpy
3 | from PIL import Image
4 | from flyai.processor.base import Base
5 | from flyai.processor.download import check_download
6 | import torch
7 |
8 | from path import DATA_PATH
9 |
10 | # 判断gpu是否可用
11 | if torch.cuda.is_available():
12 | device = 'cuda'
13 | else:
14 | device = 'cpu'
15 | device = torch.device(device)
16 | #device = torch.device('cpu')
17 |
18 | class Processor(Base):
19 | def input_x(self, image_path):
20 | '''
21 | 参数为csv中作为输入x的一条数据，该方法会被Dataset多次调用
22 | '''
23 | path = check_download(image_path, DATA_PATH)
24 | path = path.replace('\\','/')
25 | image = Image.open(path).convert('L')
26 | image = image.crop((32, 32, 223, 223))
27 | image = image.resize((224, 224))
28 | x_data = numpy.array(image)
29 | x_data = x_data.astype(numpy.float32)
30 | x_data = x_data.reshape([224, 224, 1])
31 | x_data = numpy.multiply(x_data, 1.0 / 255.0) ## scale to [0,1] from [0,255]
32 | x_data = numpy.transpose(x_data, (2, 0, 1)) ## reshape
33 | return x_data
34 |
35 | # 该参数需要与app.yaml的Model的output-->columns->name 一一对应
36 | def input_y(self, label):
37 | '''
38 | 参数为csv中作为输入y的一条数据，该方法会被Dataset多次调用
39 | '''
40 | return label
41 |
42 | def output_y(self, data):
43 | '''
44 | 验证时使用，把模型输出的y转为对应的结果
45 | '''
46 | return numpy.argmax(data)
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18/transformation.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | import torchvision.transforms as transforms
3 | import numpy as np
4 | from PIL import Image, ImageEnhance
5 | import torch
6 | class Transformation:
7 | '''
8 | 处理训练数据的类，某些情况下需要对训练的数据再一次的处理。
9 | 如无需处理的话，不用实现该方法。
10 | '''
11 |
12 | def transformation_data(self, x_train=None, y_train=None, x_test=None, y_test=None):
13 | return x_train, y_train, x_test, y_test
14 |
15 |
16 | def batch_X_transform(Xs,transforms):
17 |
18 | imgs=[]
19 | for x in Xs:
20 | img=Image.fromarray(x)
21 | img=transforms(img)
22 | imgs.append(img)
23 | imgs=torch.stack(imgs,dim=0)
24 | return imgs
25 |
26 |
27 | def src(Xs,size):
28 | tran= transforms.Compose([transforms.Resize(size),
29 | transforms.ToTensor(),
30 |
31 | ])
32 | return batch_X_transform(Xs, tran)
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18/使用jupyter调试.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "### 一、本地运行调试代码"
8 | ]
9 | },
10 | {
11 | "cell_type": "markdown",
12 | "metadata": {},
13 | "source": [
14 | "网络结构在main.py之后，执行`run main.py`可以本地调试。\n",
15 | "\n",
16 | "执行下面代码可使用快捷键`shift+enter`"
17 | ]
18 | },
19 | {
20 | "cell_type": "code",
21 | "execution_count": null,
22 | "metadata": {},
23 | "outputs": [],
24 | "source": [
25 | "run main.py"
26 | ]
27 | },
28 | {
29 | "cell_type": "markdown",
30 | "metadata": {},
31 | "source": [
32 | "### 二、提交到GPU训练"
33 | ]
34 | },
35 | {
36 | "cell_type": "markdown",
37 | "metadata": {},
38 | "source": [
39 | "#### Windows环境执行"
40 | ]
41 | },
42 | {
43 | "cell_type": "markdown",
44 | "metadata": {},
45 | "source": [
46 | "通过执行训练命令，`-e=`是整个数据集循环10次，`-b=`是每次训练读取的数据量为32 "
47 | ]
48 | },
49 | {
50 | "cell_type": "code",
51 | "execution_count": null,
52 | "metadata": {},
53 | "outputs": [],
54 | "source": [
55 | "! flyai.exe train -e=10 -b=32"
56 | ]
57 | },
58 | {
59 | "cell_type": "markdown",
60 | "metadata": {},
61 | "source": [
62 | "#### Mac和Linux执行"
63 | ]
64 | },
65 | {
66 | "cell_type": "markdown",
67 | "metadata": {},
68 | "source": [
69 | "给执行脚本授权"
70 | ]
71 | },
72 | {
73 | "cell_type": "code",
74 | "execution_count": 6,
75 | "metadata": {},
76 | "outputs": [],
77 | "source": [
78 | "! chmod +x ./flyai"
79 | ]
80 | },
81 | {
82 | "cell_type": "markdown",
83 | "metadata": {},
84 | "source": [
85 | "通过执行训练命令，整个数据集循环10次，每次训练读取的数据量为 32 "
86 | ]
87 | },
88 | {
89 | "cell_type": "code",
90 | "execution_count": null,
91 | "metadata": {},
92 | "outputs": [],
93 | "source": [
94 | "! ./flyai train -e=10 -b=32"
95 | ]
96 | },
97 | {
98 | "cell_type": "markdown",
99 | "metadata": {},
100 | "source": [
101 | "### 三、FlyAI的Python环境"
102 | ]
103 | },
104 | {
105 | "cell_type": "markdown",
106 | "metadata": {},
107 | "source": [
108 | "#### Windows用户，Python路径在："
109 | ]
110 | },
111 | {
112 | "cell_type": "code",
113 | "execution_count": null,
114 | "metadata": {},
115 | "outputs": [],
116 | "source": [
117 | "! %USERPROFILE%\\.flyai\\env\\python.exe"
118 | ]
119 | },
120 | {
121 | "cell_type": "markdown",
122 | "metadata": {},
123 | "source": [
124 | "#### pip路径："
125 | ]
126 | },
127 | {
128 | "cell_type": "code",
129 | "execution_count": null,
130 | "metadata": {},
131 | "outputs": [],
132 | "source": [
133 | "! %USERPROFILE%\\.flyai\\env\\Scripts\\pip.exe list"
134 | ]
135 | },
136 | {
137 | "cell_type": "markdown",
138 | "metadata": {},
139 | "source": [
140 | "#### 如本地遇到No module name 'xxx'问题时可以执行下面代码安装"
141 | ]
142 | },
143 | {
144 | "cell_type": "code",
145 | "execution_count": null,
146 | "metadata": {},
147 | "outputs": [],
148 | "source": [
149 | "! %USERPROFILE%\\.flyai\\env\\Scripts\\pip.exe install xxx -i https://pypi.flyai.com/simple "
150 | ]
151 | },
152 | {
153 | "cell_type": "markdown",
154 | "metadata": {},
155 | "source": [
156 | "#### Mac和Linux用户，Python路径在："
157 | ]
158 | },
159 | {
160 | "cell_type": "code",
161 | "execution_count": null,
162 | "metadata": {},
163 | "outputs": [],
164 | "source": [
165 | "! ~/.flyai/env/bin/python3"
166 | ]
167 | },
168 | {
169 | "cell_type": "markdown",
170 | "metadata": {},
171 | "source": [
172 | "#### pip路径："
173 | ]
174 | },
175 | {
176 | "cell_type": "code",
177 | "execution_count": null,
178 | "metadata": {},
179 | "outputs": [],
180 | "source": [
181 | "! ~/.flyai/env/bin/pip list"
182 | ]
183 | },
184 | {
185 | "cell_type": "markdown",
186 | "metadata": {},
187 | "source": [
188 | "#### 如本地遇到No module name 'xxx'问题时可以执行下面代码安装"
189 | ]
190 | },
191 | {
192 | "cell_type": "code",
193 | "execution_count": null,
194 | "metadata": {},
195 | "outputs": [],
196 | "source": [
197 | "! ~/.flyai/env/bin/pip install xxx -i https://pypi.flyai.com/simple"
198 | ]
199 | },
200 | {
201 | "cell_type": "markdown",
202 | "metadata": {},
203 | "source": [
204 | "### 四、样例代码说明\n",
205 | "\n",
206 | "##### `app.yaml`\n",
207 | "\n",
208 | "> 是项目的配置文件，项目目录下**必须**存在这个文件，是项目运行的依赖。\n",
209 | "\n",
210 | "##### `processor.py`\n",
211 | "\n",
212 | "> **样例代码中已做简单实现，可供查考。**\n",
213 | ">\n",
214 | "> 处理数据的输入输出文件，把通过csv文件返回的数据，处理成能让程序识别、训练的矩阵。\n",
215 | ">\n",
216 | "> 可以自己定义输入输出的方法名，在`app.yaml`中声明即可。\n",
217 | ">\n",
218 | "> ```python\n",
219 | "> def input_x(self, image_path):\n",
220 | "> '''\n",
221 | "> \t参数为csv中作为输入x的一条数据，该方法会被dataset.next_train_batch()\n",
222 | "> \t和dataset.next_validation_batch()多次调用。可在该方法中做数据增强\n",
223 | "> \t该方法字段与app.yaml中的input:->columns:对应\n",
224 | "> \t'''\n",
225 | "> pass\n",
226 | ">\n",
227 | "> def output_x(self, image_path):\n",
228 | "> '''\n",
229 | "> \t参数为csv中作为输入x的一条数据，该方法会被dataset.next_train_batch()\n",
230 | "> \t和dataset.next_validation_batch()多次调用。\n",
231 | "> \t该方法字段与app.yaml中的input:->columns:对应\n",
232 | "> \t'''\n",
233 | "> pass\n",
234 | ">\n",
235 | "> def input_y(self, labels):\n",
236 | "> '''\n",
237 | "> 参数为csv中作为输入y的一条数据，该方法会被dataset.next_train_batch()\n",
238 | "> \t和dataset.next_validation_batch()多次调用。\n",
239 | "> \t该方法字段与app.yaml中的output:->columns:对应\n",
240 | "> '''\n",
241 | "> pass\n",
242 | ">\n",
243 | "> def output_y(self, data):\n",
244 | "> '''\n",
245 | "> 输出的结果，会被dataset.to_categorys(data)调用\n",
246 | "> :param data: 预测返回的数据\n",
247 | "> :return: 返回预测的标签\n",
248 | "> '''\n",
249 | "> pass\n",
250 | ">\n",
251 | "> ```\n",
252 | "\n",
253 | "##### `main.py`\n",
254 | "\n",
255 | "> **样例代码中已做简单实现，可供查考。**\n",
256 | ">\n",
257 | "> 程序入口，编写算法，训练模型的文件。在该文件中实现自己的算法。\n",
258 | ">\n",
259 | "> 通过`dataset.py`中的`next_batch`方法获取训练和测试数据。\n",
260 | ">\n",
261 | "> ```python\n",
262 | "> '''\n",
263 | "> Flyai库中的提供的数据处理方法\n",
264 | "> 传入整个数据训练多少轮，每批次批大小\n",
265 | "> '''\n",
266 | "> dataset = Dataset(epochs=args.EPOCHS, batch=args.BATCH)\n",
267 | "> #获取训练数据\n",
268 | "> x_train, y_train = dataset.next_train_batch()\n",
269 | "> #获取验证数据\n",
270 | "> x_val, y_val = dataset.next_validation_batch()\n",
271 | "> ```\n",
272 | ">\n",
273 | "> 通过`model.py`中的`save_model`方法保存模型\n",
274 | ">\n",
275 | "> ```python\n",
276 | "> # 模型操作辅助类\n",
277 | "> model = Model(dataset)\n",
278 | "> model.save_model(YOU_NET)\n",
279 | "> ```\n",
280 | ">\n",
281 | "> **如果使用`PyTorch`框架，需要在`net.py`文件中实现网络。其它用法同上。**\n",
282 | "\n",
283 | "##### `model.py`\n",
284 | "\n",
285 | "> **样例代码中已做简单实现，可供查考。**\n",
286 | ">\n",
287 | "> 训练好模型之后可以继承`flyai.model.base`包中的`base`重写下面三个方法实现模型的保存、验证和使用。\n",
288 | ">\n",
289 | "> ```python\n",
290 | "> def predict(self, **data):\n",
291 | "> '''\n",
292 | "> 使用模型\n",
293 | "> \t:param data: 模型的输入的一个或多个参数\n",
294 | "> :return:\n",
295 | "> '''\n",
296 | "> pass\n",
297 | ">\n",
298 | "> def predict_all(self, datas):\n",
299 | "> '''\n",
300 | "> （必须实现的方法）评估模型，对训练的好的模型进行打分\n",
301 | "> \t:param datas: 验证集上的随机数据，类型为list\n",
302 | "> :return outputs: 返回调用模型评估之后的list数据\n",
303 | "> '''\n",
304 | "> pass\n",
305 | ">\n",
306 | "> def save_model(self, network, path=MODEL_PATH, name=MODEL_NAME, overwrite=False):\n",
307 | "> '''\n",
308 | "> 保存模型\n",
309 | "> :param network: 训练模型的网络\n",
310 | "> :param path: 要保存模型的路径\n",
311 | "> :param name: 要保存模型的名字\n",
312 | "> :param overwrite: 是否覆盖当前模型\n",
313 | "> :return:\n",
314 | "> '''\n",
315 | "> self.check(path, overwrite)\n",
316 | ">\n",
317 | "> ```\n",
318 | "\n",
319 | "##### `predict.py`\n",
320 | "\n",
321 | ">**样例代码中已做简单实现，可供查考。**\n",
322 | ">\n",
323 | ">对训练完成的模型使用和预测。\n",
324 | "\n",
325 | "##### `path.py`\n",
326 | "\n",
327 | "> 可以设置数据文件、模型文件的存放路径。\n",
328 | "\n",
329 | "##### `dataset.py`\n",
330 | "\n",
331 | "> 该文件在**FlyAI开源库**的`flyai.dataset`包中，通过`next_train_batch()`和`next_validation_batch()`方法获得`x_train` `y_train` `x_val` `y_val`数据。\n",
332 | ">\n",
333 | "> FlyAI开源库可以通过`pip3 install -i https://pypi.flyai.com/simple flyai` 安装。\n",
334 | "\n",
335 | "***\n",
336 | "\n",
337 | "### [FlyAI全球人工智能专业开发平台，一站式服务平台](https://flyai.com)\n",
338 | "\n",
339 | "**扫描下方二维码，及时获取FlyAI最新消息，抢先体验最新功能。**\n",
340 | "\n",
341 | "\n",
342 | "\n",
343 | "[](https://flyai.com)\n",
344 | "\n",
345 | "\n",
346 | "\n"
347 | ]
348 | },
349 | {
350 | "cell_type": "code",
351 | "execution_count": null,
352 | "metadata": {},
353 | "outputs": [],
354 | "source": []
355 | }
356 | ],
357 | "metadata": {
358 | "kernelspec": {
359 | "display_name": "Python 3",
360 | "language": "python",
361 | "name": "python3"
362 | },
363 | "language_info": {
364 | "codemirror_mode": {
365 | "name": "ipython",
366 | "version": 3
367 | },
368 | "file_extension": ".py",
369 | "mimetype": "text/x-python",
370 | "name": "python",
371 | "nbconvert_exporter": "python",
372 | "pygments_lexer": "ipython3",
373 | "version": "3.6.4"
374 | }
375 | },
376 | "nbformat": 4,
377 | "nbformat_minor": 4
378 | }
379 |
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18/常见问题.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | 常见问题
521 |
522 |
523 | FlyAI竞赛常见问题
如何获得奖金？
超过项目设置的最低分，根据公式计算，就可以获得奖金。
比赛使用什么框架？
比赛支持常用的机器学习和深度学习框架，比如TensorFlow，PyTorch，Keras，Scikit-learn等。
怎么参加比赛，需不需要提交csv文件？
FlyAI竞赛平台无需提交csv文件，在网页上点击报名，下载项目，使用你熟练的框架，修改main.py中的网络结构，和processor.py中的数据处理。使用FlyAI提供的命令提交，就可以参加比赛了。
比赛排行榜分数怎么得到的？
参加项目竞赛必须实现 model.py 中的predict_all方法。系统通过该方法，调用模型得出评分。
平台机器什么配置？
目前每个训练独占一块P40显卡，显存24G。
本地数据集在哪？
运行 flyai.exe test or ./flyai test 命令之后会下载100条数据到项目的data目录下
也可以本地使用ide运行 main.py 也会下载数据
FlyAI自带的Python环境在哪,会不会覆盖本地环境？
FlyAI不会覆盖本地现有的Python环境。
windows用户:
C:\Users{你计算机用户名}.flyai\env\python.exe
C:\Users{你计算机用户名}.flyai\env\Scripts\pip.exe
mac和linux用户:
/Users/{你计算机用户名}/.flyai/env/bin/python3.6
/Users/{你计算机用户名}/.flyai/env/bin/pip
FAI训练积分不够用怎么办？
目前GPU免费使用，可以进入到：我的积分，通过签到和分享等途径获得大量积分。
离线训练代码不符合规范问题
main.py中必须使用args.EPOCHS和args.BATCH。
项目什么时候开始，持续多长时间？
网站上能看见的项目就是已经开始的，项目会一直存在，随时可以提交。
排行榜和获奖问题
目前可能获得奖金的项目是审核之后才能上榜的，每天会审核一次。
通过审核之后，奖金才能发送到账户。
全量数据集怎么查看，数据集支不支持下载？
暂时不支持查看和下载，如果需要，可以进入数据集来源链接查看。
运行 flyai.exe train -e=xx -b=xx or ./flyai train -e=xx -b=xx 命令之后会提交代码到服务器上使用全量数据集训练。
from flyai.dataset import Dataset 报错
No module name 'flyai'
先找到ide中使用的Python对应的pip的位置。
windows用户：pip所在路径\pip.exe install -i https://pypi.flyai.com/simple flyai
mac和linux用户：pip所在路径/pip install -i https://pypi.flyai.com/simple flyai
其他 No module name 'xxxx'问题 也可以参考上面。
如果遇到其他问题还可以添加24小时客服微信：flyaixzs
扫描下方二维码，及时获取FlyAI最新消息，抢先体验最新功能。
2 |
3 | ---
4 | optimizer = Adam(cnn.parameters(), lr=0.001, betas=(0.9, 0.999)) # 选用AdamOptimizer.
5 |
6 | 1.
7 | epoch:10
8 | batch_size:32
9 | 评分：32.45
10 |
11 | 2.
12 | epoch:128
13 | batch_size:500
14 | 评分：75.41
15 |
16 | 3.
17 | epoch:128
18 | batch_size:1000
19 | 评分：50.97
20 |
21 | 4.
22 | epoch:512
23 | batch_size:1000
24 | 评分：85.93
25 |
26 | 5.
27 | epoch:512
28 | batch_size:128
29 | 评分：82.29
30 |
31 | 6.
32 | epoch:1000
33 | batch_size:128
34 | 评分：85.91
35 |
36 | 7.
37 | epoch:1000
38 | batch_size:256
39 | 评分：86.73
40 |
41 | optimizer = Adam(cnn.parameters(), lr=3e-4) # 选用AdamOptimizer
42 | 8.
43 | epoch:1000
44 | batch_size:512
45 | 评分：84.31
46 |
47 | 500/1000
48 | tensor(0.0745, device='cuda:0')
49 |
50 | 700/1000
51 | tensor(0.0565, device='cuda:0')
52 |
53 | 800/1000
54 | tensor(0.0193, device='cuda:0')
55 |
56 | 900/1000
57 | tensor(0.0051, device='cuda:0')
58 |
59 | 998/1000
60 | tensor(0.0167, device='cuda:0')
61 |
62 | ---
63 | ```python
64 | #调整学习率策略 factor=0.85, patience=20 要搭配好，factor太小容易一下子就使lr降到很小值
65 |
66 | scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.85, patience=20, verbose=True,
67 | threshold=0.0001, threshold_mode='rel', cooldown=0, min_lr=0, eps=1e-10)
68 | ```
69 | 9.
70 | epoch:500
71 | batch_size:512
72 | 评分：49.6
73 | train_loss: 0.6935; test_accuracy: 0.71875; lr: 7.1410e-06
74 | 分析：训练损失还很大，因此为欠拟合，lr降得太小了，参数更新幅度太小; 模型保存逻辑为测试集accuracy不再增加，则保存模型，那么当测试集acc=1时，就不再保存模型了，之后我们将保存逻辑调整为当测试集accuracy大于等于之前最好，就保存模型，使训练集进一步拟合。
75 |
76 |
77 | ---
78 | 参考第8次训练过程，针对patience进行处理
79 | ```python
80 | scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=800, verbose=True,
81 | threshold=0.0001, threshold_mode='rel', cooldown=0, min_lr=0, eps=1e-10)
82 | ```
83 | 10.
84 | epoch:4000
85 | batch_size:256
86 | 评分：85.13
87 |
88 | 这个版本patience=800太大，学习率基本没变化
89 |
90 |
91 | ---
92 | 按照ReduceLROnPlateau调整机制，patience最好设置的比上面稍为小一点。我们参考第8次训练loss损失，采用等距降低学习率策略
93 | ```python
94 | scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 800, gamma=0.1, last_epoch=-1)
95 | ```
96 | 10.
97 | epoch:3200
98 | batch_size:1000
99 | 评分：84.99
100 |
101 | 500/3200
102 | tensor(0.0002, device='cuda:0')
103 |
104 | 1000/3200
105 | tensor(7.8809e-05, device='cuda:0')
106 |
107 | batch size选择过大，导致其严重过拟合
108 |
109 |
110 |
111 | ---
112 | 将在测试集的损失作为保存模型的标准，原来为准确率
113 | epoch:500
114 | batch_size:512
115 | 评分：
版本41
116 |
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18_2/app.yaml:
--------------------------------------------------------------------------------
1 | ---
2 | name: malaria Cell Images
3 | author: M
4 | description: malaria Cell Images Classification
5 | topic: malaria Cell Images Classification
6 | algorithm: CNN
7 | language: python3
8 | virtualenv: flyai_env/bin/
9 |
10 | framework: PyTorch
11 |
12 | data:
13 | id: MalariaCellImages
14 | name: malaria Cell Images
15 | category: image
16 |
17 |
18 | model:
19 | processor: Processor
20 | input_x: input_x
21 | input_y: input_y
22 | output_y: output_y
23 | input:
24 | columns:
25 | - name: image_path
26 | type: string
27 | to_type: float
28 | to_shape: [-1, 1]
29 | output:
30 | columns:
31 | - name: label
32 | type: int
33 | to_type: float
34 | to_shape: [-1,7]
35 |
36 |
37 | evaluate:
38 | score: torch_accuracy
39 |
40 | servers:
41 | - id: flyai
42 | url: https://flyai.com
43 | ...
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18_2/flyai:
--------------------------------------------------------------------------------
1 | unameOut="$(uname -s)"
2 | case "${unameOut}" in
3 | Linux*) machine=Linux;;
4 | Darwin*) machine=Mac;;
5 | CYGWIN*) machine=Cygwin;;
6 | MINGW*) machine=MinGw;;
7 | *) machine="UNKNOWN:${unameOut}"
8 | esac
9 |
10 |
11 | if [ $machine = "Mac" ]; then
12 | chmod +x ./.flyai_darwin
13 | ./.flyai_darwin "$@"
14 | fi
15 |
16 | if [ $machine = "Linux" ]; then
17 | chmod +x ./.flyai_linux
18 | ./.flyai_linux "$@"
19 | fi
20 |
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18_2/flyai.exe:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zwkkk/pytorch-project-exercise/7f3a8930d82e7143afbea022f778ba52a8bfa662/MalariaCellImages_FlyAI_resnet18_2/flyai.exe
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18_2/main.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import argparse
3 | import torch
4 | import torch.nn as nn
5 | from flyai.dataset import Dataset
6 | from torch.optim import Adam
7 |
8 | from model import Model
9 | from net import resnet18
10 | from path import MODEL_PATH
11 | from transformation import src
12 | # 数据获取辅助类
13 | dataset = Dataset()
14 |
15 | # 模型操作辅助类
16 | model = Model(dataset)
17 |
18 | # 超参
19 | parser = argparse.ArgumentParser()
20 | parser.add_argument("-e", "--EPOCHS", default=100, type=int, help="train epochs")
21 | parser.add_argument("-b", "--BATCH", default=24, type=int, help="batch size")
22 | args = parser.parse_args()
23 |
24 | # 判断gpu是否可用
25 | if torch.cuda.is_available():
26 | device = 'cuda'
27 | else:
28 | device = 'cpu'
29 | # device = 'cpu'
30 | device = torch.device(device)
31 |
32 | def eval(model, x_test, y_test):
33 | cnn.eval()
34 | batch_eval = model.batch_iter(x_test, y_test)
35 | total_acc = 0.0
36 | data_len = len(x_test)
37 | for x_batch, y_batch in batch_eval:
38 | batch_len = len(x_batch)
39 | outputs = cnn(x_batch)
40 | _, prediction = torch.max(outputs.data, 1)
41 | correct = (prediction == y_batch).sum().item()
42 | acc = correct / batch_len
43 | total_acc += acc * batch_len
44 | return total_acc / data_len
45 |
46 |
47 | cnn = resnet18().to(device)
48 |
49 |
50 | #实验改动位置
51 | #################################################################################
52 | '''
53 | 针对optimizer进行实验
54 | '''
55 | optimizer = Adam(cnn.parameters(), lr=3e-4) # 选用AdamOptimizer
56 |
57 |
58 | #学习率优化若800轮训练loss未下降，则学习率*0.1
59 |
60 | scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 800, gamma=0.1, last_epoch=-1)
61 |
62 | #################################################################################
63 |
64 | loss_fn = nn.CrossEntropyLoss() # 定义损失函数
65 |
66 | # 训练并评估模型
67 |
68 | data = Dataset()
69 | model = Model(data)
70 |
71 | best_accuracy = 0
72 | for i in range(args.EPOCHS):
73 | cnn.train()
74 | x_train, y_train, x_test, y_test = data.next_batch(args.BATCH) # 读取数据
75 |
76 | x_train = torch.from_numpy(x_train)
77 | y_train = torch.from_numpy(y_train)
78 | x_train = x_train.float().to(device)
79 | y_train = y_train.long().to(device)
80 |
81 | x_test = torch.from_numpy(x_test)
82 | y_test = torch.from_numpy(y_test)
83 | x_test = x_test.float().to(device)
84 | y_test = y_test.long().to(device)
85 |
86 | outputs = cnn(x_train)
87 | _, prediction = torch.max(outputs.data, 1)
88 |
89 | optimizer.zero_grad()
90 | # print(x_train.shape,outputs.shape,y_train.shape)
91 | loss = loss_fn(outputs, y_train)
92 | loss.backward()
93 | optimizer.step() #优化器针对loss进行参数更新
94 | scheduler.step() #scheduler为针对学习率的调整策略
95 | print(loss.detach())
96 | # 若测试准确率高于当前最高准确率，则保存模型
97 | train_accuracy = eval(model, x_test, y_test)
98 | if train_accuracy >= best_accuracy:
99 | best_accuracy = train_accuracy
100 | model.save_model(cnn, MODEL_PATH, overwrite=True)
101 | print("step %d, best accuracy %g" % (i, best_accuracy))
102 |
103 | print(str(i) + "/" + str(args.EPOCHS))
104 |
105 | print(best_accuracy)
106 |
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18_2/model.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import numpy
3 | import os
4 | import torch
5 | from flyai.model.base import Base
6 | from torch.autograd import Variable
7 |
8 | from path import MODEL_PATH
9 |
10 | __import__('net', fromlist=["Net"])
11 |
12 | Torch_MODEL_NAME = "model.pkl"
13 |
14 | cuda_avail = torch.cuda.is_available()
15 |
16 |
17 | class Model(Base):
18 | def __init__(self, data):
19 | self.data = data
20 |
21 | def predict(self, **data):
22 | cnn = torch.load(os.path.join(MODEL_PATH, Torch_MODEL_NAME))
23 | if cuda_avail:
24 | cnn.cuda()
25 | x_data = self.data.predict_data(**data)
26 | x_data = torch.from_numpy(x_data)
27 | x_data = x_data.float()
28 | if cuda_avail:
29 | x_data = Variable(x_data.cuda())
30 | outputs = cnn(x_data)
31 | outputs = outputs.cpu()
32 | prediction = outputs.data.numpy()
33 | prediction = self.data.to_categorys(prediction)
34 | return prediction
35 |
36 | def predict_all(self, datas):
37 | print(os.path.join(MODEL_PATH, Torch_MODEL_NAME))
38 | cnn = torch.load(os.path.join(MODEL_PATH, Torch_MODEL_NAME))
39 | if cuda_avail:
40 | cnn.cuda()
41 | labels = []
42 | for data in datas:
43 | x_data = self.data.predict_data(**data)
44 | x_data = torch.from_numpy(x_data)
45 | x_data = x_data.float()
46 | if cuda_avail:
47 | x_data = Variable(x_data.cuda())
48 | outputs = cnn(x_data)
49 | outputs = outputs.cpu()
50 | prediction = outputs.data.numpy()
51 | prediction = self.data.to_categorys(prediction)
52 | labels.append(prediction)
53 | return labels
54 |
55 | def batch_iter(self, x, y, batch_size=128):
56 | """生成批次数据"""
57 | data_len = len(x)
58 | num_batch = int((data_len - 1) / batch_size) + 1
59 |
60 | indices = numpy.random.permutation(numpy.arange(data_len))
61 | x_shuffle = x[indices]
62 | y_shuffle = y[indices]
63 |
64 | for i in range(num_batch):
65 | start_id = i * batch_size
66 | end_id = min((i + 1) * batch_size, data_len)
67 | yield x_shuffle[start_id:end_id], y_shuffle[start_id:end_id]
68 |
69 | def save_model(self, network, path, name=Torch_MODEL_NAME, overwrite=False):
70 | super().save_model(network, path, name, overwrite)
71 | torch.save(network, os.path.join(path, name))
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18_2/net.py:
--------------------------------------------------------------------------------
1 | ## build CNN
2 | from torch import nn
3 | import math
4 | '''
5 |
6 | ## build CNN
7 | class Net(nn.Module):
8 | # def __init__(self,num_classes=10):
9 | def __init__(self):
10 | super(Net, self).__init__()
11 | self.conv1 = nn.Conv2d(1, 32, 5, stride=1, padding=2)
12 | self.relu1 = nn.ReLU(True)
13 | self.bn1 = nn.BatchNorm2d(32)
14 | self.pool1 = nn.MaxPool2d(2, 2)
15 | self.conv2 = nn.Conv2d(32, 64, 3, stride=1, padding=1)
16 | self.relu2 = nn.ReLU(True)
17 | self.bn2 = nn.BatchNorm2d(64)
18 | self.pool2 = nn.MaxPool2d(2, 2)
19 | self.conv3 = nn.Conv2d(64, 128, 3, stride=1, padding=1)
20 | self.relu3 = nn.ReLU(True)
21 | self.bn3 = nn.BatchNorm2d(128)
22 | self.pool3 = nn.MaxPool2d(2, 2)
23 | self.fc1 = nn.Linear(128 * 16 * 16, 1024)
24 | self.relu4 = nn.ReLU(True)
25 | self.fc2 = nn.Linear(1024, 7)
26 |
27 | def forward(self, input):
28 | output = self.conv1(input)
29 | output = self.relu1(output)
30 | output = self.bn1(output)
31 | output = self.pool1(output)
32 |
33 | output = self.conv2(output)
34 | output = self.relu2(output)
35 | output = self.bn2(output)
36 | output = self.pool2(output)
37 |
38 | output = self.conv3(output)
39 | output = self.relu3(output)
40 | output = self.bn3(output)
41 | output = self.pool3(output)
42 |
43 | output = output.view(-1, 128 * 16 * 16)
44 | output = self.fc1(output)
45 | output = self.relu4(output)
46 | output = self.fc2(output)
47 |
48 | return output
49 | '''
50 |
51 | '''
52 | 定义resnet18
53 | '''
54 |
55 |
56 | def conv3x3(in_planes, out_planes, stride=1):
57 | "3x3 convolution with padding"
58 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
59 | padding=1, bias=False)
60 |
61 |
62 | class BasicBlock(nn.Module):
63 | expansion = 1
64 |
65 | # inplanes其实就是channel,叫法不同
66 | def __init__(self, inplanes, planes, stride=1, downsample=None):
67 | super(BasicBlock, self).__init__()
68 | self.conv1 = conv3x3(inplanes, planes, stride)
69 | self.bn1 = nn.BatchNorm2d(planes)
70 | self.relu = nn.ReLU(inplace=True)
71 | self.conv2 = conv3x3(planes, planes)
72 | self.bn2 = nn.BatchNorm2d(planes)
73 | self.downsample = downsample
74 | self.stride = stride
75 |
76 | def forward(self, x):
77 | residual = x
78 |
79 | out = self.conv1(x)
80 | out = self.bn1(out)
81 | out = self.relu(out)
82 |
83 | out = self.conv2(out)
84 | out = self.bn2(out)
85 | # 把shortcut那的channel的维度统一
86 | if self.downsample is not None:
87 | residual = self.downsample(x)
88 |
89 | out += residual
90 | out = self.relu(out)
91 |
92 | return out
93 |
94 |
95 | class ResNet(nn.Module):
96 | def __init__(self, block, layers, num_classes=2):
97 | self.inplanes = 64
98 | super(ResNet, self).__init__()
99 | self.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, # 因为mnist为（1，28，28）灰度图，因此输入通道数为1
100 | bias=False)
101 | self.bn1 = nn.BatchNorm2d(64)
102 | self.relu = nn.ReLU(inplace=True)
103 | self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
104 | self.layer1 = self._make_layer(block, 64, layers[0])
105 | self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
106 | self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
107 | self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
108 | self.avgpool = nn.AvgPool2d(7, stride=1)
109 | self.fc = nn.Linear(512 * block.expansion, num_classes)
110 |
111 | for m in self.modules():
112 | if isinstance(m, nn.Conv2d):
113 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
114 | m.weight.data.normal_(0, math.sqrt(2. / n))
115 | elif isinstance(m, nn.BatchNorm2d):
116 | m.weight.data.fill_(1)
117 | m.bias.data.zero_()
118 |
119 | def _make_layer(self, block, planes, blocks, stride=1):
120 | # downsample 主要用来处理H(x)=F(x)+x中F(x)和xchannel维度不匹配问题
121 | downsample = None
122 | # self.inplanes为上个box_block的输出channel,planes为当前box_block块的输入channel
123 | if stride != 1 or self.inplanes != planes * block.expansion:
124 | downsample = nn.Sequential(
125 | nn.Conv2d(self.inplanes, planes * block.expansion,
126 | kernel_size=1, stride=stride, bias=False),
127 | nn.BatchNorm2d(planes * block.expansion),
128 | )
129 |
130 | layers = []
131 | layers.append(block(self.inplanes, planes, stride, downsample))
132 | self.inplanes = planes * block.expansion
133 | for i in range(1, blocks):
134 | layers.append(block(self.inplanes, planes))
135 |
136 | return nn.Sequential(*layers)
137 |
138 | def forward(self, x):
139 | x = self.conv1(x)
140 | x = self.bn1(x)
141 | x = self.relu(x)
142 | x = self.maxpool(x)
143 |
144 | x = self.layer1(x)
145 | x = self.layer2(x)
146 | x = self.layer3(x)
147 | x = self.layer4(x)
148 |
149 | x = self.avgpool(x)
150 | x = x.view(x.size(0), -1)
151 | x = self.fc(x)
152 |
153 | return x
154 |
155 |
156 | def resnet18(pretrained=False, **kwargs):
157 | """Constructs a ResNet-18 model.
158 | Args:
159 | pretrained (bool): If True, returns a model pre-trained on ImageNet
160 | """
161 | # [2, 2, 2, 2]和结构图[]X2是对应的
162 | model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs)
163 | if pretrained: # 加载模型权重
164 | model.load_state_dict(model_zoo.load_url(model_urls['resnet18']))
165 | return model
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18_2/path.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import sys
3 |
4 | import os
5 |
6 | DATA_PATH = os.path.join(sys.path[0], 'data', 'input')
7 | MODEL_PATH = os.path.join(sys.path[0], 'data', 'output', 'model')
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18_2/predict.py:
--------------------------------------------------------------------------------
1 | from flyai.dataset import Dataset
2 |
3 | from model import Model
4 |
5 | data = Dataset()
6 | model = Model(data)
7 |
8 | p = model.predict(image_path="images/C39P4thinF_original_IMG_20150622_105253_cell_111.png")
9 |
10 | print(p)
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18_2/processor.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import numpy
3 | from PIL import Image
4 | from flyai.processor.base import Base
5 | from flyai.processor.download import check_download
6 | import torch
7 |
8 | from path import DATA_PATH
9 |
10 | # 判断gpu是否可用
11 | if torch.cuda.is_available():
12 | device = 'cuda'
13 | else:
14 | device = 'cpu'
15 | device = torch.device(device)
16 | #device = torch.device('cpu')
17 |
18 | class Processor(Base):
19 | def input_x(self, image_path):
20 | '''
21 | 参数为csv中作为输入x的一条数据，该方法会被Dataset多次调用
22 | '''
23 | path = check_download(image_path, DATA_PATH)
24 | path = path.replace('\\','/')
25 | image = Image.open(path).convert('L')
26 | image = image.crop((32, 32, 223, 223))
27 | image = image.resize((224, 224))
28 | x_data = numpy.array(image)
29 | x_data = x_data.astype(numpy.float32)
30 | x_data = x_data.reshape([224, 224, 1])
31 | x_data = numpy.multiply(x_data, 1.0 / 255.0) ## scale to [0,1] from [0,255]
32 | x_data = numpy.transpose(x_data, (2, 0, 1)) ## reshape
33 | return x_data
34 |
35 | # 该参数需要与app.yaml的Model的output-->columns->name 一一对应
36 | def input_y(self, label):
37 | '''
38 | 参数为csv中作为输入y的一条数据，该方法会被Dataset多次调用
39 | '''
40 | return label
41 |
42 | def output_y(self, data):
43 | '''
44 | 验证时使用，把模型输出的y转为对应的结果
45 | '''
46 | return numpy.argmax(data)
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18_2/transformation.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | import torchvision.transforms as transforms
3 | import numpy as np
4 | from PIL import Image, ImageEnhance
5 | import torch
6 | class Transformation:
7 | '''
8 | 处理训练数据的类，某些情况下需要对训练的数据再一次的处理。
9 | 如无需处理的话，不用实现该方法。
10 | '''
11 |
12 | def transformation_data(self, x_train=None, y_train=None, x_test=None, y_test=None):
13 | return x_train, y_train, x_test, y_test
14 |
15 |
16 | def batch_X_transform(Xs,transforms):
17 |
18 | imgs=[]
19 | for x in Xs:
20 | img=Image.fromarray(x)
21 | img=transforms(img)
22 | imgs.append(img)
23 | imgs=torch.stack(imgs,dim=0)
24 | return imgs
25 |
26 |
27 | def src(Xs,size):
28 | tran= transforms.Compose([transforms.Resize(size),
29 | transforms.ToTensor(),
30 |
31 | ])
32 | return batch_X_transform(Xs, tran)
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet18_2/使用jupyter调试.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "### 一、本地运行调试代码"
8 | ]
9 | },
10 | {
11 | "cell_type": "markdown",
12 | "metadata": {},
13 | "source": [
14 | "网络结构在main.py之后，执行`run main.py`可以本地调试。\n",
15 | "\n",
16 | "执行下面代码可使用快捷键`shift+enter`"
17 | ]
18 | },
19 | {
20 | "cell_type": "code",
21 | "execution_count": null,
22 | "metadata": {},
23 | "outputs": [],
24 | "source": [
25 | "run main.py"
26 | ]
27 | },
28 | {
29 | "cell_type": "markdown",
30 | "metadata": {},
31 | "source": [
32 | "### 二、提交到GPU训练"
33 | ]
34 | },
35 | {
36 | "cell_type": "markdown",
37 | "metadata": {},
38 | "source": [
39 | "#### Windows环境执行"
40 | ]
41 | },
42 | {
43 | "cell_type": "markdown",
44 | "metadata": {},
45 | "source": [
46 | "通过执行训练命令，`-e=`是整个数据集循环10次，`-b=`是每次训练读取的数据量为32 "
47 | ]
48 | },
49 | {
50 | "cell_type": "code",
51 | "execution_count": null,
52 | "metadata": {},
53 | "outputs": [],
54 | "source": [
55 | "! flyai.exe train -e=10 -b=32"
56 | ]
57 | },
58 | {
59 | "cell_type": "markdown",
60 | "metadata": {},
61 | "source": [
62 | "#### Mac和Linux执行"
63 | ]
64 | },
65 | {
66 | "cell_type": "markdown",
67 | "metadata": {},
68 | "source": [
69 | "给执行脚本授权"
70 | ]
71 | },
72 | {
73 | "cell_type": "code",
74 | "execution_count": 6,
75 | "metadata": {},
76 | "outputs": [],
77 | "source": [
78 | "! chmod +x ./flyai"
79 | ]
80 | },
81 | {
82 | "cell_type": "markdown",
83 | "metadata": {},
84 | "source": [
85 | "通过执行训练命令，整个数据集循环10次，每次训练读取的数据量为 32 "
86 | ]
87 | },
88 | {
89 | "cell_type": "code",
90 | "execution_count": null,
91 | "metadata": {},
92 | "outputs": [],
93 | "source": [
94 | "! ./flyai train -e=10 -b=32"
95 | ]
96 | },
97 | {
98 | "cell_type": "markdown",
99 | "metadata": {},
100 | "source": [
101 | "### 三、FlyAI的Python环境"
102 | ]
103 | },
104 | {
105 | "cell_type": "markdown",
106 | "metadata": {},
107 | "source": [
108 | "#### Windows用户，Python路径在："
109 | ]
110 | },
111 | {
112 | "cell_type": "code",
113 | "execution_count": null,
114 | "metadata": {},
115 | "outputs": [],
116 | "source": [
117 | "! %USERPROFILE%\\.flyai\\env\\python.exe"
118 | ]
119 | },
120 | {
121 | "cell_type": "markdown",
122 | "metadata": {},
123 | "source": [
124 | "#### pip路径："
125 | ]
126 | },
127 | {
128 | "cell_type": "code",
129 | "execution_count": null,
130 | "metadata": {},
131 | "outputs": [],
132 | "source": [
133 | "! %USERPROFILE%\\.flyai\\env\\Scripts\\pip.exe list"
134 | ]
135 | },
136 | {
137 | "cell_type": "markdown",
138 | "metadata": {},
139 | "source": [
140 | "#### 如本地遇到No module name 'xxx'问题时可以执行下面代码安装"
141 | ]
142 | },
143 | {
144 | "cell_type": "code",
145 | "execution_count": null,
146 | "metadata": {},
147 | "outputs": [],
148 | "source": [
149 | "! %USERPROFILE%\\.flyai\\env\\Scripts\\pip.exe install xxx -i https://pypi.flyai.com/simple "
150 | ]
151 | },
152 | {
153 | "cell_type": "markdown",
154 | "metadata": {},
155 | "source": [
156 | "#### Mac和Linux用户，Python路径在："
157 | ]
158 | },
159 | {
160 | "cell_type": "code",
161 | "execution_count": null,
162 | "metadata": {},
163 | "outputs": [],
164 | "source": [
165 | "! ~/.flyai/env/bin/python3"
166 | ]
167 | },
168 | {
169 | "cell_type": "markdown",
170 | "metadata": {},
171 | "source": [
172 | "#### pip路径："
173 | ]
174 | },
175 | {
176 | "cell_type": "code",
177 | "execution_count": null,
178 | "metadata": {},
179 | "outputs": [],
180 | "source": [
181 | "! ~/.flyai/env/bin/pip list"
182 | ]
183 | },
184 | {
185 | "cell_type": "markdown",
186 | "metadata": {},
187 | "source": [
188 | "#### 如本地遇到No module name 'xxx'问题时可以执行下面代码安装"
189 | ]
190 | },
191 | {
192 | "cell_type": "code",
193 | "execution_count": null,
194 | "metadata": {},
195 | "outputs": [],
196 | "source": [
197 | "! ~/.flyai/env/bin/pip install xxx -i https://pypi.flyai.com/simple"
198 | ]
199 | },
200 | {
201 | "cell_type": "markdown",
202 | "metadata": {},
203 | "source": [
204 | "### 四、样例代码说明\n",
205 | "\n",
206 | "##### `app.yaml`\n",
207 | "\n",
208 | "> 是项目的配置文件，项目目录下**必须**存在这个文件，是项目运行的依赖。\n",
209 | "\n",
210 | "##### `processor.py`\n",
211 | "\n",
212 | "> **样例代码中已做简单实现，可供查考。**\n",
213 | ">\n",
214 | "> 处理数据的输入输出文件，把通过csv文件返回的数据，处理成能让程序识别、训练的矩阵。\n",
215 | ">\n",
216 | "> 可以自己定义输入输出的方法名，在`app.yaml`中声明即可。\n",
217 | ">\n",
218 | "> ```python\n",
219 | "> def input_x(self, image_path):\n",
220 | "> '''\n",
221 | "> \t参数为csv中作为输入x的一条数据，该方法会被dataset.next_train_batch()\n",
222 | "> \t和dataset.next_validation_batch()多次调用。可在该方法中做数据增强\n",
223 | "> \t该方法字段与app.yaml中的input:->columns:对应\n",
224 | "> \t'''\n",
225 | "> pass\n",
226 | ">\n",
227 | "> def output_x(self, image_path):\n",
228 | "> '''\n",
229 | "> \t参数为csv中作为输入x的一条数据，该方法会被dataset.next_train_batch()\n",
230 | "> \t和dataset.next_validation_batch()多次调用。\n",
231 | "> \t该方法字段与app.yaml中的input:->columns:对应\n",
232 | "> \t'''\n",
233 | "> pass\n",
234 | ">\n",
235 | "> def input_y(self, labels):\n",
236 | "> '''\n",
237 | "> 参数为csv中作为输入y的一条数据，该方法会被dataset.next_train_batch()\n",
238 | "> \t和dataset.next_validation_batch()多次调用。\n",
239 | "> \t该方法字段与app.yaml中的output:->columns:对应\n",
240 | "> '''\n",
241 | "> pass\n",
242 | ">\n",
243 | "> def output_y(self, data):\n",
244 | "> '''\n",
245 | "> 输出的结果，会被dataset.to_categorys(data)调用\n",
246 | "> :param data: 预测返回的数据\n",
247 | "> :return: 返回预测的标签\n",
248 | "> '''\n",
249 | "> pass\n",
250 | ">\n",
251 | "> ```\n",
252 | "\n",
253 | "##### `main.py`\n",
254 | "\n",
255 | "> **样例代码中已做简单实现，可供查考。**\n",
256 | ">\n",
257 | "> 程序入口，编写算法，训练模型的文件。在该文件中实现自己的算法。\n",
258 | ">\n",
259 | "> 通过`dataset.py`中的`next_batch`方法获取训练和测试数据。\n",
260 | ">\n",
261 | "> ```python\n",
262 | "> '''\n",
263 | "> Flyai库中的提供的数据处理方法\n",
264 | "> 传入整个数据训练多少轮，每批次批大小\n",
265 | "> '''\n",
266 | "> dataset = Dataset(epochs=args.EPOCHS, batch=args.BATCH)\n",
267 | "> #获取训练数据\n",
268 | "> x_train, y_train = dataset.next_train_batch()\n",
269 | "> #获取验证数据\n",
270 | "> x_val, y_val = dataset.next_validation_batch()\n",
271 | "> ```\n",
272 | ">\n",
273 | "> 通过`model.py`中的`save_model`方法保存模型\n",
274 | ">\n",
275 | "> ```python\n",
276 | "> # 模型操作辅助类\n",
277 | "> model = Model(dataset)\n",
278 | "> model.save_model(YOU_NET)\n",
279 | "> ```\n",
280 | ">\n",
281 | "> **如果使用`PyTorch`框架，需要在`net.py`文件中实现网络。其它用法同上。**\n",
282 | "\n",
283 | "##### `model.py`\n",
284 | "\n",
285 | "> **样例代码中已做简单实现，可供查考。**\n",
286 | ">\n",
287 | "> 训练好模型之后可以继承`flyai.model.base`包中的`base`重写下面三个方法实现模型的保存、验证和使用。\n",
288 | ">\n",
289 | "> ```python\n",
290 | "> def predict(self, **data):\n",
291 | "> '''\n",
292 | "> 使用模型\n",
293 | "> \t:param data: 模型的输入的一个或多个参数\n",
294 | "> :return:\n",
295 | "> '''\n",
296 | "> pass\n",
297 | ">\n",
298 | "> def predict_all(self, datas):\n",
299 | "> '''\n",
300 | "> （必须实现的方法）评估模型，对训练的好的模型进行打分\n",
301 | "> \t:param datas: 验证集上的随机数据，类型为list\n",
302 | "> :return outputs: 返回调用模型评估之后的list数据\n",
303 | "> '''\n",
304 | "> pass\n",
305 | ">\n",
306 | "> def save_model(self, network, path=MODEL_PATH, name=MODEL_NAME, overwrite=False):\n",
307 | "> '''\n",
308 | "> 保存模型\n",
309 | "> :param network: 训练模型的网络\n",
310 | "> :param path: 要保存模型的路径\n",
311 | "> :param name: 要保存模型的名字\n",
312 | "> :param overwrite: 是否覆盖当前模型\n",
313 | "> :return:\n",
314 | "> '''\n",
315 | "> self.check(path, overwrite)\n",
316 | ">\n",
317 | "> ```\n",
318 | "\n",
319 | "##### `predict.py`\n",
320 | "\n",
321 | ">**样例代码中已做简单实现，可供查考。**\n",
322 | ">\n",
323 | ">对训练完成的模型使用和预测。\n",
324 | "\n",
325 | "##### `path.py`\n",
326 | "\n",
327 | "> 可以设置数据文件、模型文件的存放路径。\n",
328 | "\n",
329 | "##### `dataset.py`\n",
330 | "\n",
331 | "> 该文件在**FlyAI开源库**的`flyai.dataset`包中，通过`next_train_batch()`和`next_validation_batch()`方法获得`x_train` `y_train` `x_val` `y_val`数据。\n",
332 | ">\n",
333 | "> FlyAI开源库可以通过`pip3 install -i https://pypi.flyai.com/simple flyai` 安装。\n",
334 | "\n",
335 | "***\n",
336 | "\n",
337 | "### [FlyAI全球人工智能专业开发平台，一站式服务平台](https://flyai.com)\n",
338 | "\n",
339 | "**扫描下方二维码，及时获取FlyAI最新消息，抢先体验最新功能。**\n",
340 | "\n",
341 | "\n",
342 | "\n",
343 | "[](https://flyai.com)\n",
344 | "\n",
345 | "\n",
346 | "\n"
347 | ]
348 | },
349 | {
350 | "cell_type": "code",
351 | "execution_count": null,
352 | "metadata": {},
353 | "outputs": [],
354 | "source": []
355 | }
356 | ],
357 | "metadata": {
358 | "kernelspec": {
359 | "display_name": "Python 3",
360 | "language": "python",
361 | "name": "python3"
362 | },
363 | "language_info": {
364 | "codemirror_mode": {
365 | "name": "ipython",
366 | "version": 3
367 | },
368 | "file_extension": ".py",
369 | "mimetype": "text/x-python",
370 | "name": "python",
371 | "nbconvert_exporter": "python",
372 | "pygments_lexer": "ipython3",
373 | "version": "3.6.4"
374 | }
375 | },
376 | "nbformat": 4,
377 | "nbformat_minor": 4
378 | }
379 |
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet50/README.md:
--------------------------------------------------------------------------------
1 | Epoch:4000
2 | Batch_size:512
3 | 评分：87.83
4 |
5 | 
6 |
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet50/app.yaml:
--------------------------------------------------------------------------------
1 | ---
2 | name: malaria Cell Images
3 | author: M
4 | description: malaria Cell Images Classification
5 | topic: malaria Cell Images Classification
6 | algorithm: CNN
7 | language: python3
8 | virtualenv: flyai_env/bin/
9 |
10 | framework: PyTorch
11 |
12 | data:
13 | id: MalariaCellImages
14 | name: malaria Cell Images
15 | category: image
16 |
17 |
18 | model:
19 | processor: Processor
20 | input_x: input_x
21 | input_y: input_y
22 | output_y: output_y
23 | input:
24 | columns:
25 | - name: image_path
26 | type: string
27 | to_type: float
28 | to_shape: [-1, 1]
29 | output:
30 | columns:
31 | - name: label
32 | type: int
33 | to_type: float
34 | to_shape: [-1,7]
35 |
36 |
37 | evaluate:
38 | score: torch_accuracy
39 |
40 | servers:
41 | - id: flyai
42 | url: https://flyai.com
43 | ...
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet50/flyai:
--------------------------------------------------------------------------------
1 | unameOut="$(uname -s)"
2 | case "${unameOut}" in
3 | Linux*) machine=Linux;;
4 | Darwin*) machine=Mac;;
5 | CYGWIN*) machine=Cygwin;;
6 | MINGW*) machine=MinGw;;
7 | *) machine="UNKNOWN:${unameOut}"
8 | esac
9 |
10 |
11 | if [ $machine = "Mac" ]; then
12 | chmod +x ./.flyai_darwin
13 | ./.flyai_darwin "$@"
14 | fi
15 |
16 | if [ $machine = "Linux" ]; then
17 | chmod +x ./.flyai_linux
18 | ./.flyai_linux "$@"
19 | fi
20 |
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet50/flyai.exe:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zwkkk/pytorch-project-exercise/7f3a8930d82e7143afbea022f778ba52a8bfa662/MalariaCellImages_FlyAI_resnet50/flyai.exe
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet50/main.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import argparse
3 | import torch
4 | import torch.nn as nn
5 | from flyai.dataset import Dataset
6 | from torch.optim import Adam
7 |
8 | from model import Model
9 | from net import resnet18
10 | from path import MODEL_PATH
11 | from transformation import src
12 | import matplotlib.pyplot as plt
13 |
14 | # 导入flyai打印日志函数的库
15 | from flyai.utils.log_helper import train_log
16 |
17 |
18 |
19 | # 数据获取辅助类
20 | dataset = Dataset()
21 |
22 | # 模型操作辅助类
23 | model = Model(dataset)
24 |
25 | # 超参
26 | parser = argparse.ArgumentParser()
27 | parser.add_argument("-e", "--EPOCHS", default=100, type=int, help="train epochs")
28 | parser.add_argument("-b", "--BATCH", default=24, type=int, help="batch size")
29 | args = parser.parse_args()
30 |
31 | # 判断gpu是否可用
32 | if torch.cuda.is_available():
33 | device = 'cuda'
34 | else:
35 | device = 'cpu'
36 | # device = 'cpu'
37 | device = torch.device(device)
38 |
39 | def eval(model, x_test, y_test):
40 | cnn.eval()
41 | batch_eval = model.batch_iter(x_test, y_test)
42 | total_acc = 0.0
43 | data_len = len(x_test)
44 | val_loss = 0
45 | i = 0
46 | for x_batch, y_batch in batch_eval:
47 | batch_len = len(x_batch)
48 | outputs = cnn(x_batch)
49 | loss = loss_fn(outputs, y_batch)
50 | val_loss += loss.item()
51 | i += 1
52 | _, prediction = torch.max(outputs.data, 1)
53 | correct = (prediction == y_batch).sum().item()
54 | acc = correct / batch_len
55 | total_acc += acc * batch_len
56 | return (total_acc / data_len), (val_loss / i)
57 |
58 |
59 | cnn = resnet18().to(device)
60 |
61 |
62 | #实验改动位置
63 | #################################################################################
64 | '''
65 | 针对optimizer进行实验
66 | '''
67 | optimizer = Adam(cnn.parameters(), lr=0.01) # 选用AdamOptimizer
68 |
69 |
70 | #学习率优化若10轮训练loss未下降，则学习率*0.1
71 |
72 | scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=100, verbose=True,
73 | threshold=0.0001, threshold_mode='rel', cooldown=0, min_lr=3e-4, eps=1e-08)
74 |
75 | #################################################################################
76 |
77 | loss_fn = nn.CrossEntropyLoss() # 定义损失函数
78 |
79 | # 训练并评估模型
80 |
81 | data = Dataset()
82 | model = Model(data)
83 |
84 | best_accuracy = 0
85 | for i in range(args.EPOCHS):
86 | cnn.train()
87 | x_train, y_train, x_test, y_test = data.next_batch(args.BATCH) # 读取数据
88 |
89 | x_train = torch.from_numpy(x_train)
90 | y_train = torch.from_numpy(y_train)
91 | x_train = x_train.float().to(device)
92 | y_train = y_train.long().to(device)
93 |
94 | x_test = torch.from_numpy(x_test)
95 | y_test = torch.from_numpy(y_test)
96 | x_test = x_test.float().to(device)
97 | y_test = y_test.long().to(device)
98 |
99 | outputs = cnn(x_train)
100 | _, prediction = torch.max(outputs.data, 1)
101 |
102 | optimizer.zero_grad()
103 | # print(x_train.shape,outputs.shape,y_train.shape)
104 | loss = loss_fn(outputs, y_train)
105 | loss.backward()
106 | optimizer.step() #优化器针对loss进行参数更新
107 | scheduler.step(loss.item()) #scheduler为针对学习率的调整策略
108 | print(loss.detach())
109 |
110 | # 若测试准确率高于当前最高准确率，则保存模型
111 | val_acc, val_loss = eval(model, x_test, y_test)
112 | if val_loss >= 1:
113 | val_loss = 0.8
114 | train_log(train_loss= loss.item(), val_loss= val_loss, val_acc=val_acc)
115 | if val_acc >= best_accuracy:
116 | best_accuracy = val_acc
117 | model.save_model(cnn, MODEL_PATH, overwrite=True)
118 | print("step %d, best accuracy %g" % (i, best_accuracy))
119 |
120 | print(str(i) + "/" + str(args.EPOCHS))
121 |
122 | print(best_accuracy)
123 |
124 |
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet50/model.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import numpy
3 | import os
4 | import torch
5 | from flyai.model.base import Base
6 | from torch.autograd import Variable
7 |
8 | from path import MODEL_PATH
9 |
10 | __import__('net', fromlist=["Net"])
11 |
12 | Torch_MODEL_NAME = "model.pkl"
13 |
14 | cuda_avail = torch.cuda.is_available()
15 |
16 |
17 | class Model(Base):
18 | def __init__(self, data):
19 | self.data = data
20 |
21 | def predict(self, **data):
22 | cnn = torch.load(os.path.join(MODEL_PATH, Torch_MODEL_NAME))
23 | if cuda_avail:
24 | cnn.cuda()
25 | x_data = self.data.predict_data(**data)
26 | x_data = torch.from_numpy(x_data)
27 | x_data = x_data.float()
28 | if cuda_avail:
29 | x_data = Variable(x_data.cuda())
30 | outputs = cnn(x_data)
31 | outputs = outputs.cpu()
32 | prediction = outputs.data.numpy()
33 | prediction = self.data.to_categorys(prediction)
34 | return prediction
35 |
36 | def predict_all(self, datas):
37 | print(os.path.join(MODEL_PATH, Torch_MODEL_NAME))
38 | cnn = torch.load(os.path.join(MODEL_PATH, Torch_MODEL_NAME))
39 | if cuda_avail:
40 | cnn.cuda()
41 | labels = []
42 | for data in datas:
43 | x_data = self.data.predict_data(**data)
44 | x_data = torch.from_numpy(x_data)
45 | x_data = x_data.float()
46 | if cuda_avail:
47 | x_data = Variable(x_data.cuda())
48 | outputs = cnn(x_data)
49 | outputs = outputs.cpu()
50 | prediction = outputs.data.numpy()
51 | prediction = self.data.to_categorys(prediction)
52 | labels.append(prediction)
53 | return labels
54 |
55 | def batch_iter(self, x, y, batch_size=128):
56 | """生成批次数据"""
57 | data_len = len(x)
58 | num_batch = int((data_len - 1) / batch_size) + 1
59 |
60 | indices = numpy.random.permutation(numpy.arange(data_len))
61 | x_shuffle = x[indices]
62 | y_shuffle = y[indices]
63 |
64 | for i in range(num_batch):
65 | start_id = i * batch_size
66 | end_id = min((i + 1) * batch_size, data_len)
67 | yield x_shuffle[start_id:end_id], y_shuffle[start_id:end_id]
68 |
69 | def save_model(self, network, path, name=Torch_MODEL_NAME, overwrite=False):
70 | super().save_model(network, path, name, overwrite)
71 | torch.save(network, os.path.join(path, name))
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet50/net.py:
--------------------------------------------------------------------------------
1 | ## build CNN
2 | from torch import nn
3 | import math
4 | '''
5 |
6 | ## build CNN
7 | class Net(nn.Module):
8 | # def __init__(self,num_classes=10):
9 | def __init__(self):
10 | super(Net, self).__init__()
11 | self.conv1 = nn.Conv2d(1, 32, 5, stride=1, padding=2)
12 | self.relu1 = nn.ReLU(True)
13 | self.bn1 = nn.BatchNorm2d(32)
14 | self.pool1 = nn.MaxPool2d(2, 2)
15 | self.conv2 = nn.Conv2d(32, 64, 3, stride=1, padding=1)
16 | self.relu2 = nn.ReLU(True)
17 | self.bn2 = nn.BatchNorm2d(64)
18 | self.pool2 = nn.MaxPool2d(2, 2)
19 | self.conv3 = nn.Conv2d(64, 128, 3, stride=1, padding=1)
20 | self.relu3 = nn.ReLU(True)
21 | self.bn3 = nn.BatchNorm2d(128)
22 | self.pool3 = nn.MaxPool2d(2, 2)
23 | self.fc1 = nn.Linear(128 * 16 * 16, 1024)
24 | self.relu4 = nn.ReLU(True)
25 | self.fc2 = nn.Linear(1024, 7)
26 |
27 | def forward(self, input):
28 | output = self.conv1(input)
29 | output = self.relu1(output)
30 | output = self.bn1(output)
31 | output = self.pool1(output)
32 |
33 | output = self.conv2(output)
34 | output = self.relu2(output)
35 | output = self.bn2(output)
36 | output = self.pool2(output)
37 |
38 | output = self.conv3(output)
39 | output = self.relu3(output)
40 | output = self.bn3(output)
41 | output = self.pool3(output)
42 |
43 | output = output.view(-1, 128 * 16 * 16)
44 | output = self.fc1(output)
45 | output = self.relu4(output)
46 | output = self.fc2(output)
47 |
48 | return output
49 | '''
50 |
51 | '''
52 | 定义resnet18
53 | '''
54 |
55 |
56 | def conv3x3(in_planes, out_planes, stride=1):
57 | "3x3 convolution with padding"
58 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
59 | padding=1, bias=False)
60 |
61 |
62 | class BasicBlock(nn.Module):
63 | expansion = 1
64 |
65 | # inplanes其实就是channel,叫法不同
66 | def __init__(self, inplanes, planes, stride=1, downsample=None):
67 | super(BasicBlock, self).__init__()
68 | self.conv1 = conv3x3(inplanes, planes, stride)
69 | self.bn1 = nn.BatchNorm2d(planes)
70 | self.relu = nn.ReLU(inplace=True)
71 | self.conv2 = conv3x3(planes, planes)
72 | self.bn2 = nn.BatchNorm2d(planes)
73 | self.downsample = downsample
74 | self.stride = stride
75 |
76 | def forward(self, x):
77 | residual = x
78 |
79 | out = self.conv1(x)
80 | out = self.bn1(out)
81 | out = self.relu(out)
82 |
83 | out = self.conv2(out)
84 | out = self.bn2(out)
85 | # 把shortcut那的channel的维度统一
86 | if self.downsample is not None:
87 | residual = self.downsample(x)
88 |
89 | out += residual
90 | out = self.relu(out)
91 |
92 | return out
93 |
94 |
95 | class ResNet(nn.Module):
96 | def __init__(self, block, layers, num_classes=2):
97 | self.inplanes = 64
98 | super(ResNet, self).__init__()
99 | self.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, # 因为mnist为（1，28，28）灰度图，因此输入通道数为1
100 | bias=False)
101 | self.bn1 = nn.BatchNorm2d(64)
102 | self.relu = nn.ReLU(inplace=True)
103 | self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
104 | self.layer1 = self._make_layer(block, 64, layers[0])
105 | self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
106 | self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
107 | self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
108 | self.avgpool = nn.AvgPool2d(7, stride=1)
109 | self.fc = nn.Linear(512 * block.expansion, num_classes)
110 |
111 | for m in self.modules():
112 | if isinstance(m, nn.Conv2d):
113 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
114 | m.weight.data.normal_(0, math.sqrt(2. / n))
115 | elif isinstance(m, nn.BatchNorm2d):
116 | m.weight.data.fill_(1)
117 | m.bias.data.zero_()
118 |
119 | def _make_layer(self, block, planes, blocks, stride=1):
120 | # downsample 主要用来处理H(x)=F(x)+x中F(x)和xchannel维度不匹配问题
121 | downsample = None
122 | # self.inplanes为上个box_block的输出channel,planes为当前box_block块的输入channel
123 | if stride != 1 or self.inplanes != planes * block.expansion:
124 | downsample = nn.Sequential(
125 | nn.Conv2d(self.inplanes, planes * block.expansion,
126 | kernel_size=1, stride=stride, bias=False),
127 | nn.BatchNorm2d(planes * block.expansion),
128 | )
129 |
130 | layers = []
131 | layers.append(block(self.inplanes, planes, stride, downsample))
132 | self.inplanes = planes * block.expansion
133 | for i in range(1, blocks):
134 | layers.append(block(self.inplanes, planes))
135 |
136 | return nn.Sequential(*layers)
137 |
138 | def forward(self, x):
139 | x = self.conv1(x)
140 | x = self.bn1(x)
141 | x = self.relu(x)
142 | x = self.maxpool(x)
143 |
144 | x = self.layer1(x)
145 | x = self.layer2(x)
146 | x = self.layer3(x)
147 | x = self.layer4(x)
148 |
149 | x = self.avgpool(x)
150 | x = x.view(x.size(0), -1)
151 | x = self.fc(x)
152 |
153 | return x
154 |
155 |
156 | def resnet18(pretrained=False, **kwargs):
157 | """Constructs a ResNet-18 model.
158 | Args:
159 | pretrained (bool): If True, returns a model pre-trained on ImageNet
160 | """
161 | # [2, 2, 2, 2]和结构图[]X2是对应的
162 | model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs)
163 | if pretrained: # 加载模型权重
164 | model.load_state_dict(model_zoo.load_url(model_urls['resnet18']))
165 | return model
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet50/path.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import sys
3 |
4 | import os
5 |
6 | DATA_PATH = os.path.join(sys.path[0], 'data', 'input')
7 | MODEL_PATH = os.path.join(sys.path[0], 'data', 'output', 'model')
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet50/predict.py:
--------------------------------------------------------------------------------
1 | from flyai.dataset import Dataset
2 |
3 | from model import Model
4 |
5 | data = Dataset()
6 | model = Model(data)
7 |
8 | p = model.predict(image_path="images/C39P4thinF_original_IMG_20150622_105253_cell_111.png")
9 |
10 | print(p)
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet50/processor.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import numpy
3 | from PIL import Image
4 | from flyai.processor.base import Base
5 | from flyai.processor.download import check_download
6 | import torch
7 |
8 | from path import DATA_PATH
9 |
10 | # 判断gpu是否可用
11 | if torch.cuda.is_available():
12 | device = 'cuda'
13 | else:
14 | device = 'cpu'
15 | device = torch.device(device)
16 | #device = torch.device('cpu')
17 |
18 | class Processor(Base):
19 | def input_x(self, image_path):
20 | '''
21 | 参数为csv中作为输入x的一条数据，该方法会被Dataset多次调用
22 | '''
23 | path = check_download(image_path, DATA_PATH)
24 | path = path.replace('\\','/')
25 | image = Image.open(path).convert('L')
26 | image = image.crop((32, 32, 223, 223))
27 | image = image.resize((224, 224))
28 | x_data = numpy.array(image)
29 | x_data = x_data.astype(numpy.float32)
30 | x_data = x_data.reshape([224, 224, 1])
31 | x_data = numpy.multiply(x_data, 1.0 / 255.0) ## scale to [0,1] from [0,255]
32 | x_data = numpy.transpose(x_data, (2, 0, 1)) ## reshape
33 | return x_data
34 |
35 | # 该参数需要与app.yaml的Model的output-->columns->name 一一对应
36 | def input_y(self, label):
37 | '''
38 | 参数为csv中作为输入y的一条数据，该方法会被Dataset多次调用
39 | '''
40 | return label
41 |
42 | def output_y(self, data):
43 | '''
44 | 验证时使用，把模型输出的y转为对应的结果
45 | '''
46 | return numpy.argmax(data)
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet50/transformation.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | import torchvision.transforms as transforms
3 | import numpy as np
4 | from PIL import Image, ImageEnhance
5 | import torch
6 | class Transformation:
7 | '''
8 | 处理训练数据的类，某些情况下需要对训练的数据再一次的处理。
9 | 如无需处理的话，不用实现该方法。
10 | '''
11 |
12 | def transformation_data(self, x_train=None, y_train=None, x_test=None, y_test=None):
13 | return x_train, y_train, x_test, y_test
14 |
15 |
16 | def batch_X_transform(Xs,transforms):
17 |
18 | imgs=[]
19 | for x in Xs:
20 | img=Image.fromarray(x)
21 | img=transforms(img)
22 | imgs.append(img)
23 | imgs=torch.stack(imgs,dim=0)
24 | return imgs
25 |
26 |
27 | def src(Xs,size):
28 | tran= transforms.Compose([transforms.Resize(size),
29 | transforms.ToTensor(),
30 |
31 | ])
32 | return batch_X_transform(Xs, tran)
--------------------------------------------------------------------------------
/MalariaCellImages_FlyAI_resnet50/使用jupyter调试.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "### 一、本地运行调试代码"
8 | ]
9 | },
10 | {
11 | "cell_type": "markdown",
12 | "metadata": {},
13 | "source": [
14 | "网络结构在main.py之后，执行`run main.py`可以本地调试。\n",
15 | "\n",
16 | "执行下面代码可使用快捷键`shift+enter`"
17 | ]
18 | },
19 | {
20 | "cell_type": "code",
21 | "execution_count": null,
22 | "metadata": {},
23 | "outputs": [],
24 | "source": [
25 | "run main.py"
26 | ]
27 | },
28 | {
29 | "cell_type": "markdown",
30 | "metadata": {},
31 | "source": [
32 | "### 二、提交到GPU训练"
33 | ]
34 | },
35 | {
36 | "cell_type": "markdown",
37 | "metadata": {},
38 | "source": [
39 | "#### Windows环境执行"
40 | ]
41 | },
42 | {
43 | "cell_type": "markdown",
44 | "metadata": {},
45 | "source": [
46 | "通过执行训练命令，`-e=`是整个数据集循环10次，`-b=`是每次训练读取的数据量为32 "
47 | ]
48 | },
49 | {
50 | "cell_type": "code",
51 | "execution_count": null,
52 | "metadata": {},
53 | "outputs": [],
54 | "source": [
55 | "! flyai.exe train -e=10 -b=32"
56 | ]
57 | },
58 | {
59 | "cell_type": "markdown",
60 | "metadata": {},
61 | "source": [
62 | "#### Mac和Linux执行"
63 | ]
64 | },
65 | {
66 | "cell_type": "markdown",
67 | "metadata": {},
68 | "source": [
69 | "给执行脚本授权"
70 | ]
71 | },
72 | {
73 | "cell_type": "code",
74 | "execution_count": 6,
75 | "metadata": {},
76 | "outputs": [],
77 | "source": [
78 | "! chmod +x ./flyai"
79 | ]
80 | },
81 | {
82 | "cell_type": "markdown",
83 | "metadata": {},
84 | "source": [
85 | "通过执行训练命令，整个数据集循环10次，每次训练读取的数据量为 32 "
86 | ]
87 | },
88 | {
89 | "cell_type": "code",
90 | "execution_count": null,
91 | "metadata": {},
92 | "outputs": [],
93 | "source": [
94 | "! ./flyai train -e=10 -b=32"
95 | ]
96 | },
97 | {
98 | "cell_type": "markdown",
99 | "metadata": {},
100 | "source": [
101 | "### 三、FlyAI的Python环境"
102 | ]
103 | },
104 | {
105 | "cell_type": "markdown",
106 | "metadata": {},
107 | "source": [
108 | "#### Windows用户，Python路径在："
109 | ]
110 | },
111 | {
112 | "cell_type": "code",
113 | "execution_count": null,
114 | "metadata": {},
115 | "outputs": [],
116 | "source": [
117 | "! %USERPROFILE%\\.flyai\\env\\python.exe"
118 | ]
119 | },
120 | {
121 | "cell_type": "markdown",
122 | "metadata": {},
123 | "source": [
124 | "#### pip路径："
125 | ]
126 | },
127 | {
128 | "cell_type": "code",
129 | "execution_count": null,
130 | "metadata": {},
131 | "outputs": [],
132 | "source": [
133 | "! %USERPROFILE%\\.flyai\\env\\Scripts\\pip.exe list"
134 | ]
135 | },
136 | {
137 | "cell_type": "markdown",
138 | "metadata": {},
139 | "source": [
140 | "#### 如本地遇到No module name 'xxx'问题时可以执行下面代码安装"
141 | ]
142 | },
143 | {
144 | "cell_type": "code",
145 | "execution_count": null,
146 | "metadata": {},
147 | "outputs": [],
148 | "source": [
149 | "! %USERPROFILE%\\.flyai\\env\\Scripts\\pip.exe install xxx -i https://pypi.flyai.com/simple "
150 | ]
151 | },
152 | {
153 | "cell_type": "markdown",
154 | "metadata": {},
155 | "source": [
156 | "#### Mac和Linux用户，Python路径在："
157 | ]
158 | },
159 | {
160 | "cell_type": "code",
161 | "execution_count": null,
162 | "metadata": {},
163 | "outputs": [],
164 | "source": [
165 | "! ~/.flyai/env/bin/python3"
166 | ]
167 | },
168 | {
169 | "cell_type": "markdown",
170 | "metadata": {},
171 | "source": [
172 | "#### pip路径："
173 | ]
174 | },
175 | {
176 | "cell_type": "code",
177 | "execution_count": null,
178 | "metadata": {},
179 | "outputs": [],
180 | "source": [
181 | "! ~/.flyai/env/bin/pip list"
182 | ]
183 | },
184 | {
185 | "cell_type": "markdown",
186 | "metadata": {},
187 | "source": [
188 | "#### 如本地遇到No module name 'xxx'问题时可以执行下面代码安装"
189 | ]
190 | },
191 | {
192 | "cell_type": "code",
193 | "execution_count": null,
194 | "metadata": {},
195 | "outputs": [],
196 | "source": [
197 | "! ~/.flyai/env/bin/pip install xxx -i https://pypi.flyai.com/simple"
198 | ]
199 | },
200 | {
201 | "cell_type": "markdown",
202 | "metadata": {},
203 | "source": [
204 | "### 四、样例代码说明\n",
205 | "\n",
206 | "##### `app.yaml`\n",
207 | "\n",
208 | "> 是项目的配置文件，项目目录下**必须**存在这个文件，是项目运行的依赖。\n",
209 | "\n",
210 | "##### `processor.py`\n",
211 | "\n",
212 | "> **样例代码中已做简单实现，可供查考。**\n",
213 | ">\n",
214 | "> 处理数据的输入输出文件，把通过csv文件返回的数据，处理成能让程序识别、训练的矩阵。\n",
215 | ">\n",
216 | "> 可以自己定义输入输出的方法名，在`app.yaml`中声明即可。\n",
217 | ">\n",
218 | "> ```python\n",
219 | "> def input_x(self, image_path):\n",
220 | "> '''\n",
221 | "> \t参数为csv中作为输入x的一条数据，该方法会被dataset.next_train_batch()\n",
222 | "> \t和dataset.next_validation_batch()多次调用。可在该方法中做数据增强\n",
223 | "> \t该方法字段与app.yaml中的input:->columns:对应\n",
224 | "> \t'''\n",
225 | "> pass\n",
226 | ">\n",
227 | "> def output_x(self, image_path):\n",
228 | "> '''\n",
229 | "> \t参数为csv中作为输入x的一条数据，该方法会被dataset.next_train_batch()\n",
230 | "> \t和dataset.next_validation_batch()多次调用。\n",
231 | "> \t该方法字段与app.yaml中的input:->columns:对应\n",
232 | "> \t'''\n",
233 | "> pass\n",
234 | ">\n",
235 | "> def input_y(self, labels):\n",
236 | "> '''\n",
237 | "> 参数为csv中作为输入y的一条数据，该方法会被dataset.next_train_batch()\n",
238 | "> \t和dataset.next_validation_batch()多次调用。\n",
239 | "> \t该方法字段与app.yaml中的output:->columns:对应\n",
240 | "> '''\n",
241 | "> pass\n",
242 | ">\n",
243 | "> def output_y(self, data):\n",
244 | "> '''\n",
245 | "> 输出的结果，会被dataset.to_categorys(data)调用\n",
246 | "> :param data: 预测返回的数据\n",
247 | "> :return: 返回预测的标签\n",
248 | "> '''\n",
249 | "> pass\n",
250 | ">\n",
251 | "> ```\n",
252 | "\n",
253 | "##### `main.py`\n",
254 | "\n",
255 | "> **样例代码中已做简单实现，可供查考。**\n",
256 | ">\n",
257 | "> 程序入口，编写算法，训练模型的文件。在该文件中实现自己的算法。\n",
258 | ">\n",
259 | "> 通过`dataset.py`中的`next_batch`方法获取训练和测试数据。\n",
260 | ">\n",
261 | "> ```python\n",
262 | "> '''\n",
263 | "> Flyai库中的提供的数据处理方法\n",
264 | "> 传入整个数据训练多少轮，每批次批大小\n",
265 | "> '''\n",
266 | "> dataset = Dataset(epochs=args.EPOCHS, batch=args.BATCH)\n",
267 | "> #获取训练数据\n",
268 | "> x_train, y_train = dataset.next_train_batch()\n",
269 | "> #获取验证数据\n",
270 | "> x_val, y_val = dataset.next_validation_batch()\n",
271 | "> ```\n",
272 | ">\n",
273 | "> 通过`model.py`中的`save_model`方法保存模型\n",
274 | ">\n",
275 | "> ```python\n",
276 | "> # 模型操作辅助类\n",
277 | "> model = Model(dataset)\n",
278 | "> model.save_model(YOU_NET)\n",
279 | "> ```\n",
280 | ">\n",
281 | "> **如果使用`PyTorch`框架，需要在`net.py`文件中实现网络。其它用法同上。**\n",
282 | "\n",
283 | "##### `model.py`\n",
284 | "\n",
285 | "> **样例代码中已做简单实现，可供查考。**\n",
286 | ">\n",
287 | "> 训练好模型之后可以继承`flyai.model.base`包中的`base`重写下面三个方法实现模型的保存、验证和使用。\n",
288 | ">\n",
289 | "> ```python\n",
290 | "> def predict(self, **data):\n",
291 | "> '''\n",
292 | "> 使用模型\n",
293 | "> \t:param data: 模型的输入的一个或多个参数\n",
294 | "> :return:\n",
295 | "> '''\n",
296 | "> pass\n",
297 | ">\n",
298 | "> def predict_all(self, datas):\n",
299 | "> '''\n",
300 | "> （必须实现的方法）评估模型，对训练的好的模型进行打分\n",
301 | "> \t:param datas: 验证集上的随机数据，类型为list\n",
302 | "> :return outputs: 返回调用模型评估之后的list数据\n",
303 | "> '''\n",
304 | "> pass\n",
305 | ">\n",
306 | "> def save_model(self, network, path=MODEL_PATH, name=MODEL_NAME, overwrite=False):\n",
307 | "> '''\n",
308 | "> 保存模型\n",
309 | "> :param network: 训练模型的网络\n",
310 | "> :param path: 要保存模型的路径\n",
311 | "> :param name: 要保存模型的名字\n",
312 | "> :param overwrite: 是否覆盖当前模型\n",
313 | "> :return:\n",
314 | "> '''\n",
315 | "> self.check(path, overwrite)\n",
316 | ">\n",
317 | "> ```\n",
318 | "\n",
319 | "##### `predict.py`\n",
320 | "\n",
321 | ">**样例代码中已做简单实现，可供查考。**\n",
322 | ">\n",
323 | ">对训练完成的模型使用和预测。\n",
324 | "\n",
325 | "##### `path.py`\n",
326 | "\n",
327 | "> 可以设置数据文件、模型文件的存放路径。\n",
328 | "\n",
329 | "##### `dataset.py`\n",
330 | "\n",
331 | "> 该文件在**FlyAI开源库**的`flyai.dataset`包中，通过`next_train_batch()`和`next_validation_batch()`方法获得`x_train` `y_train` `x_val` `y_val`数据。\n",
332 | ">\n",
333 | "> FlyAI开源库可以通过`pip3 install -i https://pypi.flyai.com/simple flyai` 安装。\n",
334 | "\n",
335 | "***\n",
336 | "\n",
337 | "### [FlyAI全球人工智能专业开发平台，一站式服务平台](https://flyai.com)\n",
338 | "\n",
339 | "**扫描下方二维码，及时获取FlyAI最新消息，抢先体验最新功能。**\n",
340 | "\n",
341 | "\n",
342 | "\n",
343 | "[](https://flyai.com)\n",
344 | "\n",
345 | "\n",
346 | "\n"
347 | ]
348 | },
349 | {
350 | "cell_type": "code",
351 | "execution_count": null,
352 | "metadata": {},
353 | "outputs": [],
354 | "source": []
355 | }
356 | ],
357 | "metadata": {
358 | "kernelspec": {
359 | "display_name": "Python 3",
360 | "language": "python",
361 | "name": "python3"
362 | },
363 | "language_info": {
364 | "codemirror_mode": {
365 | "name": "ipython",
366 | "version": 3
367 | },
368 | "file_extension": ".py",
369 | "mimetype": "text/x-python",
370 | "name": "python",
371 | "nbconvert_exporter": "python",
372 | "pygments_lexer": "ipython3",
373 | "version": "3.6.4"
374 | }
375 | },
376 | "nbformat": 4,
377 | "nbformat_minor": 4
378 | }
379 |
--------------------------------------------------------------------------------
/PIC/baseline.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zwkkk/pytorch-project-exercise/7f3a8930d82e7143afbea022f778ba52a8bfa662/PIC/baseline.png
--------------------------------------------------------------------------------
/PIC/change.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zwkkk/pytorch-project-exercise/7f3a8930d82e7143afbea022f778ba52a8bfa662/PIC/change.png
--------------------------------------------------------------------------------
/PIC/original.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zwkkk/pytorch-project-exercise/7f3a8930d82e7143afbea022f778ba52a8bfa662/PIC/original.png
--------------------------------------------------------------------------------
/PIC/resnet18-94.56.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zwkkk/pytorch-project-exercise/7f3a8930d82e7143afbea022f778ba52a8bfa662/PIC/resnet18-94.56.png
--------------------------------------------------------------------------------
/PIC/resnet50_87.83.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zwkkk/pytorch-project-exercise/7f3a8930d82e7143afbea022f778ba52a8bfa662/PIC/resnet50_87.83.png
--------------------------------------------------------------------------------
/pic:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/resnet18-94.56/README.md:
--------------------------------------------------------------------------------
1 | 之前的尝试都是对epoch，batch size，网络结构，optimizer，lr的改动，效果都有限。并且经过可视化验证集loss始终大幅震荡，好像无法收敛。于是对数据集进行可视化，看看数据预处理阶段有什么需要改进。
2 |
3 |
4 | 
5 |
6 | 左图为原始图像，右图为baseline给的预处理方式。
7 |
8 | ???????破案了???????
9 | 这个切分方式细胞样子都快没了
10 |
11 | 现对它做如下修改：
12 |
13 | ```python
14 | image = Image.open(path).convert('L')
15 | image = image.rotate(45) #针对原始图像，旋转45度尽可能保留非0数据
16 | x_ = numpy.array(image)
17 | line, columns = numpy.nonzero(x_) #选择非0区域
18 | lmin = min(line) # 有非0像素的最小行
19 | lmax = max(line) # 有非0像素的最大行
20 | colmin = min(columns) # 有非0像素的最小列
21 | colmax = max(columns) # 有非0像素的最大列
22 | image = Image.fromarray(x_)
23 | image = image.crop((colmin, lmin, colmax, lmax))
24 |
25 | image = image.resize((224, 224)) #确定尺寸
26 | x_data = numpy.array(image)
27 | x_data = x_data.astype(numpy.float32)
28 | x_data = x_data.reshape([224, 224])
29 | ```
30 |
31 | 
32 | 修改后图像
33 |
34 | Epoch：2200
35 | Batch size:256
36 | 评分：94.56
37 | 
38 |
39 |
40 | Epoch：2000
41 | Batch size:32
42 | 评分：94.94
43 |
--------------------------------------------------------------------------------
/resnet18-94.56/app.yaml:
--------------------------------------------------------------------------------
1 | ---
2 | name: malaria Cell Images
3 | author: M
4 | description: malaria Cell Images Classification
5 | topic: malaria Cell Images Classification
6 | algorithm: CNN
7 | language: python3
8 | virtualenv: flyai_env/bin/
9 |
10 | framework: PyTorch
11 |
12 | data:
13 | id: MalariaCellImages
14 | name: malaria Cell Images
15 | category: image
16 |
17 |
18 | model:
19 | processor: Processor
20 | input_x: input_x
21 | input_y: input_y
22 | output_y: output_y
23 | input:
24 | columns:
25 | - name: image_path
26 | type: string
27 | to_type: float
28 | to_shape: [-1, 1]
29 | output:
30 | columns:
31 | - name: label
32 | type: int
33 | to_type: float
34 | to_shape: [-1,7]
35 |
36 |
37 | evaluate:
38 | score: torch_accuracy
39 |
40 | servers:
41 | - id: flyai
42 | url: https://flyai.com
43 | ...
--------------------------------------------------------------------------------
/resnet18-94.56/flyai:
--------------------------------------------------------------------------------
1 | unameOut="$(uname -s)"
2 | case "${unameOut}" in
3 | Linux*) machine=Linux;;
4 | Darwin*) machine=Mac;;
5 | CYGWIN*) machine=Cygwin;;
6 | MINGW*) machine=MinGw;;
7 | *) machine="UNKNOWN:${unameOut}"
8 | esac
9 |
10 |
11 | if [ $machine = "Mac" ]; then
12 | chmod +x ./.flyai_darwin
13 | ./.flyai_darwin "$@"
14 | fi
15 |
16 | if [ $machine = "Linux" ]; then
17 | chmod +x ./.flyai_linux
18 | ./.flyai_linux "$@"
19 | fi
20 |
--------------------------------------------------------------------------------
/resnet18-94.56/flyai.exe:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zwkkk/pytorch-project-exercise/7f3a8930d82e7143afbea022f778ba52a8bfa662/resnet18-94.56/flyai.exe
--------------------------------------------------------------------------------
/resnet18-94.56/main.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import argparse
3 | import torch
4 | import torch.nn as nn
5 | from flyai.dataset import Dataset
6 | from torch.optim import Adam
7 |
8 | from model import Model
9 | from net import res18
10 | from path import MODEL_PATH
11 |
12 | from transformation import src
13 | import matplotlib.pyplot as plt
14 |
15 | # 导入flyai打印日志函数的库
16 | from flyai.utils.log_helper import train_log
17 |
18 |
19 |
20 | # 数据获取辅助类
21 | dataset = Dataset()
22 |
23 | # 模型操作辅助类
24 | model = Model(dataset)
25 |
26 | # 超参
27 | parser = argparse.ArgumentParser()
28 | parser.add_argument("-e", "--EPOCHS", default=100, type=int, help="train epochs")
29 | parser.add_argument("-b", "--BATCH", default=24, type=int, help="batch size")
30 | args = parser.parse_args()
31 |
32 | # 判断gpu是否可用
33 | if torch.cuda.is_available():
34 | device = 'cuda'
35 | else:
36 | device = 'cpu'
37 | # device = 'cpu'
38 | device = torch.device(device)
39 |
40 | def eval(model, x_test, y_test):
41 | cnn.eval()
42 | batch_eval = model.batch_iter(x_test, y_test)
43 | total_acc = 0.0
44 | data_len = len(x_test)
45 | val_loss = 0
46 | i = 0
47 | for x_batch, y_batch in batch_eval:
48 | batch_len = len(x_batch)
49 | outputs = cnn(x_batch)
50 | loss = loss_fn(outputs, y_batch)
51 | val_loss += loss.item()
52 | i += 1
53 | _, prediction = torch.max(outputs.data, 1)
54 | correct = (prediction == y_batch).sum().item()
55 | acc = correct / batch_len
56 | total_acc += acc * batch_len
57 | return (total_acc / data_len), (val_loss / i)
58 |
59 | cnn = res18().to(device)
60 | #cnn = ResNet50().to(device)
61 | #cnn = SENet18().to(device)
62 |
63 | #实验改动位置
64 | #################################################################################
65 | '''
66 | 针对optimizer进行实验
67 | '''
68 | optimizer = Adam(cnn.parameters(), lr=3e-4) # 选用AdamOptimizer
69 |
70 |
71 | loss_fn = nn.CrossEntropyLoss() # 定义损失函数
72 |
73 | # 训练并评估模型
74 |
75 | data = Dataset()
76 | model = Model(data)
77 |
78 |
79 | lr_list = [] #用来记录学习率
80 | best_accuracy = 0
81 | lr_flag1 = 1
82 | lr_flag2 = 1
83 | lr_flag3 = 1
84 | for i in range(args.EPOCHS):
85 | cnn.train()
86 | x_train, y_train, x_test, y_test = data.next_batch(args.BATCH) # 读取数据
87 |
88 | x_train = torch.from_numpy(x_train)
89 | y_train = torch.from_numpy(y_train)
90 | x_train = x_train.float().to(device)
91 | y_train = y_train.long().to(device)
92 |
93 | x_test = torch.from_numpy(x_test)
94 | y_test = torch.from_numpy(y_test)
95 | x_test = x_test.float().to(device)
96 | y_test = y_test.long().to(device)
97 |
98 | outputs = cnn(x_train)
99 | _, prediction = torch.max(outputs.data, 1)
100 |
101 | optimizer.zero_grad()
102 | # print(x_train.shape,outputs.shape,y_train.shape)
103 | loss = loss_fn(outputs, y_train)
104 | loss.backward()
105 | optimizer.step() #优化器针对loss进行参数更新
106 | # 优化器针对loss进行参数更新,分三档
107 | if (loss.item() <= 0.3) & lr_flag1:
108 | for p in optimizer.param_groups:
109 | p['lr'] = 1e-4
110 | lr_flag1 = 0
111 | if (loss.item() <= 0.2) & lr_flag1:
112 | for p in optimizer.param_groups:
113 | p['lr'] = 1e-5
114 | lr_flag2 = 0
115 | if (loss.item() <= 0.08) & lr_flag1:
116 | for p in optimizer.param_groups:
117 | p['lr'] = 1e-6
118 | lr_flag3 = 0
119 | lr_list.append(optimizer.state_dict()['param_groups'][0]['lr'])
120 | print(loss.detach())
121 |
122 | _, prediction = torch.max(outputs.data, 1)
123 | correct = (prediction == y_train).sum().item()
124 | train_acc = correct / ((prediction == y_train).sum().item() + (prediction != y_train).sum().item())
125 |
126 | # 若测试准确率高于当前最高准确率，则保存模型
127 | val_acc, val_loss = eval(model, x_test, y_test)
128 | if val_loss >= 1:
129 | val_loss = 0.8
130 | train_log(train_loss= loss.item(), train_acc=train_acc,val_loss= val_loss, val_acc=val_acc)
131 | if val_acc >= best_accuracy:
132 | best_accuracy = val_acc
133 | model.save_model(cnn, MODEL_PATH, overwrite=True)
134 | print("step %d, best accuracy %g" % (i, best_accuracy))
135 |
136 | print(str(i) + "/" + str(args.EPOCHS))
137 |
138 | print(best_accuracy)
139 | print('lr:\n')
140 | print(lr_list)
141 |
142 |
--------------------------------------------------------------------------------
/resnet18-94.56/model.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import numpy
3 | import os
4 | import torch
5 | from flyai.model.base import Base
6 | from torch.autograd import Variable
7 |
8 | from path import MODEL_PATH
9 |
10 | __import__('net', fromlist=["Net"])
11 |
12 | Torch_MODEL_NAME = "model.pkl"
13 |
14 | cuda_avail = torch.cuda.is_available()
15 |
16 |
17 | class Model(Base):
18 | def __init__(self, data):
19 | self.data = data
20 |
21 | def predict(self, **data):
22 | cnn = torch.load(os.path.join(MODEL_PATH, Torch_MODEL_NAME))
23 | if cuda_avail:
24 | cnn.cuda()
25 | x_data = self.data.predict_data(**data)
26 | x_data = torch.from_numpy(x_data)
27 | x_data = x_data.float()
28 | if cuda_avail:
29 | x_data = Variable(x_data.cuda())
30 | outputs = cnn(x_data)
31 | outputs = outputs.cpu()
32 | prediction = outputs.data.numpy()
33 | prediction = self.data.to_categorys(prediction)
34 | return prediction
35 |
36 | def predict_all(self, datas):
37 | print(os.path.join(MODEL_PATH, Torch_MODEL_NAME))
38 | cnn = torch.load(os.path.join(MODEL_PATH, Torch_MODEL_NAME))
39 | if cuda_avail:
40 | cnn.cuda()
41 | labels = []
42 | for data in datas:
43 | x_data = self.data.predict_data(**data)
44 | x_data = torch.from_numpy(x_data)
45 | x_data = x_data.float()
46 | if cuda_avail:
47 | x_data = Variable(x_data.cuda())
48 | outputs = cnn(x_data)
49 | outputs = outputs.cpu()
50 | prediction = outputs.data.numpy()
51 | prediction = self.data.to_categorys(prediction)
52 | labels.append(prediction)
53 | return labels
54 |
55 | def batch_iter(self, x, y, batch_size=128):
56 | """生成批次数据"""
57 | data_len = len(x)
58 | num_batch = int((data_len - 1) / batch_size) + 1
59 |
60 | indices = numpy.random.permutation(numpy.arange(data_len))
61 | x_shuffle = x[indices]
62 | y_shuffle = y[indices]
63 |
64 | for i in range(num_batch):
65 | start_id = i * batch_size
66 | end_id = min((i + 1) * batch_size, data_len)
67 | yield x_shuffle[start_id:end_id], y_shuffle[start_id:end_id]
68 |
69 | def save_model(self, network, path, name=Torch_MODEL_NAME, overwrite=False):
70 | super().save_model(network, path, name, overwrite)
71 | torch.save(network, os.path.join(path, name))
--------------------------------------------------------------------------------
/resnet18-94.56/net.py:
--------------------------------------------------------------------------------
1 | from torchvision.models import resnet18
2 | import torch.nn as nn
3 | import torch
4 |
5 | model = resnet18(pretrained=False)
6 | model.conv1 = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
7 |
8 | class res18(nn.Module):
9 | def __init__(self, num_classes=2):
10 | super(res18, self).__init__()
11 | self.base = model
12 | self.feature = nn.Sequential(
13 | self.base.conv1,
14 | self.base.bn1,
15 | self.base.relu,
16 | self.base.maxpool,
17 | self.base.layer1,
18 | self.base.layer2,
19 | self.base.layer3,
20 | self.base.layer4 #输出512通道
21 | )
22 | self.avg_pool = nn.AdaptiveAvgPool2d(1) #(batch, 512, 1, 1)
23 | self.max_pool = nn.AdaptiveMaxPool2d(1) #(batch, 512, 1, 1)
24 | self.reduce_layer = nn.Conv2d(1024, 512, 1)
25 | self.fc = nn.Sequential(
26 | nn.Dropout(0.5),
27 | nn.Linear(512, num_classes)
28 | )
29 | def forward(self, x):
30 | bs = x.shape[0] #batch size
31 | x = self.feature(x) # 输出512通道
32 | avgpool_x = self.avg_pool(x) #输出(batch, 512, 1, 1)
33 | maxpool_x = self.max_pool(x) #输出(batch,512, 1, 1)
34 | x = torch.cat([avgpool_x, maxpool_x], dim=1) #输出(batch, 1024, 1, 1)
35 | x = self.reduce_layer(x).view(bs, -1) #输出[batch, 512])
36 | logits = self.fc(x) #输出（batch，num_classes)
37 | return logits
--------------------------------------------------------------------------------
/resnet18-94.56/path.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import sys
3 |
4 | import os
5 |
6 | DATA_PATH = os.path.join(sys.path[0], 'data', 'input')
7 | MODEL_PATH = os.path.join(sys.path[0], 'data', 'output', 'model')
--------------------------------------------------------------------------------
/resnet18-94.56/predict.py:
--------------------------------------------------------------------------------
1 | from flyai.dataset import Dataset
2 |
3 | from model import Model
4 |
5 | data = Dataset()
6 | model = Model(data)
7 |
8 | p = model.predict(image_path="images/C39P4thinF_original_IMG_20150622_105253_cell_111.png")
9 |
10 | print(p)
--------------------------------------------------------------------------------
/resnet18-94.56/processor.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*
2 | import numpy
3 | from PIL import Image
4 | from flyai.processor.base import Base
5 | from flyai.processor.download import check_download
6 | import torch
7 |
8 | from path import DATA_PATH
9 |
10 | # 判断gpu是否可用
11 | if torch.cuda.is_available():
12 | device = 'cuda'
13 | else:
14 | device = 'cpu'
15 | device = torch.device(device)
16 | #device = torch.device('cpu')
17 |
18 | class Processor(Base):
19 | def input_x(self, image_path):
20 | '''
21 | 参数为csv中作为输入x的一条数据，该方法会被Dataset多次调用
22 | '''
23 | path = check_download(image_path, DATA_PATH)
24 | path = path.replace('\\','/')
25 | image = Image.open(path).convert('L')
26 | image = image.rotate(45) #针对原始图像，旋转45度尽可能保留非0数据
27 | x_data = numpy.array(image)
28 | line, columns = numpy.nonzero(x_data) #选择非0区域
29 | lmin = min(line) # 有非0像素的最小行
30 | lmax = max(line)
31 | colmin = min(columns)
32 | colmax = max(columns)
33 | image = Image.fromarray(x_data)
34 | image = image.crop((colmin, lmin, colmax, lmax))
35 |
36 | image = image.resize((224, 224)) #确定尺寸
37 | # image = image.resize((32, 32))
38 | x_data = numpy.array(image)
39 | x_data = x_data.astype(numpy.float32)
40 | x_data = x_data.reshape([224, 224, 1])
41 | # x_data = x_data.reshape([32, 32, 1])
42 | x_data = numpy.transpose(x_data, (2, 0, 1)) ## reshape
43 | return x_data
44 |
45 | # 该参数需要与app.yaml的Model的output-->columns->name 一一对应
46 | def input_y(self, label):
47 | '''
48 | 参数为csv中作为输入y的一条数据，该方法会被Dataset多次调用
49 | '''
50 | return label
51 |
52 | def output_y(self, data):
53 | '''
54 | 验证时使用，把模型输出的y转为对应的结果
55 | '''
56 | return numpy.argmax(data)
--------------------------------------------------------------------------------
/resnet18-94.56/transformation.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | import torchvision.transforms as transforms
3 | import numpy as np
4 | from PIL import Image, ImageEnhance
5 | import torch
6 | class Transformation:
7 | '''
8 | 处理训练数据的类，某些情况下需要对训练的数据再一次的处理。
9 | 如无需处理的话，不用实现该方法。
10 | '''
11 |
12 | def transformation_data(self, x_train=None, y_train=None, x_test=None, y_test=None):
13 | return x_train, y_train, x_test, y_test
14 |
15 |
16 | def batch_X_transform(Xs,transforms):
17 |
18 | imgs=[]
19 | for x in Xs:
20 | img=Image.fromarray(x)
21 | img=transforms(img)
22 | imgs.append(img)
23 | imgs=torch.stack(imgs,dim=0)
24 | return imgs
25 |
26 |
27 | def src(Xs,size):
28 | tran= transforms.Compose([transforms.Resize(size),
29 | transforms.ToTensor(),
30 |
31 | ])
32 | return batch_X_transform(Xs, tran)
--------------------------------------------------------------------------------
/resnet18-94.56/使用jupyter调试.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "### 一、本地运行调试代码"
8 | ]
9 | },
10 | {
11 | "cell_type": "markdown",
12 | "metadata": {},
13 | "source": [
14 | "网络结构在main.py之后，执行`run main.py`可以本地调试。\n",
15 | "\n",
16 | "执行下面代码可使用快捷键`shift+enter`"
17 | ]
18 | },
19 | {
20 | "cell_type": "code",
21 | "execution_count": null,
22 | "metadata": {},
23 | "outputs": [],
24 | "source": [
25 | "run main.py"
26 | ]
27 | },
28 | {
29 | "cell_type": "markdown",
30 | "metadata": {},
31 | "source": [
32 | "### 二、提交到GPU训练"
33 | ]
34 | },
35 | {
36 | "cell_type": "markdown",
37 | "metadata": {},
38 | "source": [
39 | "#### Windows环境执行"
40 | ]
41 | },
42 | {
43 | "cell_type": "markdown",
44 | "metadata": {},
45 | "source": [
46 | "通过执行训练命令，`-e=`是整个数据集循环10次，`-b=`是每次训练读取的数据量为32 "
47 | ]
48 | },
49 | {
50 | "cell_type": "code",
51 | "execution_count": null,
52 | "metadata": {},
53 | "outputs": [],
54 | "source": [
55 | "! flyai.exe train -e=10 -b=32"
56 | ]
57 | },
58 | {
59 | "cell_type": "markdown",
60 | "metadata": {},
61 | "source": [
62 | "#### Mac和Linux执行"
63 | ]
64 | },
65 | {
66 | "cell_type": "markdown",
67 | "metadata": {},
68 | "source": [
69 | "给执行脚本授权"
70 | ]
71 | },
72 | {
73 | "cell_type": "code",
74 | "execution_count": 6,
75 | "metadata": {},
76 | "outputs": [],
77 | "source": [
78 | "! chmod +x ./flyai"
79 | ]
80 | },
81 | {
82 | "cell_type": "markdown",
83 | "metadata": {},
84 | "source": [
85 | "通过执行训练命令，整个数据集循环10次，每次训练读取的数据量为 32 "
86 | ]
87 | },
88 | {
89 | "cell_type": "code",
90 | "execution_count": null,
91 | "metadata": {},
92 | "outputs": [],
93 | "source": [
94 | "! ./flyai train -e=10 -b=32"
95 | ]
96 | },
97 | {
98 | "cell_type": "markdown",
99 | "metadata": {},
100 | "source": [
101 | "### 三、FlyAI的Python环境"
102 | ]
103 | },
104 | {
105 | "cell_type": "markdown",
106 | "metadata": {},
107 | "source": [
108 | "#### Windows用户，Python路径在："
109 | ]
110 | },
111 | {
112 | "cell_type": "code",
113 | "execution_count": null,
114 | "metadata": {},
115 | "outputs": [],
116 | "source": [
117 | "! %USERPROFILE%\\.flyai\\env\\python.exe"
118 | ]
119 | },
120 | {
121 | "cell_type": "markdown",
122 | "metadata": {},
123 | "source": [
124 | "#### pip路径："
125 | ]
126 | },
127 | {
128 | "cell_type": "code",
129 | "execution_count": null,
130 | "metadata": {},
131 | "outputs": [],
132 | "source": [
133 | "! %USERPROFILE%\\.flyai\\env\\Scripts\\pip.exe list"
134 | ]
135 | },
136 | {
137 | "cell_type": "markdown",
138 | "metadata": {},
139 | "source": [
140 | "#### 如本地遇到No module name 'xxx'问题时可以执行下面代码安装"
141 | ]
142 | },
143 | {
144 | "cell_type": "code",
145 | "execution_count": null,
146 | "metadata": {},
147 | "outputs": [],
148 | "source": [
149 | "! %USERPROFILE%\\.flyai\\env\\Scripts\\pip.exe install xxx -i https://pypi.flyai.com/simple "
150 | ]
151 | },
152 | {
153 | "cell_type": "markdown",
154 | "metadata": {},
155 | "source": [
156 | "#### Mac和Linux用户，Python路径在："
157 | ]
158 | },
159 | {
160 | "cell_type": "code",
161 | "execution_count": null,
162 | "metadata": {},
163 | "outputs": [],
164 | "source": [
165 | "! ~/.flyai/env/bin/python3"
166 | ]
167 | },
168 | {
169 | "cell_type": "markdown",
170 | "metadata": {},
171 | "source": [
172 | "#### pip路径："
173 | ]
174 | },
175 | {
176 | "cell_type": "code",
177 | "execution_count": null,
178 | "metadata": {},
179 | "outputs": [],
180 | "source": [
181 | "! ~/.flyai/env/bin/pip list"
182 | ]
183 | },
184 | {
185 | "cell_type": "markdown",
186 | "metadata": {},
187 | "source": [
188 | "#### 如本地遇到No module name 'xxx'问题时可以执行下面代码安装"
189 | ]
190 | },
191 | {
192 | "cell_type": "code",
193 | "execution_count": null,
194 | "metadata": {},
195 | "outputs": [],
196 | "source": [
197 | "! ~/.flyai/env/bin/pip install xxx -i https://pypi.flyai.com/simple"
198 | ]
199 | },
200 | {
201 | "cell_type": "markdown",
202 | "metadata": {},
203 | "source": [
204 | "### 四、样例代码说明\n",
205 | "\n",
206 | "##### `app.yaml`\n",
207 | "\n",
208 | "> 是项目的配置文件，项目目录下**必须**存在这个文件，是项目运行的依赖。\n",
209 | "\n",
210 | "##### `processor.py`\n",
211 | "\n",
212 | "> **样例代码中已做简单实现，可供查考。**\n",
213 | ">\n",
214 | "> 处理数据的输入输出文件，把通过csv文件返回的数据，处理成能让程序识别、训练的矩阵。\n",
215 | ">\n",
216 | "> 可以自己定义输入输出的方法名，在`app.yaml`中声明即可。\n",
217 | ">\n",
218 | "> ```python\n",
219 | "> def input_x(self, image_path):\n",
220 | "> '''\n",
221 | "> \t参数为csv中作为输入x的一条数据，该方法会被dataset.next_train_batch()\n",
222 | "> \t和dataset.next_validation_batch()多次调用。可在该方法中做数据增强\n",
223 | "> \t该方法字段与app.yaml中的input:->columns:对应\n",
224 | "> \t'''\n",
225 | "> pass\n",
226 | ">\n",
227 | "> def output_x(self, image_path):\n",
228 | "> '''\n",
229 | "> \t参数为csv中作为输入x的一条数据，该方法会被dataset.next_train_batch()\n",
230 | "> \t和dataset.next_validation_batch()多次调用。\n",
231 | "> \t该方法字段与app.yaml中的input:->columns:对应\n",
232 | "> \t'''\n",
233 | "> pass\n",
234 | ">\n",
235 | "> def input_y(self, labels):\n",
236 | "> '''\n",
237 | "> 参数为csv中作为输入y的一条数据，该方法会被dataset.next_train_batch()\n",
238 | "> \t和dataset.next_validation_batch()多次调用。\n",
239 | "> \t该方法字段与app.yaml中的output:->columns:对应\n",
240 | "> '''\n",
241 | "> pass\n",
242 | ">\n",
243 | "> def output_y(self, data):\n",
244 | "> '''\n",
245 | "> 输出的结果，会被dataset.to_categorys(data)调用\n",
246 | "> :param data: 预测返回的数据\n",
247 | "> :return: 返回预测的标签\n",
248 | "> '''\n",
249 | "> pass\n",
250 | ">\n",
251 | "> ```\n",
252 | "\n",
253 | "##### `main.py`\n",
254 | "\n",
255 | "> **样例代码中已做简单实现，可供查考。**\n",
256 | ">\n",
257 | "> 程序入口，编写算法，训练模型的文件。在该文件中实现自己的算法。\n",
258 | ">\n",
259 | "> 通过`dataset.py`中的`next_batch`方法获取训练和测试数据。\n",
260 | ">\n",
261 | "> ```python\n",
262 | "> '''\n",
263 | "> Flyai库中的提供的数据处理方法\n",
264 | "> 传入整个数据训练多少轮，每批次批大小\n",
265 | "> '''\n",
266 | "> dataset = Dataset(epochs=args.EPOCHS, batch=args.BATCH)\n",
267 | "> #获取训练数据\n",
268 | "> x_train, y_train = dataset.next_train_batch()\n",
269 | "> #获取验证数据\n",
270 | "> x_val, y_val = dataset.next_validation_batch()\n",
271 | "> ```\n",
272 | ">\n",
273 | "> 通过`model.py`中的`save_model`方法保存模型\n",
274 | ">\n",
275 | "> ```python\n",
276 | "> # 模型操作辅助类\n",
277 | "> model = Model(dataset)\n",
278 | "> model.save_model(YOU_NET)\n",
279 | "> ```\n",
280 | ">\n",
281 | "> **如果使用`PyTorch`框架，需要在`net.py`文件中实现网络。其它用法同上。**\n",
282 | "\n",
283 | "##### `model.py`\n",
284 | "\n",
285 | "> **样例代码中已做简单实现，可供查考。**\n",
286 | ">\n",
287 | "> 训练好模型之后可以继承`flyai.model.base`包中的`base`重写下面三个方法实现模型的保存、验证和使用。\n",
288 | ">\n",
289 | "> ```python\n",
290 | "> def predict(self, **data):\n",
291 | "> '''\n",
292 | "> 使用模型\n",
293 | "> \t:param data: 模型的输入的一个或多个参数\n",
294 | "> :return:\n",
295 | "> '''\n",
296 | "> pass\n",
297 | ">\n",
298 | "> def predict_all(self, datas):\n",
299 | "> '''\n",
300 | "> （必须实现的方法）评估模型，对训练的好的模型进行打分\n",
301 | "> \t:param datas: 验证集上的随机数据，类型为list\n",
302 | "> :return outputs: 返回调用模型评估之后的list数据\n",
303 | "> '''\n",
304 | "> pass\n",
305 | ">\n",
306 | "> def save_model(self, network, path=MODEL_PATH, name=MODEL_NAME, overwrite=False):\n",
307 | "> '''\n",
308 | "> 保存模型\n",
309 | "> :param network: 训练模型的网络\n",
310 | "> :param path: 要保存模型的路径\n",
311 | "> :param name: 要保存模型的名字\n",
312 | "> :param overwrite: 是否覆盖当前模型\n",
313 | "> :return:\n",
314 | "> '''\n",
315 | "> self.check(path, overwrite)\n",
316 | ">\n",
317 | "> ```\n",
318 | "\n",
319 | "##### `predict.py`\n",
320 | "\n",
321 | ">**样例代码中已做简单实现，可供查考。**\n",
322 | ">\n",
323 | ">对训练完成的模型使用和预测。\n",
324 | "\n",
325 | "##### `path.py`\n",
326 | "\n",
327 | "> 可以设置数据文件、模型文件的存放路径。\n",
328 | "\n",
329 | "##### `dataset.py`\n",
330 | "\n",
331 | "> 该文件在**FlyAI开源库**的`flyai.dataset`包中，通过`next_train_batch()`和`next_validation_batch()`方法获得`x_train` `y_train` `x_val` `y_val`数据。\n",
332 | ">\n",
333 | "> FlyAI开源库可以通过`pip3 install -i https://pypi.flyai.com/simple flyai` 安装。\n",
334 | "\n",
335 | "***\n",
336 | "\n",
337 | "### [FlyAI全球人工智能专业开发平台，一站式服务平台](https://flyai.com)\n",
338 | "\n",
339 | "**扫描下方二维码，及时获取FlyAI最新消息，抢先体验最新功能。**\n",
340 | "\n",
341 | "\n",
342 | "\n",
343 | "[](https://flyai.com)\n",
344 | "\n",
345 | "\n",
346 | "\n"
347 | ]
348 | },
349 | {
350 | "cell_type": "code",
351 | "execution_count": null,
352 | "metadata": {},
353 | "outputs": [],
354 | "source": []
355 | }
356 | ],
357 | "metadata": {
358 | "kernelspec": {
359 | "display_name": "Python 3",
360 | "language": "python",
361 | "name": "python3"
362 | },
363 | "language_info": {
364 | "codemirror_mode": {
365 | "name": "ipython",
366 | "version": 3
367 | },
368 | "file_extension": ".py",
369 | "mimetype": "text/x-python",
370 | "name": "python",
371 | "nbconvert_exporter": "python",
372 | "pygments_lexer": "ipython3",
373 | "version": "3.6.4"
374 | }
375 | },
376 | "nbformat": 4,
377 | "nbformat_minor": 4
378 | }
379 |
--------------------------------------------------------------------------------
/resnet18-94.56/常见问题.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | 常见问题
521 |
522 |
523 | FlyAI竞赛常见问题
如何获得奖金？
超过项目设置的最低分，根据公式计算，就可以获得奖金。
比赛使用什么框架？
比赛支持常用的机器学习和深度学习框架，比如TensorFlow，PyTorch，Keras，Scikit-learn等。
怎么参加比赛，需不需要提交csv文件？
FlyAI竞赛平台无需提交csv文件，在网页上点击报名，下载项目，使用你熟练的框架，修改main.py中的网络结构，和processor.py中的数据处理。使用FlyAI提供的命令提交，就可以参加比赛了。
比赛排行榜分数怎么得到的？
参加项目竞赛必须实现 model.py 中的predict_all方法。系统通过该方法，调用模型得出评分。
平台机器什么配置？
目前每个训练独占一块P40显卡，显存24G。
本地数据集在哪？
运行 flyai.exe test or ./flyai test 命令之后会下载100条数据到项目的data目录下
也可以本地使用ide运行 main.py 也会下载数据
FlyAI自带的Python环境在哪,会不会覆盖本地环境？
FlyAI不会覆盖本地现有的Python环境。
windows用户:
C:\Users{你计算机用户名}.flyai\env\python.exe
C:\Users{你计算机用户名}.flyai\env\Scripts\pip.exe
mac和linux用户:
/Users/{你计算机用户名}/.flyai/env/bin/python3.6
/Users/{你计算机用户名}/.flyai/env/bin/pip
FAI训练积分不够用怎么办？
目前GPU免费使用，可以进入到：我的积分，通过签到和分享等途径获得大量积分。
离线训练代码不符合规范问题
main.py中必须使用args.EPOCHS和args.BATCH。
项目什么时候开始，持续多长时间？
网站上能看见的项目就是已经开始的，项目会一直存在，随时可以提交。
排行榜和获奖问题
目前可能获得奖金的项目是审核之后才能上榜的，每天会审核一次。
通过审核之后，奖金才能发送到账户。
全量数据集怎么查看，数据集支不支持下载？
暂时不支持查看和下载，如果需要，可以进入数据集来源链接查看。
运行 flyai.exe train -e=xx -b=xx or ./flyai train -e=xx -b=xx 命令之后会提交代码到服务器上使用全量数据集训练。
from flyai.dataset import Dataset 报错
No module name 'flyai'
先找到ide中使用的Python对应的pip的位置。
windows用户：pip所在路径\pip.exe install -i https://pypi.flyai.com/simple flyai
mac和linux用户：pip所在路径/pip install -i https://pypi.flyai.com/simple flyai
其他 No module name 'xxxx'问题 也可以参考上面。
如果遇到其他问题还可以添加24小时客服微信：flyaixzs
扫描下方二维码，及时获取FlyAI最新消息，抢先体验最新功能。