├── .gitignore
├── .idea
├── .gitignore
├── dbnavigator.xml
├── inspectionProfiles
│ ├── Project_Default.xml
│ └── profiles_settings.xml
├── misc.xml
├── modules.xml
├── tensorflow_serving_examples.iml
└── vcs.xml
├── README.md
├── docker-compose.yml
├── multi_ckpt_2_pb.py
├── multi_model.py
├── multi_tf_serving.py
├── requirements.txt
├── single_ckpt_2_pb.py
├── single_model.py
├── single_tf_serving.py
├── version_control_ckpt_2_pb.py
├── version_control_model.py
└── version_control_tf_serving.py
/.gitignore:
--------------------------------------------------------------------------------
1 | ckpt_models
2 | pb_models
--------------------------------------------------------------------------------
/.idea/.gitignore:
--------------------------------------------------------------------------------
1 | # Default ignored files
2 | /shelf/
3 | /workspace.xml
4 |
--------------------------------------------------------------------------------
/.idea/dbnavigator.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
146 |
147 |
148 |
149 |
150 |
151 |
152 |
153 |
154 |
155 |
156 |
157 |
158 |
159 |
160 |
161 |
162 |
163 |
164 |
165 |
166 |
167 |
168 |
169 |
170 |
171 |
172 |
173 |
174 |
175 |
176 |
177 |
178 |
179 |
180 |
181 |
182 |
183 |
184 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
193 |
194 |
195 |
196 |
197 |
198 |
199 |
200 |
201 |
202 |
203 |
204 |
205 |
206 |
207 |
208 |
209 |
210 |
211 |
212 |
213 |
214 |
215 |
216 |
217 |
218 |
219 |
220 |
221 |
222 |
223 |
224 |
225 |
226 |
227 |
228 |
229 |
230 |
231 |
232 |
233 |
234 |
235 |
236 |
237 |
238 |
239 |
240 |
241 |
242 |
243 |
244 |
245 |
246 |
247 |
248 |
249 |
250 |
251 |
252 |
253 |
254 |
255 |
256 |
257 |
258 |
259 |
260 |
261 |
262 |
263 |
264 |
265 |
266 |
267 |
268 |
269 |
270 |
271 |
272 |
273 |
274 |
275 |
276 |
277 |
278 |
279 |
280 |
281 |
282 |
283 |
284 |
285 |
286 |
287 |
288 |
289 |
290 |
291 |
292 |
293 |
294 |
295 |
296 |
297 |
298 |
299 |
300 |
301 |
302 |
303 |
304 |
305 |
306 |
307 |
308 |
309 |
310 |
311 |
312 |
313 |
314 |
315 |
316 |
317 |
318 |
319 |
320 |
321 |
322 |
323 |
324 |
325 |
326 |
327 |
328 |
329 |
330 |
331 |
332 |
333 |
334 |
335 |
336 |
337 |
338 |
339 |
340 |
341 |
342 |
343 |
344 |
345 |
346 |
347 |
348 |
349 |
350 |
351 |
352 |
353 |
354 |
355 |
356 |
357 |
358 |
359 |
360 |
361 |
362 |
363 |
364 |
365 |
366 |
367 |
368 |
369 |
370 |
371 |
372 |
373 |
374 |
375 |
376 |
377 |
378 |
379 |
380 |
381 |
382 |
383 |
384 |
385 |
386 |
387 |
388 |
389 |
390 |
391 |
392 |
393 |
394 |
395 |
396 |
397 |
398 |
399 |
400 |
401 |
402 |
403 |
404 |
405 |
406 |
407 |
408 |
409 |
410 |
411 |
412 |
413 |
414 |
415 |
416 |
417 |
418 |
419 |
420 |
421 |
422 |
423 |
424 |
425 |
426 |
427 |
428 |
429 |
430 |
431 |
432 |
433 |
434 |
435 |
436 |
437 |
438 |
439 |
440 |
441 |
442 |
443 |
444 |
445 |
446 |
447 |
448 |
449 |
450 |
451 |
452 |
453 |
454 |
455 |
456 |
457 |
--------------------------------------------------------------------------------
/.idea/inspectionProfiles/Project_Default.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
30 |
31 |
32 |
--------------------------------------------------------------------------------
/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/tensorflow_serving_examples.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | 本项目用于演示如何使用tensorflow/serving进行单模型与多模型部署以及模型预测。
2 |
3 | 拉取tensorflow/serving镜像：
4 |
5 | ```
6 | docker pull tensorflow/serving:1.14.0
7 | ```
8 |
9 | 单模型部署命令：
10 |
11 | ```
12 | docker run -t --rm -p 8551:8501 -v "absolute_path_to_pb_models/pb_models/add:/models/add" -e MODEL_NAME=add tensorflow/serving:1.14.0
13 | ```
14 |
15 | 多模型部署命令：
16 |
17 | ```
18 | docker run -t -d --rm -p 8551:8501 -v "absolute_path_to_pb_models/pb_models:/models" tensorflow/serving:1.14.0 --model_config_file=/models/models.config
19 | ```
20 |
21 | 模型多版本部署命令：
22 |
23 | ```
24 | docker run -t -d --rm -p 8551:8501 -v "absolute_path_to_pb_models/pb_models:/models" tensorflow/serving:1.14.0 --model_config_file=/models/models.config
25 | ```
26 |
27 | 模型预测：
28 |
29 | 参考`single_tf_serving.py`及`multi_tf_serving.py`、`version_control_tf_serving.py`。
30 |
31 | curl命令：
32 |
33 | ```
34 | curl --location --request POST 'http://192.168.1.193:8551/v1/models/add/versions/2:predict' \
35 | --header 'Content-Type: application/json' \
36 | --data-raw '{
37 | "instances": [{"t": 2}]
38 | }'
39 | ```
40 |
41 | 输出：
42 |
43 | ```
44 | {
45 | "predictions": [
46 | 8.0
47 | ]
48 | }
49 | ```
50 |
51 | ### 使用Prometheus进行服务监控
52 |
53 | 1. Prothemus的配置文件位于./pb_models/prometheus.yml; Tensorflow/Serving支持Prothemus的配置文件位于pb_models/monitoring_config.txt;
54 | 2. 启动服务：
55 |
56 | ```
57 | docker-compose up -d
58 | ```
59 |
60 | 3. 在浏览器中输入http://IP:9090, 选择Graph, 配置好Expression（比如:tensorflow:core:graph_run_time_usecs_histogram_count，调用次数）即可使用。
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: "3.4"
2 | services:
3 | model_serving:
4 | image: tensorflow/serving:1.14.0
5 | container_name: model_serving
6 | ports:
7 | - "8511:8501"
8 | restart: always
9 | networks:
10 | - tf_net
11 | volumes:
12 | - ./pb_models:/models
13 | command: --model_config_file=/models/models.config && --monitoring_config_file=/models/monitoring_config.txt
14 |
15 | prometheus:
16 | image: prom/prometheus:latest
17 | ports:
18 | - "9090:9090"
19 | restart: always
20 | container_name: prometheus
21 | networks:
22 | - tf_net
23 | volumes:
24 | - ./pb_models/prometheus.yml:/etc/prometheus/prometheus.yml
25 |
26 | networks:
27 | tf_net:
28 |
--------------------------------------------------------------------------------
/multi_ckpt_2_pb.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 2020/11/5 23:45
3 | # @Author : Jclian91
4 | # @File : multi_ckpt_2_pb.py
5 | # @Place : Yangpu, Shanghai
6 |
7 | import tensorflow as tf
8 | from tensorflow.python import saved_model
9 |
10 |
11 | # change ckpt file to pb file
12 | def model_export(model_name):
13 | export_path = "pb_models/{}/1".format(model_name)
14 | graph = tf.Graph()
15 | saver = tf.train.import_meta_graph("./ckpt_models/{}/{}.ckpt.meta".format(model_name, model_name),
16 | graph=graph)
17 | with tf.Session(graph=graph) as sess:
18 | saver.restore(sess, tf.train.latest_checkpoint("./ckpt_models/{}".format(model_name)))
19 | saved_model.simple_save(session=sess,
20 | export_dir=export_path,
21 | inputs={"t": graph.get_operation_by_name('t').outputs[0]},
22 | outputs={"z": graph.get_operation_by_name('z').outputs[0]})
23 |
24 |
25 | model_export("add")
26 | model_export("subtract")
27 | model_export("multiply")
28 | model_export("divide")
29 |
--------------------------------------------------------------------------------
/multi_model.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 2020/11/5 23:39
3 | # @Author : Jclian91
4 | # @File : multi_model.py
5 | # @Place : Yangpu, Shanghai
6 | import tensorflow as tf
7 |
8 | # add model
9 | with tf.Graph().as_default() as g:
10 | x = tf.Variable(2.0, dtype=tf.float32, name="x")
11 | y = tf.Variable(2.0, dtype=tf.float32, name="y")
12 | xy = x * y
13 | t = tf.placeholder(shape=None, dtype=tf.float32, name="t")
14 | z = tf.add(xy, t, name="z")
15 |
16 |
17 | with tf.Session(graph=g) as sess:
18 | sess.run(tf.global_variables_initializer())
19 | result = sess.run(z, feed_dict={t: 3.0})
20 | print("result: ", result)
21 |
22 | # save the model
23 | saver = tf.train.Saver()
24 | saver.save(sess, save_path='./ckpt_models/add/add.ckpt')
25 |
26 | # substract model
27 | with tf.Graph().as_default() as g:
28 | x = tf.Variable(2.0, dtype=tf.float32, name="x")
29 | y = tf.Variable(2.0, dtype=tf.float32, name="y")
30 | xy = x * y
31 | t = tf.placeholder(shape=None, dtype=tf.float32, name="t")
32 | z = tf.subtract(xy, t, name="z")
33 |
34 |
35 | with tf.Session(graph=g) as sess:
36 | sess.run(tf.global_variables_initializer())
37 | result = sess.run(z, feed_dict={t: 3.0})
38 | print("result: ", result)
39 |
40 | # save the model
41 | saver = tf.train.Saver()
42 | saver.save(sess, save_path='./ckpt_models/subtract/subtract.ckpt')
43 |
44 | # multipy
45 | with tf.Graph().as_default() as g:
46 | x = tf.Variable(2.0, dtype=tf.float32, name="x")
47 | y = tf.Variable(2.0, dtype=tf.float32, name="y")
48 | xy = x * y
49 | t = tf.placeholder(shape=None, dtype=tf.float32, name="t")
50 | z = tf.multiply(xy, t, name="z")
51 |
52 |
53 | with tf.Session(graph=g) as sess:
54 | sess.run(tf.global_variables_initializer())
55 | result = sess.run(z, feed_dict={t: 3.0})
56 | print("result: ", result)
57 |
58 | # save the model
59 | saver = tf.train.Saver()
60 | saver.save(sess, save_path='./ckpt_models/multiply/multiply.ckpt')
61 |
62 | # divide model
63 | with tf.Graph().as_default() as g:
64 | x = tf.Variable(2.0, dtype=tf.float32, name="x")
65 | y = tf.Variable(2.0, dtype=tf.float32, name="y")
66 | xy = x * y
67 | t = tf.placeholder(shape=None, dtype=tf.float32, name="t")
68 | z = tf.divide(xy, t, name="z")
69 |
70 |
71 | with tf.Session(graph=g) as sess:
72 | sess.run(tf.global_variables_initializer())
73 | result = sess.run(z, feed_dict={t: 3.0})
74 | print("result: ", result)
75 |
76 | # save the model
77 | saver = tf.train.Saver()
78 | saver.save(sess, save_path='./ckpt_models/divide/divide.ckpt')
--------------------------------------------------------------------------------
/multi_tf_serving.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 2020/11/5 23:55
3 | # @Author : Jclian91
4 | # @File : multi_tf_serving.py
5 | # @Place : Yangpu, Shanghai
6 | import requests
7 |
8 |
9 | # 利用tensorflow/serving的HTTP接口请求进行预测
10 | def model_predict(model_name):
11 | t = 4.0
12 | tensor = {"instances": [{"t": t}]}
13 |
14 | url = "http://192.168.4.193:8511/v1/models/{}:predict".format(model_name)
15 | req = requests.post(url, json=tensor)
16 | if req.status_code == 200:
17 | z = req.json()['predictions'][0]
18 | print("model_{}: ".format(model_name), z)
19 |
20 |
21 | model_predict("add")
22 | model_predict("subtract")
23 | model_predict("multiply")
24 | model_predict("divide")
25 |
26 | '''
27 | model_add: 8.0
28 | model_subtract: 0.0
29 | model_multiply: 16.0
30 | model_divide: 1.0
31 | '''
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | requests==2.18.4
2 | tensorflow==1.14.0
3 |
--------------------------------------------------------------------------------
/single_ckpt_2_pb.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 2020/11/5 23:10
3 | # @Author : Jclian91
4 | # @File : single_ckpt_2_pb.py
5 | # @Place : Yangpu, Shanghai
6 | import tensorflow as tf
7 | from tensorflow.python import saved_model
8 |
9 | export_path = "pb_models/add/1"
10 |
11 | graph = tf.Graph()
12 | saver = tf.train.import_meta_graph("./ckpt_models/add/add.ckpt.meta", graph=graph)
13 | with tf.Session(graph=graph) as sess:
14 | saver.restore(sess, tf.train.latest_checkpoint("./ckpt_models/add"))
15 | saved_model.simple_save(session=sess,
16 | export_dir=export_path,
17 | inputs={"t": graph.get_operation_by_name('t').outputs[0]},
18 | outputs={"z": graph.get_operation_by_name('z').outputs[0]})
19 |
--------------------------------------------------------------------------------
/single_model.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | import tensorflow as tf
3 |
4 | g = tf.Graph()
5 | with g.as_default() as g:
6 | x = tf.Variable(2.0, dtype=tf.float32, name="x")
7 | y = tf.Variable(2.0, dtype=tf.float32, name="y")
8 | xy = x * y
9 | t = tf.placeholder(shape=None, dtype=tf.float32, name="t")
10 | z = tf.add(xy, t, name="z")
11 |
12 |
13 | with tf.Session(graph=g) as sess:
14 | sess.run(tf.global_variables_initializer())
15 | result = sess.run(z, feed_dict={t: 1.0})
16 | print("result: ", result)
17 |
18 | # save the model
19 | saver = tf.train.Saver()
20 | saver.save(sess, save_path='./ckpt_models/add/add.ckpt')
--------------------------------------------------------------------------------
/single_tf_serving.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 2020/11/5 23:36
3 | # @Author : Jclian91
4 | # @File : single_tf_serving.py
5 | # @Place : Yangpu, Shanghai
6 | import requests
7 |
8 | # 利用tensorflow/serving的HTTP接口请求进行预测
9 | t = 1.0
10 | tensor = {"instances": [{"t": t}]}
11 |
12 | url = "http://192.168.1.193:8551/v1/models/add:predict"
13 | req = requests.post(url, json=tensor)
14 | if req.status_code == 200:
15 | z = req.json()['predictions'][0]
16 | print("model_add:", z)
17 |
--------------------------------------------------------------------------------
/version_control_ckpt_2_pb.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 2020/11/6 9:50
3 | # @Author : Jclian91
4 | # @File : version_control_ckpt_2_pb.py
5 | # @Place : Yangpu, Shanghai
6 | import tensorflow as tf
7 | from tensorflow.python import saved_model
8 |
9 |
10 | # change ckpt file to pb file
11 | def model_export(model_version, tf_version):
12 | export_path = "pb_models/add/{}".format(tf_version)
13 | graph = tf.Graph()
14 | saver = tf.train.import_meta_graph("./ckpt_models/add/{}.ckpt.meta".format(model_version),
15 | graph=graph)
16 | with tf.Session(graph=graph) as sess:
17 | saver.restore(sess, tf.train.latest_checkpoint("./ckpt_models/add"))
18 | saved_model.simple_save(session=sess,
19 | export_dir=export_path,
20 | inputs={"t": graph.get_operation_by_name('t').outputs[0]},
21 | outputs={"z": graph.get_operation_by_name('z').outputs[0]})
22 |
23 |
24 | model_export("add", 1)
25 | model_export("add2", 2)
26 | model_export("add3", 3)
--------------------------------------------------------------------------------
/version_control_model.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 2020/11/6 9:48
3 | # @Author : Jclian91
4 | # @File : version_control_model.py
5 | # @Place : Yangpu, Shanghai
6 | import tensorflow as tf
7 |
8 | # 第一个模型
9 | g = tf.Graph()
10 | with g.as_default() as g:
11 | x = tf.Variable(2.0, dtype=tf.float32, name="x")
12 | y = tf.Variable(2.0, dtype=tf.float32, name="y")
13 | xy = x * y
14 | t = tf.placeholder(shape=None, dtype=tf.float32, name="t")
15 | z = tf.add(xy, t, name="z")
16 |
17 |
18 | with tf.Session(graph=g) as sess:
19 | sess.run(tf.global_variables_initializer())
20 | result = sess.run(z, feed_dict={t: 1.0})
21 | print("result: ", result)
22 |
23 | # save the model
24 | saver = tf.train.Saver()
25 | saver.save(sess, save_path='./ckpt_models/add/add.ckpt')
26 |
27 |
28 | # 第二个模型
29 | g = tf.Graph()
30 | with g.as_default() as g:
31 | x = tf.Variable(2.0, dtype=tf.float32, name="x")
32 | y = tf.Variable(2.0, dtype=tf.float32, name="y")
33 | xy = x * y
34 | t = tf.placeholder(shape=None, dtype=tf.float32, name="t")
35 | z = tf.add(xy, 2*t, name="z")
36 |
37 |
38 | with tf.Session(graph=g) as sess:
39 | sess.run(tf.global_variables_initializer())
40 | result = sess.run(z, feed_dict={t: 1.0})
41 | print("result: ", result)
42 |
43 | # save the model
44 | saver = tf.train.Saver()
45 | saver.save(sess, save_path='./ckpt_models/add/add2.ckpt')
46 |
47 |
48 | # 第三个模型
49 | g = tf.Graph()
50 | with g.as_default() as g:
51 | x = tf.Variable(2.0, dtype=tf.float32, name="x")
52 | y = tf.Variable(2.0, dtype=tf.float32, name="y")
53 | xy = x * y
54 | t = tf.placeholder(shape=None, dtype=tf.float32, name="t")
55 | z = tf.add(xy, 3*t, name="z")
56 |
57 |
58 | with tf.Session(graph=g) as sess:
59 | sess.run(tf.global_variables_initializer())
60 | result = sess.run(z, feed_dict={t: 1.0})
61 | print("result: ", result)
62 |
63 | # save the model
64 | saver = tf.train.Saver()
65 | saver.save(sess, save_path='./ckpt_models/add/add3.ckpt')
66 |
--------------------------------------------------------------------------------
/version_control_tf_serving.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 2020/11/6 9:58
3 | # @Author : Jclian91
4 | # @File : version_control_tf_serving.py
5 | # @Place : Yangpu, Shanghai
6 | import requests
7 |
8 |
9 | # 利用tensorflow/serving的HTTP接口请求进行预测
10 | def model_predict(model_version):
11 | t = 4.0
12 | tensor = {"instances": [{"t": t}]}
13 |
14 | url = "http://192.168.1.193:8551/v1/models/add/versions/{}:predict".format(model_version)
15 | req = requests.post(url, json=tensor)
16 | if req.status_code == 200:
17 | z = req.json()['predictions'][0]
18 | print("model_version{}: ".format(model_version), z)
19 |
20 |
21 | model_predict("1")
22 | model_predict("2")
23 | model_predict("3")
24 |
--------------------------------------------------------------------------------