getAccumulatedData(@Param("plantform") String plantform, @Param("start_date") Date start_date, @Param("end_date") Date end_date);
23 | }
24 |
--------------------------------------------------------------------------------
/python-opencv/blog12-sobel/demo-sobel.py:
--------------------------------------------------------------------------------
1 | import cv2 as cv
2 | import matplotlib.pyplot as plt
3 |
4 | # 读取图像
5 | img = cv.imread('maliao.jpg', cv.COLOR_BGR2GRAY)
6 | rgb_img = cv.cvtColor(img, cv.COLOR_BGR2RGB)
7 |
8 | # 灰度化处理图像
9 | grayImage = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
10 |
11 | # Sobel 算子
12 | x = cv.Sobel(grayImage, cv.CV_16S, 1, 0)
13 | y = cv.Sobel(grayImage, cv.CV_16S, 0, 1)
14 |
15 | # 转 uint8 ,图像融合
16 | absX = cv.convertScaleAbs(x)
17 | absY = cv.convertScaleAbs(y)
18 | Sobel = cv.addWeighted(absX, 0.5, absY, 0.5, 0)
19 |
20 | # 用来正常显示中文标签
21 | plt.rcParams['font.sans-serif'] = ['SimHei']
22 |
23 | # 显示图形
24 | titles = ['原始图像', 'Sobel 算子']
25 | images = [rgb_img, Sobel]
26 |
27 | for i in range(2):
28 | plt.subplot(1, 2, i + 1), plt.imshow(images[i], 'gray')
29 | plt.title(titles[i])
30 | plt.xticks([]), plt.yticks([])
31 | plt.show()
--------------------------------------------------------------------------------
/python-spider/first_scrapy/first_scrapy/spiders/quotes.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | import scrapy
3 | from first_scrapy.items import QuoteItem
4 |
5 | class QuotesSpider(scrapy.Spider):
6 | name = 'quotes'
7 | allowed_domains = ['quotes.toscrape.com']
8 | start_urls = ['http://quotes.toscrape.com/']
9 |
10 | def parse(self, response):
11 | quotes = response.css('.quote')
12 | for quote in quotes:
13 | item = QuoteItem()
14 | item['text'] = quote.css('.text::text').extract_first()
15 | item['author'] = quote.css('.author::text').extract_first()
16 | item['tags'] = quote.css('.tags .tag::text').extract()
17 | yield item
18 |
19 | next = response.css('.pager .next a::attr("href")').extract_first()
20 | url = response.urljoin(next)
21 | yield scrapy.Request(url=url, callback=self.parse)
--------------------------------------------------------------------------------
/python-data-analysis/matplotlib/PieDemo1.py:
--------------------------------------------------------------------------------
1 | import matplotlib.pyplot as plt
2 |
3 | # 中文和负号的正常显示
4 | plt.rcParams['font.sans-serif']=['SimHei']
5 | plt.rcParams['axes.unicode_minus'] = False
6 |
7 | # 数据
8 | edu = [0.2515,0.3724,0.3336,0.0368,0.0057]
9 | labels = ['中专','大专','本科','硕士','其他']
10 |
11 | # 让本科学历离圆心远一点
12 | explode = [0,0,0.1,0,0]
13 |
14 | # 将横、纵坐标轴标准化处理，保证饼图是一个正圆，否则为椭圆
15 | plt.axes(aspect='equal')
16 |
17 | # 自定义颜色
18 | colors=['#9999ff','#ff9999','#7777aa','#2442aa','#dd5555'] # 自定义颜色
19 |
20 | # 绘制饼图
21 | plt.pie(x=edu, # 绘图数据
22 | explode = explode, # 突出显示大专人群
23 | labels = labels, # 添加教育水平标签
24 | colors = colors, # 设置饼图的自定义填充色
25 | autopct = '%.1f%%', # 设置百分比的格式，这里保留一位小数
26 | wedgeprops = {'width': 0.3, 'edgecolor':'green'}
27 | )
28 |
29 | # 添加图标题
30 | plt.title('xxx 公司员工教育水平分布')
31 |
32 | # 保存图形
33 | plt.savefig('pie_demo1.png')
--------------------------------------------------------------------------------
/python-data-analysis/pyecharts_map/GlobeMap.py:
--------------------------------------------------------------------------------
1 | import pyecharts.options as opts
2 | from pyecharts.charts import MapGlobe
3 | from pyecharts.faker import POPULATION
4 |
5 |
6 | data = [x for _, x in POPULATION[1:]]
7 | low, high = min(data), max(data)
8 |
9 | c = (
10 | MapGlobe()
11 | .add_schema()
12 | .add(
13 | maptype="world",
14 | series_name="World Population",
15 | data_pair=POPULATION[1:],
16 | is_map_symbol_show=False,
17 | label_opts=opts.LabelOpts(is_show=False),
18 | )
19 | .set_global_opts(
20 | visualmap_opts=opts.VisualMapOpts(
21 | min_=low,
22 | max_=high,
23 | range_text=["max", "min"],
24 | is_calculable=True,
25 | range_color=["lightskyblue", "yellow", "orangered"],
26 | )
27 |
28 | )
29 | .render("map_globe.html")
30 | )
--------------------------------------------------------------------------------
/python-spider/urllib-request/cookies_lwp.txt:
--------------------------------------------------------------------------------
1 | #LWP-Cookies-2.0
2 | Set-Cookie3: BAIDUID="823ED2595594806E60BCE6477089CABB:FG=1"; path="/"; domain=".baidu.com"; path_spec; domain_dot; expires="2020-11-30 02:49:16Z"; comment=bd; version=0
3 | Set-Cookie3: BIDUPSID=823ED2595594806EF45F17B3045C8D5B; path="/"; domain=".baidu.com"; path_spec; domain_dot; expires="2087-12-19 06:03:23Z"; version=0
4 | Set-Cookie3: H_PS_PSSID=1427_21109_30210_26350; path="/"; domain=".baidu.com"; path_spec; domain_dot; discard; version=0
5 | Set-Cookie3: PSTM=1575168556; path="/"; domain=".baidu.com"; path_spec; domain_dot; expires="2087-12-19 06:03:23Z"; version=0
6 | Set-Cookie3: delPer=0; path="/"; domain=".baidu.com"; path_spec; domain_dot; discard; version=0
7 | Set-Cookie3: BDSVRTM=0; path="/"; domain="www.baidu.com"; path_spec; discard; version=0
8 | Set-Cookie3: BD_HOME=0; path="/"; domain="www.baidu.com"; path_spec; discard; version=0
9 |
--------------------------------------------------------------------------------
/python-spider/xpath-demo/xpath_demo.py:
--------------------------------------------------------------------------------
1 | from lxml import etree
2 | import requests
3 |
4 | response = requests.get('https://www.geekdigging.com/')
5 | html_str = response.content.decode('UTF-8')
6 | html = etree.HTML(html_str)
7 | result = etree.tostring(html, encoding = 'UTF-8').decode('UTF-8')
8 | # 输出太长，先注释
9 | # print(result)
10 |
11 | result_1 = html.xpath('//*')
12 | print(result_1)
13 |
14 | result_2 = html.xpath('//meta')
15 | print(result_2)
16 |
17 | result_3 = html.xpath('//main/article')
18 | print(result_3)
19 |
20 | result_4 = html.xpath('//main//div')
21 | print(result_4)
22 |
23 | result_5 = html.xpath('//img[@alt="小白学 Python 爬虫（16）：urllib 实战之爬取妹子图"]/../@href')
24 | print(result_5)
25 |
26 | result_6 = html.xpath('//img[@alt="小白学 Python 爬虫（16）：urllib 实战之爬取妹子图"]/parent::*/@href')
27 | print(result_6)
28 |
29 | result_7 = html.xpath('//section/div[@class="container"]')
30 | print(result_7)
--------------------------------------------------------------------------------
/data_structure/StackNode.py:
--------------------------------------------------------------------------------
1 | class Node(object):
2 | '''
3 | 节点实现
4 | '''
5 | def __init__(self,elem):
6 | self.elem = elem
7 | self.next = None
8 |
9 |
10 | class Stack(object):
11 | def __init__(self):
12 | '''
13 | 初始化链表头
14 | '''
15 | self.__head = None
16 |
17 | def is_empty(self):
18 | return self.__head is None
19 |
20 | def push(self, item):
21 | '''
22 | 压栈
23 | :param item:
24 | :return:
25 | '''
26 | node = Node(item)
27 | node.next = self.__head
28 | self.__head = node
29 |
30 | def pop(self):
31 | '''
32 | 弹出栈
33 | :return:
34 | '''
35 | if self.is_empty():
36 | return
37 | else:
38 | p = self.__head
39 | self.__head = p.next
40 | return p.elem
--------------------------------------------------------------------------------
/python-data-analysis/pyecharts/Calendar_base_demo.py:
--------------------------------------------------------------------------------
1 | import datetime
2 | import random
3 |
4 | from pyecharts import options as opts
5 | from pyecharts.charts import Calendar
6 |
7 |
8 | begin = datetime.date(2019, 1, 1)
9 | end = datetime.date(2019, 12, 31)
10 | data = [
11 | [str(begin + datetime.timedelta(days=i)), random.randint(1, 20)]
12 | for i in range((end - begin).days + 1)
13 | ]
14 |
15 | c = (
16 | Calendar()
17 | .add("", data, calendar_opts=opts.CalendarOpts(range_="2019"))
18 | .set_global_opts(
19 | title_opts=opts.TitleOpts(title="Calendar-2019 Github 提交情况"),
20 | visualmap_opts=opts.VisualMapOpts(
21 | max_=20,
22 | min_=1,
23 | orient="horizontal",
24 | is_piecewise=True,
25 | pos_top="230px",
26 | pos_left="100px",
27 | ),
28 | )
29 | .render("calendar_base.html")
30 | )
31 |
--------------------------------------------------------------------------------
/spider-blog/spider_data.sql:
--------------------------------------------------------------------------------
1 | SET NAMES utf8mb4;
2 | SET FOREIGN_KEY_CHECKS = 0;
3 |
4 | -- ----------------------------
5 | -- Table structure for spider_data
6 | -- ----------------------------
7 | DROP TABLE IF EXISTS `spider_data`;
8 | CREATE TABLE `spider_data` (
9 | `id` varchar(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '主键 UUID',
10 | `plantform` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '平台类型',
11 | `read_num` int(11) NULL DEFAULT NULL COMMENT '总阅读数',
12 | `fans_num` int(11) NULL DEFAULT NULL COMMENT '总粉丝数',
13 | `rank_num` int(11) NULL DEFAULT NULL COMMENT '排名',
14 | `like_num` int(11) NULL DEFAULT NULL COMMENT '总点赞数',
15 | `create_date` datetime(0) NULL DEFAULT NULL COMMENT '创建时间',
16 | PRIMARY KEY (`id`) USING BTREE
17 | ) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
18 |
19 | SET FOREIGN_KEY_CHECKS = 1;
20 |
--------------------------------------------------------------------------------
/base-data-str/Demo.py:
--------------------------------------------------------------------------------
1 | print('www.geekdigging.com')
2 |
3 | print("www.geekdigging.com")
4 |
5 | print('小明说："吃了吗？"')
6 |
7 | print("小明说：\"吃了吗？\"")
8 |
9 | print('''
10 | 这是一个多行字符串的实例
11 | 多行字符串可以使用制表符
12 | TAB ( \t )。
13 | 也可以使用换行符 [ \n ]。
14 | ''')
15 |
16 | print('I \nlike Python!')
17 | print("")
18 | print('I\tlike\tPython!')
19 |
20 | print("横向制表符：\\t")
21 |
22 | print(r"横向制表符：\t")
23 |
24 | print(len('I like Python!'))
25 |
26 | print(len('I like 小明!'))
27 |
28 | print(len(''))
29 |
30 | print(len(' '))
31 |
32 | print(len("\n"))
33 |
34 | a = "Hello"
35 | b = "Python"
36 |
37 | print("a + b 输出结果：", a + " " + b)
38 |
39 | print("a * 2 输出结果：", a * 2)
40 |
41 | print("a[1] 输出结果：", a[1])
42 |
43 | print("a[1:4] 输出结果：", a[1:4])
44 |
45 | if "H" in a:
46 | print("H 在变量 a 中")
47 | else:
48 | print("H 不在变量 a 中")
49 |
50 | if "M" not in a:
51 | print("M 不在变量 a 中")
52 | else:
53 | print("M 在变量 a 中")
--------------------------------------------------------------------------------
/python-spider/douyin/demo-video.py:
--------------------------------------------------------------------------------
1 | import requests
2 | import re
3 |
4 | # 创建一个请求头
5 | headers = {
6 | 'user-agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1',
7 | }
8 |
9 | # 分享路径
10 | share_url = "https://v.douyin.com/JefvNdx/"
11 |
12 | session = requests.Session()
13 | res = session.get(share_url, headers = headers)
14 |
15 | # 获取视频 id
16 | item_ids = re.compile(r'itemId: "([0-9]+)"').findall(res.text)[0]
17 |
18 | # 拼接请求
19 | item_info_url = f"https://www.iesdouyin.com/web/api/v2/aweme/iteminfo/?item_ids={item_ids}"
20 | res_json = session.get(item_info_url, headers = headers).json()
21 |
22 | # 获取视频源路径
23 | vedio_url = res_json['item_list'][0]['video']['play_addr']['url_list'][0]
24 |
25 | res = requests.get(vedio_url, headers = headers)
26 | with open('demo.mp4', 'wb') as fb:
27 | fb.write(res.content)
28 |
29 | print("视频下载完成~~~")
--------------------------------------------------------------------------------
/python-opencv/blog15-contours/approx.py:
--------------------------------------------------------------------------------
1 | import cv2 as cv
2 |
3 | img = cv.imread("number.png")
4 |
5 | gray_img = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
6 | # 降噪
7 | ret, thresh = cv.threshold(gray_img, 127, 255, 0)
8 | # 寻找轮廓
9 | contours, hierarchy = cv.findContours(gray_img, cv.RETR_TREE, cv.CHAIN_APPROX_NONE)
10 |
11 | cnt = contours[0]
12 |
13 | # 计算 epsilon ，按照周长百分比进行计算，分别取周长 1% 和 10%
14 | epsilon_1 = 0.1 * cv.arcLength(cnt, True)
15 | epsilon_2 = 0.01 * cv.arcLength(cnt, True)
16 |
17 | # 进行多边形逼近
18 | approx_1 = cv.approxPolyDP(cnt, epsilon_1, True)
19 | approx_2 = cv.approxPolyDP(cnt, epsilon_2, True)
20 |
21 | # 画出多边形
22 | image_1 = cv.cvtColor(gray_img, cv.COLOR_GRAY2BGR)
23 | image_2 = cv.cvtColor(gray_img, cv.COLOR_GRAY2BGR)
24 |
25 | cv.polylines(image_1, [approx_1], True, (0, 0, 255), 2)
26 | cv.polylines(image_2, [approx_2], True, (0, 0, 255), 2)
27 |
28 | cv.imshow("image_1", image_1)
29 | cv.imshow("image_2", image_2)
30 | cv.waitKey(0)
31 | cv.destroyAllWindows()
--------------------------------------------------------------------------------
/python-opencv/blog6-threshold/demo-quanjiafu.py:
--------------------------------------------------------------------------------
1 | import cv2 as cv
2 | import numpy as np
3 | import matplotlib.pyplot as plt
4 |
5 | # 读取图像
6 | img=cv.imread('maliao.jpg')
7 | lenna_img = cv.cvtColor(img,cv.COLOR_BGR2RGB)
8 | gray_img=cv.cvtColor(img,cv.COLOR_BGR2GRAY)
9 |
10 | # 阈值化处理
11 | ret1, thresh1=cv.threshold(gray_img, 127, 255, cv.THRESH_BINARY)
12 | ret2, thresh2=cv.threshold(gray_img, 127, 255, cv.THRESH_BINARY_INV)
13 | ret3, thresh3=cv.threshold(gray_img, 127, 255, cv.THRESH_TRUNC)
14 | ret4, thresh4=cv.threshold(gray_img, 127, 255, cv.THRESH_TOZERO)
15 | ret5, thresh5=cv.threshold(gray_img, 127, 255, cv.THRESH_TOZERO_INV)
16 |
17 | # 显示结果
18 | titles = ['Gray Img','BINARY','BINARY_INV','TRUNC','TOZERO','TOZERO_INV']
19 | images = [gray_img, thresh1, thresh2, thresh3, thresh4, thresh5]
20 |
21 | # matplotlib 绘图
22 | for i in range(6):
23 | plt.subplot(2, 3, i+1), plt.imshow(images[i],'gray')
24 | plt.title(titles[i])
25 | plt.xticks([]),plt.yticks([])
26 |
27 | plt.show()
28 |
--------------------------------------------------------------------------------
/base-data-list/Demo.py:
--------------------------------------------------------------------------------
1 | list1 = [1, 2, 3, 4, 5]
2 | print(list1)
3 |
4 | list2 = ['a', 'b', 'c', 'd', 'e']
5 | print(list2)
6 |
7 | list3 = [1, 2, 3, 'a', 'b']
8 | print(list3)
9 |
10 | list4 = [1, 2.33, 'a', list3]
11 | print(list4)
12 |
13 | list5 = []
14 | print(list5)
15 |
16 | print(type(list4))
17 |
18 | list1 = [1, 2, 3, 4, 5]
19 | print(list1[0])
20 | # 异常索引打印
21 | # print(list1[5])
22 |
23 | print(list1[-1])
24 |
25 | print(list1 + list2)
26 |
27 | for i in list1:
28 | print(i)
29 |
30 | print(len(list1))
31 |
32 | print(len(list1 + list2))
33 |
34 | print('a' in list1)
35 | print(1 in list1)
36 |
37 | # 删除 list1 示例
38 | # list1 = [1, 2, 3, 4, 5]
39 | #
40 | # del list1
41 | # print(list1)
42 |
43 | list1 = [1, 2, 3, 4, 5]
44 |
45 | del list1[2]
46 | print(list1)
47 |
48 | print(list1)
49 | print(max(list1))
50 | # 比较会报错，注释
51 | # print(list4)
52 | # print(max(list4))
53 |
54 | print(list1)
55 | print(min(list1))
56 | print(list4)
57 | print(min(list4))
--------------------------------------------------------------------------------
/python-data-analysis/mojito/damu-wordcloud.py:
--------------------------------------------------------------------------------
1 | import jieba
2 | import pandas as pd
3 | import wordcloud
4 |
5 | # 读取弹幕 txt 文件
6 | with open("dan_mu.txt", encoding="utf-8") as f:
7 | txt = f.read()
8 | danmu_list = txt.split("\n")
9 |
10 | # jieba 分词
11 | danmu_cut = [jieba.lcut(item) for item in danmu_list]
12 |
13 | # 获取停用词
14 | with open("baidu_stopwords.txt",encoding="utf-8") as f:
15 | stop = f.read()
16 | stop_words = stop.split()
17 |
18 | # 去掉停用词后的最终词
19 | s_data_cut = pd.Series(danmu_cut)
20 | all_words_after = s_data_cut.apply(lambda x:[i for i in x if i not in stop])
21 |
22 | # 词频统计
23 | all_words = []
24 | for i in all_words_after:
25 | all_words.extend(i)
26 | word_count = pd.Series(all_words).value_counts()
27 |
28 | wordcloud.WordCloud(
29 | font_path='msyh.ttc',
30 | background_color="#fff",
31 | max_words=1000,
32 | max_font_size=200,
33 | random_state=42,
34 | width=900,
35 | height=1600
36 | ).fit_words(word_count).to_file("wordcloud.png")
--------------------------------------------------------------------------------
/python-opencv/blog12-sobel/demo-roberts.py:
--------------------------------------------------------------------------------
1 | import cv2 as cv
2 | import numpy as np
3 | import matplotlib.pyplot as plt
4 |
5 | # 读取图像
6 | img = cv.imread('maliao.jpg', cv.COLOR_BGR2GRAY)
7 | rgb_img = cv.cvtColor(img, cv.COLOR_BGR2RGB)
8 |
9 | # 灰度化处理图像
10 | grayImage = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
11 |
12 | # Roberts 算子
13 | kernelx = np.array([[-1, 0], [0, 1]], dtype=int)
14 | kernely = np.array([[0, -1], [1, 0]], dtype=int)
15 |
16 | x = cv.filter2D(grayImage, cv.CV_16S, kernelx)
17 | y = cv.filter2D(grayImage, cv.CV_16S, kernely)
18 |
19 | # 转 uint8 ,图像融合
20 | absX = cv.convertScaleAbs(x)
21 | absY = cv.convertScaleAbs(y)
22 | Roberts = cv.addWeighted(absX, 0.5, absY, 0.5, 0)
23 |
24 | # 用来正常显示中文标签
25 | plt.rcParams['font.sans-serif'] = ['SimHei']
26 |
27 | # 显示图形
28 | titles = ['原始图像', 'Roberts 算子']
29 | images = [rgb_img, Roberts]
30 |
31 | for i in range(2):
32 | plt.subplot(1, 2, i + 1), plt.imshow(images[i], 'gray')
33 | plt.title(titles[i])
34 | plt.xticks([]), plt.yticks([])
35 | plt.show()
--------------------------------------------------------------------------------
/base-except/Demo.py:
--------------------------------------------------------------------------------
1 | def division(x, y):
2 | try:
3 | return x / y
4 | except:
5 | print('程序报错啦！！！')
6 | return None
7 |
8 | print(division(15, 5))
9 |
10 | def division1(x, y):
11 | try:
12 | return x / y
13 | except ZeroDivisionError:
14 | print('程序报错啦！！！')
15 | return None
16 |
17 | print(division1(15, 0))
18 |
19 | def division2(x, y):
20 | try:
21 | return x / y
22 | except ZeroDivisionError:
23 | print('您输出的除数为 0 ！！！')
24 | return None
25 | except TypeError:
26 | print('您输出的参数类型非法！！！')
27 | return None
28 |
29 | print(division2('python', 0))
30 |
31 | def division3(x, y):
32 | try:
33 | return x / y
34 | except ZeroDivisionError:
35 | print('您输出的除数为 0 ！！！')
36 | return None
37 | except TypeError:
38 | print('您输出的参数类型非法！！！')
39 | return None
40 | finally:
41 | print('你一定能看到我！！！')
42 |
43 | print(division3(15, 3))
44 | print(division3('python', 0))
--------------------------------------------------------------------------------
/python-opencv/blog12-sobel/demo-prewitt.py:
--------------------------------------------------------------------------------
1 | import cv2 as cv
2 | import numpy as np
3 | import matplotlib.pyplot as plt
4 |
5 | # 读取图像
6 | img = cv.imread('maliao.jpg', cv.COLOR_BGR2GRAY)
7 | rgb_img = cv.cvtColor(img, cv.COLOR_BGR2RGB)
8 |
9 | # 灰度化处理图像
10 | grayImage = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
11 |
12 | # Prewitt 算子
13 | kernelx = np.array([[1,1,1],[0,0,0],[-1,-1,-1]],dtype=int)
14 | kernely = np.array([[-1,0,1],[-1,0,1],[-1,0,1]],dtype=int)
15 |
16 | x = cv.filter2D(grayImage, cv.CV_16S, kernelx)
17 | y = cv.filter2D(grayImage, cv.CV_16S, kernely)
18 |
19 | # 转 uint8 ,图像融合
20 | absX = cv.convertScaleAbs(x)
21 | absY = cv.convertScaleAbs(y)
22 | Prewitt = cv.addWeighted(absX, 0.5, absY, 0.5, 0)
23 |
24 | # 用来正常显示中文标签
25 | plt.rcParams['font.sans-serif'] = ['SimHei']
26 |
27 | # 显示图形
28 | titles = ['原始图像', 'Prewitt 算子']
29 | images = [rgb_img, Prewitt]
30 |
31 | for i in range(2):
32 | plt.subplot(1, 2, i + 1), plt.imshow(images[i], 'gray')
33 | plt.title(titles[i])
34 | plt.xticks([]), plt.yticks([])
35 | plt.show()
--------------------------------------------------------------------------------
/python-data-analysis/matplotlib/PolarDemo.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import matplotlib.pyplot as plt
3 |
4 | # 中文和负号的正常显示
5 | plt.rcParams['font.sans-serif']=['SimHei']
6 | plt.rcParams['axes.unicode_minus'] = False
7 |
8 | # 使用ggplot的绘图风格
9 | plt.style.use('ggplot')
10 |
11 | # 构造数据
12 | values = [3.2, 2.1, 3.5, 2.8, 3]
13 | feature = ['攻击力', '防御力', '恢复力', '法术强度', '生命值']
14 |
15 | N = len(values)
16 | # 设置雷达图的角度，用于平分切开一个圆面
17 | angles = np.linspace(0, 2 * np.pi, N, endpoint=False)
18 |
19 | # 为了使雷达图一圈封闭起来，需要下面的步骤
20 | values = np.concatenate((values, [values[0]]))
21 | angles = np.concatenate((angles, [angles[0]]))
22 |
23 | # 绘图
24 | fig = plt.figure()
25 | # 这里一定要设置为极坐标格式
26 | ax = fig.add_subplot(111, polar=True)
27 | # 绘制折线图
28 | ax.plot(angles, values, 'o-', linewidth=2)
29 | # 填充颜色
30 | ax.fill(angles, values, alpha=0.25)
31 | # 添加每个特征的标签
32 | ax.set_thetagrids(angles * 180 / np.pi, feature)
33 | # 设置雷达图的范围
34 | ax.set_ylim(0, 5)
35 | # 添加标题
36 | plt.title('游戏人物属性')
37 | # 添加网格线
38 | ax.grid(True)
39 | # 显示图形
40 | plt.savefig('polar_demo.png')
--------------------------------------------------------------------------------
/python-spider/aiohttp-demo/aio-basic-demo.py:
--------------------------------------------------------------------------------
1 | import aiohttp
2 | import asyncio
3 |
4 | async def aio_1():
5 | async with aiohttp.ClientSession() as session:
6 | async with session.get('https://www.baidu.com/') as resp:
7 | print(resp.status)
8 | # print(await resp.text())
9 |
10 | loop = asyncio.get_event_loop()
11 | loop.run_until_complete(aio_1())
12 |
13 | async def aio_2():
14 | async with aiohttp.ClientSession() as session:
15 | async with session.get('https://www.geekdigging.com/') as resp:
16 | print(resp.status)
17 | print(await resp.text())
18 |
19 | loop = asyncio.get_event_loop()
20 | loop.run_until_complete(aio_2())
21 |
22 | async def aio_3():
23 | timeout = aiohttp.ClientTimeout(total=60)
24 | async with aiohttp.ClientSession(timeout = timeout) as session:
25 | async with session.get('https://www.geekdigging.com/', timeout = timeout) as resp:
26 | print(resp.status)
27 |
28 | loop = asyncio.get_event_loop()
29 | loop.run_until_complete(aio_3())
--------------------------------------------------------------------------------
/base-data-set/Demo1.py:
--------------------------------------------------------------------------------
1 | set1 = {1, 2, 3, 4, 5, 6}
2 | set2 = {4, 5, 6, 7, 7, 9}
3 |
4 | # 求交集
5 | set3 = set1.intersection(set2)
6 | print('交集：', set3)
7 |
8 | # 求并集
9 | set4 = set1.union(set2)
10 | print('并集：', set4)
11 |
12 | # 做差
13 | set5 = set1.difference(set2)
14 | print('做差：', set5)
15 |
16 | set6 = {1, 2, 3}
17 | set6.add(4)
18 | print(set6)
19 | set6.add('python')
20 | print(set6)
21 | set6.add((1, 2))
22 | print(set6)
23 |
24 | set7 = {1, 2}
25 | set7.update({3, 4, 'python', (4, 5)})
26 | print(set7)
27 |
28 | set7.pop()
29 | print(set7)
30 |
31 | set8 = {1, 2, 3, 4}
32 | set8.remove(4)
33 | print(set8)
34 | # 删除不存在元素，报错
35 | # set8.remove(9)
36 |
37 | set8.discard(9)
38 | print(set8)
39 |
40 | set9 = {1, 2, 3}
41 | set9.clear()
42 | print(set9)
43 |
44 | set10 = {1, 2, 3}
45 | set11 = {1, 2}
46 | set12 = {4, 5}
47 | print(set10.isdisjoint(set11))
48 | print(set10.isdisjoint(set12))
49 |
50 | print(set11.issubset(set10))
51 | print(set12.issubset(set10))
52 |
53 | print(set10.issuperset(set11))
54 | print(set10.issuperset(set12))
--------------------------------------------------------------------------------
/base-process/Demo.py:
--------------------------------------------------------------------------------
1 | # 示例 1
2 |
3 | # width = input("请输入长方形的宽：")
4 | # height = input("请输入长方形的高：")
5 | # area = int(width) * int(height)
6 | # print("长方形的面积为：", area)
7 |
8 | # 示例 2
9 |
10 | # weight = input("请输入当前的体重：")
11 | #
12 | # if float(weight) >= 200:
13 | # print("你和加菲猫一样肥！！")
14 | # else:
15 | # print("你还是很苗条的么！！")
16 |
17 | # 示例 3
18 |
19 | # weight = input("请输入您当前的体重：")
20 | #
21 | # if float(weight) >= 200:
22 | # print("你和加菲猫一样肥！！")
23 | # elif float(weight) >= 100:
24 | # print("你的身材真棒！！")
25 | # else:
26 | # print("有点瘦哦，要多吃肉！！")
27 |
28 | # 示例 4
29 | gender = input("请输入您的性别（M或者F）:")
30 | height = input("请输入您的身高：")
31 |
32 | if gender == 'M':
33 | if float(height) >= 185:
34 | print("海拔太高了，可能会导致缺氧！！！")
35 | elif float(height) >= 175:
36 | print("男神身高！！！")
37 | else:
38 | print("哥们，该补钙了！！！")
39 | else:
40 | if float(height) >= 175:
41 | print("您可以去当模特了！！！")
42 | elif float(height) >= 165:
43 | print("女神身高，您是一位美丽的女孩子！！！")
44 | else:
45 | print("美女，多晒晒太阳吧！！！")
--------------------------------------------------------------------------------
/python-data-analysis/pyecharts_map/ShanghaiMap.py:
--------------------------------------------------------------------------------
1 | from pyecharts import options as opts
2 | from pyecharts.charts import Map
3 | from pyecharts.globals import ThemeType
4 |
5 | shanghai_list = ['黄浦区', '徐汇区', '长宁区', '静安区', '普陀区', '虹口区', '杨浦区', '闵行区', '宝山区', '嘉定区', '金山区', '松江区', '青浦区', '奉贤区', '崇明区', '浦东新区']
6 |
7 | shanghai_people = [65.38, 108.44, 69.4, 106.28, 128.19, 79.7, 131.27, 254.35, 204.23, 158.89, 80.5, 176.22, 121.9, 115.2, 68.81, 555.02]
8 |
9 | BAIDU_LINK='https://baike.baidu.com/item/%E4%B8%8A%E6%B5%B7%E8%A1%8C%E6%94%BF%E5%8C%BA%E5%88%92/7426389?fr=aladdin'
10 |
11 | c = (
12 | Map(init_opts=opts.InitOpts(theme=ThemeType.DARK, bg_color='#404a59', width='1600px', height='900px'))
13 | .add("上海市-常住人口", [list(z) for z in zip(shanghai_list, shanghai_people)], "上海")
14 | .set_global_opts(
15 | title_opts=opts.TitleOpts(
16 | title="上海地图-常住人口（单位：万人）",
17 | subtitle="常住人口数据来自百度百科",
18 | subtitle_link=BAIDU_LINK,
19 | ),
20 | visualmap_opts=opts.VisualMapOpts()
21 | )
22 | .render("map_shanghai.html")
23 | )
--------------------------------------------------------------------------------
/python-data-analysis/pyecharts/Line3d_autorotate_demo.py:
--------------------------------------------------------------------------------
1 | import math
2 |
3 | from pyecharts import options as opts
4 | from pyecharts.charts import Line3D
5 | from pyecharts.faker import Faker
6 |
7 | data = []
8 | for t in range(0, 25000):
9 | _t = t / 1000
10 | x = (1 + 0.25 * math.cos(75 * _t)) * math.cos(_t)
11 | y = (1 + 0.25 * math.cos(75 * _t)) * math.sin(_t)
12 | z = _t + 2.0 * math.sin(75 * _t)
13 | data.append([x, y, z])
14 | c = (
15 | Line3D()
16 | .add(
17 | "",
18 | data,
19 | xaxis3d_opts=opts.Axis3DOpts(Faker.clock, type_="value"),
20 | yaxis3d_opts=opts.Axis3DOpts(Faker.week_en, type_="value"),
21 | grid3d_opts=opts.Grid3DOpts(
22 | width=100, depth=100, rotate_speed=150, is_rotate=True
23 | ),
24 | )
25 | .set_global_opts(
26 | visualmap_opts=opts.VisualMapOpts(
27 | max_=30, min_=0, range_color=Faker.visual_color
28 | ),
29 | title_opts=opts.TitleOpts(title="Line3D-旋转的弹簧"),
30 | )
31 | .render("line3d_autorotate.html")
32 | )
33 |
--------------------------------------------------------------------------------
/python-data-analysis/pandas-demo/GroupByDemo.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 |
3 | # 数据导入
4 | epidemic_dxy = pd.read_excel("epidemic_dxy.xlsx")
5 |
6 | print(epidemic_dxy.groupby(['continents']))
7 |
8 | print(epidemic_dxy.groupby(['continents']).count())
9 |
10 | # pd.set_option('display.max_columns', None)
11 | print(epidemic_dxy.groupby(['continents']).sum())
12 |
13 | print(epidemic_dxy.groupby(['continents'])['confirmedCount', 'suspectedCount', 'curedCount', 'deadCount'].sum())
14 |
15 | print(epidemic_dxy.groupby(epidemic_dxy['continents'])['confirmedCount', 'suspectedCount', 'curedCount', 'deadCount'].sum())
16 |
17 | print(epidemic_dxy.groupby([epidemic_dxy['continents'], epidemic_dxy['provinceName']])['confirmedCount', 'suspectedCount', 'curedCount', 'deadCount'].sum())
18 |
19 | print(epidemic_dxy.groupby(epidemic_dxy['continents'])['confirmedCount', 'suspectedCount', 'curedCount', 'deadCount'].aggregate(['count', 'sum']))
20 |
21 | new_dataframe = epidemic_dxy.groupby(epidemic_dxy['continents'])['confirmedCount', 'suspectedCount', 'curedCount', 'deadCount'].sum().reset_index()
22 |
23 | print(new_dataframe)
--------------------------------------------------------------------------------
/python-spider/bs4-demo/bs4_demo.py:
--------------------------------------------------------------------------------
1 | import requests
2 | from bs4 import BeautifulSoup
3 |
4 | response = requests.get('https://www.geekdigging.com/')
5 | soup = BeautifulSoup(response.content, "html5lib")
6 | # 打印内容过长，先注释
7 | # print(soup.prettify())
8 |
9 | print(soup.title)
10 | print(type(soup.title))
11 | print(soup.title.string)
12 | print(soup.a)
13 |
14 | tag = soup.section
15 | print(tag.name)
16 |
17 | print(tag['class'])
18 | print(tag.attrs)
19 |
20 | print(soup.title.string)
21 |
22 | print(soup.a.img)
23 | print(type(soup.a.img))
24 | print(soup.a.img.attrs)
25 |
26 | # 获取子节点
27 | print(soup.article.contents)
28 |
29 | for i, child in enumerate(soup.article.children):
30 | print(i, child)
31 |
32 | # 获取所有子孙节点
33 | for i, child in enumerate(soup.article.descendants):
34 | print(i, child)
35 | # 父节点
36 | print(soup.title.parent)
37 |
38 | # 兄弟节点
39 | print('next_sibling：', soup.title.next_sibling)
40 | print('previous_sibling：', soup.title.previous_sibling)
41 | print('next_siblings：', soup.title.next_siblings)
42 | print('previous_siblings：', soup.title.previous_siblings)
--------------------------------------------------------------------------------
/python-data-analysis/pandas-demo/TableJoinDemo.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 |
3 | df1 = pd.read_excel("table_join_exp.xlsx", sheet_name='Sheet1')
4 | print(df1)
5 |
6 | df2 = pd.read_excel("table_join_exp.xlsx", sheet_name='Sheet2')
7 | print(df2)
8 |
9 | print(pd.merge(df1, df2))
10 |
11 | df3 = pd.read_excel("table_join_exp.xlsx", sheet_name='Sheet3')
12 | print(df3)
13 |
14 | print(pd.merge(df1, df3, on='编号'))
15 |
16 | df4 = pd.read_excel("table_join_exp.xlsx", sheet_name='Sheet4')
17 | print(df4)
18 |
19 | print(pd.merge(df4, df3, on='编号'))
20 |
21 | df5 = pd.read_excel("table_join_exp.xlsx", sheet_name='Sheet5')
22 | print(df5)
23 |
24 | # 内连接
25 | print(pd.merge(df5, df3, on='编号', how='inner'))
26 |
27 | # 左连接
28 | print(pd.merge(df5, df3, on='编号', how='left'))
29 |
30 | # 右连接
31 | print(pd.merge(df5, df3, on='编号', how='right'))
32 |
33 | # 外连接
34 | print(pd.merge(df5, df3, on='编号', how='outer'))
35 |
36 | df6 = pd.read_excel("table_join_exp.xlsx", sheet_name='Sheet6')
37 | print(df6)
38 |
39 | # 纵向拼接
40 | print(pd.concat([df5, df6]))
41 |
42 | print(pd.concat([df5, df6], ignore_index=True))
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2020 meteor1993
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/python-opencv/blog13-scharr/demo-summary.py:
--------------------------------------------------------------------------------
1 | import cv2 as cv
2 | import numpy as np
3 | import matplotlib.pyplot as plt
4 |
5 | #读取图像
6 | img = cv.imread("maliao.jpg")
7 | rgb_img = cv.cvtColor(img, cv.COLOR_BGR2RGB)
8 |
9 | gray_img = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
10 |
11 | #高斯滤波
12 | gaussianBlur = cv.GaussianBlur(gray_img, (3,3), 0)
13 |
14 | #阈值处理
15 | ret, binary = cv.threshold(gaussianBlur, 127, 255, cv.THRESH_BINARY)
16 |
17 | #Scharr算子
18 | x = cv.Scharr(gray_img, cv.CV_32F, 1, 0) #X方向
19 | y = cv.Scharr(gray_img, cv.CV_32F, 0, 1) #Y方向
20 | absX = cv.convertScaleAbs(x)
21 | absY = cv.convertScaleAbs(y)
22 | Scharr = cv.addWeighted(absX, 0.5, absY, 0.5, 0)
23 |
24 | #LOG算子
25 | gaussian = cv.GaussianBlur(gray_img, (3,3), 0) #先通过高斯滤波降噪
26 | dst = cv.Laplacian(gaussian, cv.CV_16S, ksize = 3) #再通过拉普拉斯算子做边缘检测
27 | LOG = cv.convertScaleAbs(dst)
28 |
29 | #效果图
30 | titles = ['Source Image', 'Gray Image', 'Scharr Image', 'LOG Image']
31 | images = [rgb_img, gray_img, Scharr, LOG]
32 |
33 | for i in np.arange(4):
34 | plt.subplot(2, 2, i+1),plt.imshow(images[i],'gray')
35 | plt.title(titles[i])
36 | plt.xticks([]),plt.yticks([])
37 | plt.show()
38 |
--------------------------------------------------------------------------------
/python-spider/urllib-request/Demo_Error.py:
--------------------------------------------------------------------------------
1 | from urllib import request, error
2 | import socket
3 |
4 | # 访问明显不存在的地址，报错：Not Found
5 | try:
6 | response = request.urlopen('https://www.geekdigging.com/aa')
7 | except error.URLError as e:
8 | print(e.reason)
9 |
10 | # 访问超时，报错：timed out
11 | try:
12 | response = request.urlopen('https://www.baidu.com', timeout=0.001)
13 | except error.URLError as e:
14 | print(e.reason)
15 |
16 | # 异常类型示例
17 | try:
18 | response = request.urlopen('https://www.baidu.com', timeout=0.001)
19 | except error.URLError as e:
20 | print(type(e.reason))
21 | if isinstance(e.reason, socket.timeout):
22 | print('TIME OUT')
23 |
24 | # 访问明显不存在的地址，使用 HTTPError 捕捉异常
25 | try:
26 | response = request.urlopen('https://www.geekdigging.com/aa')
27 | except error.HTTPError as e:
28 | print(e.reason, e.code, e.headers, sep='\n')
29 |
30 | # 优化异常捕捉代码
31 | try:
32 | response = request.urlopen('https://www.geekdigging.com/aa')
33 | except error.HTTPError as e:
34 | print(e.reason, e.code, e.headers, sep='\n')
35 | except error.URLError as e:
36 | print(e.reason)
37 | else:
38 | print('Request Success!')
--------------------------------------------------------------------------------
/base-data-tuple/Demo.py:
--------------------------------------------------------------------------------
1 | tuple1 = "Python", "Java", 2011, 2015
2 | print(tuple1)
3 |
4 | tuple2 = ("Python", "Java", 2011, 2015)
5 | print(tuple2)
6 | print(type(tuple2))
7 |
8 | tuple3 = ("Python", "Java", [1 ,2, 'python', 'java'], 2011, 2015)
9 | print(tuple3)
10 |
11 | tuple4 = (0 ,1, 2, 3, 4, 5, 6, 7, 8, 9)
12 | print(tuple4)
13 | # 索引
14 | print(tuple4[2])
15 | # 索引
16 | print(tuple4[-2])
17 | # 切片
18 | print(tuple4[0:8:2])
19 | # 切片
20 | print(tuple4[8:1:-1])
21 |
22 | tuple5 = (2333, '98k')
23 | # 连接
24 | print(tuple4 + tuple5)
25 | # 循环
26 | for index in tuple4:
27 | print(index)
28 | # 查找元素是否存在
29 | print(1 in tuple4)
30 | print(11 in tuple4)
31 | # 删除元组
32 | # del tuple5
33 | # print(tuple5)
34 |
35 | # 取最大
36 | print(max(tuple4))
37 | # 取最小
38 | print(min(tuple4))
39 | # 元组长度
40 | print(len(tuple4))
41 | # 修改元组
42 | # tuple4[0] = 11
43 |
44 | # 相互转化
45 | print(type(tuple4))
46 | print(list(tuple4))
47 | print(type(list(tuple4)))
48 | list1 = [0 ,1, 2, 3, 4, 5, 6, 7, 8, 9]
49 | print(type(list1))
50 | print(tuple(list1))
51 | print(type(tuple(list1)))
52 |
53 | # 元组解包
54 | tuple6 = (1, 2, 3)
55 | print(tuple6)
56 | a, b, c = tuple6
57 | print(a, b, c)
--------------------------------------------------------------------------------
/python-opencv/blog9-open/demo-noise.py:
--------------------------------------------------------------------------------
1 | import cv2 as cv
2 | import numpy as np
3 | import matplotlib.pyplot as plt
4 |
5 | # 读取图片
6 | img = cv.imread("demo.png", cv.IMREAD_UNCHANGED)
7 | source = cv.cvtColor(img, cv.COLOR_BGR2RGB)
8 | rows, cols, chn = source.shape
9 |
10 | # 加噪声-白点噪声
11 | for i in range(500):
12 | x = np.random.randint(0, rows)
13 | y = np.random.randint(0, cols)
14 | source[x, y, :] = 255
15 |
16 | # 图像保存 白点噪声图像
17 | cv.imwrite("demo_noise_white.jpg", source)
18 | print("白点噪声添加完成")
19 |
20 | # 重新读取图像
21 | img1 = cv.imread("demo.png", cv.IMREAD_UNCHANGED)
22 | source1 = cv.cvtColor(img1, cv.COLOR_BGR2RGB)
23 |
24 | # 加噪声-黑点噪声
25 | for i in range(1000):
26 | x = np.random.randint(0, rows)
27 | y = np.random.randint(0, cols)
28 | source1[x, y, :] = 0
29 |
30 | # 图像保存 黑点噪声图像
31 | cv.imwrite("demo_noise_black.jpg", source1)
32 | print("黑点噪声添加完成")
33 |
34 | # 显示结果
35 | titles = ['White Img','Black Img']
36 | images = [source, source1]
37 |
38 | # matplotlib 绘图
39 | for i in range(2):
40 | plt.subplot(1, 2, i+1), plt.imshow(images[i],'gray')
41 | plt.title(titles[i])
42 | plt.xticks([]),plt.yticks([])
43 |
44 | plt.show()
45 |
--------------------------------------------------------------------------------
/python-data-analysis/series/demo1.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 |
4 | s = pd.Series(np.random.rand(5), index=['a', 'b', 'c', 'd', 'e'])
5 | print(s)
6 | print(s.index)
7 |
8 | s1 = pd.Series(np.random.randn(5))
9 | print(s1)
10 |
11 | d = {'b': 1, 'a': 0, 'c': 2}
12 | s2 = pd.Series(d)
13 | print(s2)
14 |
15 | s3 = pd.Series(d, index=['b', 'c', 'd', 'a'])
16 | print(s3)
17 |
18 | s4 = pd.Series(5., index=['a', 'b', 'c', 'd', 'e'])
19 | print(s4)
20 |
21 | print(s[0])
22 | print(s[:3])
23 | print(s[s > s.median()])
24 | print(s[[4, 3, 1]])
25 | # 打印 e 的幂次方， e 是一个常数为 2.71828
26 | print (np.exp(s))
27 | # 打印 s 里每个元素的开方
28 | print (np.sqrt(s))
29 | print(s.dtype)
30 | print(s.array)
31 | print(s.to_numpy())
32 |
33 | print(s['a'])
34 | s['e'] = 12.
35 | print(s)
36 |
37 | print('e' in s)
38 | print('f' in s)
39 | # 抛出 KeyError 异常
40 | # print(s['f'])
41 |
42 | print(s.get('f'))
43 | print(s.get('f', np.nan))
44 |
45 | print(s[1:] + s[:-1])
46 |
47 | s5 = pd.Series(np.random.randn(5), name='my_series')
48 | print(s5)
49 | print(s5.name)
50 | print(id(s5))
51 |
52 | # 重命名 series
53 | s6 = s5.rename("my_series_different")
54 | print(s6)
55 | print(id(s6))
--------------------------------------------------------------------------------
/python-spider/scrapy_selenium_demo/scrapy_selenium_demo/pipelines.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Define your item pipelines here
4 | #
5 | # Don't forget to add your pipeline to the ITEM_PIPELINES setting
6 | # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
7 |
8 |
9 | class ScrapySeleniumDemoPipeline(object):
10 | def process_item(self, item, spider):
11 | return item
12 |
13 |
14 | import pymongo
15 |
16 | class MongoPipeline(object):
17 | def __init__(self, mongo_uri, mongo_db):
18 | self.mongo_uri = mongo_uri
19 | self.mongo_db = mongo_db
20 |
21 | @classmethod
22 | def from_crawler(cls, crawler):
23 | return cls(mongo_uri=crawler.settings.get('MONGO_URI'),
24 | mongo_db=crawler.settings.get('MONGO_DB')
25 | )
26 |
27 | def open_spider(self, spider):
28 | self.client = pymongo.MongoClient(self.mongo_uri)
29 | self.db = self.client[self.mongo_db]
30 |
31 | def process_item(self, item, spider):
32 | name = item.__class__.__name__
33 | self.db[name].insert(dict(item))
34 | return item
35 |
36 | def close_spider(self, spider):
37 | self.client.close()
--------------------------------------------------------------------------------
/base-data-dict/Demo1.py:
--------------------------------------------------------------------------------
1 | import copy
2 |
3 | dict1 = {'name': 'geekdigging', 'age': 2}
4 |
5 | print(dict1.keys())
6 | print(list(dict1.keys()))
7 | print(type(list(dict1.keys())))
8 |
9 | print(dict1.values())
10 | print(list(dict1.values()))
11 | print(type(list(dict1.values())))
12 |
13 | print(dict1.items())
14 | print(list(dict1.items()))
15 | print(type(list(dict1.items())))
16 |
17 | print(dict1.get('name'))
18 | print(dict1.get('geekdigging'))
19 |
20 | print(dict1.pop('age'))
21 | print(dict1)
22 |
23 | dict1.setdefault('age')
24 | print(dict1)
25 |
26 | dict2 = {'sex': 'male'}
27 | dict1.update(dict2)
28 | print(dict1)
29 |
30 | dict2.clear()
31 | print(dict2)
32 |
33 | dict3 = {'name': 'geekdigging', 'age': [1, 2, 3]}
34 | # 浅拷贝: 引用对象
35 | dict4 = dict3
36 | print(id(dict3))
37 | print(id(dict4))
38 | # 浅拷贝：深拷贝父对象（一级目录），子对象（二级目录）不拷贝，还是引用
39 | dict5 = dict3.copy()
40 | dict3['age'].remove(1)
41 | print(dict3)
42 | print(dict5)
43 | print(id(dict3))
44 | print(id(dict5))
45 |
46 | dict3 = {'name': 'geekdigging', 'age': [1, 2, 3]}
47 | dict6 = copy.deepcopy(dict3)
48 | dict3['age'].remove(1)
49 | print(dict3)
50 | print(dict6)
51 | print(id(dict3))
52 | print(id(dict6))
--------------------------------------------------------------------------------
/python-spider/first_scrapy/first_scrapy/spiders/MzituSpider.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | from scrapy import Spider, Request
3 | from first_scrapy.items import ImageItem
4 |
5 | class MziTuSpider(Spider):
6 | name = 'MziTuSpider'
7 | allowed_domains = ['www.mzitu.com']
8 | start_urls = ['https://www.mzitu.com/mm/']
9 |
10 | def start_requests(self):
11 | headers = {
12 | 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36',
13 | 'referer': 'https://www.mzitu.com/'
14 | }
15 | yield Request('https://www.mzitu.com/mm/', self.parse, headers = headers)
16 |
17 | def parse(self, response):
18 | imageList = response.css('.postlist ul li')
19 | for image in imageList:
20 | item = ImageItem()
21 | item['id'] = image.css('a::attr("href")').extract_first().split('/')[3]
22 | item['url'] = image.css('a::attr("href")').extract_first()
23 | item['title'] = image.css('a img::attr("alt")').extract_first()
24 | item['thumb'] = image.css('a img::attr("data-original")').extract_first()
25 | yield item
--------------------------------------------------------------------------------
/base-excel/Demo1.py:
--------------------------------------------------------------------------------
1 | import xlsxwriter
2 | import datetime
3 |
4 | workbook = xlsxwriter.Workbook('demo.xlsx')
5 |
6 | sheet1 = workbook.add_worksheet('test_sheet')
7 |
8 | workfomat = workbook.add_format()
9 | # 字体加粗
10 | workfomat.set_bold(True)
11 | # 单元格边框宽度
12 | workfomat.set_border(1)
13 | # 对齐方式
14 | workfomat.set_align('left')
15 | # 格式化数据格式为小数点后两位
16 | workfomat.set_num_format('0.00')
17 |
18 | heads = ['', '语文', '数学', '英语']
19 | datas = [
20 | ['小明', 76, 85, 95],
21 | ['小红', 85, 58, 92],
22 | ['小王', 98, 96, 91]
23 | ]
24 |
25 | sheet1.write_row('A1', heads, workfomat)
26 |
27 | sheet1.write_row('A2', datas[0], workfomat)
28 | sheet1.write_row('A3', datas[1], workfomat)
29 | sheet1.write_row('A4', datas[2], workfomat)
30 |
31 | fomat1 = workbook.add_format({'num_format': 'yyyy/mm/dd/ hh:mm:ss'})
32 |
33 | sheet1.write_datetime('E5', datetime.datetime(2019, 11, 9, 22, 44, 26), fomat1)
34 |
35 | sheet1.insert_image('I6', 'wx.jpg')
36 |
37 | chart = workbook.add_chart({'type': 'column'})
38 |
39 | chart.add_series({'values': '=test_sheet!$B$2:$B$4'})
40 | chart.add_series({'values': '=test_sheet!$C$2:$C$4'})
41 | chart.add_series({'values': '=test_sheet!$D$2:$D$4'})
42 |
43 | sheet1.insert_chart('A7', chart)
44 |
45 | workbook.close()
--------------------------------------------------------------------------------
/python-data-analysis/pandas-demo/DataOperation.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 |
3 | # 数据导入
4 | data_train = pd.read_csv("train.csv")
5 |
6 | # 数据查看
7 | print(data_train.info())
8 |
9 | pd.set_option('display.max_columns', None)
10 | print(data_train.describe())
11 |
12 | print(data_train)
13 |
14 | print(data_train['SibSp'] + data_train['Parch'])
15 |
16 | print(data_train['SibSp'] + data_train['Parch'] + 1 - data_train['Survived'])
17 |
18 | print(data_train['Survived'] > (data_train['SibSp'] + data_train['Parch']))
19 |
20 | # 按列统计
21 | print(data_train.count())
22 | # 按行统计
23 | print(data_train.count(axis=1))
24 | # 某一列单独统计
25 | print(data_train['Age'].count())
26 |
27 | # 按列求和
28 | print(data_train.sum())
29 | # 按行求和
30 | print(data_train.sum(axis=1))
31 |
32 | print(data_train.mean())
33 |
34 | print(data_train.max())
35 | print('-----------------------------------------')
36 | print(data_train.min())
37 |
38 | print(data_train.median())
39 |
40 | print(data_train.mode())
41 | # 单独获取某列众数
42 | print(data_train['Sex'].mode())
43 |
44 | print(data_train.var())
45 |
46 | print(data_train.std())
47 |
48 | print(data_train.quantile(0.25))
49 | print('-----------------------------------------')
50 | print(data_train.quantile(0.5))
51 |
52 | print(data_train.corr(method='pearson'))
--------------------------------------------------------------------------------
/python-data-analysis/pandas-demo/demo1.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 |
4 | dates = pd.date_range('20200101', periods=6)
5 | df = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list('ABCD'))
6 | print(df)
7 |
8 | # 查看头部数据
9 | print(df.head(1))
10 | # 查看尾部数据
11 | print(df.tail(2))
12 | # 获取索引
13 | print(df.index)
14 | # 获取列名
15 | print(df.columns)
16 | # 查看数据的统计摘要
17 | print(df.describe())
18 | # 转置数据
19 | print(df.T)
20 | # 参考 https://www.jianshu.com/p/f0ed06cd5003
21 | # 两种排序 sort_index() 和 sort_values()
22 | df1 = pd.DataFrame({'b' :[1,2,3,2],'a':[4,3,2,1],'c':[1,3,8,2]},index=[2,0,1,3])
23 | print(df1)
24 | # sort_values() 排序
25 | # 按 b 列升序排序
26 | print(df1.sort_values(by='b'))
27 | # 先按 b 列降序，再按 a 列升序排序
28 | print(df1.sort_values(by=['b','a'],axis=0,ascending=[False,True]))
29 | # 按行 3 升序排列,必须指定 axis = 1
30 | print(df1.sort_values(by=3,axis=1))
31 | # 按行 3 升序，行 0 降排列
32 | print(df1.sort_values(by=[3,0],axis=1,ascending=[True,False]))
33 | # sort_index() 排序
34 | # 默认按「行标签」升序排列
35 | print(df1.sort_index())
36 | # 按「列标签」升序排列
37 | print(df1.sort_index(axis=1))
38 | # 先按 b 列「降序」排列，因为 b 列中有相同值，相同值再按 a 列的「升序」排列
39 | print(df1.sort_index(by=['b','a'],ascending=[False,True]))
40 | # 先按 a 列「降序」排列，而 a 列中没有相同值，因此这里按 b 列的「升序」排列不起作用。
41 | print(df1.sort_index(by=['a','b'],ascending=[False,True]))
--------------------------------------------------------------------------------
/python-data-analysis/pandas-demo/demo2.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 |
4 | dates = pd.date_range('20200101', periods=6)
5 | df = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list('ABCD'))
6 | print(df)
7 |
8 | # 获取单列，获得 Series
9 | print(df['A'])
10 | # 行切片
11 | print(df[0:3])
12 | print(df['20200101' : '20200103'])
13 |
14 | # loc，可以使用 column 名和 index 名进行定位
15 | # 用标签提取一行数据
16 | print(df.loc[dates[0]])
17 | # 用标签提取多列数据
18 | print(df.loc[:, ['A', 'B']])
19 | # 用标签进行切片操作，同时制定行与列的结束点
20 | print(df.loc['20200101':'20200103', ['A', 'B']])
21 | # 返回一行中的两列
22 | print(df.loc['20200101', ['A', 'B']])
23 | # 获取某个标量值
24 | print(df.loc[dates[0], 'A'])
25 |
26 | # iloc，即 index locate 用 index 索引进行定位，所以参数是整型
27 | # 用整数位置选择
28 | print(df.iloc[3])
29 | # 使用整数按行和列进行切片操作
30 | print(df.iloc[3:5, 0:2])
31 | # 用整数列表按位置切片
32 | print(df.iloc[[1, 2, 4], [0, 2]])
33 | # 整行切片
34 | print(df.iloc[1:3, :])
35 | # 整列切片
36 | print(df.iloc[:, 1:3])
37 | # 获取某个标量值 同上
38 | print(df.iloc[1, 1])
39 |
40 | # at 使用方法与 loc 类似，但是比 loc 有更快的访问数据的速度，而且只能访问单个元素，不能访问多个元素。
41 | print(df.at[dates[0], 'A'])
42 |
43 | # iat iat 对于 iloc 的关系就像 at 对于 loc 的关系，是一种更快的基于索引位置的选择方法，同 at 一样只能访问单个元素。
44 | print(df.iat[1, 1])
45 |
46 | # 用单列的值选择数据
47 | print(df[df.A > 0])
48 |
49 | # 选择 df 里满足条件的值
50 | print(df[df < 0])
51 |
52 |
--------------------------------------------------------------------------------
/python-spider/xpath-demo/xpath_demo_advanced.py:
--------------------------------------------------------------------------------
1 | from lxml import etree
2 | import requests
3 |
4 | response = requests.get('https://www.geekdigging.com/')
5 | html_str = response.content.decode('UTF-8')
6 | html = etree.HTML(html_str)
7 |
8 | result_1 = html.xpath('/html/body/section/div/div/main/article[1]/div[2]/div/h3/a/text()')
9 | print(result_1)
10 |
11 | result_2 = html.xpath('/html/body/section/div/div/main/article[1]/div[2]/div/h3/a/@href')
12 | print(result_2)
13 |
14 | result_3 = html.xpath('//div[contains(@class, "post-head")]')
15 | print(result_3)
16 |
17 | result_4 = html.xpath('//img[@class="img-ajax" and @alt="小白学 Python 爬虫（18）：Requests 进阶操作"]')
18 | print(result_4)
19 |
20 | result_5 = html.xpath('//article/div/div/h3[@class="post-title"]/a/text()')
21 | print(result_5)
22 | result_6 = html.xpath('//article[1]/div/div/h3[@class="post-title"]/a/text()')
23 | print(result_6)
24 | result_7 = html.xpath('//article[last()]/div/div/h3[@class="post-title"]/a/text()')
25 | print(result_7)
26 | result_8 = html.xpath('//article[position() < 5]/div/div/h3[@class="post-title"]/a/text()')
27 | print(result_8)
28 |
29 | # 节点轴示例
30 | # 获取所有祖先节点
31 | result_9 = html.xpath('//article/ancestor::*')
32 | print(result_9)
33 | # 获取祖先节点 main 节点
34 | result_10 = html.xpath('//article/ancestor::main')
35 | print(result_10)
--------------------------------------------------------------------------------
/base-data-list/Demo1.py:
--------------------------------------------------------------------------------
1 | list1 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
2 |
3 | # 省略步长时默认为 1
4 | print(list1[3:8])
5 | # 步长为 2
6 | print(list1[3:8:2])
7 | # 从索引 3 开始取到最后
8 | print(list1[3:])
9 | # 从头开始取，取到索引 8 ，并且索引 8 娶不到
10 | print(list1[:8])
11 | # 取所有，步长为 3
12 | print(list1[::3])
13 | # 从索引 1 开始，取到倒数第 2 个，并且倒数第 2 个 取不到
14 | print(list1[1:-2])
15 | # 取所有
16 | print(list1[:])
17 | # 取逆序列表
18 | print(list1[::-1])
19 | # 取逆序，并且步长为 2
20 | print(list1[8:1:-2])
21 |
22 | list1.append("Python")
23 | print(list1)
24 |
25 | list2 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
26 |
27 | list2.append("Python")
28 | list2.append("Python")
29 | list2.append("Python")
30 | list2.append(1)
31 | print(list2.count("Python"))
32 | print(list2.count(1))
33 |
34 | list1.extend(list2)
35 | print(list1)
36 |
37 | print(list1.index("Python"))
38 |
39 | list1.insert(0, "Hello")
40 | print(list1)
41 |
42 | list3 = [0, 1, 2]
43 | list4 = [2, 2]
44 | list3.insert(1, list4)
45 | print(list3)
46 |
47 | list3.pop()
48 | print(list3)
49 |
50 | list3.pop(1)
51 | print(list3)
52 |
53 | list5 = [1, 2, 3, 4, 4, 5]
54 | list5.remove(4)
55 | print(list5)
56 | print(id(list5))
57 | list5.reverse()
58 | print(list5)
59 | print(id(list5))
60 | print(id(list5[::-1]))
61 |
62 | list6 = [2, 5, 1, 9, 6, 3]
63 | list6.sort()
64 | print(list6)
65 | list6.sort(reverse=True)
66 | print(list6)
--------------------------------------------------------------------------------
/base-generator/Demo.py:
--------------------------------------------------------------------------------
1 | list1 = [x*x for x in range(10)]
2 | print(list1)
3 | # list2 列表创建语句慎重执行，先注释
4 | # list2 = [x*x for x in range(1000000000000000000000000)]
5 |
6 | generator1 = (x*x for x in range(1000000000000000000000000))
7 | print(generator1)
8 | print(type(generator1))
9 |
10 | generator2 = (x*x for x in range(3))
11 | print(next(generator2))
12 | print(next(generator2))
13 | print(next(generator2))
14 | # print(next(generator2))
15 |
16 | generator3 = (x*x for x in range(5))
17 | for index in generator3:
18 | print(index)
19 |
20 | def print_a(max):
21 | i = 0
22 | while i < max:
23 | i += 1
24 | yield i
25 |
26 | a = print_a(10)
27 | print(a)
28 | print(type(a))
29 |
30 | print(next(a))
31 | print(next(a))
32 | print(next(a))
33 | print(next(a))
34 |
35 | print(a.__next__())
36 | print(a.__next__())
37 |
38 | def print_b(max):
39 | i = 0
40 | while i < max:
41 | i += 1
42 | args = yield i
43 | print('传入参数为：' + args)
44 |
45 | b = print_b(20)
46 | print(next(b))
47 | print(b.send('Python'))
48 |
49 | def print_c():
50 | while True:
51 | print('执行 A ')
52 | yield None
53 | def print_d():
54 | while True:
55 | print('执行 B ')
56 | yield None
57 |
58 | c = print_c()
59 | d = print_d()
60 | while True:
61 | c.__next__()
62 | d.__next__()
--------------------------------------------------------------------------------
/base-operator/Demo-1.py:
--------------------------------------------------------------------------------
1 | # 对应《小白学 Python（6）：基础运算符（下）》示例代码
2 |
3 | a = 10
4 | b = 20
5 |
6 | c = a + b
7 | print("c = a + b 的值为：", c)
8 |
9 | c += a
10 | print("c += a 的值为：", c)
11 |
12 | c *= a
13 | print("c *= a 的值为：", c)
14 |
15 | c /= a
16 | print("c /= a 的值为：", c)
17 |
18 | c = 2
19 | c %= a
20 | print("c %= a 的值为：", c)
21 |
22 | c **= a
23 | print("c **= a 的值为：", c)
24 |
25 | c //= a
26 | print("c //= a 的值为：", c)
27 |
28 | print(True and True)
29 | # True
30 | print(True and False)
31 | # False
32 | print(True or True)
33 | # True
34 | print(True or False)
35 | # True
36 | print(False or False)
37 | # False
38 | print(not True)
39 | # False
40 | print(not False)
41 | # True
42 |
43 | str = "asdfghjkl"
44 |
45 | if 'a' in str:
46 | print('a 在字符串 str 中')
47 | else:
48 | print('a 不在字符串 str 中')
49 |
50 | if 'a' not in str:
51 | print('a 不在字符串 str 中')
52 | else:
53 | print('a 在字符串 str 中')
54 |
55 | a = 20
56 | b = 20
57 |
58 | if a is b:
59 | print("a 和 b 有相同的标识")
60 | else:
61 | print("a 和 b 没有相同的标识")
62 |
63 | if id(a) == id(b):
64 | print("a 和 b 有相同的标识")
65 | else:
66 | print("a 和 b 没有相同的标识")
67 |
68 | # 修改变量 b 的值
69 | b = 30
70 | if a is b:
71 | print("a 和 b 有相同的标识")
72 | else:
73 | print("a 和 b 没有相同的标识")
74 |
75 | if a is not b:
76 | print("a 和 b 没有相同的标识")
77 | else:
78 | print("a 和 b 有相同的标识")
--------------------------------------------------------------------------------
/spider-blog/报表sql.sql:
--------------------------------------------------------------------------------
1 | -- 因为掘金使用的是排名是掘力值，顾取出来的数据是负数
2 | SELECT a.read_num - (
3 | SELECT b.read_num
4 | FROM spider_data b
5 | WHERE b.plantform = a.plantform
6 | AND DATE_FORMAT(b.create_date, '%Y-%m-%d') = date_sub(DATE_FORMAT(a.create_date, '%Y-%m-%d'), INTERVAL 1 DAY)
7 | ORDER BY b.create_date DESC LIMIT 1
8 | ) AS read_num, a.fans_num - (
9 | SELECT b.fans_num
10 | FROM spider_data b
11 | WHERE b.plantform = a.plantform
12 | AND DATE_FORMAT(b.create_date, '%Y-%m-%d') = date_sub(DATE_FORMAT(a.create_date, '%Y-%m-%d'), INTERVAL 1 DAY)
13 | ORDER BY b.create_date DESC LIMIT 1
14 | ) AS fans_num
15 | , a.like_num - (
16 | SELECT b.like_num
17 | FROM spider_data b
18 | WHERE b.plantform = a.plantform
19 | AND DATE_FORMAT(b.create_date, '%Y-%m-%d') = date_sub(DATE_FORMAT(a.create_date, '%Y-%m-%d'), INTERVAL 1 DAY)
20 | ORDER BY b.create_date DESC LIMIT 1
21 | ) AS like_num, (
22 | SELECT b.rank_num
23 | FROM spider_data b
24 | WHERE b.plantform = a.plantform
25 | AND DATE_FORMAT(b.create_date, '%Y-%m-%d') = date_sub(DATE_FORMAT(a.create_date, '%Y-%m-%d'), INTERVAL 1 DAY)
26 | ORDER BY b.create_date DESC LIMIT 1
27 | ) - a.rank_num AS rank_num
28 | , a.create_date,a.plantform
29 | FROM (SELECT * FROM spider_data ORDER BY create_date DESC LIMIT 1000000000000000) a
30 |
31 | GROUP BY DATE_FORMAT(a.create_date, '%Y-%m-%d'), a.plantform
32 | ORDER BY a.create_date DESC;
--------------------------------------------------------------------------------
/spider-blog/tongji/src/main/resources/application.yml:
--------------------------------------------------------------------------------
1 | server:
2 | port: 8080
3 | spring:
4 | application:
5 | name: tongji
6 | thymeleaf:
7 | # 关闭thymeleaf缓存 开发时使用 否则没有实时画面
8 | cache: false
9 | # 检查模板是否存在，然后再呈现
10 | check-template-location: true
11 | # Content-Type value.
12 | servlet:
13 | content-type: text/html
14 | # 启用MVC Thymeleaf视图分辨率
15 | enabled: true
16 | # Template encoding.
17 | encoding: UTF-8
18 | # 关闭严格模式
19 | mode: LEGACYHTML5
20 | # Prefix that gets prepended to view names when building a URL.
21 | prefix: classpath:/templates/
22 | # Suffix that gets appended to view names when building a URL.
23 | suffix: .html
24 | datasource:
25 | url: jdbc:mysql://ip:port/blog_data?serverTimezone=Asia/Shanghai&useUnicode=true&characterEncoding=UTF-8&useSSL=false
26 | username: root
27 | password: xxxxxxxxxxx
28 | driver-class-name: com.mysql.cj.jdbc.Driver
29 | type: com.zaxxer.hikari.HikariDataSource
30 | hikari:
31 | auto-commit: true
32 | minimum-idle: 2
33 | idle-timeout: 60000
34 | connection-timeout: 30000
35 | max-lifetime: 1800000
36 | pool-name: DatebookHikariCP
37 | maximum-pool-size: 5
38 | # 配置slq打印日志
39 | mybatis:
40 | type-aliases-package: com.geekdigging.tongji.model
41 | config-location: classpath:mybatis/mybatis-config.xml
42 | mapper-locations: classpath:mybatis/mapper/*.xml
--------------------------------------------------------------------------------
/python-spider/database-docker-conf/mysql/my.cnf:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
2 | #
3 | # This program is free software; you can redistribute it and/or modify
4 | # it under the terms of the GNU General Public License, version 2.0,
5 | # as published by the Free Software Foundation.
6 | #
7 | # This program is also distributed with certain software (including
8 | # but not limited to OpenSSL) that is licensed under separate terms,
9 | # as designated in a particular file or component or in included license
10 | # documentation. The authors of MySQL hereby grant you an additional
11 | # permission to link the program and your derivative works with the
12 | # separately licensed software that they have included with MySQL.
13 | #
14 | # This program is distributed in the hope that it will be useful,
15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 | # GNU General Public License, version 2.0, for more details.
18 | #
19 | # You should have received a copy of the GNU General Public License
20 | # along with this program; if not, write to the Free Software
21 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
22 |
23 | !includedir /etc/mysql/conf.d/
24 | !includedir /etc/mysql/mysql.conf.d/
25 |
26 | sql_mode=STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION
--------------------------------------------------------------------------------
/python-spider/bs4-demo/bs4_demo1.py:
--------------------------------------------------------------------------------
1 | from bs4 import BeautifulSoup
2 |
3 | html_doc = """
4 | The Dormouse's story
5 |
6 | The Dormouse's story
7 |
8 | Once upon a time there were three little sisters; and their names were
9 | Elsie,
10 | Lacie and
11 | Tillie;
12 | and they lived at the bottom of a well.
13 |
14 | ...
15 | """
16 |
17 | soup = BeautifulSoup(html_doc, 'lxml')
18 |
19 | print(soup.find_all(name = "a"))
20 | print(type(soup.find_all(name = "a")[0]))
21 |
22 | for a in soup.find_all(name = "a"):
23 | print(a.string)
24 |
25 | print(soup.find_all(attrs={'id': 'link1'}))
26 | print(soup.find_all(attrs={'id': 'link2'}))
27 | print(type(soup.find_all(attrs={'id': 'link1'})))
28 | print(type(soup.find_all(attrs={'id': 'link2'})))
29 |
30 | import re
31 |
32 | print(soup.find_all(text=re.compile('sisters')))
33 |
34 | print(soup.find_all(id='link1'))
35 | print(soup.find_all(class_='title'))
36 |
37 | print(soup.find_all(href=re.compile("elsie"), id='link1'))
38 |
39 | print(soup.find(name = "a"))
40 | print(type(soup.find(name = "a")))
41 |
42 | print(soup.select('#link1'))
43 | print(type(soup.select('#link1')[0]))
44 | print(soup.select('.story .sister'))
--------------------------------------------------------------------------------
/python-opencv/blog14-pyramid/demo-pyramid.py:
--------------------------------------------------------------------------------
1 | import cv2 as cv
2 |
3 | #高斯金字塔
4 | def gaussian_pyramid(image):
5 | level = 3 #设置金字塔的层数为3
6 | temp = image.copy() #拷贝图像
7 | gaussian_images = [] #建立一个空列表
8 | for i in range(level):
9 | dst = cv.pyrDown(temp) #先对图像进行高斯平滑，然后再进行降采样（将图像尺寸行和列方向缩减一半）
10 | gaussian_images.append(dst) #在列表末尾添加新的对象
11 | cv.imshow("gaussian"+str(i), dst)
12 | temp = dst.copy()
13 | return gaussian_images
14 |
15 |
16 | #拉普拉斯金字塔
17 | def laplacian_pyramid(image):
18 | gaussian_images = gaussian_pyramid(image) #做拉普拉斯金字塔必须用到高斯金字塔的结果
19 | level = len(gaussian_images)
20 | for i in range(level-1, -1, -1):
21 | if (i-1) < 0:
22 | expand = cv.pyrUp(gaussian_images[i], dstsize = image.shape[:2])
23 | laplacian = cv.subtract(image, expand)
24 | # 展示差值图像
25 | cv.imshow("laplacian_down_"+str(i), laplacian)
26 | else:
27 | expand = cv.pyrUp(gaussian_images[i], dstsize = gaussian_images[i-1].shape[:2])
28 | laplacian = cv.subtract(gaussian_images[i-1], expand)
29 | # 展示差值图像
30 | cv.imshow("laplacian_down_"+str(i), laplacian)
31 |
32 |
33 | src = cv.imread('maliao.jpg')
34 | print(src.shape)
35 | # 先将图像转化成正方形，否则会报错
36 | input_image = cv.resize(src, (560, 560))
37 | # 设置为 WINDOW_NORMAL 可以任意缩放
38 | cv.namedWindow('input_image', cv.WINDOW_AUTOSIZE)
39 | cv.imshow('input_image', input_image)
40 | laplacian_pyramid(input_image)
41 | cv.waitKey(0)
42 | cv.destroyAllWindows()
--------------------------------------------------------------------------------