├── codecov.yml ├── web ├── static │ ├── .eslintignore │ ├── favicon.ico │ ├── build │ │ ├── logo.png │ │ ├── vue-loader.conf.js │ │ ├── check-versions.js │ │ ├── build.js │ │ ├── utils.js │ │ ├── webpack.dev.conf.js │ │ └── webpack.base.conf.js │ ├── src │ │ ├── assets │ │ │ ├── 401_images │ │ │ │ └── 401.gif │ │ │ ├── 404_images │ │ │ │ ├── 404.png │ │ │ │ └── 404_cloud.png │ │ │ ├── default_avatar │ │ │ │ └── gopher.png │ │ │ └── custom-theme │ │ │ │ └── fonts │ │ │ │ ├── element-icons.ttf │ │ │ │ └── element-icons.woff │ │ ├── App.vue │ │ ├── views │ │ │ ├── svg-icons │ │ │ │ ├── generateIconsView.js │ │ │ │ └── index.vue │ │ │ ├── layout │ │ │ │ ├── components │ │ │ │ │ ├── index.js │ │ │ │ │ ├── AppMain.vue │ │ │ │ │ └── Sidebar │ │ │ │ │ │ ├── index.vue │ │ │ │ │ │ └── SidebarItem.vue │ │ │ │ ├── mixin │ │ │ │ │ └── ResizeHandler.js │ │ │ │ └── Layout.vue │ │ │ ├── login │ │ │ │ └── authredirect.vue │ │ │ ├── dashboard │ │ │ │ └── index.vue │ │ │ ├── errorPage │ │ │ │ └── 401.vue │ │ │ └── exportdb │ │ │ │ └── add.vue │ │ ├── utils │ │ │ ├── createUniqueString.js │ │ │ ├── auth.js │ │ │ ├── i18n.js │ │ │ ├── permission.js │ │ │ ├── clipboard.js │ │ │ ├── openWindow.js │ │ │ ├── validate.js │ │ │ └── request.js │ │ ├── styles │ │ │ ├── variables.scss │ │ │ ├── transition.scss │ │ │ ├── element-ui.scss │ │ │ ├── mixin.scss │ │ │ ├── btn.scss │ │ │ ├── sidebar.scss │ │ │ └── index.scss │ │ ├── directive │ │ │ ├── waves │ │ │ │ ├── index.js │ │ │ │ ├── waves.css │ │ │ │ └── waves.js │ │ │ ├── permission │ │ │ │ ├── index.js │ │ │ │ └── permission.js │ │ │ └── sticky.js │ │ ├── store │ │ │ ├── modules │ │ │ │ ├── errorLog.js │ │ │ │ ├── app.js │ │ │ │ ├── permission.js │ │ │ │ ├── tagsView.js │ │ │ │ └── user.js │ │ │ ├── index.js │ │ │ └── getters.js │ │ ├── api │ │ │ ├── login.js │ │ │ ├── exportdb.js │ │ │ └── task.js │ │ ├── icons │ │ │ ├── index.js │ │ │ └── svg │ │ │ │ ├── chart.svg │ │ │ │ ├── money.svg │ │ │ │ ├── guide.svg │ │ │ │ ├── email.svg │ │ │ │ ├── component.svg │ │ │ │ ├── documentation.svg │ │ │ │ ├── user.svg │ │ │ │ ├── excel.svg │ │ │ │ ├── message.svg │ │ │ │ ├── drag.svg │ │ │ │ ├── edit.svg │ │ │ │ ├── icon.svg │ │ │ │ ├── example.svg │ │ │ │ ├── lock.svg │ │ │ │ ├── theme.svg │ │ │ │ ├── list.svg │ │ │ │ ├── tab.svg │ │ │ │ ├── nested.svg │ │ │ │ ├── star.svg │ │ │ │ ├── password.svg │ │ │ │ ├── clipboard.svg │ │ │ │ ├── peoples.svg │ │ │ │ ├── language.svg │ │ │ │ ├── 404.svg │ │ │ │ ├── shoppingCard.svg │ │ │ │ ├── bug.svg │ │ │ │ ├── people.svg │ │ │ │ ├── table.svg │ │ │ │ ├── eye.svg │ │ │ │ ├── international.svg │ │ │ │ ├── dashboard.svg │ │ │ │ ├── wechat.svg │ │ │ │ ├── zip.svg │ │ │ │ └── form.svg │ │ ├── errorLog.js │ │ ├── components │ │ │ ├── SvgIcon │ │ │ │ └── index.vue │ │ │ ├── LangSelect │ │ │ │ └── index.vue │ │ │ ├── Breadcrumb │ │ │ │ └── index.vue │ │ │ ├── Hamburger │ │ │ │ └── index.vue │ │ │ ├── Screenfull │ │ │ │ └── index.vue │ │ │ ├── ScrollPane │ │ │ │ └── index.vue │ │ │ └── ErrorLog │ │ │ │ └── index.vue │ │ ├── lang │ │ │ ├── index.js │ │ │ ├── zh.js │ │ │ └── en.js │ │ ├── main.js │ │ ├── filters │ │ │ └── index.js │ │ └── permission.js │ ├── config │ │ ├── sit.env.js │ │ ├── dev.env.js │ │ ├── prod.env.js │ │ └── index.js │ ├── .editorconfig │ ├── .postcssrc.js │ ├── .babelrc │ ├── .gitignore │ ├── index.html │ └── LICENSE ├── service │ ├── cron_test.go │ ├── taskstatus.go │ ├── checktask.go │ └── cron.go ├── router │ ├── task │ │ ├── lock_test.go │ │ ├── task_lock.go │ │ ├── get_task_list.go │ │ ├── get_task_by_id.go │ │ ├── stop_task.go │ │ ├── create_task.go │ │ ├── restart_task.go │ │ └── start_task.go │ ├── user │ │ └── get_user_info.go │ ├── exportdb │ │ ├── delete.go │ │ ├── get_list.go │ │ └── create.go │ ├── rule │ │ └── get_rules.go │ └── route.go ├── model │ ├── model_test.go │ ├── task_test.go │ ├── exportdb_test.go │ ├── exportdb.go │ ├── user_test.go │ └── user.go ├── web.go └── core │ ├── db.go │ └── db_test.go ├── gospider-task-create.png ├── gospider-task-list.png ├── gospider-exportdb-list.png ├── common ├── types.go ├── enu.go └── db.go ├── .travis.yml ├── .gitignore ├── spider ├── task_test.go ├── task.go ├── task_rule_test.go ├── errors.go └── task_rule.go ├── example_gospider_test.go ├── _example ├── main.go └── rule │ └── baidunews │ └── baidu_news.go ├── LICENSE └── go.mod /codecov.yml: -------------------------------------------------------------------------------- 1 | ignore: 2 | - "web/model/autogenerated_*.go" -------------------------------------------------------------------------------- /web/static/.eslintignore: -------------------------------------------------------------------------------- 1 | build/*.js 2 | config/*.js 3 | src/assets 4 | -------------------------------------------------------------------------------- /gospider-task-create.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nange/gospider/HEAD/gospider-task-create.png -------------------------------------------------------------------------------- /gospider-task-list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nange/gospider/HEAD/gospider-task-list.png -------------------------------------------------------------------------------- /web/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nange/gospider/HEAD/web/static/favicon.ico -------------------------------------------------------------------------------- /gospider-exportdb-list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nange/gospider/HEAD/gospider-exportdb-list.png -------------------------------------------------------------------------------- /web/static/build/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nange/gospider/HEAD/web/static/build/logo.png -------------------------------------------------------------------------------- /web/static/src/assets/401_images/401.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nange/gospider/HEAD/web/static/src/assets/401_images/401.gif -------------------------------------------------------------------------------- /web/static/src/assets/404_images/404.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nange/gospider/HEAD/web/static/src/assets/404_images/404.png -------------------------------------------------------------------------------- /web/static/config/sit.env.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | NODE_ENV: '"production"', 3 | ENV_CONFIG: '"sit"', 4 | BASE_API: '"/"' 5 | } 6 | -------------------------------------------------------------------------------- /web/static/config/dev.env.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | NODE_ENV: '"development"', 3 | ENV_CONFIG: '"dev"', 4 | BASE_API: '"/"' 5 | } 6 | -------------------------------------------------------------------------------- /web/static/config/prod.env.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | NODE_ENV: '"production"', 3 | ENV_CONFIG: '"prod"', 4 | BASE_API: '"/"' 5 | } 6 | -------------------------------------------------------------------------------- /web/static/src/assets/404_images/404_cloud.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nange/gospider/HEAD/web/static/src/assets/404_images/404_cloud.png -------------------------------------------------------------------------------- /web/static/build/vue-loader.conf.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | module.exports = { 4 | //You can set the vue-loader configuration by yourself. 5 | } 6 | -------------------------------------------------------------------------------- /web/static/src/assets/default_avatar/gopher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nange/gospider/HEAD/web/static/src/assets/default_avatar/gopher.png -------------------------------------------------------------------------------- /web/static/src/assets/custom-theme/fonts/element-icons.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nange/gospider/HEAD/web/static/src/assets/custom-theme/fonts/element-icons.ttf -------------------------------------------------------------------------------- /web/static/src/assets/custom-theme/fonts/element-icons.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nange/gospider/HEAD/web/static/src/assets/custom-theme/fonts/element-icons.woff -------------------------------------------------------------------------------- /web/static/.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | -------------------------------------------------------------------------------- /web/static/src/App.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 12 | -------------------------------------------------------------------------------- /web/static/src/views/svg-icons/generateIconsView.js: -------------------------------------------------------------------------------- 1 | const data = { 2 | state: { 3 | iconsMap: [] 4 | }, 5 | generate(iconsMap) { 6 | this.state.iconsMap = iconsMap 7 | } 8 | } 9 | 10 | export default data 11 | -------------------------------------------------------------------------------- /common/types.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | const ( 4 | OutputTypeMySQL = "mysql" 5 | OutputTypeCSV = "csv" 6 | OutputTypeStdout = "stdout" 7 | ) 8 | 9 | type MTS struct { 10 | ID uint64 11 | Status TaskStatus 12 | } 13 | -------------------------------------------------------------------------------- /web/static/.postcssrc.js: -------------------------------------------------------------------------------- 1 | // https://github.com/michael-ciniawsky/postcss-load-config 2 | 3 | module.exports = { 4 | "plugins": { 5 | // to edit target browsers: use "browserlist" field in package.json 6 | "autoprefixer": {} 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /web/static/src/utils/createUniqueString.js: -------------------------------------------------------------------------------- 1 | export default function createUniqueString() { 2 | const timestamp = +new Date() + '' 3 | const randomNum = parseInt((1 + Math.random()) * 65536) + '' 4 | return (+(randomNum + timestamp)).toString(32) 5 | } 6 | -------------------------------------------------------------------------------- /web/static/src/views/layout/components/index.js: -------------------------------------------------------------------------------- 1 | export { default as Navbar } from './Navbar' 2 | export { default as Sidebar } from './Sidebar/index.vue' 3 | export { default as TagsView } from './TagsView' 4 | export { default as AppMain } from './AppMain' 5 | -------------------------------------------------------------------------------- /web/static/src/styles/variables.scss: -------------------------------------------------------------------------------- 1 | $blue:#324157; 2 | $light-blue:#3A71A8; 3 | $red:#C03639; 4 | $pink: #E65D6E; 5 | $green: #30B08F; 6 | $tiffany: #4AB7BD; 7 | $yellow:#FEC171; 8 | $panGreen: #30B08F; 9 | 10 | //sidebar 11 | $menuBg:#304156; 12 | $subMenuBg:#1f2d3d; 13 | $menuHover:#001528; 14 | -------------------------------------------------------------------------------- /web/static/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | ["env", { "modules": false }], 4 | "stage-2" 5 | ], 6 | "plugins": ["transform-runtime"], 7 | "comments": false, 8 | "env": { 9 | "test": { 10 | "presets": ["env", "stage-2"], 11 | "plugins": [ "istanbul" ] 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /web/static/src/directive/waves/index.js: -------------------------------------------------------------------------------- 1 | import waves from './waves' 2 | 3 | const install = function(Vue) { 4 | Vue.directive('waves', waves) 5 | } 6 | 7 | if (window.Vue) { 8 | window.waves = waves 9 | Vue.use(install); // eslint-disable-line 10 | } 11 | 12 | waves.install = install 13 | export default waves 14 | -------------------------------------------------------------------------------- /web/static/src/views/login/authredirect.vue: -------------------------------------------------------------------------------- 1 | 11 | -------------------------------------------------------------------------------- /web/static/src/utils/auth.js: -------------------------------------------------------------------------------- 1 | import Cookies from 'js-cookie' 2 | 3 | const TokenKey = 'jwt' 4 | 5 | export function getToken() { 6 | return Cookies.get(TokenKey) 7 | } 8 | 9 | export function setToken(token) { 10 | return Cookies.set(TokenKey, token) 11 | } 12 | 13 | export function removeToken() { 14 | return Cookies.remove(TokenKey) 15 | } 16 | -------------------------------------------------------------------------------- /web/static/src/directive/permission/index.js: -------------------------------------------------------------------------------- 1 | import permission from './permission' 2 | 3 | const install = function(Vue) { 4 | Vue.directive('permission', permission) 5 | } 6 | 7 | if (window.Vue) { 8 | window['permission'] = permission 9 | Vue.use(install); // eslint-disable-line 10 | } 11 | 12 | permission.install = install 13 | export default permission 14 | -------------------------------------------------------------------------------- /web/static/src/store/modules/errorLog.js: -------------------------------------------------------------------------------- 1 | const errorLog = { 2 | state: { 3 | logs: [] 4 | }, 5 | mutations: { 6 | ADD_ERROR_LOG: (state, log) => { 7 | state.logs.push(log) 8 | } 9 | }, 10 | actions: { 11 | addErrorLog({ commit }, log) { 12 | commit('ADD_ERROR_LOG', log) 13 | } 14 | } 15 | } 16 | 17 | export default errorLog 18 | -------------------------------------------------------------------------------- /web/service/cron_test.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "log" 5 | "testing" 6 | 7 | "github.com/robfig/cron" 8 | ) 9 | 10 | type testJob struct { 11 | c *cron.Cron 12 | } 13 | 14 | func (j testJob) Run() { 15 | log.Println("run once...") 16 | } 17 | 18 | func TestCron(t *testing.T) { 19 | c := cron.New() 20 | c.AddJob("0 */1 * * * *", testJob{c: c}) 21 | c.Start() 22 | 23 | } 24 | -------------------------------------------------------------------------------- /web/static/src/utils/i18n.js: -------------------------------------------------------------------------------- 1 | // translate router.meta.title, be used in breadcrumb sidebar tagsview 2 | export function generateTitle(title) { 3 | const hasKey = this.$te('route.' + title) 4 | const translatedTitle = this.$t('route.' + title) // $t :this method from vue-i18n, inject in @/lang/index.js 5 | 6 | if (hasKey) { 7 | return translatedTitle 8 | } 9 | return title 10 | } 11 | -------------------------------------------------------------------------------- /web/static/src/views/dashboard/index.vue: -------------------------------------------------------------------------------- 1 | 8 | 9 | 18 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: false 3 | go: 4 | - 1.11.x 5 | - 1.12.x 6 | 7 | env: 8 | - GO111MODULE=on 9 | 10 | cache: 11 | directories: 12 | - $HOME/.cache/go-build 13 | - $HOME/gopath/pkg/mod 14 | 15 | script: 16 | - go vet ./... 17 | - go test -race -v -coverprofile=coverage.txt -covermode=atomic ./... 18 | - go build 19 | 20 | after_success: 21 | - bash <(curl -s https://codecov.io/bash) -------------------------------------------------------------------------------- /web/static/.gitignore: -------------------------------------------------------------------------------- 1 | # maven ignore 2 | target/ 3 | 4 | # eclipse ignore 5 | .settings/ 6 | .project 7 | .classpath 8 | 9 | # idea ignore 10 | .idea/ 11 | *.ipr 12 | *.iml 13 | *.iws 14 | 15 | # temp ignore 16 | *.log 17 | *.cache 18 | *.diff 19 | *.patch 20 | *.tmp 21 | 22 | # system ignore 23 | .DS_Store 24 | Thumbs.db 25 | 26 | # project ignore 27 | **/tmp 28 | pom.xml.versionsBackup 29 | 30 | # node ignore 31 | node_modules/ 32 | dist/ 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 14 | .glide/ 15 | 16 | .DS_Store 17 | .idea/ 18 | node_modules/ 19 | _example/_example 20 | main 21 | csv_output/ 22 | 23 | package-lock.json -------------------------------------------------------------------------------- /spider/task_test.go: -------------------------------------------------------------------------------- 1 | package spider 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestTask(t *testing.T) { 11 | _, cancel := context.WithCancel(context.Background()) 12 | err := addTaskCtrl(1, cancel) 13 | assert.Nil(t, err) 14 | err = addTaskCtrl(1, cancel) 15 | assert.NotNil(t, err) 16 | 17 | ok := CancelTask(1) 18 | assert.True(t, ok) 19 | ok = CancelTask(1) 20 | assert.False(t, ok) 21 | } 22 | -------------------------------------------------------------------------------- /web/static/src/api/login.js: -------------------------------------------------------------------------------- 1 | import request from '@/utils/request' 2 | 3 | export function loginByUsername(username, password) { 4 | const data = { 5 | username, 6 | password 7 | } 8 | return request({ 9 | url: '/login', 10 | method: 'post', 11 | data 12 | }) 13 | } 14 | 15 | export function getUserInfo(token) { 16 | return request({ 17 | url: '/api/user/info', 18 | method: 'get', 19 | params: { token } 20 | }) 21 | } 22 | 23 | -------------------------------------------------------------------------------- /web/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | gospider 9 | 10 | 11 |
12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /web/router/task/lock_test.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestLock(t *testing.T) { 10 | for i := 0; i < 3; i++ { 11 | taskID := uint64(1) 12 | go op1(taskID) 13 | } 14 | time.Sleep(5 * time.Second) 15 | } 16 | 17 | func op1(taskID uint64) { 18 | if taskLock.IsRunning(taskID) { 19 | fmt.Println("task is running...") 20 | return 21 | } 22 | defer taskLock.Complete(taskID) 23 | fmt.Println("run task") 24 | time.Sleep(1 * time.Second) 25 | } 26 | -------------------------------------------------------------------------------- /web/static/src/store/index.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import Vuex from 'vuex' 3 | import app from './modules/app' 4 | import errorLog from './modules/errorLog' 5 | import permission from './modules/permission' 6 | import tagsView from './modules/tagsView' 7 | import user from './modules/user' 8 | import getters from './getters' 9 | 10 | Vue.use(Vuex) 11 | 12 | const store = new Vuex.Store({ 13 | modules: { 14 | app, 15 | errorLog, 16 | permission, 17 | tagsView, 18 | user 19 | }, 20 | getters 21 | }) 22 | 23 | export default store 24 | -------------------------------------------------------------------------------- /web/static/src/icons/index.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import SvgIcon from '@/components/SvgIcon'// svg组件 3 | import generateIconsView from '@/views/svg-icons/generateIconsView.js'// just for @/views/icons , you can delete it 4 | 5 | // register globally 6 | Vue.component('svg-icon', SvgIcon) 7 | 8 | const requireAll = requireContext => requireContext.keys().map(requireContext) 9 | const req = require.context('./svg', false, /\.svg$/) 10 | const iconMap = requireAll(req) 11 | 12 | generateIconsView.generate(iconMap) // just for @/views/icons , you can delete it 13 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/chart.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example_gospider_test.go: -------------------------------------------------------------------------------- 1 | package gospider_test 2 | 3 | import ( 4 | "github.com/nange/gospider" 5 | log "github.com/sirupsen/logrus" 6 | ) 7 | 8 | //quitstart 9 | func Example() { 10 | // if gospider.New() has no argments, will use env parameters 11 | // gs := gospider.New() 12 | 13 | gs := gospider.New( 14 | gospider.BackendMySQL(), 15 | gospider.MySQLHost("127.0.0.1"), 16 | gospider.MySQLPort(3306), 17 | gospider.MySQLDBName("test"), 18 | gospider.MySQLUser("root"), 19 | gospider.MySQLPassword(""), 20 | gospider.WebPort(8080), 21 | ) 22 | log.Fatal(gs.Run()) 23 | } 24 | -------------------------------------------------------------------------------- /_example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/nange/gospider" 5 | _ "github.com/nange/gospider/_example/rule/baidunews" 6 | _ "github.com/nange/gospider/_example/rule/dianping" 7 | _ "github.com/nange/gospider/_example/rule/mojitianqi" 8 | _ "github.com/nange/gospider/_example/rule/stackoverflow" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | func init() { 13 | log.SetFormatter(&log.JSONFormatter{TimestampFormat: "2006-01-02 15:04:05.000"}) 14 | log.SetLevel(log.DebugLevel) 15 | } 16 | 17 | func main() { 18 | gs := gospider.New() 19 | log.Fatal(gs.Run()) 20 | } 21 | -------------------------------------------------------------------------------- /web/static/src/api/exportdb.js: -------------------------------------------------------------------------------- 1 | import request from '@/utils/request' 2 | 3 | // 数据列表 4 | export function fetchExportDBList(params) { 5 | return request({ 6 | url: '/api/exportdb', 7 | method: 'get', 8 | params 9 | }) 10 | } 11 | 12 | // 创建导出数据库记录 13 | export function createExportDB(data) { 14 | return request({ 15 | url: '/api/exportdb', 16 | method: 'post', 17 | data 18 | }) 19 | } 20 | 21 | // 删除导出数据库记录 22 | export function deleteExportDB(params) { 23 | return request({ 24 | url: '/api/exportdb/' + params.id, 25 | method: 'delete' 26 | }) 27 | } 28 | 29 | -------------------------------------------------------------------------------- /web/router/user/get_user_info.go: -------------------------------------------------------------------------------- 1 | package user 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | 7 | jwt "github.com/appleboy/gin-jwt/v2" 8 | "github.com/gin-gonic/gin" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | func GetUserInfo(c *gin.Context) { 13 | v, exists := c.Get(jwt.IdentityKey) 14 | if !exists { 15 | log.Warnf("jwt identity key not found") 16 | c.AbortWithStatus(http.StatusBadRequest) 17 | return 18 | } 19 | m := v.(map[string]interface{}) 20 | log.Debugf("identity:%+v", m) 21 | 22 | rolesStr := m["roles"].(string) 23 | m["roles"] = strings.Split(rolesStr, ",") 24 | 25 | c.JSON(http.StatusOK, m) 26 | } 27 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/money.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/router/task/task_lock.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | var taskLock *TaskLock 8 | 9 | type TaskLock struct { 10 | taskLock map[uint64]bool 11 | sync.Mutex 12 | } 13 | 14 | func init() { 15 | taskLock = &TaskLock{ 16 | taskLock: make(map[uint64]bool), 17 | } 18 | } 19 | 20 | func (tl *TaskLock) IsRunning(taskid uint64) bool { 21 | tl.Lock() 22 | defer tl.Unlock() 23 | if tl.taskLock[taskid] { 24 | return true 25 | } 26 | tl.taskLock[taskid] = true 27 | return false 28 | } 29 | 30 | func (tl *TaskLock) Complete(taskid uint64) { 31 | tl.Lock() 32 | defer tl.Unlock() 33 | delete(tl.taskLock, taskid) 34 | } 35 | -------------------------------------------------------------------------------- /web/static/src/errorLog.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import store from './store' 3 | 4 | // you can set only in production env show the error-log 5 | // if (process.env.NODE_ENV === 'production') { 6 | 7 | Vue.config.errorHandler = function(err, vm, info, a) { 8 | // Don't ask me why I use Vue.nextTick, it just a hack. 9 | // detail see https://forum.vuejs.org/t/dispatch-in-vue-config-errorhandler-has-some-problem/23500 10 | Vue.nextTick(() => { 11 | store.dispatch('addErrorLog', { 12 | err, 13 | vm, 14 | info, 15 | url: window.location.href 16 | }) 17 | console.error(err, info) 18 | }) 19 | } 20 | 21 | // } 22 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/guide.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/email.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/component.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/directive/permission/permission.js: -------------------------------------------------------------------------------- 1 | 2 | import store from '@/store' 3 | 4 | export default{ 5 | inserted(el, binding, vnode) { 6 | const { value } = binding 7 | const roles = store.getters && store.getters.roles 8 | 9 | if (value && value instanceof Array && value.length > 0) { 10 | const permissionRoles = value 11 | 12 | const hasPermission = roles.some(role => { 13 | return permissionRoles.includes(role) 14 | }) 15 | 16 | if (!hasPermission) { 17 | el.parentNode && el.parentNode.removeChild(el) 18 | } 19 | } else { 20 | throw new Error(`need roles! Like v-permission="['admin','editor']"`) 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /web/static/src/utils/permission.js: -------------------------------------------------------------------------------- 1 | import store from '@/store' 2 | 3 | /** 4 | * @param {Array} value 5 | * @returns {Boolean} 6 | * @example see @/views/permission/directive.vue 7 | */ 8 | export default function checkPermission(value) { 9 | if (value && value instanceof Array && value.length > 0) { 10 | const roles = store.getters && store.getters.roles 11 | const permissionRoles = value 12 | 13 | const hasPermission = roles.some(role => { 14 | return permissionRoles.includes(role) 15 | }) 16 | 17 | if (!hasPermission) { 18 | return false 19 | } 20 | return true 21 | } else { 22 | console.error(`need roles! Like v-permission="['admin','editor']"`) 23 | return false 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/documentation.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/model/model_test.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "database/sql" 5 | "testing" 6 | 7 | "github.com/DATA-DOG/go-sqlmock" 8 | "github.com/jinzhu/gorm" 9 | "github.com/stretchr/testify/suite" 10 | ) 11 | 12 | type testModelSuite struct { 13 | suite.Suite 14 | db *sql.DB 15 | mock sqlmock.Sqlmock 16 | gdb *gorm.DB 17 | } 18 | 19 | func (s *testModelSuite) SetupSuite() { 20 | db, mock, err := sqlmock.New() 21 | s.Require().NoError(err) 22 | s.db = db 23 | s.mock = mock 24 | gdb, err := gorm.Open("mysql", db) 25 | s.Require().NoError(err) 26 | s.gdb = gdb 27 | } 28 | 29 | func (s *testModelSuite) TearDownSuite() { 30 | if s.db != nil { 31 | s.db.Close() 32 | } 33 | } 34 | 35 | func TestModel(t *testing.T) { 36 | suite.Run(t, new(testModelSuite)) 37 | } 38 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/user.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/store/getters.js: -------------------------------------------------------------------------------- 1 | const getters = { 2 | sidebar: state => state.app.sidebar, 3 | language: state => state.app.language, 4 | device: state => state.app.device, 5 | visitedViews: state => state.tagsView.visitedViews, 6 | cachedViews: state => state.tagsView.cachedViews, 7 | token: state => state.user.token, 8 | avatar: state => state.user.avatar, 9 | name: state => state.user.name, 10 | introduction: state => state.user.introduction, 11 | status: state => state.user.status, 12 | roles: state => state.user.roles, 13 | setting: state => state.user.setting, 14 | permission_routers: state => state.permission.routers, 15 | addRouters: state => state.permission.addRouters, 16 | errorLogs: state => state.errorLog.logs 17 | } 18 | export default getters 19 | -------------------------------------------------------------------------------- /web/static/src/views/layout/components/AppMain.vue: -------------------------------------------------------------------------------- 1 | 10 | 11 | 24 | 25 | 33 | 34 | -------------------------------------------------------------------------------- /web/router/exportdb/delete.go: -------------------------------------------------------------------------------- 1 | package exportdb 2 | 3 | import ( 4 | "net/http" 5 | "strconv" 6 | 7 | "github.com/gin-gonic/gin" 8 | "github.com/nange/gospider/web/core" 9 | "github.com/nange/gospider/web/model" 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | func DeleteExportDB(c *gin.Context) { 14 | idStr := c.Param("id") 15 | log.Infof("delete exportdb id [%v]", idStr) 16 | id, err := strconv.ParseUint(idStr, 10, 64) 17 | if err != nil { 18 | c.AbortWithStatus(http.StatusBadRequest) 19 | return 20 | } 21 | edb := &model.ExportDB{ID: id} 22 | if err := edb.Delete(core.GetGormDB()); err != nil { 23 | log.Errorf("delete export db err [%+v]", err) 24 | c.AbortWithStatus(http.StatusInternalServerError) 25 | return 26 | } 27 | 28 | c.Data(http.StatusNoContent, "", nil) 29 | } 30 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/excel.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/components/SvgIcon/index.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 33 | 34 | 43 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/message.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/utils/clipboard.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import Clipboard from 'clipboard' 3 | 4 | function clipboardSuccess() { 5 | Vue.prototype.$message({ 6 | message: 'Copy successfully', 7 | type: 'success', 8 | duration: 1500 9 | }) 10 | } 11 | 12 | function clipboardError() { 13 | Vue.prototype.$message({ 14 | message: 'Copy failed', 15 | type: 'error' 16 | }) 17 | } 18 | 19 | export default function handleClipboard(text, event) { 20 | const clipboard = new Clipboard(event.target, { 21 | text: () => text 22 | }) 23 | clipboard.on('success', () => { 24 | clipboardSuccess() 25 | clipboard.off('error') 26 | clipboard.off('success') 27 | clipboard.destroy() 28 | }) 29 | clipboard.on('error', () => { 30 | clipboardError() 31 | clipboard.off('error') 32 | clipboard.off('success') 33 | clipboard.destroy() 34 | }) 35 | clipboard.onClick(event) 36 | } 37 | -------------------------------------------------------------------------------- /web/static/src/views/layout/components/Sidebar/index.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 34 | -------------------------------------------------------------------------------- /web/static/src/styles/transition.scss: -------------------------------------------------------------------------------- 1 | //globl transition css 2 | 3 | /*fade*/ 4 | .fade-enter-active, 5 | .fade-leave-active { 6 | transition: opacity 0.28s; 7 | } 8 | 9 | .fade-enter, 10 | .fade-leave-active { 11 | opacity: 0; 12 | } 13 | 14 | /*fade-transform*/ 15 | .fade-transform-leave-active, 16 | .fade-transform-enter-active { 17 | transition: all .5s; 18 | } 19 | .fade-transform-enter { 20 | opacity: 0; 21 | transform: translateX(-30px); 22 | } 23 | .fade-transform-leave-to { 24 | opacity: 0; 25 | transform: translateX(30px); 26 | } 27 | 28 | /*breadcrumb transition*/ 29 | .breadcrumb-enter-active, 30 | .breadcrumb-leave-active { 31 | transition: all .5s; 32 | } 33 | 34 | .breadcrumb-enter, 35 | .breadcrumb-leave-active { 36 | opacity: 0; 37 | transform: translateX(20px); 38 | } 39 | 40 | .breadcrumb-move { 41 | transition: all .5s; 42 | } 43 | 44 | .breadcrumb-leave-active { 45 | position: absolute; 46 | } 47 | 48 | -------------------------------------------------------------------------------- /web/web.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/gin-gonic/gin" 7 | "github.com/nange/gospider/web/core" 8 | "github.com/nange/gospider/web/model" 9 | "github.com/nange/gospider/web/router" 10 | "github.com/nange/gospider/web/service" 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | type Server struct { 15 | IP string 16 | Port int 17 | } 18 | 19 | func (s *Server) Run() error { 20 | if err := core.AutoMigrate(); err != nil { 21 | return errors.Wrap(err, "model auto migrate failed") 22 | } 23 | if err := model.InitAdminUserIfNeeded(core.GetGormDB()); err != nil { 24 | return errors.Wrap(err, "init admin user failed") 25 | } 26 | 27 | // 启动服务时，先检查task相关状态 28 | go service.CheckTask() 29 | // 管理task状态(如task运行完成之后需要将状态标为已完成) 30 | go service.ManageTaskStatus() 31 | 32 | engine := gin.Default() 33 | router.Route(engine) 34 | 35 | return errors.WithStack(engine.Run(fmt.Sprintf("%s:%d", s.IP, s.Port))) 36 | } 37 | -------------------------------------------------------------------------------- /web/static/src/directive/waves/waves.css: -------------------------------------------------------------------------------- 1 | .waves-ripple { 2 | position: absolute; 3 | border-radius: 100%; 4 | background-color: rgba(0, 0, 0, 0.15); 5 | background-clip: padding-box; 6 | pointer-events: none; 7 | -webkit-user-select: none; 8 | -moz-user-select: none; 9 | -ms-user-select: none; 10 | user-select: none; 11 | -webkit-transform: scale(0); 12 | -ms-transform: scale(0); 13 | transform: scale(0); 14 | opacity: 1; 15 | } 16 | 17 | .waves-ripple.z-active { 18 | opacity: 0; 19 | -webkit-transform: scale(2); 20 | -ms-transform: scale(2); 21 | transform: scale(2); 22 | -webkit-transition: opacity 1.2s ease-out, -webkit-transform 0.6s ease-out; 23 | transition: opacity 1.2s ease-out, -webkit-transform 0.6s ease-out; 24 | transition: opacity 1.2s ease-out, transform 0.6s ease-out; 25 | transition: opacity 1.2s ease-out, transform 0.6s ease-out, -webkit-transform 0.6s ease-out; 26 | } -------------------------------------------------------------------------------- /web/static/src/lang/index.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import VueI18n from 'vue-i18n' 3 | import Cookies from 'js-cookie' 4 | import elementEnLocale from 'element-ui/lib/locale/lang/en' // element-ui lang 5 | import elementZhLocale from 'element-ui/lib/locale/lang/zh-CN'// element-ui lang 6 | import enLocale from './en' 7 | import zhLocale from './zh' 8 | 9 | Vue.use(VueI18n) 10 | 11 | const messages = { 12 | en: { 13 | ...enLocale, 14 | ...elementEnLocale 15 | }, 16 | zh: { 17 | ...zhLocale, 18 | ...elementZhLocale 19 | } 20 | } 21 | 22 | const i18n = new VueI18n({ 23 | // set locale 24 | // options: en or zh 25 | locale: Cookies.get('language') || getBrowserLanguage(), 26 | // set locale messages 27 | messages 28 | }) 29 | 30 | export function getBrowserLanguage() { 31 | const lang = navigator.language || navigator.browserLanguage || navigator.userLanguage 32 | return lang.substr(0, 2) 33 | } 34 | 35 | export default i18n 36 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/drag.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/main.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | 3 | import 'normalize.css/normalize.css' // A modern alternative to CSS resets 4 | 5 | import Element from 'element-ui' 6 | import 'element-ui/lib/theme-chalk/index.css' 7 | 8 | import '@/styles/index.scss' // global css 9 | 10 | import App from './App' 11 | import router from './router' 12 | import store from './store' 13 | 14 | import i18n from './lang' // Internationalization 15 | import './icons' // icon 16 | import './errorLog' // error log 17 | import './permission' // permission control 18 | 19 | import * as filters from './filters' // global filters 20 | 21 | Vue.use(Element, { 22 | size: 'medium', // set element-ui default size 23 | i18n: (key, value) => i18n.t(key, value) 24 | }) 25 | 26 | // register global utility filters. 27 | Object.keys(filters).forEach(key => { 28 | Vue.filter(key, filters[key]) 29 | }) 30 | 31 | Vue.config.productionTip = false 32 | 33 | new Vue({ 34 | el: '#app', 35 | router, 36 | store, 37 | i18n, 38 | render: h => h(App) 39 | }) 40 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/edit.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/icon.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/example.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/service/taskstatus.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "github.com/nange/gospider/common" 5 | "github.com/nange/gospider/web/core" 6 | "github.com/nange/gospider/web/model" 7 | "github.com/pkg/errors" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | var mtsCh = make(chan common.MTS, 1) 12 | 13 | func GetMTSChan() chan common.MTS { 14 | return mtsCh 15 | } 16 | 17 | func ManageTaskStatus() { 18 | log.Infof("starting manage task status goroutine") 19 | for { 20 | select { 21 | case mts := <-mtsCh: 22 | task := &model.Task{} 23 | err := model.NewTaskQuerySet(core.GetGormDB()).IDEq(mts.ID).One(task) 24 | if err != nil { 25 | log.Errorf("query model task err: %+v", err) 26 | break 27 | } 28 | 29 | task.Status = mts.Status 30 | if mts.Status == common.TaskStatusCompleted { 31 | task.Counts += 1 32 | } 33 | 34 | if err := task.Update(core.GetGormDB(), model.TaskDBSchema.Status, model.TaskDBSchema.Counts); err != nil { 35 | log.Errorf("update task status err:%+v", errors.WithStack(err)) 36 | break 37 | } 38 | 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/lock.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/theme.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /web/router/task/get_task_list.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/gin-gonic/gin" 7 | "github.com/nange/gospider/web/core" 8 | "github.com/nange/gospider/web/model" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | type GetTaskListReq struct { 13 | Size int `json:"size" form:"size"` 14 | Offset int `json:"offset" form:"offset"` 15 | } 16 | 17 | type GetTaskListResp struct { 18 | Total int `json:"total"` 19 | Data []model.Task `json:"data"` 20 | } 21 | 22 | func GetTaskList(c *gin.Context) { 23 | var req GetTaskListReq 24 | if err := c.BindQuery(&req); err != nil { 25 | log.Warnf("query param is invalid") 26 | c.String(http.StatusBadRequest, "") 27 | return 28 | } 29 | log.Infof("get task list req:%+v", req) 30 | 31 | tasks, count, err := model.GetTaskList(core.GetGormDB(), req.Size, req.Offset) 32 | if err != nil { 33 | log.Errorf("GetTaskList failed! err:%+v", err) 34 | c.String(http.StatusInternalServerError, "") 35 | return 36 | } 37 | 38 | c.JSON(http.StatusOK, &GetTaskListResp{ 39 | Total: count, 40 | Data: tasks, 41 | }) 42 | } 43 | -------------------------------------------------------------------------------- /web/router/task/get_task_by_id.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "net/http" 5 | "strconv" 6 | 7 | "github.com/gin-gonic/gin" 8 | "github.com/nange/gospider/web/core" 9 | "github.com/nange/gospider/web/model" 10 | "github.com/sirupsen/logrus" 11 | ) 12 | 13 | func GetTaskByID(c *gin.Context) { 14 | taskIDStr := c.Param("id") 15 | if taskIDStr == "" { 16 | logrus.Warnf("GetTaskByID taskID is empty") 17 | c.AbortWithStatus(http.StatusBadRequest) 18 | return 19 | } 20 | taskID, err := strconv.ParseUint(taskIDStr, 10, 64) 21 | if err != nil { 22 | logrus.Warnf("GetTaskByID taskID format is invalid, taskID:%v", taskIDStr) 23 | c.AbortWithStatus(http.StatusBadRequest) 24 | return 25 | } 26 | logrus.Infof("GetTaskByID:%+v", taskID) 27 | 28 | // query task info from db 29 | task := &model.Task{} 30 | err = model.NewTaskQuerySet(core.GetGormDB()).IDEq(taskID).One(task) 31 | if err != nil { 32 | logrus.Errorf("GetTaskByID query model task fail, taskID: %v , err: %+v", taskIDStr, err) 33 | c.String(http.StatusInternalServerError, "") 34 | return 35 | } 36 | 37 | c.JSON(http.StatusOK, task) 38 | } 39 | -------------------------------------------------------------------------------- /web/static/src/utils/openWindow.js: -------------------------------------------------------------------------------- 1 | export default function openWindow(url, title, w, h) { 2 | // Fixes dual-screen position Most browsers Firefox 3 | const dualScreenLeft = window.screenLeft !== undefined ? window.screenLeft : screen.left 4 | const dualScreenTop = window.screenTop !== undefined ? window.screenTop : screen.top 5 | 6 | const width = window.innerWidth ? window.innerWidth : document.documentElement.clientWidth ? document.documentElement.clientWidth : screen.width 7 | const height = window.innerHeight ? window.innerHeight : document.documentElement.clientHeight ? document.documentElement.clientHeight : screen.height 8 | 9 | const left = ((width / 2) - (w / 2)) + dualScreenLeft 10 | const top = ((height / 2) - (h / 2)) + dualScreenTop 11 | const newWindow = window.open(url, title, 'toolbar=no, location=no, directories=no, status=no, menubar=no, scrollbars=no, resizable=yes, copyhistory=no, width=' + w + ', height=' + h + ', top=' + top + ', left=' + left) 12 | 13 | // Puts focus on the newWindow 14 | if (window.focus) { 15 | newWindow.focus() 16 | } 17 | } 18 | 19 | -------------------------------------------------------------------------------- /web/core/db.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "database/sql" 5 | "reflect" 6 | 7 | "github.com/jinzhu/gorm" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | var modelList []Model 12 | 13 | var db *gorm.DB 14 | 15 | func SetGormDB(gdb *gorm.DB) { 16 | db = gdb 17 | } 18 | 19 | func GetGormDB() *gorm.DB { 20 | return db 21 | } 22 | 23 | func GetDB() *sql.DB { 24 | return db.DB() 25 | } 26 | 27 | type Model interface { 28 | TableName() string 29 | } 30 | 31 | func Register(model Model) { 32 | rv := reflect.ValueOf(model) 33 | if rv.IsNil() { 34 | panic("register model failed, model is nil") 35 | } 36 | for _, m := range modelList { 37 | if m.TableName() == model.TableName() { 38 | panic("register model failed, already have the table name:" + model.TableName()) 39 | } 40 | } 41 | modelList = append(modelList, model) 42 | } 43 | 44 | func AutoMigrate() error { 45 | for _, model := range modelList { 46 | if err := db.Debug().Set("gorm:table_options", "CHARSET=utf8mb4").AutoMigrate(model).Error; err != nil { 47 | return errors.Wrap(err, "db auto migrate failed") 48 | } 49 | } 50 | 51 | return nil 52 | } 53 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 LanceLi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/list.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/components/LangSelect/index.vue: -------------------------------------------------------------------------------- 1 | 12 | 13 | 32 | 33 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /web/static/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017-present PanJiaChen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /spider/task.go: -------------------------------------------------------------------------------- 1 | package spider 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | // Task is a task define 12 | type Task struct { 13 | ID uint64 14 | TaskRule 15 | TaskConfig 16 | } 17 | 18 | // NewTask return a new task object 19 | func NewTask(id uint64, rule TaskRule, config TaskConfig) *Task { 20 | return &Task{ 21 | ID: id, 22 | TaskRule: rule, 23 | TaskConfig: config, 24 | } 25 | } 26 | 27 | var ( 28 | ctlMu = &sync.RWMutex{} 29 | ctlMap = make(map[uint64]context.CancelFunc) 30 | ) 31 | 32 | func addTaskCtrl(taskID uint64, cancelFunc context.CancelFunc) error { 33 | ctlMu.Lock() 34 | defer ctlMu.Unlock() 35 | 36 | if _, ok := ctlMap[taskID]; ok { 37 | return errors.WithStack(fmt.Errorf("duplicate taskID:%d", taskID)) 38 | } 39 | ctlMap[taskID] = cancelFunc 40 | 41 | return nil 42 | } 43 | 44 | // CancelTask cancel a task by taskID 45 | func CancelTask(taskID uint64) bool { 46 | ctlMu.Lock() 47 | defer ctlMu.Unlock() 48 | 49 | cancel, ok := ctlMap[taskID] 50 | if !ok { 51 | return false 52 | } 53 | cancel() 54 | delete(ctlMap, taskID) 55 | 56 | return true 57 | } 58 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/tab.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/views/layout/mixin/ResizeHandler.js: -------------------------------------------------------------------------------- 1 | import store from '@/store' 2 | 3 | const { body } = document 4 | const WIDTH = 1024 5 | const RATIO = 3 6 | 7 | export default { 8 | watch: { 9 | $route(route) { 10 | if (this.device === 'mobile' && this.sidebar.opened) { 11 | store.dispatch('closeSideBar', { withoutAnimation: false }) 12 | } 13 | } 14 | }, 15 | beforeMount() { 16 | window.addEventListener('resize', this.resizeHandler) 17 | }, 18 | mounted() { 19 | const isMobile = this.isMobile() 20 | if (isMobile) { 21 | store.dispatch('toggleDevice', 'mobile') 22 | store.dispatch('closeSideBar', { withoutAnimation: true }) 23 | } 24 | }, 25 | methods: { 26 | isMobile() { 27 | const rect = body.getBoundingClientRect() 28 | return rect.width - RATIO < WIDTH 29 | }, 30 | resizeHandler() { 31 | if (!document.hidden) { 32 | const isMobile = this.isMobile() 33 | store.dispatch('toggleDevice', isMobile ? 'mobile' : 'desktop') 34 | 35 | if (isMobile) { 36 | store.dispatch('closeSideBar', { withoutAnimation: true }) 37 | } 38 | } 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/nested.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/router/exportdb/get_list.go: -------------------------------------------------------------------------------- 1 | package exportdb 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/gin-gonic/gin" 7 | "github.com/nange/gospider/web/core" 8 | "github.com/nange/gospider/web/model" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | type GetExportDBListReq struct { 13 | Size int `json:"size" form:"size"` 14 | Offset int `json:"offset" form:"offset"` 15 | } 16 | 17 | type GetExportDBListResp struct { 18 | Total int `json:"total"` 19 | Data []model.ExportDB `json:"data"` 20 | } 21 | 22 | func GetExportDBList(c *gin.Context) { 23 | var req GetExportDBListReq 24 | if err := c.BindQuery(&req); err != nil { 25 | log.Warnf("query param is invalid") 26 | c.String(http.StatusBadRequest, "") 27 | return 28 | } 29 | log.Infof("get exportdb list req:%+v", req) 30 | 31 | exportdbs, count, err := model.GetExportDBList(core.GetGormDB(), req.Size, req.Offset) 32 | if err != nil { 33 | log.Errorf("GetExportDBList failed! err [%+v]", err) 34 | c.String(http.StatusInternalServerError, "") 35 | return 36 | } 37 | log.Infof("exportdbs:%+v", exportdbs) 38 | 39 | c.JSON(http.StatusOK, &GetExportDBListResp{ 40 | Total: count, 41 | Data: exportdbs, 42 | }) 43 | } 44 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/star.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/router/exportdb/create.go: -------------------------------------------------------------------------------- 1 | package exportdb 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/gin-gonic/gin" 8 | "github.com/nange/gospider/web/core" 9 | "github.com/nange/gospider/web/model" 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | type CreateExportDBReq struct { 14 | model.ExportDB 15 | } 16 | 17 | type CreateExportDBResp struct { 18 | ID uint64 `json:"id"` 19 | CreateAt time.Time `json:"create_at"` 20 | } 21 | 22 | func CreateExportDB(c *gin.Context) { 23 | var req CreateExportDBReq 24 | if err := c.BindJSON(&req); err != nil { 25 | log.Errorf("bind json failed! err:%+v", err) 26 | c.String(http.StatusBadRequest, "") 27 | return 28 | } 29 | log.Infof("req:%+v", req) 30 | 31 | if req.Host == "" { 32 | req.Host = "127.0.0.1" 33 | } 34 | if req.Port == 0 { 35 | req.Port = 3306 36 | } 37 | if req.User == "" { 38 | req.User = "root" 39 | } 40 | 41 | db := core.GetGormDB() 42 | if err := req.Create(db); err != nil { 43 | log.Errorf("create sysdb err: %+v", err) 44 | c.AbortWithError(http.StatusInternalServerError, err) 45 | return 46 | } 47 | 48 | c.JSON(http.StatusOK, &CreateExportDBResp{ 49 | ID: req.ID, 50 | CreateAt: req.CreatedAt, 51 | }) 52 | } 53 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/password.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/utils/validate.js: -------------------------------------------------------------------------------- 1 | export function isvalidUsername(str) { 2 | const valid_map = ['admin', 'editor'] 3 | return valid_map.indexOf(str.trim()) >= 0 4 | } 5 | 6 | /* 合法uri*/ 7 | export function validateURL(textval) { 8 | const urlregex = /^(https?|ftp):\/\/([a-zA-Z0-9.-]+(:[a-zA-Z0-9.&%$-]+)*@)*((25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}|([a-zA-Z0-9-]+\.)*[a-zA-Z0-9-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(:[0-9]+)*(\/($|[a-zA-Z0-9.,?'\\+&%$#=~_-]+))*$/ 9 | return urlregex.test(textval) 10 | } 11 | 12 | /* 小写字母*/ 13 | export function validateLowerCase(str) { 14 | const reg = /^[a-z]+$/ 15 | return reg.test(str) 16 | } 17 | 18 | /* 大写字母*/ 19 | export function validateUpperCase(str) { 20 | const reg = /^[A-Z]+$/ 21 | return reg.test(str) 22 | } 23 | 24 | /* 大小写字母*/ 25 | export function validateAlphabets(str) { 26 | const reg = /^[A-Za-z]+$/ 27 | return reg.test(str) 28 | } 29 | 30 | /** 31 | * validate email 32 | * @param email 33 | * @returns {boolean} 34 | */ 35 | export function validateEmail(email) { 36 | const re = /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/ 37 | return re.test(email) 38 | } 39 | -------------------------------------------------------------------------------- /web/static/src/api/task.js: -------------------------------------------------------------------------------- 1 | import request from '@/utils/request' 2 | 3 | // 数据列表 4 | export function fetchTaskList(params) { 5 | return request({ 6 | url: '/api/tasks', 7 | method: 'get', 8 | params 9 | }) 10 | } 11 | 12 | // 根据id查询数据 13 | export function getTask(id) { 14 | return request({ 15 | url: '/api/tasks/' + id, 16 | method: 'get' 17 | }) 18 | } 19 | 20 | // 查询所有rules 21 | export function getRules() { 22 | return request({ 23 | url: '/api/rules', 24 | method: 'get' 25 | }) 26 | } 27 | 28 | // 根据id停止任务 29 | export function stopTask(id) { 30 | return request({ 31 | url: '/api/tasks/' + id + '/stop', 32 | method: 'put' 33 | }) 34 | } 35 | // 根据id启动非定时任务 36 | export function startTask(id) { 37 | return request({ 38 | url: '/api/tasks/' + id + '/start', 39 | method: 'put' 40 | }) 41 | } 42 | // 根据id重启定时任务 43 | export function restartTask(id) { 44 | return request({ 45 | url: '/api/tasks/' + id + '/restart', 46 | method: 'put' 47 | }) 48 | } 49 | // 添加数据 50 | export function saveTask(data) { 51 | return request({ 52 | url: '/api/tasks', 53 | method: 'post', 54 | data 55 | }) 56 | } 57 | 58 | // 修改数据 59 | export function updateTask(id, data) { 60 | return request({ 61 | url: '/api/tasks/' + id, 62 | method: 'put', 63 | data 64 | }) 65 | } 66 | 67 | -------------------------------------------------------------------------------- /web/static/src/filters/index.js: -------------------------------------------------------------------------------- 1 | // set function parseTime,formatTime to filter 2 | export { parseTime, formatTime } from '@/utils' 3 | 4 | function pluralize(time, label) { 5 | if (time === 1) { 6 | return time + label 7 | } 8 | return time + label + 's' 9 | } 10 | 11 | export function timeAgo(time) { 12 | const between = Date.now() / 1000 - Number(time) 13 | if (between < 3600) { 14 | return pluralize(~~(between / 60), ' minute') 15 | } else if (between < 86400) { 16 | return pluralize(~~(between / 3600), ' hour') 17 | } else { 18 | return pluralize(~~(between / 86400), ' day') 19 | } 20 | } 21 | 22 | /* 数字 格式化*/ 23 | export function numberFormatter(num, digits) { 24 | const si = [ 25 | { value: 1E18, symbol: 'E' }, 26 | { value: 1E15, symbol: 'P' }, 27 | { value: 1E12, symbol: 'T' }, 28 | { value: 1E9, symbol: 'G' }, 29 | { value: 1E6, symbol: 'M' }, 30 | { value: 1E3, symbol: 'k' } 31 | ] 32 | for (let i = 0; i < si.length; i++) { 33 | if (num >= si[i].value) { 34 | return (num / si[i].value + 0.1).toFixed(digits).replace(/\.0+$|(\.[0-9]*[1-9])0+$/, '$1') + si[i].symbol 35 | } 36 | } 37 | return num.toString() 38 | } 39 | 40 | export function toThousandslsFilter(num) { 41 | return (+num || 0).toString().replace(/^-?\d+/g, m => m.replace(/(?=(?!\b)(\d{3})+$)/g, ',')) 42 | } 43 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/clipboard.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/peoples.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/router/rule/get_rules.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "net/http" 7 | "sort" 8 | 9 | "github.com/gin-gonic/gin" 10 | log "github.com/sirupsen/logrus" 11 | "golang.org/x/text/encoding/simplifiedchinese" 12 | "golang.org/x/text/transform" 13 | 14 | "github.com/nange/gospider/spider" 15 | ) 16 | 17 | type GetRuleListResp struct { 18 | Total int `json:"total"` 19 | Data []string `json:"data"` 20 | } 21 | 22 | func GetRuleList(c *gin.Context) { 23 | keys := spider.GetTaskRuleKeys() 24 | if len(keys) == 0 { 25 | log.Warnf("task rule is empty") 26 | } else { 27 | sort.Sort(Pinyin(keys)) 28 | } 29 | 30 | c.JSON(http.StatusOK, &GetRuleListResp{ 31 | Total: len(keys), 32 | Data: keys, 33 | }) 34 | } 35 | 36 | type Pinyin []string 37 | 38 | func (s Pinyin) Len() int { return len(s) } 39 | func (s Pinyin) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 40 | func (s Pinyin) Less(i, j int) bool { 41 | a, _ := UTF82GB18030(s[i]) 42 | b, _ := UTF82GB18030(s[j]) 43 | bLen := len(b) 44 | for idx, chr := range a { 45 | if idx > bLen-1 { 46 | return false 47 | } 48 | if chr != b[idx] { 49 | return chr < b[idx] 50 | } 51 | } 52 | return true 53 | } 54 | 55 | func UTF82GB18030(src string) ([]byte, error) { 56 | GB18030 := simplifiedchinese.All[0] 57 | return ioutil.ReadAll(transform.NewReader(bytes.NewReader([]byte(src)), GB18030.NewEncoder())) 58 | } 59 | -------------------------------------------------------------------------------- /web/core/db_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | sqlmock "github.com/DATA-DOG/go-sqlmock" 8 | "github.com/jinzhu/gorm" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | type testModel struct { 14 | ID uint64 `gorm:"column:id;type:bigint unsigned AUTO_INCREMENT;primary_key"` 15 | Name string `gorm:"column:user_name;type:varchar(32) not null"` 16 | } 17 | 18 | func (t *testModel) TableName() string { 19 | return "gospider_testmodel" 20 | } 21 | 22 | func TestRegisterAndAutoMigrate(t *testing.T) { 23 | assert.Panics(t, func() { 24 | Register(nil) 25 | }) 26 | assert.Panics(t, func() { 27 | var tm *testModel 28 | Register(tm) 29 | }) 30 | 31 | Register(&testModel{}) 32 | assert.Panics(t, func() { 33 | Register(&testModel{}) 34 | }) 35 | 36 | db, mock, err := sqlmock.New() 37 | require.NoError(t, err) 38 | defer db.Close() 39 | gdb, err := gorm.Open("mysql", db) 40 | require.NoError(t, err) 41 | SetGormDB(gdb) 42 | 43 | mock.ExpectExec("(?i)create table `gospider_testmodel`"). 44 | WillReturnResult(sqlmock.NewResult(0, 0)) 45 | 46 | assert.NoError(t, AutoMigrate()) 47 | assert.NoError(t, mock.ExpectationsWereMet()) 48 | 49 | mock.ExpectExec("(?i)create table `gospider_testmodel`"). 50 | WillReturnError(fmt.Errorf("some error")) 51 | assert.NotNil(t, AutoMigrate()) 52 | assert.NoError(t, mock.ExpectationsWereMet()) 53 | 54 | } 55 | -------------------------------------------------------------------------------- /spider/task_rule_test.go: -------------------------------------------------------------------------------- 1 | package spider 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/pkg/errors" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestTaskRule(t *testing.T) { 11 | r1, err := GetTaskRule("test_rule_not_exist") 12 | assert.Nil(t, r1) 13 | assert.Equal(t, ErrTaskRuleNotExist, errors.Cause(err)) 14 | 15 | assert.Panics(t, func() { 16 | Register(nil) 17 | }) 18 | assert.Panics(t, func() { 19 | Register(&TaskRule{}) 20 | }) 21 | assert.Panics(t, func() { 22 | Register(&TaskRule{Namespace: ""}) 23 | }) 24 | assert.Panics(t, func() { 25 | Register(&TaskRule{Namespace: ""}) 26 | }) 27 | assert.Panics(t, func() { 28 | Register(&TaskRule{ 29 | Name: "test", 30 | Rule: &Rule{}, 31 | }) 32 | }) 33 | assert.Panics(t, func() { 34 | Register(&TaskRule{ 35 | Name: "test", 36 | Rule: &Rule{ 37 | Head: func(ctx *Context) error { 38 | return nil 39 | }, 40 | Nodes: map[int]*Node{ 41 | 0: {}, 42 | 2: {}, 43 | }, 44 | }, 45 | }) 46 | }) 47 | 48 | Register(&TaskRule{ 49 | Name: "test_rule_name", 50 | Namespace: "test_namespace", 51 | OutputFields: []string{"test_field"}, 52 | Rule: &Rule{ 53 | Head: func(ctx *Context) error { 54 | return nil 55 | }, 56 | Nodes: map[int]*Node{ 57 | 0: {}, 58 | }, 59 | }, 60 | }) 61 | r2, err := GetTaskRule("test_rule_name") 62 | assert.NoError(t, err) 63 | assert.NotNil(t, r2) 64 | 65 | keys := GetTaskRuleKeys() 66 | assert.Contains(t, keys, "test_rule_name") 67 | } 68 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/language.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /web/static/src/styles/element-ui.scss: -------------------------------------------------------------------------------- 1 | //覆盖一些element-ui样式 2 | 3 | .el-breadcrumb__inner, .el-breadcrumb__inner a{ 4 | font-weight: 400!important; 5 | } 6 | 7 | .el-upload { 8 | input[type="file"] { 9 | display: none !important; 10 | } 11 | } 12 | 13 | .el-upload__input { 14 | display: none; 15 | } 16 | 17 | .cell { 18 | .el-tag { 19 | margin-right: 0px; 20 | } 21 | } 22 | 23 | .small-padding { 24 | .cell { 25 | padding-left: 5px; 26 | padding-right: 5px; 27 | } 28 | } 29 | 30 | .fixed-width{ 31 | .el-button--mini{ 32 | padding: 7px 10px; 33 | width: 60px; 34 | } 35 | } 36 | 37 | .status-col { 38 | .cell { 39 | padding: 0 10px; 40 | text-align: center; 41 | .el-tag { 42 | margin-right: 0px; 43 | } 44 | } 45 | } 46 | 47 | //暂时性解决dialog 问题 https://github.com/ElemeFE/element/issues/2461 48 | .el-dialog { 49 | transform: none; 50 | left: 0; 51 | position: relative; 52 | margin: 0 auto; 53 | } 54 | 55 | //文章页textarea修改样式 56 | .article-textarea { 57 | textarea { 58 | padding-right: 40px; 59 | resize: none; 60 | border: none; 61 | border-radius: 0px; 62 | border-bottom: 1px solid #bfcbd9; 63 | } 64 | } 65 | 66 | //element ui upload 67 | .upload-container { 68 | .el-upload { 69 | width: 100%; 70 | .el-upload-dragger { 71 | width: 100%; 72 | height: 200px; 73 | } 74 | } 75 | } 76 | 77 | //dropdown 78 | .el-dropdown-menu{ 79 | a{ 80 | display: block 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /web/model/task_test.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/DATA-DOG/go-sqlmock" 7 | ) 8 | 9 | func (s *testModelSuite) TestGetTaskList() { 10 | rows1 := sqlmock.NewRows([]string{"count(*)"}).AddRow(10) 11 | s.mock.ExpectQuery("(?i)select count\\(\\*\\) from `gospider_task`"). 12 | WillReturnRows(rows1) 13 | rows2 := sqlmock.NewRows([]string{"id"}). 14 | AddRow(uint64(1)). 15 | AddRow(uint64(2)). 16 | AddRow(uint64(3)). 17 | AddRow(uint64(4)). 18 | AddRow(uint64(5)) 19 | s.mock.ExpectQuery("(?i)select \\* from `gospider_task`"). 20 | WillReturnRows(rows2) 21 | 22 | list, count, err := GetTaskList(s.gdb, 5, 0) 23 | s.Require().NoError(err) 24 | s.Equal(10, count) 25 | s.Equal(5, len(list)) 26 | s.Equal(uint64(1), list[0].ID) 27 | s.NoError(s.mock.ExpectationsWereMet()) 28 | 29 | s.mock.ExpectQuery("(?i)select count\\(\\*\\) from `gospider_task`"). 30 | WillReturnError(fmt.Errorf("some error")) 31 | list, count, err = GetTaskList(s.gdb, 5, 0) 32 | s.Require().NotNil(err) 33 | s.Equal(0, count) 34 | s.Nil(list) 35 | s.NoError(s.mock.ExpectationsWereMet()) 36 | 37 | rows3 := sqlmock.NewRows([]string{"count(*)"}).AddRow(10) 38 | s.mock.ExpectQuery("(?i)select count\\(\\*\\) from `gospider_task`"). 39 | WillReturnRows(rows3) 40 | s.mock.ExpectQuery("(?i)select \\* from `gospider_task` order by"). 41 | WillReturnError(fmt.Errorf("some error")) 42 | list2, count2, err2 := GetTaskList(s.gdb, 5, 0) 43 | s.Require().NotNil(err2) 44 | s.Equal(0, count2) 45 | s.Nil(list2) 46 | s.NoError(s.mock.ExpectationsWereMet()) 47 | } 48 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/404.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/styles/mixin.scss: -------------------------------------------------------------------------------- 1 | @mixin clearfix { 2 | &:after { 3 | content: ""; 4 | display: table; 5 | clear: both; 6 | } 7 | } 8 | 9 | @mixin scrollBar { 10 | &::-webkit-scrollbar-track-piece { 11 | background: #d3dce6; 12 | } 13 | &::-webkit-scrollbar { 14 | width: 6px; 15 | } 16 | &::-webkit-scrollbar-thumb { 17 | background: #99a9bf; 18 | border-radius: 20px; 19 | } 20 | } 21 | 22 | @mixin relative { 23 | position: relative; 24 | width: 100%; 25 | height: 100%; 26 | } 27 | 28 | @mixin pct($pct) { 29 | width: #{$pct}; 30 | position: relative; 31 | margin: 0 auto; 32 | } 33 | 34 | @mixin triangle($width, $height, $color, $direction) { 35 | $width: $width/2; 36 | $color-border-style: $height solid $color; 37 | $transparent-border-style: $width solid transparent; 38 | height: 0; 39 | width: 0; 40 | @if $direction==up { 41 | border-bottom: $color-border-style; 42 | border-left: $transparent-border-style; 43 | border-right: $transparent-border-style; 44 | } 45 | @else if $direction==right { 46 | border-left: $color-border-style; 47 | border-top: $transparent-border-style; 48 | border-bottom: $transparent-border-style; 49 | } 50 | @else if $direction==down { 51 | border-top: $color-border-style; 52 | border-left: $transparent-border-style; 53 | border-right: $transparent-border-style; 54 | } 55 | @else if $direction==left { 56 | border-right: $color-border-style; 57 | border-top: $transparent-border-style; 58 | border-bottom: $transparent-border-style; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /web/static/src/store/modules/app.js: -------------------------------------------------------------------------------- 1 | import Cookies from 'js-cookie' 2 | import { getBrowserLanguage } from '../../lang' 3 | 4 | const app = { 5 | state: { 6 | sidebar: { 7 | opened: !+Cookies.get('sidebarStatus'), 8 | withoutAnimation: false 9 | }, 10 | device: 'desktop', 11 | language: Cookies.get('language') || getBrowserLanguage() 12 | }, 13 | mutations: { 14 | TOGGLE_SIDEBAR: state => { 15 | if (state.sidebar.opened) { 16 | Cookies.set('sidebarStatus', 1) 17 | } else { 18 | Cookies.set('sidebarStatus', 0) 19 | } 20 | state.sidebar.opened = !state.sidebar.opened 21 | state.sidebar.withoutAnimation = false 22 | }, 23 | CLOSE_SIDEBAR: (state, withoutAnimation) => { 24 | Cookies.set('sidebarStatus', 1) 25 | state.sidebar.opened = false 26 | state.sidebar.withoutAnimation = withoutAnimation 27 | }, 28 | TOGGLE_DEVICE: (state, device) => { 29 | state.device = device 30 | }, 31 | SET_LANGUAGE: (state, language) => { 32 | state.language = language 33 | Cookies.set('language', language) 34 | } 35 | }, 36 | actions: { 37 | toggleSideBar({ commit }) { 38 | commit('TOGGLE_SIDEBAR') 39 | }, 40 | closeSideBar({ commit }, { withoutAnimation }) { 41 | commit('CLOSE_SIDEBAR', withoutAnimation) 42 | }, 43 | toggleDevice({ commit }, device) { 44 | commit('TOGGLE_DEVICE', device) 45 | }, 46 | setLanguage({ commit }, language) { 47 | commit('SET_LANGUAGE', language) 48 | } 49 | } 50 | } 51 | 52 | export default app 53 | -------------------------------------------------------------------------------- /web/model/exportdb_test.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/DATA-DOG/go-sqlmock" 7 | ) 8 | 9 | func (s *testModelSuite) TestGetExportDBList() { 10 | rows1 := sqlmock.NewRows([]string{"count(*)"}).AddRow(10) 11 | s.mock.ExpectQuery("(?i)select count\\(\\*\\) from `gospider_exportdb`"). 12 | WillReturnRows(rows1) 13 | rows2 := sqlmock.NewRows([]string{"id"}). 14 | AddRow(uint64(1)). 15 | AddRow(uint64(2)). 16 | AddRow(uint64(3)). 17 | AddRow(uint64(4)). 18 | AddRow(uint64(5)) 19 | s.mock.ExpectQuery("(?i)select \\* from `gospider_exportdb`"). 20 | WillReturnRows(rows2) 21 | 22 | list, count, err := GetExportDBList(s.gdb, 5, 0) 23 | s.Require().NoError(err) 24 | s.Equal(10, count) 25 | s.Equal(5, len(list)) 26 | s.Equal(uint64(1), list[0].ID) 27 | s.NoError(s.mock.ExpectationsWereMet()) 28 | 29 | s.mock.ExpectQuery("(?i)select count\\(\\*\\) from `gospider_exportdb`"). 30 | WillReturnError(fmt.Errorf("some error")) 31 | list, count, err = GetExportDBList(s.gdb, 5, 0) 32 | s.Require().NotNil(err) 33 | s.Equal(0, count) 34 | s.Nil(list) 35 | s.NoError(s.mock.ExpectationsWereMet()) 36 | 37 | rows3 := sqlmock.NewRows([]string{"count(*)"}).AddRow(10) 38 | s.mock.ExpectQuery("(?i)select count\\(\\*\\) from `gospider_exportdb`"). 39 | WillReturnRows(rows3) 40 | s.mock.ExpectQuery("(?i)select \\* from `gospider_exportdb` order by"). 41 | WillReturnError(fmt.Errorf("some error")) 42 | list2, count2, err2 := GetExportDBList(s.gdb, 5, 0) 43 | s.Require().NotNil(err2) 44 | s.Equal(0, count2) 45 | s.Nil(list2) 46 | s.NoError(s.mock.ExpectationsWereMet()) 47 | 48 | } 49 | -------------------------------------------------------------------------------- /web/router/task/stop_task.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "net/http" 5 | "strconv" 6 | 7 | "github.com/gin-gonic/gin" 8 | "github.com/nange/gospider/common" 9 | "github.com/nange/gospider/spider" 10 | "github.com/nange/gospider/web/core" 11 | "github.com/nange/gospider/web/model" 12 | "github.com/nange/gospider/web/service" 13 | "github.com/pkg/errors" 14 | log "github.com/sirupsen/logrus" 15 | ) 16 | 17 | // 根据任务id停止任务 18 | func StopTask(c *gin.Context) { 19 | taskIDStr := c.Param("id") 20 | if taskIDStr == "" { 21 | log.Warnf("StopTaskReq taskID is empty") 22 | c.AbortWithStatus(http.StatusBadRequest) 23 | return 24 | } 25 | taskID, err := strconv.ParseUint(taskIDStr, 10, 64) 26 | if err != nil { 27 | log.Warnf("StopTaskReq taskID format is invalid, taskID:%v", taskIDStr) 28 | c.AbortWithStatus(http.StatusBadRequest) 29 | return 30 | } 31 | log.Infof("StopTaskReq:%+v", taskID) 32 | 33 | if taskLock.IsRunning(taskID) { 34 | c.String(http.StatusConflict, "other operation is running") 35 | return 36 | } 37 | defer taskLock.Complete(taskID) 38 | 39 | // stop cron task 40 | if ct := service.GetCronTask(taskID); ct != nil { 41 | ct.Stop() 42 | } 43 | 44 | // cancel spider task 45 | spider.CancelTask(taskID) 46 | 47 | // set task status to TaskStatusStopped 48 | err = model.NewTaskQuerySet(core.GetGormDB()).IDEq(taskID).GetUpdater().SetStatus(common.TaskStatusStopped).Update() 49 | if err != nil { 50 | log.Errorf("update task status err:%+v", errors.WithStack(err)) 51 | c.String(http.StatusInternalServerError, "") 52 | return 53 | } 54 | 55 | c.String(http.StatusOK, "success") 56 | } 57 | -------------------------------------------------------------------------------- /web/static/src/components/Breadcrumb/index.vue: -------------------------------------------------------------------------------- 1 | 11 | 12 | 42 | 43 | 55 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/shoppingCard.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/build/check-versions.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const chalk = require('chalk') 3 | const semver = require('semver') 4 | const packageConfig = require('../package.json') 5 | const shell = require('shelljs') 6 | 7 | function exec(cmd) { 8 | return require('child_process') 9 | .execSync(cmd) 10 | .toString() 11 | .trim() 12 | } 13 | 14 | const versionRequirements = [ 15 | { 16 | name: 'node', 17 | currentVersion: semver.clean(process.version), 18 | versionRequirement: packageConfig.engines.node 19 | } 20 | ] 21 | 22 | if (shell.which('npm')) { 23 | versionRequirements.push({ 24 | name: 'npm', 25 | currentVersion: exec('npm --version'), 26 | versionRequirement: packageConfig.engines.npm 27 | }) 28 | } 29 | 30 | module.exports = function() { 31 | const warnings = [] 32 | 33 | for (let i = 0; i < versionRequirements.length; i++) { 34 | const mod = versionRequirements[i] 35 | 36 | if (!semver.satisfies(mod.currentVersion, mod.versionRequirement)) { 37 | warnings.push( 38 | mod.name + 39 | ': ' + 40 | chalk.red(mod.currentVersion) + 41 | ' should be ' + 42 | chalk.green(mod.versionRequirement) 43 | ) 44 | } 45 | } 46 | 47 | if (warnings.length) { 48 | console.log('') 49 | console.log( 50 | chalk.yellow( 51 | 'To use this template, you must update following to modules:' 52 | ) 53 | ) 54 | console.log() 55 | 56 | for (let i = 0; i < warnings.length; i++) { 57 | const warning = warnings[i] 58 | console.log(' ' + warning) 59 | } 60 | 61 | console.log() 62 | process.exit(1) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /web/static/src/store/modules/permission.js: -------------------------------------------------------------------------------- 1 | import { asyncRouterMap, constantRouterMap } from '@/router' 2 | 3 | /** 4 | * 通过meta.role判断是否与当前用户权限匹配 5 | * @param roles 6 | * @param route 7 | */ 8 | function hasPermission(roles, route) { 9 | if (route.meta && route.meta.roles) { 10 | return roles.some(role => route.meta.roles.indexOf(role) >= 0) 11 | } else { 12 | return true 13 | } 14 | } 15 | 16 | /** 17 | * 递归过滤异步路由表，返回符合用户角色权限的路由表 18 | * @param asyncRouterMap 19 | * @param roles 20 | */ 21 | function filterAsyncRouter(asyncRouterMap, roles) { 22 | const accessedRouters = asyncRouterMap.filter(route => { 23 | if (hasPermission(roles, route)) { 24 | if (route.children && route.children.length) { 25 | route.children = filterAsyncRouter(route.children, roles) 26 | } 27 | return true 28 | } 29 | return false 30 | }) 31 | return accessedRouters 32 | } 33 | 34 | const permission = { 35 | state: { 36 | routers: constantRouterMap, 37 | addRouters: [] 38 | }, 39 | mutations: { 40 | SET_ROUTERS: (state, routers) => { 41 | state.addRouters = routers 42 | state.routers = constantRouterMap.concat(routers) 43 | } 44 | }, 45 | actions: { 46 | GenerateRoutes({ commit }, data) { 47 | return new Promise(resolve => { 48 | const { roles } = data 49 | let accessedRouters 50 | if (roles.indexOf('admin') >= 0) { 51 | accessedRouters = asyncRouterMap 52 | } else { 53 | accessedRouters = filterAsyncRouter(asyncRouterMap, roles) 54 | } 55 | commit('SET_ROUTERS', accessedRouters) 56 | resolve() 57 | }) 58 | } 59 | } 60 | } 61 | 62 | export default permission 63 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/bug.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/people.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/directive/waves/waves.js: -------------------------------------------------------------------------------- 1 | import './waves.css' 2 | 3 | export default{ 4 | bind(el, binding) { 5 | el.addEventListener('click', e => { 6 | const customOpts = Object.assign({}, binding.value) 7 | const opts = Object.assign({ 8 | ele: el, // 波纹作用元素 9 | type: 'hit', // hit点击位置扩散center中心点扩展 10 | color: 'rgba(0, 0, 0, 0.15)' // 波纹颜色 11 | }, customOpts) 12 | const target = opts.ele 13 | if (target) { 14 | target.style.position = 'relative' 15 | target.style.overflow = 'hidden' 16 | const rect = target.getBoundingClientRect() 17 | let ripple = target.querySelector('.waves-ripple') 18 | if (!ripple) { 19 | ripple = document.createElement('span') 20 | ripple.className = 'waves-ripple' 21 | ripple.style.height = ripple.style.width = Math.max(rect.width, rect.height) + 'px' 22 | target.appendChild(ripple) 23 | } else { 24 | ripple.className = 'waves-ripple' 25 | } 26 | switch (opts.type) { 27 | case 'center': 28 | ripple.style.top = (rect.height / 2 - ripple.offsetHeight / 2) + 'px' 29 | ripple.style.left = (rect.width / 2 - ripple.offsetWidth / 2) + 'px' 30 | break 31 | default: 32 | ripple.style.top = (e.pageY - rect.top - ripple.offsetHeight / 2 - document.documentElement.scrollTop || document.body.scrollTop) + 'px' 33 | ripple.style.left = (e.pageX - rect.left - ripple.offsetWidth / 2 - document.documentElement.scrollLeft || document.body.scrollLeft) + 'px' 34 | } 35 | ripple.style.backgroundColor = opts.color 36 | ripple.className = 'waves-ripple z-active' 37 | return false 38 | } 39 | }, false) 40 | } 41 | } 42 | 43 | -------------------------------------------------------------------------------- /common/enu.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | // TaskStatus the task status type define 10 | type TaskStatus uint8 11 | 12 | const ( 13 | TaskStatusUnknown TaskStatus = iota 14 | TaskStatusRunning 15 | TaskStatusPaused 16 | TaskStatusStopped 17 | TaskStatusUnexceptedExited 18 | TaskStatusCompleted 19 | TaskStatusRunningTimeout 20 | ) 21 | 22 | var tsMap = map[TaskStatus]string{ 23 | TaskStatusUnknown: "未知状态", 24 | TaskStatusRunning: "运行中", 25 | TaskStatusPaused: "暂停", 26 | TaskStatusStopped: "停止", 27 | TaskStatusUnexceptedExited: "异常退出", 28 | TaskStatusCompleted: "完成", 29 | TaskStatusRunningTimeout: "运行超时", 30 | } 31 | 32 | var InvalidTaskStatus = errors.New("invalid task status") 33 | 34 | func (ts TaskStatus) String() string { 35 | s, ok := tsMap[ts] 36 | if !ok { 37 | panic(fmt.Sprintf("unexcepted TaskStatus %d", ts)) 38 | } 39 | return s 40 | } 41 | 42 | func ParseTaskStatusFromString(s string) (TaskStatus, error) { 43 | switch s { 44 | case "未知状态": 45 | return TaskStatusUnknown, nil 46 | case "运行中": 47 | return TaskStatusRunning, nil 48 | case "暂停": 49 | return TaskStatusPaused, nil 50 | case "停止": 51 | return TaskStatusStopped, nil 52 | case "异常退出": 53 | return TaskStatusUnexceptedExited, nil 54 | case "完成": 55 | return TaskStatusCompleted, nil 56 | case "运行超时": 57 | return TaskStatusRunningTimeout, nil 58 | } 59 | return TaskStatusUnknown, InvalidTaskStatus 60 | } 61 | 62 | func (ts TaskStatus) MarshalJSON() ([]byte, error) { 63 | return []byte("\"" + ts.String() + "\""), nil 64 | } 65 | 66 | func (ts *TaskStatus) UnmarshalJSON(data []byte) (err error) { 67 | s := strings.Trim(strings.ToUpper(string(data)), "\"") 68 | *ts, err = ParseTaskStatusFromString(s) 69 | return 70 | } 71 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/nange/gospider 2 | 3 | require ( 4 | cloud.google.com/go v0.26.0 // indirect 5 | github.com/DATA-DOG/go-sqlmock v1.3.2 6 | github.com/PuerkitoBio/goquery v1.4.1 7 | github.com/andybalholm/cascadia v1.0.0 // indirect 8 | github.com/antchfx/htmlquery v0.0.0-20180524052033-b4407197cfe8 // indirect 9 | github.com/antchfx/xmlquery v0.0.0-20180524052823-9188d8442369 // indirect 10 | github.com/antchfx/xpath v0.0.0-20180524052354-077bca4d2caa // indirect 11 | github.com/appleboy/gin-jwt/v2 v2.6.2 12 | github.com/denisenkom/go-mssqldb v0.0.0-20180824013952-8fac8b954edb // indirect 13 | github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 // indirect 14 | github.com/gin-contrib/sse v0.1.0 // indirect 15 | github.com/gin-gonic/gin v1.4.0 16 | github.com/go-sql-driver/mysql v1.4.1 17 | github.com/go-xorm/builder v0.3.4 18 | github.com/gobuffalo/packr v1.26.0 19 | github.com/gobwas/glob v0.2.3 // indirect 20 | github.com/gocolly/colly v1.2.0 21 | github.com/google/go-cmp v0.2.0 // indirect 22 | github.com/jinzhu/gorm v1.9.1 23 | github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect 24 | github.com/jinzhu/now v0.0.0-20180511015916-ed742868f2ae // indirect 25 | github.com/kennygrant/sanitize v1.2.3 // indirect 26 | github.com/pkg/errors v0.8.1 27 | github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967 28 | github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect 29 | github.com/sirupsen/logrus v1.4.2 30 | github.com/stretchr/testify v1.3.0 31 | github.com/temoto/robotstxt v0.0.0-20170603013557-9e4646fa7053 // indirect 32 | github.com/ugorji/go/codec v1.1.7 // indirect 33 | golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf 34 | golang.org/x/text v0.3.0 35 | google.golang.org/appengine v1.1.0 // indirect 36 | ) 37 | 38 | go 1.13 39 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/table.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/styles/btn.scss: -------------------------------------------------------------------------------- 1 | @import './variables.scss'; 2 | 3 | @mixin colorBtn($color) { 4 | background: $color; 5 | &:hover { 6 | color: $color; 7 | &:before, 8 | &:after { 9 | background: $color; 10 | } 11 | } 12 | } 13 | 14 | .blue-btn { 15 | @include colorBtn($blue) 16 | } 17 | 18 | .light-blue-btn { 19 | @include colorBtn($light-blue) 20 | } 21 | 22 | .red-btn { 23 | @include colorBtn($red) 24 | } 25 | 26 | .pink-btn { 27 | @include colorBtn($pink) 28 | } 29 | 30 | .green-btn { 31 | @include colorBtn($green) 32 | } 33 | 34 | .tiffany-btn { 35 | @include colorBtn($tiffany) 36 | } 37 | 38 | .yellow-btn { 39 | @include colorBtn($yellow) 40 | } 41 | 42 | .pan-btn { 43 | font-size: 14px; 44 | color: #fff; 45 | padding: 14px 36px; 46 | border-radius: 8px; 47 | border: none; 48 | outline: none; 49 | margin-right: 25px; 50 | transition: 600ms ease all; 51 | position: relative; 52 | display: inline-block; 53 | &:hover { 54 | background: #fff; 55 | &:before, 56 | &:after { 57 | width: 100%; 58 | transition: 600ms ease all; 59 | } 60 | } 61 | &:before, 62 | &:after { 63 | content: ''; 64 | position: absolute; 65 | top: 0; 66 | right: 0; 67 | height: 2px; 68 | width: 0; 69 | transition: 400ms ease all; 70 | } 71 | &::after { 72 | right: inherit; 73 | top: inherit; 74 | left: 0; 75 | bottom: 0; 76 | } 77 | } 78 | 79 | .custom-button { 80 | display: inline-block; 81 | line-height: 1; 82 | white-space: nowrap; 83 | cursor: pointer; 84 | background: #fff; 85 | color: #fff; 86 | -webkit-appearance: none; 87 | text-align: center; 88 | box-sizing: border-box; 89 | outline: 0; 90 | margin: 0; 91 | padding: 10px 15px; 92 | font-size: 14px; 93 | border-radius: 4px; 94 | } 95 | 96 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/eye.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/model/exportdb.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/jinzhu/gorm" 7 | "github.com/nange/gospider/web/core" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | //go:generate goqueryset -in exportdb.go 12 | // gen:qs 13 | type ExportDB struct { 14 | ID uint64 `json:"id" gorm:"column:id;type:bigint unsigned AUTO_INCREMENT;primary_key"` 15 | ShowName string `json:"show_name" gorm:"column:show_name;type:varchar(64);not null;unique_index:uk_show_name"` 16 | Host string `json:"host" gorm:"column:host;type:varchar(128);not null"` 17 | Port int `json:"port" gorm:"column:port;type:int;not null"` 18 | User string `json:"user" gorm:"column:user;type:varchar(32);not null"` 19 | Password string `json:"password" gorm:"column:password;type:varchar(32);not null;default:''"` 20 | DBName string `json:"db_name" gorm:"column:db_name;type:varchar(64);not null"` 21 | CreatedAt time.Time `json:"created_at" gorm:"column:created_at;type:datetime;not null;default:CURRENT_TIMESTAMP;index:idx_created_at"` 22 | UpdatedAt time.Time `json:"updated_at" gorm:"column:updated_at;type:datetime;not null;default:CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"` 23 | } 24 | 25 | func (o *ExportDB) TableName() string { 26 | return "gospider_exportdb" 27 | } 28 | 29 | func init() { 30 | core.Register(&ExportDB{}) 31 | } 32 | 33 | func GetExportDBList(db *gorm.DB, size, offset int) ([]ExportDB, int, error) { 34 | queryset := NewExportDBQuerySet(db) 35 | count, err := queryset.Count() 36 | if err != nil { 37 | return nil, 0, errors.WithStack(err) 38 | } 39 | 40 | queryset = NewExportDBQuerySet(db.Limit(size).Offset(offset)) 41 | ret := make([]ExportDB, 0) 42 | if err := queryset.OrderDescByCreatedAt().All(&ret); err != nil { 43 | return nil, 0, errors.WithStack(err) 44 | } 45 | 46 | return ret, count, nil 47 | } 48 | -------------------------------------------------------------------------------- /web/static/src/views/layout/Layout.vue: -------------------------------------------------------------------------------- 1 | 12 | 13 | 49 | 50 | 72 | -------------------------------------------------------------------------------- /web/static/src/lang/zh.js: -------------------------------------------------------------------------------- 1 | export default { 2 | route: { 3 | dashboard: '首页', 4 | icons: '图标', 5 | errorPages: '错误页面', 6 | page401: '401', 7 | page404: '404', 8 | errorLog: '错误日志', 9 | theme: '换肤', 10 | i18n: '国际化', 11 | taskManage: '任务管理', 12 | createTask: '创建任务', 13 | editTask: '修改任务', 14 | taskList: '任务列表', 15 | dbManage: '数据库管理', 16 | addExpDb: '添加导出数据库', 17 | expDbList: '导出数据库列表' 18 | }, 19 | task: { 20 | name: '任务名称', 21 | rule: '任务规则名', 22 | desc: '任务描述', 23 | cron: '定时执行', 24 | proxy: '代理列表', 25 | agent: '用户代理', 26 | maxDepth: '爬虫最大深度', 27 | allowDomains: '允许访问的域名', 28 | urlFilter: 'URL过滤', 29 | maxBody: '最大body值', 30 | requestTimeout: '请求超时时间', 31 | outType: '导出类型', 32 | autoMigrate: '自动建表', 33 | limitEn: '频率限制', 34 | limitDomainGlob: '域名glob匹配', 35 | limitDelay: '延迟', 36 | limitRandomDelay: '随机延迟', 37 | limitPara: '请求并发度', 38 | add: '创建任务', 39 | update: '更新任务', 40 | id: '序号', 41 | status: '状态', 42 | counts: '运行次数', 43 | iscron: '定时任务', 44 | create_at: '创建时间', 45 | actions: '操作', 46 | info: '详情', 47 | edit: '修改', 48 | stop: '停止', 49 | start: '启动', 50 | restart: '重启' 51 | }, 52 | exportdb: { 53 | add: '新建数据库', 54 | id: '序号', 55 | showname: '显示名称', 56 | host: '主机地址', 57 | port: '端口', 58 | user: '用户名', 59 | password: '密码', 60 | dbname: '数据库名', 61 | actions: '操作', 62 | edit: '修改', 63 | delete: '删除' 64 | }, 65 | navbar: { 66 | logOut: '退出登录', 67 | dashboard: '首页', 68 | screenfull: '全屏', 69 | theme: '换肤' 70 | }, 71 | login: { 72 | title: '系统登录', 73 | logIn: '登录', 74 | username: '账号', 75 | password: '密码' 76 | }, 77 | tagsView: { 78 | close: '关闭', 79 | closeOthers: '关闭其它', 80 | closeAll: '关闭所有' 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/international.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/model/user_test.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "github.com/DATA-DOG/go-sqlmock" 5 | ) 6 | 7 | func (s *testModelSuite) TestGenUserHashPassword() { 8 | hash, err := GenUserHashPassword("admin") 9 | s.NoErrorf(err, "should gen hash password success") 10 | s.T().Logf("hash:%s", hash) 11 | } 12 | 13 | func (s *testModelSuite) TestInitAdminUserIfNeeded() { 14 | row := sqlmock.NewRows([]string{"id"}).AddRow(uint64(1)) 15 | s.mock.ExpectQuery("(?i)select \\* from `gospider_user` where \\(user_name = \\?\\)"). 16 | WithArgs("admin"). 17 | WillReturnRows(row) 18 | err := InitAdminUserIfNeeded(s.gdb) 19 | s.NoError(err) 20 | s.NoError(s.mock.ExpectationsWereMet()) 21 | 22 | s.mock.ExpectQuery("(?i)select \\* from `gospider_user` where \\(user_name = \\?\\)"). 23 | WithArgs("admin"). 24 | WillReturnRows(sqlmock.NewRows([]string{"id"})) 25 | s.mock.ExpectExec("(?i)insert into `gospider_user` (.+) values"). 26 | WillReturnResult(sqlmock.NewResult(1, 1)) 27 | err = InitAdminUserIfNeeded(s.gdb) 28 | s.NoError(err) 29 | s.NoError(s.mock.ExpectationsWereMet()) 30 | 31 | } 32 | 33 | func (s *testModelSuite) TestIsValidUser() { 34 | pw, err := GenUserHashPassword("admin") 35 | s.Require().NoError(err) 36 | 37 | s.mock.ExpectQuery("(?i)select \\* from `gospider_user` where \\(user_name = \\?\\)"). 38 | WithArgs("admin"). 39 | WillReturnRows(sqlmock.NewRows([]string{"id"})) 40 | valid, user, err := IsValidUser(s.gdb, "admin", pw) 41 | s.Nil(err) 42 | s.False(valid) 43 | s.Nil(user) 44 | s.NoError(s.mock.ExpectationsWereMet()) 45 | 46 | row := sqlmock.NewRows([]string{"id", "password"}).AddRow(uint64(1), pw) 47 | s.mock.ExpectQuery("(?i)select \\* from `gospider_user` where \\(user_name = \\?\\)"). 48 | WithArgs("admin"). 49 | WillReturnRows(row) 50 | valid2, user2, err2 := IsValidUser(s.gdb, "admin", "admin") 51 | s.NoError(err2) 52 | s.True(valid2) 53 | s.NotNil(user2) 54 | s.NoError(s.mock.ExpectationsWereMet()) 55 | 56 | } 57 | -------------------------------------------------------------------------------- /web/static/src/components/Hamburger/index.vue: -------------------------------------------------------------------------------- 1 | 14 | 15 | 30 | 31 | 46 | -------------------------------------------------------------------------------- /web/static/build/build.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | require('./check-versions')() 3 | 4 | const ora = require('ora') 5 | const rm = require('rimraf') 6 | const path = require('path') 7 | const chalk = require('chalk') 8 | const webpack = require('webpack') 9 | const config = require('../config') 10 | const webpackConfig = require('./webpack.prod.conf') 11 | var connect = require('connect') 12 | var serveStatic = require('serve-static') 13 | 14 | const spinner = ora( 15 | 'building for ' + process.env.env_config + ' environment...' 16 | ) 17 | spinner.start() 18 | 19 | rm(path.join(config.build.assetsRoot, config.build.assetsSubDirectory), err => { 20 | if (err) throw err 21 | webpack(webpackConfig, (err, stats) => { 22 | spinner.stop() 23 | if (err) throw err 24 | process.stdout.write( 25 | stats.toString({ 26 | colors: true, 27 | modules: false, 28 | children: false, 29 | chunks: false, 30 | chunkModules: false 31 | }) + '\n\n' 32 | ) 33 | 34 | if (stats.hasErrors()) { 35 | console.log(chalk.red(' Build failed with errors.\n')) 36 | process.exit(1) 37 | } 38 | 39 | console.log(chalk.cyan(' Build complete.\n')) 40 | console.log( 41 | chalk.yellow( 42 | ' Tip: built files are meant to be served over an HTTP server.\n' + 43 | " Opening index.html over file:// won't work.\n" 44 | ) 45 | ) 46 | 47 | if (process.env.npm_config_preview) { 48 | const port = 9526 49 | const host = 'http://localhost:' + port 50 | const basePath = config.build.assetsPublicPath 51 | const app = connect() 52 | 53 | app.use( 54 | basePath, 55 | serveStatic('./dist', { 56 | index: ['index.html', '/'] 57 | }) 58 | ) 59 | 60 | app.listen(port, function() { 61 | console.log( 62 | chalk.green(`> Listening at http://localhost:${port}${basePath}`) 63 | ) 64 | }) 65 | } 66 | }) 67 | }) 68 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/dashboard.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/views/svg-icons/index.vue: -------------------------------------------------------------------------------- 1 | 22 | 23 | 50 | 51 | 78 | -------------------------------------------------------------------------------- /web/service/checktask.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "github.com/jinzhu/gorm" 5 | "github.com/nange/gospider/common" 6 | "github.com/nange/gospider/web/core" 7 | "github.com/nange/gospider/web/model" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | // check task status 12 | // 1. set status to TaskStatusRunning if status is TaskStatusUnexceptedExited 13 | // 2. restart task if status is TaskStatusCompleted, TaskStatusRunning 14 | func CheckTask() { 15 | log.Infof("starting check task goroutine") 16 | qs := model.NewTaskQuerySet(core.GetGormDB()) 17 | 18 | tasks := make([]model.Task, 0) 19 | if err := qs.All(&tasks); err != nil { 20 | if err == gorm.ErrRecordNotFound { 21 | log.Infof("no task found, exit service.CheckTask method") 22 | return 23 | } 24 | log.Errorf("query task list err: %+v", err) 25 | return 26 | } 27 | 28 | for _, task := range tasks { 29 | if task.Status == common.TaskStatusRunning { 30 | log.Infof("set task status to TaskStatusUnexceptedExited, taskID:%v", task.ID) 31 | err := model.NewTaskQuerySet(core.GetGormDB()).IDEq(task.ID). 32 | GetUpdater().SetStatus(common.TaskStatusUnexceptedExited).Update() 33 | if err != nil { 34 | log.Errorf("update task status err: %+v", err) 35 | continue 36 | } 37 | } 38 | 39 | if (task.Status == common.TaskStatusCompleted || task.Status == common.TaskStatusRunning || 40 | task.Status == common.TaskStatusUnexceptedExited) && task.CronSpec != "" { 41 | if err := CreateCronTask(task); err != nil { 42 | log.Errorf("restart task err:%+v", err) 43 | continue 44 | } 45 | } 46 | } 47 | 48 | } 49 | 50 | func CreateCronTask(task model.Task) error { 51 | log.Infof("restarting task, taskID:%v", task.ID) 52 | 53 | ct, err := NewCronTask(task.ID, task.CronSpec, GetMTSChan()) 54 | if err != nil { 55 | log.Errorf("new cron task failed! err:%+v", err) 56 | return err 57 | } 58 | 59 | if err := ct.Start(); err != nil { 60 | log.Errorf("start cron task failed! err:%+v", err) 61 | return err 62 | } 63 | 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /web/router/task/create_task.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/gin-gonic/gin" 8 | "github.com/nange/gospider/common" 9 | "github.com/nange/gospider/spider" 10 | "github.com/nange/gospider/web/core" 11 | "github.com/nange/gospider/web/model" 12 | "github.com/nange/gospider/web/service" 13 | log "github.com/sirupsen/logrus" 14 | ) 15 | 16 | type CreateTaskReq struct { 17 | model.Task 18 | } 19 | 20 | type CreateTaskResp struct { 21 | ID uint64 `json:"id"` 22 | CreateAt time.Time `json:"create_at"` 23 | } 24 | 25 | func CreateTask(c *gin.Context) { 26 | var req CreateTaskReq 27 | if err := c.BindJSON(&req); err != nil { 28 | log.Errorf("bind json failed! err:%+v", err) 29 | c.AbortWithStatus(http.StatusBadRequest) 30 | return 31 | } 32 | log.Infof("req:%+v", req) 33 | 34 | task := req.Task 35 | task.Status = common.TaskStatusRunning 36 | if err := task.Create(core.GetGormDB()); err != nil { 37 | log.Errorf("create task failed! err:%+v", err) 38 | c.AbortWithStatus(http.StatusInternalServerError) 39 | return 40 | } 41 | 42 | spiderTask, err := service.GetSpiderTaskByModel(&task) 43 | if err != nil { 44 | log.Errorf("spider get model task failed! err:%+v", err) 45 | c.AbortWithStatus(http.StatusInternalServerError) 46 | return 47 | } 48 | s := spider.New(spiderTask, service.GetMTSChan()) 49 | if err := s.Run(); err != nil { 50 | log.Errorf("spider run task failed! err:%+v", err) 51 | c.AbortWithStatus(http.StatusInternalServerError) 52 | return 53 | } 54 | 55 | if task.CronSpec != "" { 56 | log.Infof("starting cron task:%s", task.CronSpec) 57 | ct, err := service.NewCronTask(task.ID, task.CronSpec, service.GetMTSChan()) 58 | if err != nil { 59 | log.Errorf("new cron task failed! err:%+v", err) 60 | } else { 61 | if err := ct.Start(); err != nil { 62 | log.Errorf("start cron task failed! err:%+v", err) 63 | } 64 | } 65 | 66 | } 67 | 68 | c.JSON(http.StatusOK, &CreateTaskResp{ 69 | ID: task.ID, 70 | CreateAt: task.CreatedAt, 71 | }) 72 | } 73 | -------------------------------------------------------------------------------- /web/static/src/components/Screenfull/index.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 55 | 56 | 66 | -------------------------------------------------------------------------------- /web/static/src/lang/en.js: -------------------------------------------------------------------------------- 1 | export default { 2 | route: { 3 | dashboard: 'Dashboard', 4 | icons: 'Icons', 5 | errorPages: 'Error Pages', 6 | page401: '401', 7 | page404: '404', 8 | errorLog: 'Error Log', 9 | theme: 'Theme', 10 | i18n: 'I18n', 11 | taskManage: 'Task Manage', 12 | createTask: 'Create Task', 13 | editTask: 'Edit Task', 14 | taskList: 'Task List', 15 | dbManage: 'Database Manage', 16 | addExpDb: 'Add Export DB', 17 | expDbList: 'Export DB List' 18 | }, 19 | task: { 20 | name: 'Task Alias', 21 | rule: 'Task Rule Name', 22 | desc: 'Task Describe', 23 | cron: 'Crontab Config', 24 | proxy: 'Proxy Urls', 25 | agent: 'User Agent', 26 | maxDepth: 'Max Depth', 27 | allowDomains: 'Allow Domains', 28 | urlFilter: 'URL Filters', 29 | maxBody: 'Max Body Size', 30 | requestTimeout: 'Request Timeout', 31 | outType: 'Output Type', 32 | autoMigrate: 'Auto Migrate', 33 | limitEn: 'Limit Enable', 34 | limitDomainGlob: 'Limit Domain Glob', 35 | limitDelay: 'Limit Delay', 36 | limitRandomDelay: 'Limit Random Delay', 37 | limitPara: 'Limit Parallelism', 38 | add: 'Create Task', 39 | update: 'Update Task', 40 | id: 'ID', 41 | status: 'Status', 42 | counts: 'Run Counts', 43 | iscron: 'Is Cron', 44 | create_at: 'Create Time', 45 | actions: 'Operation', 46 | info: 'Info', 47 | edit: 'Edit', 48 | stop: 'Stop', 49 | start: 'Start', 50 | restart: 'Restart' 51 | }, 52 | exportdb: { 53 | id: 'ID', 54 | showname: 'Show Name', 55 | host: 'Host', 56 | port: 'Post', 57 | user: 'User', 58 | password: 'Password', 59 | dbname: 'Db Name', 60 | actions: 'Operation', 61 | edit: 'Edit', 62 | delete: 'Delete' 63 | }, 64 | navbar: { 65 | logOut: 'Log Out', 66 | dashboard: 'Dashboard', 67 | screenfull: 'screenfull', 68 | theme: 'theme' 69 | }, 70 | login: { 71 | title: 'Login Form', 72 | logIn: 'Log in', 73 | username: 'Username', 74 | password: 'Password' 75 | }, 76 | tagsView: { 77 | close: 'Close', 78 | closeOthers: 'Close Others', 79 | closeAll: 'Close All' 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/wechat.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /spider/errors.go: -------------------------------------------------------------------------------- 1 | package spider 2 | 3 | import "github.com/pkg/errors" 4 | 5 | var ( 6 | // ErrTaskRuleNotExist is the error type for task rule not exist 7 | ErrTaskRuleNotExist = errors.New("task rule not exist") 8 | // ErrTaskRuleIsNil is the error thrown when a nil rule registered 9 | ErrTaskRuleIsNil = errors.New("task rule is nil") 10 | // ErrTaskRuleNameIsEmpty is the error thrown when the ruleName is empty 11 | ErrTaskRuleNameIsEmpty = errors.New("task rule name is empty") 12 | // ErrTaskRuleNameDuplicated is the error thrown if the rule name is duplicated 13 | ErrTaskRuleNameDuplicated = errors.New("task rule name is Duplicated") 14 | // ErrTaskRuleHeadIsNil is the error thrown if the rule's head is nil 15 | ErrTaskRuleHeadIsNil = errors.New("task rule head is nil") 16 | // ErrTaskRuleNodesLenInvalid is the error thrown if the rule's nodes len is invalid 17 | ErrTaskRuleNodesLenInvalid = errors.New("task rule nodes len is invalid") 18 | // ErrTaskRuleNodesKeyInvalid is the error thrown if the rule's key len is invalid 19 | ErrTaskRuleNodesKeyInvalid = errors.New("task rule nodes key should start from 0 and monotonically increasing") 20 | // ErrTaskRunningTimeout is the error type for task running timeout 21 | ErrTaskRunningTimeout = errors.New("task running timeout") 22 | ) 23 | 24 | var ( 25 | // ErrOutputFieldsNotMatchOutputRow is the error type for output fields not match out put row 26 | ErrOutputFieldsNotMatchOutputRow = errors.New("output fields not match out put row") 27 | // ErrTooManyOutputNamespace is the error type for for too many output namespace 28 | ErrTooManyOutputNamespace = errors.New("too many output namespace") 29 | // ErrOutputToMultipleTableDisabled is the error thrown if "OutputToMultipleTable" is false 30 | ErrOutputToMultipleTableDisabled = errors.New("output to multiple tables disabled") 31 | // ErrOutputTypeNotSupported is the error type for unknown output type 32 | ErrOutputTypeNotSupported = errors.New("output type not supported") 33 | // ErrMultConfNamespaceNotFound is the error type for mult conf namespace not found 34 | ErrMultConfNamespaceNotFound = errors.New("mult conf namespace not found") 35 | // ErrOutputParamNotSupported is the error type for unknown output param 36 | ErrOutputParamNotSupported = errors.New("output param not supported ") 37 | ) 38 | -------------------------------------------------------------------------------- /web/static/src/components/ScrollPane/index.vue: -------------------------------------------------------------------------------- 1 | 8 | 9 | 61 | 62 | 73 | -------------------------------------------------------------------------------- /web/static/src/views/errorPage/401.vue: -------------------------------------------------------------------------------- 1 | 25 | 26 | 49 | 50 | 88 | -------------------------------------------------------------------------------- /web/static/src/utils/request.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios' 2 | import { Message } from 'element-ui' 3 | import store from '@/store' 4 | import { getToken } from '@/utils/auth' 5 | 6 | // create an axios instance 7 | const service = axios.create({ 8 | baseURL: process.env.BASE_API, // api的base_url 9 | timeout: 5000 // request timeout 10 | }) 11 | 12 | // request interceptor 13 | service.interceptors.request.use( 14 | config => { 15 | // Do something before request is sent 16 | if (store.getters.token) { 17 | // 让每个请求携带token-- ['X-Token']为自定义key 请根据实际情况自行修改 18 | config.headers['X-Token'] = getToken() 19 | } 20 | return config 21 | }, 22 | error => { 23 | // Do something with request error 24 | console.log(error) // for debug 25 | Promise.reject(error) 26 | } 27 | ) 28 | 29 | // respone interceptor 30 | service.interceptors.response.use( 31 | response => response, 32 | /** 33 | * 下面的注释为通过在response里，自定义code来标示请求状态 34 | * 当code返回如下情况则说明权限有问题，登出并返回到登录页 35 | * 如想通过xmlhttprequest来状态码标识 逻辑可写在下面error中 36 | * 以下代码均为样例，请结合自生需求加以修改，若不需要，则可删除 37 | */ 38 | // response => { 39 | // const res = response.data 40 | // if (res.code !== 20000) { 41 | // Message({ 42 | // message: res.message, 43 | // type: 'error', 44 | // duration: 5 * 1000 45 | // }) 46 | // // 50008:非法的token; 50012:其他客户端登录了; 50014:Token 过期了; 47 | // if (res.code === 50008 || res.code === 50012 || res.code === 50014) { 48 | // // 请自行在引入 MessageBox 49 | // // import { Message, MessageBox } from 'element-ui' 50 | // MessageBox.confirm('你已被登出，可以取消继续留在该页面，或者重新登录', '确定登出', { 51 | // confirmButtonText: '重新登录', 52 | // cancelButtonText: '取消', 53 | // type: 'warning' 54 | // }).then(() => { 55 | // store.dispatch('FedLogOut').then(() => { 56 | // location.reload() // 为了重新实例化vue-router对象 避免bug 57 | // }) 58 | // }) 59 | // } 60 | // return Promise.reject('error') 61 | // } else { 62 | // return response.data 63 | // } 64 | // }, 65 | error => { 66 | console.log('err' + error) // for debug 67 | Message({ 68 | message: error.message, 69 | type: 'error', 70 | duration: 5 * 1000 71 | }) 72 | return Promise.reject(error) 73 | } 74 | ) 75 | 76 | export default service 77 | -------------------------------------------------------------------------------- /web/static/src/store/modules/tagsView.js: -------------------------------------------------------------------------------- 1 | const tagsView = { 2 | state: { 3 | visitedViews: [], 4 | cachedViews: [] 5 | }, 6 | mutations: { 7 | ADD_VISITED_VIEWS: (state, view) => { 8 | if (state.visitedViews.some(v => v.path === view.path)) return 9 | state.visitedViews.push( 10 | Object.assign({}, view, { 11 | title: view.meta.title || 'no-name' 12 | }) 13 | ) 14 | if (!view.meta.noCache) { 15 | state.cachedViews.push(view.name) 16 | } 17 | }, 18 | DEL_VISITED_VIEWS: (state, view) => { 19 | for (const [i, v] of state.visitedViews.entries()) { 20 | if (v.path === view.path) { 21 | state.visitedViews.splice(i, 1) 22 | break 23 | } 24 | } 25 | for (const i of state.cachedViews) { 26 | if (i === view.name) { 27 | const index = state.cachedViews.indexOf(i) 28 | state.cachedViews.splice(index, 1) 29 | break 30 | } 31 | } 32 | }, 33 | DEL_OTHERS_VIEWS: (state, view) => { 34 | for (const [i, v] of state.visitedViews.entries()) { 35 | if (v.path === view.path) { 36 | state.visitedViews = state.visitedViews.slice(i, i + 1) 37 | break 38 | } 39 | } 40 | for (const i of state.cachedViews) { 41 | if (i === view.name) { 42 | const index = state.cachedViews.indexOf(i) 43 | state.cachedViews = state.cachedViews.slice(index, index + 1) 44 | break 45 | } 46 | } 47 | }, 48 | DEL_ALL_VIEWS: state => { 49 | state.visitedViews = [] 50 | state.cachedViews = [] 51 | } 52 | }, 53 | actions: { 54 | addVisitedViews({ commit }, view) { 55 | commit('ADD_VISITED_VIEWS', view) 56 | }, 57 | delVisitedViews({ commit, state }, view) { 58 | return new Promise(resolve => { 59 | commit('DEL_VISITED_VIEWS', view) 60 | resolve([...state.visitedViews]) 61 | }) 62 | }, 63 | delOthersViews({ commit, state }, view) { 64 | return new Promise(resolve => { 65 | commit('DEL_OTHERS_VIEWS', view) 66 | resolve([...state.visitedViews]) 67 | }) 68 | }, 69 | delAllViews({ commit, state }) { 70 | return new Promise(resolve => { 71 | commit('DEL_ALL_VIEWS') 72 | resolve([...state.visitedViews]) 73 | }) 74 | } 75 | } 76 | } 77 | 78 | export default tagsView 79 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/zip.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /web/static/src/icons/svg/form.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/static/src/permission.js: -------------------------------------------------------------------------------- 1 | import router from './router' 2 | import store from './store' 3 | import { Message } from 'element-ui' 4 | import NProgress from 'nprogress' // progress bar 5 | import 'nprogress/nprogress.css'// progress bar style 6 | import { getToken } from '@/utils/auth' // getToken from cookie 7 | 8 | NProgress.configure({ showSpinner: false })// NProgress Configuration 9 | 10 | // permission judge function 11 | function hasPermission(roles, permissionRoles) { 12 | if (roles.indexOf('admin') >= 0) return true // admin permission passed directly 13 | if (!permissionRoles) return true 14 | return roles.some(role => permissionRoles.indexOf(role) >= 0) 15 | } 16 | 17 | const whiteList = ['/login', '/authredirect']// no redirect whitelist 18 | 19 | router.beforeEach((to, from, next) => { 20 | NProgress.start() // start progress bar 21 | if (getToken()) { // determine if there has token 22 | /* has token*/ 23 | if (to.path === '/login') { 24 | next({ path: '/' }) 25 | NProgress.done() // if current page is dashboard will not trigger afterEach hook, so manually handle it 26 | } else { 27 | if (store.getters.roles.length === 0) { // 判断当前用户是否已拉取完user_info信息 28 | store.dispatch('GetUserInfo').then(res => { // 拉取user_info 29 | const roles = res.data.roles // note: roles must be a array! such as: ['editor','develop'] 30 | store.dispatch('GenerateRoutes', { roles }).then(() => { // 根据roles权限生成可访问的路由表 31 | router.addRoutes(store.getters.addRouters) // 动态添加可访问路由表 32 | next({ ...to, replace: true }) // hack方法 确保addRoutes已完成 ,set the replace: true so the navigation will not leave a history record 33 | }) 34 | }).catch((err) => { 35 | store.dispatch('FedLogOut').then(() => { 36 | Message.error(err || 'Verification failed, please login again') 37 | next({ path: '/' }) 38 | }) 39 | }) 40 | } else { 41 | // 没有动态改变权限的需求可直接next() 删除下方权限判断 ↓ 42 | if (hasPermission(store.getters.roles, to.meta.roles)) { 43 | next() 44 | } else { 45 | next({ path: '/401', replace: true, query: { noGoBack: true }}) 46 | } 47 | // 可删 ↑ 48 | } 49 | } 50 | } else { 51 | /* has no token*/ 52 | if (whiteList.indexOf(to.path) !== -1) { // 在免登录白名单，直接进入 53 | next() 54 | } else { 55 | next('/login') // 否则全部重定向到登录页 56 | NProgress.done() // if current page is login will not trigger afterEach hook, so manually handle it 57 | } 58 | } 59 | }) 60 | 61 | router.afterEach(() => { 62 | NProgress.done() // finish progress bar 63 | }) 64 | -------------------------------------------------------------------------------- /web/router/task/restart_task.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "net/http" 5 | "strconv" 6 | 7 | "github.com/gin-gonic/gin" 8 | "github.com/nange/gospider/common" 9 | "github.com/nange/gospider/spider" 10 | "github.com/nange/gospider/web/core" 11 | "github.com/nange/gospider/web/model" 12 | "github.com/nange/gospider/web/service" 13 | "github.com/pkg/errors" 14 | log "github.com/sirupsen/logrus" 15 | ) 16 | 17 | // 根据任务id重启定时任务 18 | func RestartTask(c *gin.Context) { 19 | taskIDStr := c.Param("id") 20 | if taskIDStr == "" { 21 | log.Warnf("RestartTaskReq taskID is empty") 22 | c.AbortWithStatus(http.StatusBadRequest) 23 | return 24 | } 25 | taskID, err := strconv.ParseUint(taskIDStr, 10, 64) 26 | if err != nil { 27 | log.Warnf("RestartTaskReq taskID format is invalid, taskID:%v", taskIDStr) 28 | c.AbortWithStatus(http.StatusBadRequest) 29 | return 30 | } 31 | log.Infof("RestartTaskReq:%+v", taskID) 32 | 33 | if taskLock.IsRunning(taskID) { 34 | c.String(http.StatusConflict, "other operation is running") 35 | return 36 | } 37 | defer taskLock.Complete(taskID) 38 | 39 | // query task info from db 40 | task := &model.Task{} 41 | err = model.NewTaskQuerySet(core.GetGormDB()).IDEq(taskID).One(task) 42 | if err != nil { 43 | log.Errorf("RestartTaskReq query model task fail, taskID: %v , err: %+v", taskIDStr, err) 44 | c.AbortWithStatus(http.StatusInternalServerError) 45 | return 46 | } 47 | // only allow crontab task 48 | if task.CronSpec == "" { 49 | log.Warnf("RestartTaskReq taskID is not crontab task, taskID: %v", taskIDStr) 50 | c.AbortWithStatus(http.StatusBadRequest) 51 | return 52 | } 53 | 54 | // check task status 55 | if !taskCanBeRestart(task) { 56 | log.Warnf("StartTaskReq taskID status is non-conformance , taskID: %v", taskIDStr) 57 | c.AbortWithStatus(http.StatusBadRequest) 58 | return 59 | } 60 | 61 | // create crontab task 62 | err = service.CreateCronTask(*task) 63 | if err != nil { 64 | log.Errorf("RestartTaskReq run task fail, taskID: %v , err: %+v", taskIDStr, err) 65 | c.AbortWithStatus(http.StatusInternalServerError) 66 | return 67 | } 68 | // update task status 69 | err = model.NewTaskQuerySet(core.GetGormDB()).IDEq(taskID).GetUpdater().SetStatus(common.TaskStatusCompleted).Update() 70 | if err != nil { 71 | // stop cron task 72 | if ct := service.GetCronTask(taskID); ct != nil { 73 | ct.Stop() 74 | } 75 | 76 | // cancel spider task 77 | spider.CancelTask(taskID) 78 | 79 | log.Errorf("RestartTaskReq update task status err:%+v", errors.WithStack(err)) 80 | c.String(http.StatusInternalServerError, "") 81 | return 82 | } 83 | c.String(http.StatusOK, "success") 84 | } 85 | 86 | func taskCanBeRestart(task *model.Task) bool { 87 | return task.Status == common.TaskStatusStopped 88 | } 89 | -------------------------------------------------------------------------------- /_example/rule/baidunews/baidu_news.go: -------------------------------------------------------------------------------- 1 | package baidunews 2 | 3 | import ( 4 | "github.com/nange/gospider/spider" 5 | log "github.com/sirupsen/logrus" 6 | ) 7 | 8 | func init() { 9 | spider.Register(rule) 10 | } 11 | 12 | var ( 13 | outputFields = []string{"category", "title", "link"} 14 | outputFields2 = []string{"category", "category_link"} 15 | 16 | namespace1 = "baidu_news" 17 | namespace2 = "baidu_category" 18 | ) 19 | var multNamespaceConf = map[string]*spider.MultipleNamespaceConf{ 20 | namespace1: { 21 | OutputFields: outputFields, 22 | OutputConstraints: spider.NewStringsConstraints(outputFields, 64, 128, 512), 23 | }, 24 | namespace2: { 25 | OutputFields: outputFields2, 26 | OutputConstraints: spider.NewStringsConstraints(outputFields2, 64, 256), 27 | }, 28 | } 29 | 30 | // 演示如何在一条规则里面，同时需要导出数据到两张表 31 | var rule = &spider.TaskRule{ 32 | Name: "百度新闻规则", 33 | Description: "抓取百度新闻各个分类的最新焦点新闻以及最新的新闻分类和链接", 34 | OutputToMultipleNamespace: true, 35 | MultipleNamespaceConf: multNamespaceConf, 36 | Rule: &spider.Rule{ 37 | Head: func(ctx *spider.Context) error { 38 | return ctx.VisitForNext("http://news.baidu.com") 39 | }, 40 | Nodes: map[int]*spider.Node{ 41 | 0: step1, // 第一步: 获取所有分类 42 | 1: step2, // 第二步: 获取每个分类的新闻标题链接 43 | }, 44 | }, 45 | } 46 | 47 | var step1 = &spider.Node{ 48 | OnRequest: func(ctx *spider.Context, req *spider.Request) { 49 | log.Println("Visting", req.URL.String()) 50 | }, 51 | OnHTML: map[string]func(*spider.Context, *spider.HTMLElement) error{ 52 | `#channel-all .menu-list a`: func(ctx *spider.Context, el *spider.HTMLElement) error { // 获取所有分类 53 | category := el.Text 54 | ctx.PutReqContextValue("category", category) 55 | link := el.Attr("href") 56 | 57 | if category != "首页" { 58 | err := ctx.Output(map[int]interface{}{ 59 | 0: category, 60 | 1: ctx.AbsoluteURL(link), 61 | }, namespace2) 62 | if err != nil { 63 | return err 64 | } 65 | } 66 | 67 | return ctx.VisitForNextWithContext(link) 68 | }, 69 | }, 70 | } 71 | 72 | var step2 = &spider.Node{ 73 | OnRequest: func(ctx *spider.Context, req *spider.Request) { 74 | log.Println("Visting", req.URL.String()) 75 | }, 76 | OnHTML: map[string]func(*spider.Context, *spider.HTMLElement) error{ 77 | `#col_focus a, .focal-news a, .auto-col-focus a, .l-common .fn-c a`: func(ctx *spider.Context, el *spider.HTMLElement) error { 78 | title := el.Text 79 | link := el.Attr("href") 80 | if title == "" || link == "javascript:void(0);" { 81 | return nil 82 | } 83 | 84 | category := ctx.GetReqContextValue("category") 85 | return ctx.Output(map[int]interface{}{ 86 | 0: category, 87 | 1: title, 88 | 2: link, 89 | }, namespace1) 90 | }, 91 | }, 92 | } 93 | -------------------------------------------------------------------------------- /spider/task_rule.go: -------------------------------------------------------------------------------- 1 | package spider 2 | 3 | import "github.com/pkg/errors" 4 | 5 | var rules = make(map[string]*TaskRule) 6 | 7 | // Register register a task rule 8 | func Register(rule *TaskRule) { 9 | if err := checkRule(rule); err != nil { 10 | panic(err) 11 | } 12 | rules[rule.Name] = rule 13 | } 14 | 15 | // MultipleNamespaceConf is the mutiple namespace conf 16 | type MultipleNamespaceConf struct { 17 | OutputFields []string 18 | OutputConstraints map[string]*OutputConstraint 19 | OutputTableOpts string 20 | } 21 | 22 | // TaskRule is the task rule define 23 | type TaskRule struct { 24 | Name string 25 | Description string 26 | OutputToMultipleNamespace bool 27 | MultipleNamespaceConf map[string]*MultipleNamespaceConf 28 | Namespace string 29 | OutputFields []string 30 | OutputConstraints map[string]*OutputConstraint 31 | OutputTableOpts string 32 | DisableCookies bool 33 | AllowURLRevisit bool 34 | IgnoreRobotsTxt bool 35 | InsecureSkipVerify bool 36 | ParseHTTPErrorResponse bool 37 | Rule *Rule 38 | } 39 | 40 | // GetTaskRule get task rule by ruleName 41 | func GetTaskRule(ruleName string) (*TaskRule, error) { 42 | if rule, ok := rules[ruleName]; ok { 43 | return rule, nil 44 | } 45 | return nil, errors.WithStack(ErrTaskRuleNotExist) 46 | } 47 | 48 | // GetTaskRuleKeys return all keys of task rule 49 | func GetTaskRuleKeys() []string { 50 | keys := make([]string, 0, len(rules)) 51 | for k := range rules { 52 | keys = append(keys, k) 53 | } 54 | 55 | return keys 56 | } 57 | 58 | // Rule the rule define 59 | type Rule struct { 60 | Head func(ctx *Context) error 61 | Nodes map[int]*Node 62 | } 63 | 64 | // Node the rule node of a task 65 | type Node struct { 66 | OnRequest func(ctx *Context, req *Request) 67 | OnError func(ctx *Context, res *Response, err error) error 68 | OnResponse func(ctx *Context, res *Response) error 69 | OnHTML map[string]func(ctx *Context, el *HTMLElement) error 70 | OnXML map[string]func(ctx *Context, el *XMLElement) error 71 | OnScraped func(ctx *Context, res *Response) error 72 | } 73 | 74 | func checkRule(rule *TaskRule) error { 75 | if rule == nil || rule.Rule == nil { 76 | return ErrTaskRuleIsNil 77 | } 78 | if rule.Name == "" { 79 | return ErrTaskRuleNameIsEmpty 80 | } 81 | if rule.Rule.Head == nil { 82 | return ErrTaskRuleHeadIsNil 83 | } 84 | if len(rule.Rule.Nodes) == 0 { 85 | return ErrTaskRuleNodesLenInvalid 86 | } 87 | for i := 0; i < len(rule.Rule.Nodes); i++ { 88 | if _, ok := rule.Rule.Nodes[i]; !ok { 89 | return ErrTaskRuleNodesKeyInvalid 90 | } 91 | } 92 | if _, ok := rules[rule.Name]; ok { 93 | return ErrTaskRuleNameDuplicated 94 | } 95 | 96 | return nil 97 | } 98 | -------------------------------------------------------------------------------- /web/router/route.go: -------------------------------------------------------------------------------- 1 | package router 2 | 3 | import ( 4 | "time" 5 | 6 | jwt "github.com/appleboy/gin-jwt/v2" 7 | "github.com/gin-gonic/gin" 8 | "github.com/gobuffalo/packr" 9 | "github.com/nange/gospider/web/core" 10 | "github.com/nange/gospider/web/model" 11 | "github.com/nange/gospider/web/router/exportdb" 12 | "github.com/nange/gospider/web/router/rule" 13 | "github.com/nange/gospider/web/router/task" 14 | "github.com/nange/gospider/web/router/user" 15 | log "github.com/sirupsen/logrus" 16 | ) 17 | 18 | func Route(engine *gin.Engine) { 19 | authMiddleware, err := JwtAuth() 20 | if err != nil { 21 | log.Fatalf("get jwt middleware err [%+v]", err) 22 | } 23 | 24 | engine.POST("/login", authMiddleware.LoginHandler) 25 | api := engine.Group("/api") 26 | api.Use(authMiddleware.MiddlewareFunc()) 27 | { 28 | api.GET("/tasks", task.GetTaskList) 29 | api.GET("/tasks/:id", task.GetTaskByID) 30 | api.POST("/tasks", task.CreateTask) 31 | api.PUT("/tasks/:id", task.UpdateTask) 32 | api.PUT("/tasks/:id/stop", task.StopTask) 33 | api.PUT("/tasks/:id/start", task.StartTask) 34 | api.PUT("/tasks/:id/restart", task.RestartTask) 35 | 36 | api.GET("/rules", rule.GetRuleList) 37 | 38 | api.GET("/exportdb", exportdb.GetExportDBList) 39 | api.POST("/exportdb", exportdb.CreateExportDB) 40 | api.DELETE("/exportdb/:id", exportdb.DeleteExportDB) 41 | 42 | api.GET("/user/info", user.GetUserInfo) 43 | 44 | } 45 | 46 | box := packr.NewBox("../static/dist") 47 | box2 := packr.NewBox("../static/dist/static") 48 | engine.StaticFS("/admin", box) 49 | engine.StaticFS("/static", box2) 50 | } 51 | 52 | func JwtAuth() (*jwt.GinJWTMiddleware, error) { 53 | type login struct { 54 | Username string `form:"username" json:"username" binding:"required"` 55 | Password string `form:"password" json:"password" binding:"required"` 56 | } 57 | middle, err := jwt.New(&jwt.GinJWTMiddleware{ 58 | Key: []byte("gospider"), 59 | Timeout: 24 * time.Hour, 60 | MaxRefresh: 24 * time.Hour, 61 | PayloadFunc: func(data interface{}) jwt.MapClaims { 62 | if v, ok := data.(*model.User); ok { 63 | return jwt.MapClaims{ 64 | jwt.IdentityKey: v, 65 | } 66 | } 67 | return jwt.MapClaims{} 68 | }, 69 | Authenticator: func(c *gin.Context) (i interface{}, e error) { 70 | var loginVals login 71 | if err := c.ShouldBind(&loginVals); err != nil { 72 | return "", jwt.ErrMissingLoginValues 73 | } 74 | valid, user, err := model.IsValidUser(core.GetGormDB(), loginVals.Username, loginVals.Password) 75 | if err != nil { 76 | return nil, err 77 | } 78 | if valid { 79 | return user, nil 80 | } 81 | 82 | return nil, jwt.ErrFailedAuthentication 83 | }, 84 | TokenLookup: "header: Authorization, query: token, cookie: jwt", 85 | SendCookie: true, 86 | }) 87 | 88 | return middle, err 89 | } 90 | -------------------------------------------------------------------------------- /web/model/user.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/jinzhu/gorm" 7 | "github.com/nange/gospider/web/core" 8 | log "github.com/sirupsen/logrus" 9 | "golang.org/x/crypto/bcrypt" 10 | ) 11 | 12 | //go:generate goqueryset -in user.go 13 | // gen:qs 14 | type User struct { 15 | ID uint64 `json:"id,string" gorm:"column:id;type:bigint unsigned AUTO_INCREMENT;primary_key"` 16 | UserName string `json:"user_name" gorm:"column:user_name;type:varchar(32) not null;unique_index:uk_uname"` 17 | Password string `json:"-" gorm:"column:password;type:varchar(128) not null"` 18 | Email string `json:"email" gorm:"column:email;type:varchar(32) not null default '';index:idx_email"` 19 | Roles string `json:"roles" gorm:"column:roles;type:varchar(128) not null default ''"` 20 | Introduction string `json:"introduction" gorm:"column:introduction;type:varchar(128) not null default ''"` 21 | Avatar string `json:"avatar" gorm:"column:avatar;type:varchar(256) not null default ''"` 22 | CreatedAt time.Time `json:"created_at" gorm:"column:created_at;type:datetime;not null;default:CURRENT_TIMESTAMP;index:idx_created_at"` 23 | UpdatedAt time.Time `json:"updated_at" gorm:"column:updated_at;type:datetime;not null;default:CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP;index:idx_updated_at"` 24 | } 25 | 26 | func (o *User) TableName() string { 27 | return "gospider_user" 28 | } 29 | 30 | func init() { 31 | core.Register(&User{}) 32 | } 33 | 34 | func IsValidUser(db *gorm.DB, username, password string) (bool, *User, error) { 35 | user := &User{} 36 | query := NewUserQuerySet(db) 37 | if err := query.UserNameEq(username).One(user); err != nil { 38 | if err == gorm.ErrRecordNotFound { 39 | return false, nil, nil 40 | } 41 | log.Errorf("fetch user by name err [%+v]", err) 42 | return false, nil, err 43 | } 44 | 45 | if err := bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(password)); err != nil { 46 | return false, nil, nil 47 | } 48 | 49 | return true, user, nil 50 | } 51 | 52 | func GenUserHashPassword(password string) (string, error) { 53 | hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) 54 | if err != nil { 55 | return "", err 56 | } 57 | return string(hash), nil 58 | } 59 | 60 | func InitAdminUserIfNeeded(db *gorm.DB) error { 61 | user := &User{} 62 | query := NewUserQuerySet(db) 63 | err := query.UserNameEq("admin").One(user) 64 | if err == nil { 65 | return nil 66 | } 67 | if err == gorm.ErrRecordNotFound { 68 | user.UserName = "admin" 69 | pw, err := GenUserHashPassword("admin") 70 | if err != nil { 71 | return err 72 | } 73 | user.Password = pw 74 | user.Avatar = "/admin/gopher.png" 75 | user.Introduction = "admin user" 76 | user.Roles = "admin" 77 | 78 | return user.Create(db) 79 | } 80 | 81 | return err 82 | } 83 | -------------------------------------------------------------------------------- /common/db.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/go-sql-driver/mysql" 9 | "github.com/jinzhu/gorm" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | // MySQLConf is the mysql conf 14 | type MySQLConf struct { 15 | Host string 16 | Port int 17 | User string 18 | Password string 19 | DBName string 20 | MaxIdleConns int 21 | MaxOpenConns int 22 | MaxLifetime time.Duration 23 | } 24 | 25 | // NewGormDB return a new gorm db instance 26 | func NewGormDB(conf MySQLConf) (*gorm.DB, error) { 27 | dsn := getDSNWithDB(conf) 28 | db, err := gorm.Open("mysql", dsn) 29 | if err != nil { 30 | code, ok := GetSQLErrCode(err) 31 | if !ok { 32 | return nil, errors.WithStack(err) 33 | } 34 | if code == 1049 { // Database not exists 35 | if err := createDatabase(conf); err != nil { 36 | return nil, err 37 | } 38 | } 39 | db, err = gorm.Open("mysql", dsn) 40 | if err != nil { 41 | return nil, errors.WithStack(err) 42 | } 43 | } 44 | 45 | if conf.MaxIdleConns == 0 { 46 | db.DB().SetMaxIdleConns(3) 47 | } 48 | if conf.MaxOpenConns == 0 { 49 | db.DB().SetMaxOpenConns(5) 50 | } 51 | if conf.MaxLifetime == 0 { 52 | db.DB().SetConnMaxLifetime(time.Hour) 53 | } 54 | 55 | return db, nil 56 | } 57 | 58 | // NewDB returns a new sql.DB instance 59 | func NewDB(conf MySQLConf) (*sql.DB, error) { 60 | dsn := getDSNWithDB(conf) 61 | db, err := sql.Open("mysql", dsn) 62 | if err != nil { 63 | return nil, errors.WithStack(err) 64 | } 65 | 66 | db.SetConnMaxLifetime(time.Hour) 67 | db.SetMaxIdleConns(2) 68 | db.SetMaxOpenConns(10) 69 | 70 | return db, nil 71 | } 72 | 73 | func getbaseDSN(conf MySQLConf) string { 74 | dsn := fmt.Sprintf("%s:%s@(%s:%d)/?charset=utf8mb4&parseTime=True&loc=Local", 75 | conf.User, conf.Password, conf.Host, conf.Port) 76 | 77 | return dsn 78 | } 79 | 80 | func getDSNWithDB(conf MySQLConf) string { 81 | dsn := fmt.Sprintf("%s:%s@(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local", 82 | conf.User, conf.Password, conf.Host, conf.Port, conf.DBName) 83 | 84 | return dsn 85 | } 86 | 87 | // GetSQLErrCode returns error code if err is a mysql error 88 | func GetSQLErrCode(err error) (int, bool) { 89 | mysqlErr, ok := errors.Cause(err).(*mysql.MySQLError) 90 | if !ok { 91 | return -1, false 92 | } 93 | 94 | return int(mysqlErr.Number), true 95 | } 96 | 97 | func createDatabase(conf MySQLConf) error { 98 | dsn := getbaseDSN(conf) 99 | db, err := sql.Open("mysql", dsn) 100 | if err != nil { 101 | return errors.WithStack(err) 102 | } 103 | defer db.Close() 104 | 105 | _, err = db.Exec("CREATE DATABASE IF NOT EXISTS " + conf.DBName) 106 | if err != nil { 107 | return errors.WithStack(err) 108 | } 109 | 110 | return nil 111 | } 112 | -------------------------------------------------------------------------------- /web/static/src/directive/sticky.js: -------------------------------------------------------------------------------- 1 | const vueSticky = {} 2 | let listenAction 3 | vueSticky.install = Vue => { 4 | Vue.directive('sticky', { 5 | inserted(el, binding) { 6 | const params = binding.value || {} 7 | const stickyTop = params.stickyTop || 0 8 | const zIndex = params.zIndex || 1000 9 | const elStyle = el.style 10 | 11 | elStyle.position = '-webkit-sticky' 12 | elStyle.position = 'sticky' 13 | // if the browser support css sticky（Currently Safari, Firefox and Chrome Canary） 14 | // if (~elStyle.position.indexOf('sticky')) { 15 | // elStyle.top = `${stickyTop}px`; 16 | // elStyle.zIndex = zIndex; 17 | // return 18 | // } 19 | const elHeight = el.getBoundingClientRect().height 20 | const elWidth = el.getBoundingClientRect().width 21 | elStyle.cssText = `top: ${stickyTop}px; z-index: ${zIndex}` 22 | 23 | const parentElm = el.parentNode || document.documentElement 24 | const placeholder = document.createElement('div') 25 | placeholder.style.display = 'none' 26 | placeholder.style.width = `${elWidth}px` 27 | placeholder.style.height = `${elHeight}px` 28 | parentElm.insertBefore(placeholder, el) 29 | 30 | let active = false 31 | 32 | const getScroll = (target, top) => { 33 | const prop = top ? 'pageYOffset' : 'pageXOffset' 34 | const method = top ? 'scrollTop' : 'scrollLeft' 35 | let ret = target[prop] 36 | if (typeof ret !== 'number') { 37 | ret = window.document.documentElement[method] 38 | } 39 | return ret 40 | } 41 | 42 | const sticky = () => { 43 | if (active) { 44 | return 45 | } 46 | if (!elStyle.height) { 47 | elStyle.height = `${el.offsetHeight}px` 48 | } 49 | 50 | elStyle.position = 'fixed' 51 | elStyle.width = `${elWidth}px` 52 | placeholder.style.display = 'inline-block' 53 | active = true 54 | } 55 | 56 | const reset = () => { 57 | if (!active) { 58 | return 59 | } 60 | 61 | elStyle.position = '' 62 | placeholder.style.display = 'none' 63 | active = false 64 | } 65 | 66 | const check = () => { 67 | const scrollTop = getScroll(window, true) 68 | const offsetTop = el.getBoundingClientRect().top 69 | if (offsetTop < stickyTop) { 70 | sticky() 71 | } else { 72 | if (scrollTop < elHeight + stickyTop) { 73 | reset() 74 | } 75 | } 76 | } 77 | listenAction = () => { 78 | check() 79 | } 80 | 81 | window.addEventListener('scroll', listenAction) 82 | }, 83 | 84 | unbind() { 85 | window.removeEventListener('scroll', listenAction) 86 | } 87 | }) 88 | } 89 | 90 | export default vueSticky 91 | 92 | -------------------------------------------------------------------------------- /web/service/cron.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "github.com/nange/gospider/common" 8 | "github.com/nange/gospider/spider" 9 | "github.com/nange/gospider/web/core" 10 | "github.com/nange/gospider/web/model" 11 | "github.com/pkg/errors" 12 | "github.com/robfig/cron" 13 | log "github.com/sirupsen/logrus" 14 | ) 15 | 16 | var ( 17 | ErrCronTaskDuplicated = errors.New("cron task is Duplicated") 18 | ) 19 | 20 | var cronTaskMap = &sync.Map{} 21 | 22 | type CronTask struct { 23 | taskID uint64 24 | cronSpec string 25 | cr *cron.Cron 26 | retCh chan<- common.MTS 27 | } 28 | 29 | func NewCronTask(taskID uint64, cronSpec string, retCh chan<- common.MTS) (*CronTask, error) { 30 | ct := &CronTask{ 31 | taskID: taskID, 32 | cronSpec: cronSpec, 33 | cr: cron.New(), 34 | retCh: retCh, 35 | } 36 | 37 | if err := AddCronTask(ct); err != nil { 38 | return nil, errors.Wrap(err, "add cron task failed") 39 | } 40 | 41 | return ct, nil 42 | } 43 | 44 | func (ct *CronTask) Run() { 45 | task := &model.Task{} 46 | err := model.NewTaskQuerySet(core.GetGormDB()).IDEq(ct.taskID).One(task) 47 | if err != nil { 48 | log.Errorf("run cron task failed, query task err:%+v", errors.WithStack(err)) 49 | return 50 | } 51 | if task.Status != common.TaskStatusCompleted && task.Status != common.TaskStatusUnexceptedExited { 52 | log.Warnf("run cron task failed, status:%+v", errors.New(task.Status.String())) 53 | return 54 | } 55 | 56 | spiderTask, err := GetSpiderTaskByModel(task) 57 | if err != nil { 58 | log.Errorf("run cron task failed, err:%+v", errors.WithStack(err)) 59 | return 60 | } 61 | s := spider.New(spiderTask, ct.retCh) 62 | if err := s.Run(); err != nil { 63 | log.Errorf("run cron task failed, err:%+v", err) 64 | ct.retCh <- common.MTS{ID: task.ID, Status: common.TaskStatusUnexceptedExited} 65 | return 66 | } 67 | 68 | ct.retCh <- common.MTS{ID: task.ID, Status: common.TaskStatusRunning} 69 | } 70 | 71 | func (ct *CronTask) Start() error { 72 | if err := ct.cr.AddJob(ct.cronSpec, ct); err != nil { 73 | return errors.Wrapf(err, "cron add job failed, taskID:%d", ct.taskID) 74 | } 75 | ct.cr.Start() 76 | return nil 77 | } 78 | 79 | func (ct *CronTask) Stop() error { 80 | if ct.cr == nil { 81 | return errors.New("CronTask do not started") 82 | } 83 | cronTaskMap.Delete(ct.taskID) 84 | 85 | ct.cr.Stop() 86 | return nil 87 | } 88 | 89 | func GetCronTask(taskID uint64) *CronTask { 90 | ct, ok := cronTaskMap.Load(taskID) 91 | if !ok { 92 | return nil 93 | } 94 | return ct.(*CronTask) 95 | } 96 | 97 | func AddCronTask(ct *CronTask) error { 98 | if _, loaded := cronTaskMap.LoadOrStore(ct.taskID, ct); loaded { 99 | return errors.Wrap(ErrCronTaskDuplicated, fmt.Sprintf("add cron task failed, taskID:%d", ct.taskID)) 100 | } 101 | return nil 102 | } 103 | -------------------------------------------------------------------------------- /web/static/src/views/layout/components/Sidebar/SidebarItem.vue: -------------------------------------------------------------------------------- 1 | 31 | 32 | 81 | 82 | -------------------------------------------------------------------------------- /web/router/task/start_task.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "net/http" 5 | "strconv" 6 | 7 | "github.com/gin-gonic/gin" 8 | "github.com/nange/gospider/common" 9 | "github.com/nange/gospider/spider" 10 | "github.com/nange/gospider/web/core" 11 | "github.com/nange/gospider/web/model" 12 | "github.com/nange/gospider/web/service" 13 | "github.com/pkg/errors" 14 | log "github.com/sirupsen/logrus" 15 | ) 16 | 17 | // 根据任务id启动非定时任务 18 | func StartTask(c *gin.Context) { 19 | taskIDStr := c.Param("id") 20 | if taskIDStr == "" { 21 | log.Warnf("StartTaskReq taskID is empty") 22 | c.AbortWithStatus(http.StatusBadRequest) 23 | return 24 | } 25 | taskID, err := strconv.ParseUint(taskIDStr, 10, 64) 26 | if err != nil { 27 | log.Warnf("StartTaskReq taskID format is invalid, taskID: %v", taskIDStr) 28 | c.AbortWithStatus(http.StatusBadRequest) 29 | return 30 | } 31 | log.Infof("StartTaskReq:%+v", taskID) 32 | 33 | if taskLock.IsRunning(taskID) { 34 | c.String(http.StatusConflict, "other operation is running") 35 | return 36 | } 37 | defer taskLock.Complete(taskID) 38 | 39 | // query task info from db 40 | task := &model.Task{} 41 | err = model.NewTaskQuerySet(core.GetGormDB()).IDEq(taskID).One(task) 42 | if err != nil { 43 | log.Errorf("StartTaskReq query model task fail, taskID: %v , err: %+v", taskIDStr, err) 44 | c.AbortWithStatus(http.StatusInternalServerError) 45 | return 46 | } 47 | // not allow crontab task 48 | if task.CronSpec != "" { 49 | log.Warnf("StartTaskReq taskID is crontab task, taskID: %v", taskIDStr) 50 | c.AbortWithStatus(http.StatusBadRequest) 51 | return 52 | } 53 | 54 | // check task status 55 | if !taskCanBeStart(task) { 56 | log.Warnf("StartTaskReq taskID status is non-conformance , taskID: %v", taskIDStr) 57 | c.AbortWithStatus(http.StatusBadRequest) 58 | return 59 | } 60 | 61 | // create Task Model 62 | spiderTask, err := service.GetSpiderTaskByModel(task) 63 | if err != nil { 64 | log.Errorf("StartTaskReq get model task fail, taskID: %v , err: %+v", taskIDStr, err) 65 | c.AbortWithStatus(http.StatusInternalServerError) 66 | return 67 | } 68 | // run Task Model 69 | s := spider.New(spiderTask, service.GetMTSChan()) 70 | if err := s.Run(); err != nil { 71 | log.Errorf("StartTaskReq run task fail, taskID: %v , err: %+v", taskIDStr, err) 72 | c.AbortWithStatus(http.StatusInternalServerError) 73 | return 74 | } 75 | // update task status 76 | err = model.NewTaskQuerySet(core.GetGormDB()).IDEq(taskID).GetUpdater().SetStatus(common.TaskStatusRunning).Update() 77 | if err != nil { 78 | spider.CancelTask(taskID) 79 | log.Errorf("StartTaskReq update task status err:%+v", errors.WithStack(err)) 80 | c.String(http.StatusInternalServerError, "") 81 | return 82 | } 83 | 84 | c.String(http.StatusOK, "success") 85 | } 86 | 87 | func taskCanBeStart(task *model.Task) bool { 88 | return task.Status == common.TaskStatusStopped || 89 | task.Status == common.TaskStatusUnexceptedExited || 90 | task.Status == common.TaskStatusCompleted || 91 | task.Status == common.TaskStatusRunningTimeout 92 | } 93 | -------------------------------------------------------------------------------- /web/static/src/styles/sidebar.scss: -------------------------------------------------------------------------------- 1 | #app { 2 | // 主体区域 3 | .main-container { 4 | min-height: 100%; 5 | transition: margin-left .28s; 6 | margin-left: 200px; 7 | position: relative; 8 | } 9 | // 侧边栏 10 | .sidebar-container { 11 | transition: width 0.28s; 12 | width: 200px !important; 13 | height: 100%; 14 | position: fixed; 15 | font-size: 0px; 16 | top: 0; 17 | bottom: 0; 18 | left: 0; 19 | z-index: 1001; 20 | overflow: hidden; 21 | //reset element-ui css 22 | .horizontal-collapse-transition { 23 | transition: 0s width ease-in-out, 0s padding-left ease-in-out, 0s padding-right ease-in-out; 24 | } 25 | .scrollbar-wrapper { 26 | overflow-x: hidden!important; 27 | .el-scrollbar__view { 28 | height: 100%; 29 | } 30 | } 31 | .is-horizontal { 32 | display: none; 33 | } 34 | a { 35 | display: inline-block; 36 | width: 100%; 37 | overflow: hidden; 38 | } 39 | .svg-icon { 40 | margin-right: 16px; 41 | } 42 | .el-menu { 43 | border: none; 44 | height: 100%; 45 | width: 100% !important; 46 | } 47 | } 48 | .hideSidebar { 49 | .sidebar-container { 50 | width: 36px !important; 51 | } 52 | .main-container { 53 | margin-left: 36px; 54 | } 55 | .submenu-title-noDropdown { 56 | padding-left: 10px !important; 57 | position: relative; 58 | .el-tooltip { 59 | padding: 0 10px !important; 60 | } 61 | } 62 | .el-submenu { 63 | overflow: hidden; 64 | &>.el-submenu__title { 65 | padding-left: 10px !important; 66 | .el-submenu__icon-arrow { 67 | display: none; 68 | } 69 | } 70 | } 71 | .el-menu--collapse { 72 | .el-submenu { 73 | &>.el-submenu__title { 74 | &>span { 75 | height: 0; 76 | width: 0; 77 | overflow: hidden; 78 | visibility: hidden; 79 | display: inline-block; 80 | } 81 | } 82 | } 83 | } 84 | } 85 | .sidebar-container .nest-menu .el-submenu>.el-submenu__title, 86 | .sidebar-container .el-submenu .el-menu-item { 87 | min-width: 180px !important; 88 | background-color: $subMenuBg !important; 89 | &:hover { 90 | background-color: $menuHover !important; 91 | } 92 | } 93 | .el-menu--collapse .el-menu .el-submenu { 94 | min-width: 180px !important; 95 | } 96 | 97 | //适配移动端 98 | .mobile { 99 | .main-container { 100 | margin-left: 0px; 101 | } 102 | .sidebar-container { 103 | transition: transform .28s; 104 | width: 180px !important; 105 | } 106 | &.hideSidebar { 107 | .sidebar-container { 108 | transition-duration: 0.3s; 109 | transform: translate3d(-180px, 0, 0); 110 | } 111 | } 112 | } 113 | .withoutAnimation { 114 | .main-container, 115 | .sidebar-container { 116 | transition: none; 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /web/static/build/utils.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const path = require('path') 3 | const config = require('../config') 4 | const MiniCssExtractPlugin = require('mini-css-extract-plugin') 5 | const packageConfig = require('../package.json') 6 | 7 | exports.assetsPath = function(_path) { 8 | const assetsSubDirectory = 9 | process.env.NODE_ENV === 'production' 10 | ? config.build.assetsSubDirectory 11 | : config.dev.assetsSubDirectory 12 | 13 | return path.posix.join(assetsSubDirectory, _path) 14 | } 15 | 16 | exports.cssLoaders = function(options) { 17 | options = options || {} 18 | 19 | const cssLoader = { 20 | loader: 'css-loader', 21 | options: { 22 | sourceMap: options.sourceMap 23 | } 24 | } 25 | 26 | const postcssLoader = { 27 | loader: 'postcss-loader', 28 | options: { 29 | sourceMap: options.sourceMap 30 | } 31 | } 32 | 33 | // generate loader string to be used with extract text plugin 34 | function generateLoaders(loader, loaderOptions) { 35 | const loaders = [] 36 | 37 | // Extract CSS when that option is specified 38 | // (which is the case during production build) 39 | if (options.extract) { 40 | loaders.push(MiniCssExtractPlugin.loader) 41 | } else { 42 | loaders.push('vue-style-loader') 43 | } 44 | 45 | loaders.push(cssLoader) 46 | 47 | if (options.usePostCSS) { 48 | loaders.push(postcssLoader) 49 | } 50 | 51 | if (loader) { 52 | loaders.push({ 53 | loader: loader + '-loader', 54 | options: Object.assign({}, loaderOptions, { 55 | sourceMap: options.sourceMap 56 | }) 57 | }) 58 | } 59 | 60 | return loaders 61 | } 62 | // https://vue-loader.vuejs.org/en/configurations/extract-css.html 63 | return { 64 | css: generateLoaders(), 65 | postcss: generateLoaders(), 66 | less: generateLoaders('less'), 67 | sass: generateLoaders('sass', { 68 | indentedSyntax: true 69 | }), 70 | scss: generateLoaders('sass'), 71 | stylus: generateLoaders('stylus'), 72 | styl: generateLoaders('stylus') 73 | } 74 | } 75 | 76 | // Generate loaders for standalone style files (outside of .vue) 77 | exports.styleLoaders = function(options) { 78 | const output = [] 79 | const loaders = exports.cssLoaders(options) 80 | 81 | for (const extension in loaders) { 82 | const loader = loaders[extension] 83 | output.push({ 84 | test: new RegExp('\\.' + extension + '$'), 85 | use: loader 86 | }) 87 | } 88 | 89 | return output 90 | } 91 | 92 | exports.createNotifierCallback = () => { 93 | const notifier = require('node-notifier') 94 | 95 | return (severity, errors) => { 96 | if (severity !== 'error') return 97 | 98 | const error = errors[0] 99 | const filename = error.file && error.file.split('!').pop() 100 | 101 | notifier.notify({ 102 | title: packageConfig.name, 103 | message: severity + ': ' + error.name, 104 | subtitle: filename || '', 105 | icon: path.join(__dirname, 'logo.png') 106 | }) 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /web/static/build/webpack.dev.conf.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const path = require('path') 3 | const utils = require('./utils') 4 | const webpack = require('webpack') 5 | const config = require('../config') 6 | const merge = require('webpack-merge') 7 | const baseWebpackConfig = require('./webpack.base.conf') 8 | const HtmlWebpackPlugin = require('html-webpack-plugin') 9 | const FriendlyErrorsPlugin = require('friendly-errors-webpack-plugin') 10 | const portfinder = require('portfinder') 11 | 12 | function resolve(dir) { 13 | return path.join(__dirname, '..', dir) 14 | } 15 | 16 | const HOST = process.env.HOST 17 | const PORT = process.env.PORT && Number(process.env.PORT) 18 | 19 | const devWebpackConfig = merge(baseWebpackConfig, { 20 | mode: 'development', 21 | module: { 22 | rules: utils.styleLoaders({ 23 | sourceMap: config.dev.cssSourceMap, 24 | usePostCSS: true 25 | }) 26 | }, 27 | // cheap-module-eval-source-map is faster for development 28 | devtool: config.dev.devtool, 29 | 30 | // these devServer options should be customized in /config/index.js 31 | devServer: { 32 | clientLogLevel: 'warning', 33 | historyApiFallback: true, 34 | hot: true, 35 | compress: true, 36 | host: HOST || config.dev.host, 37 | port: PORT || config.dev.port, 38 | open: config.dev.autoOpenBrowser, 39 | overlay: config.dev.errorOverlay 40 | ? { warnings: false, errors: true } 41 | : false, 42 | publicPath: config.dev.assetsPublicPath, 43 | proxy: config.dev.proxyTable, 44 | quiet: true, // necessary for FriendlyErrorsPlugin 45 | watchOptions: { 46 | poll: config.dev.poll 47 | } 48 | }, 49 | plugins: [ 50 | new webpack.DefinePlugin({ 51 | 'process.env': require('../config/dev.env') 52 | }), 53 | new webpack.HotModuleReplacementPlugin(), 54 | // https://github.com/ampedandwired/html-webpack-plugin 55 | new HtmlWebpackPlugin({ 56 | filename: 'index.html', 57 | template: 'index.html', 58 | inject: true, 59 | favicon: resolve('favicon.ico'), 60 | title: 'gospider', 61 | path: config.dev.assetsPublicPath + config.dev.assetsSubDirectory 62 | }) 63 | ] 64 | }) 65 | 66 | module.exports = new Promise((resolve, reject) => { 67 | portfinder.basePort = process.env.PORT || config.dev.port 68 | portfinder.getPort((err, port) => { 69 | if (err) { 70 | reject(err) 71 | } else { 72 | // publish the new Port, necessary for e2e tests 73 | process.env.PORT = port 74 | // add port to devServer config 75 | devWebpackConfig.devServer.port = port 76 | 77 | // Add FriendlyErrorsPlugin 78 | devWebpackConfig.plugins.push( 79 | new FriendlyErrorsPlugin({ 80 | compilationSuccessInfo: { 81 | messages: [ 82 | `Your application is running here: http://${ 83 | devWebpackConfig.devServer.host 84 | }:${port}` 85 | ] 86 | }, 87 | onErrors: config.dev.notifyOnErrors 88 | ? utils.createNotifierCallback() 89 | : undefined 90 | }) 91 | ) 92 | 93 | resolve(devWebpackConfig) 94 | } 95 | }) 96 | }) 97 | -------------------------------------------------------------------------------- /web/static/src/views/exportdb/add.vue: -------------------------------------------------------------------------------- 1 | 33 | 73 | -------------------------------------------------------------------------------- /web/static/build/webpack.base.conf.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const path = require('path') 3 | const utils = require('./utils') 4 | const config = require('../config') 5 | const { VueLoaderPlugin } = require('vue-loader') 6 | const vueLoaderConfig = require('./vue-loader.conf') 7 | 8 | function resolve(dir) { 9 | return path.join(__dirname, '..', dir) 10 | } 11 | 12 | const createLintingRule = () => ({ 13 | test: /\.(js|vue)$/, 14 | loader: 'eslint-loader', 15 | enforce: 'pre', 16 | include: [resolve('src'), resolve('test')], 17 | options: { 18 | formatter: require('eslint-friendly-formatter'), 19 | emitWarning: !config.dev.showEslintErrorsInOverlay 20 | } 21 | }) 22 | 23 | module.exports = { 24 | context: path.resolve(__dirname, '../'), 25 | entry: { 26 | app: './src/main.js' 27 | }, 28 | output: { 29 | path: config.build.assetsRoot, 30 | filename: '[name].js', 31 | publicPath: 32 | process.env.NODE_ENV === 'production' 33 | ? config.build.assetsPublicPath 34 | : config.dev.assetsPublicPath 35 | }, 36 | resolve: { 37 | extensions: ['.js', '.vue', '.json'], 38 | alias: { 39 | '@': resolve('src') 40 | } 41 | }, 42 | module: { 43 | rules: [ 44 | ...(config.dev.useEslint ? [createLintingRule()] : []), 45 | { 46 | test: /\.vue$/, 47 | loader: 'vue-loader', 48 | options: vueLoaderConfig 49 | }, 50 | { 51 | test: /\.js$/, 52 | loader: 'babel-loader?cacheDirectory', 53 | include: [ 54 | resolve('src'), 55 | resolve('test'), 56 | resolve('node_modules/webpack-dev-server/client') 57 | ] 58 | }, 59 | { 60 | test: /\.svg$/, 61 | loader: 'svg-sprite-loader', 62 | include: [resolve('src/icons')], 63 | options: { 64 | symbolId: 'icon-[name]' 65 | } 66 | }, 67 | { 68 | test: /\.(png|jpe?g|gif|svg)(\?.*)?$/, 69 | loader: 'url-loader', 70 | exclude: [resolve('src/icons')], 71 | options: { 72 | limit: 10000, 73 | name: utils.assetsPath('img/[name].[hash:7].[ext]') 74 | } 75 | }, 76 | { 77 | test: /\.(mp4|webm|ogg|mp3|wav|flac|aac)(\?.*)?$/, 78 | loader: 'url-loader', 79 | options: { 80 | limit: 10000, 81 | name: utils.assetsPath('media/[name].[hash:7].[ext]') 82 | } 83 | }, 84 | { 85 | test: /\.(woff2?|eot|ttf|otf)(\?.*)?$/, 86 | loader: 'url-loader', 87 | options: { 88 | limit: 10000, 89 | name: utils.assetsPath('fonts/[name].[hash:7].[ext]') 90 | } 91 | } 92 | ] 93 | }, 94 | plugins: [new VueLoaderPlugin()], 95 | node: { 96 | // prevent webpack from injecting useless setImmediate polyfill because Vue 97 | // source contains it (although only uses it if it's native). 98 | setImmediate: false, 99 | // prevent webpack from injecting mocks to Node native modules 100 | // that does not make sense for the client 101 | dgram: 'empty', 102 | fs: 'empty', 103 | net: 'empty', 104 | tls: 'empty', 105 | child_process: 'empty' 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /web/static/config/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | // Template version: 1.2.6 3 | // see http://vuejs-templates.github.io/webpack for documentation. 4 | 5 | const path = require('path') 6 | 7 | module.exports = { 8 | dev: { 9 | // Paths 10 | assetsSubDirectory: 'static', 11 | assetsPublicPath: '/admin/', 12 | proxyTable: {}, 13 | 14 | // Various Dev Server settings 15 | 16 | // can be overwritten by process.env.HOST 17 | // if you want dev by ip, please set host: '0.0.0.0' 18 | host: 'localhost', 19 | port: 3000, // can be overwritten by process.env.PORT, if port is in use, a free one will be determined 20 | autoOpenBrowser: true, 21 | errorOverlay: true, 22 | notifyOnErrors: false, 23 | poll: false, // https://webpack.js.org/configuration/dev-server/#devserver-watchoptions- 24 | 25 | // Use Eslint Loader? 26 | // If true, your code will be linted during bundling and 27 | // linting errors and warnings will be shown in the console. 28 | useEslint: true, 29 | // If true, eslint errors and warnings will also be shown in the error overlay 30 | // in the browser. 31 | showEslintErrorsInOverlay: false, 32 | 33 | /** 34 | * Source Maps 35 | */ 36 | 37 | // https://webpack.js.org/configuration/devtool/#development 38 | devtool: 'cheap-source-map', 39 | 40 | // CSS Sourcemaps off by default because relative paths are "buggy" 41 | // with this option, according to the CSS-Loader README 42 | // (https://github.com/webpack/css-loader#sourcemaps) 43 | // In our experience, they generally work as expected, 44 | // just be aware of this issue when enabling this option. 45 | cssSourceMap: false 46 | }, 47 | 48 | build: { 49 | // Template for index.html 50 | index: path.resolve(__dirname, '../dist/index.html'), 51 | 52 | // Paths 53 | assetsRoot: path.resolve(__dirname, '../dist'), 54 | assetsSubDirectory: 'static', 55 | 56 | /** 57 | * You can set by youself according to actual condition 58 | * You will need to set this if you plan to deploy your site under a sub path, 59 | * for example GitHub pages. If you plan to deploy your site to https://foo.github.io/bar/, 60 | * then assetsPublicPath should be set to "/bar/". 61 | * In most cases please use '/' !!! 62 | */ 63 | assetsPublicPath: '/admin/', // If you are deployed on the root path, please use '/' 64 | 65 | /** 66 | * Source Maps 67 | */ 68 | productionSourceMap: false, 69 | // https://webpack.js.org/configuration/devtool/#production 70 | devtool: 'source-map', 71 | 72 | // Gzip off by default as many popular static hosts such as 73 | // Surge or Netlify already gzip all static assets for you. 74 | // Before setting to `true`, make sure to: 75 | // npm install --save-dev compression-webpack-plugin 76 | productionGzip: false, 77 | productionGzipExtensions: ['js', 'css'], 78 | 79 | // Run the build command with an extra argument to 80 | // View the bundle analyzer report after build finishes: 81 | // `npm run build:prod --report` 82 | // Set to `true` or `false` to always turn it on or off 83 | bundleAnalyzerReport: process.env.npm_config_report || false, 84 | 85 | // `npm run build:prod --generate_report` 86 | generateAnalyzerReport: process.env.npm_config_generate_report || false 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /web/static/src/store/modules/user.js: -------------------------------------------------------------------------------- 1 | import { loginByUsername, getUserInfo } from '@/api/login' 2 | import { getToken, setToken, removeToken } from '@/utils/auth' 3 | 4 | const user = { 5 | state: { 6 | user: '', 7 | status: '', 8 | code: '', 9 | token: getToken(), 10 | name: '', 11 | avatar: '', 12 | introduction: '', 13 | roles: [], 14 | setting: { 15 | articlePlatform: [] 16 | } 17 | }, 18 | 19 | mutations: { 20 | SET_CODE: (state, code) => { 21 | state.code = code 22 | }, 23 | SET_TOKEN: (state, token) => { 24 | state.token = token 25 | }, 26 | SET_INTRODUCTION: (state, introduction) => { 27 | state.introduction = introduction 28 | }, 29 | SET_SETTING: (state, setting) => { 30 | state.setting = setting 31 | }, 32 | SET_STATUS: (state, status) => { 33 | state.status = status 34 | }, 35 | SET_NAME: (state, name) => { 36 | state.name = name 37 | }, 38 | SET_AVATAR: (state, avatar) => { 39 | state.avatar = avatar 40 | }, 41 | SET_ROLES: (state, roles) => { 42 | state.roles = roles 43 | } 44 | }, 45 | 46 | actions: { 47 | // 用户名登录 48 | LoginByUsername({ commit }, userInfo) { 49 | const username = userInfo.username.trim() 50 | return new Promise((resolve, reject) => { 51 | loginByUsername(username, userInfo.password).then(response => { 52 | const data = response.data 53 | commit('SET_TOKEN', data.token) 54 | setToken(response.data.token) 55 | resolve() 56 | }).catch(error => { 57 | reject(error) 58 | }) 59 | }) 60 | }, 61 | 62 | // 获取用户信息 63 | GetUserInfo({ commit, state }) { 64 | return new Promise((resolve, reject) => { 65 | getUserInfo(state.token).then(response => { 66 | if (!response.data) { // 由于mockjs 不支持自定义状态码只能这样hack 67 | reject('error') 68 | } 69 | const data = response.data 70 | 71 | if (data.roles && data.roles.length > 0) { // 验证返回的roles是否是一个非空数组 72 | commit('SET_ROLES', data.roles) 73 | } else { 74 | reject('getInfo: roles must be a non-null array !') 75 | } 76 | 77 | commit('SET_NAME', data.name) 78 | commit('SET_AVATAR', data.avatar) 79 | commit('SET_INTRODUCTION', data.introduction) 80 | resolve(response) 81 | }).catch(error => { 82 | reject(error) 83 | }) 84 | }) 85 | }, 86 | 87 | // 登出 88 | LogOut({ commit }) { 89 | return new Promise(resolve => { 90 | commit('SET_TOKEN', '') 91 | commit('SET_ROLES', []) 92 | removeToken() 93 | resolve() 94 | }) 95 | }, 96 | 97 | // 前端 登出 98 | FedLogOut({ commit }) { 99 | return new Promise(resolve => { 100 | commit('SET_TOKEN', '') 101 | removeToken() 102 | resolve() 103 | }) 104 | }, 105 | 106 | // 动态修改权限 107 | ChangeRoles({ commit }, role) { 108 | return new Promise(resolve => { 109 | commit('SET_TOKEN', role) 110 | setToken(role) 111 | getUserInfo(role).then(response => { 112 | const data = response.data 113 | commit('SET_ROLES', data.roles) 114 | commit('SET_NAME', data.name) 115 | commit('SET_AVATAR', data.avatar) 116 | commit('SET_INTRODUCTION', data.introduction) 117 | resolve() 118 | }) 119 | }) 120 | } 121 | } 122 | } 123 | 124 | export default user 125 | -------------------------------------------------------------------------------- /web/static/src/components/ErrorLog/index.vue: -------------------------------------------------------------------------------- 1 | 43 | 44 | 59 | 60 | 78 | -------------------------------------------------------------------------------- /web/static/src/styles/index.scss: -------------------------------------------------------------------------------- 1 | @import './variables.scss'; 2 | @import './mixin.scss'; 3 | @import './transition.scss'; 4 | @import './element-ui.scss'; 5 | @import './sidebar.scss'; 6 | @import './btn.scss'; 7 | 8 | body { 9 | height: 100%; 10 | -moz-osx-font-smoothing: grayscale; 11 | -webkit-font-smoothing: antialiased; 12 | text-rendering: optimizeLegibility; 13 | font-family: Helvetica Neue, Helvetica, PingFang SC, Hiragino Sans GB, Microsoft YaHei, Arial, sans-serif; 14 | } 15 | 16 | label { 17 | font-weight: 700; 18 | } 19 | 20 | html { 21 | height: 100%; 22 | box-sizing: border-box; 23 | } 24 | 25 | #app{ 26 | height: 100%; 27 | } 28 | 29 | *, 30 | *:before, 31 | *:after { 32 | box-sizing: inherit; 33 | } 34 | 35 | .no-padding { 36 | padding: 0px !important; 37 | } 38 | 39 | .padding-content { 40 | padding: 4px 0; 41 | } 42 | 43 | a:focus, 44 | a:active { 45 | outline: none; 46 | } 47 | 48 | a, 49 | a:focus, 50 | a:hover { 51 | cursor: pointer; 52 | color: inherit; 53 | text-decoration: none; 54 | } 55 | 56 | div:focus{ 57 | outline: none; 58 | } 59 | 60 | .fr { 61 | float: right; 62 | } 63 | 64 | .fl { 65 | float: left; 66 | } 67 | 68 | .pr-5 { 69 | padding-right: 5px; 70 | } 71 | 72 | .pl-5 { 73 | padding-left: 5px; 74 | } 75 | 76 | .block { 77 | display: block; 78 | } 79 | 80 | .pointer { 81 | cursor: pointer; 82 | } 83 | 84 | .inlineBlock { 85 | display: block; 86 | } 87 | 88 | .clearfix { 89 | &:after { 90 | visibility: hidden; 91 | display: block; 92 | font-size: 0; 93 | content: " "; 94 | clear: both; 95 | height: 0; 96 | } 97 | } 98 | 99 | code { 100 | background: #eef1f6; 101 | padding: 15px 16px; 102 | margin-bottom: 20px; 103 | display: block; 104 | line-height: 36px; 105 | font-size: 15px; 106 | font-family: "Source Sans Pro", "Helvetica Neue", Arial, sans-serif; 107 | a { 108 | color: #337ab7; 109 | cursor: pointer; 110 | &:hover { 111 | color: rgb(32, 160, 255); 112 | } 113 | } 114 | } 115 | 116 | .warn-content{ 117 | background: rgba(66,185,131,.1); 118 | border-radius: 2px; 119 | padding: 16px; 120 | padding: 1rem; 121 | line-height: 1.6rem; 122 | word-spacing: .05rem; 123 | a{ 124 | color: #42b983; 125 | font-weight: 600; 126 | } 127 | } 128 | 129 | //main-container全局样式 130 | .app-container { 131 | padding: 20px; 132 | } 133 | 134 | .components-container { 135 | margin: 30px 50px; 136 | position: relative; 137 | } 138 | 139 | .pagination-container { 140 | margin-top: 30px; 141 | } 142 | 143 | .text-center { 144 | text-align: center 145 | } 146 | 147 | .sub-navbar { 148 | height: 50px; 149 | line-height: 50px; 150 | position: relative; 151 | width: 100%; 152 | text-align: right; 153 | padding-right: 20px; 154 | transition: 600ms ease position; 155 | background: linear-gradient(90deg, rgba(32, 182, 249, 1) 0%, rgba(32, 182, 249, 1) 0%, rgba(33, 120, 241, 1) 100%, rgba(33, 120, 241, 1) 100%); 156 | .subtitle { 157 | font-size: 20px; 158 | color: #fff; 159 | } 160 | &.draft { 161 | background: #d0d0d0; 162 | } 163 | &.deleted { 164 | background: #d0d0d0; 165 | } 166 | } 167 | 168 | .link-type, 169 | .link-type:focus { 170 | color: #337ab7; 171 | cursor: pointer; 172 | &:hover { 173 | color: rgb(32, 160, 255); 174 | } 175 | } 176 | 177 | .filter-container { 178 | padding-bottom: 10px; 179 | .filter-item { 180 | display: inline-block; 181 | vertical-align: middle; 182 | margin-bottom: 10px; 183 | } 184 | } 185 | 186 | //refine vue-multiselect plugin 187 | .multiselect { 188 | line-height: 16px; 189 | } 190 | 191 | .multiselect--active { 192 | z-index: 1000 !important; 193 | } 194 | --------------------------------------------------------------------------------