├── doc ├── tech.png ├── preview.png └── lighthouse.png ├── web ├── .env ├── src │ ├── pages │ │ ├── index │ │ │ ├── assets │ │ │ │ ├── logo.webp │ │ │ │ └── video.jpg │ │ │ ├── App.vue │ │ │ ├── scss │ │ │ │ ├── core │ │ │ │ │ ├── _all.scss │ │ │ │ │ ├── content.scss │ │ │ │ │ ├── _mixin.scss │ │ │ │ │ ├── _layout.scss │ │ │ │ │ ├── _vars.scss │ │ │ │ │ └── _init.scss │ │ │ │ ├── main.scss │ │ │ │ ├── _mixins.scss │ │ │ │ ├── _page.scss │ │ │ │ ├── _cards.scss │ │ │ │ └── _theme_dark.scss │ │ │ ├── components │ │ │ │ ├── cards │ │ │ │ │ ├── MText.vue │ │ │ │ │ ├── MRichText.vue │ │ │ │ │ ├── MVideo.vue │ │ │ │ │ └── Opt.vue │ │ │ │ ├── Main.vue │ │ │ │ ├── Footer.vue │ │ │ │ ├── HoTab.vue │ │ │ │ └── Content.vue │ │ │ ├── registerSW.js │ │ │ ├── store │ │ │ │ └── main.js │ │ │ ├── main.js │ │ │ └── router │ │ │ │ └── router.js │ │ └── admin │ │ │ ├── App.vue │ │ │ ├── styles │ │ │ ├── _mixins.scss │ │ │ ├── main.scss │ │ │ ├── libs │ │ │ │ └── _all.scss │ │ │ ├── _animate.scss │ │ │ ├── _card.scss │ │ │ ├── _element.scss │ │ │ ├── _section.scss │ │ │ ├── _custom.scss │ │ │ └── _theme-default.scss │ │ │ ├── stores │ │ │ ├── style.js │ │ │ └── main.js │ │ │ ├── components │ │ │ ├── toast │ │ │ │ ├── Toast.vue │ │ │ │ └── index.js │ │ │ ├── FormControl.vue │ │ │ ├── MenuLink.vue │ │ │ ├── BoxMain.vue │ │ │ ├── FormField.vue │ │ │ ├── BasicIcon.vue │ │ │ ├── Tile.vue │ │ │ └── MenuItem.vue │ │ │ ├── main.js │ │ │ ├── config.js │ │ │ ├── views │ │ │ ├── Home.vue │ │ │ ├── Login.vue │ │ │ └── User.vue │ │ │ ├── layouts │ │ │ └── Admin.vue │ │ │ └── router │ │ │ └── index.js │ └── lib │ │ └── http.js ├── .env.development ├── .gitignore ├── README.md ├── admin.html ├── index.html ├── package.json └── vite.config.js ├── internal ├── application │ ├── dto │ │ ├── query.go │ │ ├── oauth.go │ │ ├── user.go │ │ ├── node.go │ │ ├── favor.go │ │ └── site.go │ ├── service │ │ ├── user_test.go │ │ ├── craw_test.go │ │ ├── oauth.go │ │ ├── favor.go │ │ ├── user.go │ │ ├── craw.go │ │ └── node.go │ └── store │ │ ├── base_repo.go │ │ ├── node_repo.go │ │ ├── favor_repo.go │ │ ├── site_repo.go │ │ └── user_repo.go ├── constant │ ├── resp.go │ └── common.go ├── domain │ ├── model │ │ ├── base.go │ │ ├── favor.go │ │ ├── user.go │ │ ├── node.go │ │ └── site.go │ └── repo │ │ ├── favor.go │ │ ├── node.go │ │ ├── user.go │ │ └── site.go ├── pb │ ├── commander.proto │ └── agent.proto ├── util │ └── token.go ├── api │ ├── middleware │ │ ├── cache.go │ │ ├── online.go │ │ └── auth.go │ ├── handler │ │ ├── base_test.go │ │ ├── user.go │ │ ├── base.go │ │ ├── user_test.go │ │ ├── index.go │ │ ├── stat.go │ │ ├── admin │ │ │ ├── node.go │ │ │ └── site.go │ │ ├── auth.go │ │ └── favor.go │ └── route.go ├── config │ ├── config_test.go │ └── config.go ├── infra │ ├── cache │ │ └── redis.go │ └── db │ │ ├── mysql_test.go │ │ └── mysql.go ├── commander │ ├── commander.go │ └── job.go ├── core │ ├── rpc │ │ ├── client.go │ │ └── pool.go │ └── site │ │ ├── github_test.go │ │ ├── hacker_test.go │ │ ├── zhihu.go │ │ ├── weibo.go │ │ ├── hacker.go │ │ ├── tieba.go │ │ ├── v2ex.go │ │ ├── guanggu.go │ │ ├── zaobao.go │ │ ├── chouti.go │ │ ├── github.go │ │ ├── reddit.go │ │ └── wbvideo.go ├── agent │ ├── agent_test.go │ └── agent.go ├── agent.go └── api.go ├── .gitignore ├── pkg ├── helper │ ├── hash.go │ ├── time.go │ └── ip.go ├── oauth │ ├── oauth.go │ └── github.go ├── flow │ └── flow.go ├── http │ └── client.go └── logger │ └── log.go ├── scripts ├── dockerfiles │ ├── front.Dockerfile │ ├── agent.Dockerfile │ ├── commander.Dockerfile │ └── api.Dockerfile ├── k8s │ ├── mu-agent.yaml │ ├── mu-api.yaml │ └── mu-commander.yaml └── configs │ └── nginx.conf ├── conf └── demo.yml ├── README.md ├── cmd ├── api │ └── main.go ├── agent │ └── main.go └── commander │ └── main.go ├── test ├── setup.go └── web.go ├── LICENSE ├── mage.go └── go.mod /doc/tech.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aaronzjc/mu/HEAD/doc/tech.png -------------------------------------------------------------------------------- /doc/preview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aaronzjc/mu/HEAD/doc/preview.png -------------------------------------------------------------------------------- /doc/lighthouse.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aaronzjc/mu/HEAD/doc/lighthouse.png -------------------------------------------------------------------------------- /web/.env: -------------------------------------------------------------------------------- 1 | VITE_APP_URL=https://mu.memosa.cn 2 | VITE_APP_VERSION=$VERSION 3 | 4 | VITE_TOKEN_KEY=mu_cache_access_token -------------------------------------------------------------------------------- /web/src/pages/index/assets/logo.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aaronzjc/mu/HEAD/web/src/pages/index/assets/logo.webp -------------------------------------------------------------------------------- /web/src/pages/index/assets/video.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aaronzjc/mu/HEAD/web/src/pages/index/assets/video.jpg -------------------------------------------------------------------------------- /web/src/pages/index/App.vue: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/.env.development: -------------------------------------------------------------------------------- 1 | NODE_ENV=development 2 | VITE_APP_URL=https://mu.memosa.cn 3 | VITE_APP_VERSION=1.0 4 | 5 | VITE_TOKEN_KEY=mu_cache_access_token -------------------------------------------------------------------------------- /internal/application/dto/query.go: -------------------------------------------------------------------------------- 1 | package dto 2 | 3 | type Query struct { 4 | Query string 5 | Args []interface{} 6 | Order string 7 | Limit int 8 | } 9 | -------------------------------------------------------------------------------- /web/src/pages/admin/App.vue: -------------------------------------------------------------------------------- 1 | 4 | 5 | 8 | -------------------------------------------------------------------------------- /web/src/pages/index/scss/core/_all.scss: -------------------------------------------------------------------------------- 1 | @import "vars"; 2 | @import "mixin"; 3 | @import "init"; 4 | @import "layout"; 5 | @import "elements"; 6 | @import "content"; -------------------------------------------------------------------------------- /internal/application/dto/oauth.go: -------------------------------------------------------------------------------- 1 | package dto 2 | 3 | type OAuthPlatform struct { 4 | Name string `json:"name"` 5 | Type string `json:"type"` 6 | Url string `json:"url"` 7 | } 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | coverage.out 3 | dagger/ 4 | public/ 5 | **/dev.yml 6 | **/prod.yml 7 | **/kubeconf.yaml 8 | 9 | # Editor directories and files 10 | .idea 11 | .vscode 12 | *.suo 13 | *.ntvs* 14 | *.njsproj 15 | *.sln 16 | *.sw? -------------------------------------------------------------------------------- /internal/constant/resp.go: -------------------------------------------------------------------------------- 1 | package constant 2 | 3 | const ( 4 | CodeSuccess = 10000 5 | CodeError = 10001 6 | CodeForbidden = 10002 7 | CodeAuthFailed = 10003 8 | 9 | // 错误消息定义 10 | ERR_MSG_USERLIST = "获取用户列表失败" 11 | ) 12 | -------------------------------------------------------------------------------- /pkg/helper/hash.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "crypto/md5" 5 | "fmt" 6 | ) 7 | 8 | func Md5(input string) string { 9 | has := md5.Sum([]byte(input)) 10 | md5Str := fmt.Sprintf("%x", has) 11 | 12 | return md5Str 13 | } 14 | -------------------------------------------------------------------------------- /scripts/dockerfiles/front.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nginx:stable-alpine 2 | COPY ./dagger/frontend /usr/share/nginx/html 3 | COPY ./scripts/nginx.conf /etc/nginx/conf.d/default.conf 4 | EXPOSE 80 5 | VOLUME /usr/share/nginx/html 6 | CMD ["nginx", "-g", "daemon off;"] -------------------------------------------------------------------------------- /web/src/pages/index/scss/main.scss: -------------------------------------------------------------------------------- 1 | /** 2 | * just to reduce dst css size 3 | */ 4 | 5 | @import "core/_all.scss"; 6 | 7 | @import "theme_dark"; 8 | 9 | @import "mixins"; 10 | @import "cards"; 11 | 12 | @import "page"; 13 | 14 | @import "nprogress/nprogress.css"; -------------------------------------------------------------------------------- /internal/domain/model/base.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import "time" 4 | 5 | const DB_MU = "mu" 6 | 7 | type BaseModel struct { 8 | ID uint `gorm:"column:id" json:"id"` 9 | CreatedAt time.Time `json:"created_at"` 10 | UpdatedAt time.Time `json:"updated_at"` 11 | } 12 | -------------------------------------------------------------------------------- /scripts/dockerfiles/agent.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | ENV APP_ENV prod 3 | RUN apk add --no-cache ca-certificates tzdata && cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime 4 | RUN mkdir -p /app/bin 5 | COPY ./dagger/backend/agent /app/bin/ 6 | WORKDIR /app 7 | EXPOSE 7990 8 | CMD ["./bin/agent"] -------------------------------------------------------------------------------- /pkg/oauth/oauth.go: -------------------------------------------------------------------------------- 1 | package oauth 2 | 3 | type OAuth interface { 4 | Type() string 5 | RedirectAuth() string 6 | RequestAccessToken(string) (string, error) 7 | RequestUser(string) (User, error) 8 | } 9 | 10 | type User struct { 11 | ID int64 12 | Username string 13 | Nickname string 14 | Avatar string 15 | } 16 | -------------------------------------------------------------------------------- /internal/pb/commander.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package pb; 3 | 4 | option go_package = "./internal/pb"; 5 | 6 | service Commander { 7 | rpc UpdateCron (Cron) returns (CronRes) {} 8 | } 9 | 10 | message Empty {} 11 | 12 | message Cron { 13 | string site = 1; 14 | } 15 | message CronRes { 16 | bool success = 1; 17 | } -------------------------------------------------------------------------------- /scripts/dockerfiles/commander.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | ENV APP_ENV prod 3 | RUN apk add --no-cache ca-certificates tzdata && cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime 4 | RUN mkdir -p /app/bin /app/conf 5 | COPY ./dagger/backend/commander /app/bin 6 | EXPOSE 7970 7 | VOLUME /app/conf 8 | WORKDIR /app 9 | CMD ["./bin/commander", "-c", "conf/prod.yml"] -------------------------------------------------------------------------------- /internal/util/token.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "crypto/md5" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | func GenerateToken(input string, salt string) string { 10 | data := []byte(fmt.Sprintf("%s%s%s", input, salt, time.Now().Format("2006_01_02_15_04_05"))) 11 | has := md5.Sum(data) 12 | md5Str := fmt.Sprintf("%x", has) 13 | 14 | return md5Str 15 | } 16 | -------------------------------------------------------------------------------- /web/src/pages/admin/styles/_mixins.scss: -------------------------------------------------------------------------------- 1 | @mixin transition($t) { 2 | transition: $t 100ms ease-in-out 50ms; 3 | } 4 | 5 | @mixin icon-with-update-mark($icon-base-width) { 6 | .icon { 7 | width: $icon-base-width; 8 | 9 | &.has-update-mark:after { 10 | right: ($icon-base-width * 0.5) - 0.85; 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /internal/constant/common.go: -------------------------------------------------------------------------------- 1 | package constant 2 | 3 | type SvcName string 4 | 5 | const ( 6 | // 认证通过后用户缓存Key 7 | LoginKey = "login_user" 8 | 9 | // Commander用于选举的状态key 10 | IdMachine = "id_machine" 11 | JobVisor = "job_visor" 12 | Election = "election" 13 | 14 | // svc定义 15 | SvcCommander SvcName = "commander" 16 | SvcOnline SvcName = "online" 17 | ) 18 | -------------------------------------------------------------------------------- /web/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | -------------------------------------------------------------------------------- /internal/api/middleware/cache.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/gin-gonic/gin" 7 | ) 8 | 9 | // 给静态资源添加一个客户端缓存时间 10 | func AddCacheControlHeader() gin.HandlerFunc { 11 | return func(c *gin.Context) { 12 | if strings.HasPrefix(c.Request.RequestURI, "/static/") { 13 | c.Header("Cache-Control", "max-age=31536000") 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /web/src/pages/admin/styles/main.scss: -------------------------------------------------------------------------------- 1 | /* Theme style (colors & sizes) */ 2 | @import 'theme-default'; 3 | 4 | /* Core Libs & Lib configs */ 5 | @import 'libs/all'; 6 | 7 | /* Mixins */ 8 | @import 'mixins'; 9 | 10 | /* Theme components */ 11 | @import 'element'; 12 | @import 'menu'; 13 | @import 'card'; 14 | @import 'form'; 15 | @import 'section'; 16 | @import 'custom'; 17 | @import 'animate'; -------------------------------------------------------------------------------- /scripts/dockerfiles/api.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | ENV APP_ENV prod 3 | RUN apk add --no-cache ca-certificates tzdata && cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime 4 | RUN mkdir -p /app/bin /app/conf /app/public 5 | COPY ./dagger/backend/api /app/bin 6 | COPY ./dagger/frontend /app/public 7 | EXPOSE 7980 8 | VOLUME /app/conf 9 | WORKDIR /app 10 | CMD ["./bin/api", "-c", "conf/prod.yml"] -------------------------------------------------------------------------------- /web/src/pages/index/scss/core/content.scss: -------------------------------------------------------------------------------- 1 | .has-text-grey { 2 | color: $grey-dark; 3 | } 4 | 5 | .has-text-centered { 6 | text-align: center; 7 | } 8 | 9 | body { 10 | color: $grey-dark; 11 | } 12 | 13 | h4 { 14 | font-size: $font-size-large; 15 | } 16 | 17 | a { 18 | color: $link; 19 | cursor: pointer; 20 | text-decoration: none; 21 | &:hover { 22 | color: $grey-dark; 23 | } 24 | } -------------------------------------------------------------------------------- /web/src/pages/admin/styles/libs/_all.scss: -------------------------------------------------------------------------------- 1 | @import 'node_modules/bulma-radio/bulma-radio'; 2 | @import 'node_modules/bulma-responsive-tables/bulma-responsive-tables'; 3 | @import 'node_modules/bulma-checkbox/bulma-checkbox'; 4 | @import 'node_modules/bulma-switch-control/bulma-switch-control'; 5 | @import 'node_modules/bulma-upload-control/bulma-upload-control'; 6 | 7 | /* Bulma */ 8 | @import 'node_modules/bulma/bulma'; 9 | -------------------------------------------------------------------------------- /web/src/pages/index/scss/core/_mixin.scss: -------------------------------------------------------------------------------- 1 | $tablet: 768px; 2 | $desktop: 1024px; 3 | 4 | @mixin is($device) { 5 | @media screen and (min-width: $device) { 6 | @content; 7 | } 8 | } 9 | 10 | @mixin until($device) { 11 | @media screen and (max-width: $device - 1) { 12 | @content 13 | } 14 | } 15 | 16 | @mixin mobile() { 17 | @media screen and (max-width: $tablet - 1) { 18 | @content 19 | } 20 | } -------------------------------------------------------------------------------- /internal/domain/repo/favor.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aaronzjc/mu/internal/application/dto" 7 | "github.com/aaronzjc/mu/internal/domain/model" 8 | ) 9 | 10 | type FavorRepo interface { 11 | Get(context.Context, *dto.Query) ([]model.Favor, error) 12 | Create(context.Context, model.Favor) error 13 | Del(context.Context, model.Favor) error 14 | Sites(context.Context, *dto.Query) []string 15 | } 16 | -------------------------------------------------------------------------------- /internal/domain/repo/node.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aaronzjc/mu/internal/application/dto" 7 | "github.com/aaronzjc/mu/internal/domain/model" 8 | ) 9 | 10 | type NodeRepo interface { 11 | Get(context.Context, *dto.Query) ([]model.Node, error) 12 | Create(context.Context, model.Node) error 13 | Update(context.Context, model.Node, map[string]interface{}) error 14 | Del(context.Context, model.Node) error 15 | } 16 | -------------------------------------------------------------------------------- /web/src/pages/index/components/cards/MText.vue: -------------------------------------------------------------------------------- 1 | 10 | 11 | -------------------------------------------------------------------------------- /internal/domain/model/favor.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type Favor struct { 8 | ID int `gorm:"id"` 9 | UserId int `gorm:"user_id"` 10 | Site string `gorm:"site"` 11 | Key string `gorm:"key"` 12 | OriginUrl string `gorm:"origin_url"` 13 | Title string `gorm:"title"` 14 | CreateAt time.Time `gorm:"create_at"` 15 | } 16 | 17 | func (f *Favor) TableName() string { 18 | return "favor" 19 | } 20 | -------------------------------------------------------------------------------- /web/src/pages/admin/stores/style.js: -------------------------------------------------------------------------------- 1 | import { defineStore } from 'pinia' 2 | 3 | export const useStyleStore = defineStore('style', { 4 | state: () => ({ 5 | isAsideMobileOpen: false, 6 | isNavMobileOpen: false 7 | }), 8 | actions: { 9 | toggleAside() { 10 | this.isAsideMobileOpen = !this.isAsideMobileOpen 11 | }, 12 | toggleNav() { 13 | this.isNavMobileOpen = !this.isNavMobileOpen 14 | } 15 | } 16 | }) 17 | -------------------------------------------------------------------------------- /internal/domain/repo/user.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aaronzjc/mu/internal/application/dto" 7 | "github.com/aaronzjc/mu/internal/domain/model" 8 | ) 9 | 10 | // User 用户相关行为 11 | type UserRepo interface { 12 | GetUsers(context.Context, *dto.Query) ([]model.User, error) 13 | GetUser(context.Context, *dto.Query) (model.User, error) 14 | CreateUser(context.Context, model.User) error 15 | Update(context.Context, model.User, map[string]interface{}) error 16 | } 17 | -------------------------------------------------------------------------------- /pkg/helper/time.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import "time" 4 | 5 | const ( 6 | LocalTimezone = "Asia/Shanghai" 7 | LayoutISO = "2006-01-02 15:04:05" 8 | ) 9 | 10 | var localTimezone *time.Location 11 | 12 | func init() { 13 | localTimezone, _ = time.LoadLocation(LocalTimezone) 14 | } 15 | 16 | func TimeToLocalStr(t time.Time) string { 17 | return t.In(localTimezone).Format(LayoutISO) 18 | } 19 | 20 | func CurrentTimeStr() string { 21 | return time.Now().In(localTimezone).Format(LayoutISO) 22 | } 23 | -------------------------------------------------------------------------------- /internal/application/service/user_test.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | // func TestUserGetAll(t *testing.T) { 4 | // assert := assert.New(t) 5 | 6 | // userRepo := mocks.NewUserRepo(t) 7 | // userRepo.EXPECT().GetAll(mock.Anything).Return([]model.User{{BaseModel: model.BaseModel{ID: 1}, Username: "aaron"}}, nil) 8 | 9 | // userService := NewUserService(userRepo) 10 | // users, _ := userService.GetUserList(context.Background()) 11 | // assert.NotEmpty(users) 12 | // userRepo.AssertExpectations(t) 13 | // } 14 | -------------------------------------------------------------------------------- /web/README.md: -------------------------------------------------------------------------------- 1 | # Vue 3 + Vite 2 | 3 | This template should help get you started developing with Vue 3 in Vite. The template uses Vue 3 ` 18 | 19 | 20 | -------------------------------------------------------------------------------- /web/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Mu - 快乐每一天 8 | 9 | 10 | 16 |
17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /web/src/pages/index/components/cards/MRichText.vue: -------------------------------------------------------------------------------- 1 | 15 | 16 | -------------------------------------------------------------------------------- /internal/api/handler/base_test.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aaronzjc/mu/internal/constant" 7 | "github.com/aaronzjc/mu/test" 8 | 9 | "github.com/gin-gonic/gin" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestResp(t *testing.T) { 14 | assert := assert.New(t) 15 | h := func(c *gin.Context) { 16 | Resp(c, constant.CodeSuccess, "ok", nil) 17 | } 18 | 19 | resp := test.NewRequest(t).Handler(h).Get("/").Exec() 20 | assert.Equal(200, resp.Code()) 21 | errno, errmsg, _, _ := resp.TryDecode() 22 | assert.Equal(errno, constant.CodeSuccess) 23 | assert.Equal(errmsg, "ok") 24 | } 25 | -------------------------------------------------------------------------------- /internal/config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func setupConfig() string { 11 | f, _ := os.CreateTemp("", "config") 12 | f.WriteString("name: api\nenv: dev\nhost: 127.0.0.1\nport: 8780") 13 | return f.Name() 14 | } 15 | 16 | func TestLoadConfig(t *testing.T) { 17 | assert := assert.New(t) 18 | 19 | conf, err := LoadConfig("invalid path") 20 | assert.NotNil(err) 21 | assert.Nil(conf) 22 | 23 | f := setupConfig() 24 | defer os.Remove(f) 25 | conf, err = LoadConfig(f) 26 | assert.Nil(err) 27 | assert.Equal(conf.Http.Port, 8780) 28 | } 29 | -------------------------------------------------------------------------------- /web/src/pages/index/scss/core/_vars.scss: -------------------------------------------------------------------------------- 1 | // define colors 2 | $primary: #0088ff; 3 | 4 | $dark-primary: #308fe2ad; 5 | $dark-white: #e8e6e394; 6 | 7 | $grey-dark: #4a4a4a; 8 | $grey-light: #f1f1f1; 9 | 10 | $white: #fff; 11 | $white-ter: #fefefe; 12 | 13 | $link: #3b82f6; 14 | $link-hover: $grey-dark; 15 | 16 | $border-color: #dbdbdb; 17 | 18 | $family-sans-serif: -apple-system,system-ui,Segoe UI,Roboto,Ubuntu,Cantarell,Noto Sans,sans-serif,BlinkMacSystemFont,Helvetica Neue,PingFang SC,Hiragino Sans GB,Microsoft YaHei,Arial; 19 | $family-monospace: monospace; 20 | 21 | $font-size-small: 0.75rem; 22 | $font-size-normal: 16px; 23 | $font-size-medium: 1.25rem; 24 | $font-size-large: 1.5rem; -------------------------------------------------------------------------------- /internal/application/dto/user.go: -------------------------------------------------------------------------------- 1 | package dto 2 | 3 | import ( 4 | "github.com/aaronzjc/mu/internal/domain/model" 5 | "github.com/aaronzjc/mu/pkg/helper" 6 | ) 7 | 8 | type User struct { 9 | ID int `json:"id"` 10 | Username string `json:"username"` 11 | Nickname string `json:"nickname"` 12 | Avatar string `json:"avatar"` 13 | AuthType string `json:"auth_type"` 14 | AuthTime string `json:"auth_time"` 15 | } 16 | 17 | func (u *User) FillByModel(user model.User) *User { 18 | u.ID = user.ID 19 | u.Username = user.Username 20 | u.Nickname = user.Nickname 21 | u.Avatar = user.Avatar 22 | u.AuthType = user.AuthType 23 | u.AuthTime = helper.TimeToLocalStr(user.AuthTime) 24 | return u 25 | } 26 | -------------------------------------------------------------------------------- /web/src/pages/index/store/main.js: -------------------------------------------------------------------------------- 1 | import { defineStore } from 'pinia' 2 | 3 | export const useMainStore = defineStore('main', { 4 | state: () => ({ 5 | userInfo: { 6 | username: '', 7 | avatar: '' 8 | } 9 | }), 10 | getters: { 11 | isLogin: (state) => { 12 | return state.userInfo.username != '' 13 | } 14 | }, 15 | actions: { 16 | setUser(payload) { 17 | if (payload.username) { 18 | this.userInfo.username = payload.username 19 | } 20 | if (payload.avatar) { 21 | this.userInfo.avatar = payload.avatar 22 | } 23 | } 24 | } 25 | }) -------------------------------------------------------------------------------- /internal/application/dto/node.go: -------------------------------------------------------------------------------- 1 | package dto 2 | 3 | import ( 4 | "github.com/aaronzjc/mu/internal/domain/model" 5 | "github.com/aaronzjc/mu/pkg/helper" 6 | ) 7 | 8 | type Node struct { 9 | ID int `json:"id"` 10 | Name string `json:"name"` 11 | Addr string `json:"addr"` 12 | Type int8 `json:"type"` 13 | Enable int8 `json:"enable"` 14 | Ping int8 `json:"ping"` 15 | CreateAt string `json:"create_at"` 16 | } 17 | 18 | func (n *Node) FillByModel(node model.Node) *Node { 19 | n.ID = node.ID 20 | n.Name = node.Name 21 | n.Addr = node.Addr 22 | n.Type = node.Type 23 | n.Enable = node.Enable 24 | n.Ping = node.Ping 25 | n.CreateAt = helper.TimeToLocalStr(node.CreateAt) 26 | return n 27 | } 28 | -------------------------------------------------------------------------------- /web/src/pages/admin/components/toast/Toast.vue: -------------------------------------------------------------------------------- 1 | 8 | 9 | 27 | -------------------------------------------------------------------------------- /internal/api/handler/user.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "github.com/aaronzjc/mu/internal/application/service" 5 | "github.com/aaronzjc/mu/internal/application/store" 6 | "github.com/aaronzjc/mu/internal/constant" 7 | "github.com/gin-gonic/gin" 8 | ) 9 | 10 | type User struct { 11 | svc service.UserService 12 | } 13 | 14 | func (u *User) List(ctx *gin.Context) { 15 | users, err := u.svc.GetUserList(ctx) 16 | if err != nil { 17 | Resp(ctx, constant.CodeError, err.Error(), nil) 18 | return 19 | } 20 | Resp(ctx, constant.CodeSuccess, "success", users) 21 | } 22 | 23 | func NewUser() *User { 24 | repo := store.NewUserRepo() 25 | svc := service.NewUserService(repo) 26 | return &User{ 27 | svc: svc, 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /internal/api/handler/base.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/gin-gonic/gin" 8 | ) 9 | 10 | type RespSt struct { 11 | Code int `json:"code"` 12 | Msg string `json:"msg"` 13 | Data interface{} `json:"data"` 14 | } 15 | 16 | func Resp(ctx *gin.Context, code int, msg string, data interface{}) { 17 | if data == nil { 18 | data = make(map[string]struct{}) 19 | } 20 | ctx.JSON(http.StatusOK, &RespSt{ 21 | Code: code, 22 | Msg: msg, 23 | Data: data, 24 | }) 25 | } 26 | 27 | func SetCookies(ctx *gin.Context, data map[string]string, domain string) { 28 | for k, v := range data { 29 | ctx.SetCookie(k, v, int(time.Hour*24*30), "", domain, false, false) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /internal/infra/cache/redis.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/aaronzjc/mu/internal/config" 9 | "github.com/go-redis/redis" 10 | ) 11 | 12 | var ( 13 | client *redis.Client 14 | once sync.Once 15 | ) 16 | 17 | func Setup(conf *config.RedisConfig) error { 18 | once.Do(func() { 19 | c := redis.NewClient(&redis.Options{ 20 | Addr: fmt.Sprintf("%s:%d", conf.Host, conf.Port), 21 | Password: conf.Password, 22 | DB: 0, 23 | }) 24 | if _, err := c.Ping().Result(); err != nil { 25 | return 26 | } 27 | client = c 28 | }) 29 | if client == nil { 30 | return errors.New("init redis err") 31 | } 32 | return nil 33 | } 34 | 35 | func Get() *redis.Client { 36 | return client 37 | } 38 | -------------------------------------------------------------------------------- /internal/infra/db/mysql_test.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aaronzjc/mu/internal/config" 7 | 8 | "github.com/stretchr/testify/require" 9 | "gorm.io/gorm" 10 | ) 11 | 12 | var ( 13 | conf = config.Config{ 14 | Database: map[string]config.DbConfig{ 15 | "demo": { 16 | Host: "127.0.0.1", 17 | Port: 3306, 18 | Username: "root", 19 | Password: "123456", 20 | Charset: "utf8", 21 | }, 22 | }, 23 | } 24 | ) 25 | 26 | func TestDb(t *testing.T) { 27 | require := require.New(t) 28 | 29 | err := Setup(&conf, &gorm.Config{}) 30 | require.Nil(err) 31 | 32 | demo, ok := Get("demo") 33 | require.True(ok) 34 | require.NotEmpty(demo) 35 | 36 | db, _ := demo.DB() 37 | require.Nil(db.Ping()) 38 | } 39 | -------------------------------------------------------------------------------- /web/src/pages/admin/main.js: -------------------------------------------------------------------------------- 1 | import { createApp } from "vue"; 2 | import App from "./App.vue"; 3 | import "./styles/main.scss"; 4 | 5 | const app = createApp(App); 6 | 7 | /** 路由 */ 8 | import router from "./router"; 9 | app.use(router); 10 | 11 | 12 | import client from "@/lib/http"; 13 | client.interceptors.response.use((resp) => { 14 | let res = resp.data; 15 | if (res.code === 10003) { 16 | router.push({ name: "login" }).catch(() => {}); 17 | return Promise.reject(resp); 18 | } 19 | 20 | return resp; 21 | }); 22 | 23 | /** 全局状态管理 */ 24 | import { createPinia } from "pinia"; 25 | const pinia = createPinia(); 26 | app.use(pinia); 27 | 28 | /** 自定义组件 */ 29 | import { Toast } from "@adm/components/toast"; 30 | app.use(Toast); 31 | 32 | app.mount("#app"); 33 | -------------------------------------------------------------------------------- /web/src/pages/admin/components/FormControl.vue: -------------------------------------------------------------------------------- 1 | 10 | 11 | 27 | -------------------------------------------------------------------------------- /conf/demo.yml: -------------------------------------------------------------------------------- 1 | name: mu # 项目名称 2 | env: dev # 环境类型 3 | salt: testing # 用户密码加密 4 | log: 5 | level: debug # 日志级别 6 | file: /var/log/mu/app.log # 日志文件 7 | http: 8 | tls: false # https 9 | url: 127.0.0.1:8980 # 服务地址 10 | port: 7980 # 监听端口，默认是7980 11 | commander: 12 | port: 7981 # commander监听端口，某人是7981 13 | redis: 14 | host: 127.0.0.1 15 | port: 7379 16 | password: 17 | database: 18 | mu: 19 | host: 127.0.0.1 20 | port: 3306 21 | username: root 22 | password: 123456 23 | charset: "UTF8" 24 | service: 25 | commander: 26 | url: 127.0.0.1:8981 27 | oauth: 28 | github: 29 | clientId: 1111 30 | clientSecret: 11111 31 | admins: 32 | - "11111" 33 | weibo: 34 | clientId: 11111 35 | clientSecret: 11111 36 | admins: 37 | - "1111" -------------------------------------------------------------------------------- /web/src/pages/index/scss/_mixins.scss: -------------------------------------------------------------------------------- 1 | @include is($desktop){ 2 | .container { 3 | max-width: 960px !important; 4 | } 5 | .search-form { 6 | width: 50%; 7 | } 8 | } 9 | 10 | @include until($desktop) { 11 | .content-box { 12 | padding: 0 0.75rem; 13 | } 14 | .switch { 15 | position: -webkit-sticky; 16 | position: sticky; 17 | z-index: 99; 18 | top: 0; 19 | left: 0; 20 | 21 | &.sticky { 22 | background: #fff; 23 | box-shadow: 0px 2px 6px 0px #dadada; 24 | } 25 | } 26 | html.dark { 27 | .switch.sticky { 28 | background: #181a1b; 29 | box-shadow: 0px 2px 6px 0px #0c0c0c;; 30 | } 31 | } 32 | .navbar-item.navbar-opt { 33 | display: none; 34 | } 35 | .mini-navbar-opt { 36 | display: block; 37 | } 38 | } -------------------------------------------------------------------------------- /web/src/pages/index/scss/core/_init.scss: -------------------------------------------------------------------------------- 1 | 2 | html { 3 | box-sizing: border-box; 4 | background-color: $white; 5 | font-size: $font-size-normal; 6 | -moz-osx-font-smoothing: grayscale; 7 | -webkit-font-smoothing: antialiased; 8 | text-rendering: optimizeLegibility; 9 | } 10 | 11 | body { 12 | overflow: auto; 13 | } 14 | 15 | * { 16 | padding: 0; 17 | margin: 0; 18 | -webkit-tap-highlight-color: transparent; 19 | 20 | &, &:before, &:after { 21 | box-sizing: inherit; 22 | } 23 | &:focus { 24 | outline: none; 25 | } 26 | } 27 | 28 | section { 29 | display: block; 30 | } 31 | 32 | ul { 33 | list-style: none; 34 | } 35 | 36 | body,button { 37 | font-size: 1rem; 38 | font-weight: 400; 39 | line-height: 1.5rem; 40 | font-family: $family-sans-serif; 41 | } -------------------------------------------------------------------------------- /web/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "web", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite", 8 | "build": "vite build", 9 | "preview": "vite preview" 10 | }, 11 | "dependencies": { 12 | "@mdi/js": "^7.1.96", 13 | "axios": "^1.3.4", 14 | "bulma": "^0.9.4", 15 | "bulma-checkbox": "^1.2.1", 16 | "bulma-radio": "^1.2.0", 17 | "bulma-responsive-tables": "^1.2.5", 18 | "bulma-switch-control": "^1.2.2", 19 | "bulma-upload-control": "^1.2.0", 20 | "nprogress": "^0.2.0", 21 | "pinia": "^2.0.33", 22 | "sass": "^1.59.3", 23 | "vue": "^3.2.47", 24 | "vue-router": "^4.1.6" 25 | }, 26 | "devDependencies": { 27 | "@vitejs/plugin-vue": "^4.1.0", 28 | "vite": "^4.2.0" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## 介绍 2 | 3 | `mu`是一个热榜帖子聚合网站。 4 | 5 | 这个项目的目的是实践自己掌握的新的技术。目前的技术栈 6 | 7 | + Golang, Gin 8 | + Vue 3.0, PWA 9 | + k3s 10 | + dagger 11 | 12 | ## 预览&架构 13 | 14 |
15 | preview 16 | tech 17 |
18 |
19 | preview 20 |
21 | 22 | ## 构建部署 23 | 24 | 因为windows原生不支持`make`，项目采用`mage`和`dagger`作为CI/CD工具集。 25 | 26 | 安装参考`magefile/mage`项目。 27 | 28 | ```shell 29 | # 查看可用构建 30 | $ mage 31 | # 构建前端页面 32 | $ mage build_frontend 33 | # 构建后端，会同时编译api,commander,agent 34 | $ mage build_backend 35 | # 打包镜像 36 | $ mage build_image 37 | # 部署到k8s，配置格式参考demo.yml 38 | $ mage deploy 39 | ``` 40 | 41 | ### 授权 42 | 43 | MIT -------------------------------------------------------------------------------- /internal/commander/commander.go: -------------------------------------------------------------------------------- 1 | package commander 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aaronzjc/mu/internal/constant" 7 | "github.com/aaronzjc/mu/internal/infra/cache" 8 | "github.com/aaronzjc/mu/internal/pb" 9 | "github.com/aaronzjc/mu/pkg/logger" 10 | ) 11 | 12 | type CommanderServer struct { 13 | pb.UnimplementedCommanderServer 14 | } 15 | 16 | var _ pb.CommanderServer = &CommanderServer{} 17 | 18 | func (commander *CommanderServer) UpdateCron(ctx context.Context, req *pb.Cron) (*pb.CronRes, error) { 19 | redis := cache.Get() 20 | redis.LPush(constant.JobVisor, req.Site) 21 | logger.Info("Rpc UpdateCron [site = %s] success !", req.Site) 22 | return &pb.CronRes{Success: true}, nil 23 | } 24 | 25 | func NewCommanderServer() *CommanderServer { 26 | return &CommanderServer{} 27 | } 28 | -------------------------------------------------------------------------------- /web/src/pages/index/main.js: -------------------------------------------------------------------------------- 1 | import { createApp } from "vue"; 2 | import App from "./App.vue"; 3 | 4 | const app = createApp(App); 5 | 6 | /** 样式 */ 7 | import "./scss/main.scss"; 8 | 9 | /** 路由 */ 10 | import router from "./router/router"; 11 | app.use(router); 12 | 13 | import client from "@/lib/http"; 14 | client.interceptors.response.use((resp) => { 15 | let res = resp.data; 16 | if (res.code === 10003) { 17 | localStorage.removeItem(import.meta.env.VITE_TOKEN_KEY); 18 | router.push({ name: "login" }).catch(() => {}); 19 | return Promise.reject(resp); 20 | } 21 | return resp; 22 | }); 23 | 24 | /** 状态 */ 25 | import { createPinia } from "pinia"; 26 | const pinia = createPinia(); 27 | app.use(pinia); 28 | 29 | app.mount("#app"); 30 | 31 | /* register sw */ 32 | import "./registerSW"; 33 | -------------------------------------------------------------------------------- /web/vite.config.js: -------------------------------------------------------------------------------- 1 | import { fileURLToPath, URL } from 'node:url' 2 | import { resolve } from 'path' 3 | 4 | import { defineConfig } from 'vite' 5 | import vue from '@vitejs/plugin-vue' 6 | 7 | // https://vitejs.dev/config/ 8 | export default defineConfig({ 9 | plugins: [vue()], 10 | resolve: { 11 | alias: { 12 | '@': fileURLToPath(new URL('./src', import.meta.url)), 13 | '@idx': fileURLToPath(new URL('./src/pages/index', import.meta.url)), 14 | '@adm': fileURLToPath(new URL('./src/pages/admin', import.meta.url)) 15 | } 16 | }, 17 | build: { 18 | rollupOptions: { 19 | input: { 20 | index: resolve(__dirname, 'index.html'), 21 | admin: resolve(__dirname, 'admin.html'), 22 | }, 23 | }, 24 | outDir: '../public' 25 | } 26 | }) 27 | -------------------------------------------------------------------------------- /web/src/pages/admin/styles/_animate.scss: -------------------------------------------------------------------------------- 1 | @keyframes fadeOut { 2 | from { 3 | opacity: 1; 4 | } 5 | 6 | to { 7 | opacity: 0; 8 | } 9 | } 10 | .fadeOut { 11 | animation-name: fadeOut; 12 | } 13 | 14 | @keyframes fadeInDown { 15 | from { 16 | opacity: 0.5; 17 | transform: translate3d(0, -100%, 0); 18 | } 19 | to { 20 | opacity: 1; 21 | transform: none; 22 | } 23 | } 24 | .fadeInDown { 25 | animation-name: fadeInDown; 26 | } 27 | 28 | // 页面切换 29 | .page-leave-active, 30 | .page-enter-active { 31 | transition: all 0.3s; 32 | } 33 | .page-enter-from { 34 | opacity: 0; 35 | transform: translateX(-30px); 36 | } 37 | .page-leave-to { 38 | opacity: 0; 39 | transform: translateX(30px); 40 | display: none; 41 | } 42 | -------------------------------------------------------------------------------- /internal/api/middleware/online.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | 7 | "github.com/aaronzjc/mu/internal/config" 8 | "github.com/aaronzjc/mu/internal/constant" 9 | "github.com/aaronzjc/mu/pkg/helper" 10 | "github.com/gin-gonic/gin" 11 | ) 12 | 13 | func SetOnline() gin.HandlerFunc { 14 | return func(ctx *gin.Context) { 15 | defer ctx.Next() 16 | clientIp := helper.ClientIp(ctx.Request) 17 | if clientIp == "" { 18 | return 19 | } 20 | svcUrl := config.Get().GetServiceUrl(constant.SvcOnline) 21 | if svcUrl == "" { 22 | return 23 | } 24 | url := fmt.Sprintf("%s/online/%s/%s", svcUrl, "mu", clientIp) 25 | req, _ := http.NewRequest("POST", url, nil) 26 | resp, err := (new(http.Client)).Do(req) 27 | if err == nil { 28 | resp.Body.Close() 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /cmd/api/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "github.com/aaronzjc/mu/internal" 8 | 9 | "github.com/urfave/cli" 10 | ) 11 | 12 | var ( 13 | appName = "mu-api" 14 | usage = "run mu-api server" 15 | desc = `mu-api is the api server for mu project, it provides index api & admin api services. ` 16 | version = "7.0" 17 | ) 18 | 19 | func main() { 20 | app := *cli.NewApp() 21 | app.Name = appName 22 | app.Usage = usage 23 | app.Description = desc 24 | app.Version = version 25 | app.Flags = []cli.Flag{ 26 | &cli.StringFlag{ 27 | Name: "config,c", 28 | Usage: "(config) Load configuration from `FILE`", 29 | }, 30 | } 31 | app.Before = internal.SetupApi 32 | app.Action = internal.RunApi 33 | 34 | if err := app.Run(os.Args); err != nil { 35 | log.Fatal(err) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /internal/core/rpc/client.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "github.com/aaronzjc/mu/internal/pb" 5 | "google.golang.org/grpc" 6 | "google.golang.org/grpc/credentials/insecure" 7 | ) 8 | 9 | func NewCommanderClient(addr string) (pb.CommanderClient, error) { 10 | opts := []grpc.DialOption{ 11 | grpc.WithTransportCredentials(insecure.NewCredentials()), 12 | } 13 | conn, err := grpc.Dial(addr, opts...) 14 | if err != nil { 15 | return nil, err 16 | } 17 | return pb.NewCommanderClient(conn), nil 18 | } 19 | 20 | func NewAgentClient(addr string) (pb.AgentClient, error) { 21 | opts := []grpc.DialOption{ 22 | grpc.WithTransportCredentials(insecure.NewCredentials()), 23 | } 24 | conn, err := grpc.Dial(addr, opts...) 25 | if err != nil { 26 | return nil, err 27 | } 28 | return pb.NewAgentClient(conn), nil 29 | } 30 | -------------------------------------------------------------------------------- /internal/agent/agent_test.go: -------------------------------------------------------------------------------- 1 | package agent 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/aaronzjc/mu/internal/core/site" 8 | "github.com/aaronzjc/mu/internal/pb" 9 | "github.com/aaronzjc/mu/test" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | var ( 14 | agent = NewAgentServer() 15 | ) 16 | 17 | func TestCraw(t *testing.T) { 18 | test.SetupProxy() 19 | defer test.ClearProxy() 20 | 21 | cases := []site.Site{ 22 | site.NewGithub().Site, 23 | site.NewHacker().Site, 24 | } 25 | for _, v := range cases { 26 | result, err := agent.Craw(context.Background(), &pb.Job{ 27 | Name: v.Key, 28 | }) 29 | assert.Nil(t, err) 30 | for _, vv := range v.Tabs { 31 | tabStr, ok := result.HotMap[vv.Tag] 32 | assert.True(t, ok) 33 | assert.True(t, tabStr != "[]") 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /scripts/k8s/mu-agent.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: mu-agent-zyra 5 | namespace: k3s-apps 6 | labels: 7 | app: mu-agent-zyra 8 | spec: 9 | containers: 10 | - name: mu-agent-zyra 11 | image: aaronzjc/mu-agent:latest 12 | imagePullPolicy: Always 13 | ports: 14 | - containerPort: 7990 15 | resources: 16 | limits: 17 | cpu: 50m 18 | memory: 50Mi 19 | 20 | --- 21 | apiVersion: v1 22 | kind: Pod 23 | metadata: 24 | name: mu-agent-nami 25 | namespace: k3s-apps 26 | labels: 27 | app: mu-agent-nami 28 | spec: 29 | containers: 30 | - name: mu-agent-nami 31 | image: aaronzjc/mu-agent:latest 32 | imagePullPolicy: Always 33 | ports: 34 | - containerPort: 7990 35 | resources: 36 | limits: 37 | cpu: 50m 38 | memory: 50Mi -------------------------------------------------------------------------------- /internal/application/dto/favor.go: -------------------------------------------------------------------------------- 1 | package dto 2 | 3 | import ( 4 | "github.com/aaronzjc/mu/internal/domain/model" 5 | "github.com/aaronzjc/mu/pkg/helper" 6 | ) 7 | 8 | type Favor struct { 9 | ID int `json:"id"` 10 | UserId int `json:"user_id"` 11 | Site string `json:"site"` 12 | Key string `json:"key"` 13 | OriginUrl string `json:"origin_url"` 14 | Title string `json:"title"` 15 | CreateAt string `json:"create_at"` 16 | } 17 | 18 | func (f *Favor) FillByModel(favor model.Favor) *Favor { 19 | f.ID = favor.ID 20 | f.UserId = favor.UserId 21 | f.Site = favor.Site 22 | f.OriginUrl = favor.OriginUrl 23 | f.Title = favor.Title 24 | f.CreateAt = helper.TimeToLocalStr(favor.CreateAt) 25 | return f 26 | } 27 | 28 | type FavorList struct { 29 | Tabs []*IndexSite `json:"tabs"` 30 | List []*Favor `json:"list"` 31 | } 32 | -------------------------------------------------------------------------------- /cmd/agent/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "github.com/aaronzjc/mu/internal" 8 | 9 | "github.com/urfave/cli" 10 | ) 11 | 12 | var ( 13 | appName = "mu-agent" 14 | usage = "run mu-agent server" 15 | desc = `mu-agent is the worker to craw pages, since some websites are blocked in china, distributed agents can be useful. ` 16 | version = "7.0" 17 | ) 18 | 19 | func main() { 20 | app := *cli.NewApp() 21 | app.Name = appName 22 | app.Usage = usage 23 | app.Description = desc 24 | app.Version = version 25 | app.Flags = []cli.Flag{ 26 | &cli.StringFlag{ 27 | Name: "config,c", 28 | Usage: "(config) Load configuration from `FILE`", 29 | }, 30 | } 31 | app.Before = internal.SetupAgent 32 | app.Action = internal.RunAgent 33 | 34 | if err := app.Run(os.Args); err != nil { 35 | log.Fatal(err) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /cmd/commander/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "github.com/aaronzjc/mu/internal" 8 | 9 | "github.com/urfave/cli" 10 | ) 11 | 12 | var ( 13 | appName = "mu-commander" 14 | usage = "run mu-commander server" 15 | desc = `mu-commander is the schedule server for dispatching craw jobs, monitor agent status, etc. ` 16 | version = "7.0" 17 | ) 18 | 19 | func main() { 20 | app := *cli.NewApp() 21 | app.Name = appName 22 | app.Usage = usage 23 | app.Description = desc 24 | app.Version = version 25 | app.Flags = []cli.Flag{ 26 | &cli.StringFlag{ 27 | Name: "config,c", 28 | Usage: "(config) Load configuration from `FILE`", 29 | }, 30 | } 31 | app.Before = internal.SetupCommander 32 | app.Action = internal.RunCommander 33 | 34 | if err := app.Run(os.Args); err != nil { 35 | log.Fatal(err) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /pkg/flow/flow.go: -------------------------------------------------------------------------------- 1 | package flow 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | type Step struct { 10 | name string 11 | fn func(context.Context) error 12 | } 13 | 14 | type Flow struct { 15 | ctx context.Context 16 | steps []Step 17 | } 18 | 19 | func (f *Flow) Step(name string, fn func(context.Context) error) *Flow { 20 | f.steps = append(f.steps, Step{name: name, fn: fn}) 21 | return f 22 | } 23 | 24 | func (f *Flow) Run() { 25 | start := time.Now() 26 | for _, v := range f.steps { 27 | if err := v.fn(f.ctx); err != nil { 28 | fmt.Printf("[x]run %s failed, err = %v\n", v.name, err) 29 | break 30 | } 31 | fmt.Printf("[o]run %s done\n", v.name) 32 | } 33 | fmt.Printf("[o]flow took %.2fs\n", time.Since(start).Seconds()) 34 | } 35 | 36 | func NewFlow(ctx context.Context) *Flow { 37 | return &Flow{ctx: ctx} 38 | } 39 | -------------------------------------------------------------------------------- /web/src/pages/admin/config.js: -------------------------------------------------------------------------------- 1 | import { 2 | mdiAccount, 3 | mdiDotsGrid, 4 | mdiFirefox, 5 | mdiLogin, 6 | mdiMonitor, 7 | mdiServer, 8 | mdiSquareEditOutline, 9 | mdiTable 10 | } from '@mdi/js' 11 | 12 | export const menus = [ 13 | { 14 | route: 'home', 15 | title: '后台总览', 16 | icon: mdiMonitor, 17 | active: false 18 | }, 19 | { 20 | route: 'site', 21 | title: '网站管理', 22 | icon: mdiFirefox 23 | }, 24 | { 25 | route: 'node', 26 | title: '节点管理', 27 | icon: mdiServer 28 | }, 29 | { 30 | route: 'user', 31 | title: '用户管理', 32 | icon: mdiAccount 33 | } 34 | ] 35 | 36 | export const nodeType = { 37 | 1: "国内", 38 | 2: "海外" 39 | }; 40 | 41 | export const crawType = { 42 | 1: "JSON", 43 | 2: "HTML" 44 | }; -------------------------------------------------------------------------------- /web/src/pages/index/components/cards/MVideo.vue: -------------------------------------------------------------------------------- 1 | 20 | 21 | -------------------------------------------------------------------------------- /web/src/pages/admin/components/MenuLink.vue: -------------------------------------------------------------------------------- 1 | 15 | 16 | 33 | -------------------------------------------------------------------------------- /internal/core/site/github_test.go: -------------------------------------------------------------------------------- 1 | package site 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aaronzjc/mu/test" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestCrawGithub(t *testing.T) { 11 | test.SetupProxy() 12 | defer test.ClearProxy() 13 | 14 | assert := assert.New(t) 15 | 16 | c := &Github{ 17 | Site{ 18 | Name: "Github", 19 | Key: SITE_GITHUB, 20 | Root: "https://github.com", 21 | Desc: "Github.com", 22 | CrawType: CrawHtml, 23 | Tabs: GithubTabs, 24 | }, 25 | } 26 | links, _ := c.BuildUrl() 27 | headers := make(map[string]string) 28 | headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:102.0) Gecko/20100101 Firefox/102.0" 29 | for _, link := range links { 30 | page, err := c.CrawPage(link, headers) 31 | assert.Nil(err) 32 | assert.NotEmpty(page.List) 33 | } 34 | t.Log("fetch github done .") 35 | } 36 | -------------------------------------------------------------------------------- /internal/api/handler/user_test.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aaronzjc/mu/internal/application/dto" 7 | "github.com/aaronzjc/mu/internal/constant" 8 | "github.com/aaronzjc/mu/internal/mocks" 9 | "github.com/aaronzjc/mu/test" 10 | 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/mock" 13 | ) 14 | 15 | func mockUser(t *testing.T) *User { 16 | svc := mocks.NewUserService(t) 17 | svc.EXPECT().GetUserList(mock.Anything).Return([]*dto.User{{ID: 1, Username: "aaron"}}, nil) 18 | return &User{svc: svc} 19 | } 20 | 21 | func TestGetUserList(t *testing.T) { 22 | assert := assert.New(t) 23 | user := mockUser(t) 24 | 25 | resp := test.NewRequest(t).Handler(user.List).Get("/user/list").Exec() 26 | assert.Equal(200, resp.Code()) 27 | errno, _, _, err := resp.TryDecode() 28 | assert.Equal(errno, constant.CodeSuccess) 29 | assert.Nil(err) 30 | } 31 | -------------------------------------------------------------------------------- /internal/core/site/hacker_test.go: -------------------------------------------------------------------------------- 1 | package site 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aaronzjc/mu/test" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestCrawHacker(t *testing.T) { 11 | test.SetupProxy() 12 | defer test.ClearProxy() 13 | 14 | assert := assert.New(t) 15 | 16 | c := &Hacker{ 17 | Site{ 18 | Name: "Hacker", 19 | Key: SITE_HACKER, 20 | Root: "https://news.ycombinator.com/", 21 | Desc: "Hacker News", 22 | CrawType: CrawHtml, 23 | Tabs: HackerTabs, 24 | }, 25 | } 26 | links, _ := c.BuildUrl() 27 | headers := make(map[string]string) 28 | headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:102.0) Gecko/20100101 Firefox/102.0" 29 | for _, link := range links { 30 | page, err := c.CrawPage(link, headers) 31 | assert.Nil(err) 32 | assert.NotEmpty(page.List) 33 | } 34 | t.Log("fetch hacker news done .") 35 | } 36 | -------------------------------------------------------------------------------- /scripts/configs/nginx.conf: -------------------------------------------------------------------------------- 1 | # 前端Docker打包的nginx配置 2 | server { 3 | listen 80; 4 | listen [::]:80; 5 | server_name localhost; 6 | 7 | gzip on; 8 | gzip_types text/plain text/css application/xml application/javascript; 9 | gzip_proxied no-cache no-store private expired auth; 10 | gzip_min_length 1000; 11 | 12 | root /usr/share/nginx/html; 13 | 14 | location / { 15 | index index.html; 16 | if ($request_filename ~ .*\.(css|js|webp|png)$) { 17 | add_header Cache-Control max-age=31536000; 18 | } 19 | } 20 | 21 | location /admin { 22 | index admin.html; 23 | try_files $uri $uri/ /admin.html; 24 | if ($request_filename ~ .*\.(css|js|webp|png)$) { 25 | add_header Cache-Control max-age=31536000; 26 | } 27 | } 28 | 29 | error_page 500 502 503 504 /50x.html; 30 | location = /50x.html { 31 | root /usr/share/nginx/html; 32 | } 33 | } -------------------------------------------------------------------------------- /web/src/pages/admin/components/BoxMain.vue: -------------------------------------------------------------------------------- 1 | 17 | 18 | 36 | -------------------------------------------------------------------------------- /test/setup.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/aaronzjc/mu/internal/config" 7 | "github.com/aaronzjc/mu/internal/infra/cache" 8 | "github.com/aaronzjc/mu/internal/infra/db" 9 | 10 | "gorm.io/gorm" 11 | ) 12 | 13 | var ( 14 | conf = config.Config{ 15 | Database: map[string]config.DbConfig{ 16 | "mu": { 17 | Host: "127.0.0.1", 18 | Port: 3306, 19 | Username: "root", 20 | Password: "123456", 21 | Charset: "utf8", 22 | }, 23 | }, 24 | Redis: config.RedisConfig{ 25 | Host: "127.0.0.1", 26 | Port: 7379, 27 | }, 28 | } 29 | ) 30 | 31 | func SetupDb() error { 32 | return db.Setup(&conf, &gorm.Config{}) 33 | } 34 | 35 | func SetupCache() error { 36 | return cache.Setup(&conf.Redis) 37 | } 38 | 39 | func SetupProxy() { 40 | os.Setenv("HTTP_PROXY", "http://127.0.0.1:51081") 41 | os.Setenv("HTTPS_PROXY", "http://127.0.0.1:51081") 42 | } 43 | 44 | func ClearProxy() { 45 | os.Unsetenv("HTTP_PROXY") 46 | os.Unsetenv("HTTPS_PROXY") 47 | } 48 | -------------------------------------------------------------------------------- /internal/application/store/base_repo.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/aaronzjc/mu/internal/application/dto" 7 | "github.com/aaronzjc/mu/internal/domain/model" 8 | "github.com/aaronzjc/mu/internal/infra/db" 9 | "gorm.io/gorm" 10 | ) 11 | 12 | type BaseRepoImpl struct { 13 | db *gorm.DB 14 | } 15 | 16 | func (r *BaseRepoImpl) prepare(q *dto.Query) *gorm.DB { 17 | clone := r.db 18 | if q == nil { 19 | return clone 20 | } 21 | if q.Order != "" { 22 | clone = clone.Order(q.Order) 23 | } 24 | 25 | if q.Query != "" { 26 | clone = clone.Where(q.Query, q.Args...) 27 | } 28 | 29 | if q.Limit > 0 { 30 | clone = clone.Limit(q.Limit) 31 | } 32 | return clone 33 | } 34 | 35 | func (r *BaseRepoImpl) create(m interface{}) error { 36 | return r.db.Create(m).Error 37 | } 38 | 39 | func NewBaseImpl() (BaseRepoImpl, error) { 40 | mu, ok := db.Get(model.DB_MU) 41 | if !ok { 42 | return BaseRepoImpl{}, errors.New("db not connected") 43 | } 44 | return BaseRepoImpl{db: mu}, nil 45 | } 46 | -------------------------------------------------------------------------------- /scripts/k8s/mu-api.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: mu-api 5 | namespace: k3s-apps 6 | spec: 7 | selector: 8 | matchLabels: 9 | app: mu-api 10 | replicas: 2 11 | revisionHistoryLimit: 1 12 | template: 13 | metadata: 14 | labels: 15 | app: mu-api 16 | spec: 17 | volumes: 18 | - name: tz-config 19 | hostPath: 20 | path: /usr/share/zoneinfo/Asia/Shanghai 21 | - name: mu-conf 22 | configMap: 23 | name: mu-config-next 24 | - name: mu-log-dir 25 | emptyDir: {} 26 | containers: 27 | - name: mu-api 28 | image: aaronzjc/mu-api:latest 29 | imagePullPolicy: Always 30 | volumeMounts: 31 | - name: mu-conf 32 | mountPath: /app/conf 33 | - name: mu-log-dir 34 | mountPath: /var/log 35 | ports: 36 | - containerPort: 7980 37 | resources: 38 | limits: 39 | cpu: 50m 40 | memory: 50Mi -------------------------------------------------------------------------------- /scripts/k8s/mu-commander.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: mu-commander 5 | namespace: k3s-apps 6 | spec: 7 | selector: 8 | matchLabels: 9 | app: mu-commander 10 | replicas: 2 11 | revisionHistoryLimit: 1 12 | template: 13 | metadata: 14 | labels: 15 | app: mu-commander 16 | spec: 17 | volumes: 18 | - name: mu-conf 19 | configMap: 20 | name: mu-config-next 21 | - name: tz-config 22 | hostPath: 23 | path: /usr/share/zoneinfo/Asia/Shanghai 24 | - name: mu-log-dir 25 | emptyDir: {} 26 | containers: 27 | - name: mu-commander 28 | image: aaronzjc/mu-commander:latest 29 | imagePullPolicy: Always 30 | volumeMounts: 31 | - name: mu-conf 32 | mountPath: /app/conf 33 | - name: mu-log-dir 34 | mountPath: /var/log 35 | ports: 36 | - containerPort: 7970 37 | resources: 38 | limits: 39 | cpu: 50m 40 | memory: 50Mi -------------------------------------------------------------------------------- /web/src/pages/admin/components/toast/index.js: -------------------------------------------------------------------------------- 1 | import { createApp } from 'vue' 2 | import Toast from '@adm/components/toast/Toast.vue' 3 | 4 | const toastContainer = document.createElement('div') 5 | toastContainer.className = 'notice-container' 6 | 7 | const ToastInstance = { 8 | show(text, level = '') { 9 | toastContainer.innerHTML = '' 10 | const toastApp = createApp(Toast, { text, level }).mount( 11 | document.createElement('div') 12 | ) 13 | toastContainer.insertAdjacentElement('afterbegin', toastApp.$el) 14 | document.getElementById('app').appendChild(toastContainer) 15 | }, 16 | error(text) { 17 | this.show(text, 'danger') 18 | }, 19 | warn(text) { 20 | this.show(text, 'warning') 21 | }, 22 | success(text) { 23 | this.show(text, 'success') 24 | } 25 | } 26 | 27 | Toast.install = (app, options) => { 28 | app.config.globalProperties.$toast = ToastInstance 29 | } 30 | 31 | function useToast() { 32 | return ToastInstance 33 | } 34 | 35 | export {Toast, useToast} -------------------------------------------------------------------------------- /web/src/pages/index/components/Main.vue: -------------------------------------------------------------------------------- 1 | 7 | 8 | 41 | -------------------------------------------------------------------------------- /web/src/pages/admin/components/FormField.vue: -------------------------------------------------------------------------------- 1 | 19 | 20 | 41 | -------------------------------------------------------------------------------- /web/src/lib/http.js: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | 3 | const client = axios.create({ 4 | baseURL: import.meta.env.VITE_APP_URL, 5 | withCredentials: true 6 | }); 7 | 8 | client.interceptors.request.use(req => { 9 | let token = localStorage.getItem(import.meta.env.VITE_TOKEN_KEY) 10 | if (token) { 11 | req.headers.Authorization = token; 12 | } 13 | return req; 14 | }); 15 | 16 | export function Get(url, params, headers) { 17 | if (!params) { 18 | params = {}; 19 | } 20 | 21 | let config = { 22 | method: "get", 23 | url: url, 24 | params: params 25 | }; 26 | if (headers) { 27 | config.headers = headers; 28 | } 29 | 30 | return client(config) 31 | } 32 | 33 | export function Post(url, data, headers) { 34 | let config= { 35 | method: 'post', 36 | url: url, 37 | params: {} 38 | }; 39 | if (headers) { 40 | config.headers = headers; 41 | } 42 | if (data) { 43 | config.data = data; 44 | } 45 | return client(config); 46 | } 47 | 48 | export default client; -------------------------------------------------------------------------------- /internal/api/handler/index.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "github.com/aaronzjc/mu/internal/application/dto" 5 | "github.com/aaronzjc/mu/internal/application/service" 6 | "github.com/aaronzjc/mu/internal/application/store" 7 | "github.com/aaronzjc/mu/internal/constant" 8 | "github.com/gin-gonic/gin" 9 | ) 10 | 11 | type Index struct { 12 | svc service.SiteService 13 | } 14 | 15 | func (idx *Index) Sites(ctx *gin.Context) { 16 | sites, _ := idx.svc.ListOfIndex(ctx) 17 | Resp(ctx, constant.CodeSuccess, "", sites) 18 | } 19 | 20 | func (idx *Index) News(ctx *gin.Context) { 21 | k := ctx.Request.URL.Query()["key"][0] 22 | kk := ctx.Request.URL.Query()["hkey"][0] 23 | 24 | loginUser := ctx.GetInt(constant.LoginKey) 25 | news, _ := idx.svc.News(ctx, loginUser, k, kk) 26 | if news == nil { 27 | news = &dto.News{List: make([]dto.NewsItem, 0)} 28 | } 29 | Resp(ctx, constant.CodeSuccess, "success", news) 30 | } 31 | 32 | func NewIndex() *Index { 33 | repo := store.NewSiteRepo() 34 | favorRepo := store.NewFavorRepo() 35 | return &Index{ 36 | svc: service.NewSiteService(repo, favorRepo), 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /web/src/pages/index/components/Footer.vue: -------------------------------------------------------------------------------- 1 | 14 | 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 aaronzhang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /web/src/pages/admin/components/BasicIcon.vue: -------------------------------------------------------------------------------- 1 | 34 | 35 | 48 | -------------------------------------------------------------------------------- /pkg/helper/ip.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "net" 5 | "net/http" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | func LocalHostname() string { 11 | hostname, _ := os.Hostname() 12 | return hostname 13 | } 14 | 15 | func LocalAddr() string { 16 | ifaces, err := net.Interfaces() 17 | if err != nil { 18 | return "" 19 | } 20 | var ip net.IP 21 | for _, i := range ifaces { 22 | addrs, err := i.Addrs() 23 | if err != nil { 24 | continue 25 | } 26 | for _, addr := range addrs { 27 | switch v := addr.(type) { 28 | case *net.IPNet: 29 | case *net.IPAddr: 30 | ip = v.IP 31 | } 32 | } 33 | } 34 | return ip.String() 35 | } 36 | 37 | func ClientIp(r *http.Request) string { 38 | ip := r.Header.Get("X-Forward-For") 39 | for _, i := range strings.Split(ip, ",") { 40 | if res := net.ParseIP(i); res != nil { 41 | return res.String() 42 | } 43 | } 44 | ip = r.Header.Get("X-Real-IP") 45 | if res := net.ParseIP(ip); res != nil { 46 | return res.String() 47 | } 48 | ip, _, err := net.SplitHostPort(r.RemoteAddr) 49 | if err != nil { 50 | return "" 51 | } 52 | if net.ParseIP(ip) != nil { 53 | return ip 54 | } 55 | 56 | return "" 57 | } 58 | -------------------------------------------------------------------------------- /internal/application/store/node_repo.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/aaronzjc/mu/internal/application/dto" 8 | "github.com/aaronzjc/mu/internal/domain/model" 9 | "github.com/aaronzjc/mu/internal/domain/repo" 10 | ) 11 | 12 | type NodeRepoImpl struct { 13 | BaseRepoImpl 14 | } 15 | 16 | var _ repo.NodeRepo = &NodeRepoImpl{} 17 | 18 | func (r *NodeRepoImpl) Get(ctx context.Context, q *dto.Query) ([]model.Node, error) { 19 | nodes := []model.Node{} 20 | if err := r.prepare(q).Find(&nodes).Error; err != nil { 21 | return nil, errors.New("get nodes err") 22 | } 23 | return nodes, nil 24 | } 25 | 26 | func (r *NodeRepoImpl) Create(ctx context.Context, node model.Node) error { 27 | return r.create(&node) 28 | } 29 | 30 | func (r *NodeRepoImpl) Update(ctx context.Context, node model.Node, data map[string]interface{}) error { 31 | return r.db.Model(&node).Updates(data).Error 32 | } 33 | 34 | func (r *NodeRepoImpl) Del(ctx context.Context, node model.Node) error { 35 | return r.db.Delete(&node).Error 36 | } 37 | 38 | func NewNodeRepo() *NodeRepoImpl { 39 | base, _ := NewBaseImpl() 40 | return &NodeRepoImpl{BaseRepoImpl: base} 41 | } 42 | -------------------------------------------------------------------------------- /web/src/pages/admin/styles/_card.scss: -------------------------------------------------------------------------------- 1 | .card:not(:last-child) { 2 | margin-bottom: $default-padding; 3 | } 4 | 5 | .card { 6 | border-radius: $radius-large; 7 | border: $card-border; 8 | 9 | &.has-table { 10 | .card-content { 11 | padding: 0; 12 | } 13 | .b-table { 14 | border-radius: $radius-large; 15 | overflow: hidden; 16 | } 17 | } 18 | 19 | &.is-card-widget { 20 | .card-content { 21 | padding: $default-padding * 0.5; 22 | } 23 | } 24 | 25 | .card-header { 26 | border-bottom: 1px solid $base-color-light; 27 | flex-direction: column; 28 | 29 | .card-header-action { 30 | padding: 1rem 0.75rem; 31 | } 32 | } 33 | 34 | .card-content { 35 | hr { 36 | margin-left: $card-content-padding * -1; 37 | margin-right: $card-content-padding * -1; 38 | } 39 | } 40 | 41 | .is-widget-icon { 42 | .icon { 43 | width: 5rem; 44 | height: 5rem; 45 | } 46 | } 47 | 48 | .is-widget-label { 49 | .subtitle { 50 | color: $grey; 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /internal/application/service/craw_test.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/aaronzjc/mu/internal/application/dto" 8 | "github.com/aaronzjc/mu/internal/application/store" 9 | "github.com/aaronzjc/mu/internal/core/site" 10 | "github.com/aaronzjc/mu/internal/domain/model" 11 | "github.com/aaronzjc/mu/test" 12 | "github.com/stretchr/testify/assert" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | func TestCraw(t *testing.T) { 17 | require.Nil(t, test.SetupDb()) 18 | require.Nil(t, test.SetupCache()) 19 | 20 | nodeRepo := store.NewNodeRepo() 21 | siteRepo := store.NewSiteRepo() 22 | crawSvc := NewCrawService(siteRepo, nodeRepo) 23 | ctx := context.Background() 24 | 25 | ds := &dto.Site{ 26 | Key: site.SITE_GITHUB, 27 | NodeOption: model.ByHosts, 28 | NodeHosts: []int{4}, 29 | } 30 | // pick agent 31 | node, err := crawSvc.PickAgent(ctx, ds) 32 | require.Nil(t, err) 33 | require.Equal(t, node.ID, 4) 34 | 35 | // craw data 36 | err = crawSvc.Craw(ctx, ds) 37 | require.Nil(t, err) 38 | s := site.NewGithub() 39 | for _, tab := range s.Tabs { 40 | news, err := siteRepo.GetNews(ctx, s.Key, tab.Tag) 41 | require.Nil(t, err) 42 | assert.NotEmpty(t, news.List) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /internal/infra/db/mysql.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/aaronzjc/mu/internal/config" 9 | "github.com/aaronzjc/mu/pkg/logger" 10 | 11 | "gorm.io/driver/mysql" 12 | "gorm.io/gorm" 13 | ) 14 | 15 | type DBPool struct { 16 | dbMap map[string]*gorm.DB 17 | } 18 | 19 | var ( 20 | pool *DBPool 21 | once sync.Once 22 | ) 23 | 24 | func init() { 25 | pool = &DBPool{ 26 | dbMap: make(map[string]*gorm.DB), 27 | } 28 | } 29 | 30 | func Setup(conf *config.Config, config *gorm.Config) error { 31 | var err error 32 | once.Do(func() { 33 | for dbname, v := range conf.Database { 34 | // 初始化DB等 35 | dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=%s&parseTime=True", 36 | v.Username, 37 | v.Password, 38 | v.Host, 39 | v.Port, 40 | dbname, 41 | v.Charset, 42 | ) 43 | if client, errr := gorm.Open(mysql.Open(dsn), config); errr != nil { 44 | logger.Error("init db err, ", errr.Error()) 45 | err = errors.New("connect to " + dbname + " err") 46 | return 47 | } else { 48 | pool.dbMap[dbname] = client 49 | } 50 | } 51 | }) 52 | return err 53 | } 54 | 55 | func Get(dbname string) (*gorm.DB, bool) { 56 | db, ok := pool.dbMap[dbname] 57 | return db, ok 58 | } 59 | -------------------------------------------------------------------------------- /internal/agent.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "net" 5 | "os" 6 | "os/signal" 7 | "syscall" 8 | 9 | "github.com/aaronzjc/mu/internal/agent" 10 | "github.com/aaronzjc/mu/internal/pb" 11 | "github.com/aaronzjc/mu/pkg/logger" 12 | "github.com/aaronzjc/mu/test" 13 | "github.com/urfave/cli" 14 | "google.golang.org/grpc" 15 | ) 16 | 17 | func SetupAgent(ctx *cli.Context) error { 18 | err := logger.Setup("agent", "") 19 | if err != nil { 20 | return err 21 | } 22 | 23 | // 测试环境配置代理 24 | if os.Getenv("APP_ENV") != "prod" { 25 | test.SetupProxy() 26 | } 27 | return nil 28 | } 29 | 30 | func RunAgent(ctx *cli.Context) error { 31 | addr := ":7990" 32 | listener, err := net.Listen("tcp", addr) // no need to use config file 33 | if err != nil { 34 | logger.Fatal("bind socket failed") 35 | } 36 | 37 | var opts []grpc.ServerOption 38 | rpcServer := grpc.NewServer(opts...) 39 | rpcServer.RegisterService(&pb.Agent_ServiceDesc, agent.NewAgentServer()) 40 | 41 | go rpcServer.Serve(listener) 42 | logger.Info("[START] agent listen at ", addr) 43 | 44 | // 监听关闭信号 45 | sig := make(chan os.Signal, 1) 46 | signal.Notify(sig, syscall.SIGQUIT, os.Interrupt, syscall.SIGTERM) 47 | <-sig 48 | 49 | // 关闭服务 50 | rpcServer.GracefulStop() 51 | logger.Info("[STOP] agent stop done") 52 | return nil 53 | } 54 | -------------------------------------------------------------------------------- /internal/domain/model/site.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type CrawType int 4 | 5 | const ( 6 | CrawHtml CrawType = 1 // 网站是HTML 7 | CrawApi CrawType = 2 // 网站是JSON接口 8 | 9 | ByType = 1 // 通过服务器类型 10 | ByHosts = 2 // 服务器IPs 11 | 12 | Disable = 0 // 禁用 13 | Enable = 1 // 启用 14 | ) 15 | 16 | type Site struct { 17 | ID int `gorm:"primaryKey"` 18 | Name string `gorm:"name"` 19 | Root string `gorm:"root"` 20 | Key string `gorm:"key"` 21 | Desc string `gorm:"desc"` 22 | Type int8 `gorm:"type"` 23 | Tags string `gorm:"tags"` 24 | Cron string `gorm:"cron"` 25 | Enable int8 `gorm:"enable"` 26 | NodeOption int8 `gorm:"node_option"` 27 | NodeType int8 `gorm:"node_type"` 28 | NodeHosts string `gorm:"node_hosts"` 29 | ReqHeaders string `gorm:"req_headers"` 30 | } 31 | 32 | func (s *Site) TableName() string { 33 | return "site" 34 | } 35 | 36 | type NewsItem struct { 37 | Key string `json:"key"` 38 | Title string `json:"title"` 39 | Desc string `json:"desc"` 40 | Rank float64 `json:"rank"` 41 | OriginUrl string `json:"origin_url"` 42 | Card uint8 `json:"card_type"` 43 | Ext map[string]string `json:"ext"` 44 | } 45 | type News struct { 46 | T string `json:"t"` 47 | List []NewsItem `json:"list"` 48 | } 49 | -------------------------------------------------------------------------------- /web/src/pages/admin/views/Home.vue: -------------------------------------------------------------------------------- 1 | 19 | 20 | 46 | -------------------------------------------------------------------------------- /internal/application/store/favor_repo.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/aaronzjc/mu/internal/application/dto" 8 | "github.com/aaronzjc/mu/internal/domain/model" 9 | "github.com/aaronzjc/mu/internal/domain/repo" 10 | ) 11 | 12 | type FavorRepoImpl struct { 13 | BaseRepoImpl 14 | } 15 | 16 | var _ repo.FavorRepo = &FavorRepoImpl{} 17 | 18 | func (r *FavorRepoImpl) Get(ctx context.Context, q *dto.Query) ([]model.Favor, error) { 19 | favors := []model.Favor{} 20 | if err := r.prepare(q).Find(&favors).Error; err != nil { 21 | return nil, errors.New("get favors err") 22 | } 23 | return favors, nil 24 | } 25 | 26 | func (r *FavorRepoImpl) Create(ctx context.Context, favor model.Favor) error { 27 | return r.create(&favor) 28 | } 29 | 30 | func (r *FavorRepoImpl) Del(ctx context.Context, f model.Favor) error { 31 | r.db.Delete(&f) 32 | return nil 33 | } 34 | 35 | func (r *FavorRepoImpl) Sites(ctx context.Context, q *dto.Query) []string { 36 | var sites []string 37 | var favors []model.Favor 38 | if err := r.prepare(q).Select("DISTINCT(`site`)").Find(&favors).Error; err != nil { 39 | return sites 40 | } 41 | for _, v := range favors { 42 | sites = append(sites, v.Site) 43 | } 44 | return sites 45 | } 46 | 47 | func NewFavorRepo() *FavorRepoImpl { 48 | base, _ := NewBaseImpl() 49 | return &FavorRepoImpl{ 50 | BaseRepoImpl: base, 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /web/src/pages/admin/styles/_element.scss: -------------------------------------------------------------------------------- 1 | .is-divider { 2 | display: flex; 3 | align-items: center; 4 | color: #aaa; 5 | margin: 1rem 0; 6 | font-size: $size-base * 0.85; 7 | align-self: stretch; 8 | gap: 1rem; 9 | word-break: keep-all; 10 | &::before, 11 | &::after { 12 | content: ''; 13 | width: 100%; 14 | height: 1px; 15 | background-color: #dadada; 16 | } 17 | } 18 | 19 | .notice-container { 20 | width: 100%; 21 | height: 100%; 22 | position: fixed; 23 | z-index: 999; 24 | pointer-events: none; 25 | background-color: transparent; 26 | left: 0; 27 | top: 0; 28 | overflow: hidden; 29 | display: flex; 30 | flex-direction: column; 31 | align-items: center; 32 | padding: 4rem; 33 | .toast { 34 | display: block; 35 | background-color: $primary; 36 | color: #fff; 37 | padding: 0.5rem 2rem; 38 | border-radius: $radius; 39 | margin: 0.25rem 0; 40 | z-index: auto; 41 | animation-duration: 150ms; 42 | box-shadow: 0 1px 4px rgba(0, 0, 0, 0.12), 0 0 6px rgba(0, 0, 0, 0.04); 43 | &.danger { 44 | background-color: $danger; 45 | } 46 | &.warning { 47 | background-color: $warning; 48 | } 49 | &.success { 50 | background-color: $success; 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /web/src/pages/index/router/router.js: -------------------------------------------------------------------------------- 1 | import { createRouter, createWebHashHistory } from "vue-router"; 2 | import Main from "../components/Main.vue"; 3 | import Content from "../components/Content.vue"; 4 | 5 | const Favor = () => import("../components/Favor.vue"); 6 | const Login = () => import("../components/Login.vue"); 7 | 8 | const routes = [ 9 | { 10 | path: "/", 11 | name: "default", 12 | title: "首页", 13 | component: Main, 14 | redirect: "index", 15 | children: [ 16 | { 17 | path: "/", 18 | name: "index", 19 | title: "首页", 20 | component: Content, 21 | }, 22 | { 23 | path: "/favor", 24 | name: "favor", 25 | title: "我的收藏", 26 | component: Favor, 27 | }, 28 | ], 29 | }, 30 | ]; 31 | 32 | export { routes }; 33 | 34 | const publicRoutes = [ 35 | { 36 | path: "/login", 37 | name: "login", 38 | component: Login, 39 | }, 40 | ]; 41 | 42 | const router = createRouter({ 43 | history: createWebHashHistory(), 44 | routes: routes.concat(publicRoutes), 45 | }); 46 | 47 | router.beforeEach((to, from, next) => { 48 | let token = to.query.token; 49 | if (token != "" && token != undefined && token != null) { 50 | localStorage.setItem(import.meta.env.VITE_TOKEN_KEY, token); 51 | router.replace({ path: "/" }); 52 | } else { 53 | next(); 54 | } 55 | }); 56 | 57 | export default router; 58 | -------------------------------------------------------------------------------- /internal/commander/job.go: -------------------------------------------------------------------------------- 1 | package commander 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aaronzjc/mu/internal/application/dto" 7 | "github.com/aaronzjc/mu/internal/application/service" 8 | "github.com/aaronzjc/mu/internal/application/store" 9 | "github.com/aaronzjc/mu/pkg/logger" 10 | "github.com/robfig/cron/v3" 11 | ) 12 | 13 | type CrawJob struct { 14 | site *dto.Site 15 | 16 | svc service.CrawService 17 | } 18 | 19 | var _ cron.Job = &CrawJob{} 20 | 21 | func (j *CrawJob) Run() { 22 | ctx := context.Background() 23 | err := j.svc.Craw(ctx, j.site) 24 | if err != nil { 25 | logger.Error("craw job run err " + err.Error()) 26 | } 27 | } 28 | 29 | func NewCrawJob(site *dto.Site) *CrawJob { 30 | siteRepo := store.NewSiteRepo() 31 | nodeRepo := store.NewNodeRepo() 32 | crawSvc := service.NewCrawService(siteRepo, nodeRepo) 33 | return &CrawJob{ 34 | site: site, 35 | svc: crawSvc, 36 | } 37 | } 38 | 39 | /** 40 | * 服务存活检查任务 41 | */ 42 | type CheckJob struct { 43 | Name string 44 | Spec string 45 | 46 | svc service.NodeService 47 | } 48 | 49 | var _ cron.Job = &CheckJob{} 50 | 51 | func (j *CheckJob) Run() { 52 | ctx := context.Background() 53 | j.svc.CheckNodes(ctx, &Pool) 54 | } 55 | 56 | func NewCheckJob(name string, spec string) *CheckJob { 57 | repo := store.NewNodeRepo() 58 | return &CheckJob{ 59 | Name: name, 60 | Spec: spec, 61 | svc: service.NewNodeService(repo), 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /web/src/pages/admin/layouts/Admin.vue: -------------------------------------------------------------------------------- 1 | 19 | 20 | 49 | -------------------------------------------------------------------------------- /test/web.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | "github.com/gin-gonic/gin" 10 | ) 11 | 12 | type Request struct { 13 | t *testing.T 14 | g *gin.Engine 15 | h gin.HandlerFunc 16 | 17 | Method string 18 | Url string 19 | } 20 | 21 | func NewRequest(t *testing.T) *Request { 22 | return &Request{ 23 | t: t, 24 | g: gin.Default(), 25 | } 26 | } 27 | 28 | func (r *Request) Handler(h gin.HandlerFunc) *Request { 29 | r.h = h 30 | return r 31 | } 32 | 33 | func (r *Request) Get(url string) *Request { 34 | r.Method = "GET" 35 | r.Url = url 36 | r.g.GET(url, r.h) 37 | return r 38 | } 39 | 40 | func (r *Request) Exec() *Response { 41 | resp := &Response{ 42 | resp: httptest.NewRecorder(), 43 | } 44 | req, _ := http.NewRequest(r.Method, r.Url, nil) 45 | r.g.ServeHTTP(resp.resp, req) 46 | return resp 47 | } 48 | 49 | type Response struct { 50 | resp *httptest.ResponseRecorder 51 | } 52 | 53 | func (r *Response) Code() int { 54 | return r.resp.Code 55 | } 56 | 57 | func (r *Response) Body() string { 58 | return r.resp.Body.String() 59 | } 60 | 61 | func (r *Response) TryDecode() (code int, msg string, data any, err error) { 62 | var dataSt struct { 63 | Code int `json:"code"` 64 | Msg string `json:"msg"` 65 | Data interface{} `json:"data"` 66 | } 67 | err = json.Unmarshal([]byte(r.Body()), &dataSt) 68 | if err != nil { 69 | return 70 | } 71 | code, msg, data = dataSt.Code, dataSt.Msg, dataSt.Data 72 | return 73 | } 74 | -------------------------------------------------------------------------------- /web/src/pages/admin/components/Tile.vue: -------------------------------------------------------------------------------- 1 | 35 | 41 | -------------------------------------------------------------------------------- /web/src/pages/index/components/HoTab.vue: -------------------------------------------------------------------------------- 1 | 19 | 20 | 50 | -------------------------------------------------------------------------------- /internal/core/rpc/pool.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "sync" 7 | "time" 8 | 9 | "github.com/aaronzjc/mu/internal/pb" 10 | "github.com/aaronzjc/mu/pkg/logger" 11 | "google.golang.org/grpc" 12 | ) 13 | 14 | type RpcClient struct { 15 | Conn *grpc.ClientConn 16 | Client *pb.AgentClient 17 | } 18 | 19 | type RpcPool struct { 20 | Clients map[string]*RpcClient 21 | Lock sync.RWMutex 22 | } 23 | 24 | func (r *RpcPool) Get(addr string) (*RpcClient, error) { 25 | r.Lock.RLock() 26 | rc, ok := r.Clients[addr] 27 | r.Lock.RUnlock() 28 | if ok { 29 | return rc, nil 30 | } 31 | 32 | client, err := r.Set(addr) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | return client, nil 38 | } 39 | 40 | func (r *RpcPool) Set(addr string) (*RpcClient, error) { 41 | r.Lock.Lock() 42 | defer r.Lock.Unlock() 43 | 44 | rc, ok := r.Clients[addr] 45 | if ok { 46 | return rc, nil 47 | } 48 | 49 | opts := []grpc.DialOption{ 50 | grpc.WithInsecure(), 51 | } 52 | 53 | conn, err := grpc.Dial(addr, opts...) 54 | if err != nil { 55 | logger.Error("connect error " + err.Error()) 56 | return nil, errors.New("dial server " + addr + " failed") 57 | } 58 | 59 | client := pb.NewAgentClient(conn) 60 | _, cancel := context.WithTimeout(context.Background(), time.Second*3) 61 | defer cancel() 62 | 63 | r.Clients[addr] = &RpcClient{ 64 | Conn: conn, 65 | Client: &client, 66 | } 67 | 68 | return r.Clients[addr], nil 69 | } 70 | 71 | func (r *RpcPool) Release(addr string) bool { 72 | r.Lock.Lock() 73 | rc, ok := r.Clients[addr] 74 | r.Lock.Unlock() 75 | if !ok { 76 | return true 77 | } 78 | 79 | delete(r.Clients, addr) 80 | _ = rc.Conn.Close() 81 | 82 | return true 83 | } 84 | -------------------------------------------------------------------------------- /internal/agent/agent.go: -------------------------------------------------------------------------------- 1 | package agent 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "sync" 9 | 10 | "github.com/aaronzjc/mu/internal/core/site" 11 | "github.com/aaronzjc/mu/internal/pb" 12 | "github.com/aaronzjc/mu/pkg/helper" 13 | "github.com/aaronzjc/mu/pkg/logger" 14 | ) 15 | 16 | type AgentServer struct { 17 | pb.UnimplementedAgentServer 18 | } 19 | 20 | var _ pb.AgentServer = &AgentServer{} 21 | 22 | func (agent *AgentServer) Craw(ctx context.Context, msg *pb.Job) (*pb.Result, error) { 23 | var wg sync.WaitGroup 24 | 25 | pageMap := make(map[string]site.Page) 26 | headers := make(map[string]string) 27 | 28 | h := msg.Headers 29 | for _, v := range h { 30 | headers[v.Key] = v.Val 31 | } 32 | spider, ok := site.SiteMap[msg.Name] 33 | if !ok { 34 | return nil, errors.New("not supported site " + msg.Name) 35 | } 36 | 37 | links, _ := spider.BuildUrl() 38 | for _, link := range links { 39 | wg.Add(1) 40 | go func(link site.Link) { 41 | defer wg.Done() 42 | page, err := spider.CrawPage(link, headers) 43 | if err != nil { 44 | logger.Error("craw page error, err " + err.Error()) 45 | return 46 | } 47 | logger.Info(fmt.Sprintf("craw page %s done", link.Url)) 48 | pageMap[link.Tag] = page 49 | }(link) 50 | } 51 | 52 | wg.Wait() 53 | 54 | result := new(pb.Result) 55 | result.T = helper.CurrentTimeStr() 56 | result.HotMap = make(map[string]string) 57 | for tag, page := range pageMap { 58 | res, _ := json.Marshal(page.List) 59 | result.HotMap[tag] = string(res) 60 | } 61 | return result, nil 62 | } 63 | 64 | func (agent *AgentServer) Check(ctx context.Context, msg *pb.Ping) (*pb.Pong, error) { 65 | logger.Info("receive health check") 66 | return &pb.Pong{ 67 | Pong: msg.Ping, 68 | }, nil 69 | } 70 | 71 | func NewAgentServer() *AgentServer { 72 | return &AgentServer{} 73 | } 74 | -------------------------------------------------------------------------------- /web/src/pages/admin/styles/_section.scss: -------------------------------------------------------------------------------- 1 | section.section.is-main-section { 2 | padding: $default-padding; 3 | padding-bottom: $default-padding * 2; 4 | } 5 | 6 | section.section.is-title-bar { 7 | padding: $size-base $default-padding; 8 | border-bottom: $light-border; 9 | 10 | ul { 11 | li { 12 | display: inline-block; 13 | padding: 0 $default-padding * 0.5 0 0; 14 | font-size: $default-padding; 15 | color: $title-bar-color; 16 | 17 | &:after { 18 | display: inline-block; 19 | content: '/'; 20 | padding-left: $default-padding * 0.5; 21 | } 22 | 23 | &:last-child { 24 | padding-right: 0; 25 | color: $title-bar-active-color; 26 | 27 | &:after { 28 | display: none; 29 | } 30 | } 31 | } 32 | } 33 | } 34 | 35 | section.hero.is-hero-bar { 36 | background-color: $hero-bar-background; 37 | border-bottom: $light-border; 38 | 39 | .hero-body { 40 | padding: $default-padding; 41 | 42 | .level-item { 43 | &.is-hero-avatar-item { 44 | margin-right: $default-padding; 45 | } 46 | 47 | > div > .level { 48 | margin-bottom: $default-padding * 0.5; 49 | } 50 | 51 | .subtitle + p { 52 | margin-top: $default-padding * 0.5; 53 | } 54 | } 55 | 56 | .button { 57 | &.is-hero-button { 58 | background-color: rgba($white, 0.5); 59 | font-weight: 300; 60 | @include transition(background-color); 61 | 62 | &:hover { 63 | background-color: $white; 64 | } 65 | } 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /web/src/pages/admin/styles/_custom.scss: -------------------------------------------------------------------------------- 1 | .is-tiles-wrapper { 2 | margin-bottom: $default-padding; 3 | } 4 | 5 | .modal-card { 6 | width: $modal-card-width; 7 | } 8 | 9 | .modal-card-foot { 10 | background-color: $modal-card-foot-background-color; 11 | } 12 | 13 | @include mobile { 14 | .modal .modal-card { 15 | width: $modal-card-width-mobile; 16 | margin: 0 auto; 17 | } 18 | } 19 | 20 | .is-user-avatar { 21 | &.has-max-width { 22 | max-width: $size-base * 7; 23 | } 24 | 25 | &.is-aligned-center { 26 | margin: 0 auto; 27 | } 28 | 29 | img { 30 | margin: 0 auto; 31 | border-radius: $radius-rounded; 32 | } 33 | } 34 | 35 | .icon.has-update-mark { 36 | position: relative; 37 | 38 | &:after { 39 | content: ''; 40 | width: $icon-update-mark-size; 41 | height: $icon-update-mark-size; 42 | position: absolute; 43 | top: 1px; 44 | right: 1px; 45 | background-color: $icon-update-mark-color; 46 | border-radius: $radius-rounded; 47 | } 48 | } 49 | 50 | // 登录页面 51 | .login.hero { 52 | height: 100%; 53 | justify-content: center; 54 | background-image: linear-gradient(to right top, #004080, #ffc383); 55 | 56 | .card-header { 57 | border-bottom-width: 0; 58 | } 59 | .card-header-title { 60 | padding: 1rem 1rem; 61 | 62 | span { 63 | font-size: $size-base * 1.3; 64 | font-weight: 550; 65 | color: $base-color; 66 | } 67 | } 68 | .hero-body { 69 | flex-grow: 0; 70 | } 71 | .field.password { 72 | margin-top: 1rem; 73 | margin-bottom: 1.5rem; 74 | } 75 | .options { 76 | display: flex; 77 | justify-content: space-between; 78 | font-size: $size-base * 0.9; 79 | .forgot { 80 | cursor: pointer; 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /internal/core/site/zhihu.go: -------------------------------------------------------------------------------- 1 | package site 2 | 3 | import ( 4 | "regexp" 5 | "time" 6 | 7 | "github.com/PuerkitoBio/goquery" 8 | ) 9 | 10 | const SITE_ZHIHU = "zhihu" 11 | 12 | var ZhihuTabs = []SiteTab{ 13 | { 14 | Tag: "all", 15 | Url: "https://www.zhihu.com/hot", 16 | Name: "知乎热榜", 17 | }, 18 | } 19 | 20 | type Zhihu struct { 21 | Site 22 | } 23 | 24 | func (z *Zhihu) GetSite() *Site { 25 | return &z.Site 26 | } 27 | 28 | func (z *Zhihu) BuildUrl() ([]Link, error) { 29 | var list []Link 30 | for _, tab := range ZhihuTabs { 31 | url := tab.Url 32 | link := Link{ 33 | Key: url, 34 | Url: url, 35 | Tag: tab.Tag, 36 | } 37 | list = append(list, link) 38 | } 39 | 40 | return list, nil 41 | } 42 | 43 | func (z *Zhihu) CrawPage(link Link, headers map[string]string) (Page, error) { 44 | page, err := z.FetchData(link, nil, headers) 45 | if err != nil { 46 | return Page{}, err 47 | } 48 | var data []Hot 49 | doc := page.Doc 50 | doc.Find(".HotList-list .HotItem-content").Each(func(i int, s *goquery.Selection) { 51 | url := s.Find("a").AttrOr("href", "") 52 | text := s.Find("h2").Text() 53 | if text == "" { 54 | return 55 | } 56 | hot := Hot{ 57 | Title: text, 58 | OriginUrl: url, 59 | } 60 | hot.Key = z.FetchKey(hot.OriginUrl) 61 | if hot.Key == "" { 62 | return 63 | } 64 | data = append(data, hot) 65 | }) 66 | 67 | page.T = time.Now() 68 | page.List = data 69 | 70 | return page, nil 71 | } 72 | 73 | func (z *Zhihu) FetchKey(link string) string { 74 | reg := regexp.MustCompile(`.*/question/(\d+)`) 75 | id := reg.ReplaceAllString(link, "$1") 76 | return id 77 | } 78 | 79 | func NewZhihu() *Zhihu { 80 | return &Zhihu{ 81 | Site{ 82 | Name: "知乎", 83 | Key: SITE_ZHIHU, 84 | Root: "https://zhihu.com", 85 | Desc: "知乎热榜", 86 | CrawType: CrawHtml, 87 | Tabs: ZhihuTabs, 88 | }, 89 | } 90 | } 91 | 92 | var _ Spider = &Zhihu{} 93 | 94 | func init() { 95 | RegistSite(SITE_ZHIHU, NewZhihu()) 96 | } 97 | -------------------------------------------------------------------------------- /web/src/pages/admin/components/MenuItem.vue: -------------------------------------------------------------------------------- 1 | 24 | 25 | 72 | -------------------------------------------------------------------------------- /internal/api/handler/stat.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | 9 | "github.com/aaronzjc/mu/internal/config" 10 | "github.com/aaronzjc/mu/internal/constant" 11 | "github.com/gin-gonic/gin" 12 | ) 13 | 14 | type Stat struct{} 15 | 16 | func (ctr *Stat) Online(c *gin.Context) { 17 | svcUrl := config.Get().GetServiceUrl(constant.SvcOnline) 18 | if svcUrl == "" { 19 | Resp(c, constant.CodeSuccess, "success", map[string]string{ 20 | "count": "", 21 | }) 22 | return 23 | } 24 | url := fmt.Sprintf("%s/online/%s", svcUrl, "mu") 25 | resp, err := http.Get(url) 26 | if err != nil { 27 | Resp(c, constant.CodeSuccess, "success", map[string]string{ 28 | "count": "", 29 | }) 30 | return 31 | } 32 | defer resp.Body.Close() 33 | body, err := io.ReadAll(resp.Body) 34 | if err != nil { 35 | Resp(c, constant.CodeSuccess, "success", map[string]string{ 36 | "count": "", 37 | }) 38 | return 39 | } 40 | Resp(c, constant.CodeSuccess, "success", map[string]string{ 41 | "count": string(body), 42 | }) 43 | } 44 | 45 | func (ctr *Stat) OnlineList(c *gin.Context) { 46 | onlineList := []string{} 47 | svcUrl := config.Get().GetServiceUrl(constant.SvcOnline) 48 | if svcUrl == "" { 49 | Resp(c, constant.CodeSuccess, "success", map[string][]string{ 50 | "onlineList": onlineList, 51 | }) 52 | return 53 | } 54 | url := fmt.Sprintf("%s/online/%s/dump", svcUrl, "mu") 55 | resp, err := http.Get(url) 56 | if err != nil { 57 | Resp(c, constant.CodeSuccess, "success", map[string][]string{ 58 | "onlineList": onlineList, 59 | }) 60 | return 61 | } 62 | defer resp.Body.Close() 63 | body, err := io.ReadAll(resp.Body) 64 | if err != nil { 65 | Resp(c, constant.CodeSuccess, "success", map[string][]string{ 66 | "onlineList": onlineList, 67 | }) 68 | return 69 | } 70 | json.Unmarshal(body, &onlineList) 71 | Resp(c, constant.CodeSuccess, "success", map[string][]string{ 72 | "onlineList": onlineList, 73 | }) 74 | } 75 | 76 | func NewStat() *Stat { 77 | return &Stat{} 78 | } 79 | -------------------------------------------------------------------------------- /web/src/pages/admin/router/index.js: -------------------------------------------------------------------------------- 1 | import { createRouter, createWebHashHistory } from "vue-router"; 2 | 3 | import Admin from "@adm/layouts/Admin.vue"; 4 | import Home from "@adm/views/Home.vue"; 5 | import Site from "@adm/views/Site.vue"; 6 | import SiteEdit from "@adm/views/SiteEdit.vue"; 7 | import Node from "@adm/views/Node.vue"; 8 | import NodeEdit from "@adm/views/NodeEdit.vue"; 9 | import User from "@adm/views/User.vue"; 10 | import Login from "@adm/views/Login.vue"; 11 | 12 | const routes = [ 13 | { 14 | path: "/", 15 | component: Admin, 16 | redirect: "home", 17 | children: [ 18 | { 19 | path: "/home", 20 | name: "home", 21 | component: Home, 22 | }, 23 | { 24 | path: "/site", 25 | name: "site", 26 | component: Site, 27 | }, 28 | { 29 | meta: { 30 | hl: "site", 31 | }, 32 | path: "/site/edit", 33 | name: "siteEdit", 34 | component: SiteEdit, 35 | }, 36 | { 37 | path: "/node", 38 | name: "node", 39 | component: Node, 40 | }, 41 | { 42 | meta: { 43 | hl: "node", 44 | }, 45 | path: "/node/edit", 46 | name: "nodeEdit", 47 | component: NodeEdit, 48 | }, 49 | { 50 | path: "/user", 51 | name: "user", 52 | component: User, 53 | }, 54 | ], 55 | }, 56 | { 57 | path: "/login", 58 | name: "login", 59 | component: Login, 60 | }, 61 | ]; 62 | 63 | const router = createRouter({ 64 | history: createWebHashHistory(), 65 | routes, 66 | scrollBehavior(to, from, savedPostion) { 67 | return savedPostion || { top: 0 }; 68 | }, 69 | }); 70 | 71 | router.beforeEach((to, from, next) => { 72 | let token = to.query.token; 73 | if (token != "" && token != undefined && token != null) { 74 | localStorage.setItem(import.meta.env.VITE_TOKEN_KEY, token); 75 | router.replace({ path: "/" }); 76 | } else { 77 | next(); 78 | } 79 | }); 80 | 81 | export default router; 82 | -------------------------------------------------------------------------------- /internal/application/store/site_repo.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | 8 | "github.com/aaronzjc/mu/internal/application/dto" 9 | "github.com/aaronzjc/mu/internal/domain/model" 10 | "github.com/aaronzjc/mu/internal/domain/repo" 11 | "github.com/aaronzjc/mu/internal/infra/cache" 12 | "github.com/go-redis/redis" 13 | "gorm.io/gorm" 14 | ) 15 | 16 | type SiteRepoImpl struct { 17 | BaseRepoImpl 18 | cache *redis.Client 19 | } 20 | 21 | var _ repo.SiteRepo = &SiteRepoImpl{} 22 | 23 | func (r *SiteRepoImpl) Get(ctx context.Context, q *dto.Query) ([]model.Site, error) { 24 | sites := []model.Site{} 25 | if err := r.prepare(q).Find(&sites).Error; err != nil { 26 | if err.Error() != gorm.ErrRecordNotFound.Error() { 27 | return nil, errors.New("get sites err") 28 | } 29 | } 30 | return sites, nil 31 | } 32 | 33 | func (s *SiteRepoImpl) Create(ctx context.Context, site model.Site) error { 34 | return s.create(&site) 35 | } 36 | 37 | func (s *SiteRepoImpl) Update(ctx context.Context, site model.Site, data map[string]interface{}) error { 38 | return s.db.Model(&site).Updates(data).Error 39 | } 40 | 41 | func (s *SiteRepoImpl) Del(ctx context.Context, site model.Site) error { 42 | return s.db.Delete(&site).Error 43 | } 44 | 45 | func (s *SiteRepoImpl) GetNews(ctx context.Context, k string, kk string) (model.News, error) { 46 | data, err := s.cache.HGet(k, kk).Result() 47 | if err != nil { 48 | return model.News{}, errors.New("get news err") 49 | } 50 | var news model.News 51 | if err := json.Unmarshal([]byte(data), &news); err != nil { 52 | return model.News{}, errors.New("get news err") 53 | } 54 | return news, nil 55 | } 56 | 57 | func (s *SiteRepoImpl) SaveNews(ctx context.Context, site string, tag string, data string) error { 58 | redis := cache.Get() 59 | _, err := redis.HSet(site, tag, data).Result() 60 | return err 61 | } 62 | 63 | func NewSiteRepo() *SiteRepoImpl { 64 | base, _ := NewBaseImpl() 65 | cache := cache.Get() 66 | return &SiteRepoImpl{ 67 | BaseRepoImpl: base, 68 | cache: cache, 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /internal/api/middleware/auth.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "encoding/base64" 5 | "net/url" 6 | "strings" 7 | 8 | "github.com/aaronzjc/mu/internal/api/handler" 9 | "github.com/aaronzjc/mu/internal/application/dto" 10 | "github.com/aaronzjc/mu/internal/application/service" 11 | "github.com/aaronzjc/mu/internal/application/store" 12 | "github.com/aaronzjc/mu/internal/config" 13 | "github.com/aaronzjc/mu/internal/constant" 14 | "github.com/gin-gonic/gin" 15 | ) 16 | 17 | func ApiAuth(admin bool) gin.HandlerFunc { 18 | return func(ctx *gin.Context) { 19 | token := ctx.GetHeader("Authorization") 20 | if token == "" { 21 | handler.Resp(ctx, constant.CodeAuthFailed, "认证失败", nil) 22 | ctx.Abort() 23 | return 24 | } 25 | token, _ = url.QueryUnescape(token) 26 | authBytes, err := base64.StdEncoding.DecodeString(token) 27 | if err != nil { 28 | handler.Resp(ctx, constant.CodeAuthFailed, "解析失败", nil) 29 | ctx.Abort() 30 | return 31 | } 32 | segs := strings.Split(string(authBytes), ";") 33 | if len(segs) != 2 || segs[0] == "" || segs[1] == "" { 34 | handler.Resp(ctx, constant.CodeAuthFailed, "token格式有误", nil) 35 | ctx.Abort() 36 | return 37 | } 38 | 39 | username, token := segs[0], segs[1] 40 | 41 | userRepo := store.NewUserRepo() 42 | userService := service.NewUserService(userRepo) 43 | 44 | if ok := userService.VerifyToken(ctx, username, token); !ok { 45 | handler.Resp(ctx, constant.CodeAuthFailed, "禁止访问", nil) 46 | ctx.Abort() 47 | return 48 | } 49 | 50 | u, _ := userService.GetUser(ctx, &dto.Query{ 51 | Query: "`username` = ?", 52 | Args: []interface{}{username}, 53 | }) 54 | 55 | if admin { 56 | admins := []string{"everyone"} 57 | oauthConfig, ok := config.Get().OAuth[u.AuthType] 58 | if ok { 59 | admins = oauthConfig.Admins 60 | } 61 | for _, v := range admins { 62 | if u.Username != v && v != "everyone" { 63 | handler.Resp(ctx, constant.CodeForbidden, "不好意思，您没有权限。请联系管理员", nil) 64 | ctx.Abort() 65 | return 66 | } 67 | } 68 | } 69 | ctx.Set(constant.LoginKey, u.ID) 70 | ctx.Next() 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /internal/application/store/user_repo.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/aaronzjc/mu/internal/application/dto" 8 | "github.com/aaronzjc/mu/internal/domain/model" 9 | "github.com/aaronzjc/mu/internal/domain/repo" 10 | "gorm.io/gorm" 11 | ) 12 | 13 | type UserRepoImpl struct { 14 | BaseRepoImpl 15 | } 16 | 17 | var _ repo.UserRepo = &UserRepoImpl{} 18 | 19 | func (r *UserRepoImpl) GetAll(ctx context.Context) ([]model.User, error) { 20 | users := []model.User{} 21 | st := r.db.Find(&users) 22 | if st.Error != nil { 23 | return nil, errors.New("get users err") 24 | } 25 | return users, nil 26 | } 27 | 28 | func (r *UserRepoImpl) GetUser(ctx context.Context, query *dto.Query) (model.User, error) { 29 | var user model.User 30 | err := r.prepare(query).First(&user).Error 31 | if err != nil { 32 | if err.Error() == gorm.ErrRecordNotFound.Error() { 33 | return model.User{}, nil 34 | } 35 | return model.User{}, errors.New("fetch user info failed") 36 | } 37 | 38 | return user, nil 39 | } 40 | 41 | func (r *UserRepoImpl) GetUsers(ctx context.Context, query *dto.Query) ([]model.User, error) { 42 | var users []model.User 43 | err := r.prepare(query).Find(&users).Error 44 | if err != nil { 45 | return nil, errors.New("fetch user info failed") 46 | } 47 | 48 | return users, nil 49 | } 50 | 51 | func (r *UserRepoImpl) CreateUser(ctx context.Context, user model.User) error { 52 | var exist model.User 53 | var err error 54 | exist, err = r.GetUser(ctx, &dto.Query{ 55 | Query: "`username` = ?", 56 | Args: []interface{}{user.Username}, 57 | }) 58 | if err != nil { 59 | return errors.New("find user err") 60 | } 61 | if exist.ID > 0 { 62 | return errors.New("user exist") 63 | } 64 | if err = r.create(&user); err != nil { 65 | return errors.New("create user err") 66 | } 67 | return nil 68 | } 69 | 70 | func (r *UserRepoImpl) Update(ctx context.Context, user model.User, data map[string]interface{}) error { 71 | return r.db.Model(&user).Updates(data).Error 72 | } 73 | 74 | func NewUserRepo() *UserRepoImpl { 75 | base, _ := NewBaseImpl() 76 | return &UserRepoImpl{ 77 | base, 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /internal/core/site/weibo.go: -------------------------------------------------------------------------------- 1 | package site 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/PuerkitoBio/goquery" 8 | "github.com/aaronzjc/mu/pkg/helper" 9 | ) 10 | 11 | const SITE_WEIBO = "weibo" 12 | 13 | var WeiboTabs = []SiteTab{ 14 | { 15 | Tag: "hot", 16 | Url: "https://s.weibo.com/top/summary?cate=realtimehot", 17 | Name: "热搜", 18 | }, 19 | } 20 | 21 | type Weibo struct { 22 | Site 23 | } 24 | 25 | func (w *Weibo) GetSite() *Site { 26 | return &w.Site 27 | } 28 | 29 | func (w *Weibo) BuildUrl() ([]Link, error) { 30 | var list []Link 31 | for _, tab := range WeiboTabs { 32 | url := tab.Url 33 | link := Link{ 34 | Key: url, 35 | Url: url, 36 | Tag: tab.Tag, 37 | } 38 | list = append(list, link) 39 | } 40 | 41 | return list, nil 42 | } 43 | 44 | func (w *Weibo) CrawPage(link Link, headers map[string]string) (Page, error) { 45 | page, err := w.FetchData(link, nil, headers) 46 | if err != nil { 47 | return Page{}, err 48 | } 49 | var data []Hot 50 | doc := page.Doc 51 | doc.Find("tbody td.td-02").Each(func(i int, s *goquery.Selection) { 52 | link := s.Find("a").First() 53 | text, url := link.Text(), link.AttrOr("href", "#") 54 | if text == "" { 55 | return 56 | } 57 | hot := Hot{ 58 | Title: text, 59 | OriginUrl: fmt.Sprintf("%s%s", w.Root, url), 60 | Rank: 0, 61 | } 62 | hot.Key = w.FetchKey(hot.OriginUrl) 63 | if hot.Key == "" { 64 | return 65 | } 66 | data = append(data, hot) 67 | }) 68 | 69 | page.T = time.Now() 70 | page.List = data 71 | 72 | return page, nil 73 | } 74 | 75 | func (w *Weibo) FetchKey(link string) string { 76 | if link == "" { 77 | return "" 78 | } 79 | return helper.Md5(link) 80 | } 81 | 82 | func NewWeibo() *Weibo { 83 | return &Weibo{ 84 | Site{ 85 | Name: "微博", 86 | Key: SITE_WEIBO, 87 | Root: "https://s.weibo.com", 88 | Desc: "微博热搜", 89 | CrawType: CrawHtml, 90 | Tabs: WeiboTabs, 91 | }, 92 | } 93 | } 94 | 95 | var _ Spider = &Weibo{} 96 | 97 | func init() { 98 | RegistSite(SITE_WEIBO, NewWeibo()) 99 | } 100 | -------------------------------------------------------------------------------- /web/src/pages/index/scss/_page.scss: -------------------------------------------------------------------------------- 1 | /** 2 | * 全局样式 3 | */ 4 | * { 5 | -webkit-tap-highlight-color: transparent; 6 | } 7 | 8 | section.section { 9 | padding: 0px 0 0 0; 10 | } 11 | 12 | /** 13 | * 导航菜单 14 | */ 15 | .navbar { 16 | margin-bottom: 1rem; 17 | 18 | .navbar-item img { 19 | width: 28px; 20 | height: 28px; 21 | } 22 | 23 | .navbar-burger:hover { 24 | background: none; 25 | } 26 | .navbar-item:hover { 27 | background-color: unset; 28 | } 29 | .mini-navbar-opt { 30 | background: $white-ter; 31 | } 32 | // fix a bug during compile 33 | .navbar-link:not(.is-arrowless):after { 34 | border-color: #08f; 35 | } 36 | } 37 | 38 | /** 39 | * 内容区 40 | */ 41 | .content-box { 42 | .columns { 43 | margin-bottom: 0 !important; 44 | } 45 | .switch { 46 | .tabs { 47 | margin-bottom: 0.8rem; 48 | scrollbar-width: none; 49 | } 50 | .tabs::-webkit-scrollbar { 51 | display: none; 52 | } 53 | .tag { 54 | cursor: pointer; 55 | } 56 | } 57 | .hot-container { 58 | margin-top:0; 59 | min-height: 20rem; 60 | 61 | .hot-list { 62 | padding-top: 0; 63 | } 64 | } 65 | } 66 | 67 | /** 68 | * 其他杂项 69 | */ 70 | .login { 71 | margin: 4rem auto !important; 72 | height: 40%; 73 | 74 | .backidx { 75 | background: none; 76 | border:none; 77 | font-size: 0.8rem; 78 | color: #666; 79 | 80 | &:hover { 81 | cursor: pointer; 82 | } 83 | } 84 | } 85 | 86 | .tag:not(body).is-light-dark { 87 | background-color: rgba(0,0,0,0.7); 88 | color:#fff; 89 | } 90 | 91 | .hot-ts { 92 | color: $grey-dark; 93 | font-size: 0.8rem; 94 | padding: 2px 0px; 95 | } 96 | 97 | .hot-list { 98 | padding-top:0; 99 | flex-basis: unset; 100 | width: 100%; 101 | } 102 | 103 | .search-form { 104 | margin: 0 auto; 105 | padding: 0 1rem; 106 | } 107 | 108 | .copyright { 109 | font-size: 0.85rem; 110 | } 111 | 112 | .user-declare { 113 | padding: .5rem; 114 | } 115 | 116 | .backtop { 117 | padding-top:1rem; 118 | a { 119 | cursor: pointer; 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /web/src/pages/index/components/cards/Opt.vue: -------------------------------------------------------------------------------- 1 | 15 | 16 | -------------------------------------------------------------------------------- /pkg/http/client.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "strconv" 10 | "time" 11 | 12 | "github.com/aaronzjc/mu/pkg/logger" 13 | ) 14 | 15 | type HttpClient struct { 16 | client *http.Client 17 | Timeout time.Duration 18 | } 19 | 20 | var httpclient *HttpClient 21 | 22 | func init() { 23 | httpclient = &HttpClient{ 24 | client: new(http.Client), 25 | Timeout: time.Second * 3, 26 | } 27 | httpclient.client.Timeout = httpclient.Timeout 28 | } 29 | 30 | func (c *HttpClient) Do(ctx context.Context, method string, url string, params map[string]interface{}, body []byte, headers map[string]string) (*http.Response, error) { 31 | // 处理POST的参数 32 | var buf io.Reader 33 | if len(body) > 0 { 34 | buf = bytes.NewBuffer(body) 35 | } 36 | 37 | req, _ := http.NewRequestWithContext(ctx, method, url, buf) 38 | 39 | // 补充headers 40 | for k, v := range headers { 41 | req.Header.Add(k, v) 42 | } 43 | 44 | // 格式化GET的参数 45 | pasrseUrlParams(req, params) 46 | 47 | // 发送请求&记录请求耗时 48 | var err error 49 | var resp *http.Response 50 | start := time.Now().UnixMilli() 51 | resp, err = c.client.Do(req) 52 | ts, _ := strconv.ParseFloat(fmt.Sprintf("%.3f", (float64)(time.Now().UnixMilli()-start)/1000), 64) 53 | logger.Request(req, resp, ts, err) 54 | if err != nil { 55 | return nil, err 56 | } 57 | defer resp.Body.Close() 58 | 59 | return req.Response, nil 60 | } 61 | 62 | func (c *HttpClient) Get(ctx context.Context, url string, params map[string]interface{}) (string, error) { 63 | resp, err := c.Do(ctx, "GET", url, params, nil, nil) 64 | if err != nil { 65 | return "", err 66 | } 67 | body, err := io.ReadAll(resp.Body) 68 | if err != nil { 69 | return "", err 70 | } 71 | return string(body), nil 72 | } 73 | 74 | func pasrseUrlParams(req *http.Request, params map[string]interface{}) { 75 | if len(params) == 0 { 76 | return 77 | } 78 | query := req.URL.Query() 79 | for k, v := range params { 80 | switch v := v.(type) { 81 | case string: 82 | query.Add(k, v) 83 | case int: 84 | query.Add(k, strconv.Itoa(v)) 85 | case float64: 86 | query.Add(k, strconv.FormatFloat(v, 'f', -1, 64)) 87 | } 88 | } 89 | req.URL.RawQuery = query.Encode() 90 | } 91 | -------------------------------------------------------------------------------- /internal/core/site/hacker.go: -------------------------------------------------------------------------------- 1 | package site 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/PuerkitoBio/goquery" 7 | "github.com/aaronzjc/mu/pkg/helper" 8 | ) 9 | 10 | const SITE_HACKER = "hacker" 11 | 12 | var HackerTabs = []SiteTab{ 13 | { 14 | Tag: "new", 15 | Name: "最新", 16 | Url: "https://news.ycombinator.com/", 17 | }, 18 | { 19 | Tag: "show", 20 | Name: "作品展示", 21 | Url: "https://news.ycombinator.com/shownew", 22 | }, 23 | } 24 | 25 | type Hacker struct { 26 | Site 27 | } 28 | 29 | func (h *Hacker) GetSite() *Site { 30 | return &h.Site 31 | } 32 | 33 | func (h *Hacker) BuildUrl() ([]Link, error) { 34 | var list []Link 35 | for _, tab := range HackerTabs { 36 | url := tab.Url 37 | link := Link{ 38 | Key: url, 39 | Url: url, 40 | Tag: tab.Tag, 41 | } 42 | list = append(list, link) 43 | } 44 | 45 | return list, nil 46 | } 47 | 48 | func (h *Hacker) CrawPage(link Link, headers map[string]string) (Page, error) { 49 | page, err := h.FetchData(link, nil, headers) 50 | if err != nil { 51 | return Page{}, err 52 | } 53 | var data []Hot 54 | doc := page.Doc 55 | doc.Find(".athing").Each(func(i int, s *goquery.Selection) { 56 | ele := s.Find(".title").Find("a").First() 57 | url, _ := ele.Attr("href") 58 | text := ele.Text() 59 | if text == "" || url == "" { 60 | return 61 | } 62 | hot := Hot{ 63 | Title: text, 64 | OriginUrl: url, 65 | } 66 | hot.Key = h.FetchKey(hot.OriginUrl) 67 | if h.Key == "" { 68 | return 69 | } 70 | data = append(data, hot) 71 | }) 72 | 73 | page.T = time.Now() 74 | page.List = data 75 | 76 | return page, nil 77 | } 78 | 79 | func (h *Hacker) FetchKey(link string) string { 80 | if link == "" { 81 | return "" 82 | } 83 | return helper.Md5(link) 84 | } 85 | 86 | func NewHacker() *Hacker { 87 | return &Hacker{ 88 | Site{ 89 | Name: "Hacker", 90 | Key: SITE_HACKER, 91 | Root: "https://news.ycombinator.com/", 92 | Desc: "Hacker News", 93 | CrawType: CrawHtml, 94 | Tabs: HackerTabs, 95 | }, 96 | } 97 | } 98 | 99 | var _ Spider = &Hacker{} 100 | 101 | func init() { 102 | RegistSite(SITE_HACKER, NewHacker()) 103 | } 104 | -------------------------------------------------------------------------------- /mage.go: -------------------------------------------------------------------------------- 1 | //go:build mage 2 | 3 | package main 4 | 5 | import ( 6 | "github.com/magefile/mage/sh" 7 | ) 8 | 9 | // run build web 10 | func Build_frontend() error { 11 | err := sh.RunV("go", "run", "./scripts/dagger.go", "frontend") 12 | if err != nil { 13 | return err 14 | } 15 | return nil 16 | } 17 | 18 | // run build backend 19 | func Build_backend() error { 20 | err := sh.RunV("go", "run", "./scripts/dagger.go", "backend") 21 | if err != nil { 22 | return err 23 | } 24 | return nil 25 | } 26 | 27 | // run build & release image 28 | func Build_image() error { 29 | err := sh.RunV("go", "run", "./scripts/dagger.go", "image") 30 | if err != nil { 31 | return err 32 | } 33 | return nil 34 | } 35 | 36 | // run deployment of my k3s server 37 | func Deploy() error { 38 | err := sh.RunV("go", "run", "./scripts/dagger.go", "deploy") 39 | if err != nil { 40 | return err 41 | } 42 | return nil 43 | } 44 | 45 | // run gen mock test data 46 | func Gen_mock() error { 47 | // mockery --dir ./internal/domain/repo --name [a-z]+Repo --output ./internal/mocks --outpkg mocks --case underscore --with-expecter 48 | mocks := map[string]string{ 49 | "Repo": "./internal/domain/repo", 50 | "Service": "./internal/application/service", 51 | } 52 | for name, dir := range mocks { 53 | err := sh.RunV( 54 | "mockery", 55 | "--dir", dir, 56 | "--name", "[a-z]+"+name, 57 | "--output", "./internal/mocks", 58 | "--outpkg", "mocks", 59 | "--case", "underscore", 60 | "--with-expecter", 61 | ) 62 | if err != nil { 63 | return err 64 | } 65 | } 66 | return nil 67 | } 68 | 69 | // run gen proto files 70 | func Gen_proto() error { 71 | inputs := []string{"commander.proto", "agent.proto"} // os.Exec不支持通配符，只能手动了 72 | for _, v := range inputs { 73 | err := sh.RunV( 74 | "protoc", 75 | "--go_out=.", 76 | "--go_opt=paths=source_relative", 77 | "--go-grpc_out=.", 78 | "--go-grpc_opt=paths=source_relative", 79 | "./internal/pb/"+v, 80 | ) 81 | if err != nil { 82 | return err 83 | } 84 | } 85 | 86 | return nil 87 | } 88 | 89 | // run tests 90 | func Test() error { 91 | err := sh.RunV( 92 | "go", "test", "-cover", "-coverprofile=coverage.out", "./...", 93 | ) 94 | if err != nil { 95 | return err 96 | } 97 | return nil 98 | } 99 | -------------------------------------------------------------------------------- /pkg/logger/log.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "net/http" 5 | "os" 6 | "runtime" 7 | 8 | "github.com/aaronzjc/mu/pkg/helper" 9 | "github.com/sirupsen/logrus" 10 | ) 11 | 12 | const ( 13 | TYPE_COMMON = "common" 14 | TYPE_REQ = "request" 15 | ) 16 | 17 | type AppLogger struct { 18 | logger *logrus.Entry 19 | } 20 | 21 | var appLogger *AppLogger 22 | 23 | func SetLevel(l string) { 24 | level, _ := logrus.ParseLevel(l) 25 | appLogger.logger.Logger.SetLevel(level) 26 | } 27 | 28 | func Fatal(args ...interface{}) { 29 | appLogger.logger.Fatal(args...) 30 | } 31 | 32 | func Info(args ...interface{}) { 33 | appLogger.logger.Info(args...) 34 | } 35 | 36 | func Debug(args ...interface{}) { 37 | appLogger.logger.Debug(args...) 38 | } 39 | 40 | func Error(args ...interface{}) { 41 | appLogger.logger.Error(args...) 42 | } 43 | 44 | func ErrorWithStack(args ...interface{}) { 45 | stack := make([]byte, 2048) 46 | stack = stack[:runtime.Stack(stack, true)] 47 | appLogger.logger.WithFields(logrus.Fields{ 48 | "stack": string(stack), 49 | }).Error(args...) 50 | } 51 | 52 | func init() { 53 | appLogger = &AppLogger{ 54 | logger: logrus.NewEntry(logrus.New()), 55 | } 56 | } 57 | 58 | func Request(req *http.Request, resp *http.Response, ts float64, err error) { 59 | fields := logrus.Fields{ 60 | "consume": ts, 61 | } 62 | if req != nil { 63 | fields["req_host"] = req.URL.Host 64 | fields["req_path"] = req.URL.Path 65 | fields["req_params"] = req.URL.RawQuery 66 | } 67 | if resp != nil { 68 | fields["resp_code"] = resp.StatusCode 69 | } 70 | if err != nil { 71 | fields["err"] = err.Error() 72 | appLogger.logger.WithFields(fields).Error() 73 | } else { 74 | appLogger.logger.WithFields(fields).Info() 75 | } 76 | } 77 | 78 | func Setup(appName string, path string) error { 79 | log := logrus.New() 80 | log.SetFormatter(&logrus.JSONFormatter{}) 81 | log.SetOutput(os.Stdout) 82 | if path != "" { 83 | if f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755); err != nil { 84 | return err 85 | } else { 86 | log.SetOutput(f) 87 | } 88 | } 89 | appLogger.logger = log.WithFields(logrus.Fields{ 90 | "app_name": appName, 91 | "hostname": helper.LocalHostname(), 92 | "ip": helper.LocalAddr(), 93 | }) 94 | return nil 95 | } 96 | -------------------------------------------------------------------------------- /internal/application/service/oauth.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | "github.com/aaronzjc/mu/internal/application/dto" 8 | "github.com/aaronzjc/mu/internal/config" 9 | "github.com/aaronzjc/mu/pkg/oauth" 10 | ) 11 | 12 | type OAuthService interface { 13 | Platforms(string) []dto.OAuthPlatform 14 | GetPlatform(string) oauth.OAuth 15 | Redirect(string) string 16 | Auth(string, string) (oauth.User, error) 17 | } 18 | 19 | type OAuthServiceImpl struct { 20 | } 21 | 22 | var _ OAuthService = &OAuthServiceImpl{} 23 | 24 | func (s *OAuthServiceImpl) GetPlatform(t string) oauth.OAuth { 25 | conf, ok := config.Get().OAuth[t] 26 | if !ok { 27 | return nil 28 | } 29 | callback := fmt.Sprintf("%s%s", config.Get().ServerUrl(), "/auth/callback") 30 | if t == oauth.OauthGithub { 31 | return oauth.NewGithubOauth(conf.ClientId, conf.ClientSecret, callback) 32 | } 33 | if t == oauth.OauthWeibo { 34 | return oauth.NewWeiboOauth(conf.ClientId, conf.ClientSecret, callback) 35 | } 36 | return nil 37 | } 38 | func (s *OAuthServiceImpl) Platforms(from string) []dto.OAuthPlatform { 39 | path := "/auth/redirect" // 重定向地址 40 | cnf := config.Get() 41 | return []dto.OAuthPlatform{ 42 | { 43 | Name: "Github登录", 44 | Type: oauth.OauthGithub, 45 | Url: fmt.Sprintf("%s%s?from=%s&by=%s", cnf.ServerUrl(), path, from, oauth.OauthGithub), 46 | }, 47 | { 48 | Name: "微博登录", 49 | Type: oauth.OauthWeibo, 50 | Url: fmt.Sprintf("%s%s?from=%s&by=%s", cnf.ServerUrl(), path, from, oauth.OauthWeibo), 51 | }, 52 | } 53 | } 54 | 55 | func (s *OAuthServiceImpl) Redirect(t string) string { 56 | var a oauth.OAuth 57 | if a = s.GetPlatform(t); a == nil { 58 | return "" 59 | } 60 | return a.RedirectAuth() 61 | } 62 | 63 | func (s *OAuthServiceImpl) Auth(t string, code string) (oauth.User, error) { 64 | var a oauth.OAuth 65 | if a = s.GetPlatform(t); a == nil { 66 | return oauth.User{}, errors.New("invalid platform") 67 | } 68 | accessToken, err := a.RequestAccessToken(code) 69 | if err != nil { 70 | return oauth.User{}, errors.New("get access token err") 71 | } 72 | usr, err := a.RequestUser(accessToken) 73 | if err != nil { 74 | return oauth.User{}, errors.New("get oauth user err") 75 | } 76 | return usr, nil 77 | } 78 | 79 | func NewOAuthService() *OAuthServiceImpl { 80 | return &OAuthServiceImpl{} 81 | } 82 | -------------------------------------------------------------------------------- /web/src/pages/index/scss/_cards.scss: -------------------------------------------------------------------------------- 1 | .hot { 2 | width: 100%; 3 | min-height: 2rem; 4 | height: auto; 5 | display: flex; 6 | flex-direction: row; 7 | 8 | &:not(:first-child) { 9 | margin: 0.5rem 0; 10 | } 11 | .hot-opt { 12 | padding-left: 3px; 13 | display: flex; 14 | align-items: center; 15 | 16 | &:hover { 17 | cursor: pointer; 18 | } 19 | } 20 | 21 | .divider { 22 | width: 2px; 23 | margin: 10px 4px; 24 | background: $grey-light; 25 | } 26 | 27 | &:hover { 28 | .divider { 29 | background: $grey-dark; 30 | } 31 | } 32 | } 33 | 34 | .hot-item { 35 | width: 100%; 36 | margin-right: 2px; 37 | display: flex; 38 | flex-direction: column; 39 | align-items: flex-start; 40 | word-break: break-word; 41 | justify-content: center; 42 | } 43 | 44 | .card0 { 45 | width: 100%; 46 | margin-right: 2px; 47 | word-break: break-word; 48 | } 49 | 50 | .card1 { 51 | .hot-item { 52 | align-items: flex-start; 53 | flex-direction: column; 54 | 55 | .hot-desc { 56 | padding: 2px 0; 57 | p { 58 | font-size: 0.8rem; 59 | } 60 | } 61 | } 62 | } 63 | 64 | .card2 { 65 | color: $grey-dark; 66 | .hot-item { 67 | &:hover { 68 | cursor: pointer; 69 | } 70 | display: flex; 71 | flex-direction: row; 72 | justify-content: left; 73 | align-items: center; 74 | .hot-cover { 75 | margin-right: 8px; 76 | position: relative; 77 | img { 78 | border-radius: 4px 4px; 79 | width: 120px; 80 | display: block; 81 | } 82 | .rank { 83 | display: block; 84 | width: 24px; 85 | height: 16px; 86 | line-height: 16px; 87 | text-align: center; 88 | font-size: 8px; 89 | background: #ffaf24; 90 | color:#fff; 91 | position: absolute; 92 | top: 0; 93 | left: 0; 94 | } 95 | } 96 | .hot-content { 97 | height: 100%; 98 | display: flex; 99 | flex-direction: column; 100 | align-items: flex-start; 101 | justify-content: space-between; 102 | .hot-ext { 103 | font-size: .8rem; 104 | p { 105 | color: #666; 106 | } 107 | span { 108 | color: #096; 109 | } 110 | } 111 | } 112 | } 113 | } -------------------------------------------------------------------------------- /internal/core/site/tieba.go: -------------------------------------------------------------------------------- 1 | package site 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "time" 7 | 8 | "github.com/PuerkitoBio/goquery" 9 | ) 10 | 11 | const SITE_TIEBA = "tieba" 12 | 13 | var TiebaTabs = []SiteTab{ 14 | { 15 | Tag: "beiguo", 16 | Name: "抗压背锅吧", 17 | Url: "https://tieba.baidu.com/f?ie=utf-8&kw=抗压背锅&fr=search", 18 | }, 19 | { 20 | Tag: "ruozhi", 21 | Name: "弱智吧", 22 | Url: "https://tieba.baidu.com/f?ie=utf-8&kw=弱智&fr=search", 23 | }, 24 | } 25 | 26 | type Tieba struct { 27 | Site 28 | } 29 | 30 | func (t *Tieba) GetSite() *Site { 31 | return &t.Site 32 | } 33 | 34 | func (t *Tieba) BuildUrl() ([]Link, error) { 35 | var list []Link 36 | for _, tab := range TiebaTabs { 37 | url := tab.Url 38 | link := Link{ 39 | Key: url, 40 | Url: url, 41 | Tag: tab.Tag, 42 | } 43 | list = append(list, link) 44 | } 45 | 46 | return list, nil 47 | } 48 | 49 | func (t *Tieba) CrawPage(link Link, headers map[string]string) (Page, error) { 50 | page, err := t.FetchData(link, nil, nil) 51 | if err != nil { 52 | return Page{}, err 53 | } 54 | var data []Hot 55 | doc := page.Doc 56 | doc.Find(".tl_shadow_new").Each(func(i int, s *goquery.Selection) { 57 | num := s.Find(".btn_icon").Text() 58 | url, _ := s.Find(".j_common").Attr("href") 59 | text := s.Find(".j_common").Find(".ti_title span").Text() 60 | if text == "" || url == "" { 61 | return 62 | } 63 | if num == "" { 64 | num = "0" 65 | } 66 | hot := Hot{ 67 | Title: fmt.Sprintf("%s - %s", text, num), 68 | OriginUrl: fmt.Sprintf("%s%s", t.Root, url), 69 | } 70 | hot.Key = t.FetchKey(hot.OriginUrl) 71 | if t.Key == "" { 72 | return 73 | } 74 | data = append(data, hot) 75 | }) 76 | page.T = time.Now() 77 | page.List = data 78 | 79 | return page, nil 80 | } 81 | 82 | func (t *Tieba) FetchKey(link string) string { 83 | reg := regexp.MustCompile(`.*/p/(\d+).*`) 84 | id := reg.ReplaceAllString(link, "$1") 85 | return id 86 | } 87 | 88 | func NewTieba() *Tieba { 89 | return &Tieba{ 90 | Site{ 91 | Name: "贴吧", 92 | Key: SITE_TIEBA, 93 | Root: "https://tieba.baidu.com", 94 | Desc: "鱼龙混杂的社区", 95 | CrawType: CrawHtml, 96 | Tabs: TiebaTabs, 97 | }, 98 | } 99 | } 100 | 101 | var _ Spider = &Tieba{} 102 | 103 | func init() { 104 | RegistSite(SITE_TIEBA, NewTieba()) 105 | } 106 | -------------------------------------------------------------------------------- /internal/core/site/v2ex.go: -------------------------------------------------------------------------------- 1 | package site 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/PuerkitoBio/goquery" 10 | ) 11 | 12 | const SITE_V2EX = "v2ex" 13 | 14 | var V2exTabs = []SiteTab{ 15 | { 16 | Tag: "all", 17 | Name: "全部", 18 | }, 19 | { 20 | Tag: "hot", 21 | Name: "最热", 22 | }, 23 | } 24 | 25 | type V2ex struct { 26 | Site 27 | } 28 | 29 | func (v *V2ex) GetSite() *Site { 30 | return &v.Site 31 | } 32 | 33 | func (v *V2ex) BuildUrl() ([]Link, error) { 34 | f := func(site string, tab string) string { 35 | return fmt.Sprintf("%s/?tab=%s", site, tab) 36 | } 37 | 38 | var list []Link 39 | for _, tab := range V2exTabs { 40 | url := f(v.Root, tab.Tag) 41 | link := Link{ 42 | Key: url, 43 | Url: url, 44 | Tag: tab.Tag, 45 | } 46 | list = append(list, link) 47 | } 48 | 49 | return list, nil 50 | } 51 | 52 | func (v *V2ex) CrawPage(link Link, headers map[string]string) (Page, error) { 53 | page, err := v.FetchData(link, nil, nil) 54 | if err != nil { 55 | return Page{}, err 56 | } 57 | var data []Hot 58 | doc := page.Doc 59 | doc.Find(".cell tr").Each(func(i int, s *goquery.Selection) { 60 | url, _ := s.Find(".item_title").Find("a").Attr("href") 61 | text := s.Find(".item_title").Find("a").Text() 62 | comment := s.Find(".count_livid").Text() 63 | if text == "" || url == "" { 64 | return 65 | } 66 | if comment == "" { 67 | comment = "0" 68 | } 69 | h := Hot{ 70 | Title: text, 71 | OriginUrl: fmt.Sprintf("%s%s", v.Root, url), 72 | Rank: (func() float64 { 73 | val, _ := strconv.ParseFloat(comment, 32) 74 | return float64(val) 75 | })(), 76 | } 77 | h.Key = v.FetchKey(h.OriginUrl) 78 | if h.Key == "" { 79 | return 80 | } 81 | data = append(data, h) 82 | }) 83 | 84 | page.T = time.Now() 85 | page.List = data 86 | 87 | return page, nil 88 | } 89 | 90 | func (v *V2ex) FetchKey(link string) string { 91 | reg := regexp.MustCompile(`.*/t/(\\d+).*`) 92 | id := reg.ReplaceAllString(link, "$1") 93 | return id 94 | } 95 | 96 | func NewV2ex() *V2ex { 97 | return &V2ex{ 98 | Site{ 99 | Name: "v2ex", 100 | Key: SITE_V2EX, 101 | Root: "https://www.v2ex.com", 102 | Desc: "way to explore", 103 | CrawType: CrawHtml, 104 | Tabs: V2exTabs, 105 | }, 106 | } 107 | } 108 | 109 | var _ Spider = &V2ex{} 110 | 111 | func init() { 112 | RegistSite(SITE_V2EX, NewV2ex()) 113 | } 114 | -------------------------------------------------------------------------------- /internal/core/site/guanggu.go: -------------------------------------------------------------------------------- 1 | package site 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/PuerkitoBio/goquery" 10 | ) 11 | 12 | const SITE_GUANGGU = "guanggu" 13 | 14 | var GuangGuTabs = []SiteTab{ 15 | { 16 | Tag: "default", 17 | Name: "默认", 18 | Url: "https://www.guozaoke.com/", 19 | }, 20 | { 21 | Tag: "latest", 22 | Name: "最新", 23 | Url: "https://www.guozaoke.com/?tab=latest", 24 | }, 25 | } 26 | 27 | type Guanggu struct { 28 | Site 29 | } 30 | 31 | func (g *Guanggu) GetSite() *Site { 32 | return &g.Site 33 | } 34 | 35 | func (g *Guanggu) BuildUrl() ([]Link, error) { 36 | var list []Link 37 | for _, tab := range GuangGuTabs { 38 | link := Link{ 39 | Key: tab.Url, 40 | Url: tab.Url, 41 | Tag: tab.Tag, 42 | } 43 | list = append(list, link) 44 | } 45 | 46 | return list, nil 47 | } 48 | 49 | func (g *Guanggu) CrawPage(link Link, headers map[string]string) (Page, error) { 50 | page, err := g.FetchData(link, nil, nil) 51 | if err != nil { 52 | return Page{}, err 53 | } 54 | var data []Hot 55 | doc := page.Doc 56 | doc.Find(".topic-item").Each(func(i int, s *goquery.Selection) { 57 | url, _ := s.Find(".main .title").Find("a").Attr("href") 58 | text := s.Find(".main .title").Find("a").Text() 59 | comment := s.Find(".count").Find("a").Text() 60 | if text == "" || url == "" { 61 | return 62 | } 63 | if comment == "" { 64 | comment = "0" 65 | } 66 | h := Hot{ 67 | Title: text, 68 | OriginUrl: fmt.Sprintf("%s%s", g.Root, url), 69 | Rank: (func() float64 { 70 | val, _ := strconv.ParseFloat(comment, 32) 71 | return float64(val) 72 | })(), 73 | } 74 | h.Key = g.FetchKey(h.OriginUrl) 75 | if h.Key == "" { 76 | return 77 | } 78 | data = append(data, h) 79 | }) 80 | 81 | page.T = time.Now() 82 | page.List = data 83 | 84 | return page, nil 85 | } 86 | 87 | func (g *Guanggu) FetchKey(link string) string { 88 | reg := regexp.MustCompile(`.*/t/(\d+).*`) 89 | id := reg.ReplaceAllString(link, "$1") 90 | return id 91 | } 92 | 93 | func NewGuanggu() *Guanggu { 94 | return &Guanggu{ 95 | Site{ 96 | Name: "光谷", 97 | Key: SITE_GUANGGU, 98 | Root: "https://www.guozaoke.com", 99 | Desc: "武汉光谷社区", 100 | CrawType: CrawHtml, 101 | Tabs: GuangGuTabs, 102 | }, 103 | } 104 | } 105 | 106 | var _ Spider = &Guanggu{} 107 | 108 | func init() { 109 | RegistSite(SITE_GUANGGU, NewGuanggu()) 110 | } 111 | -------------------------------------------------------------------------------- /internal/core/site/zaobao.go: -------------------------------------------------------------------------------- 1 | package site 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/PuerkitoBio/goquery" 8 | "github.com/aaronzjc/mu/pkg/helper" 9 | ) 10 | 11 | const SITE_ZAOBAO = "zaobao" 12 | 13 | var ZaobaoTabs = []SiteTab{ 14 | { 15 | Tag: "focus", 16 | Url: "http://www.zaobao.com/", 17 | Name: "今日焦点", 18 | }, 19 | } 20 | 21 | type Zaobao struct { 22 | Site 23 | } 24 | 25 | func (z *Zaobao) GetSite() *Site { 26 | return &z.Site 27 | } 28 | 29 | func (z *Zaobao) BuildUrl() ([]Link, error) { 30 | var list []Link 31 | for _, tab := range ZaobaoTabs { 32 | url := tab.Url 33 | link := Link{ 34 | Key: url, 35 | Url: url, 36 | Tag: tab.Tag, 37 | } 38 | list = append(list, link) 39 | } 40 | 41 | return list, nil 42 | } 43 | 44 | func (z *Zaobao) CrawPage(link Link, headers map[string]string) (Page, error) { 45 | page, err := z.FetchData(link, nil, nil) 46 | if err != nil { 47 | return Page{}, err 48 | } 49 | var data []Hot 50 | doc := page.Doc 51 | doc.Find("#piping-hot .post-item-special p a").Each(func(i int, s *goquery.Selection) { 52 | url := s.AttrOr("href", "") 53 | text := s.Text() 54 | if text == "" || url == "" { 55 | return 56 | } 57 | h := Hot{ 58 | Title: text, 59 | OriginUrl: fmt.Sprintf("%s%s", z.Root, url), 60 | } 61 | h.Key = z.FetchKey(h.OriginUrl) 62 | if h.Key == "" { 63 | return 64 | } 65 | data = append(data, h) 66 | }) 67 | doc.Find("#piping-hot a").Each(func(i int, s *goquery.Selection) { 68 | url := s.AttrOr("href", "") 69 | text := s.Find("span.post-title").Text() 70 | if text == "" || url == "" { 71 | return 72 | } 73 | h := Hot{ 74 | Title: text, 75 | OriginUrl: fmt.Sprintf("%s%s", z.Root, url), 76 | } 77 | h.Key = z.FetchKey(h.OriginUrl) 78 | if h.Key == "" { 79 | return 80 | } 81 | data = append(data, h) 82 | }) 83 | 84 | page.T = time.Now() 85 | page.List = data 86 | 87 | return page, nil 88 | } 89 | 90 | func (z *Zaobao) FetchKey(link string) string { 91 | if link == "" { 92 | return "" 93 | } 94 | return helper.Md5(link) 95 | } 96 | 97 | func NewZaobao() *Zaobao { 98 | return &Zaobao{ 99 | Site{ 100 | Name: "联合早报", 101 | Key: SITE_ZAOBAO, 102 | Root: "http://www.zaobao.com", 103 | Desc: "新加坡新闻", 104 | CrawType: CrawHtml, 105 | Tabs: ZaobaoTabs, 106 | }, 107 | } 108 | } 109 | 110 | var _ Spider = &Zaobao{} 111 | 112 | func init() { 113 | RegistSite(SITE_ZAOBAO, NewZaobao()) 114 | } 115 | -------------------------------------------------------------------------------- /internal/core/site/chouti.go: -------------------------------------------------------------------------------- 1 | package site 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/aaronzjc/mu/pkg/helper" 9 | "github.com/aaronzjc/mu/pkg/logger" 10 | ) 11 | 12 | const SITE_CT = "chouti" 13 | 14 | var ChoutiTabs = []SiteTab{ 15 | { 16 | Url: "/link/hot", 17 | Tag: "hot", 18 | Name: "新热榜", 19 | }, 20 | { 21 | Url: "/top/24hr", 22 | Tag: "24hr", 23 | Name: "24小时最热", 24 | }, 25 | { 26 | Url: "/top/72hr", 27 | Tag: "72hr", 28 | Name: "3天最热", 29 | }, 30 | } 31 | 32 | type ChoutiList struct { 33 | Data []map[string]interface{} `json:"data"` 34 | Code int `json:"code"` 35 | Success bool `json:"success"` 36 | } 37 | 38 | type Chouti struct { 39 | Site 40 | } 41 | 42 | func (c *Chouti) GetSite() *Site { 43 | return &c.Site 44 | } 45 | 46 | func (c *Chouti) BuildUrl() ([]Link, error) { 47 | var list []Link 48 | for _, item := range ChoutiTabs { 49 | link := Link{ 50 | Key: item.Url, 51 | Url: fmt.Sprintf("%s%s", c.Root, item.Url), 52 | Tag: item.Tag, 53 | } 54 | 55 | list = append(list, link) 56 | } 57 | 58 | return list, nil 59 | } 60 | 61 | func (c *Chouti) CrawPage(link Link, headers map[string]string) (Page, error) { 62 | page, err := c.FetchData(link, nil, nil) 63 | if err != nil { 64 | return Page{}, err 65 | } 66 | 67 | var list ChoutiList 68 | if err := json.Unmarshal([]byte(page.Content), &list); err != nil { 69 | logger.Error(err.Error()) 70 | return Page{}, err 71 | } 72 | page.Json = list.Data 73 | 74 | var data []Hot 75 | for _, v := range page.Json { 76 | h := Hot{ 77 | Title: v["title"].(string), 78 | OriginUrl: v["originalUrl"].(string), 79 | Rank: v["score"].(float64), 80 | } 81 | h.Key = c.FetchKey(h.OriginUrl) 82 | if h.Key == "" { 83 | continue 84 | } 85 | data = append(data, h) 86 | } 87 | 88 | page.T = time.Now() 89 | page.List = data 90 | 91 | return page, nil 92 | } 93 | 94 | func (c *Chouti) FetchKey(link string) string { 95 | if link == "" { 96 | return "" 97 | } 98 | return helper.Md5(link) 99 | } 100 | 101 | func NewChouti() *Chouti { 102 | return &Chouti{ 103 | Site{ 104 | Name: "抽屉", 105 | Key: SITE_CT, 106 | Root: "https://dig.chouti.com", 107 | Desc: "抽屉新热榜", 108 | CrawType: CrawApi, 109 | Tabs: ChoutiTabs, 110 | }, 111 | } 112 | } 113 | 114 | var _ Spider = &Chouti{} 115 | 116 | func init() { 117 | RegistSite(SITE_CT, NewChouti()) 118 | } 119 | -------------------------------------------------------------------------------- /internal/core/site/github.go: -------------------------------------------------------------------------------- 1 | package site 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "time" 7 | 8 | "github.com/PuerkitoBio/goquery" 9 | "github.com/aaronzjc/mu/pkg/helper" 10 | ) 11 | 12 | const SITE_GITHUB = "github" 13 | 14 | var GithubTabs = []SiteTab{ 15 | { 16 | Tag: "trending", 17 | Url: "https://github.com/trending", 18 | Name: "Trending", 19 | }, 20 | { 21 | Tag: "trending-php", 22 | Url: "https://github.com/trending/php?since=daily", 23 | Name: "Trending-PHP", 24 | }, 25 | { 26 | Tag: "trending-go", 27 | Url: "https://github.com/trending/go?since=daily", 28 | Name: "Trending-Go", 29 | }, 30 | } 31 | 32 | type Github struct { 33 | Site 34 | } 35 | 36 | func (g *Github) GetSite() *Site { 37 | return &g.Site 38 | } 39 | 40 | func (g *Github) BuildUrl() ([]Link, error) { 41 | var list []Link 42 | for _, tab := range GithubTabs { 43 | url := tab.Url 44 | link := Link{ 45 | Key: url, 46 | Url: url, 47 | Tag: tab.Tag, 48 | } 49 | list = append(list, link) 50 | } 51 | 52 | return list, nil 53 | } 54 | 55 | func (g *Github) CrawPage(link Link, headers map[string]string) (Page, error) { 56 | page, err := g.FetchData(link, nil, nil) 57 | if err != nil { 58 | return Page{}, err 59 | } 60 | var data []Hot 61 | doc := page.Doc 62 | doc.Find(".Box .Box-row").Each(func(i int, s *goquery.Selection) { 63 | url := s.Find(" h1 a").AttrOr("href", "") 64 | desc := s.Find("p").Text() 65 | desc = strings.Trim(desc, "\n ") 66 | text := url[1:] 67 | 68 | if text == "" || url == "" { 69 | return 70 | } 71 | h := Hot{ 72 | Title: strings.Replace(text, "/", " • ", 1), 73 | Desc: desc, 74 | OriginUrl: fmt.Sprintf("%s%s", g.Root, url), 75 | Card: CardRichText, 76 | } 77 | h.Key = g.FetchKey(h.OriginUrl) 78 | if h.Key == "" { 79 | return 80 | } 81 | data = append(data, h) 82 | }) 83 | 84 | page.T = time.Now() 85 | page.List = data 86 | 87 | return page, nil 88 | } 89 | 90 | func (g *Github) FetchKey(link string) string { 91 | if link == "" { 92 | return "" 93 | } 94 | return helper.Md5(link) 95 | } 96 | 97 | func NewGithub() *Github { 98 | return &Github{ 99 | Site{ 100 | Name: "Github", 101 | Key: SITE_GITHUB, 102 | Root: "https://github.com", 103 | Desc: "Github.com", 104 | CrawType: CrawHtml, 105 | Tabs: GithubTabs, 106 | }, 107 | } 108 | } 109 | 110 | var _ Spider = &Github{} 111 | 112 | func init() { 113 | RegistSite(SITE_GITHUB, NewGithub()) 114 | } 115 | -------------------------------------------------------------------------------- /internal/api.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "os" 9 | "os/signal" 10 | "syscall" 11 | "time" 12 | 13 | "github.com/aaronzjc/mu/internal/api" 14 | "github.com/aaronzjc/mu/internal/application/service" 15 | "github.com/aaronzjc/mu/internal/application/store" 16 | "github.com/aaronzjc/mu/internal/config" 17 | "github.com/aaronzjc/mu/internal/infra/cache" 18 | "github.com/aaronzjc/mu/internal/infra/db" 19 | "github.com/aaronzjc/mu/pkg/logger" 20 | 21 | "github.com/gin-gonic/gin" 22 | "github.com/urfave/cli" 23 | "gorm.io/gorm" 24 | ) 25 | 26 | func SetupApi(ctx *cli.Context) error { 27 | var err error 28 | if ctx.String("config") == "" { 29 | return errors.New("invalid config option, use -h get full doc") 30 | } 31 | // 初始化项目配置 32 | configFile := ctx.String("config") 33 | if _, err := os.Stat(configFile); os.IsNotExist(err) { 34 | return errors.New("config file not found") 35 | } 36 | var conf *config.Config 37 | if conf, err = config.LoadConfig(configFile); err != nil { 38 | return err 39 | } 40 | // 初始化日志组件 41 | if err = logger.Setup(conf.Name, conf.Log.File); err != nil { 42 | return err 43 | } 44 | // 初始化DB 45 | if err = db.Setup(conf, &gorm.Config{}); err != nil { 46 | return err 47 | } 48 | // 初始化缓存 49 | if err = cache.Setup(&conf.Redis); err != nil { 50 | return err 51 | } 52 | 53 | // 初始化网站配置 54 | service.NewSiteService(store.NewSiteRepo(), nil).Init(context.Background()) 55 | 56 | // 设置调试模式 57 | if conf.Env != "prod" { 58 | logger.SetLevel(conf.Log.Level) 59 | gin.SetMode(gin.DebugMode) 60 | } 61 | return nil 62 | } 63 | 64 | func RunApi(ctx *cli.Context) error { 65 | conf := config.Get() 66 | var addr string 67 | if conf.Env != "prod" { 68 | addr = fmt.Sprintf("127.0.0.1:%d", conf.Http.Port) 69 | } else { 70 | addr = fmt.Sprintf(":%d", conf.Http.Port) 71 | } 72 | // 启动服务器 73 | app := gin.New() 74 | api.SetupRoute(app) 75 | server := &http.Server{ 76 | Addr: addr, 77 | Handler: app, 78 | ReadTimeout: time.Second * 5, 79 | WriteTimeout: time.Second * 10, 80 | } 81 | go server.ListenAndServe() 82 | logger.Info("[START] server listen at :", conf.Http.Port) 83 | 84 | // 监听关闭信号 85 | sig := make(chan os.Signal, 1) 86 | signal.Notify(sig, syscall.SIGQUIT, os.Interrupt, syscall.SIGTERM) 87 | <-sig 88 | 89 | // 收到关闭信号，主动回收连接 90 | ctxTimeout, cancel := context.WithTimeout(context.Background(), time.Second*10) 91 | defer cancel() 92 | if err := server.Shutdown(ctxTimeout); err != nil { 93 | logger.Error("[STOP] server shutdown error", err) 94 | return err 95 | } 96 | logger.Info("[STOP] server shutdown ok") 97 | return nil 98 | } 99 | -------------------------------------------------------------------------------- /internal/application/service/favor.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "time" 7 | 8 | "github.com/aaronzjc/mu/internal/application/dto" 9 | "github.com/aaronzjc/mu/internal/domain/model" 10 | "github.com/aaronzjc/mu/internal/domain/repo" 11 | ) 12 | 13 | type FavorService interface { 14 | UserFavors(context.Context, int, string, string) ([]*dto.Favor, error) 15 | UserFavorSites(context.Context, int, string) ([]string, error) 16 | 17 | Add(context.Context, *dto.Favor) error 18 | Del(context.Context, int, string, string) error 19 | } 20 | 21 | type FavorServiceImpl struct { 22 | repo repo.FavorRepo 23 | } 24 | 25 | func (s *FavorServiceImpl) Add(ctx context.Context, favor *dto.Favor) error { 26 | fs, _ := s.repo.Get(ctx, &dto.Query{ 27 | Query: "`user_id` = ? AND `site` = ? AND `key` = ?", 28 | Args: []interface{}{favor.UserId, favor.Site, favor.Key}, 29 | }) 30 | if len(fs) > 0 { 31 | return errors.New("重复内容") 32 | } 33 | if err := s.repo.Create(ctx, model.Favor{ 34 | UserId: favor.UserId, 35 | Key: favor.Key, 36 | Site: favor.Site, 37 | OriginUrl: favor.OriginUrl, 38 | Title: favor.Title, 39 | CreateAt: time.Now(), 40 | }); err != nil { 41 | return errors.New("添加失败") 42 | } 43 | return nil 44 | } 45 | 46 | func (s *FavorServiceImpl) Del(ctx context.Context, uid int, site string, key string) error { 47 | return s.repo.Del(ctx, model.Favor{ 48 | UserId: uid, 49 | Site: site, 50 | Key: key, 51 | }) 52 | } 53 | 54 | func (s *FavorServiceImpl) UserFavors(ctx context.Context, uid int, site string, keyword string) ([]*dto.Favor, error) { 55 | var favors []*dto.Favor 56 | q := &dto.Query{} 57 | if keyword == "" { 58 | q.Query = "`user_id` = ? AND `site` = ?" 59 | q.Args = []interface{}{uid, site} 60 | } else { 61 | q.Query = "`user_id` = ? AND `site` = ? AND `title` like ?" 62 | q.Args = []interface{}{uid, site, "%" + keyword + "%"} 63 | } 64 | 65 | mfs, _ := s.repo.Get(ctx, q) 66 | for _, v := range mfs { 67 | favors = append(favors, (&dto.Favor{}).FillByModel(v)) 68 | } 69 | 70 | return favors, nil 71 | } 72 | 73 | func (s *FavorServiceImpl) UserFavorSites(ctx context.Context, uid int, keyword string) ([]string, error) { 74 | var sites []string 75 | q := &dto.Query{} 76 | if keyword == "" { 77 | q.Query = "`user_id` = ?" 78 | q.Args = []interface{}{uid} 79 | } else { 80 | q.Query = "`user_id` = ? AND `title` like ?" 81 | q.Args = []interface{}{uid, "%" + keyword + "%"} 82 | } 83 | 84 | sites = s.repo.Sites(ctx, q) 85 | return sites, nil 86 | } 87 | 88 | func NewFavorService(repo repo.FavorRepo) *FavorServiceImpl { 89 | return &FavorServiceImpl{ 90 | repo: repo, 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /internal/api/handler/admin/node.go: -------------------------------------------------------------------------------- 1 | package admin 2 | 3 | import ( 4 | "strconv" 5 | 6 | "github.com/aaronzjc/mu/internal/api/handler" 7 | "github.com/aaronzjc/mu/internal/application/dto" 8 | "github.com/aaronzjc/mu/internal/application/service" 9 | "github.com/aaronzjc/mu/internal/application/store" 10 | "github.com/aaronzjc/mu/internal/constant" 11 | "github.com/gin-gonic/gin" 12 | ) 13 | 14 | type Node struct { 15 | svc service.NodeService 16 | } 17 | 18 | type ListForm struct { 19 | Id int `form:"id"` 20 | Keyword string `form:"keyword"` 21 | } 22 | 23 | func (n *Node) List(ctx *gin.Context) { 24 | var r ListForm 25 | if err := ctx.ShouldBindQuery(&r); err != nil { 26 | handler.Resp(ctx, constant.CodeError, "参数错误", nil) 27 | return 28 | } 29 | 30 | q := &dto.Query{} 31 | if r.Id > 0 { 32 | q.Query = "`id` = ?" 33 | q.Args = []interface{}{r.Id} 34 | } 35 | if r.Keyword != "" { 36 | q.Query = "`title` = ?" 37 | q.Args = []interface{}{r.Keyword} 38 | } 39 | nodes, err := n.svc.Get(ctx, q) 40 | if err != nil { 41 | handler.Resp(ctx, constant.CodeError, err.Error(), nil) 42 | return 43 | } 44 | 45 | handler.Resp(ctx, constant.CodeSuccess, "success", nodes) 46 | } 47 | 48 | type UpsertForm struct { 49 | ID int `form:"id"` 50 | Name string `form:"name"` 51 | Addr string `form:"addr"` 52 | Type int8 `form:"type"` 53 | Enable int8 `form:"enable"` 54 | } 55 | 56 | func (n *Node) Upsert(ctx *gin.Context) { 57 | var r UpsertForm 58 | if err := ctx.ShouldBind(&r); err != nil { 59 | handler.Resp(ctx, constant.CodeError, "参数错误", nil) 60 | return 61 | } 62 | id, err := strconv.Atoi(ctx.Param("id")) 63 | if err != nil { 64 | handler.Resp(ctx, constant.CodeError, "参数错误", nil) 65 | return 66 | } 67 | err = n.svc.Upsert(ctx, &dto.Node{ 68 | ID: id, 69 | Name: r.Name, 70 | Addr: r.Addr, 71 | Type: r.Type, 72 | Enable: r.Enable, 73 | }) 74 | if err != nil { 75 | handler.Resp(ctx, constant.CodeError, err.Error(), nil) 76 | return 77 | } 78 | handler.Resp(ctx, constant.CodeSuccess, "success", nil) 79 | } 80 | 81 | func (n *Node) Del(ctx *gin.Context) { 82 | id, err := strconv.Atoi(ctx.Param("id")) 83 | if err != nil { 84 | handler.Resp(ctx, constant.CodeError, "参数错误", nil) 85 | return 86 | } 87 | err = n.svc.Del(ctx, id) 88 | if err != nil { 89 | handler.Resp(ctx, constant.CodeError, err.Error(), nil) 90 | return 91 | } 92 | handler.Resp(ctx, constant.CodeSuccess, "success", nil) 93 | } 94 | 95 | func NewNode() *Node { 96 | repo := store.NewNodeRepo() 97 | return &Node{ 98 | svc: service.NewNodeService(repo), 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /web/src/pages/admin/views/Login.vue: -------------------------------------------------------------------------------- 1 | 38 | 39 | 72 | -------------------------------------------------------------------------------- /internal/application/dto/site.go: -------------------------------------------------------------------------------- 1 | package dto 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/aaronzjc/mu/internal/domain/model" 7 | ) 8 | 9 | type Tag struct { 10 | Key string `json:"key"` 11 | Name string `json:"name"` 12 | Enable int8 `json:"enable"` 13 | } 14 | 15 | type Header struct { 16 | Key string `json:"key"` 17 | Val string `json:"val"` 18 | } 19 | 20 | const ( 21 | CrawHtml int8 = 1 // 网站是HTML 22 | CrawApi int8 = 2 // 网站是JSON接口 23 | 24 | ByType int8 = 1 // 通过服务器类型 25 | ByHosts int8 = 2 // 服务器IPs 26 | 27 | Disable int8 = 0 // 禁用 28 | Enable int8 = 1 // 启用 29 | ) 30 | 31 | type Site struct { 32 | ID int `json:"id"` 33 | Name string `json:"name"` 34 | Key string `json:"key"` 35 | Root string `json:"root"` 36 | Desc string `json:"desc"` 37 | Tags []Tag `json:"tags"` 38 | Type int8 `json:"type"` 39 | Cron string `json:"cron"` 40 | NodeOption int8 `json:"node_option"` 41 | NodeType int8 `json:"node_type"` 42 | NodeHosts []int `json:"node_hosts"` 43 | Enable int8 `json:"enable"` 44 | ReqHeaders []Header `json:"req_headers"` 45 | } 46 | 47 | func (s *Site) FillByModel(site model.Site) *Site { 48 | s.ID = site.ID 49 | s.Name = site.Name 50 | s.Key = site.Key 51 | s.Root = site.Root 52 | s.Desc = site.Desc 53 | s.Type = site.Type 54 | s.Cron = site.Cron 55 | s.NodeOption = site.NodeOption 56 | s.NodeType = site.NodeType 57 | s.Enable = site.Enable 58 | 59 | var err error 60 | tags := []Tag{} 61 | headers := []Header{} 62 | hosts := []int{} 63 | s.Tags = []Tag{} 64 | if site.Tags != "" { 65 | if err = json.Unmarshal([]byte(site.Tags), &tags); err == nil { 66 | s.Tags = tags 67 | } 68 | } 69 | s.ReqHeaders = []Header{} 70 | if site.ReqHeaders != "" { 71 | if err = json.Unmarshal([]byte(site.ReqHeaders), &headers); err == nil { 72 | s.ReqHeaders = headers 73 | } 74 | } 75 | s.NodeHosts = []int{} 76 | if site.NodeHosts != "" { 77 | if err = json.Unmarshal([]byte(site.NodeHosts), &hosts); err == nil { 78 | s.NodeHosts = hosts 79 | } 80 | } 81 | 82 | return s 83 | } 84 | 85 | type IndexSite struct { 86 | Name string `json:"name"` 87 | Key string `json:"key"` 88 | Tags []Tag `json:"tags"` 89 | } 90 | 91 | type NewsItem struct { 92 | Key string `json:"key"` 93 | Title string `json:"title"` 94 | Desc string `json:"desc"` 95 | Rank float64 `json:"rank"` 96 | OriginUrl string `json:"origin_url"` 97 | Card uint8 `json:"card_type"` 98 | Ext map[string]string `json:"ext"` 99 | Mark bool `json:"mark"` 100 | } 101 | type News struct { 102 | T string `json:"t"` 103 | List []NewsItem `json:"list"` 104 | } 105 | -------------------------------------------------------------------------------- /internal/api/handler/auth.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/url" 7 | 8 | "github.com/aaronzjc/mu/internal/application/dto" 9 | "github.com/aaronzjc/mu/internal/application/service" 10 | "github.com/aaronzjc/mu/internal/application/store" 11 | "github.com/aaronzjc/mu/internal/config" 12 | "github.com/aaronzjc/mu/internal/constant" 13 | "github.com/gin-gonic/gin" 14 | ) 15 | 16 | type Auth struct { 17 | svc service.OAuthService 18 | usrSvc service.UserService 19 | } 20 | 21 | func (o *Auth) LoginInfo(ctx *gin.Context) { 22 | userId, ok := ctx.Get(constant.LoginKey) 23 | if !ok { 24 | Resp(ctx, constant.CodeError, "用户不存在", nil) 25 | return 26 | } 27 | user, _ := o.usrSvc.GetUser(ctx, &dto.Query{ 28 | Query: "`id` = ?", 29 | Args: []interface{}{userId.(int)}, 30 | }) 31 | Resp(ctx, constant.CodeSuccess, "success", user) 32 | } 33 | 34 | func (o *Auth) Platforms(ctx *gin.Context) { 35 | from := ctx.Query("from") 36 | if from != "admin" { 37 | from = "index" 38 | } 39 | platforms := o.svc.Platforms(from) 40 | Resp(ctx, constant.CodeSuccess, "success", platforms) 41 | } 42 | 43 | func (o *Auth) Auth(ctx *gin.Context) { 44 | from := ctx.Query("from") 45 | by := ctx.Query("by") 46 | 47 | // 设置登录后跳转 48 | SetCookies(ctx, map[string]string{ 49 | "from": from, 50 | "by": by, 51 | }, "") 52 | 53 | redirect := o.svc.Redirect(by) 54 | if redirect == "" { 55 | ctx.Abort() 56 | return 57 | } 58 | ctx.Redirect(http.StatusTemporaryRedirect, redirect) 59 | ctx.Abort() 60 | } 61 | 62 | func (o *Auth) Callback(ctx *gin.Context) { 63 | var from, by string 64 | var err error 65 | code := ctx.Query("code") 66 | from, err = ctx.Cookie("from") 67 | if err != nil || from != "admin" { 68 | // 默认跳首页 69 | from = "index" 70 | } 71 | by, err = ctx.Cookie("by") 72 | if err != nil { 73 | ctx.String(http.StatusForbidden, "get cookie err") 74 | return 75 | } 76 | 77 | if code == "" { 78 | ctx.String(http.StatusForbidden, "code缺失") 79 | return 80 | } 81 | 82 | oauthUser, err := o.svc.Auth(by, code) 83 | if err != nil { 84 | ctx.String(http.StatusForbidden, err.Error()) 85 | return 86 | } 87 | token, err := o.usrSvc.Auth(ctx, by, oauthUser) 88 | if err != nil { 89 | ctx.String(http.StatusForbidden, err.Error()) 90 | return 91 | } 92 | tokenEncode := url.QueryEscape(token) 93 | cnf := config.Get() 94 | 95 | var redirect = "" 96 | if from == "admin" { 97 | redirect = fmt.Sprintf("%s?token=%s", cnf.AdminUrl(), tokenEncode) 98 | } else { 99 | redirect = fmt.Sprintf("%s?token=%s", cnf.IndexUrl(), tokenEncode) 100 | } 101 | 102 | ctx.Redirect(http.StatusTemporaryRedirect, redirect) 103 | ctx.Abort() 104 | } 105 | 106 | func NewAuth() *Auth { 107 | return &Auth{ 108 | svc: service.NewOAuthService(), 109 | usrSvc: service.NewUserService(store.NewUserRepo()), 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/aaronzjc/mu/internal/constant" 7 | "github.com/fsnotify/fsnotify" 8 | "github.com/spf13/viper" 9 | ) 10 | 11 | type LogConfig struct { 12 | Level string `yaml:"level"` 13 | File string `yaml:"file"` 14 | } 15 | 16 | type HttpConfig struct { 17 | Tls bool `yaml:"tls"` 18 | Url string `yaml:"url"` 19 | Port int `yaml:"port"` 20 | } 21 | 22 | type CommanderConfig struct { 23 | Port int `yaml:"port"` 24 | } 25 | 26 | type RedisConfig struct { 27 | Host string `yaml:"host"` 28 | Port int `yaml:"port"` 29 | Password string `yaml:"password"` 30 | } 31 | 32 | type DbConfig struct { 33 | Host string `yaml:"host"` 34 | Port int `yaml:"port"` 35 | Username string `yaml:"username"` 36 | Password string `yaml:"password"` 37 | Charset string `yaml:"charset"` 38 | } 39 | 40 | type ServiceConfig struct { 41 | Url string `yaml:"url"` 42 | } 43 | 44 | type OAuthConfig struct { 45 | ClientId string `yaml:"clientId"` 46 | ClientSecret string `yaml:"clientSecret"` 47 | Admins []string `yaml:"admins"` 48 | } 49 | 50 | type Config struct { 51 | Name string `yaml:"name"` 52 | Env string `yaml:"env"` 53 | Salt string `yaml:"salt"` 54 | Log LogConfig `yaml:"log"` 55 | Http HttpConfig `yaml:"http"` 56 | Commander CommanderConfig `yaml:"commander"` 57 | Redis RedisConfig `yaml:"redis"` 58 | Database map[string]DbConfig `yaml:"database"` 59 | Service map[string]ServiceConfig `yaml:"service"` 60 | OAuth map[string]OAuthConfig `yaml:"oauth"` 61 | } 62 | 63 | func (c *Config) GetServiceUrl(name constant.SvcName) string { 64 | svc, ok := c.Service[string(name)] 65 | if !ok { 66 | return "" 67 | } 68 | return svc.Url 69 | } 70 | 71 | func (c *Config) ServerUrl() string { 72 | if c.Http.Tls { 73 | return "https://" + c.Http.Url 74 | } 75 | return "http://" + c.Http.Url 76 | } 77 | 78 | func (c *Config) AdminUrl() string { 79 | return c.ServerUrl() + "/admin#/" 80 | } 81 | 82 | func (c *Config) IndexUrl() string { 83 | return c.ServerUrl() + "/#/" 84 | } 85 | 86 | var ( 87 | vip *viper.Viper 88 | config *Config 89 | ) 90 | 91 | func init() { 92 | vip = viper.New() 93 | config = &Config{} 94 | } 95 | 96 | func LoadConfig(path string) (*Config, error) { 97 | //加载配置 98 | vip.SetConfigFile(path) 99 | vip.SetConfigType("yml") 100 | if err := vip.ReadInConfig(); err != nil { 101 | return nil, errors.New("read config file error") 102 | } 103 | vip.Unmarshal(&config) 104 | 105 | // 监听配置变更 106 | vip.OnConfigChange(func(in fsnotify.Event) { 107 | // do something 108 | }) 109 | vip.WatchConfig() 110 | 111 | return config, nil 112 | } 113 | 114 | func Get() *Config { 115 | return config 116 | } 117 | -------------------------------------------------------------------------------- /pkg/oauth/github.go: -------------------------------------------------------------------------------- 1 | package oauth 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "time" 10 | ) 11 | 12 | type GithubAccessToken struct { 13 | AccessToken string `json:"access_token"` 14 | Scope string `json:"scope"` 15 | TokenType string `json:"token_type"` 16 | } 17 | 18 | type GithubUser struct { 19 | ID int64 `json:"id"` 20 | Username string `json:"login"` 21 | Nickname string `json:"name"` 22 | Avatar string `json:"avatar_url"` 23 | Email string `json:"email"` 24 | Bio string `json:"bio"` 25 | } 26 | 27 | type GithubOAuth struct { 28 | clientId string 29 | clientSecret string 30 | callback string 31 | } 32 | 33 | const OauthGithub = "github" 34 | 35 | var _ OAuth = GithubOAuth{} 36 | 37 | func NewGithubOauth(clientId string, clientSecret string, callback string) *GithubOAuth { 38 | return &GithubOAuth{ 39 | clientId: clientId, 40 | clientSecret: clientSecret, 41 | callback: callback, 42 | } 43 | } 44 | 45 | func (auth GithubOAuth) Type() string { 46 | return OauthGithub 47 | } 48 | 49 | func (auth GithubOAuth) RedirectAuth() string { 50 | url := fmt.Sprintf("https://github.com/login/oauth/authorize?client_id=%s&redirect_uri=%s", auth.clientId, auth.callback) 51 | return url 52 | } 53 | 54 | func (auth GithubOAuth) RequestAccessToken(code string) (string, error) { 55 | api := "https://github.com/login/oauth/access_token" 56 | 57 | url := fmt.Sprintf("%s?client_id=%s&client_secret=%s&code=%s", api, auth.clientId, auth.clientSecret, code) 58 | client := &http.Client{ 59 | Timeout: time.Second * 3, 60 | } 61 | req, _ := http.NewRequest("GET", url, nil) 62 | req.Header.Add("Accept", "application/json") 63 | resp, err := client.Do(req) 64 | if err != nil { 65 | return "", errors.New("RequestAccessToken failed") 66 | } 67 | 68 | defer resp.Body.Close() 69 | 70 | body, _ := io.ReadAll(resp.Body) 71 | var data GithubAccessToken 72 | err = json.Unmarshal(body, &data) 73 | if err != nil || data.AccessToken == "" { 74 | return "", errors.New("RequestAccessToken decode json failed") 75 | } 76 | 77 | return data.AccessToken, nil 78 | } 79 | 80 | func (auth GithubOAuth) RequestUser(token string) (User, error) { 81 | var err error 82 | 83 | url := "https://api.github.com/user" 84 | 85 | client := &http.Client{} 86 | req, _ := http.NewRequest("GET", url, nil) 87 | req.Header.Add("Authorization", "token "+token) 88 | resp, err := client.Do(req) 89 | if err != nil { 90 | return User{}, errors.New("RequestUser failed") 91 | } 92 | 93 | defer resp.Body.Close() 94 | 95 | body, _ := io.ReadAll(resp.Body) 96 | 97 | var u GithubUser 98 | err = json.Unmarshal(body, &u) 99 | if err != nil { 100 | return User{}, err 101 | } 102 | 103 | au := User{ 104 | ID: u.ID, 105 | Username: u.Username, 106 | Nickname: u.Nickname, 107 | Avatar: u.Avatar, 108 | } 109 | 110 | return au, nil 111 | } 112 | -------------------------------------------------------------------------------- /internal/application/service/user.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "context" 5 | "encoding/base64" 6 | "errors" 7 | "fmt" 8 | "time" 9 | 10 | "github.com/aaronzjc/mu/internal/application/dto" 11 | "github.com/aaronzjc/mu/internal/config" 12 | "github.com/aaronzjc/mu/internal/domain/model" 13 | "github.com/aaronzjc/mu/internal/domain/repo" 14 | "github.com/aaronzjc/mu/internal/util" 15 | "github.com/aaronzjc/mu/pkg/oauth" 16 | ) 17 | 18 | type UserService interface { 19 | GetUserList(context.Context) ([]*dto.User, error) 20 | GetUser(context.Context, *dto.Query) (*dto.User, error) 21 | Auth(context.Context, string, oauth.User) (string, error) 22 | VerifyToken(context.Context, string, string) bool 23 | } 24 | 25 | type UserServiceImpl struct { 26 | repo repo.UserRepo 27 | } 28 | 29 | func (s *UserServiceImpl) GetUserList(ctx context.Context) ([]*dto.User, error) { 30 | userModels, err := s.repo.GetUsers(ctx, nil) 31 | if err != nil { 32 | return nil, err 33 | } 34 | var users []*dto.User 35 | for _, v := range userModels { 36 | users = append(users, (&dto.User{}).FillByModel(v)) 37 | } 38 | return users, nil 39 | } 40 | 41 | func (s *UserServiceImpl) GetUser(ctx context.Context, q *dto.Query) (*dto.User, error) { 42 | user, err := s.repo.GetUser(ctx, q) 43 | if err != nil { 44 | return nil, err 45 | } 46 | return (&dto.User{}).FillByModel(user), nil 47 | } 48 | 49 | func (s *UserServiceImpl) Auth(ctx context.Context, t string, ou oauth.User) (string, error) { 50 | exist, _ := s.repo.GetUser(ctx, &dto.Query{ 51 | Query: "`username` = ? AND `auth_type` = ?", 52 | Args: []interface{}{ou.Username, t}, 53 | }) 54 | conf := config.Get() 55 | token := util.GenerateToken(ou.Username, conf.Salt) 56 | expireAt := time.Now().Add(time.Hour * 24 * 30).Unix() 57 | if exist.ID > 0 { 58 | if err := s.repo.Update(ctx, exist, map[string]interface{}{ 59 | "token": token, 60 | "expire_at": expireAt, 61 | "auth_time": time.Now(), 62 | }); err != nil { 63 | return "", errors.New("auth update err") 64 | } 65 | } else { 66 | user := model.User{ 67 | Username: ou.Username, 68 | Nickname: ou.Nickname, 69 | Avatar: ou.Avatar, 70 | AuthType: t, 71 | AuthTime: time.Now(), 72 | Token: token, 73 | ExpireAt: expireAt, 74 | } 75 | if err := s.repo.CreateUser(ctx, user); err != nil { 76 | return "", errors.New("auth create err") 77 | } 78 | } 79 | return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s;%s", ou.Username, token))), nil 80 | } 81 | 82 | func (s *UserServiceImpl) VerifyToken(ctx context.Context, username string, token string) bool { 83 | login, _ := s.repo.GetUser(ctx, &dto.Query{ 84 | Query: "`username` = ? AND `token` = ?", 85 | Args: []interface{}{username, token}, 86 | }) 87 | if login.ID < 0 { 88 | return false 89 | } 90 | if login.ExpireAt <= time.Now().Unix() { 91 | return false 92 | } 93 | return true 94 | } 95 | 96 | func NewUserService(repo repo.UserRepo) *UserServiceImpl { 97 | return &UserServiceImpl{repo: repo} 98 | } 99 | -------------------------------------------------------------------------------- /internal/core/site/reddit.go: -------------------------------------------------------------------------------- 1 | package site 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "regexp" 7 | "time" 8 | 9 | "github.com/aaronzjc/mu/pkg/logger" 10 | ) 11 | 12 | const SITE_REDDIT = "reddit" 13 | 14 | var RedditTabs = []SiteTab{ 15 | { 16 | Tag: "AskReddit", 17 | Name: "AskReddit", 18 | Url: "AskReddit", 19 | }, 20 | { 21 | Tag: "Jokes", 22 | Name: "Jokes", 23 | Url: "Jokes", 24 | }, 25 | { 26 | Tag: "leagueoflegends", 27 | Name: "lol", 28 | Url: "leagueoflegends", 29 | }, 30 | } 31 | 32 | type RedditList struct { 33 | Posts map[string]map[string]interface{} `json:"posts"` 34 | } 35 | 36 | type Reddit struct { 37 | Site 38 | } 39 | 40 | func (r *Reddit) GetSite() *Site { 41 | return &r.Site 42 | } 43 | 44 | func (r *Reddit) BuildUrl() ([]Link, error) { 45 | str := "https://gateway.reddit.com/desktopapi/v1/subreddits/%s?rtj=only&redditWebClient=web2x&app=web2x-client-production&allow_over18=&include=prefsSubreddit&after=&dist=11&layout=card&sort=hot" 46 | 47 | var list []Link 48 | for _, tab := range RedditTabs { 49 | url := tab.Url 50 | link := Link{ 51 | Key: url, 52 | Url: fmt.Sprintf(str, url), 53 | Tag: tab.Tag, 54 | } 55 | list = append(list, link) 56 | } 57 | 58 | return list, nil 59 | } 60 | 61 | func (r *Reddit) CrawPage(link Link, headers map[string]string) (Page, error) { 62 | page, err := r.FetchData(link, nil, nil) 63 | if err != nil { 64 | return Page{}, err 65 | } 66 | 67 | var list RedditList 68 | if err := json.Unmarshal([]byte(page.Content), &list); err != nil { 69 | logger.Error(err.Error()) 70 | return Page{}, err 71 | } 72 | 73 | var listArr []map[string]interface{} 74 | re, _ := regexp.Compile(`redditads`) 75 | for k, v := range list.Posts { 76 | if ok := re.Match([]byte(v["permalink"].(string))); ok { 77 | continue 78 | } 79 | listArr = append(listArr, map[string]interface{}{ 80 | "key": k, 81 | "title": v["title"], 82 | "url": v["permalink"], 83 | "score": v["score"], 84 | }) 85 | } 86 | 87 | page.Json = listArr 88 | 89 | var data []Hot 90 | for _, v := range page.Json { 91 | h := Hot{ 92 | Title: v["title"].(string), 93 | OriginUrl: v["url"].(string), 94 | Rank: v["score"].(float64), 95 | Key: v["key"].(string), 96 | } 97 | if h.Key == "" { 98 | continue 99 | } 100 | data = append(data, h) 101 | } 102 | 103 | page.T = time.Now() 104 | page.List = data 105 | 106 | return page, nil 107 | } 108 | 109 | func (r *Reddit) FetchKey(key string) string { 110 | return key 111 | } 112 | 113 | func NewReddit() *Reddit { 114 | return &Reddit{ 115 | Site{ 116 | Name: "Reddit", 117 | Key: SITE_REDDIT, 118 | Root: "https://www.reddit.com/", 119 | Desc: "老外的贴吧", 120 | CrawType: CrawApi, 121 | Tabs: RedditTabs, 122 | }, 123 | } 124 | } 125 | 126 | var _ Spider = &Reddit{} 127 | 128 | func init() { 129 | RegistSite(SITE_REDDIT, NewReddit()) 130 | } 131 | -------------------------------------------------------------------------------- /internal/api/route.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/aaronzjc/mu/internal/api/handler" 7 | "github.com/aaronzjc/mu/internal/api/handler/admin" 8 | "github.com/aaronzjc/mu/internal/api/middleware" 9 | "github.com/gin-contrib/cors" 10 | "github.com/gin-gonic/gin" 11 | ) 12 | 13 | func SetupRoute(app *gin.Engine) { 14 | // 中间件 15 | app.Use(gin.Recovery(), gin.Logger()) 16 | app.Use(cors.New(cors.Config{ 17 | AllowOriginFunc: func(origin string) bool { return true }, 18 | AllowMethods: []string{"GET", "POST", "PUT", "DELETE", "PATCH"}, 19 | AllowHeaders: []string{"Origin", "Content-Length", "Content-Type", "Authorization"}, 20 | AllowCredentials: true, 21 | })) 22 | 23 | // auth 24 | rAuth := app.Group("/auth") 25 | { 26 | auth := handler.NewAuth() 27 | rAuth.GET("/config", auth.Platforms) 28 | rAuth.GET("/redirect", auth.Auth) 29 | rAuth.GET("/callback", auth.Callback) 30 | 31 | // 获取登录信息 32 | rAuth.Use(middleware.ApiAuth(false)).GET("/info/index", auth.LoginInfo) 33 | rAuth.Use(middleware.ApiAuth(true)).GET("/info/admin", auth.LoginInfo) 34 | } 35 | 36 | // 数据统计等 37 | statGroup := app.Group("/stat").Use(middleware.SetOnline()) 38 | { 39 | stat := handler.NewStat() 40 | statGroup.GET("/online", stat.Online) 41 | } 42 | 43 | // index页面 44 | indexGroup := app.Group("/api") 45 | { 46 | idx := handler.NewIndex() 47 | indexGroup.GET("/sites", idx.Sites) 48 | indexGroup.GET("/news", idx.News) 49 | 50 | // 需要登陆的前端页面 51 | idxAuth := indexGroup.Use(middleware.ApiAuth(false)) 52 | { 53 | // 收藏管理 54 | favor := handler.NewFavor() 55 | idxAuth.GET("/favors", favor.List) 56 | idxAuth.POST("/favors", favor.Add) 57 | idxAuth.POST("/favors/:id/del", favor.Remove) 58 | } 59 | } 60 | 61 | // admin页面 62 | adminGroup := app.Group("/admin").Use(middleware.ApiAuth(true)) 63 | { 64 | // 节点管理 65 | node := admin.NewNode() 66 | adminGroup.GET("/nodes", node.List) 67 | adminGroup.POST("/nodes/:id/upsert", node.Upsert) 68 | adminGroup.GET("/nodes/:id/del", node.Del) 69 | 70 | // 站点管理 71 | site := admin.NewSite() 72 | adminGroup.GET("/sites", site.List) 73 | adminGroup.POST("/sites/:id/upsert", site.Upsert) 74 | adminGroup.POST("/sites/:id/craw", site.Craw) 75 | 76 | // 用户管理 77 | user := handler.NewUser() 78 | adminGroup.GET("/users", user.List) 79 | 80 | // 在线人数 81 | stat := handler.NewStat() 82 | adminGroup.GET("/stats/online/list", stat.OnlineList) 83 | } 84 | 85 | // 静态资源托管 86 | RegistStatic(app) 87 | } 88 | 89 | func RegistStatic(r *gin.Engine) { 90 | r.Use(middleware.AddCacheControlHeader()) 91 | path, _ := os.Getwd() 92 | dist := "/public/" 93 | r.StaticFile("/", path+dist+"index.html") 94 | r.StaticFile("/admin", path+dist+"admin.html") 95 | for _, v := range []string{"favicon.png", "index.manifest", "sw.js"} { 96 | r.StaticFile(v, path+dist+v) 97 | } 98 | for _, v := range []string{"pwa", "assets"} { 99 | r.Static("/"+v, path+dist+""+v) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /web/src/pages/index/components/Content.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 120 | -------------------------------------------------------------------------------- /internal/application/service/craw.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "math/rand" 8 | "time" 9 | 10 | "github.com/aaronzjc/mu/internal/application/dto" 11 | "github.com/aaronzjc/mu/internal/core/rpc" 12 | "github.com/aaronzjc/mu/internal/domain/model" 13 | "github.com/aaronzjc/mu/internal/domain/repo" 14 | "github.com/aaronzjc/mu/internal/pb" 15 | "github.com/aaronzjc/mu/pkg/logger" 16 | ) 17 | 18 | type CrawService interface { 19 | PickAgent(context.Context, *dto.Site) (*dto.Node, error) 20 | Craw(context.Context, *dto.Site) error 21 | } 22 | 23 | type CrawServiceImpl struct { 24 | siteRepo repo.SiteRepo 25 | nodeRepo repo.NodeRepo 26 | } 27 | 28 | var _ CrawService = &CrawServiceImpl{} 29 | 30 | func (c *CrawServiceImpl) PickAgent(ctx context.Context, site *dto.Site) (*dto.Node, error) { 31 | rand.Seed(time.Now().UnixNano()) 32 | var nodes []model.Node 33 | 34 | q := &dto.Query{} 35 | if site.NodeOption == model.ByType { 36 | q.Query = "`type` = ? AND `ping` = ?" 37 | q.Args = []interface{}{site.NodeType, model.PingOk} 38 | } else { 39 | if len(site.NodeHosts) == 0 { 40 | return nil, errors.New("no nodes configured") 41 | } 42 | q.Query = "`id` IN (?) AND `enable` = ? AND `ping` = ?" 43 | q.Args = []interface{}{site.NodeHosts, model.Enable, model.PingOk} 44 | } 45 | nodes, err := c.nodeRepo.Get(ctx, q) 46 | if err != nil { 47 | return nil, errors.New("get nodes failed") 48 | } 49 | if len(nodes) == 0 { 50 | return nil, errors.New("no nodes avaiable") 51 | } 52 | chosen := nodes[rand.Int()%len(nodes)] 53 | return (&dto.Node{}).FillByModel(chosen), nil 54 | } 55 | 56 | func (c *CrawServiceImpl) Craw(ctx context.Context, site *dto.Site) error { 57 | picked, err := c.PickAgent(ctx, site) 58 | if err != nil { 59 | logger.Error("no agent avaiable for site ", site.Name) 60 | return errors.New("no agent avaiable") 61 | } 62 | ctx, cancel := context.WithTimeout(ctx, time.Second*5) 63 | defer cancel() 64 | 65 | rpcClient, _ := rpc.NewAgentClient(picked.Addr) 66 | 67 | // read craw config 68 | var headers []*pb.Job_Header 69 | for _, v := range site.ReqHeaders { 70 | if v.Key == "" || v.Val == "" { 71 | continue 72 | } 73 | headers = append(headers, &pb.Job_Header{ 74 | Key: v.Key, 75 | Val: v.Val, 76 | }) 77 | } 78 | 79 | // do craw 80 | var result *pb.Result 81 | if result, err = rpcClient.Craw(ctx, &pb.Job{ 82 | Name: site.Key, 83 | Headers: headers, 84 | }); err != nil { 85 | logger.Error("remote craw err " + err.Error()) 86 | return err 87 | } 88 | logger.Info("remote craw [" + site.Key + "] done") 89 | 90 | // save to cache 91 | var news dto.News 92 | news.T = result.T 93 | for tag, hotStr := range result.HotMap { 94 | json.Unmarshal([]byte(hotStr), &news.List) 95 | data, _ := json.Marshal(news) 96 | err := c.siteRepo.SaveNews(ctx, site.Key, tag, string(data)) 97 | if err != nil { 98 | logger.Error("save new err " + err.Error()) 99 | } 100 | } 101 | return nil 102 | } 103 | 104 | func NewCrawService(site repo.SiteRepo, node repo.NodeRepo) *CrawServiceImpl { 105 | return &CrawServiceImpl{ 106 | siteRepo: site, 107 | nodeRepo: node, 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/aaronzjc/mu 2 | 3 | go 1.20 4 | 5 | require ( 6 | dagger.io/dagger v0.5.1 7 | github.com/PuerkitoBio/goquery v1.8.1 8 | github.com/fsnotify/fsnotify v1.6.0 9 | github.com/gin-contrib/cors v1.4.0 10 | github.com/gin-gonic/gin v1.9.0 11 | github.com/go-redis/redis v6.15.9+incompatible 12 | github.com/magefile/mage v1.14.0 13 | github.com/robfig/cron/v3 v3.0.1 14 | github.com/sirupsen/logrus v1.9.0 15 | github.com/spf13/viper v1.15.0 16 | github.com/stretchr/testify v1.8.2 17 | github.com/urfave/cli v1.22.12 18 | google.golang.org/grpc v1.54.0 19 | google.golang.org/protobuf v1.30.0 20 | gorm.io/driver/mysql v1.4.7 21 | gorm.io/gorm v1.24.6 22 | ) 23 | 24 | require ( 25 | github.com/Khan/genqlient v0.5.0 // indirect 26 | github.com/adrg/xdg v0.4.0 // indirect 27 | github.com/andybalholm/cascadia v1.3.1 // indirect 28 | github.com/bytedance/sonic v1.8.0 // indirect 29 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect 30 | github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect 31 | github.com/davecgh/go-spew v1.1.1 // indirect 32 | github.com/gin-contrib/sse v0.1.0 // indirect 33 | github.com/go-playground/locales v0.14.1 // indirect 34 | github.com/go-playground/universal-translator v0.18.1 // indirect 35 | github.com/go-playground/validator/v10 v10.12.0 // indirect 36 | github.com/go-sql-driver/mysql v1.7.0 // indirect 37 | github.com/goccy/go-json v0.10.2 // indirect 38 | github.com/golang/protobuf v1.5.3 // indirect 39 | github.com/hashicorp/hcl v1.0.0 // indirect 40 | github.com/iancoleman/strcase v0.2.0 // indirect 41 | github.com/jinzhu/inflection v1.0.0 // indirect 42 | github.com/jinzhu/now v1.1.5 // indirect 43 | github.com/json-iterator/go v1.1.12 // indirect 44 | github.com/klauspost/cpuid/v2 v2.0.9 // indirect 45 | github.com/leodido/go-urn v1.2.2 // indirect 46 | github.com/magiconair/properties v1.8.7 // indirect 47 | github.com/mattn/go-isatty v0.0.18 // indirect 48 | github.com/mitchellh/mapstructure v1.5.0 // indirect 49 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 50 | github.com/modern-go/reflect2 v1.0.2 // indirect 51 | github.com/onsi/ginkgo v1.16.5 // indirect 52 | github.com/onsi/gomega v1.27.4 // indirect 53 | github.com/pelletier/go-toml/v2 v2.0.7 // indirect 54 | github.com/pmezard/go-difflib v1.0.0 // indirect 55 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 56 | github.com/spf13/afero v1.9.5 // indirect 57 | github.com/spf13/cast v1.5.0 // indirect 58 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 59 | github.com/spf13/pflag v1.0.5 // indirect 60 | github.com/stretchr/objx v0.5.0 // indirect 61 | github.com/subosito/gotenv v1.4.2 // indirect 62 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 63 | github.com/ugorji/go/codec v1.2.11 // indirect 64 | github.com/vektah/gqlparser/v2 v2.5.1 // indirect 65 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect 66 | golang.org/x/crypto v0.7.0 // indirect 67 | golang.org/x/net v0.8.0 // indirect 68 | golang.org/x/sync v0.1.0 // indirect 69 | golang.org/x/sys v0.6.0 // indirect 70 | golang.org/x/text v0.8.0 // indirect 71 | google.golang.org/genproto v0.0.0-20230322174352-cde4c949918d // indirect 72 | gopkg.in/ini.v1 v1.67.0 // indirect 73 | gopkg.in/yaml.v3 v3.0.1 // indirect 74 | ) 75 | -------------------------------------------------------------------------------- /internal/api/handler/favor.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "github.com/aaronzjc/mu/internal/application/dto" 5 | "github.com/aaronzjc/mu/internal/application/service" 6 | "github.com/aaronzjc/mu/internal/application/store" 7 | "github.com/aaronzjc/mu/internal/constant" 8 | "github.com/aaronzjc/mu/internal/core/site" 9 | "github.com/gin-gonic/gin" 10 | ) 11 | 12 | type Favor struct { 13 | svc service.FavorService 14 | } 15 | 16 | type ListForm struct { 17 | Site string `form:"s"` 18 | Keyword string `form:"keyword"` 19 | } 20 | 21 | func (f *Favor) List(ctx *gin.Context) { 22 | var err error 23 | var r ListForm 24 | if err = ctx.ShouldBindQuery(&r); err != nil { 25 | Resp(ctx, constant.CodeError, "参数错误", nil) 26 | return 27 | } 28 | login, exist := ctx.Get(constant.LoginKey) 29 | if !exist { 30 | Resp(ctx, constant.CodeAuthFailed, "未登录", nil) 31 | return 32 | } 33 | 34 | favorList := dto.FavorList{ 35 | Tabs: []*dto.IndexSite{}, 36 | List: []*dto.Favor{}, 37 | } 38 | 39 | sites, _ := f.svc.UserFavorSites(ctx, login.(int), r.Keyword) 40 | if len(sites) == 0 { 41 | Resp(ctx, constant.CodeSuccess, "success", favorList) 42 | return 43 | } 44 | for _, v := range sites { 45 | site, ok := site.SiteMap[v] 46 | if !ok { 47 | continue 48 | } 49 | favorList.Tabs = append(favorList.Tabs, &dto.IndexSite{ 50 | Name: site.GetSite().Name, 51 | Key: site.GetSite().Key, 52 | Tags: []dto.Tag{}, 53 | }) 54 | } 55 | 56 | site := r.Site 57 | if site == "" { 58 | site = sites[0] 59 | } 60 | favorList.List, _ = f.svc.UserFavors(ctx, login.(int), site, r.Keyword) 61 | Resp(ctx, constant.CodeSuccess, "success", favorList) 62 | } 63 | 64 | type AddForm struct { 65 | Key string `json:"key"` 66 | Site string `json:"site"` 67 | Url string `json:"url"` 68 | Title string `json:"title"` 69 | } 70 | 71 | func (f *Favor) Add(ctx *gin.Context) { 72 | var err error 73 | var r AddForm 74 | if err = ctx.ShouldBindJSON(&r); err != nil { 75 | Resp(ctx, constant.CodeError, "参数错误", nil) 76 | return 77 | } 78 | 79 | login, exist := ctx.Get(constant.LoginKey) 80 | if !exist { 81 | Resp(ctx, constant.CodeForbidden, "未登录", nil) 82 | return 83 | } 84 | 85 | err = f.svc.Add(ctx, &dto.Favor{ 86 | UserId: login.(int), 87 | Key: r.Key, 88 | Site: r.Site, 89 | OriginUrl: r.Url, 90 | Title: r.Title, 91 | }) 92 | if err != nil { 93 | Resp(ctx, constant.CodeError, err.Error(), nil) 94 | return 95 | } 96 | Resp(ctx, constant.CodeSuccess, "success", nil) 97 | } 98 | 99 | type RemoveForm struct { 100 | Site string `json:"site"` 101 | Key string `json:"key"` 102 | } 103 | 104 | func (f *Favor) Remove(ctx *gin.Context) { 105 | var err error 106 | var r RemoveForm 107 | if err = ctx.ShouldBindJSON(&r); err != nil { 108 | Resp(ctx, constant.CodeError, "参数错误", nil) 109 | return 110 | } 111 | 112 | login, exist := ctx.Get(constant.LoginKey) 113 | if !exist { 114 | Resp(ctx, constant.CodeForbidden, "未登录", nil) 115 | return 116 | } 117 | 118 | if err := f.svc.Del(ctx, login.(int), r.Site, r.Key); err != nil { 119 | Resp(ctx, constant.CodeError, "失败", nil) 120 | return 121 | } 122 | Resp(ctx, constant.CodeSuccess, "success", nil) 123 | } 124 | 125 | func NewFavor() *Favor { 126 | repo := store.NewFavorRepo() 127 | return &Favor{ 128 | svc: service.NewFavorService(repo), 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /web/src/pages/admin/views/User.vue: -------------------------------------------------------------------------------- 1 | 53 | 54 | 87 | -------------------------------------------------------------------------------- /internal/application/service/node.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/aaronzjc/mu/internal/application/dto" 10 | "github.com/aaronzjc/mu/internal/core/rpc" 11 | "github.com/aaronzjc/mu/internal/domain/model" 12 | "github.com/aaronzjc/mu/internal/domain/repo" 13 | "github.com/aaronzjc/mu/internal/pb" 14 | "github.com/aaronzjc/mu/pkg/logger" 15 | ) 16 | 17 | type NodeService interface { 18 | CheckNodes(context.Context, *rpc.RpcPool) error 19 | Upsert(context.Context, *dto.Node) error 20 | Del(context.Context, int) error 21 | Get(context.Context, *dto.Query) ([]*dto.Node, error) 22 | } 23 | 24 | type NodeServiceImpl struct { 25 | repo repo.NodeRepo 26 | } 27 | 28 | var _ NodeService = &NodeServiceImpl{} 29 | 30 | func (s *NodeServiceImpl) CheckNodes(ctx context.Context, clientPool *rpc.RpcPool) error { 31 | nodes, err := s.repo.Get(ctx, &dto.Query{ 32 | Query: "`enable` = ?", 33 | Args: []interface{}{model.Enable}, 34 | }) 35 | if err != nil { 36 | return errors.New("get nodes err") 37 | } 38 | ctx, cancel := context.WithTimeout(ctx, time.Minute) 39 | defer cancel() 40 | for _, node := range nodes { 41 | var err error 42 | var client *rpc.RpcClient 43 | if client, err = clientPool.Get(node.Addr); err != nil { 44 | if node.Ping != model.PingFail { 45 | s.repo.Update(ctx, node, map[string]interface{}{ 46 | "ping": model.PingFail, 47 | }) 48 | } 49 | continue 50 | } 51 | ping := &pb.Ping{Ping: "ping"} 52 | if result, err := (*client.Client).Check(ctx, ping); err != nil || result.Pong != ping.Ping { 53 | logger.Error(fmt.Sprintf("rpc health check : [%s] ping error, err %v.", node.Name, err)) 54 | if node.Ping != model.PingFail { 55 | s.repo.Update(ctx, node, map[string]interface{}{ 56 | "ping": model.PingFail, 57 | }) 58 | } 59 | continue 60 | } 61 | logger.Info(fmt.Sprintf("rpc health check : [%s] is online.", node.Name)) 62 | if node.Ping != model.PingOk { 63 | s.repo.Update(ctx, node, map[string]interface{}{ 64 | "ping": model.PingOk, 65 | }) 66 | } 67 | } 68 | return nil 69 | } 70 | 71 | func (s *NodeServiceImpl) Get(ctx context.Context, q *dto.Query) ([]*dto.Node, error) { 72 | var nodes []*dto.Node 73 | mns, err := s.repo.Get(ctx, q) 74 | if err != nil { 75 | return nodes, err 76 | } 77 | for _, v := range mns { 78 | nodes = append(nodes, (&dto.Node{}).FillByModel(v)) 79 | } 80 | return nodes, nil 81 | } 82 | 83 | func (s *NodeServiceImpl) Upsert(ctx context.Context, node *dto.Node) error { 84 | nodes, err := s.Get(ctx, &dto.Query{ 85 | Query: "`id` = ?", 86 | Args: []interface{}{node.ID}, 87 | }) 88 | if err != nil { 89 | return err 90 | } 91 | if len(nodes) > 0 { 92 | s.repo.Update(ctx, model.Node{ID: nodes[0].ID}, map[string]interface{}{ 93 | "name": node.Name, 94 | "addr": node.Addr, 95 | "enable": node.Enable, 96 | "type": node.Type, 97 | }) 98 | return nil 99 | } 100 | n := model.Node{ 101 | Name: node.Name, 102 | Addr: node.Addr, 103 | Type: node.Type, 104 | Enable: node.Enable, 105 | CreateAt: time.Now(), 106 | } 107 | return s.repo.Create(ctx, n) 108 | } 109 | 110 | func (s *NodeServiceImpl) Del(ctx context.Context, id int) error { 111 | return s.repo.Del(ctx, model.Node{ID: id}) 112 | } 113 | 114 | func NewNodeService(repo repo.NodeRepo) *NodeServiceImpl { 115 | return &NodeServiceImpl{ 116 | repo: repo, 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /web/src/pages/admin/styles/_theme-default.scss: -------------------------------------------------------------------------------- 1 | /* We'll need some initial vars to use here */ 2 | @import 'node_modules/bulma/sass/utilities/initial-variables'; 3 | 4 | /* Base: Size */ 5 | $size-base: 1rem; 6 | $default-padding: $size-base * 1.5; 7 | 8 | /* Default font */ 9 | $family-sans-serif: v-sans, system-ui, -apple-system, BlinkMacSystemFont, 10 | 'Segoe UI', sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 11 | 'Segoe UI Symbol'; 12 | 13 | /* Base color */ 14 | $base-color: #27272a; 15 | $base-color-light: rgba(24, 28, 33, 0.06); 16 | 17 | /* General overrides */ 18 | $primary: #14b8a6; 19 | $link: #0ea5e9; 20 | $info: #06b6d4; 21 | $success: #10b981; 22 | $warning: #eab308; 23 | $danger: #ef4444; 24 | 25 | $body-background-color: #f8f8f8; 26 | $light-border: 1px solid $base-color-light; 27 | $hr-height: 1px; 28 | 29 | /* NavBar: specifics */ 30 | $navbar-input-color: $grey-darker; 31 | $navbar-input-placeholder-color: $grey-lighter; 32 | $navbar-box-shadow: 0 1px 0 rgba(24, 28, 33, 0.04); 33 | $navbar-divider-border: 1px solid rgba($grey-lighter, 0.25); 34 | $navbar-item-h-padding: $default-padding * 0.75; 35 | $navbar-avatar-size: 1.75rem; 36 | 37 | /* Aside: Bulma override */ 38 | $menu-item-radius: 0; 39 | $menu-list-link-padding: $size-base * 0.5 0; 40 | $menu-label-color: lighten($base-color, 25%); 41 | $menu-item-color: lighten($base-color, 30%); 42 | $menu-item-hover-color: lighten($base-color, 60%); 43 | $menu-item-hover-background-color: transparent; 44 | $menu-item-active-color: $white; 45 | $menu-item-active-background-color: transparent; 46 | $menu-list-link-padding: $size-base * 0.75 $size-base * 0.75; 47 | 48 | /* Aside: specifics */ 49 | $aside-width: $size-base * 14; 50 | $aside-mobile-width: $size-base * 15; 51 | $aside-icon-width: $size-base * 3; 52 | $aside-submenu-font-size: $size-base * 0.9; 53 | $aside-box-shadow: none; 54 | $aside-background-color: $base-color; 55 | $aside-tools-background-color: darken($aside-background-color, 10%); 56 | $aside-tools-color: $white; 57 | 58 | /* Title Bar: specifics */ 59 | $title-bar-color: $grey; 60 | $title-bar-active-color: $black-ter; 61 | 62 | /* Hero Bar: specifics */ 63 | $hero-bar-background: $white; 64 | 65 | /* Card: Bulma override */ 66 | $card-shadow: none; 67 | $card-header-shadow: none; 68 | 69 | /* Card: specifics */ 70 | $card-border: 1px solid $base-color-light; 71 | $card-header-border-bottom-color: $base-color-light; 72 | 73 | /* Table: Bulma override */ 74 | $table-cell-border: 1px solid $white-bis; 75 | 76 | /* Table: specifics */ 77 | $table-avatar-size: $size-base * 1.5; 78 | $table-avatar-size-mobile: 25vw; 79 | 80 | /* Form */ 81 | $checkbox-border: 1px solid $base-color; 82 | 83 | /* Modal card: Bulma override */ 84 | $modal-card-head-background-color: $white-ter; 85 | $modal-card-title-size: $size-base; 86 | $modal-card-body-padding: $default-padding 20px; 87 | $modal-card-head-border-bottom: 1px solid $white-ter; 88 | $modal-card-foot-border-top: 0; 89 | 90 | /* Modal card: specifics */ 91 | $modal-card-width: 40vw; 92 | $modal-card-width-mobile: 90vw; 93 | $modal-card-foot-background-color: $white-ter; 94 | 95 | /* Notification: Bulma override */ 96 | $notification-padding: $default-padding * 0.75 $default-padding; 97 | 98 | /* Footer: Bulma override */ 99 | $footer-background-color: $white; 100 | $footer-padding: $default-padding * 0.33 $default-padding; 101 | 102 | /* Footer: specifics */ 103 | $footer-logo-height: $size-base * 2; 104 | 105 | /* Progress: Bulma override */ 106 | $progress-bar-background-color: $grey-lighter; 107 | 108 | /* Icon: specifics */ 109 | $icon-update-mark-size: $size-base * 0.5; 110 | $icon-update-mark-color: $yellow; 111 | -------------------------------------------------------------------------------- /web/src/pages/index/scss/_theme_dark.scss: -------------------------------------------------------------------------------- 1 | html.dark { 2 | background-color: #181a1b !important; 3 | 4 | & , body, input, textarea, select, button { 5 | border-color: #575757 !important; 6 | color: $dark-white !important; 7 | } 8 | 9 | .title { 10 | color: rgb(213, 209, 203); 11 | } 12 | 13 | a { 14 | color: $dark-primary; 15 | text-decoration-color: currentcolor; 16 | 17 | &:hover { 18 | color: $dark-white; 19 | } 20 | } 21 | 22 | .navbar { 23 | background-color: rgb(24, 26, 27); 24 | } 25 | 26 | .tabs ul { 27 | border-bottom-color: rgb(58, 58, 58); 28 | } 29 | 30 | .tabs li.is-active a { 31 | border-bottom-color: rgb(27, 78, 159); 32 | color: rgb(93, 162, 227); 33 | } 34 | 35 | .tabs a { 36 | border-bottom-color: rgb(58, 58, 58); 37 | color: rgb(205, 202, 194); 38 | 39 | &:hover { 40 | border-bottom-color: rgb(91, 91, 91); 41 | color: rgb(213, 209, 203); 42 | } 43 | } 44 | 45 | .hot .divider { 46 | background-color: rgb(45, 48, 50) !important; 47 | } 48 | 49 | .hot:hover .divider { 50 | background-color: rgb(61, 64, 67) !important; 51 | } 52 | 53 | .tag:not(body) { 54 | background-color: rgb(27, 29, 30); 55 | color: $dark-white; 56 | } 57 | .tag:not(body).is-light-dark { 58 | background-color: #4e4e4e44; 59 | color: rgb(205, 202, 194); 60 | } 61 | 62 | .mini-navbar-opt { 63 | background-color: rgb(28, 30, 31) !important;; 64 | } 65 | 66 | .navbar-link.is-active, .navbar-link:focus, .navbar-link:focus-within, .navbar-link:hover, a.navbar-item.is-active, a.navbar-item:focus, a.navbar-item:focus-within, a.navbar-item:hover { 67 | background-color: rgb(26, 27, 28); 68 | color: rgb(93, 162, 227); 69 | } 70 | a.navbar-item:hover { 71 | background-color: unset; 72 | } 73 | 74 | .navbar-dropdown a.navbar-item:focus, .navbar-dropdown { 75 | background-color: rgb(27, 29, 30); 76 | } 77 | 78 | .navbar-item.has-dropdown.is-active .navbar-link, .navbar-item.has-dropdown:focus .navbar-link, .navbar-item.has-dropdown:hover .navbar-link { 79 | background-color: rgb(26, 27, 28); 80 | } 81 | 82 | .navbar-item, .navbar-link { 83 | color: rgb(205, 202, 194); 84 | } 85 | 86 | .navbar-dropdown { 87 | background-color: rgb(24, 26, 27); 88 | border-top-color: rgb(58, 58, 58); 89 | box-shadow: rgba(10, 10, 11, 0.1) 0px 8px 8px; 90 | } 91 | 92 | .navbar-divider { 93 | background-color: rgb(27, 29, 30); 94 | border-color: currentcolor; 95 | } 96 | 97 | .input, .textarea { 98 | box-shadow: rgba(10, 10, 11, 0.1) 0px 1px 2px inset; 99 | } 100 | 101 | .input, .select select, .textarea { 102 | background-color: rgb(24, 26, 27); 103 | border-color: rgb(58, 58, 58); 104 | color: rgb(213, 209, 203); 105 | } 106 | 107 | .input:active, .input:focus, .is-active.input, .is-active.textarea, .is-focused.input, .is-focused.textarea, .select select.is-active, .select select.is-focused, .select select:active, .select select:focus, .textarea:active, .textarea:focus { 108 | border-color: rgb(27, 78, 159); 109 | box-shadow: rgba(25, 73, 149, 0.25) 0px 0px 0px 0.125em; 110 | } 111 | 112 | .input:hover, .is-hovered.input, .is-hovered.textarea, .select select.is-hovered, .select select:hover, .textarea:hover { 113 | border-color: rgb(66, 66, 66); 114 | } 115 | 116 | .button.is-white,.button.is-white:hover { 117 | background-color: rgb(24, 26, 27); 118 | border-color: transparent; 119 | color: rgb(228, 226, 223); 120 | } 121 | 122 | .button.is-info { 123 | background-color: rgb(13, 109, 172); 124 | border-color: transparent; 125 | color: rgb(255, 255, 255); 126 | } 127 | } -------------------------------------------------------------------------------- /internal/core/site/wbvideo.go: -------------------------------------------------------------------------------- 1 | package site 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | const SITE_WBVIDEO = "wbvideo" 10 | 11 | var WbvideoTabs = []SiteTab{ 12 | { 13 | Tag: "all", 14 | Url: "https://weibo.com/tv/api/component?page=%2Ftv%2Fbillboard", 15 | Name: "全站", 16 | Args: map[string]string{ 17 | "cid": "4418213501411061", 18 | }, 19 | }, 20 | { 21 | Tag: "funny", 22 | Url: "https://weibo.com/tv/api/component?page=%2Ftv%2Fbillboard%2F4418219809678869", 23 | Name: "搞笑幽默", 24 | Args: map[string]string{ 25 | "cid": "4418219809678869", 26 | }, 27 | }, 28 | } 29 | 30 | type WbVideoList struct { 31 | Code string `json:"code"` 32 | Msg string `json:"msg"` 33 | Data struct { 34 | Videos struct { 35 | Next int `json:"next_cursor"` 36 | List []struct { 37 | Title string `json:"title"` 38 | Cover string `json:"cover_image"` 39 | Id int64 `json:"mid"` 40 | Oid string `json:"oid"` 41 | Date string `json:"date"` 42 | PlayCount string `json:"play_count"` 43 | } `json:"list"` 44 | } `json:"Component_Billboard_Billboardlist"` 45 | } `json:"data"` 46 | } 47 | 48 | type Wbvideo struct { 49 | Site 50 | } 51 | 52 | func (w *Wbvideo) GetSite() *Site { 53 | return &w.Site 54 | } 55 | 56 | func (w *Wbvideo) BuildUrl() ([]Link, error) { 57 | var list []Link 58 | for _, tab := range WbvideoTabs { 59 | url := tab.Url 60 | link := Link{ 61 | Key: tab.Args["cid"], 62 | Url: url, 63 | Tag: tab.Tag, 64 | Method: "POST", 65 | } 66 | list = append(list, link) 67 | } 68 | 69 | return list, nil 70 | } 71 | 72 | func (w *Wbvideo) CrawPage(link Link, headers map[string]string) (res Page, err error) { 73 | var page Page 74 | var hotList []Hot 75 | var nextCursor int 76 | post := make(map[string]map[string]interface{}) 77 | for { 78 | if nextCursor == 0 { 79 | post = map[string]map[string]interface{}{ 80 | "Component_Billboard_Billboardcategory": {}, 81 | "Component_Billboard_Billboardlist": { 82 | "cid": link.Key, 83 | "count": 20, 84 | }, 85 | } 86 | } else { 87 | post["Component_Billboard_Billboardlist"]["next_cursor"] = nextCursor 88 | } 89 | data, _ := json.Marshal(post) 90 | 91 | videoList := WbVideoList{} 92 | page, err = w.FetchData(link, map[string]string{"data": string(data)}, headers) 93 | if err != nil { 94 | return 95 | } 96 | err = json.Unmarshal([]byte(page.Content), &videoList) 97 | if err != nil { 98 | // 但凡一次报错，全部不算了 99 | return 100 | } 101 | if len(videoList.Data.Videos.List) == 0 { 102 | break 103 | } 104 | for _, v := range videoList.Data.Videos.List { 105 | hotList = append(hotList, Hot{ 106 | Key: w.FetchKey(v.Oid), 107 | Title: v.Title, 108 | OriginUrl: fmt.Sprintf("https://weibo.com/tv/show/%s", v.Oid), 109 | Card: CardVideo, 110 | Ext: map[string]string{ 111 | "cover": v.Cover, 112 | "date": v.Date, 113 | "score": v.PlayCount, 114 | }, 115 | }) 116 | } 117 | nextCursor = videoList.Data.Videos.Next 118 | } 119 | 120 | res = Page{ 121 | Link: link, 122 | List: hotList, 123 | T: time.Now(), 124 | } 125 | 126 | return 127 | } 128 | 129 | func (w *Wbvideo) FetchKey(key string) string { 130 | return key 131 | } 132 | 133 | func NewWbvideo() *Wbvideo { 134 | return &Wbvideo{ 135 | Site{ 136 | Name: "微博视频", 137 | Key: SITE_WBVIDEO, 138 | Root: "https://weibo.com/tv/home", 139 | Desc: "微博视频榜单", 140 | CrawType: CrawApi, 141 | Tabs: WbvideoTabs, 142 | }, 143 | } 144 | } 145 | 146 | var _ Spider = &Wbvideo{} 147 | 148 | func init() { 149 | RegistSite(SITE_WBVIDEO, NewWbvideo()) 150 | } 151 | -------------------------------------------------------------------------------- /internal/api/handler/admin/site.go: -------------------------------------------------------------------------------- 1 | package admin 2 | 3 | import ( 4 | "strconv" 5 | 6 | "github.com/aaronzjc/mu/internal/api/handler" 7 | "github.com/aaronzjc/mu/internal/application/dto" 8 | "github.com/aaronzjc/mu/internal/application/service" 9 | "github.com/aaronzjc/mu/internal/application/store" 10 | "github.com/aaronzjc/mu/internal/constant" 11 | "github.com/aaronzjc/mu/internal/domain/model" 12 | "github.com/gin-gonic/gin" 13 | ) 14 | 15 | type Site struct { 16 | svc service.SiteService 17 | nodeSvc service.NodeService 18 | crawSvc service.CrawService 19 | } 20 | 21 | type SiteQueryForm struct { 22 | Id int `form:"id"` 23 | Keyword string `form:"keyword"` 24 | } 25 | 26 | func (s *Site) List(ctx *gin.Context) { 27 | var r SiteQueryForm 28 | if err := ctx.ShouldBindQuery(&r); err != nil { 29 | handler.Resp(ctx, constant.CodeError, "参数错误", nil) 30 | return 31 | } 32 | 33 | q := &dto.Query{} 34 | if r.Id > 0 { 35 | q.Query = "`id` = ?" 36 | q.Args = []interface{}{r.Id} 37 | } 38 | if r.Keyword != "" { 39 | q.Query = "`title` = ?" 40 | q.Args = []interface{}{r.Keyword} 41 | } 42 | sites, err := s.svc.Get(ctx, q) 43 | if err != nil { 44 | handler.Resp(ctx, constant.CodeError, err.Error(), nil) 45 | return 46 | } 47 | nodes, _ := s.nodeSvc.Get(ctx, &dto.Query{ 48 | Query: "`enable` = ?", 49 | Args: []interface{}{model.Enable}, 50 | }) 51 | nodeList := make(map[int]*dto.Node) 52 | for _, v := range nodes { 53 | nodeList[v.ID] = v 54 | } 55 | handler.Resp(ctx, constant.CodeSuccess, "success", map[string]interface{}{ 56 | "nodeList": nodeList, 57 | "siteList": sites, 58 | }) 59 | } 60 | 61 | type UpsertSiteForm struct { 62 | ID int `form:"id"` 63 | Name string `form:"name"` 64 | Addr string `form:"addr"` 65 | Type int8 `form:"type"` 66 | Enable int8 `form:"enable"` 67 | } 68 | 69 | func (s *Site) Upsert(ctx *gin.Context) { 70 | var r dto.Site 71 | if err := ctx.ShouldBind(&r); err != nil { 72 | handler.Resp(ctx, constant.CodeError, "参数错误", nil) 73 | return 74 | } 75 | id, err := strconv.Atoi(ctx.Param("id")) 76 | if err != nil { 77 | handler.Resp(ctx, constant.CodeError, "参数错误", nil) 78 | return 79 | } 80 | r.ID = id 81 | err = s.svc.Upsert(ctx, &r) 82 | if err != nil { 83 | handler.Resp(ctx, constant.CodeError, err.Error(), nil) 84 | return 85 | } 86 | 87 | handler.Resp(ctx, constant.CodeSuccess, "success", nil) 88 | } 89 | 90 | func (s *Site) Del(ctx *gin.Context) { 91 | var r SiteQueryForm 92 | if err := ctx.ShouldBindQuery(&r); err != nil { 93 | handler.Resp(ctx, constant.CodeError, "参数错误", nil) 94 | return 95 | } 96 | err := s.svc.Del(ctx, r.Id) 97 | if err != nil { 98 | handler.Resp(ctx, constant.CodeError, err.Error(), nil) 99 | return 100 | } 101 | handler.Resp(ctx, constant.CodeSuccess, "success", nil) 102 | } 103 | 104 | func (s *Site) Craw(ctx *gin.Context) { 105 | id, _ := strconv.Atoi(ctx.Param("id")) 106 | if id <= 0 { 107 | handler.Resp(ctx, constant.CodeError, "参数错误", nil) 108 | return 109 | } 110 | sites, _ := s.svc.Get(ctx, &dto.Query{ 111 | Query: "`id` = ?", 112 | Args: []interface{}{id}, 113 | }) 114 | if len(sites) <= 0 { 115 | handler.Resp(ctx, constant.CodeError, "站点不存在", nil) 116 | return 117 | } 118 | err := s.crawSvc.Craw(ctx, sites[0]) 119 | if err != nil { 120 | handler.Resp(ctx, constant.CodeError, err.Error(), nil) 121 | return 122 | } 123 | handler.Resp(ctx, constant.CodeSuccess, "success", nil) 124 | } 125 | 126 | func NewSite() *Site { 127 | siteRepo := store.NewSiteRepo() 128 | nodeRepo := store.NewNodeRepo() 129 | siteSvc := service.NewSiteService(siteRepo, nil) 130 | nodeSvc := service.NewNodeService(nodeRepo) 131 | crawSvc := service.NewCrawService(siteRepo, nodeRepo) 132 | return &Site{ 133 | svc: siteSvc, 134 | nodeSvc: nodeSvc, 135 | crawSvc: crawSvc, 136 | } 137 | } 138 | --------------------------------------------------------------------------------