From e020bf0d0f817e78345773b02b3c04299d66c3e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=94=85=E9=A5=AD?= <1156544355@qq.com> Date: Sun, 14 Sep 2025 23:27:48 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9C=A8=20=E6=B7=BB=E5=8A=A0rsshub?= =?UTF-8?q?=E6=8E=A8=E9=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 11 ++ go.mod | 7 + go.sum | 20 ++ main.go | 1 + plugin/rsshub/domain/job.go | 135 ++++++++++++++ plugin/rsshub/domain/model.go | 123 ++++++++++++ plugin/rsshub/domain/rawFeed.go | 110 +++++++++++ plugin/rsshub/domain/rssHub.go | 192 +++++++++++++++++++ plugin/rsshub/domain/rssHub_test.go | 105 +++++++++++ plugin/rsshub/domain/storageImpl.go | 47 +++++ plugin/rsshub/domain/storageRepo.go | 280 ++++++++++++++++++++++++++++ plugin/rsshub/main.go | 163 ++++++++++++++++ plugin/rsshub/view.go | 100 ++++++++++ 13 files changed, 1294 insertions(+) create mode 100644 plugin/rsshub/domain/job.go create mode 100644 plugin/rsshub/domain/model.go create mode 100644 plugin/rsshub/domain/rawFeed.go create mode 100644 plugin/rsshub/domain/rssHub.go create mode 100644 plugin/rsshub/domain/rssHub_test.go create mode 100644 plugin/rsshub/domain/storageImpl.go create mode 100644 plugin/rsshub/domain/storageRepo.go create mode 100644 plugin/rsshub/main.go create mode 100644 plugin/rsshub/view.go diff --git a/README.md b/README.md index e923a77fdc..2e2faadb0b 100644 --- a/README.md +++ b/README.md @@ -1284,6 +1284,17 @@ print("run[CQ:image,file="+j["img"]+"]") - [x] 打劫[对方Q号|@对方QQ] + +
+ RSSHub + +`import _ "github.com/FloatTech/ZeroBot-Plugin/plugin/rsshub"` + +- [x] 添加rsshub订阅-/bookfere/weekly +- [x] 删除rsshub订阅-/bookfere/weekly +- [x] 查看rsshub订阅列表 +- [x] rsshub同步 (使用job执行定时任务------记录在"@every 10m"触发的指令) +
在线代码运行 diff --git a/go.mod b/go.mod index bb0498903e..d73abcea98 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/kanrichan/resvg-go v0.0.2-0.20231001163256-63db194ca9f5 github.com/lithammer/fuzzysearch v1.1.8 github.com/liuzl/gocc v0.0.0-20231231122217-0372e1059ca5 + github.com/mmcdole/gofeed v1.3.0 github.com/mroth/weightedrand v1.0.0 github.com/notnil/chess v1.9.0 github.com/pkg/errors v0.9.1 @@ -54,8 +55,10 @@ require ( ) require ( + github.com/PuerkitoBio/goquery v1.8.0 // indirect github.com/adamzy/cedar-go v0.0.0-20170805034717-80a9c64b256d // indirect github.com/ajstarks/svgo v0.0.0-20200320125537-f189e35d30ca // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect github.com/antchfx/xpath v1.3.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ericpauley/go-quantize v0.0.0-20200331213906-ae555eb2afa4 // indirect @@ -70,10 +73,14 @@ require ( github.com/jfreymuth/oggvorbis v1.0.1 // indirect github.com/jfreymuth/vorbis v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/liuzl/cedar-go v0.0.0-20170805034717-80a9c64b256d // indirect github.com/liuzl/da v0.0.0-20180704015230-14771aad5b1d // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect diff --git a/go.sum b/go.sum index 9a54a9651a..d337514073 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/FloatTech/zbpctrl v1.7.0/go.mod h1:xmM4dSwHA02Gei3ogCRiG+RTrw/7Z69Pfr github.com/FloatTech/zbputils v1.7.2-0.20250812085410-2741050f465f h1:5jnrFe9FTydb/pcUhxkWHuQVCwmYIZmneOkvmgHOwGI= github.com/FloatTech/zbputils v1.7.2-0.20250812085410-2741050f465f/go.mod h1:HG/yZwExV3b1Vqu4chbqwhfX4hx7gDS07QO436JkwIg= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7 h1:S/ferNiehVjNaBMNNBxUjLtVmP/YWD6Yh79RfPv4ehU= github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7/go.mod h1:vD7Ra3Q9onRtojoY5sMCLQ7JBgjUsrXDnDKyFxqpf9w= github.com/RomiChan/websocket v1.4.3-0.20220227141055-9b2c6168c9c5 h1:bBmmB7he0iVN4m5mcehfheeRUEer/Avo4ujnxI3uCqs= @@ -31,6 +33,8 @@ github.com/adamzy/cedar-go v0.0.0-20170805034717-80a9c64b256d/go.mod h1:PRWNwWq0 github.com/ajstarks/svgo v0.0.0-20200320125537-f189e35d30ca h1:kWzLcty5V2rzOqJM7Tp/MfSX0RMSI1x4IOLApEefYxA= github.com/ajstarks/svgo v0.0.0-20200320125537-f189e35d30ca/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/antchfx/htmlquery v1.3.4 h1:Isd0srPkni2iNTWCwVj/72t7uCphFeor5Q8nCzj1jdQ= github.com/antchfx/htmlquery v1.3.4/go.mod h1:K9os0BwIEmLAvTqaNSua8tXLWRWZpocZIH73OzWQbwM= github.com/antchfx/xpath v1.3.3 h1:tmuPQa1Uye0Ym1Zn65vxPgfltWb/Lxu2jeqIGteJSRs= @@ -108,6 +112,7 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4er github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hajimehoshi/go-mp3 v0.3.0/go.mod h1:qMJj/CSDxx6CGHiZeCgbiq2DSUkbK0UbtXShQcnfyMM= @@ -128,6 +133,8 @@ github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jozsefsallai/gophersauce v1.0.1 h1:BA3ovtQRrAb1qYU9JoRLbDHpxnDunlNcEkEfhCvDDCM= github.com/jozsefsallai/gophersauce v1.0.1/go.mod h1:YVEI7djliMTmZ1Vh01YPF8bUHi+oKhe3yXgKf1T49vg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kanrichan/resvg-go v0.0.2-0.20231001163256-63db194ca9f5 h1:BXnB1Gz4y/zwQh+ZFNy7rgd+ZfMOrwRr4uZSHEI+ieY= github.com/kanrichan/resvg-go v0.0.2-0.20231001163256-63db194ca9f5/go.mod h1:c9+VS9GaommgIOzNWb5ze4lYwfT8BZ2UDyGiuQTT7yc= github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= @@ -150,6 +157,15 @@ github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/ github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/mewkiz/flac v1.0.7/go.mod h1:yU74UH277dBUpqxPouHSQIar3G1X/QIclVbFahSd1pU= github.com/mewkiz/pkg v0.0.0-20190919212034-518ade7978e2/go.mod h1:3E2FUC/qYUfM8+r9zAwpeHJzqRVVMIYnpzD/clwWxyA= +github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4= +github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE= +github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk= +github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mroth/weightedrand v1.0.0 h1:V8JeHChvl2MP1sAoXq4brElOcza+jxLkRuwvtQu8L3E= github.com/mroth/weightedrand v1.0.0/go.mod h1:3p2SIcC8al1YMzGhAIoXD+r9olo/g/cdJgAD905gyNE= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= @@ -179,6 +195,7 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= @@ -240,6 +257,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= @@ -265,6 +283,7 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -290,6 +309,7 @@ golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= diff --git a/main.go b/main.go index 4de143cfcc..01b5511e27 100644 --- a/main.go +++ b/main.go @@ -132,6 +132,7 @@ import ( _ "github.com/FloatTech/ZeroBot-Plugin/plugin/realcugan" // realcugan清晰术 _ "github.com/FloatTech/ZeroBot-Plugin/plugin/reborn" // 投胎 _ "github.com/FloatTech/ZeroBot-Plugin/plugin/robbery" // 打劫群友的ATRI币 + _ "github.com/FloatTech/ZeroBot-Plugin/plugin/rsshub" // RSSHub订阅姬 _ "github.com/FloatTech/ZeroBot-Plugin/plugin/runcode" // 在线运行代码 _ "github.com/FloatTech/ZeroBot-Plugin/plugin/saucenao" // 以图搜图 _ "github.com/FloatTech/ZeroBot-Plugin/plugin/score" // 分数 diff --git a/plugin/rsshub/domain/job.go b/plugin/rsshub/domain/job.go new file mode 100644 index 0000000000..b6e4fb642c --- /dev/null +++ b/plugin/rsshub/domain/job.go @@ -0,0 +1,135 @@ +// Package domain rsshub领域逻辑 +package domain + +import ( + "context" + + "github.com/mmcdole/gofeed" + "github.com/sirupsen/logrus" +) + +// syncRss 同步所有频道 +// 返回:更新的频道&订阅信息 map[int64]*RssClientView +// 1. 获取所有频道 +// 2. 遍历所有频道,检查频道是否更新 +// 3. 如果更新,获取更新的内容,但是返回的数据 +func (repo *rssDomain) syncRss(ctx context.Context) (updated map[int64]*RssClientView, err error) { + updated = make(map[int64]*RssClientView) + // 获取所有频道 + sources, err := repo.storage.GetSources(ctx) + if err != nil { + return + } + // 遍历所有源,获取每个channel对应的rss内容 + rssView := make([]*RssClientView, len(sources)) + for i, channel := range sources { + var feed *gofeed.Feed + // 从site获取rss内容 + feed, err = repo.rssHubClient.FetchFeed(channel.RssHubFeedPath) + // 如果获取失败,则跳过 + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub syncRss] fetch path(%+v) error: %v", channel.RssHubFeedPath, err) + continue + } + rv := convertFeedToRssView(0, channel.RssHubFeedPath, feed) + rssView[i] = rv + } + // 检查频道是否更新 + for _, cv := range rssView { + if cv == nil { + continue + } + var needUpdate bool + needUpdate, err = repo.checkSourceNeedUpdate(ctx, cv.Source) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub syncRss] checkSourceNeedUpdate error: %v", err) + err = nil + continue + } + // 保存 + logrus.WithContext(ctx).Infof("[rsshub syncRss] cv %+v, need update(real): %v", cv.Source, needUpdate) + // 如果需要更新,更新channel 和 content + if needUpdate { + err = repo.storage.UpsertSource(ctx, cv.Source) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub syncRss] upsert source error: %v", err) + err = nil + // continue + } + } + var updateChannelView = &RssClientView{Source: cv.Source, Contents: []*RssContent{}} + err = repo.processContentsUpdate(ctx, cv, err, updateChannelView) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub syncRss] processContentsUpdate error: %v", err) + continue + } + if len(updateChannelView.Contents) == 0 { + logrus.WithContext(ctx).Infof("[rsshub syncRss] cv %s, no new content", cv.Source.RssHubFeedPath) + continue + } + updateChannelView.Sort() + updated[updateChannelView.Source.ID] = updateChannelView + logrus.WithContext(ctx).Debugf("[rsshub syncRss] cv %s, new contents: %v", cv.Source.RssHubFeedPath, len(updateChannelView.Contents)) + } + return +} + +// checkSourceNeedUpdate 检查频道是否需要更新 +func (repo *rssDomain) checkSourceNeedUpdate(ctx context.Context, source *RssSource) (needUpdate bool, err error) { + var sourceInDB *RssSource + sourceInDB, err = repo.storage.GetSourceByRssHubFeedLink(ctx, source.RssHubFeedPath) + if err != nil { + return + } + if sourceInDB == nil { + logrus.WithContext(ctx).Errorf("[rsshub syncRss] source not found: %v", source.RssHubFeedPath) + return + } + source.ID = sourceInDB.ID + // 检查是否需要更新到db + if sourceInDB.IfNeedUpdate(source) { + needUpdate = true + } + return +} + +// processContentsUpdate 处理内容(s)更新 +func (repo *rssDomain) processContentsUpdate(ctx context.Context, cv *RssClientView, err error, updateChannelView *RssClientView) error { + for _, content := range cv.Contents { + if content == nil { + continue + } + content.RssSourceID = cv.Source.ID + var existed bool + existed, err = repo.processContentItemUpdate(ctx, content) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub syncRss] upsert content error: %v", err) + err = nil + continue + } + if !existed { + updateChannelView.Contents = append(updateChannelView.Contents, content) + logrus.WithContext(ctx).Infof("[rsshub syncRss] cv %s, add new content: %v", cv.Source.RssHubFeedPath, content.Title) + } + } + return err +} + +// processContentItemUpdate 处理单个内容更新 +func (repo *rssDomain) processContentItemUpdate(ctx context.Context, content *RssContent) (existed bool, err error) { + existed, err = repo.storage.IsContentHashIDExist(ctx, content.HashID) + if err != nil { + return + } + // 不需要更新&不需要发送 + if existed { + return + } + // 保存 + err = repo.storage.UpsertContent(ctx, content) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub syncRss] upsert content error: %v", err) + return + } + return +} diff --git a/plugin/rsshub/domain/model.go b/plugin/rsshub/domain/model.go new file mode 100644 index 0000000000..64c9d33400 --- /dev/null +++ b/plugin/rsshub/domain/model.go @@ -0,0 +1,123 @@ +package domain + +import ( + "encoding/hex" + "hash/fnv" + "sort" + "time" +) + +// ======== RSS ========[START] + +// type SingleFeedItem gofeed.Item + +func genHashForFeedItem(link, guid string) string { + idString := link + "||" + guid + h := fnv.New32() + _, _ = h.Write([]byte(idString)) + encoded := hex.EncodeToString(h.Sum(nil)) + return encoded +} + +// RssClientView 频道视图 +type RssClientView struct { + Source *RssSource + Contents []*RssContent +} + +// ======== RSS ========[END] + +// ======== DB ========[START] + +const ( + tableNameRssSource = "rss_source" + tableNameRssContent = "rss_content" + tableNameRssSubscribe = "rss_subscribe" +) + +// RssSource RSS频道 +type RssSource struct { + // Id 自增id + ID int64 `gorm:"column:id;primary_key;AUTO_INCREMENT"` + // RssHubFeedPath 频道路由 用于区分rss_hub 不同的频道 例如: `/bangumi/tv/calendar/today` + RssHubFeedPath string `gorm:"column:rss_hub_feed_path;not null;unique;" json:"rss_hub_feed_path"` + // Title 频道标题 + Title string `gorm:"column:title" json:"title"` + // ChannelDesc 频道描述 + ChannelDesc string `gorm:"column:channel_desc" json:"channel_desc"` + // ImageURL 频道图片 + ImageURL string `gorm:"column:image_url" json:"image_url"` + // Link 频道链接 + Link string `gorm:"column:link" json:"link"` + // UpdatedParsed RSS页面更新时间 + UpdatedParsed time.Time `gorm:"column:updated_parsed" json:"updated_parsed"` + //// Ctime create time + // Ctime int64 `gorm:"column:ctime;default:current_timestamp" json:"ctime"` + // Mtime update time + Mtime time.Time `gorm:"column:mtime;default:current_timestamp;" json:"mtime"` +} + +// TableName ... +func (RssSource) TableName() string { + return tableNameRssSource +} + +// IfNeedUpdate ... +func (r RssSource) IfNeedUpdate(cmp *RssSource) bool { + if r.Link != cmp.Link { + return false + } + return r.UpdatedParsed.Unix() < cmp.UpdatedParsed.Unix() +} + +// RssContent 订阅的RSS频道的推送信息 +type RssContent struct { + // Id 自增id + ID int64 `gorm:"column:id;primary_key;AUTO_INCREMENT"` + HashID string `gorm:"column:hash_id;unique" json:"hash_id"` + RssSourceID int64 `gorm:"column:rss_source_id;not null" json:"rss_source_id"` + Title string `gorm:"column:title" json:"title"` + Description string `gorm:"column:description" json:"description"` + Link string `gorm:"column:link" json:"link"` + Date time.Time `gorm:"column:date" json:"date"` + Author string `gorm:"column:author" json:"author"` + Thumbnail string `gorm:"column:thumbnail" json:"thumbnail"` + Content string `gorm:"column:content" json:"content"` + //// Ctime create time + // Ctime int64 `gorm:"column:ctime;default:current_timestamp" json:"ctime"` + // Mtime update time + Mtime time.Time `gorm:"column:mtime;default:current_timestamp;" json:"mtime"` +} + +// TableName ... +func (RssContent) TableName() string { + return tableNameRssContent +} + +// Sort ... order by Date desc +func (r *RssClientView) Sort() { + sort.Slice(r.Contents, func(i, j int) bool { + return r.Contents[i].Date.Unix() > r.Contents[j].Date.Unix() + }) +} + +// RssSubscribe 订阅关系表:群组-RSS频道 +type RssSubscribe struct { + // Id 自增id + ID int64 `gorm:"column:id;primary_key;AUTO_INCREMENT"` + // 订阅群组 + GroupID int64 `gorm:"column:group_id;not null;uniqueIndex:uk_sid_gid"` + // 订阅频道 + RssSourceID int64 `gorm:"column:rss_source_id;not null;uniqueIndex:uk_sid_gid"` + //// Ctime create time + // Ctime int64 `gorm:"column:ctime;default:current_timestamp" json:"ctime"` + // Mtime update time + Mtime time.Time `gorm:"column:mtime;default:current_timestamp;" json:"mtime"` +} + +// TableName ... +func (RssSubscribe) TableName() string { + return tableNameRssSubscribe +} + +// ======== DB ========[END] diff --git a/plugin/rsshub/domain/rawFeed.go b/plugin/rsshub/domain/rawFeed.go new file mode 100644 index 0000000000..a8f6ee5d66 --- /dev/null +++ b/plugin/rsshub/domain/rawFeed.go @@ -0,0 +1,110 @@ +package domain + +import ( + "bytes" + "encoding/json" + "errors" + "net/http" + "time" + + "github.com/FloatTech/floatbox/web" + "github.com/mmcdole/gofeed" + "github.com/sirupsen/logrus" +) + +// const ( +// acceptHeader = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" +// userHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36 Edg/84.0.522.63" +//) + +var ( + // RSSHubMirrors RSSHub镜像站地址列表,第一个为默认地址 + rssHubMirrors = []string{ + "https://rsshub.rssforever.com", + "https://rss.injahow.cn", + } +) + +// RssHubClient rss hub client (http) +type RssHubClient struct { + *http.Client +} + +// FetchFeed 获取rss feed信息 +func (c *RssHubClient) FetchFeed(path string) (feed *gofeed.Feed, err error) { + var data []byte + // 遍历 rssHubMirrors,直到获取成功 + for _, mirror := range rssHubMirrors { + data, err = web.RequestDataWith(c.Client, mirror+path, "GET", "", web.RandUA(), nil) + if err == nil && len(data) > 0 { + break + } + } + if err != nil { + logrus.Errorf("[rsshub FetchFeed] fetch feed error: %v", err) + return nil, err + } + if len(data) == 0 { + logrus.Errorf("[rsshub FetchFeed] fetch feed error: data is empty") + return nil, errors.New("feed data is empty") + } + // data, err = web.RequestDataWith(c.Client, domain+path, "GET", "", web.RandUA(), nil) + // if err != nil { + // return nil, err + //} + feed, err = gofeed.NewParser().Parse(bytes.NewBuffer(data)) + if err != nil { + return + } + return +} + +func convertFeedToRssView(channelID int64, cPath string, feed *gofeed.Feed) (view *RssClientView) { + var imgURL string + if feed.Image != nil { + imgURL = feed.Image.URL + } + view = &RssClientView{ + Source: &RssSource{ + ID: channelID, + RssHubFeedPath: cPath, + Title: feed.Title, + ChannelDesc: feed.Description, + ImageURL: imgURL, + Link: feed.Link, + UpdatedParsed: *(feed.UpdatedParsed), + Mtime: time.Now(), + }, + // 不用定长,后面可能会过滤一些元素再append + Contents: []*RssContent{}, + } + // convert feed items to rss content + for _, item := range feed.Items { + if item.Link == "" || item.Title == "" { + continue + } + var thumbnail string + if item.Image != nil { + thumbnail = item.Image.URL + } + var publishedParsed = item.PublishedParsed + if publishedParsed == nil { + publishedParsed = &time.Time{} + } + aus, _ := json.Marshal(item.Authors) + view.Contents = append(view.Contents, &RssContent{ + ID: 0, + HashID: genHashForFeedItem(item.Link, item.GUID), + RssSourceID: channelID, + Title: item.Title, + Description: item.Description, + Link: item.Link, + Date: *publishedParsed, + Author: string(aus), + Thumbnail: thumbnail, + Content: item.Content, + Mtime: time.Now(), + }) + } + return +} diff --git a/plugin/rsshub/domain/rssHub.go b/plugin/rsshub/domain/rssHub.go new file mode 100644 index 0000000000..1f652032ad --- /dev/null +++ b/plugin/rsshub/domain/rssHub.go @@ -0,0 +1,192 @@ +package domain + +import ( + "context" + "errors" + "net/http" + "os" + "time" + + "github.com/jinzhu/gorm" + "github.com/sirupsen/logrus" +) + +// RssDomain RssRepo定义 +type RssDomain interface { + // Subscribe 订阅Rss频道 + Subscribe(ctx context.Context, gid int64, route string) (rv *RssClientView, isChannelExisted, + isSubExisted bool, err error) + // Unsubscribe 取消订阅Rss频道 + Unsubscribe(ctx context.Context, gid int64, route string) (err error) + // GetSubscribedChannelsByGroupID 获取群组订阅的Rss频道 + GetSubscribedChannelsByGroupID(ctx context.Context, gid int64) (rv []*RssClientView, err error) + // Sync 同步Rss频道 + // 返回群组-频道推送视图 map[群组]推送内容数组 + Sync(ctx context.Context) (groupView map[int64][]*RssClientView, err error) +} + +// rssDomain RssRepo定义 +type rssDomain struct { + storage RepoStorage + rssHubClient *RssHubClient +} + +// NewRssDomain 新建RssDomain,调用方保证单例模式 +func NewRssDomain(dbPath string) (RssDomain, error) { + return newRssDomain(dbPath) +} + +func newRssDomain(dbPath string) (*rssDomain, error) { + if _, err := os.Stat(dbPath); err != nil || os.IsNotExist(err) { + // 生成文件 + f, err := os.Create(dbPath) + if err != nil { + return nil, err + } + defer f.Close() + } + orm, err := gorm.Open("sqlite3", dbPath) + if err != nil { + logrus.Errorf("[rsshub NewRssDomain] open db error: %v", err) + panic(err) + } + repo := &rssDomain{ + storage: &repoStorage{orm: orm}, + rssHubClient: &RssHubClient{Client: http.DefaultClient}, + } + err = repo.storage.initDB() + if err != nil { + logrus.Errorf("[rsshub NewRssDomain] open db error: %v", err) + panic(err) + } + return repo, nil +} + +// Subscribe QQ群订阅Rss频道 +func (repo *rssDomain) Subscribe(ctx context.Context, gid int64, feedPath string) ( + rv *RssClientView, isChannelExisted, isSubExisted bool, err error) { + // 验证 + feed, err := repo.rssHubClient.FetchFeed(feedPath) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub Subscribe] add source error: %v", err) + return + } + logrus.WithContext(ctx).Infof("[rsshub Subscribe] try get source success: %v", len(feed.Title)) + // 新建source结构体 + rv = convertFeedToRssView(0, feedPath, feed) + feedChannel, err := repo.storage.GetSourceByRssHubFeedLink(ctx, feedPath) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub Subscribe] query source by feedPath error: %v", err) + return + } + // 如果已经存在 + if feedChannel != nil { + logrus.WithContext(ctx).Warningf("[rsshub Subscribe] source existed: %v", feedChannel) + isChannelExisted = true + } else { + // 不存在的情况,要把更新时间置空,保证下一次同步时能够更新 + rv.Source.UpdatedParsed = time.Time{} + } + // 保存 + err = repo.storage.UpsertSource(ctx, rv.Source) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub Subscribe] save source error: %v", err) + return + } + logrus.Infof("[rsshub Subscribe] save/update source success %v", rv.Source.ID) + // 添加群号到订阅 + subscribe, err := repo.storage.GetSubscribeByID(ctx, gid, rv.Source.ID) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub Subscribe] query subscribe error: %v", err) + return + } + logrus.WithContext(ctx).Infof("[rsshub Subscribe] query subscribe success: %v", subscribe) + // 如果已经存在,直接返回 + if subscribe != nil { + isSubExisted = true + logrus.WithContext(ctx).Infof("[rsshub Subscribe] subscribe existed: %v", subscribe) + return + } + // 如果不存在,保存 + err = repo.storage.CreateSubscribe(ctx, gid, rv.Source.ID) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub Subscribe] save subscribe error: %v", err) + return + } + logrus.WithContext(ctx).Infof("[rsshub Subscribe] success: %v", len(rv.Contents)) + return +} + +// Unsubscribe 群组取消订阅 +func (repo *rssDomain) Unsubscribe(ctx context.Context, gid int64, feedPath string) (err error) { + existedSubscribes, ifExisted, err := repo.storage.GetIfExistedSubscribe(ctx, gid, feedPath) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub Subscribe] query sub by route error: %v", err) + return errors.New("数据库错误") + } + logrus.WithContext(ctx).Infof("[rsshub Subscribe] query source by route success: %v", existedSubscribes) + // 如果不存在订阅关系,直接返回 + if !ifExisted || existedSubscribes == nil { + logrus.WithContext(ctx).Infof("[rsshub Subscribe] source existed: %v", ifExisted) + return errors.New("频道不存在") + } + err = repo.storage.DeleteSubscribe(ctx, existedSubscribes.ID) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub Subscribe] delete source error: %v", err) + return errors.New("删除失败") + } + // 查询是否还有群订阅这个频道 + subscribesNeedsToDel, err := repo.storage.GetSubscribesBySource(ctx, feedPath) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub Subscribe] query source by route error: %v", err) + return + } + // 没有群订阅的时候,把频道删除 + if len(subscribesNeedsToDel) == 0 { + err = repo.storage.DeleteSource(ctx, existedSubscribes.RssSourceID) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub Subscribe] delete source error: %v", err) + return errors.New("清除频道信息失败") + } + } + return +} + +// GetSubscribedChannelsByGroupID 获取群对应的订阅的频道信息 +func (repo *rssDomain) GetSubscribedChannelsByGroupID(ctx context.Context, gid int64) ([]*RssClientView, error) { + channels, err := repo.storage.GetSubscribedChannelsByGroupID(ctx, gid) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub GetSubscribedChannelsByGroupID] GetSubscribedChannelsByGroupID error: %v", err) + return nil, err + } + rv := make([]*RssClientView, len(channels)) + logrus.WithContext(ctx).Infof("[rsshub GetSubscribedChannelsByGroupID] query subscribe success: %v", len(channels)) + for i, cn := range channels { + rv[i] = &RssClientView{ + Source: cn, + } + } + return rv, nil +} + +// Sync 同步任务,按照群组订阅情况做好map切片 +func (repo *rssDomain) Sync(ctx context.Context) (groupView map[int64][]*RssClientView, err error) { + groupView = make(map[int64][]*RssClientView) + // 获取所有Rss频道 + // 获取所有频道 + updatedViews, err := repo.syncRss(ctx) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub Sync] sync rss feed error: %v", err) + return + } + logrus.WithContext(ctx).Infof("[rsshub Sync] updated channels: %v", len(updatedViews)) + subscribes, err := repo.storage.GetSubscribes(ctx) + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub Sync] get subscribes error: %v", err) + return + } + for _, subscribe := range subscribes { + groupView[subscribe.GroupID] = append(groupView[subscribe.GroupID], updatedViews[subscribe.RssSourceID]) + } + return +} diff --git a/plugin/rsshub/domain/rssHub_test.go b/plugin/rsshub/domain/rssHub_test.go new file mode 100644 index 0000000000..451795931d --- /dev/null +++ b/plugin/rsshub/domain/rssHub_test.go @@ -0,0 +1,105 @@ +package domain + +import ( + "context" + "encoding/json" + "testing" +) + +func TestNewRssDomain(t *testing.T) { + dm, err := newRssDomain("rsshub.db") + if err != nil { + t.Fatal(err) + return + } + if dm == nil { + t.Fatal("domain is nil") + } +} + +//var testRssHubChannelUrl = "https://rsshub.rssforever.com/bangumi/tv/calendar/today" + +var dm, _ = newRssDomain("rsshub.db") + +func TestSub(t *testing.T) { + testCases := []struct { + name string + feedLink string + gid int64 + }{ + { + name: "test1", + feedLink: "/bangumi/tv/calendar/today", + gid: 99, + }, + { + name: "test2", + feedLink: "/go-weekly", + gid: 99, + }, + { + name: "test3", + feedLink: "/go-weekly", + gid: 123, + }, + { + name: "test3", + feedLink: "/go-weekly", + gid: 321, + }, + { + name: "test3", + feedLink: "/go-weekly", + gid: 4123, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + channel, ifExisted, ifSub, err := dm.Subscribe(ctx, tc.gid, tc.feedLink) + if err != nil { + t.Fatal(err) + return + } + t.Logf("[TEST] add sub res: %+v,%+v,%+v\n", channel, ifExisted, ifSub) + res, ext, err := dm.storage.GetIfExistedSubscribe(ctx, tc.gid, tc.feedLink) + if err != nil { + t.Fatal(err) + return + } + t.Logf("[TEST] if exist: %+v,%+v", res, ext) + channels, err := dm.GetSubscribedChannelsByGroupID(ctx, 2) + if err != nil { + t.Fatal(err) + return + } + t.Logf("[TEST] 2 channels: %+v", channels) + // del + //err = dm.Unsubscribe(ctx, tc.gid, tc.feedLink) + //if err != nil { + // t.Fatal(err) + // return + //} + //res, ext, err = dm.storage.GetIfExistedSubscribe(ctx, tc.gid, tc.feedLink) + //if err != nil { + // t.Fatal(err) + // return + //} + //t.Logf("[TEST] after del: %+v,%+v", res, ext) + //if res != nil || ext { + // t.Fatal("delete failed") + //} + + }) + } +} + +func Test_SyncFeed(t *testing.T) { + feed, err := dm.Sync(context.Background()) + if err != nil { + t.Fatal(err) + return + } + rs, _ := json.Marshal(feed) + t.Logf("[Test] feed: %+v", string(rs)) +} diff --git a/plugin/rsshub/domain/storageImpl.go b/plugin/rsshub/domain/storageImpl.go new file mode 100644 index 0000000000..842411defb --- /dev/null +++ b/plugin/rsshub/domain/storageImpl.go @@ -0,0 +1,47 @@ +package domain + +import "context" + +// RepoContent RSS 推送信息存储接口 +type RepoContent interface { + // UpsertContent 添加一条文章 + UpsertContent(ctx context.Context, content *RssContent) error + // DeleteSourceContents 删除订阅源的所有文章,返回被删除的文章数 + DeleteSourceContents(ctx context.Context, channelID int64) (int64, error) + // IsContentHashIDExist hash id 对应的文章是否已存在 + IsContentHashIDExist(ctx context.Context, hashID string) (bool, error) +} + +// RepoSource RSS 订阅源存储接口 +type RepoSource interface { + // UpsertSource 添加一个订阅源 + UpsertSource(ctx context.Context, rfc *RssSource) error + // GetSources 获取所有订阅源信息 + GetSources(ctx context.Context) ([]RssSource, error) + // GetSourceByRssHubFeedLink 通过 rssHub 的 feed 链接获取订阅源信息 + GetSourceByRssHubFeedLink(ctx context.Context, url string) (*RssSource, error) + // DeleteSource 删除一个订阅源 + DeleteSource(ctx context.Context, fID int64) error +} + +// RepoSubscribe RSS 订阅存储接口 +type RepoSubscribe interface { + // CreateSubscribe 添加一个订阅 + CreateSubscribe(ctx context.Context, gid, rssSourceID int64) error + // DeleteSubscribe 删除一个订阅 + DeleteSubscribe(ctx context.Context, subscribeID int64) error + // GetSubscribeByID 获取一个订阅 + GetSubscribeByID(ctx context.Context, gid int64, subscribeID int64) (*RssSubscribe, error) + // GetSubscribes 获取全部订阅 + GetSubscribes(ctx context.Context) ([]*RssSubscribe, error) +} + +// RepoMultiQuery 多表查询接口 +type RepoMultiQuery interface { + // GetSubscribesBySource 获取一个源对应的所有订阅群组 + GetSubscribesBySource(ctx context.Context, feedPath string) ([]*RssSubscribe, error) + // GetIfExistedSubscribe 判断一个群组是否已订阅了一个源 + GetIfExistedSubscribe(ctx context.Context, gid int64, feedPath string) (*RssSubscribe, bool, error) + // GetSubscribedChannelsByGroupID 获取该群所有的订阅 + GetSubscribedChannelsByGroupID(ctx context.Context, gid int64) ([]*RssSource, error) +} diff --git a/plugin/rsshub/domain/storageRepo.go b/plugin/rsshub/domain/storageRepo.go new file mode 100644 index 0000000000..8698a8d996 --- /dev/null +++ b/plugin/rsshub/domain/storageRepo.go @@ -0,0 +1,280 @@ +package domain + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jinzhu/gorm" + "github.com/sirupsen/logrus" +) + +// RepoStorage 定义RepoStorage接口 +type RepoStorage interface { + RepoContent + RepoSource + RepoSubscribe + RepoMultiQuery + initDB() error +} + +// repoStorage db struct for rss +type repoStorage struct { + orm *gorm.DB +} + +// initDB ... +func (s *repoStorage) initDB() (err error) { + err = s.orm.AutoMigrate(&RssSource{}, &RssContent{}, &RssSubscribe{}).Error + if err != nil { + logrus.Errorf("[rsshub initDB] error: %v", err) + return err + } + return nil + // s.orm.LogMode(true) +} + +// GetSubscribesBySource Impl +func (s *repoStorage) GetSubscribesBySource(ctx context.Context, feedPath string) ([]*RssSubscribe, error) { + logrus.WithContext(ctx).Infof("[rsshub GetSubscribesBySource] feedPath: %s", feedPath) + rs := make([]*RssSubscribe, 0) + err := s.orm.Model(&RssSubscribe{}).Joins(fmt.Sprintf("%s left join %s on %s.rss_source_id=%s.id", tableNameRssSubscribe, tableNameRssSource, tableNameRssSubscribe, tableNameRssSource)). + Where("rss_source.rss_hub_feed_path = ?", feedPath).Select("rss_subscribe.*").Find(&rs).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + logrus.WithContext(ctx).Errorf("[rsshub GetSubscribesBySource] error: %v", err) + return nil, err + } + return rs, nil +} + +// GetIfExistedSubscribe Impl +func (s *repoStorage) GetIfExistedSubscribe(ctx context.Context, gid int64, feedPath string) (*RssSubscribe, bool, error) { + rs := RssSubscribe{} + + err := s.orm.Table(tableNameRssSubscribe). + Select("rss_subscribe.id, rss_subscribe.group_id, rss_subscribe.rss_source_id, rss_subscribe.mtime"). + Joins(fmt.Sprintf("INNER JOIN %s ON %s.rss_source_id=%s.id", + tableNameRssSource, tableNameRssSubscribe, tableNameRssSource)). + Where("rss_source.rss_hub_feed_path = ? AND rss_subscribe.group_id = ?", feedPath, gid).Scan(&rs).Error + + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, false, nil + } + logrus.WithContext(ctx).Errorf("[rsshub GetIfExistedSubscribe] error: %v", err) + return nil, false, err + } + if rs.ID == 0 { + return nil, false, nil + } + return &rs, true, nil +} + +// ==================== RepoSource ==================== [Start] + +// UpsertSource Impl +func (s *repoStorage) UpsertSource(ctx context.Context, source *RssSource) (err error) { + // Update columns to default value on `id` conflict + querySource := &RssSource{RssHubFeedPath: source.RssHubFeedPath} + err = s.orm.First(querySource, "rss_hub_feed_path = ?", querySource.RssHubFeedPath).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + err = s.orm.Create(source).Omit("id").Error + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub] add source error: %v", err) + return + } + } + return + } + source.ID = querySource.ID + logrus.WithContext(ctx).Infof("[rsshub] update source: %+v", source.UpdatedParsed) + err = s.orm.Model(&source).Where(&RssSource{ID: source.ID}). + Updates(&RssSource{ + Title: source.Title, + ChannelDesc: source.ChannelDesc, + ImageURL: source.ImageURL, + Link: source.Link, + UpdatedParsed: source.UpdatedParsed, + Mtime: time.Now(), + }).Error + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub] update source error: %v", err) + return + } + logrus.Println("[rsshub] add source success: ", source.ID) + return nil +} + +// GetSources Impl +func (s *repoStorage) GetSources(ctx context.Context) (sources []RssSource, err error) { + sources = []RssSource{} + err = s.orm.Find(&sources, "id > 0").Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, errors.New("source not found") + } + logrus.WithContext(ctx).Errorf("[rsshub] get sources error: %v", err) + return + } + logrus.WithContext(ctx).Infof("[rsshub] get sources success: %d", len(sources)) + return +} + +// GetSourceByRssHubFeedLink Impl +func (s *repoStorage) GetSourceByRssHubFeedLink(ctx context.Context, rssHubFeedLink string) (source *RssSource, err error) { + source = &RssSource{RssHubFeedPath: rssHubFeedLink} + err = s.orm.Take(source, source).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + logrus.WithContext(ctx).Errorf("[rsshub] get source error: %v", err) + return + } + return +} + +// DeleteSource Impl +func (s *repoStorage) DeleteSource(ctx context.Context, fID int64) (err error) { + err = s.orm.Delete(&RssSource{}, "id = ?", fID).Error + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub] storage.DeleteSource: %v", err) + if errors.Is(err, gorm.ErrRecordNotFound) { + return errors.New("source not found") + } + return + } + return nil +} + +// ==================== RepoSource ==================== [End] + +// ==================== RepoContent ==================== [Start] + +// UpsertContent Impl +func (s *repoStorage) UpsertContent(ctx context.Context, content *RssContent) (err error) { + // check params + if content == nil { + err = errors.New("content is nil") + return + } + // check params.RssHubFeedPath and params.HashID + if content.RssSourceID < 0 || content.HashID == "" || content.Title == "" { + err = errors.New("content.RssSourceID or content.HashID or content.Title is empty") + return + } + err = s.orm.Create(content).Omit("id").Error + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub] storage.UpsertContent: %v", err) + return + } + return +} + +// DeleteSourceContents Impl +func (s *repoStorage) DeleteSourceContents(ctx context.Context, channelID int64) (rows int64, err error) { + err = s.orm.Delete(&RssSubscribe{}).Where(&RssSubscribe{RssSourceID: channelID}).Error + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub] storage.DeleteSourceContents: %v", err) + return + } + return +} + +// IsContentHashIDExist Impl +func (s *repoStorage) IsContentHashIDExist(ctx context.Context, hashID string) (bool, error) { + wanted := &RssContent{HashID: hashID} + err := s.orm.Take(wanted, wanted).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return false, nil + } + logrus.WithContext(ctx).Errorf("[rsshub] storage.IsContentHashIDExist: %v", err) + return false, err + } + return true, nil +} + +// ==================== RepoContent ==================== [End] + +// ==================== RepoSubscribe ==================== [Start] + +// CreateSubscribe Impl +func (s *repoStorage) CreateSubscribe(ctx context.Context, gid, rssSourceID int64) (err error) { + // check subscribe + if rssSourceID < 0 || gid == 0 { + err = errors.New("gid or rssSourceID is empty") + return + } + err = s.orm.Create(&RssSubscribe{GroupID: gid, RssSourceID: rssSourceID}).Omit("id").Error + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub] storage.CreateSubscribe: %v", err) + return + } + return +} + +// DeleteSubscribe Impl +func (s *repoStorage) DeleteSubscribe(ctx context.Context, subscribeID int64) (err error) { + err = s.orm.Delete(&RssSubscribe{}, "id = ?", subscribeID).Error + if err != nil { + logrus.WithContext(ctx).Errorf("[rsshub] storage.DeleteSubscribe error: %v", err) + return + } + return +} + +// GetSubscribeByID Impl +func (s *repoStorage) GetSubscribeByID(ctx context.Context, gid int64, subscribeID int64) (res *RssSubscribe, err error) { + res = &RssSubscribe{} + err = s.orm.First(res, &RssSubscribe{GroupID: gid, RssSourceID: subscribeID}).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + logrus.WithContext(ctx).Errorf("[rsshub] storage.GetSubscribeByID: %v", err) + return nil, err + } + return +} + +// GetSubscribedChannelsByGroupID Impl +func (s *repoStorage) GetSubscribedChannelsByGroupID(ctx context.Context, gid int64) (res []*RssSource, err error) { + res = make([]*RssSource, 0) + err = s.orm.Model(&RssSource{}). + Joins(fmt.Sprintf("join %s on rss_source_id=%s.id", tableNameRssSubscribe, tableNameRssSource)).Where("rss_subscribe.group_id = ?", gid). + Select("rss_source.*"). + Find(&res). + Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + err = nil + return + } + logrus.WithContext(ctx).Errorf("[rsshub] storage.GetSubscribedChannelsByGroupID: %v", err) + return + } + return +} + +// GetSubscribes Impl +func (s *repoStorage) GetSubscribes(ctx context.Context) (res []*RssSubscribe, err error) { + res = make([]*RssSubscribe, 0) + err = s.orm.Find(&res).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + err = nil + return + } + logrus.WithContext(ctx).Errorf("[rsshub] storage.GetSubscribes: %v", err) + return + } + return +} + +// ==================== RepoSubscribe ==================== [End] diff --git a/plugin/rsshub/main.go b/plugin/rsshub/main.go new file mode 100644 index 0000000000..26984eae44 --- /dev/null +++ b/plugin/rsshub/main.go @@ -0,0 +1,163 @@ +// Package rsshub rss_hub订阅插件 +package rsshub + +import ( + "context" + "fmt" + "regexp" + + ctrl "github.com/FloatTech/zbpctrl" + "github.com/FloatTech/zbputils/control" + zbpCtxExt "github.com/FloatTech/zbputils/ctxext" + "github.com/sirupsen/logrus" + zero "github.com/wdvxdr1123/ZeroBot" + "github.com/wdvxdr1123/ZeroBot/message" + + "github.com/FloatTech/ZeroBot-Plugin/plugin/rsshub/domain" +) + +// 初始化 repo +var ( + rssRepo domain.RssDomain + initErr error + //// getRssRepo repo 初始化方法,单例 + // getRssRepo = ctxext.DoOnceOnSuccess(func(ctx *zero.Ctx) bool { + // logrus.Infoln("RssHub订阅姬:初始化") + // rssRepo, initErr = domain.NewRssDomain(engine.DataFolder() + "rsshub.db") + // if initErr != nil { + // ctx.SendChain(message.Text("RssHub订阅姬:初始化失败", initErr.Error())) + // return false + // } + // return true + // }) + // regexpForSQL 防注入 + regexpForSQL = regexp.MustCompile(`[\^<>\[\]%&\*\(\)\{\}\|\=]|(union\s+select|update\s+|delete\s+|drop\s+|truncate\s+|insert\s+|exec\s+|declare\s+)`) +) + +var ( + // 注册插件 + engine = control.Register("rsshub", &ctrl.Options[*zero.Ctx]{ + // 默认不启动 + DisableOnDefault: false, + Brief: "RssHub订阅姬", + // 详细帮助 + Help: "RssHub订阅姬desu~ \n" + + "支持的详细订阅列表文档可见:\n" + + "https://rsshub.netlify.app/ \n" + + "- 添加rsshub订阅-/bookfere/weekly \n" + + "- 删除rsshub订阅-/bookfere/weekly \n" + + "- 查看rsshub订阅列表 \n" + + "- rsshub同步 \n" + + "Tips: 定时刷新rsshub订阅信息需要配合job一起使用, 全局只需要设置一个, 无视响应状态推送, 下为例子\n" + + "记录在\"@every 10m\"触发的指令)\n" + + "rsshub同步", + // 插件数据存储路径 + PrivateDataFolder: "rsshub", + OnEnable: func(ctx *zero.Ctx) { + ctx.SendChain(message.Text("RssHub订阅姬现在启动了哦")) + }, + OnDisable: func(ctx *zero.Ctx) { + ctx.SendChain(message.Text("RssHub订阅姬现在关闭了哦")) + }, + }).ApplySingle(zbpCtxExt.DefaultSingle) +) + +// init 命令路由 +func init() { + rssRepo, initErr = domain.NewRssDomain(engine.DataFolder() + "rsshub.db") + if initErr != nil { + logrus.Errorln("RssHub订阅姬:初始化失败", initErr) + panic(initErr) + } + engine.OnFullMatch("rsshub同步", zero.OnlyGroup).SetBlock(true).Handle(func(ctx *zero.Ctx) { + // 群组-频道推送视图 map[群组]推送内容数组 + groupToFeedsMap, err := rssRepo.Sync(context.Background()) + if err != nil { + logrus.Errorln("rsshub同步失败", err) + ctx.SendPrivateMessage(zero.BotConfig.SuperUsers[0], message.Text("rsshub同步失败", err)) + return + } + // 没有更新的[群组-频道推送视图]则不推送 + if len(groupToFeedsMap) == 0 { + logrus.Info("rsshub未发现更新") + return + } + sendRssUpdateMsg(ctx, groupToFeedsMap) + }) + // 添加订阅 + engine.OnRegex(`^添加rsshub订阅-(.+)$`, zero.OnlyGroup).SetBlock(true).Handle(func(ctx *zero.Ctx) { + routeStr := ctx.State["regex_matched"].([]string)[1] + input := regexpForSQL.ReplaceAllString(routeStr, "") + logrus.Debugf("添加rsshub订阅:raw(%s), replaced(%s)", routeStr, input) + rv, _, isSubExisted, err := rssRepo.Subscribe(context.Background(), ctx.Event.GroupID, input) + if err != nil { + ctx.SendChain(message.Text("RssHub订阅姬:添加失败", err.Error())) + return + } + if isSubExisted { + ctx.SendChain(message.Text("RssHub订阅姬:已存在,更新成功")) + } else { + ctx.SendChain(message.Text("RssHub订阅姬:添加成功\n", rv.Source.Title)) + } + // 添加成功,发送订阅源快照 + msg, err := newRssDetailsMsg(ctx, rv) + if len(msg) == 0 || err != nil { + ctx.SendPrivateMessage(zero.BotConfig.SuperUsers[0], message.Text("RssHub推送错误", err)) + return + } + if id := ctx.Send(msg).ID(); id == 0 { + ctx.SendChain(message.Text("ERROR: 发送订阅源快照失败,可能被风控了")) + } + }) + engine.OnRegex(`^删除rsshub订阅-(.+)$`, zero.OnlyGroup).SetBlock(true).Handle(func(ctx *zero.Ctx) { + routeStr := ctx.State["regex_matched"].([]string)[1] + input := regexpForSQL.ReplaceAllString(routeStr, "") + logrus.Debugf("删除rsshub订阅:raw(%s), replaced(%s)", routeStr, input) + err := rssRepo.Unsubscribe(context.Background(), ctx.Event.GroupID, input) + if err != nil { + ctx.SendChain(message.Text("RssHub订阅姬:删除失败 ", err.Error())) + return + } + ctx.SendChain(message.Text(fmt.Sprintf("RssHub订阅姬:删除%s成功", input))) + }) + engine.OnFullMatch("查看rsshub订阅列表", zero.OnlyGroup).SetBlock(true).Handle(func(ctx *zero.Ctx) { + rv, err := rssRepo.GetSubscribedChannelsByGroupID(context.Background(), ctx.Event.GroupID) + if err != nil { + ctx.SendChain(message.Text("RssHub订阅姬:查询失败 ", err.Error())) + return + } + // 添加成功,发送订阅源信息 + msg, err := newRssSourcesMsg(ctx, rv) + if err != nil { + ctx.SendChain(message.Text("RssHub订阅姬:查询失败 ", err.Error())) + return + } + if len(msg) == 0 { + ctx.SendChain(message.Text("ん? 没有订阅的频道哦~")) + return + } + ctx.SendChain(msg...) + }) +} + +// sendRssUpdateMsg 发送Rss更新消息 +func sendRssUpdateMsg(ctx *zero.Ctx, groupToFeedsMap map[int64][]*domain.RssClientView) { + for groupID, views := range groupToFeedsMap { + logrus.Infof("RssHub插件在群 %d 触发推送检查", groupID) + for _, view := range views { + if view == nil || len(view.Contents) == 0 { + continue + } + msg, err := newRssDetailsMsg(ctx, view) + if len(msg) == 0 || err != nil { + ctx.SendPrivateMessage(zero.BotConfig.SuperUsers[0], message.Text(rssHubPushErrMsg, err)) + continue + } + logrus.Infof("RssHub插件在群 %d 开始推送 %s", groupID, view.Source.Title) + ctx.SendGroupMessage(groupID, message.Text(fmt.Sprintf("%s\n该RssHub频道下有更新了哦~", view.Source.Title))) + if res := ctx.SendGroupForwardMessage(groupID, msg); !res.Exists() { + ctx.SendPrivateMessage(zero.BotConfig.SuperUsers[0], message.Text(rssHubPushErrMsg)) + } + } + } +} diff --git a/plugin/rsshub/view.go b/plugin/rsshub/view.go new file mode 100644 index 0000000000..1bc2ae7c9b --- /dev/null +++ b/plugin/rsshub/view.go @@ -0,0 +1,100 @@ +package rsshub + +import ( + "fmt" + "time" + + "github.com/FloatTech/floatbox/binary" + "github.com/FloatTech/zbputils/img/text" + "github.com/sirupsen/logrus" + zero "github.com/wdvxdr1123/ZeroBot" + "github.com/wdvxdr1123/ZeroBot/message" + + "github.com/FloatTech/ZeroBot-Plugin/plugin/rsshub/domain" +) + +const ( + rssHubPushErrMsg = "RssHub推送错误" +) + +// formatRssViewToMessagesSlice 格式化RssClientView为消息切片 +func formatRssViewToMessagesSlice(view *domain.RssClientView) ([]message.Message, error) { + // 取前20条 + cts := view.Contents + if len(cts) > 20 { + cts = cts[:20] + } + // 2n+1条消息 + fv := make([]message.Message, len(cts)*2+1) + // 订阅源头图 + toastPic, err := text.RenderToBase64(fmt.Sprintf("%s\n\n\n%s\n\n\n更新时间:%v\n\n\n", + view.Source.Title, view.Source.Link, view.Source.UpdatedParsed.Local().Format(time.DateTime)), + text.SakuraFontFile, 1200, 40) + if err != nil { + return nil, err + } + fv[0] = message.Message{message.Image("base64://" + binary.BytesToString(toastPic))} + // 元素信息 + for idx, item := range cts { + contentStr := fmt.Sprintf("%s\n\n\n", item.Title) + // Date为空时不显示 + if !item.Date.IsZero() { + contentStr += fmt.Sprintf("更新时间:\n%v\n", item.Date.Local().Format(time.DateTime)) + } + var content []byte + content, err = text.RenderToBase64(contentStr, text.SakuraFontFile, 1200, 40) + if err != nil { + logrus.WithError(err).Error("RssHub订阅姬渲染图片失败") + continue + } + itemMessagePic := message.Message{message.Image("base64://" + binary.BytesToString(content))} + fv[2*idx+1] = itemMessagePic + fv[2*idx+2] = message.Message{message.Text(item.Link)} + } + return fv, nil +} + +// newRssSourcesMsg Rss订阅源列表 +func newRssSourcesMsg(ctx *zero.Ctx, view []*domain.RssClientView) (message.Message, error) { + var msgSlice []message.Message + // 生成消息 + for _, v := range view { + if v == nil { + continue + } + item, err := formatRssViewToMessagesSlice(v) + if err != nil { + return nil, err + } + msgSlice = append(msgSlice, item...) + } + // 伪造一个发送者为RssHub订阅姬的消息节点 + msg := make(message.Message, len(msgSlice)) + for i, item := range msgSlice { + msg[i] = fakeSenderForwardNode(ctx.Event.SelfID, item...) + } + return msg, nil +} + +// newRssDetailsMsg Rss订阅源详情(包含文章信息列表) +func newRssDetailsMsg(ctx *zero.Ctx, view *domain.RssClientView) (message.Message, error) { + // 生成消息 + msgSlice, err := formatRssViewToMessagesSlice(view) + if err != nil { + return nil, err + } + // 伪造一个发送者为RssHub订阅姬的消息节点 + msg := make(message.Message, len(msgSlice)) + for i, item := range msgSlice { + msg[i] = fakeSenderForwardNode(ctx.Event.SelfID, item...) + } + return msg, nil +} + +// fakeSenderForwardNode 伪造一个发送者为RssHub订阅姬的消息节点 +func fakeSenderForwardNode(userID int64, msgs ...message.Segment) message.Segment { + return message.CustomNode( + "RssHub订阅姬", + userID, + msgs) +} From 5cbc9329d376bb9d7c5a0a57dcd0e623dbecc1fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=94=85=E9=A5=AD?= <1156544355@qq.com> Date: Sun, 14 Sep 2025 23:46:01 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=94=A5=20=E5=88=A0=E9=99=A4=E6=B3=A8?= =?UTF-8?q?=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugin/rsshub/domain/model.go | 8 -------- plugin/rsshub/domain/rawFeed.go | 9 --------- plugin/rsshub/main.go | 15 ++------------- 3 files changed, 2 insertions(+), 30 deletions(-) diff --git a/plugin/rsshub/domain/model.go b/plugin/rsshub/domain/model.go index 64c9d33400..8c7d34cec7 100644 --- a/plugin/rsshub/domain/model.go +++ b/plugin/rsshub/domain/model.go @@ -9,8 +9,6 @@ import ( // ======== RSS ========[START] -// type SingleFeedItem gofeed.Item - func genHashForFeedItem(link, guid string) string { idString := link + "||" + guid h := fnv.New32() @@ -51,8 +49,6 @@ type RssSource struct { Link string `gorm:"column:link" json:"link"` // UpdatedParsed RSS页面更新时间 UpdatedParsed time.Time `gorm:"column:updated_parsed" json:"updated_parsed"` - //// Ctime create time - // Ctime int64 `gorm:"column:ctime;default:current_timestamp" json:"ctime"` // Mtime update time Mtime time.Time `gorm:"column:mtime;default:current_timestamp;" json:"mtime"` } @@ -83,8 +79,6 @@ type RssContent struct { Author string `gorm:"column:author" json:"author"` Thumbnail string `gorm:"column:thumbnail" json:"thumbnail"` Content string `gorm:"column:content" json:"content"` - //// Ctime create time - // Ctime int64 `gorm:"column:ctime;default:current_timestamp" json:"ctime"` // Mtime update time Mtime time.Time `gorm:"column:mtime;default:current_timestamp;" json:"mtime"` } @@ -109,8 +103,6 @@ type RssSubscribe struct { GroupID int64 `gorm:"column:group_id;not null;uniqueIndex:uk_sid_gid"` // 订阅频道 RssSourceID int64 `gorm:"column:rss_source_id;not null;uniqueIndex:uk_sid_gid"` - //// Ctime create time - // Ctime int64 `gorm:"column:ctime;default:current_timestamp" json:"ctime"` // Mtime update time Mtime time.Time `gorm:"column:mtime;default:current_timestamp;" json:"mtime"` } diff --git a/plugin/rsshub/domain/rawFeed.go b/plugin/rsshub/domain/rawFeed.go index a8f6ee5d66..0b29fe32b6 100644 --- a/plugin/rsshub/domain/rawFeed.go +++ b/plugin/rsshub/domain/rawFeed.go @@ -12,11 +12,6 @@ import ( "github.com/sirupsen/logrus" ) -// const ( -// acceptHeader = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" -// userHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36 Edg/84.0.522.63" -//) - var ( // RSSHubMirrors RSSHub镜像站地址列表,第一个为默认地址 rssHubMirrors = []string{ @@ -48,10 +43,6 @@ func (c *RssHubClient) FetchFeed(path string) (feed *gofeed.Feed, err error) { logrus.Errorf("[rsshub FetchFeed] fetch feed error: data is empty") return nil, errors.New("feed data is empty") } - // data, err = web.RequestDataWith(c.Client, domain+path, "GET", "", web.RandUA(), nil) - // if err != nil { - // return nil, err - //} feed, err = gofeed.NewParser().Parse(bytes.NewBuffer(data)) if err != nil { return diff --git a/plugin/rsshub/main.go b/plugin/rsshub/main.go index 26984eae44..7803d4ef38 100644 --- a/plugin/rsshub/main.go +++ b/plugin/rsshub/main.go @@ -18,19 +18,8 @@ import ( // 初始化 repo var ( - rssRepo domain.RssDomain - initErr error - //// getRssRepo repo 初始化方法,单例 - // getRssRepo = ctxext.DoOnceOnSuccess(func(ctx *zero.Ctx) bool { - // logrus.Infoln("RssHub订阅姬:初始化") - // rssRepo, initErr = domain.NewRssDomain(engine.DataFolder() + "rsshub.db") - // if initErr != nil { - // ctx.SendChain(message.Text("RssHub订阅姬:初始化失败", initErr.Error())) - // return false - // } - // return true - // }) - // regexpForSQL 防注入 + rssRepo domain.RssDomain + initErr error regexpForSQL = regexp.MustCompile(`[\^<>\[\]%&\*\(\)\{\}\|\=]|(union\s+select|update\s+|delete\s+|drop\s+|truncate\s+|insert\s+|exec\s+|declare\s+)`) ) From c9033f59d290cddb2c2af4704f3bac24d342208c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=94=85=E9=A5=AD?= <1156544355@qq.com> Date: Tue, 16 Sep 2025 23:21:34 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=8E=A8=20=E4=BF=AE=E6=94=B9=E5=91=BD?= =?UTF-8?q?=E4=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugin/rsshub/domain/job.go | 5 +++-- plugin/rsshub/domain/model.go | 7 +++++-- plugin/rsshub/main.go | 8 ++++---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/plugin/rsshub/domain/job.go b/plugin/rsshub/domain/job.go index b6e4fb642c..1763743a11 100644 --- a/plugin/rsshub/domain/job.go +++ b/plugin/rsshub/domain/job.go @@ -58,7 +58,7 @@ func (repo *rssDomain) syncRss(ctx context.Context) (updated map[int64]*RssClien } } var updateChannelView = &RssClientView{Source: cv.Source, Contents: []*RssContent{}} - err = repo.processContentsUpdate(ctx, cv, err, updateChannelView) + err = repo.processContentsUpdate(ctx, cv, updateChannelView) if err != nil { logrus.WithContext(ctx).Errorf("[rsshub syncRss] processContentsUpdate error: %v", err) continue @@ -94,7 +94,8 @@ func (repo *rssDomain) checkSourceNeedUpdate(ctx context.Context, source *RssSou } // processContentsUpdate 处理内容(s)更新 -func (repo *rssDomain) processContentsUpdate(ctx context.Context, cv *RssClientView, err error, updateChannelView *RssClientView) error { +func (repo *rssDomain) processContentsUpdate(ctx context.Context, cv *RssClientView, updateChannelView *RssClientView) error { + var err error for _, content := range cv.Contents { if content == nil { continue diff --git a/plugin/rsshub/domain/model.go b/plugin/rsshub/domain/model.go index 8c7d34cec7..3e3e2cd662 100644 --- a/plugin/rsshub/domain/model.go +++ b/plugin/rsshub/domain/model.go @@ -10,9 +10,12 @@ import ( // ======== RSS ========[START] func genHashForFeedItem(link, guid string) string { - idString := link + "||" + guid h := fnv.New32() - _, _ = h.Write([]byte(idString)) + // 分三次写入数据:link、分隔符、guid + _, _ = h.Write([]byte(link)) + _, _ = h.Write([]byte("||")) + _, _ = h.Write([]byte(guid)) + encoded := hex.EncodeToString(h.Sum(nil)) return encoded } diff --git a/plugin/rsshub/main.go b/plugin/rsshub/main.go index 7803d4ef38..ff4cce1088 100644 --- a/plugin/rsshub/main.go +++ b/plugin/rsshub/main.go @@ -74,8 +74,8 @@ func init() { sendRssUpdateMsg(ctx, groupToFeedsMap) }) // 添加订阅 - engine.OnRegex(`^添加rsshub订阅-(.+)$`, zero.OnlyGroup).SetBlock(true).Handle(func(ctx *zero.Ctx) { - routeStr := ctx.State["regex_matched"].([]string)[1] + engine.OnPrefix("添加rsshub订阅-", zero.OnlyGroup).SetBlock(true).Handle(func(ctx *zero.Ctx) { + routeStr := ctx.State["args"].(string) input := regexpForSQL.ReplaceAllString(routeStr, "") logrus.Debugf("添加rsshub订阅:raw(%s), replaced(%s)", routeStr, input) rv, _, isSubExisted, err := rssRepo.Subscribe(context.Background(), ctx.Event.GroupID, input) @@ -98,8 +98,8 @@ func init() { ctx.SendChain(message.Text("ERROR: 发送订阅源快照失败,可能被风控了")) } }) - engine.OnRegex(`^删除rsshub订阅-(.+)$`, zero.OnlyGroup).SetBlock(true).Handle(func(ctx *zero.Ctx) { - routeStr := ctx.State["regex_matched"].([]string)[1] + engine.OnPrefix("删除rsshub订阅-", zero.OnlyGroup).SetBlock(true).Handle(func(ctx *zero.Ctx) { + routeStr := ctx.State["args"].(string) input := regexpForSQL.ReplaceAllString(routeStr, "") logrus.Debugf("删除rsshub订阅:raw(%s), replaced(%s)", routeStr, input) err := rssRepo.Unsubscribe(context.Background(), ctx.Event.GroupID, input) From fe9db2d70f755a362ba08e47330d83fdb90f6cd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=94=85=E9=A5=AD?= <1156544355@qq.com> Date: Tue, 16 Sep 2025 23:31:51 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E2=9C=8F=EF=B8=8F=20=E4=BF=AE=E6=94=B9lint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugin/rsshub/domain/job.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugin/rsshub/domain/job.go b/plugin/rsshub/domain/job.go index 1763743a11..078523fa10 100644 --- a/plugin/rsshub/domain/job.go +++ b/plugin/rsshub/domain/job.go @@ -53,8 +53,6 @@ func (repo *rssDomain) syncRss(ctx context.Context) (updated map[int64]*RssClien err = repo.storage.UpsertSource(ctx, cv.Source) if err != nil { logrus.WithContext(ctx).Errorf("[rsshub syncRss] upsert source error: %v", err) - err = nil - // continue } } var updateChannelView = &RssClientView{Source: cv.Source, Contents: []*RssContent{}} From ca6bf070755b94574d0986be1a6e8740d5a7ead9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=94=85=E9=A5=AD?= <1156544355@qq.com> Date: Sun, 21 Sep 2025 11:09:29 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=8E=A8=20=E5=88=A0=E9=99=A4=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugin/rsshub/domain/job.go | 8 ++--- plugin/rsshub/domain/rssHub.go | 32 ++++++-------------- plugin/rsshub/domain/storageImpl.go | 47 ----------------------------- plugin/rsshub/domain/storageRepo.go | 9 ------ plugin/rsshub/main.go | 36 +++++++++++----------- 5 files changed, 31 insertions(+), 101 deletions(-) delete mode 100644 plugin/rsshub/domain/storageImpl.go diff --git a/plugin/rsshub/domain/job.go b/plugin/rsshub/domain/job.go index 078523fa10..2b3bc34b3f 100644 --- a/plugin/rsshub/domain/job.go +++ b/plugin/rsshub/domain/job.go @@ -13,7 +13,7 @@ import ( // 1. 获取所有频道 // 2. 遍历所有频道,检查频道是否更新 // 3. 如果更新,获取更新的内容,但是返回的数据 -func (repo *rssDomain) syncRss(ctx context.Context) (updated map[int64]*RssClientView, err error) { +func (repo *RssDomain) syncRss(ctx context.Context) (updated map[int64]*RssClientView, err error) { updated = make(map[int64]*RssClientView) // 获取所有频道 sources, err := repo.storage.GetSources(ctx) @@ -73,7 +73,7 @@ func (repo *rssDomain) syncRss(ctx context.Context) (updated map[int64]*RssClien } // checkSourceNeedUpdate 检查频道是否需要更新 -func (repo *rssDomain) checkSourceNeedUpdate(ctx context.Context, source *RssSource) (needUpdate bool, err error) { +func (repo *RssDomain) checkSourceNeedUpdate(ctx context.Context, source *RssSource) (needUpdate bool, err error) { var sourceInDB *RssSource sourceInDB, err = repo.storage.GetSourceByRssHubFeedLink(ctx, source.RssHubFeedPath) if err != nil { @@ -92,7 +92,7 @@ func (repo *rssDomain) checkSourceNeedUpdate(ctx context.Context, source *RssSou } // processContentsUpdate 处理内容(s)更新 -func (repo *rssDomain) processContentsUpdate(ctx context.Context, cv *RssClientView, updateChannelView *RssClientView) error { +func (repo *RssDomain) processContentsUpdate(ctx context.Context, cv *RssClientView, updateChannelView *RssClientView) error { var err error for _, content := range cv.Contents { if content == nil { @@ -115,7 +115,7 @@ func (repo *rssDomain) processContentsUpdate(ctx context.Context, cv *RssClientV } // processContentItemUpdate 处理单个内容更新 -func (repo *rssDomain) processContentItemUpdate(ctx context.Context, content *RssContent) (existed bool, err error) { +func (repo *RssDomain) processContentItemUpdate(ctx context.Context, content *RssContent) (existed bool, err error) { existed, err = repo.storage.IsContentHashIDExist(ctx, content.HashID) if err != nil { return diff --git a/plugin/rsshub/domain/rssHub.go b/plugin/rsshub/domain/rssHub.go index 1f652032ad..4efcbe2928 100644 --- a/plugin/rsshub/domain/rssHub.go +++ b/plugin/rsshub/domain/rssHub.go @@ -12,31 +12,17 @@ import ( ) // RssDomain RssRepo定义 -type RssDomain interface { - // Subscribe 订阅Rss频道 - Subscribe(ctx context.Context, gid int64, route string) (rv *RssClientView, isChannelExisted, - isSubExisted bool, err error) - // Unsubscribe 取消订阅Rss频道 - Unsubscribe(ctx context.Context, gid int64, route string) (err error) - // GetSubscribedChannelsByGroupID 获取群组订阅的Rss频道 - GetSubscribedChannelsByGroupID(ctx context.Context, gid int64) (rv []*RssClientView, err error) - // Sync 同步Rss频道 - // 返回群组-频道推送视图 map[群组]推送内容数组 - Sync(ctx context.Context) (groupView map[int64][]*RssClientView, err error) -} - -// rssDomain RssRepo定义 -type rssDomain struct { - storage RepoStorage +type RssDomain struct { + storage *repoStorage rssHubClient *RssHubClient } // NewRssDomain 新建RssDomain,调用方保证单例模式 -func NewRssDomain(dbPath string) (RssDomain, error) { +func NewRssDomain(dbPath string) (*RssDomain, error) { return newRssDomain(dbPath) } -func newRssDomain(dbPath string) (*rssDomain, error) { +func newRssDomain(dbPath string) (*RssDomain, error) { if _, err := os.Stat(dbPath); err != nil || os.IsNotExist(err) { // 生成文件 f, err := os.Create(dbPath) @@ -50,7 +36,7 @@ func newRssDomain(dbPath string) (*rssDomain, error) { logrus.Errorf("[rsshub NewRssDomain] open db error: %v", err) panic(err) } - repo := &rssDomain{ + repo := &RssDomain{ storage: &repoStorage{orm: orm}, rssHubClient: &RssHubClient{Client: http.DefaultClient}, } @@ -63,7 +49,7 @@ func newRssDomain(dbPath string) (*rssDomain, error) { } // Subscribe QQ群订阅Rss频道 -func (repo *rssDomain) Subscribe(ctx context.Context, gid int64, feedPath string) ( +func (repo *RssDomain) Subscribe(ctx context.Context, gid int64, feedPath string) ( rv *RssClientView, isChannelExisted, isSubExisted bool, err error) { // 验证 feed, err := repo.rssHubClient.FetchFeed(feedPath) @@ -118,7 +104,7 @@ func (repo *rssDomain) Subscribe(ctx context.Context, gid int64, feedPath string } // Unsubscribe 群组取消订阅 -func (repo *rssDomain) Unsubscribe(ctx context.Context, gid int64, feedPath string) (err error) { +func (repo *RssDomain) Unsubscribe(ctx context.Context, gid int64, feedPath string) (err error) { existedSubscribes, ifExisted, err := repo.storage.GetIfExistedSubscribe(ctx, gid, feedPath) if err != nil { logrus.WithContext(ctx).Errorf("[rsshub Subscribe] query sub by route error: %v", err) @@ -153,7 +139,7 @@ func (repo *rssDomain) Unsubscribe(ctx context.Context, gid int64, feedPath stri } // GetSubscribedChannelsByGroupID 获取群对应的订阅的频道信息 -func (repo *rssDomain) GetSubscribedChannelsByGroupID(ctx context.Context, gid int64) ([]*RssClientView, error) { +func (repo *RssDomain) GetSubscribedChannelsByGroupID(ctx context.Context, gid int64) ([]*RssClientView, error) { channels, err := repo.storage.GetSubscribedChannelsByGroupID(ctx, gid) if err != nil { logrus.WithContext(ctx).Errorf("[rsshub GetSubscribedChannelsByGroupID] GetSubscribedChannelsByGroupID error: %v", err) @@ -170,7 +156,7 @@ func (repo *rssDomain) GetSubscribedChannelsByGroupID(ctx context.Context, gid i } // Sync 同步任务,按照群组订阅情况做好map切片 -func (repo *rssDomain) Sync(ctx context.Context) (groupView map[int64][]*RssClientView, err error) { +func (repo *RssDomain) Sync(ctx context.Context) (groupView map[int64][]*RssClientView, err error) { groupView = make(map[int64][]*RssClientView) // 获取所有Rss频道 // 获取所有频道 diff --git a/plugin/rsshub/domain/storageImpl.go b/plugin/rsshub/domain/storageImpl.go deleted file mode 100644 index 842411defb..0000000000 --- a/plugin/rsshub/domain/storageImpl.go +++ /dev/null @@ -1,47 +0,0 @@ -package domain - -import "context" - -// RepoContent RSS 推送信息存储接口 -type RepoContent interface { - // UpsertContent 添加一条文章 - UpsertContent(ctx context.Context, content *RssContent) error - // DeleteSourceContents 删除订阅源的所有文章,返回被删除的文章数 - DeleteSourceContents(ctx context.Context, channelID int64) (int64, error) - // IsContentHashIDExist hash id 对应的文章是否已存在 - IsContentHashIDExist(ctx context.Context, hashID string) (bool, error) -} - -// RepoSource RSS 订阅源存储接口 -type RepoSource interface { - // UpsertSource 添加一个订阅源 - UpsertSource(ctx context.Context, rfc *RssSource) error - // GetSources 获取所有订阅源信息 - GetSources(ctx context.Context) ([]RssSource, error) - // GetSourceByRssHubFeedLink 通过 rssHub 的 feed 链接获取订阅源信息 - GetSourceByRssHubFeedLink(ctx context.Context, url string) (*RssSource, error) - // DeleteSource 删除一个订阅源 - DeleteSource(ctx context.Context, fID int64) error -} - -// RepoSubscribe RSS 订阅存储接口 -type RepoSubscribe interface { - // CreateSubscribe 添加一个订阅 - CreateSubscribe(ctx context.Context, gid, rssSourceID int64) error - // DeleteSubscribe 删除一个订阅 - DeleteSubscribe(ctx context.Context, subscribeID int64) error - // GetSubscribeByID 获取一个订阅 - GetSubscribeByID(ctx context.Context, gid int64, subscribeID int64) (*RssSubscribe, error) - // GetSubscribes 获取全部订阅 - GetSubscribes(ctx context.Context) ([]*RssSubscribe, error) -} - -// RepoMultiQuery 多表查询接口 -type RepoMultiQuery interface { - // GetSubscribesBySource 获取一个源对应的所有订阅群组 - GetSubscribesBySource(ctx context.Context, feedPath string) ([]*RssSubscribe, error) - // GetIfExistedSubscribe 判断一个群组是否已订阅了一个源 - GetIfExistedSubscribe(ctx context.Context, gid int64, feedPath string) (*RssSubscribe, bool, error) - // GetSubscribedChannelsByGroupID 获取该群所有的订阅 - GetSubscribedChannelsByGroupID(ctx context.Context, gid int64) ([]*RssSource, error) -} diff --git a/plugin/rsshub/domain/storageRepo.go b/plugin/rsshub/domain/storageRepo.go index 8698a8d996..e4c4904ba5 100644 --- a/plugin/rsshub/domain/storageRepo.go +++ b/plugin/rsshub/domain/storageRepo.go @@ -10,15 +10,6 @@ import ( "github.com/sirupsen/logrus" ) -// RepoStorage 定义RepoStorage接口 -type RepoStorage interface { - RepoContent - RepoSource - RepoSubscribe - RepoMultiQuery - initDB() error -} - // repoStorage db struct for rss type repoStorage struct { orm *gorm.DB diff --git a/plugin/rsshub/main.go b/plugin/rsshub/main.go index ff4cce1088..b2d9aca59d 100644 --- a/plugin/rsshub/main.go +++ b/plugin/rsshub/main.go @@ -18,7 +18,7 @@ import ( // 初始化 repo var ( - rssRepo domain.RssDomain + rssRepo *domain.RssDomain initErr error regexpForSQL = regexp.MustCompile(`[\^<>\[\]%&\*\(\)\{\}\|\=]|(union\s+select|update\s+|delete\s+|drop\s+|truncate\s+|insert\s+|exec\s+|declare\s+)`) ) @@ -28,11 +28,11 @@ var ( engine = control.Register("rsshub", &ctrl.Options[*zero.Ctx]{ // 默认不启动 DisableOnDefault: false, - Brief: "RssHub订阅姬", + Brief: "rsshub订阅姬", // 详细帮助 - Help: "RssHub订阅姬desu~ \n" + + Help: "rsshub订阅姬desu~ \n" + "支持的详细订阅列表文档可见:\n" + - "https://rsshub.netlify.app/ \n" + + "https://rsshub.netlify.app/zh/ \n" + "- 添加rsshub订阅-/bookfere/weekly \n" + "- 删除rsshub订阅-/bookfere/weekly \n" + "- 查看rsshub订阅列表 \n" + @@ -43,10 +43,10 @@ var ( // 插件数据存储路径 PrivateDataFolder: "rsshub", OnEnable: func(ctx *zero.Ctx) { - ctx.SendChain(message.Text("RssHub订阅姬现在启动了哦")) + ctx.SendChain(message.Text("rsshub订阅姬现在启动了哦")) }, OnDisable: func(ctx *zero.Ctx) { - ctx.SendChain(message.Text("RssHub订阅姬现在关闭了哦")) + ctx.SendChain(message.Text("rsshub订阅姬现在关闭了哦")) }, }).ApplySingle(zbpCtxExt.DefaultSingle) ) @@ -55,7 +55,7 @@ var ( func init() { rssRepo, initErr = domain.NewRssDomain(engine.DataFolder() + "rsshub.db") if initErr != nil { - logrus.Errorln("RssHub订阅姬:初始化失败", initErr) + logrus.Errorln("rsshub订阅姬:初始化失败", initErr) panic(initErr) } engine.OnFullMatch("rsshub同步", zero.OnlyGroup).SetBlock(true).Handle(func(ctx *zero.Ctx) { @@ -80,18 +80,18 @@ func init() { logrus.Debugf("添加rsshub订阅:raw(%s), replaced(%s)", routeStr, input) rv, _, isSubExisted, err := rssRepo.Subscribe(context.Background(), ctx.Event.GroupID, input) if err != nil { - ctx.SendChain(message.Text("RssHub订阅姬:添加失败", err.Error())) + ctx.SendChain(message.Text("rsshub订阅姬:添加失败", err.Error())) return } if isSubExisted { - ctx.SendChain(message.Text("RssHub订阅姬:已存在,更新成功")) + ctx.SendChain(message.Text("rsshub订阅姬:已存在,更新成功")) } else { - ctx.SendChain(message.Text("RssHub订阅姬:添加成功\n", rv.Source.Title)) + ctx.SendChain(message.Text("rsshub订阅姬:添加成功\n", rv.Source.Title)) } // 添加成功,发送订阅源快照 msg, err := newRssDetailsMsg(ctx, rv) if len(msg) == 0 || err != nil { - ctx.SendPrivateMessage(zero.BotConfig.SuperUsers[0], message.Text("RssHub推送错误", err)) + ctx.SendPrivateMessage(zero.BotConfig.SuperUsers[0], message.Text("rsshub推送错误", err)) return } if id := ctx.Send(msg).ID(); id == 0 { @@ -104,21 +104,21 @@ func init() { logrus.Debugf("删除rsshub订阅:raw(%s), replaced(%s)", routeStr, input) err := rssRepo.Unsubscribe(context.Background(), ctx.Event.GroupID, input) if err != nil { - ctx.SendChain(message.Text("RssHub订阅姬:删除失败 ", err.Error())) + ctx.SendChain(message.Text("rsshub订阅姬:删除失败 ", err.Error())) return } - ctx.SendChain(message.Text(fmt.Sprintf("RssHub订阅姬:删除%s成功", input))) + ctx.SendChain(message.Text(fmt.Sprintf("rsshub订阅姬:删除%s成功", input))) }) engine.OnFullMatch("查看rsshub订阅列表", zero.OnlyGroup).SetBlock(true).Handle(func(ctx *zero.Ctx) { rv, err := rssRepo.GetSubscribedChannelsByGroupID(context.Background(), ctx.Event.GroupID) if err != nil { - ctx.SendChain(message.Text("RssHub订阅姬:查询失败 ", err.Error())) + ctx.SendChain(message.Text("rsshub订阅姬:查询失败 ", err.Error())) return } // 添加成功,发送订阅源信息 msg, err := newRssSourcesMsg(ctx, rv) if err != nil { - ctx.SendChain(message.Text("RssHub订阅姬:查询失败 ", err.Error())) + ctx.SendChain(message.Text("rsshub订阅姬:查询失败 ", err.Error())) return } if len(msg) == 0 { @@ -132,7 +132,7 @@ func init() { // sendRssUpdateMsg 发送Rss更新消息 func sendRssUpdateMsg(ctx *zero.Ctx, groupToFeedsMap map[int64][]*domain.RssClientView) { for groupID, views := range groupToFeedsMap { - logrus.Infof("RssHub插件在群 %d 触发推送检查", groupID) + logrus.Infof("rsshub插件在群 %d 触发推送检查", groupID) for _, view := range views { if view == nil || len(view.Contents) == 0 { continue @@ -142,8 +142,8 @@ func sendRssUpdateMsg(ctx *zero.Ctx, groupToFeedsMap map[int64][]*domain.RssClie ctx.SendPrivateMessage(zero.BotConfig.SuperUsers[0], message.Text(rssHubPushErrMsg, err)) continue } - logrus.Infof("RssHub插件在群 %d 开始推送 %s", groupID, view.Source.Title) - ctx.SendGroupMessage(groupID, message.Text(fmt.Sprintf("%s\n该RssHub频道下有更新了哦~", view.Source.Title))) + logrus.Infof("rsshub插件在群 %d 开始推送 %s", groupID, view.Source.Title) + ctx.SendGroupMessage(groupID, message.Text(fmt.Sprintf("%s\n该rsshub频道下有更新了哦~", view.Source.Title))) if res := ctx.SendGroupForwardMessage(groupID, msg); !res.Exists() { ctx.SendPrivateMessage(zero.BotConfig.SuperUsers[0], message.Text(rssHubPushErrMsg)) }