Merge pull request #45 from beclab/feat/templates

add xhs

Author: kaki-admin

processor/processor.go

@@ -100,7 +100,7 @@ func ArticleContentExtractor(rawContent, entryUrl, feedUrl, rules string) (strin
 	return content, pureContent
 }
 
-func NonRawContentDownloadQueryInArticle(url string) (string, string, string) {
+func NonMediaDownloadQueryInArticle(url string) (string, string, string) {
 	funcs := reflect.ValueOf(&templates.Template{})
 	_, mediaRule := getNonRawContentDownloadScraperRules(url)
 	if mediaRule != "" {
@@ -116,7 +116,7 @@ func NonRawContentDownloadQueryInArticle(url string) (string, string, string) {
 	return "", "", ""
 }
 
-func ExceptYTdlpDownloadQueryInArticle(rawContent, url string) (string, string) {
+func MediaDownloadQueryInArticle(rawContent, url string) (string, string) {
 	templateRawData := strings.NewReader(rawContent)
 	doc, _ := goquery.NewDocumentFromReader(templateRawData)
 	funcs := reflect.ValueOf(&templates.Template{})

processor/rules.go

@@ -184,6 +184,7 @@ var contentTemplatePredefinedRules = map[string]string{
 	"fandom.com":                   "FandomScrapContent",
 	"v2ex.com":                     "V2exScrapContent",
 	"okjike.com":                   "OKjikeScrapContent",
+	"xhslink.com":                  "XhsScrapContent",
 	//"screencrush.com":              "ScreencrushScrapContent",
 	/*"espn.com": "EspnScrapContent",
 	"nbcsports.com":      "NBCSportsScrapContent",
@@ -274,6 +275,7 @@ var metadataTemplatePredefinedRules = map[string]string{
 	"v2ex.com":         "V2exScrapMetaData",
 	"okjike.com":       "OKjikeScrapMetaData",
 	"xiaoyuzhoufm.com": "XiaoyuzhouScrapMetaData",
+	"xhslink.com":      "XhsScrapMetaData",
 }
 
 var publishedAtTimeStampTemplatePredefinedRules = map[string]string{
@@ -326,6 +328,7 @@ var publishedAtTimeStampTemplatePredefinedRules = map[string]string{
 	"reddit.com":                 "RedditPublishedAtTimeFromScriptMetadata",
 	"douban.com":                 "DoubanPublishedAtTimeFromScriptMetadata",
 	"okjike.com":                 "OKjikePublishedAtTimeFromScriptMetadata",
+	"xhslink.com":                "XhsPublishedAtTimeFromScriptMetadata",
 }
 
 func getPredefinedPublishedAtTimestampTemplateRules(websiteURL string) (string, string) {

templates/xhs.go

@@ -0,0 +1,96 @@
+package templates
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+func (t *Template) XhsScrapMetaData(document *goquery.Document) (string, string) {
+	author := ""
+	published_at := ""
+
+	var jsonData string
+	document.Find("script").Each(func(i int, s *goquery.Selection) {
+		scriptContent := s.Text()
+		if strings.Contains(scriptContent, "window.__INITIAL_STATE__") {
+			parts := strings.SplitN(scriptContent, "=", 2)
+			if len(parts) == 2 {
+				jsonData = strings.TrimSpace(parts[1])
+				return
+			}
+		}
+	})
+	jsonData = strings.ReplaceAll(jsonData, "undefined", "null")
+	var result map[string]interface{}
+	if err := json.Unmarshal([]byte(jsonData), &result); err != nil {
+		log.Printf("json unmarshal %v", err)
+	}
+	if note, ok := result["note"]; ok {
+		if noteDetailMap, ok := note.(map[string]interface{})["noteDetailMap"]; ok {
+			for key, item := range noteDetailMap.(map[string]interface{}) {
+				if itemNote, ok := item.(map[string]interface{})["note"]; ok {
+					if user, ok := itemNote.(map[string]interface{})["user"]; ok {
+						if nickname, ok := user.(map[string]interface{})["nickname"]; ok {
+							author = nickname.(string)
+						}
+					}
+				}
+				fmt.Printf("xhs Key: %s\n", key)
+			}
+		}
+	}
+	return author, published_at
+}
+
+func (t *Template) XhsPublishedAtTimeFromScriptMetadata(document *goquery.Document) int64 {
+	var publishedAt int64 = 0
+	var jsonData string
+	document.Find("script").Each(func(i int, s *goquery.Selection) {
+		scriptContent := s.Text()
+		if strings.Contains(scriptContent, "window.__INITIAL_STATE__") {
+			parts := strings.SplitN(scriptContent, "=", 2)
+			if len(parts) == 2 {
+				jsonData = strings.TrimSpace(parts[1])
+				return
+			}
+		}
+	})
+	jsonData = strings.ReplaceAll(jsonData, "undefined", "null")
+	var result map[string]interface{}
+	if err := json.Unmarshal([]byte(jsonData), &result); err != nil {
+		log.Printf("json unmarshal %v", err)
+	}
+	if note, ok := result["note"]; ok {
+		if noteDetailMap, ok := note.(map[string]interface{})["noteDetailMap"]; ok {
+			for key, item := range noteDetailMap.(map[string]interface{}) {
+				if itemNote, ok := item.(map[string]interface{})["note"]; ok {
+					if lastUpdateTime, ok := itemNote.(map[string]interface{})["lastUpdateTime"]; ok {
+						publishedAt = int64(lastUpdateTime.(float64) / 1000)
+					}
+				}
+				fmt.Printf("xhs Key: %s\n", key)
+			}
+		}
+	}
+	return publishedAt
+}
+
+func (t *Template) XhsScrapContent(document *goquery.Document) string {
+	contents := ""
+
+	document.Find("meta[name='og:image']").Each(func(i int, s *goquery.Selection) {
+		if content, exists := s.Attr("content"); exists {
+			contents = contents + "<img src='" + content + "' /> <br>"
+		}
+	})
+	document.Find("span.note-text").Each(func(i int, s *goquery.Selection) {
+		var content string
+		content, _ = goquery.OuterHtml(s)
+		contents += content
+	})
+	return contents
+}