From 956980102bb5943092162941ed8924fcb9ccafbe Mon Sep 17 00:00:00 2001 From: ltaoo Date: Tue, 10 Feb 2026 17:25:20 +0800 Subject: [PATCH] feat: support download article from Wechat OfficialAccount --- _example/official.go | 27 + go.mod | 14 +- go.sum | 69 +- internal/api/handler.go | 35 +- internal/api/routes.go | 3 +- internal/interceptor/inject/eslint.config.mjs | 4 + internal/interceptor/inject/src/components.js | 4 + .../interceptor/inject/src/officialaccount.js | 170 +++-- internal/officialaccount/client.go | 42 +- pkg/gopeed/go.mod | 22 +- pkg/gopeed/go.sum | 110 ++- .../protocol/officialaccount/fetcher.go | 216 ++++++ pkg/gopeed/pkg/download/model.go | 2 + .../pkg/officialaccount/officialaccount.go | 693 ++++++++++++++++++ pkg/gopeed/pkg/officialaccount/type.go | 168 +++++ pkg/gopeed/pkg/officialaccount/util.go | 220 ++++++ .../pkg/protocol/officialaccount/model.go | 11 + 17 files changed, 1698 insertions(+), 112 deletions(-) create mode 100644 _example/official.go create mode 100644 pkg/gopeed/internal/protocol/officialaccount/fetcher.go create mode 100644 pkg/gopeed/pkg/officialaccount/officialaccount.go create mode 100644 pkg/gopeed/pkg/officialaccount/type.go create mode 100644 pkg/gopeed/pkg/officialaccount/util.go create mode 100644 pkg/gopeed/pkg/protocol/officialaccount/model.go diff --git a/_example/official.go b/_example/official.go new file mode 100644 index 0000000..8c2341a --- /dev/null +++ b/_example/official.go @@ -0,0 +1,27 @@ +package main + +import ( + "fmt" + "os" + + officialaccountdownload "github.com/GopeedLab/gopeed/pkg/officialaccount" +) + +func main() { + client := &officialaccountdownload.OfficialAccountDownload{} + + // url := "https://mp.weixin.qq.com/s/dlWe4sStQMKhKXphf8kC2Q" + // url := "https://mp.weixin.qq.com/s/tEeZD35wa92VzEQmN6zXDg" + url := "https://mp.weixin.qq.com/s/906EyV6zAvdCPTUFn2oehw" + article, err := client.FetchArticle(url) + if err != nil { + fmt.Println(err) + return + } + content, err := client.BuildHTMLFromArticle(article, false) + if err != nil { + fmt.Println(err) + return + } + os.WriteFile("article.html", []byte(content), 0644) +} diff --git a/go.mod b/go.mod index aaaf81e..a7c6652 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,8 @@ require ( github.com/spf13/cobra v1.9.1 github.com/spf13/viper v1.21.0 github.com/zeebo/blake3 v0.2.4 - golang.org/x/text v0.32.0 + golang.org/x/net v0.48.0 + golang.org/x/text v0.34.0 ) replace github.com/GopeedLab/gopeed => ./pkg/gopeed @@ -30,11 +31,18 @@ require ( atomicgo.dev/cursor v0.2.0 // indirect atomicgo.dev/keyboard v0.2.9 // indirect atomicgo.dev/schedule v0.1.0 // indirect + github.com/JohannesKaufmann/dom v0.2.0 // indirect + github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0 // indirect + github.com/PuerkitoBio/goquery v1.11.0 // indirect github.com/Trisia/gosysproxy v1.1.0 // indirect + github.com/andybalholm/cascadia v1.3.3 // indirect github.com/bytedance/sonic v1.14.0 // indirect github.com/bytedance/sonic/loader v0.3.0 // indirect + github.com/chai2010/webp v1.4.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/containerd/console v1.0.5 // indirect + github.com/dlclark/regexp2 v1.11.4 // indirect + github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3 // indirect github.com/ebitengine/purego v0.9.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect @@ -43,6 +51,7 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.27.0 // indirect + github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/goccy/go-yaml v1.18.0 // indirect @@ -50,6 +59,7 @@ require ( github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gopacket v1.1.19 // indirect + github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect github.com/gookit/color v1.5.4 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -97,7 +107,7 @@ require ( golang.org/x/arch v0.20.0 // indirect golang.org/x/crypto v0.46.0 // indirect golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect - golang.org/x/net v0.48.0 // indirect + golang.org/x/image v0.36.0 // indirect golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288 // indirect golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.39.0 // indirect diff --git a/go.sum b/go.sum index 9cebccc..75ec183 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,10 @@ atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8= atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ= atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs= atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU= +github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ= +github.com/JohannesKaufmann/dom v0.2.0/go.mod h1:57iSUl5RKric4bUkgos4zu6Xt5LMHUnw3TF1l5CbGZo= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0 h1:mklaPbT4f/EiDr1Q+zPrEt9lgKAkVrIBtWf33d9GpVA= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0/go.mod h1:D56Cl9r8M5i3UwAchE+LlLc5hPN3kJtdZNVJn06lSHU= github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs= github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8= github.com/MarvinJWendt/testza v0.2.8/go.mod h1:nwIcjmr0Zz+Rcwfh3/4UhBp7ePKVhuBExvZqnKYWlII= @@ -15,12 +19,18 @@ github.com/MarvinJWendt/testza v0.3.0/go.mod h1:eFcL4I0idjtIx8P9C6KkAuLgATNKpX4/ github.com/MarvinJWendt/testza v0.4.2/go.mod h1:mSdhXiKH8sg/gQehJ63bINcCKp7RtYewEjXsvsVUPbE= github.com/MarvinJWendt/testza v0.5.2 h1:53KDo64C1z/h/d/stCYCPY69bt/OSwjq5KpFNwi+zB4= github.com/MarvinJWendt/testza v0.5.2/go.mod h1:xu53QFE5sCdjtMCKk8YMQ2MnymimEctc4n3EjyIYvEY= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw= +github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ= github.com/Trisia/gosysproxy v1.1.0 h1:rBU1mazMXLsZiiaAHXtRoDPt2gzWdA6uxjaLskzaoAA= github.com/Trisia/gosysproxy v1.1.0/go.mod h1:PTPPgpRpyTJEL/FMxEE2OcQcGNbUm961xZcgihwraZM= github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78= github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= @@ -28,6 +38,8 @@ github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQ github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/chai2010/webp v1.4.0 h1:6DA2pkkRUPnbOHvvsmGI3He1hBKf/bkRlniAiSGuEko= +github.com/chai2010/webp v1.4.0/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= @@ -38,6 +50,10 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6N github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= +github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3 h1:bVp3yUzvSAJzu9GqID+Z96P+eu5TKnIMJSV4QaZMauM= +github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4= github.com/ebitengine/oto/v3 v3.4.0 h1:br0PgASsEWaoWn38b2Goe7m1GKFYfNgnsjSd5Gg+/bQ= github.com/ebitengine/oto/v3 v3.4.0/go.mod h1:IOleLVD0m+CMak3mRVwsYY8vTctQgOM0iiL6S7Ar7eI= github.com/ebitengine/purego v0.9.0 h1:mh0zpKBIXDceC63hpvPuGLiJ8ZAa3DfrFTudmfi8A4k= @@ -65,6 +81,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= @@ -78,6 +96,7 @@ github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= @@ -88,6 +107,8 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= +github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ= github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo= github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= @@ -180,8 +201,11 @@ github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDc github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw= github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc= +github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= @@ -242,6 +266,8 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJu github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= +github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= @@ -262,14 +288,23 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= +golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc= +golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -278,6 +313,11 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -287,6 +327,10 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -306,15 +350,25 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -323,8 +377,12 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -332,6 +390,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -351,8 +411,9 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/api/handler.go b/internal/api/handler.go index b4f0816..cab011c 100644 --- a/internal/api/handler.go +++ b/internal/api/handler.go @@ -211,7 +211,7 @@ type FeedDownloadTaskBody struct { Suffix string `json:"suffix"` } -func (c *APIClient) handleCreateTask(ctx *gin.Context) { +func (c *APIClient) handleCreateFeedDownloadTask(ctx *gin.Context) { var body FeedDownloadTaskBody if err := ctx.ShouldBindJSON(&body); err != nil { result.Err(ctx, 400, "不合法的参数") @@ -276,6 +276,39 @@ func (c *APIClient) handleCreateTask(ctx *gin.Context) { result.Ok(ctx, gin.H{"id": id}) } +type DownloadTaskPayload struct { + URL string + Filename string + Dir string + Extra map[string]string +} + +func (c *APIClient) handleCreateDownloadTask(ctx *gin.Context) { + var body DownloadTaskPayload + if err := ctx.ShouldBindJSON(&body); err != nil { + result.Err(ctx, 400, "不合法的参数") + return + } + id, err := c.downloader.CreateDirect( + &base.Request{ + URL: body.URL, + Labels: body.Extra, + }, + &base.Options{ + Name: body.Filename, + Path: filepath.Join(c.cfg.DownloadDir, body.Dir), + Extra: &gopeedhttp.OptsExtra{ + Connections: 1, + }, + }, + ) + if err != nil { + result.Err(ctx, 500, "创建任务失败："+err.Error()) + return + } + result.Ok(ctx, gin.H{"id": id}) +} + type DownloadTaskSummary struct { ID string `json:"id"` Title string `json:"title"` diff --git a/internal/api/routes.go b/internal/api/routes.go index a4c1601..2837eb0 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -28,10 +28,11 @@ func (c *APIClient) SetupRoutes() { c.engine.GET("/ws/channels", c.channels.HandleChannelsWebsocket) c.engine.GET("/api/task/list", c.handleFetchTaskList) c.engine.GET("/api/task/profile", c.handleFetchTaskProfile) - c.engine.POST("/api/task/create", c.handleCreateTask) + c.engine.POST("/api/task/create", c.handleCreateFeedDownloadTask) c.engine.POST("/api/task/create_batch", c.handleBatchCreateTask) c.engine.POST("/api/task/create_channels", c.handleCreateChannelsTask) // c.engine.POST("/api/task/create_live", c.handleCreateLiveTask) + c.engine.POST("/api/task/create2", c.handleCreateDownloadTask) c.engine.POST("/api/task/start", c.handleStartTask) c.engine.POST("/api/task/pause", c.handlePauseTask) c.engine.POST("/api/task/resume", c.handleResumeTask) diff --git a/internal/interceptor/inject/eslint.config.mjs b/internal/interceptor/inject/eslint.config.mjs index 0436ce9..c237333 100644 --- a/internal/interceptor/inject/eslint.config.mjs +++ b/internal/interceptor/inject/eslint.config.mjs @@ -42,6 +42,10 @@ export default [ download_icon2: "writable", download_icon3: "writable", download_icon4: "writable", + DownloadIcon5: "writable", + DownloadIcon6: "writable", + DownloadIcon7: "writable", + DownloadIcon8: "writable", RSSIcon: "writeable", download_btn1: "writable", download_btn2: "writable", diff --git a/internal/interceptor/inject/src/components.js b/internal/interceptor/inject/src/components.js index b6a3c4f..f97f1bb 100644 --- a/internal/interceptor/inject/src/components.js +++ b/internal/interceptor/inject/src/components.js @@ -178,6 +178,10 @@ var download_icon1 = ``; var download_icon3 = ``; var download_icon4 = ``; +var DownloadIcon5 = ``; +var DownloadIcon6 = ``; +var DownloadIcon7 = ``; +var DownloadIcon8 = ``; var FileIcon = ``; var MP3Icon = ``; var MP4Icon = ``; diff --git a/internal/interceptor/inject/src/officialaccount.js b/internal/interceptor/inject/src/officialaccount.js index a70b6a6..30621b6 100644 --- a/internal/interceptor/inject/src/officialaccount.js +++ b/internal/interceptor/inject/src/officialaccount.js @@ -78,7 +78,7 @@ JSON.stringify({ id, data: body, - }) + }), ); } if (key === "key:fetch_account_home") { @@ -118,7 +118,7 @@ JSON.stringify({ type: "ping", data: page_title, - }) + }), ); } catch (e) { // ... @@ -131,7 +131,7 @@ JSON.stringify({ type: "ping", data: page_title, - }) + }), ); } catch (e) { // ... @@ -200,6 +200,30 @@ }; return $btn; } + function render_download_button() { + var $btn = document.createElement("div"); + // $btn.className = "sns_opr_btn sns_write_comment_btn bar-expand-hotarea js_wx_tap_highlight wx_tap_link"; + $btn.style.cssText = `display: flex; align-items: center; margin-left: 16px; font-size: 14px; cursor: pointer;`; + $btn.innerHTML = `${DownloadIcon8}下载`; + $btn.onclick = async function () { + var [err, data] = await WXU.request({ + method: "POST", + url: "https://" + FakeAPIServerAddr + "/api/task/create2", + body: { + url: `officialaccount://${window.location.href}`, + // filename: document.title, + }, + }); + if (err) { + WXU.error({ + msg: err.message, + }); + return; + } + WXU.toast("开始下载"); + }; + return $btn; + } function insert_rss_button(acct) { if (!acct.biz || !acct.key) { return; @@ -210,72 +234,82 @@ var $btn = render_rss_button(acct); $container.appendChild($btn); } - insert_style(); + function insert_download_button() { + var $wraps = document.querySelectorAll(".interaction_bar"); + var $container = $wraps[$wraps.length - 1]; + if (!$container || !$container.lastElementChild) { + return; + } + var $btn = render_download_button(); + $container.insertBefore($btn, $container.lastElementChild); + } + async function main() { + if (location.pathname === "/s") { + var _OfficialAccountCredentials = { + nickname: (() => { + if (window.nickname) { + return window.nickname; + } + if (window.cgiData) { + if (window.cgiData.nick_name) { + return window.cgiData.nick_name; + } + } + if (window.cgiDataNew) { + if (window.cgiDataNew.nick_name) { + return window.cgiDataNew.nick_name; + } + } + return ""; + })(), + avatar_url: (() => { + if (window.headimg) { + return window.headimg; + } + if (window.cgiData) { + if (window.cgiData.round_head_img) { + return window.cgiData.round_head_img; + } + if (window.cgiData.hd_head_img) { + return window.cgiData.hd_head_img; + } + } + if (window.cgiDataNew) { + if (window.cgiDataNew.round_head_img) { + return window.cgiDataNew.round_head_img; + } + if (window.cgiDataNew.hd_head_img) { + return window.cgiDataNew.hd_head_img; + } + } + return ""; + })(), + biz: window.biz || window.__biz, + uin: window.uin, + key: window.key, + refresh_uri: (() => { + const params = new URLSearchParams(window.location.search); + const biz = params.get("__biz"); + const mid = params.get("mid"); + const idx = params.get("idx"); + const sn = params.get("sn"); + return `https://mp.weixin.qq.com/s?__biz=${biz}&mid=${mid}&idx=${idx}&sn=${sn}`; + })(), + pass_ticket: window.pass_ticket, + appmsg_token: window.appmsg_token, + }; + WXU.observe_node(".wx_follow_media", () => { + setTimeout(() => { + insert_style(); + // insert_rss_button(_OfficialAccountCredentials); + insert_download_button(); + }, 800); + }); + } + } WXU.onWindowLoaded(() => { - async function main() { - if (location.pathname === "/s") { - var _OfficialAccountCredentials = { - nickname: (() => { - if (window.nickname) { - return window.nickname; - } - if (window.cgiData) { - if (window.cgiData.nick_name) { - return window.cgiData.nick_name; - } - } - if (window.cgiDataNew) { - if (window.cgiDataNew.nick_name) { - return window.cgiDataNew.nick_name; - } - } - return ""; - })(), - avatar_url: (() => { - if (window.headimg) { - return window.headimg; - } - if (window.cgiData) { - if (window.cgiData.round_head_img) { - return window.cgiData.round_head_img; - } - if (window.cgiData.hd_head_img) { - return window.cgiData.hd_head_img; - } - } - if (window.cgiDataNew) { - if (window.cgiDataNew.round_head_img) { - return window.cgiDataNew.round_head_img; - } - if (window.cgiDataNew.hd_head_img) { - return window.cgiDataNew.hd_head_img; - } - } - return ""; - })(), - biz: window.biz || window.__biz, - uin: window.uin, - key: window.key, - refresh_uri: (() => { - const params = new URLSearchParams(window.location.search); - const biz = params.get("__biz"); - const mid = params.get("mid"); - const idx = params.get("idx"); - const sn = params.get("sn"); - return `https://mp.weixin.qq.com/s?__biz=${biz}&mid=${mid}&idx=${idx}&sn=${sn}`; - })(), - pass_ticket: window.pass_ticket, - appmsg_token: window.appmsg_token, - }; - if (!WXU.config.officialServerDisabled) { - connect(_OfficialAccountCredentials); - } - WXU.observe_node(".wx_follow_media", () => { - setTimeout(() => { - insert_rss_button(_OfficialAccountCredentials); - }, 1200); - }); - } + if (WXU.config.officialServerDisabled) { + return; } main(); }); diff --git a/internal/officialaccount/client.go b/internal/officialaccount/client.go index 5f44921..86c4956 100644 --- a/internal/officialaccount/client.go +++ b/internal/officialaccount/client.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "html" "io" "net" "net/http" @@ -25,6 +24,7 @@ import ( "github.com/gin-gonic/gin" "github.com/gorilla/websocket" "github.com/rs/zerolog" + "golang.org/x/net/html" result "wx_channel/internal/util" "wx_channel/pkg/cache" @@ -184,26 +184,26 @@ func NewOfficialAccountClient(cfg *OfficialAccountConfig, parent_logger *zerolog } }() } - if !cfg.RemoteMode && len(c.AccountIdsRefreshInterval) > 0 { - var valid_accounts []string - acct_mu.RLock() - for _, biz := range c.AccountIdsRefreshInterval { - if _, ok := accounts[biz]; ok { - valid_accounts = append(valid_accounts, biz) - } - } - acct_mu.RUnlock() - c.AccountIdsRefreshInterval = valid_accounts - if len(c.AccountIdsRefreshInterval) > 0 { - go func() { - ticker := time.NewTicker(28 * time.Minute) - defer ticker.Stop() - for range ticker.C { - c.RefreshSpecifiedOfficialAccountList(c.AccountIdsRefreshInterval) - } - }() - } - } + // if !cfg.RemoteMode && len(c.AccountIdsRefreshInterval) > 0 { + // var valid_accounts []string + // acct_mu.RLock() + // for _, biz := range c.AccountIdsRefreshInterval { + // if _, ok := accounts[biz]; ok { + // valid_accounts = append(valid_accounts, biz) + // } + // } + // acct_mu.RUnlock() + // c.AccountIdsRefreshInterval = valid_accounts + // if len(c.AccountIdsRefreshInterval) > 0 { + // go func() { + // ticker := time.NewTicker(28 * time.Minute) + // defer ticker.Stop() + // for range ticker.C { + // c.RefreshSpecifiedOfficialAccountList(c.AccountIdsRefreshInterval) + // } + // }() + // } + // } return c } diff --git a/pkg/gopeed/go.mod b/pkg/gopeed/go.mod index 8cce80f..0479552 100644 --- a/pkg/gopeed/go.mod +++ b/pkg/gopeed/go.mod @@ -1,20 +1,28 @@ module github.com/GopeedLab/gopeed -go 1.23.0 - -toolchain go1.23.1 +go 1.24.0 require ( + github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0 + github.com/PuerkitoBio/goquery v1.11.0 + github.com/chai2010/webp v1.4.0 + github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3 github.com/matoous/go-nanoid/v2 v2.0.0 github.com/mattn/go-ieproxy v0.0.12 github.com/rs/zerolog v1.31.0 github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad go.etcd.io/bbolt v1.3.11 golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f - golang.org/x/sync v0.13.0 + golang.org/x/image v0.36.0 + golang.org/x/sync v0.19.0 ) require ( + github.com/JohannesKaufmann/dom v0.2.0 // indirect + github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/dlclark/regexp2 v1.11.4 // indirect + github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect + github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/stretchr/testify v1.10.0 // indirect ) @@ -22,7 +30,7 @@ require ( require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - golang.org/x/net v0.39.0 // indirect - golang.org/x/sys v0.32.0 // indirect - golang.org/x/text v0.24.0 + golang.org/x/net v0.47.0 + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.34.0 ) diff --git a/pkg/gopeed/go.sum b/pkg/gopeed/go.sum index a119c6e..e762a54 100644 --- a/pkg/gopeed/go.sum +++ b/pkg/gopeed/go.sum @@ -1,8 +1,29 @@ +github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ= +github.com/JohannesKaufmann/dom v0.2.0/go.mod h1:57iSUl5RKric4bUkgos4zu6Xt5LMHUnw3TF1l5CbGZo= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0 h1:mklaPbT4f/EiDr1Q+zPrEt9lgKAkVrIBtWf33d9GpVA= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0/go.mod h1:D56Cl9r8M5i3UwAchE+LlLc5hPN3kJtdZNVJn06lSHU= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw= +github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/chai2010/webp v1.4.0 h1:6DA2pkkRUPnbOHvvsmGI3He1hBKf/bkRlniAiSGuEko= +github.com/chai2010/webp v1.4.0/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= +github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3 h1:bVp3yUzvSAJzu9GqID+Z96P+eu5TKnIMJSV4QaZMauM= +github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= +github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/matoous/go-nanoid v1.5.0/go.mod h1:zyD2a71IubI24efhpvkJz+ZwfwagzgSO6UNiFsZKN7U= github.com/matoous/go-nanoid/v2 v2.0.0 h1:d19kur2QuLeHmJBkvYkFdhFBzLoo1XVm2GgTpL+9Tj0= github.com/matoous/go-nanoid/v2 v2.0.0/go.mod h1:FtS4aGPVfEkxKxhdWPAspZpZSh1cOjtM7Ej/So3hR0g= @@ -21,28 +42,101 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc= +github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad h1:QtuWk6dsrNXc/ugwCBEN0jG52q/4tXRzMvZm4fH6T9g= github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad/go.mod h1:Jj8p9bgKGTPQ+M8CdUMS9p7Mmdoxa3OAcAjJQBu0CcI= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= +github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc= +golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/gopeed/internal/protocol/officialaccount/fetcher.go b/pkg/gopeed/internal/protocol/officialaccount/fetcher.go new file mode 100644 index 0000000..3b7f96d --- /dev/null +++ b/pkg/gopeed/internal/protocol/officialaccount/fetcher.go @@ -0,0 +1,216 @@ +package officialaccountdownload + +import ( + "net/http" + "net/url" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/GopeedLab/gopeed/internal/controller" + "github.com/GopeedLab/gopeed/internal/fetcher" + "github.com/GopeedLab/gopeed/pkg/base" + + officialaccountdownload "github.com/GopeedLab/gopeed/pkg/officialaccount" +) + +type Fetcher struct { + fetcher.DefaultFetcher + mu sync.Mutex + closed bool +} + +func (f *Fetcher) Setup(ctl *controller.Controller) { + if err := f.DefaultFetcher.Setup(ctl); err != nil { + panic(err) + } + if f.DefaultFetcher.Meta == nil { + f.DefaultFetcher.Meta = &fetcher.FetcherMeta{} + } +} + +func (f *Fetcher) Meta() *fetcher.FetcherMeta { + return f.DefaultFetcher.Meta +} + +func (f *Fetcher) Resolve(req *base.Request) error { + if f.DefaultFetcher.Meta == nil { + f.DefaultFetcher.Meta = &fetcher.FetcherMeta{} + } + f.DefaultFetcher.Meta.Req = req + + oa := &officialaccountdownload.OfficialAccountDownload{} + + // Handle custom scheme + real_url := req.URL + if strings.HasPrefix(strings.ToLower(real_url), "officialaccount://") { + real_url = real_url[len("officialaccount://"):] + if !strings.HasPrefix(real_url, "http") { + real_url = "https://" + real_url + } + // Create a copy of request with real URL + req_copy := *req + req_copy.URL = real_url + f.DefaultFetcher.Meta.Req = &req_copy + } + + // Fetch the article to get the title + article, err := oa.FetchArticle(f.DefaultFetcher.Meta.Req.URL) + if err != nil { + return err + } + + // Sanitize filename + filename := strings.ReplaceAll(article.Title, "/", "_") + filename = strings.ReplaceAll(filename, "\\", "_") + filename = filename + ".html" + size := int64(article.ContentLength) + if len(article.Images) > 0 { + var wg sync.WaitGroup + var mu sync.Mutex + for _, imgURL := range article.Images { + if imgURL == "" { + continue + } + wg.Add(1) + go func(u string) { + defer wg.Done() + client := &http.Client{ + Timeout: 10 * time.Second, + } + resp, err := client.Head(u) + if err != nil { + return + } + defer resp.Body.Close() + if resp.ContentLength > 0 { + mu.Lock() + size += resp.ContentLength + mu.Unlock() + } + }(imgURL) + } + wg.Wait() + } + + f.DefaultFetcher.Meta.Res = &base.Resource{ + Name: filename, + // Size is unknown/variable because of images, but we can set 0 or estimate + Size: size, + Files: []*base.FileInfo{ + { + Name: filename, + Path: "", + Size: size, + }, + }, + } + + return nil +} + +func (f *Fetcher) Create(opts *base.Options) error { + f.DefaultFetcher.Meta.Opts = opts + return nil +} + +func (f *Fetcher) Start() error { + go func() { + oa := &officialaccountdownload.OfficialAccountDownload{} + // Use the URL from the request and Path from options + destPath := filepath.Join(f.DefaultFetcher.Meta.Opts.Path, f.DefaultFetcher.Meta.Opts.Name) + needCompress := false + if f.DefaultFetcher.Meta.Req.Labels != nil && f.DefaultFetcher.Meta.Req.Labels["compress"] == "true" { + needCompress = true + } + content, err := oa.BuildHTMLFromURL(f.DefaultFetcher.Meta.Req.URL, needCompress) + if err != nil { + f.DoneCh <- err + return + } + + // Save content to file + file, err := f.DefaultFetcher.Ctl.Touch(destPath, int64(len(content))) + if err != nil { + f.DoneCh <- err + return + } + defer file.Close() + + _, err = file.WriteString(content) + f.DoneCh <- err + }() + return nil +} + +func (f *Fetcher) Pause() error { + return nil +} + +func (f *Fetcher) Close() error { + f.mu.Lock() + defer f.mu.Unlock() + f.closed = true + return nil +} + +func (f *Fetcher) Stats() any { + return nil +} + +func (f *Fetcher) Progress() fetcher.Progress { + // Progress tracking might be difficult with current SaveURLAsHTML implementation + // as it doesn't report progress. We return 0 or maybe implemented later. + return fetcher.Progress{0} +} + +// Manager +type FetcherManager struct{} + +func (fm *FetcherManager) Name() string { + return "officialaccount" +} + +func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter { + return []*fetcher.SchemeFilter{ + { + Type: fetcher.FilterTypeUrl, + Pattern: "officialaccount", + }, + } +} + +func (fm *FetcherManager) Build() fetcher.Fetcher { + return &Fetcher{} +} + +func (fm *FetcherManager) ParseName(u string) string { + // Simple parsing, actual name resolution happens in Resolve + parsed, err := url.Parse(u) + if err != nil { + return "article.html" + } + // Maybe extract something from URL if possible, but Resolve will do better + return parsed.Path +} + +func (fm *FetcherManager) AutoRename() bool { + return true +} + +func (fm *FetcherManager) DefaultConfig() any { + return nil +} + +func (fm *FetcherManager) Store(f fetcher.Fetcher) (any, error) { + return nil, nil +} + +func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) { + return nil, nil +} + +func (fm *FetcherManager) Close() error { + return nil +} diff --git a/pkg/gopeed/pkg/download/model.go b/pkg/gopeed/pkg/download/model.go index 5c83c14..27f81fc 100644 --- a/pkg/gopeed/pkg/download/model.go +++ b/pkg/gopeed/pkg/download/model.go @@ -8,6 +8,7 @@ import ( "github.com/GopeedLab/gopeed/internal/controller" "github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/internal/protocol/http" + officialaccountdownload "github.com/GopeedLab/gopeed/internal/protocol/officialaccount" "github.com/GopeedLab/gopeed/internal/protocol/stream" "github.com/GopeedLab/gopeed/internal/protocol/zip" "github.com/GopeedLab/gopeed/pkg/base" @@ -158,6 +159,7 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig { new(http.FetcherManager), new(stream.FetcherManager), new(zip.FetcherManager), + new(officialaccountdownload.FetcherManager), } } if cfg.RefreshInterval == 0 { diff --git a/pkg/gopeed/pkg/officialaccount/officialaccount.go b/pkg/gopeed/pkg/officialaccount/officialaccount.go new file mode 100644 index 0000000..b15165a --- /dev/null +++ b/pkg/gopeed/pkg/officialaccount/officialaccount.go @@ -0,0 +1,693 @@ +package officialaccountdownload + +import ( + "crypto/md5" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2" + "github.com/PuerkitoBio/goquery" + "golang.org/x/net/html" +) + +type OfficialAccountDownload struct { + article *WechatOfficialArticle +} + +func (c *OfficialAccountDownload) SaveURLAsMarkdown(url string, dir_path string) error { + article, err := c.FetchArticle(url) + if err != nil { + return err + } + return c.ConvertHtmlToMarkdown(article, dir_path) +} + +func (c *OfficialAccountDownload) BuildHTMLFromURL(url string, need_compress_img bool) (string, error) { + article := c.article + if article == nil { + r, err := c.FetchArticle(url) + if err != nil { + return "", err + } + article = r + } + return c.BuildHTMLFromArticle(article, need_compress_img) +} + +func (c *OfficialAccountDownload) ConvertHtmlToMarkdown(article *WechatOfficialArticle, dir_path string) error { + // Update the receiver with the fetched article data + // Sanitize filename for the markdown file + filename := strings.ReplaceAll(article.Title, "/", "_") + filename = strings.ReplaceAll(filename, "\\", "_") + + // Create images directory + imagesDirName := "images" + imagesDirPath := filepath.Join(dir_path, imagesDirName) + if err := os.MkdirAll(imagesDirPath, 0755); err != nil { + return err + } + + // Process HTML content to download images and replace links + doc, err := goquery.NewDocumentFromReader(strings.NewReader(article.Content)) + if err != nil { + return err + } + + // Preserve newlines in text nodes by replacing them with a placeholder + // This is needed because HTML parsers and markdown converters often treat newlines as whitespace + newlinePlaceholder := "WECHATNEWLINEHOLDER" + var replaceNewlines func(*html.Node) + replaceNewlines = func(n *html.Node) { + if n.Type == html.TextNode { + if strings.Contains(n.Data, "\n") { + n.Data = strings.ReplaceAll(n.Data, "\n", newlinePlaceholder) + } + return + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + if c.Type == html.ElementNode { + tag := strings.ToLower(c.Data) + // Skip pre-formatted blocks where newlines should be preserved naturally + if tag == "pre" || tag == "code" || tag == "script" || tag == "style" { + continue + } + } + replaceNewlines(c) + } + } + + for _, n := range doc.Nodes { + replaceNewlines(n) + } + + doc.Find("mp-common-mpaudio").Each(func(i int, s *goquery.Selection) { + voiceEncodeFileId := s.AttrOr("voice_encode_fileid", "") + if voiceEncodeFileId != "" { + audioURL := "https://res.wx.qq.com/voice/getvoice?mediaid=" + voiceEncodeFileId + s.AppendHtml(fmt.Sprintf(``, audioURL)) + } + }) + + doc.Find("iframe.video_iframe").Each(func(i int, s *goquery.Selection) { + vid := s.AttrOr("data-vid", "") + if vid == "" { + vid = s.AttrOr("vid", "") + } + if vid == "" { + vid = s.AttrOr("data-mpvid", "") + } + if vid != "" { + for _, video := range article.Videos { + if video.VideoID == vid { + if len(video.MpVideoTransInfo) > 0 { + videoURL := video.MpVideoTransInfo[0].Url + cover := s.AttrOr("data-cover", "") + posterAttr := "" + if cover != "" { + if decodedCover, err := url.QueryUnescape(cover); err == nil { + cover = decodedCover + } + posterAttr = fmt.Sprintf(` poster="%s"`, escapeHTML(cover)) + } + videoHTML := fmt.Sprintf(``, videoURL, posterAttr) + s.ReplaceWithHtml(videoHTML) + } + break + } + } + } + }) + + doc.Find("img").Each(func(i int, s *goquery.Selection) { + imgURL := s.AttrOr("data-src", "") + if imgURL == "" { + imgURL = s.AttrOr("src", "") + } + + if imgURL != "" { + // Download image + localFileName, err := c.downloadImage(imgURL, imagesDirPath) + if err == nil { + // Replace src with local relative path + relativePath := filepath.Join(imagesDirName, localFileName) + s.SetAttr("src", relativePath) + // Remove data-src to ensure markdown converter uses src + s.RemoveAttr("data-src") + } else { + fmt.Printf("Failed to download image %s: %v\n", imgURL, err) + } + } + }) + + newHTML, err := doc.Html() + if err != nil { + return err + } + + // Workaround for
handling: Replace
with a placeholder to ensure it's preserved as a hard break + // html-to-markdown/v2 might handle
differently depending on context or configuration. + // We want explicit hard breaks (two spaces + newline) for every
tag. + brPlaceholder := "WECHATBRHOLDER" + // Replace the newline placeholder with the break placeholder + newHTML = strings.ReplaceAll(newHTML, "WECHATNEWLINEHOLDER", brPlaceholder) + + // goquery normalizes to
but we handle all cases just to be safe + newHTML = strings.ReplaceAll(newHTML, "
", brPlaceholder) + newHTML = strings.ReplaceAll(newHTML, "
", brPlaceholder) + newHTML = strings.ReplaceAll(newHTML, "
", brPlaceholder) + + markdown, err := htmltomarkdown.ConvertString(newHTML) + if err != nil { + return err + } + + // Restore line breaks + markdown = strings.ReplaceAll(markdown, brPlaceholder, " \n") + + // Process additional images from article.Images + if len(article.Images) > 0 { + markdown += "\n\n" + for _, imgURL := range article.Images { + localFileName, err := c.downloadImage(imgURL, imagesDirPath) + if err != nil { + fmt.Printf("Failed to download attached image %s: %v\n", imgURL, err) + continue + } + relative_path := filepath.Join(imagesDirName, localFileName) + markdown += fmt.Sprintf("\n![image](%s)\n", relative_path) + } + } + + if err := os.MkdirAll(dir_path, 0755); err != nil { + return err + } + + filePath := filepath.Join(dir_path, filename+".md") + + if err := os.WriteFile(filePath, []byte(markdown), 0644); err != nil { + return err + } + + return nil +} + +func (c *OfficialAccountDownload) BuildHTMLFromArticle(article *WechatOfficialArticle, need_compress_img bool) (string, error) { + isImageArticle := article.Type == 2 && len(article.Images) > 0 + bodyMaxWidth := "677px" + if isImageArticle { + bodyMaxWidth = "1024px" + } + + var htmlContent strings.Builder + htmlContent.WriteString(` + + + + + `) + htmlContent.WriteString(escapeHTML(article.Title)) + htmlContent.WriteString(` + + +`) + + if isImageArticle { + htmlContent.WriteString(`
`) + for _, imgURL := range article.Images { + imgData, mimeType, err := downloadImageBytes(imgURL) + if err == nil { + if need_compress_img { + // Compress image to reduce size + compressedData, compressedMime, errCompress := compressImage(imgData) + if errCompress == nil { + fmt.Printf("Compressed image %s: %d -> %d bytes (%.2f%%)\n", + imgURL, len(imgData), len(compressedData), float64(len(compressedData))/float64(len(imgData))*100) + imgData = compressedData + mimeType = compressedMime + } else { + fmt.Printf("Failed to compress image %s: %v\n", imgURL, errCompress) + } + } + base64Str := base64.StdEncoding.EncodeToString(imgData) + imgSrc := fmt.Sprintf("data:%s;base64,%s", mimeType, base64Str) + htmlContent.WriteString(fmt.Sprintf(" \"\"\n", imgSrc)) + } else { + fmt.Printf("Failed to download image for base64 %s: %v\n", imgURL, err) + } + } + htmlContent.WriteString(`
`) + } + + htmlContent.WriteString(`

` + article.Title + "

") + creator_html := "" + if article.Creator != "" { + creator_html = `` + article.Creator + `` + } + htmlContent.WriteString(`
` + creator_html + `` + article.AuthorNickname + `` + article.PublishTimeStr + "
") + htmlContent.WriteString(`
`) + // Process HTML content to handle newlines + doc, err := goquery.NewDocumentFromReader(strings.NewReader(article.Content)) + if err != nil { + htmlContent.WriteString(article.Content) + } else { + newlinePlaceholder := "WECHATNEWLINEHOLDER" + var replaceNewlines func(*html.Node) + replaceNewlines = func(n *html.Node) { + if n.Type == html.TextNode { + if strings.Contains(n.Data, "\n") { + n.Data = strings.ReplaceAll(n.Data, "\n", newlinePlaceholder) + } + return + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + if c.Type == html.ElementNode { + tag := strings.ToLower(c.Data) + // Skip pre-formatted blocks + if tag == "pre" || tag == "code" || tag == "script" || tag == "style" { + continue + } + } + replaceNewlines(c) + } + } + + for _, n := range doc.Nodes { + replaceNewlines(n) + } + + doc.Find("mp-common-mpaudio").Each(func(i int, s *goquery.Selection) { + voiceEncodeFileId := s.AttrOr("voice_encode_fileid", "") + if voiceEncodeFileId != "" { + audioURL := "https://res.wx.qq.com/voice/getvoice?mediaid=" + voiceEncodeFileId + name := s.AttrOr("name", "音频") + poster := s.AttrOr("poster", "") + if poster == "" { + poster = s.AttrOr("cover", "") + } + + html := fmt.Sprintf(` +
+
+ cover +
+
+
%s
+ +
+
`, escapeHTML(poster), escapeHTML(name), audioURL) + + s.ReplaceWithHtml(html) + } + }) + + doc.Find("iframe.video_iframe").Each(func(i int, s *goquery.Selection) { + vid := s.AttrOr("data-vid", "") + if vid == "" { + vid = s.AttrOr("vid", "") + } + if vid == "" { + vid = s.AttrOr("data-mpvid", "") + } + if vid != "" { + for _, video := range article.Videos { + if video.VideoID == vid { + if len(video.MpVideoTransInfo) > 0 { + videoURL := video.MpVideoTransInfo[0].Url + cover := s.AttrOr("data-cover", "") + posterAttr := "" + if cover != "" { + if decodedCover, err := url.QueryUnescape(cover); err == nil { + cover = decodedCover + } + posterAttr = fmt.Sprintf(` poster="%s"`, escapeHTML(cover)) + } + videoHTML := fmt.Sprintf(``, videoURL, posterAttr) + s.ReplaceWithHtml(videoHTML) + } + break + } + } + } + }) + + // Process images with data-src for base64 encoding + doc.Find("img").Each(func(i int, s *goquery.Selection) { + imgURL := s.AttrOr("data-src", "") + if imgURL != "" { + imgData, mimeType, err := downloadImageBytes(imgURL) + if err == nil { + if need_compress_img { + // Compress image to reduce size + compressedData, compressedMime, errCompress := compressImage(imgData) + if errCompress == nil { + fmt.Printf("Compressed image %s: %d -> %d bytes (%.2f%%)\n", + imgURL, len(imgData), len(compressedData), float64(len(compressedData))/float64(len(imgData))*100) + imgData = compressedData + mimeType = compressedMime + } else { + fmt.Printf("Failed to compress image %s: %v\n", imgURL, errCompress) + } + } + base64Str := base64.StdEncoding.EncodeToString(imgData) + imgSrc := fmt.Sprintf("data:%s;base64,%s", mimeType, base64Str) + s.SetAttr("src", imgSrc) + s.RemoveAttr("data-src") + } else { + fmt.Printf("Failed to download image for base64 %s: %v\n", imgURL, err) + } + } + }) + + // Get the content inside + newHTML, err := doc.Find("body").Html() + if err != nil { + htmlContent.WriteString(article.Content) + } else { + newHTML = strings.ReplaceAll(newHTML, newlinePlaceholder, "
") + htmlContent.WriteString(newHTML) + } + } + + if isImageArticle { + htmlContent.WriteString("
") + } + + htmlContent.WriteString(` +`) + + return htmlContent.String(), nil +} + +func (c *OfficialAccountDownload) downloadImage(imgURL string, save_dir string) (string, error) { + // Generate filename based on hash of URL + hash := md5.Sum([]byte(imgURL)) + hashStr := hex.EncodeToString(hash[:]) + + // Try to guess extension + ext := ".jpg" // Default + if strings.Contains(imgURL, "wx_fmt=png") { + ext = ".png" + } else if strings.Contains(imgURL, "wx_fmt=gif") { + ext = ".gif" + } else if strings.Contains(imgURL, "wx_fmt=jpeg") { + ext = ".jpg" + } else if strings.Contains(imgURL, "wx_fmt=webp") { + ext = ".webp" + } else { + // Try to parse from URL path if query param not present + u, err := url.Parse(imgURL) + if err == nil { + pathExt := filepath.Ext(u.Path) + if pathExt != "" { + ext = pathExt + } + } + } + + filename := hashStr + ext + filePath := filepath.Join(save_dir, filename) + + // Check if file already exists + if _, err := os.Stat(filePath); err == nil { + return filename, nil + } + + client := &http.Client{} + req, err := http.NewRequest("GET", imgURL, nil) + if err != nil { + return "", err + } + + // Set headers similar to Scrape to avoid anti-hotlinking + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + req.Header.Set("Referer", "https://mp.weixin.qq.com/") + + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("bad status: %s", resp.Status) + } + + file, err := os.Create(filePath) + if err != nil { + return "", err + } + defer file.Close() + + _, err = io.Copy(file, resp.Body) + if err != nil { + return "", err + } + + return filename, nil +} + +func (c *OfficialAccountDownload) FetchArticle(url string) (*WechatOfficialArticle, error) { + content, err := c.Scrape(url) + if err != nil { + return nil, err + } + content_str := string(content) + // Extract createTime + var publish_time_str string + re := regexp.MustCompile(`var\s+createTime\s*=\s*'([^']+)'`) + matches := re.FindStringSubmatch(content_str) + if len(matches) > 1 { + createTime := matches[1] + t, err := time.Parse("2006-01-02 15:04", createTime) + if err == nil { + publish_time_str = t.Format("2006年01月02日 15:04") + } else { + publish_time_str = createTime + } + } + data, err := parse_cgi_datanew(content_str) + if err != nil { + return nil, err + } + article := &WechatOfficialArticle{ + Type: data.PageType, + Title: data.Title, + Content: data.ContentNoEncode, + PublishTimeStr: publish_time_str, + ContentLength: len(data.ContentNoEncode), + Creator: data.Author, + AuthorNickname: data.NickName, + AuthorAvatar: data.RoundHeadImg, + AuthorID: data.UserName, + Images: make([]string, 0), + Videos: data.VideoPageInfos, + } + // isImageArticle := data.PageType == 2 + if len(data.PicturePageInfoList) > 1 { + for _, img := range data.PicturePageInfoList { + if img.CdnUrl != "" { + article.Images = append(article.Images, img.CdnUrl) + } + } + } + c.article = article + return article, nil +} + +func (c *OfficialAccountDownload) Scrape(url string) ([]byte, error) { + if url == "" { + return nil, fmt.Errorf("url is empty") + } + client := &http.Client{} + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + req.Header.Set("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") + req.Header.Set("accept-language", "zh-CN,zh;q=0.9,en;q=0.8") + req.Header.Set("cache-control", "no-cache") + req.Header.Set("pragma", "no-cache") + req.Header.Set("priority", "u=0, i") + req.Header.Set("sec-ch-ua", `"Not(A:Brand";v="8", "Chromium";v="144", "Google Chrome";v="144"`) + req.Header.Set("sec-ch-ua-mobile", "?0") + req.Header.Set("sec-ch-ua-platform", `"macOS"`) + req.Header.Set("sec-fetch-dest", "document") + req.Header.Set("sec-fetch-mode", "navigate") + req.Header.Set("sec-fetch-site", "none") + req.Header.Set("sec-fetch-user", "?1") + req.Header.Set("upgrade-insecure-requests", "1") + req.Header.Set("user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + req.Header.Set("cookie", "ua_id=zfRTujE0WVWbxqqCAAAAAL38AtjljAqWH0xPz_up8gw=; mm_lang=zh_CN; wxuin=69477998648217; xid=27df1e40bdcb601a449dc3afb35016ba; rewardsn=; wxtokenkey=777") + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return body, err +} diff --git a/pkg/gopeed/pkg/officialaccount/type.go b/pkg/gopeed/pkg/officialaccount/type.go new file mode 100644 index 0000000..7d41fda --- /dev/null +++ b/pkg/gopeed/pkg/officialaccount/type.go @@ -0,0 +1,168 @@ +package officialaccountdownload + +type CgiDataNew struct { + BaseResp struct { + Ret int `json:"ret"` + ErrMsg string `json:"errmsg"` + WxToken int `json:"wxtoken"` + CookieCount int `json:"cookie_count"` + SessionID string `json:"sessionid"` + } `json:"base_resp"` + UserName string `json:"user_name"` + NickName string `json:"nick_name"` + RoundHeadImg string `json:"round_head_img"` + Title string `json:"title"` + Desc string `json:"desc"` + ContentNoEncode string `json:"content_noencode"` + CreateTime string `json:"create_time"` + CdnUrl string `json:"cdn_url"` + Link string `json:"link"` + SourceUrl string `json:"source_url"` + CanShare int `json:"can_share"` + Alias string `json:"alias"` + Type int `json:"type"` + Author string `json:"author"` + IsLimitUser int `json:"is_limit_user"` + ShowCoverPic int `json:"show_cover_pic"` + AdvertisementNum int `json:"advertisement_num"` + AdvertisementInfo []any `json:"advertisement_info"` + OriCreateTime int `json:"ori_create_time"` + UserUin int `json:"user_uin"` + TotalItemNum int `json:"total_item_num"` + IsAsync int `json:"is_async"` + CommentID string `json:"comment_id"` + ImgFormat string `json:"img_format"` + SvrTime int `json:"svr_time"` + CopyrightInfo struct { + CopyrightStat int `json:"copyright_stat"` + IsCartoonCopyright int `json:"is_cartoon_copyright"` + } `json:"copyright_info"` + CanReward int `json:"can_reward"` + Signature string `json:"signature"` + RewardWording string `json:"reward_wording"` + InMm int `json:"in_mm"` + AppID string `json:"app_id"` + ShowComment int `json:"show_comment"` + CanUsePage int `json:"can_use_page"` + HdHeadImg string `json:"hd_head_img"` + DelReasonID int `json:"del_reason_id"` + SrcID string `json:"srcid"` + IsWxgStuffUin int `json:"is_wxg_stuff_uin"` + NeedReportCost int `json:"need_report_cost"` + BizUin string `json:"bizuin"` + Mid int `json:"mid"` + Idx int `json:"idx"` + Sn string `json:"sn"` + UseTxVideoPlayer int `json:"use_tx_video_player"` + IsOnlyRead int `json:"is_only_read"` + ReqID string `json:"req_id"` + UseOuterLink int `json:"use_outer_link"` + BanScene int `json:"ban_scene"` + CspNonceStr int `json:"csp_nonce_str"` + MsgDailyIdx int `json:"msg_daily_idx"` + OriHeadImgUrl string `json:"ori_head_img_url"` + FilterTime int `json:"filter_time"` + AppmsgFeFilter string `json:"appmsg_fe_filter"` + IsLogin int `json:"is_login"` + RewardMoney int `json:"reward_money"` + PageType int `json:"page_type"` + ItemShowType int `json:"item_show_type"` + VoiceInAppmsg []any `json:"voice_in_appmsg"` + VideoPageInfo struct { + MpVideoTransInfo []any `json:"mp_video_trans_info"` + DramaVideoInfo struct { + } `json:"drama_video_info"` + DramaInfo struct { + } `json:"drama_info"` + } `json:"video_page_info"` + MaliciousTitleReasonID int `json:"malicious_title_reason_id"` + VideoPageInfos []VideoPageInfoItem `json:"video_page_infos"` + PicturePageInfoList []PicturePageInfo `json:"picture_page_info_list"` +} + +type VideoPageInfoItem struct { + VideoID string `json:"video_id"` + OriStatus int `json:"ori_status"` + SourceBizuin string `json:"source_bizuin"` + SourceNickname string `json:"source_nickname"` + SourceLink string `json:"source_link"` + IsMpVideo int `json:"is_mp_video"` + IsMpVideoDelete int `json:"is_mp_video_delete"` + MpVideoOverseasLimit int `json:"mp_video_overseas_limit"` + IsMpVideoForbid int `json:"is_mp_video_forbid"` + MpVideoForbidReason int `json:"mp_video_forbid_reason"` + HitUsername int `json:"hit_username"` + HitBizuin int `json:"hit_bizuin"` + HitVid string `json:"hit_vid"` + ContentNoencode string `json:"content_noencode"` + IsMpVideoUrgentState int `json:"is_mp_video_urgent_state"` + IsAppmsgUnauthorized int `json:"is_appmsg_unauthorized"` + CoverUrl string `json:"cover_url"` + CoverUrl11 string `json:"cover_url_1_1"` + CoverUrl169 string `json:"cover_url_16_9"` + VideoidUploadtime int `json:"videoid_uploadtime"` + VideoidBizuin int `json:"videoid_bizuin"` + MpVideoTransInfo []MpVideoTransInfo `json:"mp_video_trans_info"` +} + +type MpVideoTransInfo struct { + DurationMs int `json:"duration_ms"` + Filesize any `json:"filesize"` + FormatID int `json:"format_id"` + Height int `json:"height"` + Url string `json:"url"` + VideoQualityLevel int `json:"video_quality_level"` + VideoQualityWording string `json:"video_quality_wording"` + Width int `json:"width"` +} + +type PicturePageInfo struct { + CdnUrl string `json:"cdn_url"` + Width int `json:"width"` + Height int `json:"height"` + ThemeColor string `json:"theme_color"` + IsQrCode int `json:"is_qr_code"` + PoiInfo []any `json:"poi_info"` + WxaInfo []any `json:"wxa_info"` + LivePhoto struct { + FormatInfo []any `json:"format_info"` + } `json:"live_photo"` + DisableThemeColor bool `json:"disable_theme_color"` + BindAdInfo []any `json:"bind_ad_info"` + CpsAdInfo []any `json:"cps_ad_info"` + PicWindowProduct struct { + ProductEncryptKey string `json:"product_encrypt_key"` + ProductType int `json:"product_type"` + Title string `json:"title"` + DataType int `json:"data_type"` + ProductID string `json:"product_id"` + } `json:"pic_window_product"` + ShowWatermark bool `json:"show_watermark"` + BottomRightBrightness float64 `json:"bottom_right_brightness"` + WatermarkInfo struct { + CdnUrl string `json:"cdn_url"` + IsUploader bool `json:"is_uploader"` + } `json:"watermark_info"` + SpotProductInfo []any `json:"spot_product_info"` + ShareCover struct { + FileID int `json:"file_id"` + Width int `json:"width"` + Height int `json:"height"` + CdnUrl string `json:"cdn_url"` + CropInfo string `json:"crop_info"` + } `json:"share_cover"` +} + +type WechatOfficialArticle struct { + Type int `json:"type"` + Title string `json:"title"` + Content string `json:"content"` + ContentLength int `json:"content_length"` + Images []string `json:"images"` + Creator string `json:"creator"` + AuthorNickname string `json:"author_nickname"` + AuthorAvatar string `json:"author_avatar"` + AuthorID string `json:"author_id"` + PublishTimeStr string `json:"publish_time_str"` + Videos []VideoPageInfoItem `json:"videos"` +} diff --git a/pkg/gopeed/pkg/officialaccount/util.go b/pkg/gopeed/pkg/officialaccount/util.go new file mode 100644 index 0000000..a5a25c0 --- /dev/null +++ b/pkg/gopeed/pkg/officialaccount/util.go @@ -0,0 +1,220 @@ +package officialaccountdownload + +import ( + "bytes" + "encoding/json" + "fmt" + "image" + "image/jpeg" + "io" + "net/http" + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/chai2010/webp" + "github.com/dop251/goja" + "golang.org/x/image/draw" +) + +func parse_cgi_datanew(htmlContent string) (*CgiDataNew, error) { + doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent)) + if err != nil { + return nil, err + } + + var scripts []string + doc.Find("script").Each(func(i int, s *goquery.Selection) { + text := s.Text() + if strings.Contains(text, "window.cgiDataNew =") || strings.Contains(text, "var videoPageInfos =") { + scripts = append(scripts, text) + } + }) + + if len(scripts) == 0 { + return nil, fmt.Errorf("cgiDataNew script not found") + } + + vm := goja.New() + + // Mock window + vm.RunString("var window = {};") + // Mock document and basic DOM/jQuery environment to prevent script errors + vm.RunString(` + var document = { + getElementById: function() { return {}; }, + getElementsByTagName: function() { return []; }, + createElement: function() { return {}; }, + head: {} + }; + var console = { log: function() {}, warn: function() {}, error: function() {} }; + var mockDom = { + html: function() { return mockDom; }, + text: function() { return mockDom; }, + val: function() { return mockDom; }, + attr: function() { return mockDom; }, + find: function() { return mockDom; }, + css: function() { return mockDom; }, + addClass: function() { return mockDom; }, + removeClass: function() { return mockDom; }, + show: function() { return mockDom; }, + hide: function() { return mockDom; }, + append: function() { return mockDom; } + }; + var $ = function() { return mockDom; }; + var jQuery = $; + `) + + // Mock JsDecode + // In the browser, JsDecode seems to decode strings, but the strings in the script + // are often already just string literals. If they contain escape sequences, + // the JS parser handles them. + // We'll treat it as an identity function for now. + vm.Set("JsDecode", func(call goja.FunctionCall) goja.Value { + return call.Argument(0) + }) + + // Run the scripts + var scriptErrs []error + for _, script := range scripts { + _, err = vm.RunString(script) + if err != nil { + scriptErrs = append(scriptErrs, err) + } + } + + // Extract cgiDataNew + val := vm.Get("window").ToObject(vm).Get("cgiDataNew") + if val == nil { + if len(scriptErrs) > 0 { + for _, err := range scriptErrs { + fmt.Printf("failed to run script: %v\n", err) + } + } + return nil, fmt.Errorf("window.cgiDataNew is nil") + } + + // Convert to JSON string + jsonStr, err := vm.RunString("JSON.stringify(window.cgiDataNew)") + if err != nil { + return nil, fmt.Errorf("failed to stringify cgiDataNew: %v", err) + } + + data := &CgiDataNew{} + if err := json.Unmarshal([]byte(jsonStr.String()), data); err != nil { + return nil, fmt.Errorf("failed to unmarshal cgiDataNew: %v", err) + } + + // Check if PicturePageInfoList is empty, if so check global window.picture_page_info_list + if len(data.PicturePageInfoList) == 0 { + valList := vm.Get("window").ToObject(vm).Get("picture_page_info_list") + if valList != nil && !goja.IsNull(valList) && !goja.IsUndefined(valList) { + jsonList, err := vm.RunString("JSON.stringify(window.picture_page_info_list)") + if err == nil { + var list []PicturePageInfo + if err := json.Unmarshal([]byte(jsonList.String()), &list); err == nil { + data.PicturePageInfoList = list + } + } + } + } + + // Check for videoPageInfos + valVideo := vm.Get("videoPageInfos") + if valVideo != nil && !goja.IsNull(valVideo) && !goja.IsUndefined(valVideo) { + jsonVideo, err := vm.RunString("JSON.stringify(videoPageInfos)") + if err == nil { + var videoInfos []VideoPageInfoItem + if err := json.Unmarshal([]byte(jsonVideo.String()), &videoInfos); err == nil { + data.VideoPageInfos = videoInfos + } + } + } + + return data, nil +} + +func escapeHTML(s string) string { + s = strings.ReplaceAll(s, "&", "&") + s = strings.ReplaceAll(s, "<", "<") + s = strings.ReplaceAll(s, ">", ">") + s = strings.ReplaceAll(s, "\"", """) + s = strings.ReplaceAll(s, "'", "'") + return s +} + +func compressImage(data []byte) ([]byte, string, error) { + img, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return nil, "", err + } + + // Resize if needed + bounds := img.Bounds() + width := bounds.Dx() + height := bounds.Dy() + maxWidth := 640 // Reduced from 800 to 640 + + var newImg image.Image = img + if width > maxWidth { + newHeight := height * maxWidth / width + dst := image.NewRGBA(image.Rect(0, 0, maxWidth, newHeight)) + draw.CatmullRom.Scale(dst, dst.Bounds(), img, bounds, draw.Over, nil) + newImg = dst + } + + // Encode to WebP + var buf bytes.Buffer + // Quality 50 provides a good balance between size and visual quality for web usage + err = webp.Encode(&buf, newImg, &webp.Options{Lossless: false, Quality: 50}) + if err != nil { + // Fallback to JPEG if WebP encoding fails (though unlikely) + fmt.Printf("WebP encoding failed, falling back to JPEG: %v\n", err) + buf.Reset() + + // Create white background for JPEG + bg := image.NewRGBA(newImg.Bounds()) + draw.Draw(bg, bg.Bounds(), &image.Uniform{image.White}, image.Point{}, draw.Src) + draw.Draw(bg, bg.Bounds(), newImg, newImg.Bounds().Min, draw.Over) + finalImg := bg + + err = jpeg.Encode(&buf, finalImg, &jpeg.Options{Quality: 40}) + if err != nil { + return nil, "", err + } + return buf.Bytes(), "image/jpeg", nil + } + + return buf.Bytes(), "image/webp", nil +} + +func downloadImageBytes(imgURL string) ([]byte, string, error) { + client := &http.Client{} + req, err := http.NewRequest("GET", imgURL, nil) + if err != nil { + return nil, "", err + } + + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + req.Header.Set("Referer", "https://mp.weixin.qq.com/") + + resp, err := client.Do(req) + if err != nil { + return nil, "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, "", fmt.Errorf("bad status: %s", resp.Status) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, "", err + } + + contentType := resp.Header.Get("Content-Type") + if contentType == "" { + contentType = http.DetectContentType(data) + } + return data, contentType, nil +} diff --git a/pkg/gopeed/pkg/protocol/officialaccount/model.go b/pkg/gopeed/pkg/protocol/officialaccount/model.go new file mode 100644 index 0000000..b72a1c1 --- /dev/null +++ b/pkg/gopeed/pkg/protocol/officialaccount/model.go @@ -0,0 +1,11 @@ +package officialaccount + +type ReqExtra struct { +} + +type OptsExtra struct { +} + +// Stats for download +type Stats struct { +}