elmgates/res-downloader
1
0
Fork 0
mirror of https://github.com/putyy/res-downloader.git synced 2026-01-12 14:14:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

perf: core optimize(add plugins)

Browse Source
This commit is contained in:
putyy
2025-05-14 17:51:36 +08:00
parent f0495c6858
commit c1cce920a4
15 changed files with 784 additions and 403 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
core/app.go
View File

@@ -8,6 +8,7 @@ import (
"os"
"path/filepath"
"regexp"
"res-downloader/core/shared"
"strconv"
"time"
)
@@ -170,7 +171,7 @@ func (a *App) UnsetSystemProxy() error {
}
func (a *App) isInstall() bool {
return FileExist(a.LockFile)
return shared.FileExist(a.LockFile)
}
func (a *App) lock() error {

51
core/config.go
View File

@@ -179,3 +179,54 @@ func (c *Config) setConfig(config Config) {
_ = globalConfig.storage.Store(jsonData)
}
}
func (c *Config) getConfig(key string) interface{} {
switch key {
case "Host":
return c.Host
case "Port":
return c.Port
case "Theme":
return c.Theme
case "Locale":
return c.Locale
case "Quality":
return c.Quality
case "SaveDirectory":
return c.SaveDirectory
case "FilenameLen":
return c.FilenameLen
case "FilenameTime":
return c.FilenameTime
case "UpstreamProxy":
return c.UpstreamProxy
case "UserAgent":
return c.UserAgent
case "OpenProxy":
return c.OpenProxy
case "DownloadProxy":
return c.DownloadProxy
case "AutoProxy":
return c.AutoProxy
case "TaskNumber":
return c.TaskNumber
case "WxAction":
return c.WxAction
case "UseHeaders":
return c.UseHeaders
case "MimeMap":
return c.MimeMap
default:
return nil
}
}
func (c *Config) TypeSuffix(mime string) (string, string) {
mimeMux.RLock()
defer mimeMux.RUnlock()
mime = strings.ToLower(strings.Split(mime, ";")[0])
if v, ok := c.MimeMap[mime]; ok {
return v.Type, v.Suffix
}
return "", ""
}

228
core/downloader.go
View File

@@ -1,25 +1,49 @@
package core
import (
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"time"
)
type ProgressCallback func(totalDownloaded float64, totalSize float64)
// 定义常量
const (
MaxRetries = 3 // 最大重试次数
RetryDelay = 3 * time.Second // 重试延迟
MinPartSize = 1 * 1024 * 1024 // 最小分片大小（1MB）
)
// 错误定义
var (
ErrInvalidFileSize = errors.New("invalid file size")
ErrTaskFailed = errors.New("download task failed")
ErrIncompleteDownload = errors.New("incomplete download")
)
// 进度回调函数
type ProgressCallback func(totalDownloaded float64, totalSize float64, taskID int, taskProgress float64)
// 进度通道
type ProgressChan struct {
taskID int
bytes int64
}
// 下载任务
type DownloadTask struct {
taskID int
rangeStart int64
rangeEnd int64
downloadedSize int64
isCompleted bool
err error
}
type FileDownloader struct {
@@ -49,12 +73,16 @@ func NewFileDownloader(url, filename string, totalTasks int, headers map[string]
}
func (fd *FileDownloader) buildClient() *http.Client {
transport := &http.Transport{}
transport := &http.Transport{
MaxIdleConnsPerHost: 100,
IdleConnTimeout: 90 * time.Second,
}
if fd.ProxyUrl != nil {
transport.Proxy = http.ProxyURL(fd.ProxyUrl)
}
return &http.Client{
Transport: transport,
Timeout: 60 * time.Second,
}
}
@@ -69,7 +97,7 @@ func (fd *FileDownloader) setHeaders(request *http.Request) {
func (fd *FileDownloader) init() error {
parsedURL, err := url.Parse(fd.Url)
if err != nil {
return err
return fmt.Errorf("parse URL failed: %w", err)
}
if parsedURL.Scheme != "" && parsedURL.Host != "" {
fd.Referer = parsedURL.Scheme + "://" + parsedURL.Host + "/"
@@ -95,23 +123,36 @@ func (fd *FileDownloader) init() error {
}
fd.setHeaders(request)
resp, err := fd.buildClient().Do(request)
var resp *http.Response
for retries := 0; retries < MaxRetries; retries++ {
resp, err = fd.buildClient().Do(request)
if err == nil {
break
}
if retries < MaxRetries-1 {
time.Sleep(RetryDelay)
globalLogger.Warn().Msgf("HEAD request failed, retrying (%d/%d): %v", retries+1, MaxRetries, err)
}
}
if err != nil {
return fmt.Errorf("HEAD request failed: %w", err)
return fmt.Errorf("HEAD request failed after %d retries: %w", MaxRetries, err)
}
defer resp.Body.Close()
fd.TotalSize = resp.ContentLength
if fd.TotalSize <= 0 {
return fmt.Errorf("invalid file size")
return ErrInvalidFileSize
}
if resp.Header.Get("Accept-Ranges") == "bytes" && fd.TotalSize > 10*1024*1024 {
if resp.Header.Get("Accept-Ranges") == "bytes" && fd.TotalSize > MinPartSize {
fd.IsMultiPart = true
}
dir := filepath.Dir(fd.FileName)
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
return err
return fmt.Errorf("create directory failed: %w", err)
}
fd.File, err = os.OpenFile(fd.FileName, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
@@ -126,10 +167,18 @@ func (fd *FileDownloader) init() error {
func (fd *FileDownloader) createDownloadTasks() {
if fd.IsMultiPart {
if int64(fd.totalTasks) > fd.TotalSize {
fd.totalTasks = int(fd.TotalSize)
if fd.totalTasks <= 0 {
fd.totalTasks = 4
}
eachSize := fd.TotalSize / int64(fd.totalTasks)
if eachSize < MinPartSize {
fd.totalTasks = int(fd.TotalSize / MinPartSize)
if fd.totalTasks < 1 {
fd.totalTasks = 1
}
eachSize = fd.TotalSize / int64(fd.totalTasks)
}
for i := 0; i < fd.totalTasks; i++ {
start := eachSize * int64(i)
end := eachSize*int64(i+1) - 1
@@ -143,91 +192,186 @@ func (fd *FileDownloader) createDownloadTasks() {
})
}
} else {
fd.DownloadTaskList = append(fd.DownloadTaskList, &DownloadTask{taskID: 0})
fd.totalTasks = 1
fd.DownloadTaskList = append(fd.DownloadTaskList, &DownloadTask{
taskID: 0,
rangeStart: 0,
rangeEnd: fd.TotalSize - 1,
})
}
}
func (fd *FileDownloader) startDownload() {
func (fd *FileDownloader) startDownload() error {
wg := &sync.WaitGroup{}
progressChan := make(chan int64)
progressChan := make(chan ProgressChan, len(fd.DownloadTaskList))
errorChan := make(chan error, len(fd.DownloadTaskList))
for _, task := range fd.DownloadTaskList {
wg.Add(1)
go fd.startDownloadTask(wg, progressChan, task)
go fd.startDownloadTask(wg, progressChan, errorChan, task)
}
go func() {
taskProgress := make([]int64, len(fd.DownloadTaskList))
totalDownloaded := int64(0)
for progress := range progressChan {
taskProgress[progress.taskID] += progress.bytes
totalDownloaded += progress.bytes
if fd.progressCallback != nil {
taskPercentage := float64(0)
if task := fd.DownloadTaskList[progress.taskID]; task != nil {
taskSize := task.rangeEnd - task.rangeStart + 1
if taskSize > 0 {
taskPercentage = float64(taskProgress[progress.taskID]) / float64(taskSize) * 100
}
}
fd.progressCallback(float64(totalDownloaded), float64(fd.TotalSize), progress.taskID, taskPercentage)
}
}
}()
go func() {
wg.Wait()
close(progressChan)
close(errorChan)
}()
if fd.progressCallback != nil {
totalDownloaded := int64(0)
for p := range progressChan {
totalDownloaded += p
fd.progressCallback(float64(totalDownloaded), float64(fd.TotalSize))
var errArr []error
for err := range errorChan {
errArr = append(errArr, err)
}
if len(errArr) > 0 {
return fmt.Errorf("download failed with %d errors: %v", len(errArr), errArr[0])
}
if err := fd.verifyDownload(); err != nil {
return err
}
return nil
}
func (fd *FileDownloader) startDownloadTask(wg *sync.WaitGroup, progressChan chan int64, task *DownloadTask) {
func (fd *FileDownloader) startDownloadTask(wg *sync.WaitGroup, progressChan chan ProgressChan, errorChan chan error, task *DownloadTask) {
defer wg.Done()
for retries := 0; retries < MaxRetries; retries++ {
err := fd.doDownloadTask(progressChan, task)
if err == nil {
task.isCompleted = true
return
}
task.err = err
globalLogger.Warn().Msgf("Task %d failed (attempt %d/%d): %v", task.taskID, retries+1, MaxRetries, err)
if retries < MaxRetries-1 {
time.Sleep(RetryDelay)
}
}
errorChan <- fmt.Errorf("task %d failed after %d attempts: %v", task.taskID, MaxRetries, task.err)
}
func (fd *FileDownloader) doDownloadTask(progressChan chan ProgressChan, task *DownloadTask) error {
request, err := http.NewRequest("GET", fd.Url, nil)
if err != nil {
globalLogger.Error().Stack().Err(err).Msgf("任务%d创建请求出错", task.taskID)
return
return fmt.Errorf("create request failed: %w", err)
}
fd.setHeaders(request)
if fd.IsMultiPart {
rangeHeader := fmt.Sprintf("bytes=%d-%d", task.rangeStart, task.rangeEnd)
rangeStart := task.rangeStart + task.downloadedSize
rangeHeader := fmt.Sprintf("bytes=%d-%d", rangeStart, task.rangeEnd)
request.Header.Set("Range", rangeHeader)
}
client := fd.buildClient()
resp, err := client.Do(request)
if err != nil {
log.Printf("任务%d发送下载请求出错！%s", task.taskID, err)
return
return fmt.Errorf("send request failed: %w", err)
}
defer resp.Body.Close()
buf := make([]byte, 8192)
if fd.IsMultiPart && resp.StatusCode != http.StatusPartialContent {
return fmt.Errorf("server does not support range requests, status: %d", resp.StatusCode)
} else if !fd.IsMultiPart && resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
}
buf := make([]byte, 32*1024)
for {
n, err := resp.Body.Read(buf)
if n > 0 {
remain := task.rangeEnd - (task.rangeStart + task.downloadedSize) + 1
n64 := int64(n)
if n64 > remain {
n = int(remain)
writeSize := int64(n)
if writeSize > remain {
writeSize = remain
}
_, writeErr := fd.File.WriteAt(buf[:n], task.rangeStart+task.downloadedSize)
_, writeErr := fd.File.WriteAt(buf[:writeSize], task.rangeStart+task.downloadedSize)
if writeErr != nil {
log.Printf("任务%d写入文件时出现错误！位置:%d, err: %s\n", task.taskID, task.rangeStart+task.downloadedSize, writeErr)
return
return fmt.Errorf("write file failed at offset %d: %w", task.rangeStart+task.downloadedSize, writeErr)
}
task.downloadedSize += n64
progressChan <- n64
task.downloadedSize += writeSize
progressChan <- ProgressChan{taskID: task.taskID, bytes: writeSize}
if task.rangeStart+task.downloadedSize-1 >= task.rangeEnd {
task.isCompleted = true
break
return nil
}
}
if err != nil {
if err == io.EOF {
task.isCompleted = true
expectedSize := task.rangeEnd - task.rangeStart + 1
if task.downloadedSize < expectedSize {
return fmt.Errorf("incomplete download: got %d bytes, expected %d", task.downloadedSize, expectedSize)
}
break
return nil
}
return fmt.Errorf("read response failed: %w", err)
}
}
}
func (fd *FileDownloader) verifyDownload() error {
for _, task := range fd.DownloadTaskList {
if !task.isCompleted {
return fmt.Errorf("task %d not completed", task.taskID)
}
expectedSize := task.rangeEnd - task.rangeStart + 1
if task.downloadedSize != expectedSize {
return fmt.Errorf("task %d size mismatch: got %d, expected %d", task.taskID, task.downloadedSize, expectedSize)
}
}
info, err := fd.File.Stat()
if err != nil {
return fmt.Errorf("get file info failed: %w", err)
}
if info.Size() != fd.TotalSize {
return fmt.Errorf("file size mismatch: got %d, expected %d", info.Size(), fd.TotalSize)
}
return nil
}
func (fd *FileDownloader) Start() error {
if err := fd.init(); err != nil {
return err
}
fd.createDownloadTasks()
fd.startDownload()
defer fd.File.Close()
return nil
err := fd.startDownload()
if fd.File != nil {
fd.File.Close()
}
return err
}

3
core/http.go
View File

@@ -13,6 +13,7 @@ import (
"os"
"os/exec"
"path/filepath"
"res-downloader/core/shared"
sysRuntime "runtime"
"strings"
)
@@ -401,7 +402,7 @@ func (h *HttpServer) batchImport(w http.ResponseWriter, r *http.Request) {
h.error(w, err.Error())
return
}
fileName := filepath.Join(globalConfig.SaveDirectory, "res-downloader-"+GetCurrentDateTimeFormatted()+".txt")
fileName := filepath.Join(globalConfig.SaveDirectory, "res-downloader-"+shared.GetCurrentDateTimeFormatted()+".txt")
err := os.WriteFile(fileName, []byte(data.Content), 0644)
if err != nil {
h.error(w, err.Error())

5
core/logger.go
View File

@@ -6,6 +6,7 @@ import (
"io"
"os"
"path/filepath"
"res-downloader/core/shared"
)
type Logger struct {
@@ -15,7 +16,7 @@ type Logger struct {
func initLogger() *Logger {
if globalLogger == nil {
globalLogger = NewLogger(!IsDevelopment(), filepath.Join(appOnce.UserDir, "logs", "app.log"))
globalLogger = NewLogger(!shared.IsDevelopment(), filepath.Join(appOnce.UserDir, "logs", "app.log"))
}
return globalLogger
}
@@ -38,7 +39,7 @@ func NewLogger(logFile bool, logPath string) *Logger {
if logFile {
// log to file
logDir := filepath.Dir(logPath)
if err := CreateDirIfNotExist(logDir); err != nil {
if err := shared.CreateDirIfNotExist(logDir); err != nil {
panic(err)
}
var (

78
core/plugins/plugin.default.go Normal file
View File

@@ -0,0 +1,78 @@
package plugins
import (
"encoding/json"
"github.com/elazarl/goproxy"
gonanoid "github.com/matoous/go-nanoid/v2"
"net/http"
"res-downloader/core/shared"
"strconv"
)
type DefaultPlugin struct {
bridge *shared.Bridge
}
func (p *DefaultPlugin) SetBridge(bridge *shared.Bridge) {
p.bridge = bridge
}
func (p *DefaultPlugin) Domains() []string {
return []string{"default"}
}
func (p *DefaultPlugin) OnRequest(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
return nil, nil
}
func (p *DefaultPlugin) OnResponse(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
if resp == nil || resp.Request == nil || (resp.StatusCode != 200 && resp.StatusCode != 206) {
return nil
}
classify, suffix := p.bridge.TypeSuffix(resp.Header.Get("Content-Type"))
if classify == "" {
return nil
}
rawUrl := resp.Request.URL.String()
isAll, _ := p.bridge.GetResType("all")
isClassify, _ := p.bridge.GetResType(classify)
urlSign := shared.Md5(rawUrl)
if ok := p.bridge.MediaIsMarked(urlSign); !ok && (isAll || isClassify) {
value, _ := strconv.ParseFloat(resp.Header.Get("content-length"), 64)
id, err := gonanoid.New()
if err != nil {
id = urlSign
}
res := shared.MediaInfo{
Id: id,
Url: rawUrl,
UrlSign: urlSign,
CoverUrl: "",
Size: shared.FormatSize(value),
Domain: shared.GetTopLevelDomain(rawUrl),
Classify: classify,
Suffix: suffix,
Status: shared.DownloadStatusReady,
SavePath: "",
DecodeKey: "",
OtherData: map[string]string{},
Description: "",
ContentType: resp.Header.Get("Content-Type"),
}
// Store entire request headers as JSON
if headers, err := json.Marshal(resp.Request.Header); err == nil {
res.OtherData["headers"] = string(headers)
}
p.bridge.MarkMedia(urlSign)
go func(res shared.MediaInfo) {
p.bridge.Send("newResources", res)
}(res)
}
return nil
}

262
core/plugins/plugin.qq.com.go Normal file
View File

@@ -0,0 +1,262 @@
package plugins
import (
"bytes"
"encoding/json"
"fmt"
"github.com/elazarl/goproxy"
gonanoid "github.com/matoous/go-nanoid/v2"
"io"
"net/http"
"regexp"
"res-downloader/core/shared"
"strconv"
"strings"
)
type QqPlugin struct {
bridge *shared.Bridge
}
func (p *QqPlugin) SetBridge(bridge *shared.Bridge) {
p.bridge = bridge
}
func (p *QqPlugin) Domains() []string {
return []string{"qq.com"}
}
func (p *QqPlugin) OnRequest(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
if strings.Contains(r.Host, "qq.com") && strings.Contains(r.URL.Path, "/res-downloader/wechat") {
if p.bridge.GetConfig("WxAction").(bool) && r.URL.Query().Get("type") == "1" {
return p.handleWechatRequest(r, ctx)
} else if !p.bridge.GetConfig("WxAction").(bool) && r.URL.Query().Get("type") == "2" {
return p.handleWechatRequest(r, ctx)
} else {
return r, p.buildEmptyResponse(r)
}
}
return r, nil
}
func (p *QqPlugin) OnResponse(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
if resp.StatusCode != 200 && resp.StatusCode != 206 {
return nil
}
host := resp.Request.Host
Path := resp.Request.URL.Path
classify, _ := p.bridge.TypeSuffix(resp.Header.Get("Content-Type"))
if classify == "video" && strings.HasSuffix(host, "finder.video.qq.com") {
return resp
}
if strings.HasSuffix(host, "channels.weixin.qq.com") &&
(strings.Contains(Path, "/web/pages/feed") || strings.Contains(Path, "/web/pages/home")) {
return p.replaceWxJsContent(resp, ".js\"", ".js?v="+p.v()+"\"")
}
if strings.HasSuffix(host, "res.wx.qq.com") {
respTemp := resp
is := false
if strings.HasSuffix(respTemp.Request.URL.RequestURI(), ".js?v="+p.v()) {
respTemp = p.replaceWxJsContent(respTemp, ".js\"", ".js?v="+p.v()+"\"")
is = true
}
if strings.Contains(Path, "web/web-finder/res/js/virtual_svg-icons-register.publish") {
body, err := io.ReadAll(respTemp.Body)
if err != nil {
return respTemp
}
bodyStr := string(body)
newBody := regexp.MustCompile(`get\s*media\(\)\{`).
ReplaceAllString(bodyStr, `
get media(){
if(this.objectDesc){
fetch("https://wxapp.tc.qq.com/res-downloader/wechat?type=1", {
method: "POST",
mode: "no-cors",
body: JSON.stringify(this.objectDesc),
});
};
`)
newBody = regexp.MustCompile(`async\s*finderGetCommentDetail\((\w+)\)\s*\{return(.*?)\s*}\s*async`).
ReplaceAllString(newBody, `
async finderGetCommentDetail($1) {
var res = await$2;
if (res?.data?.object?.objectDesc) {
fetch("https://wxapp.tc.qq.com/res-downloader/wechat?type=2", {
method: "POST",
mode: "no-cors",
body: JSON.stringify(res.data.object.objectDesc),
});
}
return res;
}async
`)
newBodyBytes := []byte(newBody)
respTemp.Body = io.NopCloser(bytes.NewBuffer(newBodyBytes))
respTemp.ContentLength = int64(len(newBodyBytes))
respTemp.Header.Set("Content-Length", fmt.Sprintf("%d", len(newBodyBytes)))
return respTemp
}
if is {
return respTemp
}
}
return nil
}
func (p *QqPlugin) handleWechatRequest(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
body, err := io.ReadAll(r.Body)
if err != nil {
return r, p.buildEmptyResponse(r)
}
isAll, _ := p.bridge.GetResType("all")
isClassify, _ := p.bridge.GetResType("video")
if !isAll && !isClassify {
return r, p.buildEmptyResponse(r)
}
go p.handleMedia(body)
return r, p.buildEmptyResponse(r)
}
func (p *QqPlugin) handleMedia(body []byte) {
var result map[string]interface{}
if err := json.Unmarshal(body, &result); err != nil {
return
}
mediaArr, ok := result["media"].([]interface{})
if !ok || len(mediaArr) == 0 {
return
}
firstMedia, ok := mediaArr[0].(map[string]interface{})
if !ok {
return
}
rawUrl, ok := firstMedia["url"].(string)
if !ok || rawUrl == "" {
return
}
urlSign := shared.Md5(rawUrl)
if p.bridge.MediaIsMarked(urlSign) {
return
}
id, err := gonanoid.New()
if err != nil {
id = urlSign
}
res := shared.MediaInfo{
Id: id,
Url: rawUrl,
UrlSign: urlSign,
CoverUrl: "",
Size: "0",
Domain: shared.GetTopLevelDomain(rawUrl),
Classify: "video",
Suffix: ".mp4",
Status: shared.DownloadStatusReady,
SavePath: "",
DecodeKey: "",
OtherData: map[string]string{},
Description: "",
ContentType: "video/mp4",
}
if mediaType, ok := firstMedia["mediaType"].(float64); ok && mediaType == 9 {
res.Classify = "image"
res.Suffix = ".png"
res.ContentType = "image/png"
}
if urlToken, ok := firstMedia["urlToken"].(string); ok {
res.Url += urlToken
}
switch size := firstMedia["fileSize"].(type) {
case float64:
res.Size = shared.FormatSize(size)
case string:
if value, err := strconv.ParseFloat(size, 64); err == nil {
res.Size = shared.FormatSize(value)
}
}
if coverUrl, ok := firstMedia["coverUrl"].(string); ok {
res.CoverUrl = coverUrl
}
if decodeKey, ok := firstMedia["decodeKey"].(string); ok {
res.DecodeKey = decodeKey
}
if desc, ok := result["description"].(string); ok {
res.Description = desc
}
if spec, ok := firstMedia["spec"].([]interface{}); ok {
var fileFormats []string
for _, item := range spec {
if m, ok := item.(map[string]interface{}); ok {
if format, ok := m["fileFormat"].(string); ok {
fileFormats = append(fileFormats, format)
}
}
}
res.OtherData["wx_file_formats"] = strings.Join(fileFormats, "#")
}
p.bridge.MarkMedia(urlSign)
go func(res shared.MediaInfo) {
p.bridge.Send("newResources", res)
}(res)
}
func (p *QqPlugin) buildEmptyResponse(r *http.Request) *http.Response {
body := "The content does not exist"
resp := &http.Response{
Status: http.StatusText(http.StatusOK),
StatusCode: http.StatusOK,
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(body)),
ContentLength: int64(len(body)),
Request: r,
}
resp.Header.Set("Content-Type", "text/plain; charset=utf-8")
return resp
}
func (p *QqPlugin) replaceWxJsContent(resp *http.Response, old, new string) *http.Response {
body, err := io.ReadAll(resp.Body)
if err != nil {
return resp
}
bodyString := string(body)
newBodyString := strings.ReplaceAll(bodyString, old, new)
newBodyBytes := []byte(newBodyString)
resp.Body = io.NopCloser(bytes.NewBuffer(newBodyBytes))
resp.ContentLength = int64(len(newBodyBytes))
resp.Header.Set("Content-Length", fmt.Sprintf("%d", len(newBodyBytes)))
return resp
}
func (p *QqPlugin) v() string {
return p.bridge.GetVersion()
}

320
core/proxy.go
View File

@@ -1,23 +1,18 @@
package core
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/url"
"regexp"
"strconv"
"res-downloader/core/plugins"
"res-downloader/core/shared"
"strings"
"time"
"github.com/elazarl/goproxy"
gonanoid "github.com/matoous/go-nanoid/v2"
)
type Proxy struct {
@@ -43,6 +38,46 @@ type MediaInfo struct {
OtherData map[string]string
}
var pluginRegistry = make(map[string]shared.Plugin)
func init() {
ps := []shared.Plugin{
&plugins.QqPlugin{},
&plugins.DefaultPlugin{},
}
bridge := &shared.Bridge{
GetVersion: func() string {
return appOnce.Version
},
GetResType: func(key string) (bool, bool) {
return resourceOnce.getResType(key)
},
TypeSuffix: func(mine string) (string, string) {
return globalConfig.TypeSuffix(mine)
},
MediaIsMarked: func(key string) bool {
return resourceOnce.mediaIsMarked(key)
},
MarkMedia: func(key string) {
resourceOnce.markMedia(key)
},
GetConfig: func(key string) interface{} {
return globalConfig.getConfig(key)
},
Send: func(t string, data interface{}) {
httpServerOnce.send(t, data)
},
}
for _, p := range ps {
p.SetBridge(bridge)
for _, domain := range p.Domains() {
pluginRegistry[domain] = p
}
}
}
func initProxy() *Proxy {
if proxyOnce == nil {
proxyOnce = &Proxy{}
@@ -54,7 +89,7 @@ func initProxy() *Proxy {
func (p *Proxy) Startup() {
err := p.setCa()
if err != nil {
DialogErr("启动代理服务失败：" + err.Error())
DialogErr("Failed to start proxy service：" + err.Error())
return
}
@@ -70,7 +105,7 @@ func (p *Proxy) Startup() {
func (p *Proxy) setCa() error {
ca, err := tls.X509KeyPair(appOnce.PublicCrt, appOnce.PrivateKey)
if err != nil {
DialogErr("启动代理服务失败1")
DialogErr("Failed to start proxy service 1")
return err
}
if ca.Leaf, err = x509.ParseCertificate(ca.Certificate[0]); err != nil {
@@ -105,264 +140,37 @@ func (p *Proxy) setTransport() {
p.Proxy.Tr = transport
}
func (p *Proxy) matchPlugin(host string) shared.Plugin {
domain := shared.GetTopLevelDomain(host)
if plugin, ok := pluginRegistry[domain]; ok {
return plugin
}
return pluginRegistry["default"]
}
func (p *Proxy) httpRequestEvent(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
if strings.Contains(r.Host, "res-downloader.666666.com") && strings.Contains(r.URL.Path, "/wechat") {
if globalConfig.WxAction && r.URL.Query().Get("type") == "1" {
return p.handleWechatRequest(r, ctx)
} else if !globalConfig.WxAction && r.URL.Query().Get("type") == "2" {
return p.handleWechatRequest(r, ctx)
} else {
return r, p.buildEmptyResponse(r)
newReq, newResp := p.matchPlugin(r.Host).OnRequest(r, ctx)
if newResp != nil {
return newReq, newResp
}
if newReq != nil {
return newReq, nil
}
return r, nil
}
func (p *Proxy) handleWechatRequest(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
body, err := io.ReadAll(r.Body)
if err != nil {
fmt.Println(err)
return r, p.buildEmptyResponse(r)
}
isAll, _ := resourceOnce.getResType("all")
isClassify, _ := resourceOnce.getResType("video")
if !isAll && !isClassify {
return r, p.buildEmptyResponse(r)
}
go func(body []byte) {
var result map[string]interface{}
err = json.Unmarshal(body, &result)
if err != nil {
return
}
media, ok := result["media"].([]interface{})
if !ok || len(media) <= 0 {
return
}
firstMedia, ok := media[0].(map[string]interface{})
if !ok {
return
}
rowUrl, ok := firstMedia["url"]
if !ok {
return
}
urlSign := Md5(rowUrl.(string))
if resourceOnce.mediaIsMarked(urlSign) {
return
}
id, err := gonanoid.New()
if err != nil {
id = urlSign
}
res := MediaInfo{
Id: id,
Url: rowUrl.(string),
UrlSign: urlSign,
CoverUrl: "",
Size: "0",
Domain: GetTopLevelDomain(rowUrl.(string)),
Classify: "video",
Suffix: ".mp4",
Status: DownloadStatusReady,
SavePath: "",
DecodeKey: "",
OtherData: map[string]string{},
Description: "",
ContentType: "video/mp4",
}
if mediaType, ok := firstMedia["mediaType"].(float64); ok && mediaType == 9 {
res.Classify = "image"
res.Suffix = ".png"
res.ContentType = "image/png"
}
if urlToken, ok := firstMedia["urlToken"].(string); ok {
res.Url = res.Url + urlToken
}
if fileSize, ok := firstMedia["fileSize"].(float64); ok {
res.Size = FormatSize(fileSize)
}
if coverUrl, ok := firstMedia["coverUrl"].(string); ok {
res.CoverUrl = coverUrl
}
if fileSize, ok := firstMedia["fileSize"].(string); ok {
value, err := strconv.ParseFloat(fileSize, 64)
if err == nil {
res.Size = FormatSize(value)
}
}
if decodeKey, ok := firstMedia["decodeKey"].(string); ok {
res.DecodeKey = decodeKey
}
if desc, ok := result["description"].(string); ok {
res.Description = desc
}
if spec, ok := firstMedia["spec"].([]interface{}); ok {
var fileFormats []string
for _, item := range spec {
if itemMap, ok := item.(map[string]interface{}); ok {
if format, exists := itemMap["fileFormat"].(string); exists {
fileFormats = append(fileFormats, format)
}
}
}
res.OtherData["wx_file_formats"] = strings.Join(fileFormats, "#")
}
resourceOnce.markMedia(urlSign)
httpServerOnce.send("newResources", res)
}(body)
return r, p.buildEmptyResponse(r)
}
func (p *Proxy) buildEmptyResponse(r *http.Request) *http.Response {
body := "The content does not exist"
resp := &http.Response{
Status: http.StatusText(http.StatusOK),
StatusCode: http.StatusOK,
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(body)),
ContentLength: int64(len(body)),
Request: r,
}
resp.Header.Set("Content-Type", "text/plain; charset=utf-8")
return resp
}
func (p *Proxy) httpResponseEvent(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
if resp == nil || resp.Request == nil || (resp.StatusCode != 200 && resp.StatusCode != 206) {
if resp == nil || resp.Request == nil {
return resp
}
host := resp.Request.Host
Path := resp.Request.URL.Path
if strings.HasSuffix(host, "channels.weixin.qq.com") &&
(strings.Contains(Path, "/web/pages/feed") || strings.Contains(Path, "/web/pages/home")) {
return p.replaceWxJsContent(resp, ".js\"", ".js?v="+p.v()+"\"")
newResp := p.matchPlugin(resp.Request.Host).OnResponse(resp, ctx)
if newResp != nil {
return newResp
}
if strings.HasSuffix(host, "res.wx.qq.com") {
respTemp := resp
is := false
if strings.HasSuffix(respTemp.Request.URL.RequestURI(), ".js?v="+p.v()) {
respTemp = p.replaceWxJsContent(respTemp, ".js\"", ".js?v="+p.v()+"\"")
is = true
}
if strings.Contains(Path, "web/web-finder/res/js/virtual_svg-icons-register.publish") {
body, err := io.ReadAll(respTemp.Body)
if err != nil {
return respTemp
}
bodyStr := string(body)
newBody := regexp.MustCompile(`get\s*media\(\)\{`).
ReplaceAllString(bodyStr, `
get media(){
if(this.objectDesc){
fetch("https://res-downloader.666666.com/wechat?type=1", {
method: "POST",
mode: "no-cors",
body: JSON.stringify(this.objectDesc),
});
};
`)
newBody = regexp.MustCompile(`async\s*finderGetCommentDetail\((\w+)\)\s*\{return(.*?)\s*}\s*async`).
ReplaceAllString(newBody, `
async finderGetCommentDetail($1) {
var res = await$2;
if (res?.data?.object?.objectDesc) {
fetch("https://res-downloader.666666.com/wechat?type=2", {
method: "POST",
mode: "no-cors",
body: JSON.stringify(res.data.object.objectDesc),
});
}
return res;
}async
`)
newBodyBytes := []byte(newBody)
respTemp.Body = io.NopCloser(bytes.NewBuffer(newBodyBytes))
respTemp.ContentLength = int64(len(newBodyBytes))
respTemp.Header.Set("Content-Length", fmt.Sprintf("%d", len(newBodyBytes)))
return respTemp
}
if is {
return respTemp
}
}
classify, suffix := TypeSuffix(resp.Header.Get("Content-Type"))
if classify == "" {
return resp
}
if classify == "video" && strings.HasSuffix(host, "finder.video.qq.com") {
//if !globalConfig.WxAction && classify == "video" && strings.HasSuffix(host, "finder.video.qq.com") {
return resp
}
rawUrl := resp.Request.URL.String()
isAll, _ := resourceOnce.getResType("all")
isClassify, _ := resourceOnce.getResType(classify)
urlSign := Md5(rawUrl)
if ok := resourceOnce.mediaIsMarked(urlSign); !ok && (isAll || isClassify) {
value, _ := strconv.ParseFloat(resp.Header.Get("content-length"), 64)
id, err := gonanoid.New()
if err != nil {
id = urlSign
}
res := MediaInfo{
Id: id,
Url: rawUrl,
UrlSign: urlSign,
CoverUrl: "",
Size: FormatSize(value),
Domain: GetTopLevelDomain(rawUrl),
Classify: classify,
Suffix: suffix,
Status: DownloadStatusReady,
SavePath: "",
DecodeKey: "",
OtherData: map[string]string{},
Description: "",
ContentType: resp.Header.Get("Content-Type"),
}
// Store entire request headers as JSON
if headers, err := json.Marshal(resp.Request.Header); err == nil {
res.OtherData["headers"] = string(headers)
}
resourceOnce.markMedia(urlSign)
httpServerOnce.send("newResources", res)
}
return resp
}
func (p *Proxy) replaceWxJsContent(resp *http.Response, old, new string) *http.Response {
body, err := io.ReadAll(resp.Body)
if err != nil {
return resp
}
bodyString := string(body)
newBodyString := strings.ReplaceAll(bodyString, old, new)
newBodyBytes := []byte(newBodyString)
resp.Body = io.NopCloser(bytes.NewBuffer(newBodyBytes))
resp.ContentLength = int64(len(newBodyBytes))
resp.Header.Set("Content-Length", fmt.Sprintf("%d", len(newBodyBytes)))
return resp
}
func (p *Proxy) v() string {
return appOnce.Version
}

29
core/resource.go
View File

@@ -9,19 +9,12 @@ import (
"os"
"path/filepath"
"regexp"
"res-downloader/core/shared"
"strconv"
"strings"
"sync"
)
const (
DownloadStatusReady string = "ready" // task create but not start
DownloadStatusRunning string = "running"
DownloadStatusError string = "error"
DownloadStatusDone string = "done"
DownloadStatusHandle string = "handle"
)
type WxFileDecodeResult struct {
SavePath string
Message string
@@ -102,7 +95,7 @@ func (r *Resource) download(mediaInfo MediaInfo, decodeStr string) {
}
go func(mediaInfo MediaInfo) {
rawUrl := mediaInfo.Url
fileName := Md5(rawUrl)
fileName := shared.Md5(rawUrl)
if mediaInfo.Description != "" {
fileName = regexp.MustCompile(`[^\w\p{Han}]`).ReplaceAllString(mediaInfo.Description, "")
fileLen := globalConfig.FilenameLen
@@ -117,7 +110,7 @@ func (r *Resource) download(mediaInfo MediaInfo, decodeStr string) {
}
if globalConfig.FilenameTime {
mediaInfo.SavePath = filepath.Join(globalConfig.SaveDirectory, fileName+"_"+GetCurrentDateTimeFormatted()+mediaInfo.Suffix)
mediaInfo.SavePath = filepath.Join(globalConfig.SaveDirectory, fileName+"_"+shared.GetCurrentDateTimeFormatted()+mediaInfo.Suffix)
} else {
mediaInfo.SavePath = filepath.Join(globalConfig.SaveDirectory, fileName+mediaInfo.Suffix)
}
@@ -147,8 +140,8 @@ func (r *Resource) download(mediaInfo MediaInfo, decodeStr string) {
headers, _ := r.parseHeaders(mediaInfo)
downloader := NewFileDownloader(rawUrl, mediaInfo.SavePath, globalConfig.TaskNumber, headers)
downloader.progressCallback = func(totalDownloaded, totalSize float64) {
r.progressEventsEmit(mediaInfo, strconv.Itoa(int(totalDownloaded*100/totalSize))+"%", DownloadStatusRunning)
downloader.progressCallback = func(totalDownloaded, totalSize float64, taskID int, taskProgress float64) {
r.progressEventsEmit(mediaInfo, strconv.Itoa(int(totalDownloaded*100/totalSize))+"%", shared.DownloadStatusRunning)
}
err := downloader.Start()
if err != nil {
@@ -156,23 +149,21 @@ func (r *Resource) download(mediaInfo MediaInfo, decodeStr string) {
return
}
if decodeStr != "" {
r.progressEventsEmit(mediaInfo, "解密中", DownloadStatusRunning)
r.progressEventsEmit(mediaInfo, "decrypting in progress", shared.DownloadStatusRunning)
if err := r.decodeWxFile(mediaInfo.SavePath, decodeStr); err != nil {
r.progressEventsEmit(mediaInfo, "解密出错"+err.Error())
r.progressEventsEmit(mediaInfo, "decryption error: "+err.Error())
return
}
}
r.progressEventsEmit(mediaInfo, "完成", DownloadStatusDone)
r.progressEventsEmit(mediaInfo, "complete", shared.DownloadStatusDone)
}(mediaInfo)
}
// 解析并组装 headers
func (r *Resource) parseHeaders(mediaInfo MediaInfo) (map[string]string, error) {
headers := make(map[string]string)
if hh, ok := mediaInfo.OtherData["headers"]; ok {
var tempHeaders map[string][]string
// 解析 JSON 字符串为 map[string][]string
if err := json.Unmarshal([]byte(hh), &tempHeaders); err != nil {
return headers, fmt.Errorf("parse headers JSON err: %v", err)
}
@@ -193,7 +184,7 @@ func (r *Resource) wxFileDecode(mediaInfo MediaInfo, fileName, decodeStr string)
return "", err
}
defer sourceFile.Close()
mediaInfo.SavePath = strings.ReplaceAll(fileName, ".mp4", "_解密.mp4")
mediaInfo.SavePath = strings.ReplaceAll(fileName, ".mp4", "_decrypt.mp4")
destinationFile, err := os.Create(mediaInfo.SavePath)
if err != nil {
@@ -213,7 +204,7 @@ func (r *Resource) wxFileDecode(mediaInfo MediaInfo, fileName, decodeStr string)
}
func (r *Resource) progressEventsEmit(mediaInfo MediaInfo, args ...string) {
Status := DownloadStatusError
Status := shared.DownloadStatusError
Message := "ok"
if len(args) > 0 {

40
core/shared/base.go Normal file
View File

@@ -0,0 +1,40 @@
package shared
import (
"github.com/elazarl/goproxy"
"net/http"
)
type Bridge struct {
GetVersion func() string
GetResType func(key string) (bool, bool)
TypeSuffix func(mime string) (string, string)
MediaIsMarked func(key string) bool
MarkMedia func(key string)
GetConfig func(key string) interface{}
Send func(t string, data interface{})
}
type MediaInfo struct {
Id string
Url string
UrlSign string
CoverUrl string
Size string
Domain string
Classify string
Suffix string
SavePath string
Status string
DecodeKey string
Description string
ContentType string
OtherData map[string]string
}
type Plugin interface {
SetBridge(*Bridge)
Domains() []string
OnRequest(*http.Request, *goproxy.ProxyCtx) (*http.Request, *http.Response)
OnResponse(*http.Response, *goproxy.ProxyCtx) *http.Response
}

9
core/shared/const.go Normal file
View File

@@ -0,0 +1,9 @@
package shared
const (
DownloadStatusReady string = "ready" // task create but not start
DownloadStatusRunning string = "running"
DownloadStatusError string = "error"
DownloadStatusDone string = "done"
DownloadStatusHandle string = "handle"
)

70
core/shared/utils.go Normal file
View File

@@ -0,0 +1,70 @@
package shared
import (
"crypto/md5"
"encoding/hex"
"fmt"
"golang.org/x/net/publicsuffix"
"net/url"
"os"
"time"
)
func Md5(data string) string {
hashNew := md5.New()
hashNew.Write([]byte(data))
hash := hashNew.Sum(nil)
return hex.EncodeToString(hash)
}
func FormatSize(size float64) string {
if size > 1048576 {
return fmt.Sprintf("%.2fMB", float64(size)/1048576)
}
if size > 1024 {
return fmt.Sprintf("%.2fKB", float64(size)/1024)
}
return fmt.Sprintf("%.0fb", size)
}
func GetTopLevelDomain(rawURL string) string {
u, err := url.Parse(rawURL)
if err == nil && u.Host != "" {
rawURL = u.Host
}
domain, err := publicsuffix.EffectiveTLDPlusOne(rawURL)
if err != nil {
return rawURL
}
return domain
}
func FileExist(file string) bool {
info, err := os.Stat(file)
if err != nil {
return false
}
return !info.IsDir()
}
func CreateDirIfNotExist(dir string) error {
if _, err := os.Stat(dir); os.IsNotExist(err) {
return os.MkdirAll(dir, 0750)
}
return nil
}
func IsDevelopment() bool {
return os.Getenv("APP_ENV") == "development"
}
func GetCurrentDateTimeFormatted() string {
now := time.Now()
return fmt.Sprintf("%04d%02d%02d%02d%02d%02d",
now.Year(),
now.Month(),
now.Day(),
now.Hour(),
now.Minute(),
now.Second())
}

3
core/storage.go
View File

@@ -3,6 +3,7 @@ package core
import (
"os"
"path"
"res-downloader/core/shared"
)
type Storage struct {
@@ -18,7 +19,7 @@ func NewStorage(filename string, def []byte) *Storage {
}
func (l *Storage) Load() ([]byte, error) {
if !FileExist(l.fileName) {
if !shared.FileExist(l.fileName) {
err := os.WriteFile(l.fileName, l.def, 0644)
if err != nil {
return nil, err

3
core/system_linux.go
View File

@@ -5,6 +5,7 @@ package core
import (
"bytes"
"fmt"
"os"
"os/exec"
"strings"
)
@@ -81,7 +82,7 @@ func (s *SystemSetup) installCert() (string, error) {
return "", err
}
distro, err := getLinuxDistro()
distro, err := s.getLinuxDistro()
if err != nil {
return "", fmt.Errorf("detect distro failed: %w", err)
}

77
core/utils.go
View File

@@ -1,14 +1,7 @@
package core
import (
"crypto/md5"
"encoding/hex"
"fmt"
"github.com/wailsapp/wails/v2/pkg/runtime"
"net/url"
"os"
"strings"
"time"
)
func DialogErr(message string) {
@@ -19,73 +12,3 @@ func DialogErr(message string) {
DefaultButton: "Cancel",
})
}
func IsDevelopment() bool {
return os.Getenv("APP_ENV") == "development"
}
func FileExist(file string) bool {
info, err := os.Stat(file)
if err != nil {
return false
}
return !info.IsDir()
}
func CreateDirIfNotExist(dir string) error {
if _, err := os.Stat(dir); os.IsNotExist(err) {
return os.MkdirAll(dir, 0750)
}
return nil
}
func TypeSuffix(mime string) (string, string) {
mimeMux.RLock()
defer mimeMux.RUnlock()
mime = strings.ToLower(strings.Split(mime, ";")[0])
if v, ok := globalConfig.MimeMap[mime]; ok {
return v.Type, v.Suffix
}
return "", ""
}
func Md5(data string) string {
hashNew := md5.New()
hashNew.Write([]byte(data))
hash := hashNew.Sum(nil)
return hex.EncodeToString(hash)
}
func FormatSize(size float64) string {
if size > 1048576 {
return fmt.Sprintf("%.2fMB", float64(size)/1048576)
}
if size > 1024 {
return fmt.Sprintf("%.2fKB", float64(size)/1024)
}
return fmt.Sprintf("%.0fb", size)
}
func GetTopLevelDomain(rawURL string) string {
parsedURL, err := url.Parse(rawURL)
if err != nil {
return ""
}
host := parsedURL.Hostname()
parts := strings.Split(host, ".")
if len(parts) < 2 {
return ""
}
return strings.Join(parts[len(parts)-2:], ".")
}
func GetCurrentDateTimeFormatted() string {
now := time.Now()
return fmt.Sprintf("%04d%02d%02d%02d%02d%02d",
now.Year(),
now.Month(),
now.Day(),
now.Hour(),
now.Minute(),
now.Second())
}