Merge pull request #176 from putyy/wails

feat: Added header cache, optional header settings...

README-EN.md

@@ -4,7 +4,7 @@
 > A cross-platform resource downloader built with Go + [Wails](https://github.com/wailsapp/wails).  
 Clean UI, easy to use, and supports a wide range of resource sniffing and downloading.
 
-### 📖 [中文](./README.md) | English 
+### 📖 [中文](./README.md) | English  
 
 ---
 

README.md

@@ -20,7 +20,7 @@
 ## 📚 文档 & 版本
 
 - 📘 [在线文档](https://res.putyy.com/)
-- 🧩 [Mini版 UI使用默认浏览器展示](https://github.com/putyy/res-downloader) ｜ [Electron旧版 支持Win7](https://github.com/putyy/res-downloader/tree/old)
+- 🧩 [Mini版 UI使用默认浏览器展示](https://github.com/putyy/resd-mini) ｜ [Electron旧版 支持Win7](https://github.com/putyy/res-downloader/tree/old)
 - 💬 [加入交流群](https://www.putyy.com/app/admin/upload/img/20250418/6801d9554dc7.webp)
   > *群满时可加微信 `AmorousWorld`，请备注“来源”*
 
@@ -32,7 +32,7 @@
 - 🆕 [蓝奏云下载（密码：9vs5）](https://wwjv.lanzoum.com/b04wgtfyb)
 - ⚠️ *Win7 用户请下载 `2.3.0` 版本*
 
----
+--- 
 
 ## 🚀 使用方法
 

build/README.md

@@ -7,8 +7,8 @@ mv -f "build/bin/res-downloader $(jq -r '.info.productVersion' wails.json).dmg"
 
 ## Windows
 ```bash
-wails build -f -nsis -platform "windows/amd64" -webview2 Embed && mv -f "build/bin/res-downloader-amd64-installer.exe" "build/bin/res-downloader_$(jq -r '.info.productVersion' wails.json)_win_amd64.exe"
-wails build -f -nsis -platform "windows/arm64" -webview2 Embed && mv -f "build/bin/res-downloader-arm64-installer.exe" "build/bin/res-downloader_$(jq -r '.info.productVersion' wails.json)_win_arm64.exe"
+wails build -f -nsis -platform "windows/amd64" -webview2 Embed -skipbindings && mv -f "build/bin/res-downloader-amd64-installer.exe" "build/bin/res-downloader_$(jq -r '.info.productVersion' wails.json)_win_amd64.exe"
+wails build -f -nsis -platform "windows/arm64" -webview2 Embed -skipbindings && mv -f "build/bin/res-downloader-arm64-installer.exe" "build/bin/res-downloader_$(jq -r '.info.productVersion' wails.json)_win_arm64.exe"
 ```
 
 ## Linux
@@ -41,7 +41,7 @@ wget -O ./build/bin/appimagetool-x86_64.AppImage https://github.com/AppImage/App
 chmod +x ./build/bin/appimagetool-x86_64.AppImage
 ./build/bin/appimagetool-x86_64.AppImage build/linux/AppImage build/bin/res-downloader_$(jq -r '.info.productVersion' wails.json)_linux_amd64.AppImage
 
-cp build/bin/res-downloader build/bin/res-downloader_$(jq -r '.info.productVersion' wails.json)_linux_amd64
+mv -f build/bin/res-downloader build/bin/res-downloader_$(jq -r '.info.productVersion' wails.json)_linux_amd64
 ```
 
 > arm64
@@ -58,7 +58,7 @@ cp build/bin/res-downloader build/linux/Debian/usr/local/bin/
 echo "$(cat build/linux/Debian/DEBIAN/.control | sed -e "s/{{Version}}/$(jq -r '.info.productVersion' wails.json)/g")" > build/linux/Debian/DEBIAN/control
 dpkg-deb --build ./build/linux/Debian build/bin/res-downloader_$(jq -r '.info.productVersion' wails.json)_linux_arm64.deb
 
-cp build/bin/res-downloader build/bin/res-downloader_$(jq -r '.info.productVersion' wails.json)_linux_arm64
+mv -f build/bin/res-downloader build/bin/res-downloader_$(jq -r '.info.productVersion' wails.json)_linux_arm64
 ```
 
 ### Arch Linux 

build/windows/installer/wails_tools.nsh

@@ -14,7 +14,7 @@
     !define INFO_PRODUCTNAME "res-downloader"
 !endif
 !ifndef INFO_PRODUCTVERSION
-    !define INFO_PRODUCTVERSION "3.0.4"
+    !define INFO_PRODUCTVERSION "3.0.5"
 !endif
 !ifndef INFO_COPYRIGHT
     !define INFO_COPYRIGHT "Copyright © 2023"

core/config.go

@@ -24,6 +24,7 @@ type Config struct {
 	WxAction      bool   `json:"WxAction"`
 	TaskNumber    int    `json:"TaskNumber"`
 	UserAgent     string `json:"UserAgent"`
+	UseHeaders    string `json:"UseHeaders"`
 }
 
 func initConfig() *Config {
@@ -43,7 +44,8 @@ func initConfig() *Config {
   "AutoProxy": true,
   "WxAction": true,
   "TaskNumber": __TaskNumber__,
-  "UserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
+  "UserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
+  "UseHeaders": "User-Agent,Referer,Authorization,Cookie"
 }
 `
 		def = strings.ReplaceAll(def, "__TaskNumber__", strconv.Itoa(runtime.NumCPU()*2))
@@ -51,9 +53,23 @@ func initConfig() *Config {
 			storage: NewStorage("config.json", []byte(def)),
 		}
 
+		defaultMap := make(map[string]interface{})
+		_ = json.Unmarshal([]byte(def), &defaultMap)
+
 		data, err := globalConfig.storage.Load()
 		if err == nil {
-			_ = json.Unmarshal(data, &globalConfig)
+			var loadedMap map[string]interface{}
+			_ = json.Unmarshal(data, &loadedMap)
+
+			for key, val := range defaultMap {
+				if _, ok := loadedMap[key]; !ok {
+					loadedMap[key] = val
+				}
+			}
+
+			finalBytes, _ := json.Marshal(loadedMap)
+			_ = json.Unmarshal(finalBytes, &globalConfig)
+
 		} else {
 			globalLogger.Esg(err, "load config err")
 		}
@@ -77,6 +93,7 @@ func (c *Config) setConfig(config Config) {
 	c.AutoProxy = config.AutoProxy
 	c.TaskNumber = config.TaskNumber
 	c.WxAction = config.WxAction
+	c.UseHeaders = config.UseHeaders
 	if oldProxy != c.UpstreamProxy {
 		proxyOnce.setTransport()
 	}

core/downloader.go

@@ -5,7 +5,6 @@ import (
 	"io"
 	"log"
 	"net/http"
-	"net/http/cookiejar"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -32,17 +31,19 @@ type FileDownloader struct {
 	totalTasks       int
 	TotalSize        int64
 	IsMultiPart      bool
+	Headers          map[string]string
 	DownloadTaskList []*DownloadTask
 	progressCallback ProgressCallback
 }
 
-func NewFileDownloader(url, filename string, totalTasks int) *FileDownloader {
+func NewFileDownloader(url, filename string, totalTasks int, headers map[string]string) *FileDownloader {
 	return &FileDownloader{
 		Url:              url,
 		FileName:         filename,
 		totalTasks:       totalTasks,
 		IsMultiPart:      false,
 		TotalSize:        0,
+		Headers:          headers,
 		DownloadTaskList: make([]*DownloadTask, 0),
 	}
 }
@@ -53,10 +54,16 @@ func (fd *FileDownloader) buildClient() *http.Client {
 		transport.Proxy = http.ProxyURL(fd.ProxyUrl)
 	}
 	// Cookie handle
-	jar, _ := cookiejar.New(nil)
 	return &http.Client{
 		Transport: transport,
-		Jar:       jar,
+	}
+}
+
+func (fd *FileDownloader) setHeaders(request *http.Request) {
+	for key, values := range fd.Headers {
+		if strings.Contains(globalConfig.UseHeaders, key) {
+			request.Header.Set(key, values)
+		}
 	}
 }
 
@@ -76,21 +83,22 @@ func (fd *FileDownloader) init() error {
 		}
 	}
 
-	req, err := http.NewRequest("HEAD", fd.Url, nil)
+	request, err := http.NewRequest("HEAD", fd.Url, nil)
 	if err != nil {
 		return fmt.Errorf("create request failed")
 	}
 
-	// 设置请求头
-	if globalConfig.UserAgent != "" {
-		req.Header.Set("User-Agent", globalConfig.UserAgent)
+	if _, ok := fd.Headers["User-Agent"]; !ok {
+		fd.Headers["User-Agent"] = globalConfig.UserAgent
 	}
 
-	if fd.Referer != "" {
-		req.Header.Set("Referer", fd.Referer)
+	if _, ok := fd.Headers["Referer"]; !ok {
+		fd.Headers["Referer"] = fd.Referer
 	}
 
-	resp, err := fd.buildClient().Do(req)
+	fd.setHeaders(request)
+
+	resp, err := fd.buildClient().Do(request)
 	if err != nil {
 		return fmt.Errorf("request failed" + err.Error())
 	}
@@ -106,8 +114,6 @@ func (fd *FileDownloader) init() error {
 		fd.IsMultiPart = true
 	}
 
-	resp.Body.Close()
-
 	fd.FileName = filepath.Clean(fd.FileName)
 
 	dir := filepath.Dir(fd.FileName)
@@ -184,8 +190,9 @@ func (fd *FileDownloader) startDownloadTask(waitGroup *sync.WaitGroup, progressC
 		globalLogger.Error().Stack().Err(err).Msgf("任务%d创建请求出错", task.taskID)
 		return
 	}
-	request.Header.Set("User-Agent", globalConfig.UserAgent)
-	request.Header.Set("Referer", fd.Referer)
+
+	fd.setHeaders(request)
+
 	if fd.IsMultiPart {
 		request.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", task.rangeStart, task.rangeEnd))
 	}

core/proxy.go

@@ -148,12 +148,12 @@ func (p *Proxy) handleWechatRequest(r *http.Request, ctx *goproxy.ProxyCtx) (*ht
 		if !ok {
 			return
 		}
-		resourceOnce.markMu.Lock()
-		defer resourceOnce.markMu.Unlock()
+
 		urlSign := Md5(rowUrl.(string))
-		if _, ok := resourceOnce.mark[urlSign]; ok {
+		if resourceOnce.mediaIsMarked(urlSign) {
 			return
 		}
+
 		id, err := gonanoid.New()
 		if err != nil {
 			id = urlSign
@@ -214,7 +214,7 @@ func (p *Proxy) handleWechatRequest(r *http.Request, ctx *goproxy.ProxyCtx) (*ht
 
 			res.OtherData["wx_file_formats"] = strings.Join(fileFormats, "#")
 		}
-		resourceOnce.mark[urlSign] = true
+		resourceOnce.markMedia(urlSign)
 		httpServerOnce.send("newResources", res)
 	}(body)
 	return r, p.buildEmptyResponse(r)
@@ -310,14 +310,11 @@ func (p *Proxy) httpResponseEvent(resp *http.Response, ctx *goproxy.ProxyCtx) *h
 	}
 
 	rawUrl := resp.Request.URL.String()
-	resourceOnce.markMu.Lock()
-	defer resourceOnce.markMu.Unlock()
-
 	isAll, _ := resourceOnce.getResType("all")
 	isClassify, _ := resourceOnce.getResType(classify)
 
 	urlSign := Md5(rawUrl)
-	if _, ok := resourceOnce.mark[urlSign]; !ok && (isAll || isClassify) {
+	if ok := resourceOnce.mediaIsMarked(urlSign); !ok && (isAll || isClassify) {
 		value, _ := strconv.ParseFloat(resp.Header.Get("content-length"), 64)
 		id, err := gonanoid.New()
 		if err != nil {
@@ -339,7 +336,13 @@ func (p *Proxy) httpResponseEvent(resp *http.Response, ctx *goproxy.ProxyCtx) *h
 			Description: "",
 			ContentType: resp.Header.Get("Content-Type"),
 		}
-		resourceOnce.mark[urlSign] = true
+
+		// Store entire request headers as JSON
+		if headers, err := json.Marshal(resp.Request.Header); err == nil {
+			res.OtherData["headers"] = string(headers)
+		}
+
+		resourceOnce.markMedia(urlSign)
 		httpServerOnce.send("newResources", res)
 	}
 	return resp

core/resource.go

@@ -2,6 +2,8 @@ package core
 
 import (
 	"encoding/base64"
+	"encoding/json"
+	"fmt"
 	"io"
 	"net/url"
 	"os"
@@ -26,8 +28,7 @@ type WxFileDecodeResult struct {
 }
 
 type Resource struct {
-	mark      map[string]bool
-	markMu    sync.RWMutex
+	mediaMark sync.Map
 	resType   map[string]bool
 	resTypeMu sync.RWMutex
 }
@@ -35,7 +36,6 @@ type Resource struct {
 func initResource() *Resource {
 	if resourceOnce == nil {
 		resourceOnce = &Resource{
-			mark: make(map[string]bool),
 			resType: map[string]bool{
 				"all":   true,
 				"image": true,
@@ -52,17 +52,13 @@ func initResource() *Resource {
 	return resourceOnce
 }
 
-func (r *Resource) getMark(key string) (bool, bool) {
-	r.markMu.RLock()
-	defer r.markMu.RUnlock()
-	value, ok := r.mark[key]
-	return value, ok
+func (r *Resource) mediaIsMarked(key string) bool {
+	_, loaded := r.mediaMark.Load(key)
+	return loaded
 }
 
-func (r *Resource) setMark(key string, value bool) {
-	r.markMu.Lock()
-	defer r.markMu.Unlock()
-	r.mark[key] = value
+func (r *Resource) markMedia(key string) {
+	r.mediaMark.Store(key, true)
 }
 
 func (r *Resource) getResType(key string) (bool, bool) {
@@ -93,15 +89,11 @@ func (r *Resource) setResType(n []string) {
 }
 
 func (r *Resource) clear() {
-	r.markMu.Lock()
-	defer r.markMu.Unlock()
-	r.mark = make(map[string]bool)
+	r.mediaMark.Clear()
 }
 
 func (r *Resource) delete(sign string) {
-	r.markMu.Lock()
-	defer r.markMu.Unlock()
-	delete(r.mark, sign)
+	r.mediaMark.Delete(sign)
 }
 
 func (r *Resource) download(mediaInfo MediaInfo, decodeStr string) {
@@ -152,7 +144,9 @@ func (r *Resource) download(mediaInfo MediaInfo, decodeStr string) {
 			}
 		}
 
-		downloader := NewFileDownloader(rawUrl, mediaInfo.SavePath, globalConfig.TaskNumber)
+		headers, _ := r.parseHeaders(mediaInfo)
+
+		downloader := NewFileDownloader(rawUrl, mediaInfo.SavePath, globalConfig.TaskNumber, headers)
 		downloader.progressCallback = func(totalDownloaded float64) {
 			r.progressEventsEmit(mediaInfo, strconv.Itoa(int(totalDownloaded))+"%", DownloadStatusRunning)
 		}
@@ -172,6 +166,27 @@ func (r *Resource) download(mediaInfo MediaInfo, decodeStr string) {
 	}(mediaInfo)
 }
 
+// 解析并组装 headers
+func (r *Resource) parseHeaders(mediaInfo MediaInfo) (map[string]string, error) {
+	headers := make(map[string]string)
+
+	if hh, ok := mediaInfo.OtherData["headers"]; ok {
+		var tempHeaders map[string][]string
+		// 解析 JSON 字符串为 map[string][]string
+		if err := json.Unmarshal([]byte(hh), &tempHeaders); err != nil {
+			return headers, fmt.Errorf("parse headers JSON err: %v", err)
+		}
+
+		for key, values := range tempHeaders {
+			if len(values) > 0 {
+				headers[key] = values[0]
+			}
+		}
+	}
+
+	return headers, nil
+}
+
 func (r *Resource) wxFileDecode(mediaInfo MediaInfo, fileName, decodeStr string) (string, error) {
 	sourceFile, err := os.Open(fileName)
 	if err != nil {

frontend/src/stores/index.ts

@@ -27,6 +27,7 @@ export const useIndexStore = defineStore("index-store", () => {
         WxAction: false,
         TaskNumber: 8,
         UserAgent: "",
+        UseHeaders: "",
     })
 
     const envInfo = ref({

frontend/src/types/app.d.ts

@@ -21,6 +21,7 @@ export namespace appType {
         WxAction: boolean
         TaskNumber: number
         UserAgent: string
+        UseHeaders: string
     }
 
     interface MediaInfo {

frontend/src/views/setting.vue

@@ -138,6 +138,17 @@
           <span>如不清楚请保持默认</span>
         </NTooltip>
       </NFormItem>
+      <NFormItem label="UseHeaders" path="UseHeaders" size="small">
+        <NInput v-model:value="formValue.UseHeaders" style="width:256px" placeholder=""/>
+        <NTooltip trigger="hover">
+          <template #trigger>
+            <NIcon size="20" class="pl-1">
+              <HelpCircleOutline />
+            </NIcon>
+          </template>
+          <span>3.0.4版本缓存了请求header信息，这个参数定义在下载时可使用的header参数:  ,分割</span>
+        </NTooltip>
+      </NFormItem>
     </NForm>
   </div>
 </template>

wails.json

@@ -13,8 +13,8 @@
   "info": {
     "companyName": "res-downloader",
     "productName": "res-downloader",
-    "productVersion": "3.0.4",
+    "productVersion": "3.0.5",
     "copyright": "Copyright © 2023",
-    "comments": "This is a high-value, high-performance, and diverse resource downloader called res-downloader."
+    "comments": "This is a high-value high-performance and diverse resource downloader called res-downloader."
   }
 }