feat: Implement proxy functionality

docker-compose.yml

@@ -9,14 +9,22 @@ services:
       - app_data:/var/lib/nysoure
     depends_on:
       - db
+      - tendis
     environment:
       - DB_HOST=db
       - DB_PORT=3306
       - DB_USER=nysoure
       - DB_PASSWORD=nysoure_password
       - DB_NAME=nysoure
+      - REDIS_HOST=tendis
+      - REDIS_PORT=6379
       - BANNED_REDIRECT_DOMAINS=example.com,example.org
     restart: unless-stopped
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
 
   db:
     image: mariadb:latest
@@ -30,7 +38,26 @@ services:
     ports:
       - "3306"
     restart: unless-stopped
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
+
+  tendis:
+    image: tendis/tendis:latest
+    volumes:
+      - tendis_data:/data
+    ports:
+      - "6379"
+    restart: unless-stopped
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
 
 volumes:
   app_data:
   db_data:
+  tendis_data:

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/blevesearch/bleve v1.0.14
 	github.com/chai2010/webp v1.4.0
 	github.com/disintegration/imaging v1.6.2
+	github.com/redis/go-redis/v9 v9.17.0
 	github.com/stretchr/testify v1.11.1
 	gorm.io/driver/mysql v1.6.0
 )
@@ -30,8 +31,10 @@ require (
 	github.com/blevesearch/zap/v13 v13.0.6 // indirect
 	github.com/blevesearch/zap/v14 v14.0.5 // indirect
 	github.com/blevesearch/zap/v15 v15.0.3 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/couchbase/vellum v1.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 // indirect
 	github.com/go-sql-driver/mysql v1.8.1 // indirect
 	github.com/golang/protobuf v1.3.2 // indirect

go.sum

@@ -29,6 +29,12 @@ github.com/blevesearch/zap/v14 v14.0.5 h1:NdcT+81Nvmp2zL+NhwSvGSLh7xNgGL8QRVZ67n
 github.com/blevesearch/zap/v14 v14.0.5/go.mod h1:bWe8S7tRrSBTIaZ6cLRbgNH4TUDaC9LZSpRGs85AsGY=
 github.com/blevesearch/zap/v15 v15.0.3 h1:Ylj8Oe+mo0P25tr9iLPp33lN6d4qcztGjaIsP51UxaY=
 github.com/blevesearch/zap/v15 v15.0.3/go.mod h1:iuwQrImsh1WjWJ0Ue2kBqY83a0rFtJTqfa9fp1rbVVU=
+github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
+github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
+github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
+github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/webp v1.4.0 h1:6DA2pkkRUPnbOHvvsmGI3He1hBKf/bkRlniAiSGuEko=
 github.com/chai2010/webp v1.4.0/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -46,6 +52,8 @@ github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
@@ -135,6 +143,8 @@ github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJ
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/redis/go-redis/v9 v9.17.0 h1:K6E+ZlYN95KSMmZeEQPbU/c++wfmEvfFB17yEAq/VhM=
+github.com/redis/go-redis/v9 v9.17.0/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
 github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=

main.go

@@ -38,7 +38,8 @@ func main() {
 		api.AddCommentRoutes(apiG)
 		api.AddConfigRoutes(apiG)
 		api.AddActivityRoutes(apiG)
-		api.AddCollectionRoutes(apiG) // 新增
+		api.AddCollectionRoutes(apiG)
+		api.AddProxyRoutes(apiG)
 	}
 
 	log.Fatal(app.Listen(":3000"))

server/api/proxy.go

@@ -0,0 +1,124 @@
+package api
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"nysoure/server/cache"
+	"nysoure/server/model"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/gofiber/fiber/v3"
+)
+
+var (
+	allowedUrlRegexps []*regexp.Regexp
+)
+
+type proxyResponse struct {
+	Content     string `json:"content"`
+	ContentType string `json:"content_type"`
+}
+
+func init() {
+	regexps := os.Getenv("ALLOWED_URL_REGEXPS")
+	for _, expr := range strings.Split(regexps, ",") {
+		if expr == "" {
+			continue
+		}
+		re, err := regexp.Compile(expr)
+		if err != nil {
+			panic("Invalid regex in ALLOWED_URL_REGEXPS: " + expr)
+		}
+		allowedUrlRegexps = append(allowedUrlRegexps, re)
+	}
+}
+
+func handleProxyCall(c fiber.Ctx) error {
+	uriBase64 := c.Query("uri")
+	if uriBase64 == "" {
+		return model.NewRequestError("Missing uri parameter")
+	}
+	uriStr, err := base64.URLEncoding.DecodeString(uriBase64)
+	if err != nil {
+		return model.NewRequestError("Invalid base64 encoding")
+	}
+	uri, err := url.Parse(string(uriStr))
+	if err != nil {
+		return model.NewRequestError("Invalid URL")
+	}
+	allowed := false
+	for _, re := range allowedUrlRegexps {
+		if re.MatchString(uri.String()) {
+			allowed = true
+			break
+		}
+	}
+	if !allowed {
+		return model.NewRequestError("URL not allowed")
+	}
+
+	var resp *proxyResponse
+
+	rawVal, err := cache.Get("proxy:" + uri.String())
+	if err == nil {
+		var r proxyResponse
+		err = json.Unmarshal([]byte(rawVal), &r)
+		if err != nil {
+			slog.ErrorContext(c, "Failed to unmarshal cached proxy response", "error", err)
+			return model.NewInternalServerError("Error")
+		}
+		resp = &r
+	} else {
+		resp, err = proxy(uri)
+		if err != nil {
+			slog.ErrorContext(c, "Proxy request failed", "error", err)
+			return model.NewInternalServerError("Error")
+		}
+	}
+
+	c.Response().Header.SetContentType(resp.ContentType)
+	return c.Send([]byte(rawVal))
+}
+
+func proxy(uri *url.URL) (*proxyResponse, error) {
+	client := http.Client{}
+	req, err := http.NewRequest("GET", uri.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	contentType := resp.Header.Get("Content-Type")
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	proxyResp := &proxyResponse{
+		Content:     string(data),
+		ContentType: contentType,
+	}
+
+	j, err := json.Marshal(proxyResp)
+	if err != nil {
+		return nil, err
+	}
+	err = cache.Set("proxy:"+uri.String(), string(j), 24*time.Hour)
+	if err != nil {
+		slog.Error("Failed to cache proxy response", "error", err)
+	}
+	return proxyResp, nil
+}
+
+func AddProxyRoutes(router fiber.Router) {
+	router.Get("/proxy", handleProxyCall)
+}

server/cache/cache.go

@@ -0,0 +1,42 @@
+package cache
+
+import (
+	"context"
+	"os"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+)
+
+var (
+	client *redis.Client
+	ctx    = context.Background()
+)
+
+func init() {
+	host := os.Getenv("REDIS_HOST")
+	port := os.Getenv("REDIS_PORT")
+
+	if host == "" {
+		host = "localhost"
+	}
+	if port == "" {
+		port = "6379"
+	}
+
+	client = redis.NewClient(&redis.Options{
+		Addr: host + ":" + port,
+	})
+}
+
+func Get(key string) (string, error) {
+	val, err := client.Get(ctx, key).Result()
+	if err != nil {
+		return "", err
+	}
+	return val, nil
+}
+
+func Set(key, value string, expiration time.Duration) error {
+	return client.Set(ctx, key, value, expiration).Err()
+}