diff --git a/docker-compose.yml b/docker-compose.yml index ea9e584..6b82004 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,14 +9,22 @@ services: - app_data:/var/lib/nysoure depends_on: - db + - tendis environment: - DB_HOST=db - DB_PORT=3306 - DB_USER=nysoure - DB_PASSWORD=nysoure_password - DB_NAME=nysoure + - REDIS_HOST=tendis + - REDIS_PORT=6379 - BANNED_REDIRECT_DOMAINS=example.com,example.org restart: unless-stopped + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" db: image: mariadb:latest @@ -30,7 +38,26 @@ services: ports: - "3306" restart: unless-stopped + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + + tendis: + image: tendis/tendis:latest + volumes: + - tendis_data:/data + ports: + - "6379" + restart: unless-stopped + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" volumes: app_data: db_data: + tendis_data: diff --git a/go.mod b/go.mod index da6bb94..9cdc652 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/blevesearch/bleve v1.0.14 github.com/chai2010/webp v1.4.0 github.com/disintegration/imaging v1.6.2 + github.com/redis/go-redis/v9 v9.17.0 github.com/stretchr/testify v1.11.1 gorm.io/driver/mysql v1.6.0 ) @@ -30,8 +31,10 @@ require ( github.com/blevesearch/zap/v13 v13.0.6 // indirect github.com/blevesearch/zap/v14 v14.0.5 // indirect github.com/blevesearch/zap/v15 v15.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/couchbase/vellum v1.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 // indirect github.com/go-sql-driver/mysql v1.8.1 // indirect github.com/golang/protobuf v1.3.2 // indirect diff --git a/go.sum b/go.sum index 28290e6..0767b82 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,12 @@ github.com/blevesearch/zap/v14 v14.0.5 h1:NdcT+81Nvmp2zL+NhwSvGSLh7xNgGL8QRVZ67n github.com/blevesearch/zap/v14 v14.0.5/go.mod h1:bWe8S7tRrSBTIaZ6cLRbgNH4TUDaC9LZSpRGs85AsGY= github.com/blevesearch/zap/v15 v15.0.3 h1:Ylj8Oe+mo0P25tr9iLPp33lN6d4qcztGjaIsP51UxaY= github.com/blevesearch/zap/v15 v15.0.3/go.mod h1:iuwQrImsh1WjWJ0Ue2kBqY83a0rFtJTqfa9fp1rbVVU= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/webp v1.4.0 h1:6DA2pkkRUPnbOHvvsmGI3He1hBKf/bkRlniAiSGuEko= github.com/chai2010/webp v1.4.0/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -46,6 +52,8 @@ github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= @@ -135,6 +143,8 @@ github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/redis/go-redis/v9 v9.17.0 h1:K6E+ZlYN95KSMmZeEQPbU/c++wfmEvfFB17yEAq/VhM= +github.com/redis/go-redis/v9 v9.17.0/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= diff --git a/main.go b/main.go index 1d97f6f..2434c50 100644 --- a/main.go +++ b/main.go @@ -38,7 +38,8 @@ func main() { api.AddCommentRoutes(apiG) api.AddConfigRoutes(apiG) api.AddActivityRoutes(apiG) - api.AddCollectionRoutes(apiG) // 新增 + api.AddCollectionRoutes(apiG) + api.AddProxyRoutes(apiG) } log.Fatal(app.Listen(":3000")) diff --git a/server/api/proxy.go b/server/api/proxy.go new file mode 100644 index 0000000..20c17bf --- /dev/null +++ b/server/api/proxy.go @@ -0,0 +1,124 @@ +package api + +import ( + "encoding/base64" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/url" + "nysoure/server/cache" + "nysoure/server/model" + "os" + "regexp" + "strings" + "time" + + "github.com/gofiber/fiber/v3" +) + +var ( + allowedUrlRegexps []*regexp.Regexp +) + +type proxyResponse struct { + Content string `json:"content"` + ContentType string `json:"content_type"` +} + +func init() { + regexps := os.Getenv("ALLOWED_URL_REGEXPS") + for _, expr := range strings.Split(regexps, ",") { + if expr == "" { + continue + } + re, err := regexp.Compile(expr) + if err != nil { + panic("Invalid regex in ALLOWED_URL_REGEXPS: " + expr) + } + allowedUrlRegexps = append(allowedUrlRegexps, re) + } +} + +func handleProxyCall(c fiber.Ctx) error { + uriBase64 := c.Query("uri") + if uriBase64 == "" { + return model.NewRequestError("Missing uri parameter") + } + uriStr, err := base64.URLEncoding.DecodeString(uriBase64) + if err != nil { + return model.NewRequestError("Invalid base64 encoding") + } + uri, err := url.Parse(string(uriStr)) + if err != nil { + return model.NewRequestError("Invalid URL") + } + allowed := false + for _, re := range allowedUrlRegexps { + if re.MatchString(uri.String()) { + allowed = true + break + } + } + if !allowed { + return model.NewRequestError("URL not allowed") + } + + var resp *proxyResponse + + rawVal, err := cache.Get("proxy:" + uri.String()) + if err == nil { + var r proxyResponse + err = json.Unmarshal([]byte(rawVal), &r) + if err != nil { + slog.ErrorContext(c, "Failed to unmarshal cached proxy response", "error", err) + return model.NewInternalServerError("Error") + } + resp = &r + } else { + resp, err = proxy(uri) + if err != nil { + slog.ErrorContext(c, "Proxy request failed", "error", err) + return model.NewInternalServerError("Error") + } + } + + c.Response().Header.SetContentType(resp.ContentType) + return c.Send([]byte(rawVal)) +} + +func proxy(uri *url.URL) (*proxyResponse, error) { + client := http.Client{} + req, err := http.NewRequest("GET", uri.String(), nil) + if err != nil { + return nil, err + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + contentType := resp.Header.Get("Content-Type") + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + proxyResp := &proxyResponse{ + Content: string(data), + ContentType: contentType, + } + + j, err := json.Marshal(proxyResp) + if err != nil { + return nil, err + } + err = cache.Set("proxy:"+uri.String(), string(j), 24*time.Hour) + if err != nil { + slog.Error("Failed to cache proxy response", "error", err) + } + return proxyResp, nil +} + +func AddProxyRoutes(router fiber.Router) { + router.Get("/proxy", handleProxyCall) +} diff --git a/server/cache/cache.go b/server/cache/cache.go new file mode 100644 index 0000000..e887653 --- /dev/null +++ b/server/cache/cache.go @@ -0,0 +1,42 @@ +package cache + +import ( + "context" + "os" + "time" + + "github.com/redis/go-redis/v9" +) + +var ( + client *redis.Client + ctx = context.Background() +) + +func init() { + host := os.Getenv("REDIS_HOST") + port := os.Getenv("REDIS_PORT") + + if host == "" { + host = "localhost" + } + if port == "" { + port = "6379" + } + + client = redis.NewClient(&redis.Options{ + Addr: host + ":" + port, + }) +} + +func Get(key string) (string, error) { + val, err := client.Get(ctx, key).Result() + if err != nil { + return "", err + } + return val, nil +} + +func Set(key, value string, expiration time.Duration) error { + return client.Set(ctx, key, value, expiration).Err() +}