Add RSS/Sitemap handling.

server/api/resource.go

@@ -2,14 +2,26 @@ package api
 
 import (
 	"encoding/json"
+	"github.com/gofiber/fiber/v3/log"
 	"net/url"
+	"nysoure/server/dao"
 	"nysoure/server/model"
 	"nysoure/server/service"
+	"nysoure/server/utils"
 	"strconv"
 
 	"github.com/gofiber/fiber/v3"
 )
 
+func updateSiteMapAndRss(baseURL string) {
+	resources, err := dao.GetAllResources()
+	if err != nil {
+		log.Error("Error getting resources: ", err)
+	}
+	utils.GenerateSiteMap(baseURL, resources)
+	utils.GenerateRss(baseURL, resources)
+}
+
 func handleCreateResource(c fiber.Ctx) error {
 	var params service.ResourceCreateParams
 	body := c.Body()
@@ -25,6 +37,7 @@ func handleCreateResource(c fiber.Ctx) error {
 	if err != nil {
 		return err
 	}
+	updateSiteMapAndRss(c.BaseURL())
 	return c.Status(fiber.StatusOK).JSON(model.Response[uint]{
 		Success: true,
 		Data:    id,
@@ -69,6 +82,7 @@ func handleDeleteResource(c fiber.Ctx) error {
 	if err != nil {
 		return err
 	}
+	updateSiteMapAndRss(c.BaseURL())
 	return c.Status(fiber.StatusOK).JSON(model.Response[any]{
 		Success: true,
 		Data:    nil,
@@ -211,6 +225,7 @@ func handleUpdateResource(c fiber.Ctx) error {
 	if err != nil {
 		return err
 	}
+	updateSiteMapAndRss(c.BaseURL())
 	return c.Status(fiber.StatusOK).JSON(model.Response[any]{
 		Success: true,
 		Data:    nil,

server/dao/resource.go

@@ -233,3 +233,13 @@ func GetResourcesByUsername(username string, page, pageSize int) ([]model.Resour
 
 	return resources, int(totalPages), nil
 }
+
+// GetAllResources retrieves all resources from the database without all related data.
+// It is used to generate a sitemap and rss feed.
+func GetAllResources() ([]model.Resource, error) {
+	var resources []model.Resource
+	if err := db.Find(&resources).Error; err != nil {
+		return nil, err
+	}
+	return resources, nil
+}

server/middleware/frontend_middleware.go

@@ -4,15 +4,13 @@ import (
 	"fmt"
 	"nysoure/server/config"
 	"nysoure/server/service"
+	"nysoure/server/utils"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/gofiber/fiber/v3"
-	"github.com/gomarkdown/markdown"
-	"github.com/gomarkdown/markdown/html"
-	"github.com/gomarkdown/markdown/parser"
-	"github.com/k3a/html2text"
 )
 
 func FrontendMiddleware(c fiber.Ctx) error {
@@ -23,6 +21,14 @@ func FrontendMiddleware(c fiber.Ctx) error {
 	path := c.Path()
 	file := "static" + path
 
+	if path == "/robots.txt" {
+		return handleRobotsTxt(c)
+	} else if path == "/sitemap.xml" {
+		return handleSiteMap(c)
+	} else if path == "/rss.xml" {
+		return handleRss(c)
+	}
+
 	if _, err := os.Stat(file); path == "/" || os.IsNotExist(err) {
 		return serveIndexHtml(c)
 	} else {
@@ -30,6 +36,23 @@ func FrontendMiddleware(c fiber.Ctx) error {
 	}
 }
 
+func handleRobotsTxt(c fiber.Ctx) error {
+	c.Set("Content-Type", "text/plain; charset=utf-8")
+	c.Set("Cache-Control", "no-cache")
+	c.Set("X-Robots-Tag", "noindex")
+	return c.SendString("User-agent: *\nDisallow: /api/\nDisallow: /admin/\n")
+}
+
+func handleSiteMap(c fiber.Ctx) error {
+	path := filepath.Join(utils.GetStoragePath(), utils.SiteMapFileName)
+	return c.SendFile(path)
+}
+
+func handleRss(c fiber.Ctx) error {
+	path := filepath.Join(utils.GetStoragePath(), utils.RssFileName)
+	return c.SendFile(path)
+}
+
 func serveIndexHtml(c fiber.Ctx) error {
 	data, err := os.ReadFile("static/index.html")
 	if err != nil {
@@ -58,7 +81,7 @@ func serveIndexHtml(c fiber.Ctx) error {
 					preview = fmt.Sprintf("%s/api/image/%d", serverBaseURL, r.Images[0].ID)
 				}
 				title = r.Title
-				description = getResourceDescription(r.Article)
+				description = utils.ArticleToDescription(r.Article, 200)
 			}
 		}
 	} else if strings.HasPrefix(path, "/user/") {
@@ -82,46 +105,3 @@ func serveIndexHtml(c fiber.Ctx) error {
 	c.Set("Content-Type", "text/html; charset=utf-8")
 	return c.SendString(content)
 }
-
-func mergeSpaces(str string) string {
-	// Replace multiple spaces with a single space
-	builder := strings.Builder{}
-	for i, r := range str {
-		if r == '\t' || r == '\r' {
-			continue
-		}
-		if r == ' ' || r == '\n' {
-			if i > 0 && str[i-1] != ' ' && str[i-1] != '\n' {
-				builder.WriteRune(' ')
-			}
-		} else {
-			builder.WriteRune(r)
-		}
-	}
-	return builder.String()
-}
-
-func getResourceDescription(article string) string {
-	htmlContent := mdToHTML([]byte(article))
-	plain := html2text.HTML2Text(string(htmlContent))
-	plain = strings.TrimSpace(plain)
-	plain = mergeSpaces(plain)
-	if len([]rune(plain)) > 200 {
-		plain = string([]rune(plain)[:197]) + "..."
-	}
-	return plain
-}
-
-func mdToHTML(md []byte) []byte {
-	// create Markdown parser with extensions
-	extensions := parser.CommonExtensions | parser.NoEmptyLineBeforeBlock | parser.MathJax
-	p := parser.NewWithExtensions(extensions)
-	doc := p.Parse(md)
-
-	// create HTML renderer with extensions
-	htmlFlags := html.CommonFlags | html.HrefTargetBlank
-	opts := html.RendererOptions{Flags: htmlFlags}
-	renderer := html.NewRenderer(opts)
-
-	return markdown.Render(doc, renderer)
-}

server/utils/article_to_description.go

@@ -0,0 +1,55 @@
+package utils
+
+import (
+	"github.com/gomarkdown/markdown"
+	"github.com/gomarkdown/markdown/html"
+	"github.com/gomarkdown/markdown/parser"
+	"github.com/k3a/html2text"
+	"strings"
+)
+
+func ArticleToDescription(article string, maxLength int) string {
+	if maxLength < 3 {
+		maxLength = 3
+	}
+	htmlContent := mdToHTML([]byte(article))
+	plain := html2text.HTML2Text(string(htmlContent))
+	plain = strings.TrimSpace(plain)
+	plain = mergeSpaces(plain)
+	if len([]rune(plain)) > maxLength {
+		plain = string([]rune(plain)[:(maxLength-3)]) + "..."
+	}
+	return plain
+}
+
+func mergeSpaces(str string) string {
+	// Replace multiple spaces with a single space
+	builder := strings.Builder{}
+	for i, r := range str {
+		if r == '\t' || r == '\r' {
+			continue
+		}
+		if r == ' ' || r == '\n' {
+			if i > 0 && str[i-1] != ' ' && str[i-1] != '\n' {
+				builder.WriteRune(' ')
+			}
+		} else {
+			builder.WriteRune(r)
+		}
+	}
+	return builder.String()
+}
+
+func mdToHTML(md []byte) []byte {
+	// create Markdown parser with extensions
+	extensions := parser.CommonExtensions | parser.NoEmptyLineBeforeBlock | parser.MathJax
+	p := parser.NewWithExtensions(extensions)
+	doc := p.Parse(md)
+
+	// create HTML renderer with extensions
+	htmlFlags := html.CommonFlags | html.HrefTargetBlank
+	opts := html.RendererOptions{Flags: htmlFlags}
+	renderer := html.NewRenderer(opts)
+
+	return markdown.Render(doc, renderer)
+}

server/utils/rss.go

@@ -0,0 +1,49 @@
+package utils
+
+import (
+	"github.com/gofiber/fiber/v3/log"
+	"net/url"
+	"nysoure/server/model"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+const (
+	RssFileName = "rss.xml"
+)
+
+func GenerateRss(baseURL string, resources []model.Resource) {
+	path := filepath.Join(GetStoragePath(), RssFileName)
+	builder := strings.Builder{}
+	builder.WriteString(`<?xml version="1.0" encoding="UTF-8"?>`)
+	builder.WriteRune('\n')
+	builder.WriteString(`<rss version="2.0">`)
+	builder.WriteRune('\n')
+	builder.WriteString(`<channel>`)
+	builder.WriteRune('\n')
+	for _, resource := range resources {
+		builder.WriteString("  <item>\n")
+		builder.WriteString("    <title>")
+		builder.WriteString(url.PathEscape(resource.Title))
+		builder.WriteString("</title>\n")
+		builder.WriteString("    <link>")
+		builder.WriteString(baseURL + "/resources/" + strconv.Itoa(int(resource.ID)))
+		builder.WriteString("</link>\n")
+		builder.WriteString("    <description>")
+		builder.WriteString(url.PathEscape(ArticleToDescription(resource.Article, 255)))
+		builder.WriteString("</description>\n")
+		builder.WriteString("    <pubDate>")
+		builder.WriteString(resource.UpdatedAt.Format("Mon, 02 Jan 2006 15:04:05 MST"))
+		builder.WriteString("</pubDate>\n")
+		builder.WriteString("  </item>\n")
+	}
+	builder.WriteString(`</channel>`)
+	builder.WriteRune('\n')
+	builder.WriteString(`</rss>`)
+	data := builder.String()
+	if err := os.WriteFile(path, []byte(data), 0644); err != nil {
+		log.Error("failed to write RSS file", err)
+	}
+}

server/utils/site_map.go

@@ -0,0 +1,39 @@
+package utils
+
+import (
+	"fmt"
+	"github.com/gofiber/fiber/v3/log"
+	"nysoure/server/model"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	SiteMapFileName = "sitemap.xml"
+)
+
+func GenerateSiteMap(baseURL string, resources []model.Resource) {
+	path := filepath.Join(GetStoragePath(), SiteMapFileName)
+	builder := strings.Builder{}
+	builder.WriteString(`<?xml version="1.0" encoding="UTF-8"?>`)
+	builder.WriteRune('\n')
+	builder.WriteString(`<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`)
+	builder.WriteRune('\n')
+	for _, resource := range resources {
+		builder.WriteString("  <url>\n")
+		builder.WriteString("    <loc>")
+		builder.WriteString(fmt.Sprintf("%s/resources/%d", baseURL, resource.ID))
+		builder.WriteString("</loc>\n")
+		builder.WriteString("    <lastmod>")
+		builder.WriteString(resource.UpdatedAt.Format("2006-01-02"))
+		builder.WriteString("</lastmod>\n")
+		builder.WriteString("  </url>\n")
+	}
+	builder.WriteString(`</urlset>`)
+	builder.WriteRune('\n')
+	data := builder.String()
+	if err := os.WriteFile(path, []byte(data), 0644); err != nil {
+		log.Error("failed to write site map file", err)
+	}
+}