fix: retrieve resource details before adding to index in createIndex

.env.example

@@ -33,3 +33,6 @@ BACKUP_RETENTION_DAYS=30
 
 # Download Configuration
 DOWNLOAD_SECRET_KEY=your_download_secret_key_here
+
+# Access Key for Development API
+DEV_ACCESS_KEY=your_dev_access_key_here

main.go

@@ -42,6 +42,7 @@ func main() {
 		api.AddActivityRoutes(apiG)
 		api.AddCollectionRoutes(apiG)
 		api.AddProxyRoutes(apiG)
+		api.AddDevAPI(apiG)
 	}
 
 	log.Fatal(app.Listen(":3000"))

server/api/dev.go

@@ -0,0 +1,29 @@
+package api
+
+import (
+	"nysoure/server/middleware"
+
+	"nysoure/server/search"
+
+	"github.com/gofiber/fiber/v3"
+)
+
+func rebuildSearchIndex(c fiber.Ctx) error {
+	err := search.RebuildSearchIndex()
+	if err != nil {
+		return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
+			"error": "Failed to rebuild search index: " + err.Error(),
+		})
+	}
+	return c.JSON(fiber.Map{
+		"message": "Search index rebuilt successfully",
+	})
+}
+
+func AddDevAPI(router fiber.Router) {
+	devGroup := router.Group("/dev")
+	devGroup.Use(middleware.DevMiddleware())
+	{
+		devGroup.Post("/rebuild_search_index", rebuildSearchIndex)
+	}
+}

server/api/resource.go

@@ -287,7 +287,11 @@ func handleGetCharactersFromVndb(c fiber.Ctx) error {
 	if vnID == "" {
 		return model.NewRequestError("VNDB ID is required")
 	}
-	characters, err := service.GetCharactersFromVndb(vnID)
+	uid, ok := c.Locals("uid").(uint)
+	if !ok {
+		return model.NewUnAuthorizedError("You must be logged in to get characters from VNDB")
+	}
+	characters, err := service.GetCharactersFromVndb(vnID, uid)
 	if err != nil {
 		return err
 	}

server/middleware/dev_middleware.go

@@ -0,0 +1,22 @@
+package middleware
+
+import (
+	"nysoure/server/model"
+	"os"
+
+	"github.com/gofiber/fiber/v3"
+)
+
+func DevMiddleware() func(c fiber.Ctx) error {
+	AccessKey := os.Getenv("DEV_ACCESS_KEY")
+	return func(c fiber.Ctx) error {
+		if AccessKey == "" {
+			return model.NewUnAuthorizedError("Unauthorized")
+		}
+		providedKey := c.Get("X-DEV-ACCESS-KEY")
+		if providedKey != AccessKey {
+			return model.NewUnAuthorizedError("Unauthorized")
+		}
+		return c.Next()
+	}
+}

server/search/resource.go

@@ -3,30 +3,54 @@ package search
 import (
 	"errors"
 	"fmt"
+	"log/slog"
 	"nysoure/server/dao"
 	"nysoure/server/model"
 	"nysoure/server/utils"
+	"os"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/blevesearch/bleve"
 )
 
+var (
+	index bleve.Index
+	mu    = sync.RWMutex{}
+)
+
 type ResourceParams struct {
-	Id        uint
-	Title     string
-	Subtitles []string
-	Time      time.Time
+	Id         uint
+	Title      string
+	Subtitles  []string
+	Time       time.Time
+	Characters []ResourceCharacter
 }
 
-var index bleve.Index
+type ResourceCharacter struct {
+	Name  string
+	Alias []string
+	CV    string
+}
 
 func AddResourceToIndex(r model.Resource) error {
+	mu.RLock()
+	defer mu.RUnlock()
+	cs := make([]ResourceCharacter, 0, len(r.Characters))
+	for _, c := range r.Characters {
+		cs = append(cs, ResourceCharacter{
+			Name:  c.Name,
+			Alias: c.Alias,
+			CV:    c.CV,
+		})
+	}
 	return index.Index(fmt.Sprintf("%d", r.ID), ResourceParams{
-		Id:        r.ID,
-		Title:     r.Title,
-		Subtitles: r.AlternativeTitles,
-		Time:      r.CreatedAt,
+		Id:         r.ID,
+		Title:      r.Title,
+		Subtitles:  r.AlternativeTitles,
+		Time:       r.CreatedAt,
+		Characters: cs,
 	})
 }
 
@@ -40,16 +64,25 @@ func createIndex() error {
 	}
 	page := 1
 	total := 1
+	current := 0
 	for page <= total {
 		res, totalPages, err := dao.GetResourceList(page, 100, model.RSortTimeAsc)
 		if err != nil {
 			return err
 		}
 		for _, r := range res {
-			err := AddResourceToIndex(r)
+			r, err := dao.GetResourceByID(r.ID)
 			if err != nil {
 				return err
 			}
+			err = AddResourceToIndex(r)
+			if err != nil {
+				return err
+			}
+			current++
+			if current%20 == 0 {
+				slog.Info("Rebuilding search index", "current", current, "total", totalPages*100)
+			}
 		}
 		page++
 		total = totalPages
@@ -80,6 +113,8 @@ func init() {
 }
 
 func SearchResource(keyword string) ([]uint, error) {
+	mu.RLock()
+	defer mu.RUnlock()
 	query := bleve.NewMatchQuery(keyword)
 	searchRequest := bleve.NewSearchRequest(query)
 	searchResults, err := index.Search(searchRequest)
@@ -112,3 +147,24 @@ func IsStopWord(word string) bool {
 	tokens := analyzer.Analyze([]byte(word))
 	return len(tokens) == 0
 }
+
+func RebuildSearchIndex() error {
+	mu.Lock()
+	defer mu.Unlock()
+	err := index.Close()
+	if err != nil {
+		return fmt.Errorf("failed to close search index: %w", err)
+	}
+	indexPath := utils.GetStoragePath() + "/resource_index.bleve"
+	err = os.RemoveAll(indexPath)
+	if err != nil {
+		return fmt.Errorf("failed to remove search index: %w", err)
+	}
+	mapping := bleve.NewIndexMapping()
+	index, err = bleve.New(indexPath, mapping)
+	if err != nil {
+		return fmt.Errorf("failed to create search index: %w", err)
+	}
+	go createIndex()
+	return nil
+}

server/service/resource.go

@@ -615,7 +615,15 @@ func GetPinnedResources() ([]model.ResourceView, error) {
 	return views, nil
 }
 
-func GetCharactersFromVndb(vnID string) ([]CharacterParams, error) {
+func GetCharactersFromVndb(vnID string, uid uint) ([]CharacterParams, error) {
+	canUpload, err := checkUserCanUpload(uid)
+	if err != nil {
+		return nil, err
+	}
+	if !canUpload {
+		return nil, model.NewUnAuthorizedError("You have not permission to fetch characters from VNDB")
+	}
+
 	client := http.Client{}
 	jsonStr := fmt.Sprintf(`
 	{