From 7e612b3dd4e5dd92305efa3ebf569b5f1aa27b25 Mon Sep 17 00:00:00 2001 From: nyne Date: Mon, 26 May 2025 20:51:02 +0800 Subject: [PATCH] Add server download task creation and related UI components --- frontend/src/i18n.ts | 6 + frontend/src/network/network.ts | 21 +++ frontend/src/pages/resource_details_page.tsx | 63 ++++++- server/api/file.go | 26 +++ server/dao/file.go | 16 ++ server/dao/statistic.go | 35 ++-- server/service/file.go | 175 ++++++++++++++++++- 7 files changed, 324 insertions(+), 18 deletions(-) diff --git a/frontend/src/i18n.ts b/frontend/src/i18n.ts index 206cc95..556cba6 100644 --- a/frontend/src/i18n.ts +++ b/frontend/src/i18n.ts @@ -165,6 +165,8 @@ export const i18nData = { "Views Descending": "Views Descending", "Downloads Ascending": "Downloads Ascending", "Downloads Descending": "Downloads Descending", + "File Url": "File Url", + "Provide a file url for the server to download, and the file will be moved to the selected storage.": "Provide a file url for the server to download, and the file will be moved to the selected storage.", } }, "zh-CN": { @@ -333,6 +335,8 @@ export const i18nData = { "Views Descending": "浏览量降序", "Downloads Ascending": "下载量升序", "Downloads Descending": "下载量降序", + "File Url": "文件链接", + "Provide a file url for the server to download, and the file will be moved to the selected storage.": "提供一个文件链接供服务器下载，文件将被移动到选定的存储中。", } }, "zh-TW": { @@ -501,6 +505,8 @@ export const i18nData = { "Views Descending": "瀏覽量降序", "Downloads Ascending": "下載量升序", "Downloads Descending": "下載量降序", + "File Url": "檔案連結", + "Provide a file url for the server to download, and the file will be moved to the selected storage.": "提供一個檔案連結供伺服器下載，檔案將被移動到選定的儲存中。", } } } diff --git a/frontend/src/network/network.ts b/frontend/src/network/network.ts index 587ddec..e395b70 100644 --- a/frontend/src/network/network.ts +++ b/frontend/src/network/network.ts @@ -666,6 +666,27 @@ class Network { } } + async createServerDownloadTask(url: string, filename: string, description: string, + resourceId: number, storageId: number): Promise> { + try { + const response = await axios.post(`${this.apiBaseUrl}/files/upload/url`, { + url, + filename, + description, + resource_id: resourceId, + storage_id: storageId + }); + return response.data; + } + catch (e: any) { + console.error(e); + return { + success: false, + message: e.toString(), + }; + } + } + async getFile(fileId: string): Promise> { try { const response = await axios.get(`${this.apiBaseUrl}/files/${fileId}`); diff --git a/frontend/src/pages/resource_details_page.tsx b/frontend/src/pages/resource_details_page.tsx index a62c06c..4c6b69c 100644 --- a/frontend/src/pages/resource_details_page.tsx +++ b/frontend/src/pages/resource_details_page.tsx @@ -402,6 +402,7 @@ function Files({files, resourceID}: { files: RFile[], resourceID: number }) { enum FileType { redirect = "redirect", upload = "upload", + serverTask = "server_task" } function CreateFileDialog({resourceId}: { resourceId: number }) { @@ -418,6 +419,8 @@ function CreateFileDialog({resourceId}: { resourceId: number }) { const [file, setFile] = useState(null) const [description, setDescription] = useState("") + const [fileUrl, setFileUrl] = useState("") + const reload = useContext(context) const [isSubmitting, setSubmitting] = useState(false) @@ -457,7 +460,7 @@ function CreateFileDialog({resourceId}: { resourceId: number }) { setError(res.message) setSubmitting(false) } - } else { + } else if (fileType === FileType.upload) { if (!file || !storage) { setError(t("Please select a file and storage")) setSubmitting(false) @@ -477,6 +480,23 @@ function CreateFileDialog({resourceId}: { resourceId: number }) { setError(res.message) setSubmitting(false) } + } else if (fileType === FileType.serverTask) { + if (!fileUrl || !filename || !storage) { + setError(t("Please fill in all fields")); + setSubmitting(false); + return; + } + const res = await network.createServerDownloadTask(fileUrl, filename, description, resourceId, storage.id); + if (res.success) { + setSubmitting(false) + const dialog = document.getElementById("upload_dialog") as HTMLDialogElement + dialog.close() + showToast({message: t("File created successfully"), type: "success"}) + reload() + } else { + setError(res.message) + setSubmitting(false) + } } } @@ -527,6 +547,9 @@ function CreateFileDialog({resourceId}: { resourceId: number }) { { setFileType(FileType.upload); }}/> + { + setFileType(FileType.serverTask); + }}/> { @@ -581,6 +604,44 @@ function CreateFileDialog({resourceId}: { resourceId: number }) { } + { + fileType === FileType.serverTask && <> +

{t("Provide a file url for the server to download, and the file will be moved to the selected storage.")}

+ + + { + setFilename(e.target.value) + }}/> + + { + setFileUrl(e.target.value) + }}/> + + { + setDescription(e.target.value) + }}/> + + } + {error && }
diff --git a/server/api/file.go b/server/api/file.go index ecf5701..8b60019 100644 --- a/server/api/file.go +++ b/server/api/file.go @@ -21,6 +21,7 @@ func AddFileRoutes(router fiber.Router) { fileGroup.Post("/upload/finish/:id", finishUpload) fileGroup.Post("/upload/cancel/:id", cancelUpload) fileGroup.Post("/redirect", createRedirectFile) + fileGroup.Post("/upload/url", createServerDownloadTask) fileGroup.Get("/:id", getFile) fileGroup.Put("/:id", updateFile) fileGroup.Delete("/:id", deleteFile) @@ -245,3 +246,28 @@ func downloadLocalFile(c fiber.Ctx) error { ByteRange: true, }) } + +func createServerDownloadTask(c fiber.Ctx) error { + uid := c.Locals("uid").(uint) + + type InitUploadRequest struct { + Url string `json:"url"` + Filename string `json:"filename"` + Description string `json:"description"` + ResourceID uint `json:"resource_id"` + StorageID uint `json:"storage_id"` + } + + var req InitUploadRequest + if err := c.Bind().Body(&req); err != nil { + return model.NewRequestError("Invalid request parameters") + } + result, err := service.CreateServerDownloadTask(uid, req.Url, req.Filename, req.Description, req.ResourceID, req.StorageID) + if err != nil { + return err + } + return c.JSON(model.Response[*model.FileView]{ + Success: true, + Data: result, + }) +} diff --git a/server/dao/file.go b/server/dao/file.go index 8b0a6cf..204280f 100644 --- a/server/dao/file.go +++ b/server/dao/file.go @@ -153,3 +153,19 @@ func SetFileStorageKey(id string, storageKey string) error { } return nil } + +func SetFileStorageKeyAndSize(id string, storageKey string, size int64) error { + f := &model.File{} + if err := db.Where("uuid = ?", id).First(f).Error; err != nil { + return err + } + f.StorageKey = storageKey + f.Size = size + if err := db.Save(f).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return model.NewNotFoundError("file not found") + } + return err + } + return nil +} diff --git a/server/dao/statistic.go b/server/dao/statistic.go index d2dbad3..33f3374 100644 --- a/server/dao/statistic.go +++ b/server/dao/statistic.go @@ -1,17 +1,10 @@ package dao -import "nysoure/server/model" - -func SetStatistic(key string, value int64) error { - statistic := &model.Statistic{ - Key: key, - Value: value, - } - if err := db.Save(statistic).Error; err != nil { - return err - } - return nil -} +import ( + "errors" + "gorm.io/gorm" + "nysoure/server/model" +) func GetStatistic(key string) int64 { statistic := &model.Statistic{} @@ -22,3 +15,21 @@ func GetStatistic(key string) int64 { } return statistic.Value } + +func UpdateStatistic(key string, offset int64) error { + return db.Transaction(func(tx *gorm.DB) error { + statistic := &model.Statistic{} + if err := tx.Where(&model.Statistic{ + Key: key, + }).First(statistic).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + statistic.Key = key + statistic.Value = offset + return tx.Create(statistic).Error + } + return err + } + statistic.Value += offset + return tx.Save(statistic).Error + }) +} diff --git a/server/service/file.go b/server/service/file.go index cf602e6..af0a522 100644 --- a/server/service/file.go +++ b/server/service/file.go @@ -3,6 +3,8 @@ package service import ( "crypto/md5" "encoding/hex" + "io" + "net/http" "nysoure/server/config" "nysoure/server/dao" "nysoure/server/model" @@ -19,7 +21,8 @@ import ( ) const ( - blockSize = 4 * 1024 * 1024 // 4MB + blockSize = 4 * 1024 * 1024 // 4MB + storageKeyUnavailable = "storage_key_unavailable" // Placeholder for unavailable storage key ) var ( @@ -44,9 +47,7 @@ func getUploadingSize() int64 { } func updateUploadingSize(offset int64) { - c := dao.GetStatistic("uploading_size") - c += offset - _ = dao.SetStatistic("uploading_size", c) + _ = dao.UpdateStatistic("uploading_size", offset) } func getTempDir() (string, error) { @@ -250,7 +251,7 @@ func FinishUploadingFile(uid uint, fid uint, md5Str string) (*model.FileView, er return nil, model.NewInternalServerError("failed to finish uploading file. please re-upload") } - dbFile, err := dao.CreateFile(uploadingFile.Filename, uploadingFile.Description, uploadingFile.TargetResourceID, &uploadingFile.TargetStorageID, "", "", uploadingFile.TotalSize, uid) + dbFile, err := dao.CreateFile(uploadingFile.Filename, uploadingFile.Description, uploadingFile.TargetResourceID, &uploadingFile.TargetStorageID, storageKeyUnavailable, "", uploadingFile.TotalSize, uid) if err != nil { log.Error("failed to create file in db: ", err) _ = os.Remove(resultFilePath) @@ -430,6 +431,9 @@ func DownloadFile(ip, fid, cfToken string) (string, string, error) { log.Error("failed to get file: ", err) return "", "", model.NewNotFoundError("file not found") } + if file.StorageKey == storageKeyUnavailable { + return "", "", model.NewRequestError("file is not available, please try again later") + } if file.StorageID == nil { if file.RedirectUrl != "" { @@ -455,3 +459,164 @@ func DownloadFile(ip, fid, cfToken string) (string, string, error) { return path, file.Filename, err } + +func testFileUrl(url string) (int64, error) { + client := http.Client{ + Timeout: 10 * time.Second, + } + req, err := http.NewRequest("HEAD", url, nil) + if err != nil { + return 0, model.NewRequestError("failed to create HTTP request") + } + resp, err := client.Do(req) + if err != nil { + return 0, model.NewRequestError("failed to send HTTP request") + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return 0, model.NewRequestError("URL is not accessible, status code: " + resp.Status) + } + if resp.Header.Get("Content-Length") == "" { + return 0, model.NewRequestError("URL does not provide content length") + } + contentLength, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) + if err != nil { + return 0, model.NewRequestError("failed to parse Content-Length header") + } + if contentLength <= 0 { + return 0, model.NewRequestError("Content-Length is not valid") + } + return contentLength, nil +} + +func CreateServerDownloadTask(uid uint, url, filename, description string, resourceID, storageID uint) (*model.FileView, error) { + canUpload, err := checkUserCanUpload(uid) + if err != nil { + log.Error("failed to check user permission: ", err) + return nil, model.NewInternalServerError("failed to check user permission") + } + if !canUpload { + return nil, model.NewUnAuthorizedError("user cannot upload file") + } + + contentLength, err := testFileUrl(url) + if err != nil { + log.Error("failed to test file URL: ", err) + return nil, model.NewRequestError("failed to test file URL: " + err.Error()) + } + + file, err := dao.CreateFile(filename, description, resourceID, &storageID, storageKeyUnavailable, "", 0, uid) + if err != nil { + log.Error("failed to create file in db: ", err) + return nil, model.NewInternalServerError("failed to create file in db") + } + + updateUploadingSize(contentLength) + + go func() { + defer func() { + updateUploadingSize(-contentLength) + }() + client := http.Client{ + Timeout: 10 * time.Second, + } + req, err := http.NewRequest("GET", url, nil) + if err != nil { + log.Error("failed to create HTTP request: ", err) + _ = dao.DeleteFile(file.UUID) + return + } + resp, err := client.Do(req) + if err != nil { + log.Error("failed to send HTTP request: ", err) + _ = dao.DeleteFile(file.UUID) + return + } + if err != nil { + log.Error("failed to parse Content-Length header: ", err) + _ = dao.DeleteFile(file.UUID) + return + } + tempPath := filepath.Join(utils.GetStoragePath(), uuid.NewString()) + tempFile, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY, os.ModePerm) + if err != nil { + log.Error("failed to open temp file: ", err) + _ = dao.DeleteFile(file.UUID) + return + } + defer func() { + _ = tempFile.Close() + if err := os.Remove(tempPath); err != nil { + log.Error("failed to remove temp file: ", err) + } + }() + if _, err := io.Copy(tempFile, resp.Body); err != nil { + log.Error("failed to copy and hash response body: ", err) + _ = dao.DeleteFile(file.UUID) + return + } + _ = resp.Body.Close() + if err := tempFile.Close(); err != nil { + log.Error("failed to close temp file: ", err) + _ = dao.DeleteFile(file.UUID) + return + } + stat, err := os.Stat(tempPath) + if err != nil { + log.Error("failed to get temp file info: ", err) + _ = dao.DeleteFile(file.UUID) + _ = os.Remove(tempPath) + return + } + size := stat.Size() + if size == 0 { + log.Error("downloaded file is empty") + _ = dao.DeleteFile(file.UUID) + _ = os.Remove(tempPath) + return + } + if size != contentLength { + log.Error("downloaded file size does not match expected size: ", size, " != ", contentLength) + _ = dao.DeleteFile(file.UUID) + _ = os.Remove(tempPath) + return + } + s, err := dao.GetStorage(storageID) + if err != nil { + log.Error("failed to get storage: ", err) + _ = dao.DeleteFile(file.UUID) + _ = os.Remove(tempPath) + return + } + iStorage := storage.NewStorage(s) + if iStorage == nil { + log.Error("failed to find storage: ", err) + _ = dao.DeleteFile(file.UUID) + _ = os.Remove(tempPath) + return + } + storageKey, err := iStorage.Upload(tempPath, filename) + if err != nil { + log.Error("failed to upload file to storage: ", err) + _ = dao.DeleteFile(file.UUID) + _ = os.Remove(tempPath) + return + } + if err := dao.SetFileStorageKeyAndSize(file.UUID, storageKey, size); err != nil { + log.Error("failed to set file storage key: ", err) + _ = dao.DeleteFile(file.UUID) + _ = iStorage.Delete(storageKey) + _ = os.Remove(tempPath) + return + } + if err := dao.AddStorageUsage(storageID, size); err != nil { + log.Error("failed to add storage usage: ", err) + _ = dao.DeleteFile(file.UUID) + _ = iStorage.Delete(storageKey) + _ = os.Remove(tempPath) + return + } + }() + + return file.ToView(), nil +}