Add server download task creation and related UI components

frontend/src/i18n.ts

@@ -165,6 +165,8 @@ export const i18nData = {
       "Views Descending": "Views Descending",
       "Downloads Ascending": "Downloads Ascending",
       "Downloads Descending": "Downloads Descending",
+      "File Url": "File Url",
+      "Provide a file url for the server to download, and the file will be moved to the selected storage.": "Provide a file url for the server to download, and the file will be moved to the selected storage.",
     }
   },
   "zh-CN": {
@@ -333,6 +335,8 @@ export const i18nData = {
       "Views Descending": "浏览量降序",
       "Downloads Ascending": "下载量升序",
       "Downloads Descending": "下载量降序",
+      "File Url": "文件链接",
+      "Provide a file url for the server to download, and the file will be moved to the selected storage.": "提供一个文件链接供服务器下载，文件将被移动到选定的存储中。",
     }
   },
   "zh-TW": {
@@ -501,6 +505,8 @@ export const i18nData = {
       "Views Descending": "瀏覽量降序",
       "Downloads Ascending": "下載量升序",
       "Downloads Descending": "下載量降序",
+      "File Url": "檔案連結",
+      "Provide a file url for the server to download, and the file will be moved to the selected storage.": "提供一個檔案連結供伺服器下載，檔案將被移動到選定的儲存中。",
     }
   }
 }

frontend/src/network/network.ts

@@ -666,6 +666,27 @@ class Network {
     }
   }
 
+  async createServerDownloadTask(url: string, filename: string, description: string,
+                                 resourceId: number, storageId: number): Promise<Response<RFile>> {
+    try {
+      const response = await axios.post(`${this.apiBaseUrl}/files/upload/url`, {
+        url,
+        filename,
+        description,
+        resource_id: resourceId,
+        storage_id: storageId
+      });
+      return response.data;
+    }
+    catch (e: any) {
+      console.error(e);
+      return {
+        success: false,
+        message: e.toString(),
+      };
+    }
+  }
+
   async getFile(fileId: string): Promise<Response<RFile>> {
     try {
       const response = await axios.get(`${this.apiBaseUrl}/files/${fileId}`);

frontend/src/pages/resource_details_page.tsx

@@ -402,6 +402,7 @@ function Files({files, resourceID}: { files: RFile[], resourceID: number }) {
 enum FileType {
   redirect = "redirect",
   upload = "upload",
+  serverTask = "server_task"
 }
 
 function CreateFileDialog({resourceId}: { resourceId: number }) {
@@ -418,6 +419,8 @@ function CreateFileDialog({resourceId}: { resourceId: number }) {
   const [file, setFile] = useState<File | null>(null)
   const [description, setDescription] = useState<string>("")
 
+  const [fileUrl, setFileUrl] = useState<string>("")
+
   const reload = useContext(context)
 
   const [isSubmitting, setSubmitting] = useState(false)
@@ -457,7 +460,7 @@ function CreateFileDialog({resourceId}: { resourceId: number }) {
         setError(res.message)
         setSubmitting(false)
       }
-    } else {
+    } else if (fileType === FileType.upload) {
       if (!file || !storage) {
         setError(t("Please select a file and storage"))
         setSubmitting(false)
@@ -477,6 +480,23 @@ function CreateFileDialog({resourceId}: { resourceId: number }) {
         setError(res.message)
         setSubmitting(false)
       }
+    } else if (fileType === FileType.serverTask) {
+      if (!fileUrl || !filename || !storage) {
+        setError(t("Please fill in all fields"));
+        setSubmitting(false);
+        return;
+      }
+      const res = await network.createServerDownloadTask(fileUrl, filename, description, resourceId, storage.id);
+      if (res.success) {
+        setSubmitting(false)
+        const dialog = document.getElementById("upload_dialog") as HTMLDialogElement
+        dialog.close()
+        showToast({message: t("File created successfully"), type: "success"})
+        reload()
+      } else {
+        setError(res.message)
+        setSubmitting(false)
+      }
     }
   }
 
@@ -527,6 +547,9 @@ function CreateFileDialog({resourceId}: { resourceId: number }) {
           <input className="btn text-sm" type="radio" name="type" aria-label={t("Upload")} onInput={() => {
             setFileType(FileType.upload);
           }}/>
+          <input className="btn text-sm" type="radio" name="type" aria-label={t("File Url")} onInput={() => {
+            setFileType(FileType.serverTask);
+          }}/>
         </form>
 
         {
@@ -581,6 +604,44 @@ function CreateFileDialog({resourceId}: { resourceId: number }) {
           </>
         }
 
+        {
+          fileType === FileType.serverTask && <>
+            <p
+              className={"text-sm p-2"}>{t("Provide a file url for the server to download, and the file will be moved to the selected storage.")}</p>
+            <select className="select select-primary w-full my-2" defaultValue={""} onChange={(e) => {
+              const id = parseInt(e.target.value)
+              if (isNaN(id)) {
+                setStorage(null)
+              } else {
+                const s = storages.current?.find((s) => s.id == id)
+                if (s) {
+                  setStorage(s)
+                }
+              }
+            }}>
+              <option value={""} disabled>{t("Select Storage")}</option>
+              {
+                storages.current?.map((s) => {
+                  return <option key={s.id}
+                                 value={s.id}>{s.name}({(s.currentSize / 1024 / 1024).toFixed(2)}/{s.maxSize / 1024 / 1024}MB)</option>
+                })
+              }
+            </select>
+
+            <input type="text" className="input w-full my-2" placeholder={t("File Name")} onChange={(e) => {
+              setFilename(e.target.value)
+            }}/>
+
+            <input type="text" className="input w-full my-2" placeholder={t("File URL")} onChange={(e) => {
+              setFileUrl(e.target.value)
+            }}/>
+
+            <input type="text" className="input w-full my-2" placeholder={t("Description")} onChange={(e) => {
+              setDescription(e.target.value)
+            }}/>
+          </>
+        }
+
         {error && <ErrorAlert className={"my-2"} message={error}/>}
 
         <div className="modal-action">

server/api/file.go

@@ -21,6 +21,7 @@ func AddFileRoutes(router fiber.Router) {
 		fileGroup.Post("/upload/finish/:id", finishUpload)
 		fileGroup.Post("/upload/cancel/:id", cancelUpload)
 		fileGroup.Post("/redirect", createRedirectFile)
+		fileGroup.Post("/upload/url", createServerDownloadTask)
 		fileGroup.Get("/:id", getFile)
 		fileGroup.Put("/:id", updateFile)
 		fileGroup.Delete("/:id", deleteFile)
@@ -245,3 +246,28 @@ func downloadLocalFile(c fiber.Ctx) error {
 		ByteRange: true,
 	})
 }
+
+func createServerDownloadTask(c fiber.Ctx) error {
+	uid := c.Locals("uid").(uint)
+
+	type InitUploadRequest struct {
+		Url         string `json:"url"`
+		Filename    string `json:"filename"`
+		Description string `json:"description"`
+		ResourceID  uint   `json:"resource_id"`
+		StorageID   uint   `json:"storage_id"`
+	}
+
+	var req InitUploadRequest
+	if err := c.Bind().Body(&req); err != nil {
+		return model.NewRequestError("Invalid request parameters")
+	}
+	result, err := service.CreateServerDownloadTask(uid, req.Url, req.Filename, req.Description, req.ResourceID, req.StorageID)
+	if err != nil {
+		return err
+	}
+	return c.JSON(model.Response[*model.FileView]{
+		Success: true,
+		Data:    result,
+	})
+}

server/dao/file.go

@@ -153,3 +153,19 @@ func SetFileStorageKey(id string, storageKey string) error {
 	}
 	return nil
 }
+
+func SetFileStorageKeyAndSize(id string, storageKey string, size int64) error {
+	f := &model.File{}
+	if err := db.Where("uuid = ?", id).First(f).Error; err != nil {
+		return err
+	}
+	f.StorageKey = storageKey
+	f.Size = size
+	if err := db.Save(f).Error; err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return model.NewNotFoundError("file not found")
+		}
+		return err
+	}
+	return nil
+}

server/dao/statistic.go

@@ -1,17 +1,10 @@
 package dao
 
-import "nysoure/server/model"
+import (
-
+	"errors"
-func SetStatistic(key string, value int64) error {
+	"gorm.io/gorm"
-	statistic := &model.Statistic{
+	"nysoure/server/model"
-		Key:   key,
+)
-		Value: value,
-	}
-	if err := db.Save(statistic).Error; err != nil {
-		return err
-	}
-	return nil
-}
 
 func GetStatistic(key string) int64 {
 	statistic := &model.Statistic{}
@@ -22,3 +15,21 @@ func GetStatistic(key string) int64 {
 	}
 	return statistic.Value
 }
+
+func UpdateStatistic(key string, offset int64) error {
+	return db.Transaction(func(tx *gorm.DB) error {
+		statistic := &model.Statistic{}
+		if err := tx.Where(&model.Statistic{
+			Key: key,
+		}).First(statistic).Error; err != nil {
+			if errors.Is(err, gorm.ErrRecordNotFound) {
+				statistic.Key = key
+				statistic.Value = offset
+				return tx.Create(statistic).Error
+			}
+			return err
+		}
+		statistic.Value += offset
+		return tx.Save(statistic).Error
+	})
+}

server/service/file.go

@@ -3,6 +3,8 @@ package service
 import (
 	"crypto/md5"
 	"encoding/hex"
+	"io"
+	"net/http"
 	"nysoure/server/config"
 	"nysoure/server/dao"
 	"nysoure/server/model"
@@ -20,6 +22,7 @@ import (
 
 const (
 	blockSize             = 4 * 1024 * 1024           // 4MB
+	storageKeyUnavailable = "storage_key_unavailable" // Placeholder for unavailable storage key
 )
 
 var (
@@ -44,9 +47,7 @@ func getUploadingSize() int64 {
 }
 
 func updateUploadingSize(offset int64) {
-	c := dao.GetStatistic("uploading_size")
+	_ = dao.UpdateStatistic("uploading_size", offset)
-	c += offset
-	_ = dao.SetStatistic("uploading_size", c)
 }
 
 func getTempDir() (string, error) {
@@ -250,7 +251,7 @@ func FinishUploadingFile(uid uint, fid uint, md5Str string) (*model.FileView, er
 		return nil, model.NewInternalServerError("failed to finish uploading file. please re-upload")
 	}
 
-	dbFile, err := dao.CreateFile(uploadingFile.Filename, uploadingFile.Description, uploadingFile.TargetResourceID, &uploadingFile.TargetStorageID, "", "", uploadingFile.TotalSize, uid)
+	dbFile, err := dao.CreateFile(uploadingFile.Filename, uploadingFile.Description, uploadingFile.TargetResourceID, &uploadingFile.TargetStorageID, storageKeyUnavailable, "", uploadingFile.TotalSize, uid)
 	if err != nil {
 		log.Error("failed to create file in db: ", err)
 		_ = os.Remove(resultFilePath)
@@ -430,6 +431,9 @@ func DownloadFile(ip, fid, cfToken string) (string, string, error) {
 		log.Error("failed to get file: ", err)
 		return "", "", model.NewNotFoundError("file not found")
 	}
+	if file.StorageKey == storageKeyUnavailable {
+		return "", "", model.NewRequestError("file is not available, please try again later")
+	}
 
 	if file.StorageID == nil {
 		if file.RedirectUrl != "" {
@@ -455,3 +459,164 @@ func DownloadFile(ip, fid, cfToken string) (string, string, error) {
 
 	return path, file.Filename, err
 }
+
+func testFileUrl(url string) (int64, error) {
+	client := http.Client{
+		Timeout: 10 * time.Second,
+	}
+	req, err := http.NewRequest("HEAD", url, nil)
+	if err != nil {
+		return 0, model.NewRequestError("failed to create HTTP request")
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return 0, model.NewRequestError("failed to send HTTP request")
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return 0, model.NewRequestError("URL is not accessible, status code: " + resp.Status)
+	}
+	if resp.Header.Get("Content-Length") == "" {
+		return 0, model.NewRequestError("URL does not provide content length")
+	}
+	contentLength, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
+	if err != nil {
+		return 0, model.NewRequestError("failed to parse Content-Length header")
+	}
+	if contentLength <= 0 {
+		return 0, model.NewRequestError("Content-Length is not valid")
+	}
+	return contentLength, nil
+}
+
+func CreateServerDownloadTask(uid uint, url, filename, description string, resourceID, storageID uint) (*model.FileView, error) {
+	canUpload, err := checkUserCanUpload(uid)
+	if err != nil {
+		log.Error("failed to check user permission: ", err)
+		return nil, model.NewInternalServerError("failed to check user permission")
+	}
+	if !canUpload {
+		return nil, model.NewUnAuthorizedError("user cannot upload file")
+	}
+
+	contentLength, err := testFileUrl(url)
+	if err != nil {
+		log.Error("failed to test file URL: ", err)
+		return nil, model.NewRequestError("failed to test file URL: " + err.Error())
+	}
+
+	file, err := dao.CreateFile(filename, description, resourceID, &storageID, storageKeyUnavailable, "", 0, uid)
+	if err != nil {
+		log.Error("failed to create file in db: ", err)
+		return nil, model.NewInternalServerError("failed to create file in db")
+	}
+
+	updateUploadingSize(contentLength)
+
+	go func() {
+		defer func() {
+			updateUploadingSize(-contentLength)
+		}()
+		client := http.Client{
+			Timeout: 10 * time.Second,
+		}
+		req, err := http.NewRequest("GET", url, nil)
+		if err != nil {
+			log.Error("failed to create HTTP request: ", err)
+			_ = dao.DeleteFile(file.UUID)
+			return
+		}
+		resp, err := client.Do(req)
+		if err != nil {
+			log.Error("failed to send HTTP request: ", err)
+			_ = dao.DeleteFile(file.UUID)
+			return
+		}
+		if err != nil {
+			log.Error("failed to parse Content-Length header: ", err)
+			_ = dao.DeleteFile(file.UUID)
+			return
+		}
+		tempPath := filepath.Join(utils.GetStoragePath(), uuid.NewString())
+		tempFile, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY, os.ModePerm)
+		if err != nil {
+			log.Error("failed to open temp file: ", err)
+			_ = dao.DeleteFile(file.UUID)
+			return
+		}
+		defer func() {
+			_ = tempFile.Close()
+			if err := os.Remove(tempPath); err != nil {
+				log.Error("failed to remove temp file: ", err)
+			}
+		}()
+		if _, err := io.Copy(tempFile, resp.Body); err != nil {
+			log.Error("failed to copy and hash response body: ", err)
+			_ = dao.DeleteFile(file.UUID)
+			return
+		}
+		_ = resp.Body.Close()
+		if err := tempFile.Close(); err != nil {
+			log.Error("failed to close temp file: ", err)
+			_ = dao.DeleteFile(file.UUID)
+			return
+		}
+		stat, err := os.Stat(tempPath)
+		if err != nil {
+			log.Error("failed to get temp file info: ", err)
+			_ = dao.DeleteFile(file.UUID)
+			_ = os.Remove(tempPath)
+			return
+		}
+		size := stat.Size()
+		if size == 0 {
+			log.Error("downloaded file is empty")
+			_ = dao.DeleteFile(file.UUID)
+			_ = os.Remove(tempPath)
+			return
+		}
+		if size != contentLength {
+			log.Error("downloaded file size does not match expected size: ", size, " != ", contentLength)
+			_ = dao.DeleteFile(file.UUID)
+			_ = os.Remove(tempPath)
+			return
+		}
+		s, err := dao.GetStorage(storageID)
+		if err != nil {
+			log.Error("failed to get storage: ", err)
+			_ = dao.DeleteFile(file.UUID)
+			_ = os.Remove(tempPath)
+			return
+		}
+		iStorage := storage.NewStorage(s)
+		if iStorage == nil {
+			log.Error("failed to find storage: ", err)
+			_ = dao.DeleteFile(file.UUID)
+			_ = os.Remove(tempPath)
+			return
+		}
+		storageKey, err := iStorage.Upload(tempPath, filename)
+		if err != nil {
+			log.Error("failed to upload file to storage: ", err)
+			_ = dao.DeleteFile(file.UUID)
+			_ = os.Remove(tempPath)
+			return
+		}
+		if err := dao.SetFileStorageKeyAndSize(file.UUID, storageKey, size); err != nil {
+			log.Error("failed to set file storage key: ", err)
+			_ = dao.DeleteFile(file.UUID)
+			_ = iStorage.Delete(storageKey)
+			_ = os.Remove(tempPath)
+			return
+		}
+		if err := dao.AddStorageUsage(storageID, size); err != nil {
+			log.Error("failed to add storage usage: ", err)
+			_ = dao.DeleteFile(file.UUID)
+			_ = iStorage.Delete(storageKey)
+			_ = os.Remove(tempPath)
+			return
+		}
+	}()
+
+	return file.ToView(), nil
+}