From f0079003f29d92143667c92be61260a4f13eb757 Mon Sep 17 00:00:00 2001 From: nyne Date: Fri, 5 Sep 2025 14:06:51 +0800 Subject: [PATCH] add hash field to File model and update file creation functions to support hash --- server/dao/file.go | 6 ++++-- server/model/file.go | 4 ++++ server/service/file.go | 41 ++++++++++++++++++++++++----------------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/server/dao/file.go b/server/dao/file.go index c117c89..36497aa 100644 --- a/server/dao/file.go +++ b/server/dao/file.go @@ -73,7 +73,7 @@ func GetUploadingFilesOlderThan(time time.Time) ([]model.UploadingFile, error) { return files, nil } -func CreateFile(filename string, description string, resourceID uint, storageID *uint, storageKey string, redirectUrl string, size int64, userID uint) (*model.File, error) { +func CreateFile(filename string, description string, resourceID uint, storageID *uint, storageKey string, redirectUrl string, size int64, userID uint, hash string) (*model.File, error) { if storageID == nil && redirectUrl == "" { return nil, errors.New("storageID and redirectUrl cannot be both empty") } @@ -88,6 +88,7 @@ func CreateFile(filename string, description string, resourceID uint, storageID StorageKey: storageKey, Size: size, UserID: userID, + Hash: hash, } err := db.Transaction(func(tx *gorm.DB) error { @@ -200,13 +201,14 @@ func SetFileStorageKey(id string, storageKey string) error { return nil } -func SetFileStorageKeyAndSize(id string, storageKey string, size int64) error { +func SetFileStorageKeyAndSize(id string, storageKey string, size int64, hash string) error { f := &model.File{} if err := db.Where("uuid = ?", id).First(f).Error; err != nil { return err } f.StorageKey = storageKey f.Size = size + f.Hash = hash if err := db.Save(f).Error; err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return model.NewNotFoundError("file not found") diff --git a/server/model/file.go b/server/model/file.go index 9dc0ab4..95ab3a6 100644 --- a/server/model/file.go +++ b/server/model/file.go @@ -18,6 +18,7 @@ type File struct { UserID uint User User `gorm:"foreignKey:UserID"` Size int64 + Hash string `gorm:"default:null"` } type FileView struct { @@ -28,6 +29,7 @@ type FileView struct { IsRedirect bool `json:"is_redirect"` User UserView `json:"user"` Resource *ResourceView `json:"resource,omitempty"` + Hash string `json:"hash,omitempty"` } func (f *File) ToView() *FileView { @@ -38,6 +40,7 @@ func (f *File) ToView() *FileView { Size: f.Size, IsRedirect: f.RedirectUrl != "", User: f.User.ToView(), + Hash: f.Hash, } } @@ -56,5 +59,6 @@ func (f *File) ToViewWithResource() *FileView { IsRedirect: f.RedirectUrl != "", User: f.User.ToView(), Resource: resource, + Hash: f.Hash, } } diff --git a/server/service/file.go b/server/service/file.go index be0a354..fa07e2b 100644 --- a/server/service/file.go +++ b/server/service/file.go @@ -236,7 +236,7 @@ func FinishUploadingFile(uid uint, fid uint, md5Str string) (*model.FileView, er return nil, model.NewInternalServerError("failed to finish uploading file. please re-upload") } - dbFile, err := dao.CreateFile(uploadingFile.Filename, uploadingFile.Description, uploadingFile.TargetResourceID, &uploadingFile.TargetStorageID, storageKeyUnavailable, "", uploadingFile.TotalSize, uid) + dbFile, err := dao.CreateFile(uploadingFile.Filename, uploadingFile.Description, uploadingFile.TargetResourceID, &uploadingFile.TargetStorageID, storageKeyUnavailable, "", uploadingFile.TotalSize, uid, sumStr) if err != nil { log.Error("failed to create file in db: ", err) _ = os.Remove(resultFilePath) @@ -310,7 +310,7 @@ func CreateRedirectFile(uid uint, filename string, description string, resourceI return nil, model.NewUnAuthorizedError("user cannot upload file") } - file, err := dao.CreateFile(filename, description, resourceID, nil, "", redirectUrl, 0, uid) + file, err := dao.CreateFile(filename, description, resourceID, nil, "", redirectUrl, 0, uid, "") if err != nil { log.Error("failed to create file in db: ", err) return nil, model.NewInternalServerError("failed to create file in db") @@ -496,57 +496,61 @@ func testFileUrl(url string) (int64, error) { } // downloadFile return nil if the download is successful or the context is cancelled -func downloadFile(ctx context.Context, url string, path string) error { +func downloadFile(ctx context.Context, url string, path string) (string, error) { if _, err := os.Stat(path); err == nil { _ = os.Remove(path) // Remove the file if it already exists } req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { - return model.NewRequestError("failed to create HTTP request") + return "", model.NewRequestError("failed to create HTTP request") } client := http.Client{} resp, err := client.Do(req) if err != nil { // Check if the error is due to context cancellation if ctx.Err() != nil { - return nil + return "", nil } - return model.NewRequestError("failed to send HTTP request") + return "", model.NewRequestError("failed to send HTTP request") } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return model.NewRequestError("URL is not accessible, status code: " + resp.Status) + return "", model.NewRequestError("URL is not accessible, status code: " + resp.Status) } file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, os.ModePerm) if err != nil { - return model.NewInternalServerError("failed to open file for writing") + return "", model.NewInternalServerError("failed to open file for writing") } defer file.Close() writer := bufio.NewWriter(file) + h := md5.New() + buf := make([]byte, 64*1024) for { select { case <-ctx.Done(): - return nil + return "", nil default: n, readErr := resp.Body.Read(buf) if n > 0 { if _, writeErr := writer.Write(buf[:n]); writeErr != nil { - return model.NewInternalServerError("failed to write to file") + return "", model.NewInternalServerError("failed to write to file") } + h.Write(buf[:n]) } if readErr != nil { if readErr == io.EOF { if err := writer.Flush(); err != nil { - return model.NewInternalServerError("failed to flush writer") + return "", model.NewInternalServerError("failed to flush writer") } - return nil // Download completed successfully + md5Sum := hex.EncodeToString(h.Sum(nil)) + return md5Sum, nil // Download completed successfully } if ctx.Err() != nil { - return nil // Context cancelled, return nil + return "", nil // Context cancelled, return nil } - return model.NewInternalServerError("failed to read response body") + return "", model.NewInternalServerError("failed to read response body") } } } @@ -573,7 +577,7 @@ func CreateServerDownloadTask(uid uint, url, filename, description string, resou return nil, model.NewRequestError("server is busy, please try again later") } - file, err := dao.CreateFile(filename, description, resourceID, &storageID, storageKeyUnavailable, "", 0, uid) + file, err := dao.CreateFile(filename, description, resourceID, &storageID, storageKeyUnavailable, "", 0, uid, "") if err != nil { log.Error("failed to create file in db: ", err) return nil, model.NewInternalServerError("failed to create file in db") @@ -624,11 +628,14 @@ func CreateServerDownloadTask(uid uint, url, filename, description string, resou } }() + hash := "" + for i := range 3 { if done.Load() { return } - if err := downloadFile(ctx, url, tempPath); err != nil { + hash, err = downloadFile(ctx, url, tempPath) + if err != nil { log.Error("failed to download file: ", err) if i == 2 { _ = dao.DeleteFile(file.UUID) @@ -689,7 +696,7 @@ func CreateServerDownloadTask(uid uint, url, filename, description string, resou _ = os.Remove(tempPath) return } - if err := dao.SetFileStorageKeyAndSize(file.UUID, storageKey, size); err != nil { + if err := dao.SetFileStorageKeyAndSize(file.UUID, storageKey, size, hash); err != nil { log.Error("failed to set file storage key: ", err) _ = dao.DeleteFile(file.UUID) _ = iStorage.Delete(storageKey)