From 543132851575f25e7a00ad0afb36fe49983f28a6 Mon Sep 17 00:00:00 2001 From: nyne Date: Fri, 16 May 2025 15:16:01 +0800 Subject: [PATCH] add SHA-1 hash calculation for file uploads and update related API and model structures --- frontend/package-lock.json | 104 +++++++++++++++++++++++++++++- frontend/package.json | 1 + frontend/src/network/network.ts | 13 ++-- frontend/src/network/uploading.ts | 34 ++++++++-- server/api/file.go | 3 +- server/dao/file.go | 8 ++- server/model/uploading_file.go | 6 +- server/service/file.go | 25 ++++++- 8 files changed, 174 insertions(+), 20 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 2dca2d6..6036c0a 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -8,6 +8,7 @@ "name": "frontend", "version": "0.0.0", "dependencies": { + "@aws-crypto/sha1-browser": "^5.2.0", "@marsidev/react-turnstile": "^1.1.0", "@tailwindcss/vite": "^4.1.5", "axios": "^1.9.0", @@ -51,6 +52,60 @@ "node": ">=6.0.0" } }, + "node_modules/@aws-crypto/sha1-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha1-browser/-/sha1-browser-5.2.0.tgz", + "integrity": "sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==", + "dependencies": { + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.804.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.804.0.tgz", + "integrity": "sha512-A9qnsy9zQ8G89vrPPlNG9d1d8QcKRGqJKqwyGgS0dclJpwy6d1EWgQLIolKPl6vcFpLoe6avLOLxr+h8ur5wpg==", + "dependencies": { + "@smithy/types": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/util-locate-window": { + "version": "3.804.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.804.0.tgz", + "integrity": "sha512-zVoRfpmBVPodYlnMjgVjfGoEZagyRF5IPn3Uo6ZvOZp24chnW/FRstH7ESDHDDRga4z3V+ElUQHKpFDXWyBW5A==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", @@ -1447,6 +1502,52 @@ "win32" ] }, + "node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.2.0.tgz", + "integrity": "sha512-7eMk09zQKCO+E/ivsjQv+fDlOupcFUCSC/L2YUPgwhvowVGWbPQHjEFcmjt7QQ4ra5lyowS92SV53Zc6XD4+fg==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@tailwindcss/node": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.1.5.tgz", @@ -5906,8 +6007,7 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD", - "optional": true + "license": "0BSD" }, "node_modules/turbo-stream": { "version": "2.4.0", diff --git a/frontend/package.json b/frontend/package.json index 263362f..8a8d329 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,6 +10,7 @@ "preview": "vite preview" }, "dependencies": { + "@aws-crypto/sha1-browser": "^5.2.0", "@marsidev/react-turnstile": "^1.1.0", "@tailwindcss/vite": "^4.1.5", "axios": "^1.9.0", diff --git a/frontend/src/network/network.ts b/frontend/src/network/network.ts index 42be073..ddbb8f0 100644 --- a/frontend/src/network/network.ts +++ b/frontend/src/network/network.ts @@ -449,11 +449,11 @@ class Network { } async createS3Storage( - name: string, - endPoint: string, + name: string, + endPoint: string, accessKeyID: string, - secretAccessKey: string, - bucketName: string, + secretAccessKey: string, + bucketName: string, maxSizeInMB: number, domain: string): Promise> { try { @@ -520,14 +520,15 @@ class Network { } async initFileUpload(filename: string, description: string, fileSize: number, - resourceId: number, storageId: number): Promise> { + resourceId: number, storageId: number, sha1: string): Promise> { try { const response = await axios.post(`${this.apiBaseUrl}/files/upload/init`, { filename, description, file_size: fileSize, resource_id: resourceId, - storage_id: storageId + storage_id: storageId, + sha1 }); return response.data; } catch (e: any) { diff --git a/frontend/src/network/uploading.ts b/frontend/src/network/uploading.ts index d17efa9..70de55b 100644 --- a/frontend/src/network/uploading.ts +++ b/frontend/src/network/uploading.ts @@ -1,5 +1,6 @@ -import {Response} from "./models.ts"; -import {network} from "./network.ts"; +import { Sha1 } from "@aws-crypto/sha1-browser"; +import { Response } from "./models.ts"; +import { network } from "./network.ts"; enum UploadingStatus { PENDING = "pending", @@ -130,7 +131,7 @@ export class UploadingTask extends Listenable { class UploadingManager extends Listenable { tasks: UploadingTask[] = []; - onTaskStatusChanged = () => { + onTaskStatusChanged = () => { if (this.tasks.length === 0) { return; } @@ -149,12 +150,35 @@ class UploadingManager extends Listenable { } async addTask(file: File, resourceID: number, storageID: number, description: string, onFinished: () => void): Promise> { + // Calculate SHA-1 hash of the file + async function calculateSHA1(file: File): Promise { + const hash = new Sha1(); + const chunkSize = 4 * 1024 * 1024; + const totalChunks = Math.ceil(file.size / chunkSize); + for (let i = 0; i < totalChunks; i++) { + const start = i * chunkSize; + const end = Math.min(start + chunkSize, file.size); + const chunk = file.slice(start, end); + const arrayBuffer = await chunk.arrayBuffer(); + hash.update(arrayBuffer); + } + const hashBuffer = await hash.digest(); + const hashArray = new Uint8Array(hashBuffer); + const hashHex = Array.from(hashArray) + .map(byte => byte.toString(16).padStart(2, "0")) + .join(""); + return hashHex; + } + + const sha1 = await calculateSHA1(file); + const res = await network.initFileUpload( file.name, description, file.size, resourceID, - storageID + storageID, + sha1, ) if (!res.success) { return { @@ -166,7 +190,7 @@ class UploadingManager extends Listenable { task.addListener(this.onTaskStatusChanged); this.tasks.push(task); this.onTaskStatusChanged(); - return { + return { success: true, message: "ok", } diff --git a/server/api/file.go b/server/api/file.go index fd541eb..c50e26d 100644 --- a/server/api/file.go +++ b/server/api/file.go @@ -34,6 +34,7 @@ func initUpload(c fiber.Ctx) error { FileSize int64 `json:"file_size"` ResourceID uint `json:"resource_id"` StorageID uint `json:"storage_id"` + Sha1 string `json:"sha1"` } var req InitUploadRequest @@ -41,7 +42,7 @@ func initUpload(c fiber.Ctx) error { return model.NewRequestError("Invalid request parameters") } - result, err := service.CreateUploadingFile(uid, req.Filename, req.Description, req.FileSize, req.ResourceID, req.StorageID) + result, err := service.CreateUploadingFile(uid, req.Filename, req.Description, req.FileSize, req.ResourceID, req.StorageID, req.Sha1) if err != nil { return err } diff --git a/server/dao/file.go b/server/dao/file.go index 646c278..2a5473f 100644 --- a/server/dao/file.go +++ b/server/dao/file.go @@ -2,14 +2,15 @@ package dao import ( "errors" + "nysoure/server/model" + "time" + "github.com/google/uuid" "gorm.io/gorm" "gorm.io/gorm/clause" - "nysoure/server/model" - "time" ) -func CreateUploadingFile(filename string, description string, fileSize int64, blockSize int64, tempPath string, resourceID, storageID, userID uint) (*model.UploadingFile, error) { +func CreateUploadingFile(filename string, description string, fileSize int64, blockSize int64, tempPath string, resourceID, storageID, userID uint, sha1 string) (*model.UploadingFile, error) { blocksCount := (fileSize + blockSize - 1) / blockSize uf := &model.UploadingFile{ Filename: filename, @@ -21,6 +22,7 @@ func CreateUploadingFile(filename string, description string, fileSize int64, bl TargetResourceID: resourceID, TargetStorageID: storageID, UserID: userID, + Sha1: sha1, } if err := db.Create(uf).Error; err != nil { return nil, err diff --git a/server/model/uploading_file.go b/server/model/uploading_file.go index bc23f8d..3259ddc 100644 --- a/server/model/uploading_file.go +++ b/server/model/uploading_file.go @@ -2,9 +2,10 @@ package model import ( "context" + "reflect" + "gorm.io/gorm" "gorm.io/gorm/schema" - "reflect" ) type UploadingFile struct { @@ -20,6 +21,7 @@ type UploadingFile struct { TempPath string Resource Resource `gorm:"foreignKey:TargetResourceID"` Storage Storage `gorm:"foreignKey:TargetStorageID"` + Sha1 string } func (uf *UploadingFile) BlocksCount() int { @@ -84,6 +86,7 @@ type UploadingFileView struct { BlocksCount int `json:"blocksCount"` StorageID uint `json:"storageId"` ResourceID uint `json:"resourceId"` + Sha1 string `json:"sha1"` } func (uf *UploadingFile) ToView() *UploadingFileView { @@ -96,5 +99,6 @@ func (uf *UploadingFile) ToView() *UploadingFileView { BlocksCount: uf.BlocksCount(), StorageID: uf.TargetStorageID, ResourceID: uf.TargetResourceID, + Sha1: uf.Sha1, } } diff --git a/server/service/file.go b/server/service/file.go index e2425b2..d3f9895 100644 --- a/server/service/file.go +++ b/server/service/file.go @@ -1,6 +1,8 @@ package service import ( + "crypto/sha1" + "encoding/hex" "nysoure/server/config" "nysoure/server/dao" "nysoure/server/model" @@ -82,10 +84,13 @@ func init() { }() } -func CreateUploadingFile(uid uint, filename string, description string, fileSize int64, resourceID, storageID uint) (*model.UploadingFileView, error) { +func CreateUploadingFile(uid uint, filename string, description string, fileSize int64, resourceID, storageID uint, sha1Str string) (*model.UploadingFileView, error) { if filename == "" { return nil, model.NewRequestError("filename is empty") } + if sha1Str == "" { + return nil, model.NewRequestError("sha1 is empty") + } if len([]rune(filename)) > 128 { return nil, model.NewRequestError("filename is too long") } @@ -113,7 +118,7 @@ func CreateUploadingFile(uid uint, filename string, description string, fileSize log.Error("failed to create temp dir: ", err) return nil, model.NewInternalServerError("failed to create temp dir") } - uploadingFile, err := dao.CreateUploadingFile(filename, description, fileSize, blockSize, tempPath, resourceID, storageID, uid) + uploadingFile, err := dao.CreateUploadingFile(filename, description, fileSize, blockSize, tempPath, resourceID, storageID, uid, sha1Str) if err != nil { log.Error("failed to create uploading file: ", err) _ = os.Remove(tempPath) @@ -197,6 +202,8 @@ func FinishUploadingFile(uid uint, fid uint) (*model.FileView, error) { return nil, model.NewInternalServerError("failed to finish uploading file. please re-upload") } + h := sha1.New() + for i := 0; i < uploadingFile.BlocksCount(); i++ { blockPath := filepath.Join(uploadingFile.TempPath, strconv.Itoa(i)) data, err := os.ReadFile(blockPath) @@ -206,6 +213,13 @@ func FinishUploadingFile(uid uint, fid uint) (*model.FileView, error) { _ = os.Remove(resultFilePath) return nil, model.NewInternalServerError("failed to finish uploading file. please re-upload") } + _, err = h.Write(data) + if err != nil { + log.Error("failed to write block data to sha1: ", err) + _ = file.Close() + _ = os.Remove(resultFilePath) + return nil, model.NewInternalServerError("failed to finish uploading file. please re-upload") + } if _, err := file.Write(data); err != nil { log.Error("failed to write result file: ", err) _ = file.Close() @@ -218,6 +232,13 @@ func FinishUploadingFile(uid uint, fid uint) (*model.FileView, error) { _ = os.RemoveAll(uploadingFile.TempPath) tempRemoved = true + sum := h.Sum(nil) + sumStr := hex.EncodeToString(sum) + if sumStr != uploadingFile.Sha1 { + _ = os.Remove(resultFilePath) + return nil, model.NewRequestError("sha1 checksum is not correct") + } + s, err := dao.GetStorage(uploadingFile.TargetStorageID) if err != nil { log.Error("failed to get storage: ", err)