nyne/nysoure
1
0
Fork 0
mirror of https://github.com/wgh136/nysoure.git synced 2025-09-27 12:17:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

add SHA-1 hash calculation for file uploads and update related API and model structures

Browse Source
This commit is contained in:
nyne
2025-05-16 15:16:01 +08:00
parent a827b67c41
commit 5431328515
8 changed files with 174 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
frontend/package-lock.json generated
View File

@@ -8,6 +8,7 @@
"name": "frontend",
"version": "0.0.0",
"dependencies": {
"@aws-crypto/sha1-browser": "^5.2.0",
"@marsidev/react-turnstile": "^1.1.0",
"@tailwindcss/vite": "^4.1.5",
"axios": "^1.9.0",
@@ -51,6 +52,60 @@
"node": ">=6.0.0"
}
},
"node_modules/@aws-crypto/sha1-browser": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/@aws-crypto/sha1-browser/-/sha1-browser-5.2.0.tgz",
"integrity": "sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==",
"dependencies": {
"@aws-crypto/supports-web-crypto": "^5.2.0",
"@aws-crypto/util": "^5.2.0",
"@aws-sdk/types": "^3.222.0",
"@aws-sdk/util-locate-window": "^3.0.0",
"@smithy/util-utf8": "^2.0.0",
"tslib": "^2.6.2"
}
},
"node_modules/@aws-crypto/supports-web-crypto": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz",
"integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==",
"dependencies": {
"tslib": "^2.6.2"
}
},
"node_modules/@aws-crypto/util": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz",
"integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==",
"dependencies": {
"@aws-sdk/types": "^3.222.0",
"@smithy/util-utf8": "^2.0.0",
"tslib": "^2.6.2"
}
},
"node_modules/@aws-sdk/types": {
"version": "3.804.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.804.0.tgz",
"integrity": "sha512-A9qnsy9zQ8G89vrPPlNG9d1d8QcKRGqJKqwyGgS0dclJpwy6d1EWgQLIolKPl6vcFpLoe6avLOLxr+h8ur5wpg==",
"dependencies": {
"@smithy/types": "^4.2.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@aws-sdk/util-locate-window": {
"version": "3.804.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.804.0.tgz",
"integrity": "sha512-zVoRfpmBVPodYlnMjgVjfGoEZagyRF5IPn3Uo6ZvOZp24chnW/FRstH7ESDHDDRga4z3V+ElUQHKpFDXWyBW5A==",
"dependencies": {
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@babel/code-frame": {
"version": "7.27.1",
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz",
@@ -1447,6 +1502,52 @@
"win32"
]
},
"node_modules/@smithy/is-array-buffer": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz",
"integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==",
"dependencies": {
"tslib": "^2.6.2"
},
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/@smithy/types": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.2.0.tgz",
"integrity": "sha512-7eMk09zQKCO+E/ivsjQv+fDlOupcFUCSC/L2YUPgwhvowVGWbPQHjEFcmjt7QQ4ra5lyowS92SV53Zc6XD4+fg==",
"dependencies": {
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/util-buffer-from": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz",
"integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==",
"dependencies": {
"@smithy/is-array-buffer": "^2.2.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/@smithy/util-utf8": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz",
"integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==",
"dependencies": {
"@smithy/util-buffer-from": "^2.2.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/@tailwindcss/node": {
"version": "4.1.5",
"resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.1.5.tgz",
@@ -5906,8 +6007,7 @@
"version": "2.8.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
"license": "0BSD",
"optional": true
"license": "0BSD"
},
"node_modules/turbo-stream": {
"version": "2.4.0",

1
frontend/package.json
View File

@@ -10,6 +10,7 @@
"preview": "vite preview"
},
"dependencies": {
"@aws-crypto/sha1-browser": "^5.2.0",
"@marsidev/react-turnstile": "^1.1.0",
"@tailwindcss/vite": "^4.1.5",
"axios": "^1.9.0",

13
frontend/src/network/network.ts
View File

@@ -449,11 +449,11 @@ class Network {
}
async createS3Storage(
name: string,
endPoint: string,
name: string,
endPoint: string,
accessKeyID: string,
secretAccessKey: string,
bucketName: string,
secretAccessKey: string,
bucketName: string,
maxSizeInMB: number,
domain: string): Promise<Response<any>> {
try {
@@ -520,14 +520,15 @@ class Network {
}
async initFileUpload(filename: string, description: string, fileSize: number,
resourceId: number, storageId: number): Promise<Response<UploadingFile>> {
resourceId: number, storageId: number, sha1: string): Promise<Response<UploadingFile>> {
try {
const response = await axios.post(`${this.apiBaseUrl}/files/upload/init`, {
filename,
description,
file_size: fileSize,
resource_id: resourceId,
storage_id: storageId
storage_id: storageId,
sha1
});
return response.data;
} catch (e: any) {

34
frontend/src/network/uploading.ts
View File

@@ -1,5 +1,6 @@
import {Response} from "./models.ts";
import {network} from "./network.ts";
import { Sha1 } from "@aws-crypto/sha1-browser";
import { Response } from "./models.ts";
import { network } from "./network.ts";
enum UploadingStatus {
PENDING = "pending",
@@ -130,7 +131,7 @@ export class UploadingTask extends Listenable {
class UploadingManager extends Listenable {
tasks: UploadingTask[] = [];
onTaskStatusChanged = () => {
onTaskStatusChanged = () => {
if (this.tasks.length === 0) {
return;
}
@@ -149,12 +150,35 @@ class UploadingManager extends Listenable {
}
async addTask(file: File, resourceID: number, storageID: number, description: string, onFinished: () => void): Promise<Response<void>> {
// Calculate SHA-1 hash of the file
async function calculateSHA1(file: File): Promise<string> {
const hash = new Sha1();
const chunkSize = 4 * 1024 * 1024;
const totalChunks = Math.ceil(file.size / chunkSize);
for (let i = 0; i < totalChunks; i++) {
const start = i * chunkSize;
const end = Math.min(start + chunkSize, file.size);
const chunk = file.slice(start, end);
const arrayBuffer = await chunk.arrayBuffer();
hash.update(arrayBuffer);
}
const hashBuffer = await hash.digest();
const hashArray = new Uint8Array(hashBuffer);
const hashHex = Array.from(hashArray)
.map(byte => byte.toString(16).padStart(2, "0"))
.join("");
return hashHex;
}
const sha1 = await calculateSHA1(file);
const res = await network.initFileUpload(
file.name,
description,
file.size,
resourceID,
storageID
storageID,
sha1,
)
if (!res.success) {
return {
@@ -166,7 +190,7 @@ class UploadingManager extends Listenable {
task.addListener(this.onTaskStatusChanged);
this.tasks.push(task);
this.onTaskStatusChanged();
return {
return {
success: true,
message: "ok",
}

3
server/api/file.go
View File

@@ -34,6 +34,7 @@ func initUpload(c fiber.Ctx) error {
FileSize int64 `json:"file_size"`
ResourceID uint `json:"resource_id"`
StorageID uint `json:"storage_id"`
Sha1 string `json:"sha1"`
}
var req InitUploadRequest
@@ -41,7 +42,7 @@ func initUpload(c fiber.Ctx) error {
return model.NewRequestError("Invalid request parameters")
}
result, err := service.CreateUploadingFile(uid, req.Filename, req.Description, req.FileSize, req.ResourceID, req.StorageID)
result, err := service.CreateUploadingFile(uid, req.Filename, req.Description, req.FileSize, req.ResourceID, req.StorageID, req.Sha1)
if err != nil {
return err
}

8
server/dao/file.go
View File

@@ -2,14 +2,15 @@ package dao
import (
"errors"
"nysoure/server/model"
"time"
"github.com/google/uuid"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"nysoure/server/model"
"time"
)
func CreateUploadingFile(filename string, description string, fileSize int64, blockSize int64, tempPath string, resourceID, storageID, userID uint) (*model.UploadingFile, error) {
func CreateUploadingFile(filename string, description string, fileSize int64, blockSize int64, tempPath string, resourceID, storageID, userID uint, sha1 string) (*model.UploadingFile, error) {
blocksCount := (fileSize + blockSize - 1) / blockSize
uf := &model.UploadingFile{
Filename: filename,
@@ -21,6 +22,7 @@ func CreateUploadingFile(filename string, description string, fileSize int64, bl
TargetResourceID: resourceID,
TargetStorageID: storageID,
UserID: userID,
Sha1: sha1,
}
if err := db.Create(uf).Error; err != nil {
return nil, err

6
server/model/uploading_file.go
View File

@@ -2,9 +2,10 @@ package model
import (
"context"
"reflect"
"gorm.io/gorm"
"gorm.io/gorm/schema"
"reflect"
)
type UploadingFile struct {
@@ -20,6 +21,7 @@ type UploadingFile struct {
TempPath string
Resource Resource `gorm:"foreignKey:TargetResourceID"`
Storage Storage `gorm:"foreignKey:TargetStorageID"`
Sha1 string
}
func (uf *UploadingFile) BlocksCount() int {
@@ -84,6 +86,7 @@ type UploadingFileView struct {
BlocksCount int `json:"blocksCount"`
StorageID uint `json:"storageId"`
ResourceID uint `json:"resourceId"`
Sha1 string `json:"sha1"`
}
func (uf *UploadingFile) ToView() *UploadingFileView {
@@ -96,5 +99,6 @@ func (uf *UploadingFile) ToView() *UploadingFileView {
BlocksCount: uf.BlocksCount(),
StorageID: uf.TargetStorageID,
ResourceID: uf.TargetResourceID,
Sha1: uf.Sha1,
}
}

25
server/service/file.go
View File

@@ -1,6 +1,8 @@
package service
import (
"crypto/sha1"
"encoding/hex"
"nysoure/server/config"
"nysoure/server/dao"
"nysoure/server/model"
@@ -82,10 +84,13 @@ func init() {
}()
}
func CreateUploadingFile(uid uint, filename string, description string, fileSize int64, resourceID, storageID uint) (*model.UploadingFileView, error) {
func CreateUploadingFile(uid uint, filename string, description string, fileSize int64, resourceID, storageID uint, sha1Str string) (*model.UploadingFileView, error) {
if filename == "" {
return nil, model.NewRequestError("filename is empty")
}
if sha1Str == "" {
return nil, model.NewRequestError("sha1 is empty")
}
if len([]rune(filename)) > 128 {
return nil, model.NewRequestError("filename is too long")
}
@@ -113,7 +118,7 @@ func CreateUploadingFile(uid uint, filename string, description string, fileSize
log.Error("failed to create temp dir: ", err)
return nil, model.NewInternalServerError("failed to create temp dir")
}
uploadingFile, err := dao.CreateUploadingFile(filename, description, fileSize, blockSize, tempPath, resourceID, storageID, uid)
uploadingFile, err := dao.CreateUploadingFile(filename, description, fileSize, blockSize, tempPath, resourceID, storageID, uid, sha1Str)
if err != nil {
log.Error("failed to create uploading file: ", err)
_ = os.Remove(tempPath)
@@ -197,6 +202,8 @@ func FinishUploadingFile(uid uint, fid uint) (*model.FileView, error) {
return nil, model.NewInternalServerError("failed to finish uploading file. please re-upload")
}
h := sha1.New()
for i := 0; i < uploadingFile.BlocksCount(); i++ {
blockPath := filepath.Join(uploadingFile.TempPath, strconv.Itoa(i))
data, err := os.ReadFile(blockPath)
@@ -206,6 +213,13 @@ func FinishUploadingFile(uid uint, fid uint) (*model.FileView, error) {
_ = os.Remove(resultFilePath)
return nil, model.NewInternalServerError("failed to finish uploading file. please re-upload")
}
_, err = h.Write(data)
if err != nil {
log.Error("failed to write block data to sha1: ", err)
_ = file.Close()
_ = os.Remove(resultFilePath)
return nil, model.NewInternalServerError("failed to finish uploading file. please re-upload")
}
if _, err := file.Write(data); err != nil {
log.Error("failed to write result file: ", err)
_ = file.Close()
@@ -218,6 +232,13 @@ func FinishUploadingFile(uid uint, fid uint) (*model.FileView, error) {
_ = os.RemoveAll(uploadingFile.TempPath)
tempRemoved = true
sum := h.Sum(nil)
sumStr := hex.EncodeToString(sum)
if sumStr != uploadingFile.Sha1 {
_ = os.Remove(resultFilePath)
return nil, model.NewRequestError("sha1 checksum is not correct")
}
s, err := dao.GetStorage(uploadingFile.TargetStorageID)
if err != nil {
log.Error("failed to get storage: ", err)