From c60b75bd3b43d8ffb5c70d6402396eeee854e9ac Mon Sep 17 00:00:00 2001 From: Akilan Date: Sun, 16 Apr 2023 11:43:46 +0100 Subject: [PATCH] added upload status --- warehouse/Store.go | 14 +++++--- webapi/API.go | 9 ++++- webapi/Upload.go | 83 +++++++++++++++++++++++++++++++++++++++++++++ webapi/Warehouse.go | 43 +++++++++++++++++++++-- 4 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 webapi/Upload.go diff --git a/warehouse/Store.go b/warehouse/Store.go index 73cd83e..e23770d 100644 --- a/warehouse/Store.go +++ b/warehouse/Store.go @@ -38,7 +38,7 @@ const ( // CreateFile creates a new file in the warehouse // If fileSize is provided, creating the merkle tree is significantly faster as it will be created on the fly. If the file size is unknown, set the size to 0. -func (wh *Warehouse) CreateFile(data io.Reader, fileSize uint64) (hash []byte, status int, err error) { +func (wh *Warehouse) CreateFile(data io.Reader, fileSize uint64, uploadStatus io.Writer) (hash []byte, status int, err error) { // create a temporary file to hold the body content tmpFile, err := wh.tempFile() if err != nil { @@ -55,8 +55,14 @@ func (wh *Warehouse) CreateFile(data io.Reader, fileSize uint64) (hash []byte, s // create the hash-writer hashWriter := blake3.New(hashSize, nil) - // the multi-writer writes to the temp-file and the hash simultaneously - mw := io.MultiWriter(tmpFile, hashWriter) + var mw io.Writer + + if uploadStatus != nil { + // the multi-writer writes to the temp-file and the hash simultaneously + mw = io.MultiWriter(tmpFile, hashWriter, uploadStatus) + } else { + mw = io.MultiWriter(tmpFile, hashWriter) + } // copy into the multiwriter if _, err = io.Copy(mw, data); err != nil { @@ -132,7 +138,7 @@ func (wh *Warehouse) CreateFileFromPath(file string) (hash []byte, status int, e } // create the file using the opened file - return wh.CreateFile(fileHandle, fileSize) + return wh.CreateFile(fileHandle, fileSize, nil) } // ReadFile reads a file from the warehouse and outputs it to the writer diff --git a/webapi/API.go b/webapi/API.go index 319dbbb..94de23b 100644 --- a/webapi/API.go +++ b/webapi/API.go @@ -36,6 +36,10 @@ type WebapiInstance struct { // download info downloads map[uuid.UUID]*downloadInfo downloadsMutex sync.RWMutex + + // upload info + uploads map[uuid.UUID]*UploadStatus + uploadsMutex sync.RWMutex } // WSUpgrader is used for websocket functionality. It allows all requests. @@ -62,6 +66,7 @@ func Start(Backend *core.Backend, ListenAddresses []string, UseSSL bool, Certifi AllowKeyInParam: []string{"/file/read", "/file/view"}, allJobs: make(map[uuid.UUID]*SearchJob), downloads: make(map[uuid.UUID]*downloadInfo), + uploads: make(map[uuid.UUID]*UploadStatus), } if APIKey != uuid.Nil { @@ -96,7 +101,9 @@ func Start(Backend *core.Backend, ListenAddresses []string, UseSSL bool, Certifi api.Router.HandleFunc("/download/start", api.apiDownloadStart).Methods("GET") api.Router.HandleFunc("/download/status", api.apiDownloadStatus).Methods("GET") api.Router.HandleFunc("/download/action", api.apiDownloadAction).Methods("GET") - api.Router.HandleFunc("/warehouse/create", api.apiWarehouseCreateFile).Methods("POST") + api.Router.HandleFunc("/warehouse/create", api.ApiWarehouseCreateFile).Methods("POST") + api.Router.HandleFunc("/warehouse/create/uploadID", api.apiUploadID).Methods("GET") + api.Router.HandleFunc("/warehouse/create/track/uploadID", api.apiUploadInfo).Methods("GET") api.Router.HandleFunc("/warehouse/create/path", api.apiWarehouseCreateFilePath).Methods("GET") api.Router.HandleFunc("/warehouse/read", api.apiWarehouseReadFile).Methods("GET") api.Router.HandleFunc("/warehouse/read/path", api.apiWarehouseReadFilePath).Methods("GET") diff --git a/webapi/Upload.go b/webapi/Upload.go new file mode 100644 index 0000000..0331534 --- /dev/null +++ b/webapi/Upload.go @@ -0,0 +1,83 @@ +package webapi + +import ( + "github.com/google/uuid" + "math" + "net/http" +) + +type UploadStatus struct { + APIStatus int `json:"apistatus"` // Status of the API call. See DownloadResponseX. + ID uuid.UUID `json:"id"` // Download ID. This can be used to query the latest status and take actions. + UploadStatus int `json:"uploadstatus"` // Status of the download. See DownloadX. + Progress struct { + TotalSize uint64 `json:"totalsize"` // Total size in bytes. + UploadedSize uint64 `json:"uploadedsize"` // Count of bytes download so far. + Percentage float64 `json:"percentage"` // Percentage downloaded. Rounded to 2 decimal points. Between 0.00 and 100.00. + } `json:"progress"` // Progress of the download. Only valid for status >= DownloadWaitSwarm. +} + +func (api *WebapiInstance) uploadAdd(info *UploadStatus) { + api.uploadsMutex.Lock() + api.uploads[info.ID] = info + api.uploadsMutex.Unlock() +} + +func (api *WebapiInstance) uploadDelete(id uuid.UUID) { + api.uploadsMutex.Lock() + delete(api.uploads, id) + api.uploadsMutex.Unlock() +} + +func (api *WebapiInstance) uploadLookup(id uuid.UUID) (info *UploadStatus) { + api.uploadsMutex.Lock() + info = api.uploads[id] + api.uploadsMutex.Unlock() + return info +} + +// UploadID API to set a Status ID to track the upload +func (api *WebapiInstance) apiUploadID(w http.ResponseWriter, r *http.Request) { + var info UploadStatus + info.ID = uuid.New() + + // Create a upload ID for adding the upload + // metadata later one + api.uploadAdd(&info) + + EncodeJSON(api.Backend, w, r, info) +} + +// Write is used to satisfy the io.Writer interface. +// Instead of writing somewhere, it simply aggregates +// the total bytes on each read +func (uploadStatus *UploadStatus) Write(p []byte) (n int, err error) { + n = len(p) + uploadStatus.Progress.UploadedSize += uint64(n) + uploadStatus.Progress.Percentage = math.Round(float64(uploadStatus.Progress.UploadedSize)/float64(uploadStatus.Progress.TotalSize)*100*100) / 100 + return +} + +// Get information about upload file status +func (api *WebapiInstance) apiUploadInfo(w http.ResponseWriter, r *http.Request) { + ID := r.URL.Query().Get("id") + if ID == "" { + api.Backend.LogError("upload.UploadInformation", "error: %v", "ID parameter not passed") + http.Error(w, "", http.StatusBadRequest) + return + } + + IDUUID, err := uuid.Parse(ID) + if err != nil { + api.Backend.LogError("upload.UploadInformation", "error: %v", err) + return + } + + info := api.uploadLookup(IDUUID) + if info == nil { + EncodeJSON(api.Backend, w, r, UploadStatus{APIStatus: DownloadResponseIDNotFound}) + return + } + + EncodeJSON(api.Backend, w, r, info) +} diff --git a/webapi/Warehouse.go b/webapi/Warehouse.go index 95c5c2b..01d18d1 100644 --- a/webapi/Warehouse.go +++ b/webapi/Warehouse.go @@ -7,6 +7,7 @@ Author: Peter Kleissner package webapi import ( + "github.com/google/uuid" "net/http" "strconv" @@ -20,16 +21,52 @@ type WarehouseResult struct { } /* -apiWarehouseCreateFile creates a file in the warehouse. +ApiWarehouseCreateFile creates a file in the warehouse. Request: POST /warehouse/create with raw data to create as new file Response: 200 with JSON structure WarehouseResult */ -func (api *WebapiInstance) apiWarehouseCreateFile(w http.ResponseWriter, r *http.Request) { - hash, status, err := api.Backend.UserWarehouse.CreateFile(r.Body, 0) +func (api *WebapiInstance) ApiWarehouseCreateFile(w http.ResponseWriter, r *http.Request) { + // changing parameter to take ID as a parameter for upload and file itself + ID := r.FormValue("ID") + file, handler, err := r.FormFile("File") + if err != nil { + api.Backend.LogError("warehouse.CreateFile", "error: %v", err) + http.Error(w, "", http.StatusBadRequest) + return + } + + var hash []byte + var status int + + if ID != "" { + IDUUID, err := uuid.Parse(ID) + if err != nil { + api.Backend.LogError("warehouse.CreateFile", "error: %v", err) + http.Error(w, "", http.StatusBadRequest) + return + } + info := api.uploadLookup(IDUUID) + if info == nil { + var newInfo UploadStatus + newInfo.ID = IDUUID + newInfo.Progress.TotalSize = uint64(handler.Size) + api.uploadAdd(&newInfo) + hash, status, err = api.Backend.UserWarehouse.CreateFile(file, uint64(handler.Size), &newInfo) + } else { + info.Progress.TotalSize = uint64(handler.Size) + hash, status, err = api.Backend.UserWarehouse.CreateFile(file, uint64(handler.Size), info) + } + + } else { + // File := r. + hash, status, err = api.Backend.UserWarehouse.CreateFile(file, uint64(handler.Size), nil) + } if err != nil { api.Backend.LogError("warehouse.CreateFile", "status %d error: %v", status, err) + http.Error(w, "", http.StatusBadRequest) + return } EncodeJSON(api.Backend, w, r, WarehouseResult{Status: status, Hash: hash})