Peernet-Archive/core
1
0
Fork 0
mirror of https://github.com/PeernetOfficial/core.git synced 2026-07-19 03:27:50 +01:00
Code Issues Packages Projects Releases Wiki Activity

added abstracted feature for add-file

Browse Source
This commit is contained in:
Akilan Selvacoumar
2022-09-28 02:39:00 +01:00
parent 0e3e8dc988
commit b8ac4c3dc4
9 changed files with 199 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -4,3 +4,6 @@ log.txt
test.*
# Ignore the IDE gitignore files
.idea/
# Test cases generate the following
data/
Config.yaml

107
Abstrations/files.go Normal file
View File

@@ -0,0 +1,107 @@
/*
File Name: abstractions.go
Copyright: 2021 Peernet s.r.o.
Authors: Peter Kleissner, Akilan Selvacoumar
*/
package Abstrations
import (
"encoding/hex"
"errors"
"github.com/PeernetOfficial/core"
"github.com/PeernetOfficial/core/blockchain"
"github.com/PeernetOfficial/core/protocol"
"github.com/PeernetOfficial/core/warehouse"
"github.com/PeernetOfficial/core/webapi"
"github.com/google/uuid"
"path/filepath"
"time"
)
/*
Library description
to about abstracted function to easily add and remove files.
*/
type TouchReturn struct {
BlockchainHeight uint64
BlockchainVersion uint64
}
// Touch abstracted function that creates a file
// and adds the file to the warehouse and
// blockchain
// returns blockchain version and height
func Touch(b *core.Backend, filePath string) (*TouchReturn, error) {
// Creates a File in the warehouse
hash, _, err := b.UserWarehouse.CreateFileFromPath(filePath)
if err != nil {
return nil, err
}
// Add the File to the local blockchain
var input webapi.ApiBlockAddFiles
var inputFiles []webapi.ApiFile
var inputFile webapi.ApiFile
// Write File information to the input File
inputFile.Date = time.Now()
// Folder and File name
dir, file := filepath.Split(filePath)
inputFile.Folder = dir
inputFile.Name = file
inputFile.ID = uuid.New()
inputFile.Hash = hash
// Get the public key of the current node
_, publicKey := b.ExportPrivateKey()
inputFile.NodeID = []byte(hex.EncodeToString(publicKey.SerializeCompressed()))
inputFiles = append(inputFiles, inputFile)
input.Files = inputFiles
var filesAdd []blockchain.BlockRecordFile
for _, File := range input.Files {
if len(File.Hash) != protocol.HashSize {
return nil, errors.New("bad request")
}
if File.ID == uuid.Nil { // if the ID is not provided by the caller, set it
File.ID = uuid.New()
}
// Verify that the File exists in the warehouse. Folders are exempt from this check as they are only virtual.
if !File.IsVirtualFolder() {
if _, err := warehouse.ValidateHash(File.Hash); err != nil {
return nil, errors.New("bad request when validating hash")
} else if _, fileInfo, status, _ := b.UserWarehouse.FileExists(File.Hash); status != warehouse.StatusOK {
//EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse})
return nil, errors.New("file not in warehouse")
} else {
File.Size = fileInfo
}
} else {
File.Hash = protocol.HashData(nil)
File.Size = 0
}
blockRecord := webapi.BlockRecordFileFromAPI(File)
// Set the merkle tree info as appropriate.
if !webapi.SetFileMerkleInfo(b, &blockRecord) {
return nil, errors.New("merkle information not set")
}
filesAdd = append(filesAdd, blockRecord)
}
newHeight, newVersion, _ := b.UserBlockchain.AddFiles(filesAdd)
// Creating object for custom return type
var touchReturn TouchReturn
touchReturn.BlockchainHeight = newHeight
touchReturn.BlockchainVersion = newVersion
return &touchReturn, nil
}

35
Abstrations/files_test.go Normal file
View File

@@ -0,0 +1,35 @@
package Abstrations
import (
"fmt"
"github.com/PeernetOfficial/core"
"github.com/PeernetOfficial/core/webapi"
"github.com/google/uuid"
"testing"
"time"
)
func InitPeernet() *core.Backend {
backendInit, status, err := core.Init("Your application/1.0", "Config.yaml", nil, nil)
if status != core.ExitSuccess {
fmt.Printf("Error %d initializing config: %s\n", status, err.Error())
return nil
}
// start config api server
webapi.Start(backendInit, []string{"127.0.0.1:5125"}, false, "", "", 10*time.Second, 10*time.Second, uuid.Nil)
backendInit.Connect()
return backendInit
}
// Testing the touch function if the file is added or not
func TestBackend_Touch(t *testing.T) {
backend := InitPeernet()
touch, err := Touch(backend, "Config.yaml")
if err != nil {
t.Fail()
}
fmt.Printf("blockchain height: %v", touch.BlockchainHeight)
}

2
webapi/Blockchain.go
View File

@@ -109,7 +109,7 @@ func (api *WebapiInstance) apiBlockchainRead(w http.ResponseWriter, r *http.Requ
for _, record := range block.RecordsDecoded {
switch v := record.(type) {
case blockchain.BlockRecordFile:
result.RecordsDecoded = append(result.RecordsDecoded, blockRecordFileToAPI(v))
result.RecordsDecoded = append(result.RecordsDecoded, BlockRecordFileToAPI(v))
case blockchain.BlockRecordProfile:
result.RecordsDecoded = append(result.RecordsDecoded, blockRecordProfileToAPI(v))

4
webapi/Download.go
View File

@@ -23,7 +23,7 @@ type apiResponseDownloadStatus struct {
APIStatus int `json:"apistatus"` // Status of the API call. See DownloadResponseX.
ID uuid.UUID `json:"id"` // Download ID. This can be used to query the latest status and take actions.
DownloadStatus int `json:"downloadstatus"` // Status of the download. See DownloadX.
File apiFile `json:"file"` // File information. Only available for status >= DownloadWaitSwarm.
File ApiFile `json:"file"` // File information. Only available for status >= DownloadWaitSwarm.
Progress struct {
TotalSize uint64 `json:"totalsize"` // Total size in bytes.
DownloadedSize uint64 `json:"downloadedsize"` // Count of bytes download so far.
@@ -189,7 +189,7 @@ type downloadInfo struct {
created time.Time // When the download was created.
ended time.Time // When the download was finished (only status = DownloadFinished).
file apiFile // File metadata (only status >= DownloadWaitSwarm)
file ApiFile // File metadata (only status >= DownloadWaitSwarm)
DiskFile struct { // Target file on disk to store downloaded data
Name string // File name

66
webapi/File.go
View File

@@ -18,8 +18,8 @@ import (
"github.com/google/uuid"
)
// apiFileMetadata contains metadata information.
type apiFileMetadata struct {
// ApiFileMetadata contains metadata information.
type ApiFileMetadata struct {
Type uint16 `json:"type"` // See core.TagX constants.
Name string `json:"name"` // User friendly name of the metadata type. Use the Type fields to identify the metadata as this name may change.
// Depending on the exact type, one of the below fields is used for proper encoding:
@@ -29,8 +29,8 @@ type apiFileMetadata struct {
Number uint64 `json:"number"` // Number
}
// apiFile is the metadata of a file published on the blockchain
type apiFile struct {
// ApiFile is the metadata of a file published on the blockchain
type ApiFile struct {
ID uuid.UUID `json:"id"` // Unique ID.
Hash []byte `json:"hash"` // Blake3 hash of the file data
Type uint8 `json:"type"` // File Type. For example audio or document. See TypeX.
@@ -41,13 +41,13 @@ type apiFile struct {
Description string `json:"description"` // Description. This is expected to be multiline and contain hashtags!
Date time.Time `json:"date"` // Date shared
NodeID []byte `json:"nodeid"` // Node ID, owner of the file. Read only.
Metadata []apiFileMetadata `json:"metadata"` // Additional metadata.
Metadata []ApiFileMetadata `json:"metadata"` // Additional metadata.
}
// --- conversion from core to API data ---
func blockRecordFileToAPI(input blockchain.BlockRecordFile) (output apiFile) {
output = apiFile{ID: input.ID, Hash: input.Hash, NodeID: input.NodeID, Type: input.Type, Format: input.Format, Size: input.Size, Metadata: []apiFileMetadata{}}
func BlockRecordFileToAPI(input blockchain.BlockRecordFile) (output ApiFile) {
output = ApiFile{ID: input.ID, Hash: input.Hash, NodeID: input.NodeID, Type: input.Type, Format: input.Format, Size: input.Size, Metadata: []ApiFileMetadata{}}
for _, tag := range input.Tags {
switch tag.Type {
@@ -65,23 +65,23 @@ func blockRecordFileToAPI(input blockchain.BlockRecordFile) (output apiFile) {
case blockchain.TagDateCreated:
date, _ := tag.Date()
output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Name: "Date Created", Date: date})
output.Metadata = append(output.Metadata, ApiFileMetadata{Type: tag.Type, Name: "Date Created", Date: date})
case blockchain.TagSharedByCount:
output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Name: "Shared By Count", Number: tag.Number()})
output.Metadata = append(output.Metadata, ApiFileMetadata{Type: tag.Type, Name: "Shared By Count", Number: tag.Number()})
case blockchain.TagSharedByGeoIP:
output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Name: "Shared By GeoIP", Text: tag.Text()})
output.Metadata = append(output.Metadata, ApiFileMetadata{Type: tag.Type, Name: "Shared By GeoIP", Text: tag.Text()})
default:
output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Blob: tag.Data})
output.Metadata = append(output.Metadata, ApiFileMetadata{Type: tag.Type, Blob: tag.Data})
}
}
return output
}
func blockRecordFileFromAPI(input apiFile) (output blockchain.BlockRecordFile) {
func BlockRecordFileFromAPI(input ApiFile) (output blockchain.BlockRecordFile) {
output = blockchain.BlockRecordFile{ID: input.ID, Hash: input.Hash, Type: input.Type, Format: input.Format, Size: input.Size}
if input.Name != "" {
@@ -115,9 +115,9 @@ func blockRecordFileFromAPI(input apiFile) (output blockchain.BlockRecordFile) {
// --- File API ---
// apiBlockAddFiles contains a list of files from the blockchain
type apiBlockAddFiles struct {
Files []apiFile `json:"files"` // List of files
// ApiBlockAddFiles contains a list of files from the blockchain
type ApiBlockAddFiles struct {
Files []ApiFile `json:"files"` // List of files
Status int `json:"status"` // Status of the operation, only used when this structure is returned from the API.
}
@@ -128,12 +128,12 @@ If any file is not stored in the Warehouse, the function aborts with the status
If the block record encoding fails for any file, this function aborts with the status code StatusCorruptBlockRecord.
In case the function aborts, the blockchain remains unchanged.
Request: POST /blockchain/file/add with JSON structure apiBlockAddFiles
Request: POST /blockchain/file/add with JSON structure ApiBlockAddFiles
Response: 200 with JSON structure apiBlockchainBlockStatus
400 if invalid input
*/
func (api *WebapiInstance) apiBlockchainFileAdd(w http.ResponseWriter, r *http.Request) {
var input apiBlockAddFiles
var input ApiBlockAddFiles
if err := DecodeJSON(w, r, &input); err != nil {
return
}
@@ -165,10 +165,10 @@ func (api *WebapiInstance) apiBlockchainFileAdd(w http.ResponseWriter, r *http.R
file.Size = 0
}
blockRecord := blockRecordFileFromAPI(file)
blockRecord := BlockRecordFileFromAPI(file)
// Set the merkle tree info as appropriate.
if !setFileMerkleInfo(api.backend, &blockRecord) {
if !SetFileMerkleInfo(api.backend, &blockRecord) {
EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse})
return
}
@@ -185,15 +185,15 @@ func (api *WebapiInstance) apiBlockchainFileAdd(w http.ResponseWriter, r *http.R
apiBlockchainFileList lists all files stored on the blockchain.
Request: GET /blockchain/file/list
Response: 200 with JSON structure apiBlockAddFiles
Response: 200 with JSON structure ApiBlockAddFiles
*/
func (api *WebapiInstance) apiBlockchainFileList(w http.ResponseWriter, r *http.Request) {
files, status := api.backend.UserBlockchain.ListFiles()
var result apiBlockAddFiles
var result ApiBlockAddFiles
for _, file := range files {
result.Files = append(result.Files, blockRecordFileToAPI(file))
result.Files = append(result.Files, BlockRecordFileToAPI(file))
}
result.Status = status
@@ -205,11 +205,11 @@ func (api *WebapiInstance) apiBlockchainFileList(w http.ResponseWriter, r *http.
apiBlockchainFileDelete deletes files with the provided IDs. Other fields are ignored.
It will automatically delete the file in the Warehouse if there are no other references.
Request: POST /blockchain/file/delete with JSON structure apiBlockAddFiles
Request: POST /blockchain/file/delete with JSON structure ApiBlockAddFiles
Response: 200 with JSON structure apiBlockchainBlockStatus
*/
func (api *WebapiInstance) apiBlockchainFileDelete(w http.ResponseWriter, r *http.Request) {
var input apiBlockAddFiles
var input ApiBlockAddFiles
if err := DecodeJSON(w, r, &input); err != nil {
return
}
@@ -237,12 +237,12 @@ func (api *WebapiInstance) apiBlockchainFileDelete(w http.ResponseWriter, r *htt
/*
apiBlockchainSelfUpdateFile updates files that are already published on the blockchain.
Request: POST /blockchain/file/update with JSON structure apiBlockAddFiles
Request: POST /blockchain/file/update with JSON structure ApiBlockAddFiles
Response: 200 with JSON structure apiBlockchainBlockStatus
400 if invalid input
*/
func (api *WebapiInstance) apiBlockchainFileUpdate(w http.ResponseWriter, r *http.Request) {
var input apiBlockAddFiles
var input ApiBlockAddFiles
if err := DecodeJSON(w, r, &input); err != nil {
return
}
@@ -274,10 +274,10 @@ func (api *WebapiInstance) apiBlockchainFileUpdate(w http.ResponseWriter, r *htt
file.Size = 0
}
blockRecord := blockRecordFileFromAPI(file)
blockRecord := BlockRecordFileFromAPI(file)
// Set the merkle tree info as appropriate.
if !setFileMerkleInfo(api.backend, &blockRecord) {
if !SetFileMerkleInfo(api.backend, &blockRecord) {
EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse})
return
}
@@ -293,7 +293,7 @@ func (api *WebapiInstance) apiBlockchainFileUpdate(w http.ResponseWriter, r *htt
// ---- metadata functions ----
// GetMetadata returns the specified metadata or nil if not available.
func (file *apiFile) GetMetadata(Type uint16) (info *apiFileMetadata) {
func (file *ApiFile) GetMetadata(Type uint16) (info *ApiFileMetadata) {
for n := range file.Metadata {
if file.Metadata[n].Type == Type {
return &file.Metadata[n]
@@ -304,7 +304,7 @@ func (file *apiFile) GetMetadata(Type uint16) (info *apiFileMetadata) {
}
// GetNumber returns the data as number. 0 if not available.
func (info *apiFileMetadata) GetNumber() uint64 {
func (info *ApiFileMetadata) GetNumber() uint64 {
if info == nil {
return 0
}
@@ -313,12 +313,12 @@ func (info *apiFileMetadata) GetNumber() uint64 {
}
// IsVirtualFolder returns true if the file is a virtual folder
func (file *apiFile) IsVirtualFolder() bool {
func (file *ApiFile) IsVirtualFolder() bool {
return file.Type == core.TypeFolder && file.Format == core.FormatFolder
}
// setFileMerkleInfo sets the merkle fields in the BlockRecordFile
func setFileMerkleInfo(backend *core.Backend, file *blockchain.BlockRecordFile) (valid bool) {
// SetFileMerkleInfo sets the merkle fields in the BlockRecordFile
func SetFileMerkleInfo(backend *core.Backend, file *blockchain.BlockRecordFile) (valid bool) {
if file.Size <= merkle.MinimumFragmentSize {
// If smaller or equal than the minimum fragment size, the merkle tree is not used.
file.MerkleRootHash = file.Hash

2
webapi/Search Dispatch.go
View File

@@ -68,7 +68,7 @@ resultLoop:
}
// new result
newFile := blockRecordFileToAPI(file)
newFile := BlockRecordFileToAPI(file)
job.Files = append(job.Files, &newFile)
job.AllFiles = append(job.AllFiles, &newFile)

20
webapi/Search Job.go
View File

@@ -56,14 +56,14 @@ type SearchJob struct {
clientsMutex sync.Mutex // mutex for manipulating client list
// List of files found but not yet returned via API to the caller. They are subject to sorting.
Files []*apiFile
Files []*ApiFile
requireSort bool // if Files requires sort before returning the results
// FreezeFiles is a list of files that were already finally delivered via the API. They may NOT change in sorting.
FreezeFiles []*apiFile
FreezeFiles []*ApiFile
// List of all files. Does not change based on sorting or runtime filters. This list only gets expanded.
AllFiles []*apiFile
AllFiles []*ApiFile
ResultSync sync.Mutex // ResultSync ensures unique access to the file results
@@ -101,7 +101,7 @@ func (api *WebapiInstance) CreateSearchJob(Timeout time.Duration, MaxResults int
}
// ReturnResult returns the selected results.
func (job *SearchJob) ReturnResult(Offset, Limit int) (Result []*apiFile) {
func (job *SearchJob) ReturnResult(Offset, Limit int) (Result []*ApiFile) {
if Limit == 0 {
return Result
}
@@ -159,7 +159,7 @@ func (job *SearchJob) ReturnResult(Offset, Limit int) (Result []*apiFile) {
}
// ReturnNext returns the next results. Call must be serialized.
func (job *SearchJob) ReturnNext(Limit int) (Result []*apiFile) {
func (job *SearchJob) ReturnNext(Limit int) (Result []*ApiFile) {
Result = job.ReturnResult(job.currentOffset, Limit)
job.currentOffset += len(Result)
@@ -167,7 +167,7 @@ func (job *SearchJob) ReturnNext(Limit int) (Result []*apiFile) {
}
// PeekResult returns the selected results but will not change any frozen files or impact auto offset
func (job *SearchJob) PeekResult(Offset, Limit int) (Result []*apiFile) {
func (job *SearchJob) PeekResult(Offset, Limit int) (Result []*ApiFile) {
job.ResultSync.Lock()
defer job.ResultSync.Unlock()
@@ -218,7 +218,7 @@ func (job *SearchJob) RuntimeFilter(Filter SearchFilter) {
job.currentOffset = 0
// files remain in AllFiles, but Files needs to be filtered based on the new filter
job.Files = []*apiFile{}
job.Files = []*ApiFile{}
job.requireSort = false // Sorting is done immediately below
// set Files based on AllFiles with the filter
@@ -236,7 +236,7 @@ func (job *SearchJob) RuntimeFilter(Filter SearchFilter) {
}
// isFileFiltered returns true if the file conforms to the runtime filter. If there is no runtime filter, it always returns true.
func (job *SearchJob) isFileFiltered(file *apiFile) bool {
func (job *SearchJob) isFileFiltered(file *ApiFile) bool {
if job.filtersRuntime.FileType >= 0 && file.Type != uint8(job.filtersRuntime.FileType) {
return false
}
@@ -258,7 +258,7 @@ func (job *SearchJob) isFileFiltered(file *apiFile) bool {
}
// SortFiles sorts a list of files. It returns a sorted list. 0 = no sorting, 1 = Relevance ASC, 2 = Relevance DESC, 3 = Date ASC, 4 = Date DESC, 5 = Name ASC, 6 = Name DESC
func SortFiles(files []*apiFile, Sort int) (sorted []*apiFile) {
func SortFiles(files []*ApiFile, Sort int) (sorted []*ApiFile) {
switch Sort {
case SortRelevanceAsc:
sort.SliceStable(files, func(i, j int) bool { return files[i].Date.Before(files[j].Date) }) // first as date for secondary sorting
@@ -424,7 +424,7 @@ func (job *SearchJob) Statistics() (result SearchStatisticData) {
}
// statsAdd counts the files in the statistics
func (job *SearchJob) statsAdd(files ...*apiFile) {
func (job *SearchJob) statsAdd(files ...*ApiFile) {
job.stats.Lock()
defer job.stats.Unlock()

14
webapi/Search.go
View File

@@ -62,7 +62,7 @@ type SearchRequestResponse struct {
// SearchResult contains the search results.
type SearchResult struct {
Status int `json:"status"` // Status: 0 = Success with results, 1 = No more results available, 2 = Search ID not found, 3 = No results yet available keep trying
Files []apiFile `json:"files"` // List of files found
Files []ApiFile `json:"files"` // List of files found
Statistic interface{} `json:"statistic"` // Statistics of all results (independent from applied filters), if requested. Only set if files are returned (= if statistics changed). See SearchStatisticData.
}
@@ -161,7 +161,7 @@ func (api *WebapiInstance) apiSearchResult(w http.ResponseWriter, r *http.Reques
}
// query all results
var resultFiles []*apiFile
var resultFiles []*ApiFile
if errOffset == nil {
resultFiles = job.ReturnResult(offset, limit)
} else {
@@ -169,7 +169,7 @@ func (api *WebapiInstance) apiSearchResult(w http.ResponseWriter, r *http.Reques
}
var result SearchResult
result.Files = []apiFile{}
result.Files = []ApiFile{}
// loop over results
for n := range resultFiles {
@@ -241,7 +241,7 @@ func (api *WebapiInstance) apiSearchResultStream(w http.ResponseWriter, r *http.
// Only exit if limit is reached if used, otherwise only if there are no result or the connection breaks.
for {
// query all results
var resultFiles []*apiFile
var resultFiles []*ApiFile
queryCount := 1
if useLimit {
@@ -255,7 +255,7 @@ func (api *WebapiInstance) apiSearchResultStream(w http.ResponseWriter, r *http.
// loop over results
var result SearchResult
result.Files = []apiFile{}
result.Files = []ApiFile{}
for n := range resultFiles {
result.Files = append(result.Files, *resultFiles[n])
@@ -366,11 +366,11 @@ func (api *WebapiInstance) apiExplore(w http.ResponseWriter, r *http.Request) {
resultFiles := api.queryRecentShared(api.backend, fileType, uint64(limit*20/100), uint64(limit))
var result SearchResult
result.Files = []apiFile{}
result.Files = []ApiFile{}
// loop over results
for n := range resultFiles {
result.Files = append(result.Files, blockRecordFileToAPI(resultFiles[n]))
result.Files = append(result.Files, BlockRecordFileToAPI(resultFiles[n]))
}
if len(result.Files) == 0 {