From 58b4603744526a8dcb1f9efba107529159f02a78 Mon Sep 17 00:00:00 2001 From: Kleissner Date: Thu, 23 Sep 2021 02:32:07 +0200 Subject: [PATCH] Initial search API with dummy results --- File Formats.go | 3 +- webapi/API.go | 4 + webapi/Search Temp.go | 249 ++++++++++++++++++++++++++++++++++++++++++ webapi/Search.go | 213 ++++++++++++++++++++++++++++++++++++ webapi/readme.md | 104 ++++++++++++++++++ 5 files changed, 572 insertions(+), 1 deletion(-) create mode 100644 webapi/Search Temp.go create mode 100644 webapi/Search.go diff --git a/File Formats.go b/File Formats.go index afabf9e..d762461 100644 --- a/File Formats.go +++ b/File Formats.go @@ -37,11 +37,12 @@ const ( FormatHTML // HTML file FormatText // Text file FormatEbook // Ebook file - FormatCompressed // Compressed file. + FormatCompressed // Compressed file FormatDatabase // Database file FormatEmail // Single email FormatCSV // CSV file FormatFolder // Virtual folder + FormatExecutable // Executable file ) // Future tags to be defined for audio/video: Artist, Album, Title, Length, Bitrate, Codec diff --git a/webapi/API.go b/webapi/API.go index 4d292c6..78f3ba4 100644 --- a/webapi/API.go +++ b/webapi/API.go @@ -52,6 +52,10 @@ func Start(ListenAddresses []string, UseSSL bool, CertificateFile, CertificateKe Router.HandleFunc("/profile/read", apiProfileRead).Methods("GET") Router.HandleFunc("/profile/write", apiProfileWrite).Methods("POST") Router.HandleFunc("/profile/delete", apiProfileDelete).Methods("POST") + Router.HandleFunc("/search", apiSearch).Methods("POST") + Router.HandleFunc("/search/result", apiSearchResult).Methods("GET") + //Router.HandleFunc("/search/result/ws", apiSearchResultStream).Methods("GET") + Router.HandleFunc("/search/terminate", apiSearchTerminate).Methods("GET") for _, listen := range ListenAddresses { go startWebServer(listen, UseSSL, CertificateFile, CertificateKey, Router, "API", TimeoutRead, TimeoutWrite) diff --git a/webapi/Search Temp.go b/webapi/Search Temp.go new file mode 100644 index 0000000..377e377 --- /dev/null +++ b/webapi/Search Temp.go @@ -0,0 +1,249 @@ +/* +File Name: Search Temp.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner + +Temporary search code to provide dummy results for testing. To be replaced! +*/ + +package webapi + +import ( + "encoding/hex" + "math/rand" + "sync" + "time" + + "github.com/PeernetOfficial/core" + "github.com/google/uuid" +) + +func dispatchSearch(input SearchRequest) (job *SearchJob) { + job = &SearchJob{id: uuid.New(), timeout: time.Duration(input.Timeout) * time.Second, maxResult: input.MaxResults, sortOrder: input.Sort, fileType: -1, fileFormat: -1} + + allJobsMutex.Lock() + allJobs[job.id] = job + allJobsMutex.Unlock() + + // Create test data + // * Between 0-100 results + // * Delay of first result is 0-5 seconds + // * File name, type, format, ID and hash are randomized + go func(job *SearchJob) { + rand.Seed(time.Now().UnixNano()) + waitTime := time.Duration(rand.Intn(5)) * time.Second + countResults := rand.Intn(100) + + time.Sleep(waitTime) + job.ResultSync.Lock() + + for n := 0; n < countResults; n++ { + newFile := createTestResult() + job.filesCurrent = append(job.filesCurrent, &newFile) + } + + job.ResultSync.Unlock() + + }(job) + + job.RemoveDefer(job.timeout + time.Second*20) + + return job +} + +// ---- job list management ---- + +// SearchJob is a collection of search jobs +type SearchJob struct { + // input settings + id uuid.UUID // The job id + timeout time.Duration // timeout set for all searches + maxResult int // max results user-facing. + sortOrder int // 0 = No sorting, 1 = Relevance ASC, 2 = Relevance DESC, 3 = Date ASC, 4 = Date DESC + + // additional optional filters + isDates bool // whether the from/to dates are valid, both are required. + dateFrom time.Time // optional date from + dateTo time.Time // optional date to + fileType int // File type such as binary, text document etc. See core.TypeX. -1 not used. + fileFormat int // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 not used. + + // runtime data + //clients []*SearchClient // all search clients + //clientsMutex sync.Mutex // mutex for manipulating client list + + filesCurrent []*core.BlockRecordFile // Current result list of files, not yet fetched by caller. + FilesAll []*core.BlockRecordFile // List of all files. This list only gets expanded. + ResultSync sync.Mutex // ResultSync ensures unique access to the file results +} + +// job list management +var ( + allJobs map[uuid.UUID]*SearchJob = make(map[uuid.UUID]*SearchJob) + allJobsMutex sync.RWMutex +) + +// Remove removes the structure from the list. Terminate should be called before. Unless the search is manually removed, it stays forever in the list. +func (job *SearchJob) Remove() { + allJobsMutex.Lock() + delete(allJobs, job.id) // delete is safe to call multiple times, so auto-removal and manual one are fine and need no syncing + allJobsMutex.Unlock() +} + +// RemoveDefer removes the search job after a given time after all searches are terminated. This can be used for automated time delayed removal. Do not create additional search clients after deferal removing. +func (job *SearchJob) RemoveDefer(Duration time.Duration) { + go func() { + // for _, client := range job.clients { + // <-client.TerminateSignal + // } + + <-time.After(Duration) + job.Remove() + }() +} + +// JobLookup looks up a job. Returns nil if not found. +func JobLookup(id uuid.UUID) (job *SearchJob) { + allJobsMutex.RLock() + job = allJobs[id] + allJobsMutex.RUnlock() + + return job +} + +// Terminate terminates all searches +func (job *SearchJob) Terminate() { + //job.clientsMutex.Lock() + //defer job.clientsMutex.Unlock() + + // for n := range job.clients { + // if !job.clients[n].IsTerminated { + // job.clients[n].Terminate(true) + // } + // } +} + +// ReturnResult returns the selected results. +func (job *SearchJob) ReturnResult(Limit int) (Result []*core.BlockRecordFile) { + if Limit == 0 { + return Result + } + + job.ResultSync.Lock() + defer job.ResultSync.Unlock() + + if len(job.filesCurrent) == 0 { + return Result + } else if Limit > len(job.filesCurrent) { + Limit = len(job.filesCurrent) + } + + Result = job.filesCurrent[:Limit] + job.filesCurrent = job.filesCurrent[Limit:] + + return Result +} + +// ReturnNext returns the next results. Call must be serialized. +func (job *SearchJob) ReturnNext(Limit int) (Result []*core.BlockRecordFile) { + return job.ReturnResult(Limit) +} + +// createTestResult creates a test file +func createTestResult() (file core.BlockRecordFile) { + randomData := make([]byte, 10) + rand.Read(randomData) + + file.Hash = core.Data2Hash(randomData) + file.Format = uint16(rand.Intn(core.FormatCSV)) + file.ID = uuid.New() + file.Size = uint64(len(randomData)) + + switch file.Format { + case core.FormatCSV, core.FormatEmail, core.FormatText, core.FormatHTML: + file.Type = core.TypeText + + case core.FormatDatabase: + file.Type = core.TypeBinary + + case core.FormatCompressed: + file.Type = core.TypeCompressed + + case core.FormatContainer: + file.Type = core.TypeContainer + + case core.FormatEbook: + file.Type = core.TypeEbook + + case core.FormatVideo: + file.Type = core.TypeVideo + + case core.FormatAudio: + file.Type = core.TypeAudio + + case core.FormatPicture: + file.Type = core.TypePicture + + case core.FormatPowerpoint, core.FormatExcel, core.FormatWord, core.FormatPDF: + file.Type = core.TypeDocument + + case core.FormatFolder: + file.Type = core.TypeFolder + + default: + file.Type = core.TypeBinary + + } + + var extension string + switch file.Type { + case core.TypeBinary: + extension = "bin" + + case core.TypeText: + extension = "txt" + + case core.TypePicture: + extension = "jpg" + + case core.TypeVideo: + extension = "mp4" + + case core.TypeAudio: + extension = "mp3" + + case core.TypeDocument: + extension = "pdf" + + case core.TypeExecutable: + extension = "exe" + + case core.TypeContainer: + extension = "zip" + + case core.TypeCompressed: + extension = "gz" + + case core.TypeFolder: + extension = "" + + case core.TypeEbook: + extension = "" + + default: + extension = ".bin" + } + + file.TagsDecoded = append(file.TagsDecoded, core.FileTagName{Name: TempFileName("", extension)}) + //file.TagsDecoded = append(file.TagsDecoded, core.FileTagFolder{Name: "not set"}) + file.TagsDecoded = append(file.TagsDecoded, core.FileTagDateShared{Date: time.Now().UTC()}) + + return +} + +// TempFileName generates a temporary filename for use in testing or whatever +func TempFileName(prefix, suffix string) string { + randBytes := make([]byte, 16) + rand.Read(randBytes) + return prefix + hex.EncodeToString(randBytes) + "." + suffix +} diff --git a/webapi/Search.go b/webapi/Search.go new file mode 100644 index 0000000..af12bb2 --- /dev/null +++ b/webapi/Search.go @@ -0,0 +1,213 @@ +/* +File Name: Search.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner + +/search Submit a search request +/search/result Return search results +/search/terminate Terminate a search +/search/result/ws Websocket to return search results as stream (future) +/search/statistic Statistics about the results (future) + +*/ + +package webapi + +import ( + "net/http" + "strconv" + + "github.com/google/uuid" +) + +// SearchRequest is the information from the end-user for the search. +type SearchRequest struct { + Term string `json:"term"` // Search term. + Timeout int `json:"timeout"` // Timeout in seconds. 0 means default. This is the entire time the search may take. Found results are still available after this timeout. + MaxResults int `json:"maxresults"` // Total number of max results. 0 means default. + DateFrom string `json:"datefrom"` // Date from, both from/to are required if set. + DateTo string `json:"dateto"` // Date to, both from/to are required if set. + Sort int `json:"sort"` // Sort order: 0 = No sorting, 1 = Relevance ASC, 2 = Relevance DESC (this should be default), 3 = Date ASC, 4 = Date DESC + TerminateID []uuid.UUID `json:"terminate"` // Optional: Previous search IDs to terminate. This is if the user makes a new search from the same tab. Same as first calling /search/terminate. + TypeFilter int `json:"typefilter"` // 0 = No filters used, 1 = Use file type filter, 2 = Use file format filter. + FileType int `json:"filetype"` // File type such as binary, text document etc. See core.TypeX. + FileFormat int `json:"fileformat"` // File format such as PDF, Word, Ebook, etc. See core.FormatX. +} + +// SearchRequestResponse is the result to the initial search request +type SearchRequestResponse struct { + ID uuid.UUID `json:"id"` // ID of the search job. This is used to get the results. + Status int `json:"status"` // Status of the search: 0 = Success (ID valid), 1 = Invalid Term, 2 = Error Max Concurrent Searches +} + +// SearchResult contains the search results. +type SearchResult struct { + Status int `json:"status"` // Status: 0 = Success with results, 1 = No more results available, 2 = Search ID not found, 3 = No results yet available keep trying + Files []apiBlockRecordFile `json:"files"` // List of files found +} + +/* +apiSearch submits a search request +Request: POST /search with JSON SearchRequest +Result: 200 on success with JSON SearchRequestResponse + 400 on invalid JSON +*/ +func apiSearch(w http.ResponseWriter, r *http.Request) { + + var input SearchRequest + if err := DecodeJSON(w, r, &input); err != nil { + return + } + + if input.Timeout <= 0 { + input.Timeout = 20 + } + if input.MaxResults <= 0 { + input.MaxResults = 200 + } + + // Terminate previous searches, if their IDs were supplied. This allows terminating the old search immediately without making a separate /search/terminate request. + for _, terminate := range input.TerminateID { + if job := JobLookup(terminate); job != nil { + job.Terminate() + job.Remove() + } + } + + job := dispatchSearch(input) + + EncodeJSON(w, r, SearchRequestResponse{Status: 0, ID: job.id}) +} + +/* +apiSearchResult returns results. The default limit is 100. +Request: GET /search/result?id=[UUID]&limit=[max records] +Result: 200 with JSON structure SearchResult. Check the field status. +*/ +func apiSearchResult(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + jobID, err := uuid.Parse(r.Form.Get("id")) + if err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } + limit, err := strconv.Atoi(r.Form.Get("limit")) + if err != nil { + limit = 100 + } + + // find the job ID + job := JobLookup(jobID) + if job == nil { + EncodeJSON(w, r, SearchResult{Status: 2}) + return + } + + // query all results + resultFiles := job.ReturnNext(limit) + + var result SearchResult + result.Files = []apiBlockRecordFile{} + + // loop over results + for n := range resultFiles { + result.Files = append(result.Files, blockRecordFileToAPI(*resultFiles[n])) + } + + if len(result.Files) == 0 { + result.Status = 3 // No results yet available keep trying + } + + result.Status = 1 // No more results to expect + + EncodeJSON(w, r, result) +} + +/* +apiSearchResultStream runs a websocket to return results +Request: GET /search/result/ws?id=[UUID]&limit=[optional max records] +Result: If successful, upgrades to a web-socket and sends JSON structure SearchResult messages + Limit is optional. Not used if ommitted or 0. +*/ /* +func apiSearchResultStream(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + jobID, err := uuid.Parse(r.Form.Get("id")) + if err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } + limit, err := strconv.Atoi(r.Form.Get("limit")) + useLimit := err == nil + + // look up the job + job := JobLookup(jobID) + if job == nil { + EncodeJSON(w, r, SearchResult{Status: 2}) + return + } + + // upgrade to web-socket + conn, err := WSUpgrader.Upgrade(w, r, nil) + if err != nil { + // gorilla will automatically respond with "400 Bad Request", no other response is therefore necessary + return + } + + // loop to get new results and send out via the web socket. + // Only exit if limit is reached if used, otherwise only if there are no result or the connection breaks. + for { + // query all results + + var result SearchResult + result.Files = []apiBlockRecordFile{} + + // loop over the results + + // if no results, stall + if len(result.Files) == 0 { + time.Sleep(time.Millisecond * 100) + continue + } + + // send out the results via the web-socket + if err := conn.WriteJSON(result); err != nil { + conn.Close() + return + } + + // Check whether to continue. If the limit is used break once all done. + if (useLimit && limit <= 0) || result.Status == 1 { + break + } + } +}*/ + +/* +apiSearchTerminate terminates a search +Request: GET /search/terminate?id=[UUID] +Response: 204 Empty + 400 Invalid input + 404 ID not found +*/ +func apiSearchTerminate(w http.ResponseWriter, r *http.Request) { + + r.ParseForm() + jobID, err := uuid.Parse(r.Form.Get("id")) + if err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } + + // look up the job + job := JobLookup(jobID) + if job == nil { + http.Error(w, "", http.StatusNotFound) + return + } + + // terminate and remove it from the list + job.Terminate() + job.Remove() + + w.WriteHeader(http.StatusNoContent) +} diff --git a/webapi/readme.md b/webapi/readme.md index 574c873..0806d0e 100644 --- a/webapi/readme.md +++ b/webapi/readme.md @@ -41,6 +41,10 @@ These are the functions provided by the API: /profile/read Reads a specific users profile field or blob /profile/write Writes a specific users profile field or blob /profile/delete Deletes profile fields or blobs + +/search Submit a search request +/search/result Return search results +/search/terminate Terminate a search ``` # API Documentation @@ -468,3 +472,103 @@ One practical use case is deleting the profile picture (by specifying blob 0) wi }] } ``` + +## Search API + +The search API provides a high-level function to search for files in Peernet. Searching is always asynchronous. `/search` returns an UUID which is used to loop over `/search/result` until the search is terminated. + +The current implementation of the underlying search algorithm only searches file names. Optional filters are supported. + +### Submitting a Search Request + +``` +Request: POST /search with JSON SearchRequest +Response: 200 on success with JSON SearchRequestResponse +``` + +```go +type SearchRequest struct { + Term string `json:"term"` // Search term. + Timeout time.Duration `json:"timeout"` // Timeout in seconds. 0 means default. This is the entire time the search may take. Found results are still available after this timeout. + MaxResults int `json:"maxresults"` // Total number of max results. 0 means default. + DateFrom string `json:"datefrom"` // Date from, both from/to are required if set. + DateTo string `json:"dateto"` // Date to, both from/to are required if set. + Sort int `json:"sort"` // Sort order: 0 = No sorting, 1 = Relevance ASC, 2 = Relevance DESC (this should be default), 3 = Date ASC, 4 = Date DESC + TerminateID []uuid.UUID `json:"terminate"` // Optional: Previous search IDs to terminate. This is if the user makes a new search from the same tab. Same as first calling /search/terminate. + TypeFilter int `json:"typefilter"` // 0 = No filters used, 1 = Use file type filter, 2 = Use file format filter. + FileType int `json:"filetype"` // File type such as binary, text document etc. See core.TypeX. + FileFormat int `json:"fileformat"` // File format such as PDF, Word, Ebook, etc. See core.FormatX. +} + +type SearchRequestResponse struct { + ID uuid.UUID `json:"id"` // ID of the search job. This is used to get the results. + Status int `json:"status"` // Status of the search: 0 = Success (ID valid), 1 = Invalid Term, 2 = Error Max Concurrent Searches +} +``` + +Example POST request to `http://127.0.0.1:112/search`: + +```json +{ + "term": "Test Search", + "timeout": 10, + "maxresults": 1000 +} +``` + +Example response: + +```json +{ + "id": "ac5efa64-d403-4a57-8259-c7b7dfb09667", + "status": 0 +} +``` + +### Returning Search Results + +This function returns search results. + +``` +Request: GET /search/result?id=[UUID]&limit=[max records] +Result: 200 with JSON structure SearchResult. Check the field status. +``` + +```go +type SearchResult struct { + Status int `json:"status"` // Status: 0 = Success with results, 1 = No more results available, 2 = Search ID not found, 3 = No results yet available keep trying + Files []apiBlockRecordFile `json:"files"` // List of files found +} +``` + +Example request: `http://127.0.0.1:112/search/result?id=ac5efa64-d403-4a57-8259-c7b7dfb09667&limit=10` + +Example response with dummy data: + +```json +{ + "status": 1, + "files": [{ + "id": "b5b0706c-817c-492f-8203-5005c59f110c", + "hash": "Mv6O773ytkJ5jSjLoy2EvHQaM5KfVppJHeTppMc7alA=", + "type": 1, + "format": 14, + "size": 10, + "folder": "", + "name": "88d8cc57d5c2a5fea881ceea09503ee4.txt", + "description": "", + "date": "2021-09-23T00:00:00Z", + "metadata": [], + "tagsraw": [] + }] +} +``` + +### Terminating a Search + +The user can terminate a search early using this function. This helps save system resources and should be considered best practice once a search is no longer needed (for example when the user closes the tab or window that shows the results). + +``` +Request: GET /search/terminate?id=[UUID] +Response: 204 Empty +``` \ No newline at end of file