Initial search API with dummy results

File Formats.go

@@ -37,11 +37,12 @@ const (
 	FormatHTML              // HTML file
 	FormatText              // Text file
 	FormatEbook             // Ebook file
-	FormatCompressed        // Compressed file.
+	FormatCompressed        // Compressed file
 	FormatDatabase          // Database file
 	FormatEmail             // Single email
 	FormatCSV               // CSV file
 	FormatFolder            // Virtual folder
+	FormatExecutable        // Executable file
 )
 
 // Future tags to be defined for audio/video: Artist, Album, Title, Length, Bitrate, Codec

webapi/API.go

@@ -52,6 +52,10 @@ func Start(ListenAddresses []string, UseSSL bool, CertificateFile, CertificateKe
 	Router.HandleFunc("/profile/read", apiProfileRead).Methods("GET")
 	Router.HandleFunc("/profile/write", apiProfileWrite).Methods("POST")
 	Router.HandleFunc("/profile/delete", apiProfileDelete).Methods("POST")
+	Router.HandleFunc("/search", apiSearch).Methods("POST")
+	Router.HandleFunc("/search/result", apiSearchResult).Methods("GET")
+	//Router.HandleFunc("/search/result/ws", apiSearchResultStream).Methods("GET")
+	Router.HandleFunc("/search/terminate", apiSearchTerminate).Methods("GET")
 
 	for _, listen := range ListenAddresses {
 		go startWebServer(listen, UseSSL, CertificateFile, CertificateKey, Router, "API", TimeoutRead, TimeoutWrite)

webapi/Search Temp.go

@@ -0,0 +1,249 @@
+/*
+File Name:  Search Temp.go
+Copyright:  2021 Peernet Foundation s.r.o.
+Author:     Peter Kleissner
+
+Temporary search code to provide dummy results for testing. To be replaced!
+*/
+
+package webapi
+
+import (
+	"encoding/hex"
+	"math/rand"
+	"sync"
+	"time"
+
+	"github.com/PeernetOfficial/core"
+	"github.com/google/uuid"
+)
+
+func dispatchSearch(input SearchRequest) (job *SearchJob) {
+	job = &SearchJob{id: uuid.New(), timeout: time.Duration(input.Timeout) * time.Second, maxResult: input.MaxResults, sortOrder: input.Sort, fileType: -1, fileFormat: -1}
+
+	allJobsMutex.Lock()
+	allJobs[job.id] = job
+	allJobsMutex.Unlock()
+
+	// Create test data
+	// * Between 0-100 results
+	// * Delay of first result is 0-5 seconds
+	// * File name, type, format, ID and hash are randomized
+	go func(job *SearchJob) {
+		rand.Seed(time.Now().UnixNano())
+		waitTime := time.Duration(rand.Intn(5)) * time.Second
+		countResults := rand.Intn(100)
+
+		time.Sleep(waitTime)
+		job.ResultSync.Lock()
+
+		for n := 0; n < countResults; n++ {
+			newFile := createTestResult()
+			job.filesCurrent = append(job.filesCurrent, &newFile)
+		}
+
+		job.ResultSync.Unlock()
+
+	}(job)
+
+	job.RemoveDefer(job.timeout + time.Second*20)
+
+	return job
+}
+
+// ---- job list management ----
+
+// SearchJob is a collection of search jobs
+type SearchJob struct {
+	// input settings
+	id        uuid.UUID     // The job id
+	timeout   time.Duration // timeout set for all searches
+	maxResult int           // max results user-facing.
+	sortOrder int           // 0 = No sorting, 1 = Relevance ASC, 2 = Relevance DESC, 3 = Date ASC, 4 = Date DESC
+
+	// additional optional filters
+	isDates    bool      // whether the from/to dates are valid, both are required.
+	dateFrom   time.Time // optional date from
+	dateTo     time.Time // optional date to
+	fileType   int       // File type such as binary, text document etc. See core.TypeX. -1 not used.
+	fileFormat int       // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 not used.
+
+	// runtime data
+	//clients        []*SearchClient // all search clients
+	//clientsMutex sync.Mutex // mutex for manipulating client list
+
+	filesCurrent []*core.BlockRecordFile // Current result list of files, not yet fetched by caller.
+	FilesAll     []*core.BlockRecordFile // List of all files. This list only gets expanded.
+	ResultSync   sync.Mutex              // ResultSync ensures unique access to the file results
+}
+
+// job list management
+var (
+	allJobs      map[uuid.UUID]*SearchJob = make(map[uuid.UUID]*SearchJob)
+	allJobsMutex sync.RWMutex
+)
+
+// Remove removes the structure from the list. Terminate should be called before. Unless the search is manually removed, it stays forever in the list.
+func (job *SearchJob) Remove() {
+	allJobsMutex.Lock()
+	delete(allJobs, job.id) // delete is safe to call multiple times, so auto-removal and manual one are fine and need no syncing
+	allJobsMutex.Unlock()
+}
+
+// RemoveDefer removes the search job after a given time after all searches are terminated. This can be used for automated time delayed removal. Do not create additional search clients after deferal removing.
+func (job *SearchJob) RemoveDefer(Duration time.Duration) {
+	go func() {
+		// for _, client := range job.clients {
+		// 	<-client.TerminateSignal
+		// }
+
+		<-time.After(Duration)
+		job.Remove()
+	}()
+}
+
+// JobLookup looks up a job. Returns nil if not found.
+func JobLookup(id uuid.UUID) (job *SearchJob) {
+	allJobsMutex.RLock()
+	job = allJobs[id]
+	allJobsMutex.RUnlock()
+
+	return job
+}
+
+// Terminate terminates all searches
+func (job *SearchJob) Terminate() {
+	//job.clientsMutex.Lock()
+	//defer job.clientsMutex.Unlock()
+
+	// for n := range job.clients {
+	// 	if !job.clients[n].IsTerminated {
+	// 		job.clients[n].Terminate(true)
+	// 	}
+	// }
+}
+
+// ReturnResult returns the selected results.
+func (job *SearchJob) ReturnResult(Limit int) (Result []*core.BlockRecordFile) {
+	if Limit == 0 {
+		return Result
+	}
+
+	job.ResultSync.Lock()
+	defer job.ResultSync.Unlock()
+
+	if len(job.filesCurrent) == 0 {
+		return Result
+	} else if Limit > len(job.filesCurrent) {
+		Limit = len(job.filesCurrent)
+	}
+
+	Result = job.filesCurrent[:Limit]
+	job.filesCurrent = job.filesCurrent[Limit:]
+
+	return Result
+}
+
+// ReturnNext returns the next results. Call must be serialized.
+func (job *SearchJob) ReturnNext(Limit int) (Result []*core.BlockRecordFile) {
+	return job.ReturnResult(Limit)
+}
+
+// createTestResult creates a test file
+func createTestResult() (file core.BlockRecordFile) {
+	randomData := make([]byte, 10)
+	rand.Read(randomData)
+
+	file.Hash = core.Data2Hash(randomData)
+	file.Format = uint16(rand.Intn(core.FormatCSV))
+	file.ID = uuid.New()
+	file.Size = uint64(len(randomData))
+
+	switch file.Format {
+	case core.FormatCSV, core.FormatEmail, core.FormatText, core.FormatHTML:
+		file.Type = core.TypeText
+
+	case core.FormatDatabase:
+		file.Type = core.TypeBinary
+
+	case core.FormatCompressed:
+		file.Type = core.TypeCompressed
+
+	case core.FormatContainer:
+		file.Type = core.TypeContainer
+
+	case core.FormatEbook:
+		file.Type = core.TypeEbook
+
+	case core.FormatVideo:
+		file.Type = core.TypeVideo
+
+	case core.FormatAudio:
+		file.Type = core.TypeAudio
+
+	case core.FormatPicture:
+		file.Type = core.TypePicture
+
+	case core.FormatPowerpoint, core.FormatExcel, core.FormatWord, core.FormatPDF:
+		file.Type = core.TypeDocument
+
+	case core.FormatFolder:
+		file.Type = core.TypeFolder
+
+	default:
+		file.Type = core.TypeBinary
+
+	}
+
+	var extension string
+	switch file.Type {
+	case core.TypeBinary:
+		extension = "bin"
+
+	case core.TypeText:
+		extension = "txt"
+
+	case core.TypePicture:
+		extension = "jpg"
+
+	case core.TypeVideo:
+		extension = "mp4"
+
+	case core.TypeAudio:
+		extension = "mp3"
+
+	case core.TypeDocument:
+		extension = "pdf"
+
+	case core.TypeExecutable:
+		extension = "exe"
+
+	case core.TypeContainer:
+		extension = "zip"
+
+	case core.TypeCompressed:
+		extension = "gz"
+
+	case core.TypeFolder:
+		extension = ""
+
+	case core.TypeEbook:
+		extension = ""
+
+	default:
+		extension = ".bin"
+	}
+
+	file.TagsDecoded = append(file.TagsDecoded, core.FileTagName{Name: TempFileName("", extension)})
+	//file.TagsDecoded = append(file.TagsDecoded, core.FileTagFolder{Name: "not set"})
+	file.TagsDecoded = append(file.TagsDecoded, core.FileTagDateShared{Date: time.Now().UTC()})
+
+	return
+}
+
+// TempFileName generates a temporary filename for use in testing or whatever
+func TempFileName(prefix, suffix string) string {
+	randBytes := make([]byte, 16)
+	rand.Read(randBytes)
+	return prefix + hex.EncodeToString(randBytes) + "." + suffix
+}

webapi/Search.go

@@ -0,0 +1,213 @@
+/*
+File Name:  Search.go
+Copyright:  2021 Peernet Foundation s.r.o.
+Author:     Peter Kleissner
+
+/search                 Submit a search request
+/search/result          Return search results
+/search/terminate       Terminate a search
+/search/result/ws       Websocket to return search results as stream (future)
+/search/statistic       Statistics about the results (future)
+
+*/
+
+package webapi
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/google/uuid"
+)
+
+// SearchRequest is the information from the end-user for the search.
+type SearchRequest struct {
+	Term        string      `json:"term"`       // Search term.
+	Timeout     int         `json:"timeout"`    // Timeout in seconds. 0 means default. This is the entire time the search may take. Found results are still available after this timeout.
+	MaxResults  int         `json:"maxresults"` // Total number of max results. 0 means default.
+	DateFrom    string      `json:"datefrom"`   // Date from, both from/to are required if set.
+	DateTo      string      `json:"dateto"`     // Date to, both from/to are required if set.
+	Sort        int         `json:"sort"`       // Sort order: 0 = No sorting, 1 = Relevance ASC, 2 = Relevance DESC (this should be default), 3 = Date ASC, 4 = Date DESC
+	TerminateID []uuid.UUID `json:"terminate"`  // Optional: Previous search IDs to terminate. This is if the user makes a new search from the same tab. Same as first calling /search/terminate.
+	TypeFilter  int         `json:"typefilter"` // 0 = No filters used, 1 = Use file type filter, 2 = Use file format filter.
+	FileType    int         `json:"filetype"`   // File type such as binary, text document etc. See core.TypeX.
+	FileFormat  int         `json:"fileformat"` // File format such as PDF, Word, Ebook, etc. See core.FormatX.
+}
+
+// SearchRequestResponse is the result to the initial search request
+type SearchRequestResponse struct {
+	ID     uuid.UUID `json:"id"`     // ID of the search job. This is used to get the results.
+	Status int       `json:"status"` // Status of the search: 0 = Success (ID valid), 1 = Invalid Term, 2 = Error Max Concurrent Searches
+}
+
+// SearchResult contains the search results.
+type SearchResult struct {
+	Status int                  `json:"status"` // Status: 0 = Success with results, 1 = No more results available, 2 = Search ID not found, 3 = No results yet available keep trying
+	Files  []apiBlockRecordFile `json:"files"`  // List of files found
+}
+
+/*
+apiSearch submits a search request
+Request:    POST /search with JSON SearchRequest
+Result:     200 on success with JSON SearchRequestResponse
+            400 on invalid JSON
+*/
+func apiSearch(w http.ResponseWriter, r *http.Request) {
+
+	var input SearchRequest
+	if err := DecodeJSON(w, r, &input); err != nil {
+		return
+	}
+
+	if input.Timeout <= 0 {
+		input.Timeout = 20
+	}
+	if input.MaxResults <= 0 {
+		input.MaxResults = 200
+	}
+
+	// Terminate previous searches, if their IDs were supplied. This allows terminating the old search immediately without making a separate /search/terminate request.
+	for _, terminate := range input.TerminateID {
+		if job := JobLookup(terminate); job != nil {
+			job.Terminate()
+			job.Remove()
+		}
+	}
+
+	job := dispatchSearch(input)
+
+	EncodeJSON(w, r, SearchRequestResponse{Status: 0, ID: job.id})
+}
+
+/*
+apiSearchResult returns results. The default limit is 100.
+Request:    GET /search/result?id=[UUID]&limit=[max records]
+Result:     200 with JSON structure SearchResult. Check the field status.
+*/
+func apiSearchResult(w http.ResponseWriter, r *http.Request) {
+	r.ParseForm()
+	jobID, err := uuid.Parse(r.Form.Get("id"))
+	if err != nil {
+		http.Error(w, "", http.StatusBadRequest)
+		return
+	}
+	limit, err := strconv.Atoi(r.Form.Get("limit"))
+	if err != nil {
+		limit = 100
+	}
+
+	// find the job ID
+	job := JobLookup(jobID)
+	if job == nil {
+		EncodeJSON(w, r, SearchResult{Status: 2})
+		return
+	}
+
+	// query all results
+	resultFiles := job.ReturnNext(limit)
+
+	var result SearchResult
+	result.Files = []apiBlockRecordFile{}
+
+	// loop over results
+	for n := range resultFiles {
+		result.Files = append(result.Files, blockRecordFileToAPI(*resultFiles[n]))
+	}
+
+	if len(result.Files) == 0 {
+		result.Status = 3 // No results yet available keep trying
+	}
+
+	result.Status = 1 // No more results to expect
+
+	EncodeJSON(w, r, result)
+}
+
+/*
+apiSearchResultStream runs a websocket to return results
+Request:    GET /search/result/ws?id=[UUID]&limit=[optional max records]
+Result:     If successful, upgrades to a web-socket and sends JSON structure SearchResult messages
+            Limit is optional. Not used if ommitted or 0.
+*/ /*
+func apiSearchResultStream(w http.ResponseWriter, r *http.Request) {
+	r.ParseForm()
+	jobID, err := uuid.Parse(r.Form.Get("id"))
+	if err != nil {
+		http.Error(w, "", http.StatusBadRequest)
+		return
+	}
+	limit, err := strconv.Atoi(r.Form.Get("limit"))
+	useLimit := err == nil
+
+	// look up the job
+	job := JobLookup(jobID)
+	if job == nil {
+		EncodeJSON(w, r, SearchResult{Status: 2})
+		return
+	}
+
+	// upgrade to web-socket
+	conn, err := WSUpgrader.Upgrade(w, r, nil)
+	if err != nil {
+		// gorilla will automatically respond with "400 Bad Request", no other response is therefore necessary
+		return
+	}
+
+	// loop to get new results and send out via the web socket.
+	// Only exit if limit is reached if used, otherwise only if there are no result or the connection breaks.
+	for {
+		// query all results
+
+		var result SearchResult
+		result.Files = []apiBlockRecordFile{}
+
+		// loop over the results
+
+		// if no results, stall
+		if len(result.Files) == 0 {
+			time.Sleep(time.Millisecond * 100)
+			continue
+		}
+
+		// send out the results via the web-socket
+		if err := conn.WriteJSON(result); err != nil {
+			conn.Close()
+			return
+		}
+
+		// Check whether to continue. If the limit is used break once all done.
+		if (useLimit && limit <= 0) || result.Status == 1 {
+			break
+		}
+	}
+}*/
+
+/*
+apiSearchTerminate terminates a search
+Request:    GET /search/terminate?id=[UUID]
+Response:   204 Empty
+            400 Invalid input
+            404 ID not found
+*/
+func apiSearchTerminate(w http.ResponseWriter, r *http.Request) {
+
+	r.ParseForm()
+	jobID, err := uuid.Parse(r.Form.Get("id"))
+	if err != nil {
+		http.Error(w, "", http.StatusBadRequest)
+		return
+	}
+
+	// look up the job
+	job := JobLookup(jobID)
+	if job == nil {
+		http.Error(w, "", http.StatusNotFound)
+		return
+	}
+
+	// terminate and remove it from the list
+	job.Terminate()
+	job.Remove()
+
+	w.WriteHeader(http.StatusNoContent)
+}

webapi/readme.md

@@ -41,6 +41,10 @@ These are the functions provided by the API:
 /profile/read                   Reads a specific users profile field or blob
 /profile/write                  Writes a specific users profile field or blob
 /profile/delete                 Deletes profile fields or blobs
+
+/search                         Submit a search request
+/search/result                  Return search results
+/search/terminate               Terminate a search
 ```
 
 # API Documentation
@@ -468,3 +472,103 @@ One practical use case is deleting the profile picture (by specifying blob 0) wi
     }]
 }
 ```
+
+## Search API
+
+The search API provides a high-level function to search for files in Peernet. Searching is always asynchronous. `/search` returns an UUID which is used to loop over `/search/result` until the search is terminated.
+
+The current implementation of the underlying search algorithm only searches file names. Optional filters are supported.
+
+### Submitting a Search Request
+
+```
+Request:    POST /search with JSON SearchRequest
+Response:   200 on success with JSON SearchRequestResponse
+```
+
+```go
+type SearchRequest struct {
+	Term        string        `json:"term"`       // Search term.
+	Timeout     time.Duration `json:"timeout"`    // Timeout in seconds. 0 means default. This is the entire time the search may take. Found results are still available after this timeout.
+	MaxResults  int           `json:"maxresults"` // Total number of max results. 0 means default.
+	DateFrom    string        `json:"datefrom"`   // Date from, both from/to are required if set.
+	DateTo      string        `json:"dateto"`     // Date to, both from/to are required if set.
+	Sort        int           `json:"sort"`       // Sort order: 0 = No sorting, 1 = Relevance ASC, 2 = Relevance DESC (this should be default), 3 = Date ASC, 4 = Date DESC
+	TerminateID []uuid.UUID   `json:"terminate"`  // Optional: Previous search IDs to terminate. This is if the user makes a new search from the same tab. Same as first calling /search/terminate.
+	TypeFilter  int           `json:"typefilter"` // 0 = No filters used, 1 = Use file type filter, 2 = Use file format filter.
+	FileType    int           `json:"filetype"`   // File type such as binary, text document etc. See core.TypeX.
+	FileFormat  int           `json:"fileformat"` // File format such as PDF, Word, Ebook, etc. See core.FormatX.
+}
+
+type SearchRequestResponse struct {
+	ID     uuid.UUID `json:"id"`     // ID of the search job. This is used to get the results.
+	Status int       `json:"status"` // Status of the search: 0 = Success (ID valid), 1 = Invalid Term, 2 = Error Max Concurrent Searches
+}
+```
+
+Example POST request to `http://127.0.0.1:112/search`:
+
+```json
+{
+    "term": "Test Search",
+    "timeout": 10,
+    "maxresults": 1000
+}
+```
+
+Example response:
+
+```json
+{
+    "id": "ac5efa64-d403-4a57-8259-c7b7dfb09667",
+    "status": 0
+}
+```
+
+### Returning Search Results
+
+This function returns search results.
+
+```
+Request:    GET /search/result?id=[UUID]&limit=[max records]
+Result:     200 with JSON structure SearchResult. Check the field status.
+```
+
+```go
+type SearchResult struct {
+	Status int                  `json:"status"` // Status: 0 = Success with results, 1 = No more results available, 2 = Search ID not found, 3 = No results yet available keep trying
+	Files  []apiBlockRecordFile `json:"files"`  // List of files found
+}
+```
+
+Example request: `http://127.0.0.1:112/search/result?id=ac5efa64-d403-4a57-8259-c7b7dfb09667&limit=10`
+
+Example response with dummy data:
+
+```json
+{
+    "status": 1,
+    "files": [{
+        "id": "b5b0706c-817c-492f-8203-5005c59f110c",
+        "hash": "Mv6O773ytkJ5jSjLoy2EvHQaM5KfVppJHeTppMc7alA=",
+        "type": 1,
+        "format": 14,
+        "size": 10,
+        "folder": "",
+        "name": "88d8cc57d5c2a5fea881ceea09503ee4.txt",
+        "description": "",
+        "date": "2021-09-23T00:00:00Z",
+        "metadata": [],
+        "tagsraw": []
+    }]
+}
+```
+
+### Terminating a Search
+
+The user can terminate a search early using this function. This helps save system resources and should be considered best practice once a search is no longer needed (for example when the user closes the tab or window that shows the results).
+
+```
+Request:    GET /search/terminate?id=[UUID]
+Response:   204 Empty
+```