Peernet-Archive/core
1
0
Fork 0
mirror of https://github.com/PeernetOfficial/core.git synced 2026-07-16 18:37:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

webapi search: Add min/max file size as filter.

Browse Source
This commit is contained in:
Kleissner
2021-10-10 03:03:35 +02:00
parent 5707132687
commit 7c3fc8938f
4 changed files with 69 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
webapi/Search Job.go
View File

@@ -23,6 +23,8 @@ type SearchFilter struct {
FileType int // File type such as binary, text document etc. See core.TypeX. -1 = not used.
FileFormat int // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 = not used.
Sort int // Sort order. See SortX.
SizeMin int // Min file size in bytes. -1 = not used.
SizeMax int // Max file size in bytes. -1 = not used.
}
// SearchJob is a collection of search jobs
@@ -58,22 +60,13 @@ type SearchJob struct {
// CreateSearchJob creates a new search job and adds it to the lookup list.
// Timeout and MaxResults must be set and must not be 0.
func CreateSearchJob(Timeout time.Duration, MaxResults, Sort, FileType, FileFormat int, DateFrom, DateTo time.Time) (job *SearchJob) {
func CreateSearchJob(Timeout time.Duration, MaxResults int, Filter SearchFilter) (job *SearchJob) {
job = &SearchJob{}
job.id = uuid.New()
job.timeout = Timeout
job.maxResult = MaxResults
job.filtersStart = SearchFilter{Sort: Sort, FileType: FileType, FileFormat: FileFormat}
if !DateFrom.IsZero() && !DateTo.IsZero() && DateFrom.Before(DateTo) {
job.filtersStart.DateFrom = DateFrom
job.filtersStart.DateTo = DateTo
job.filtersStart.IsDates = true
}
// initialize the runtime filters as the same
job.filtersRuntime = job.filtersStart
job.filtersStart = Filter
job.filtersRuntime = Filter // initialize the runtime filters as the same
// add to the list of jobs
allJobsMutex.Lock()
@@ -189,30 +182,17 @@ func (job *SearchJob) PeekResult(Offset, Limit int) (Result []*apiFile) {
return Result
}
// RuntimeFilter allows to apply filters at runtime to search jobs that already started.
// To remove the filters, call this function without the filters set. Sort 0 = none, file type and format -1 = not used, dates 0 = not used.
func (job *SearchJob) RuntimeFilter(Sort, FileType, FileFormat int, DateFrom, DateTo time.Time) {
// RuntimeFilter allows to apply filters at runtime to search jobs that already started. To remove the filters, call this function without the filters set.
func (job *SearchJob) RuntimeFilter(Filter SearchFilter) {
job.ResultSync.Lock()
defer job.ResultSync.Unlock()
job.filtersRuntime = Filter
// FreezeFiles and current offset is reset
job.FreezeFiles = nil
job.currentOffset = 0
job.filtersRuntime.Sort = Sort
job.filtersRuntime.FileType = FileType
job.filtersRuntime.FileFormat = FileFormat
if !DateFrom.IsZero() && !DateTo.IsZero() {
job.filtersRuntime.DateFrom = DateFrom
job.filtersRuntime.DateTo = DateTo
job.filtersRuntime.IsDates = true
} else {
job.filtersRuntime.DateFrom = time.Time{}
job.filtersRuntime.DateTo = time.Time{}
job.filtersRuntime.IsDates = false
}
// files remain in AllFiles, but Files needs to be filtered based on the new filter
job.Files = []*apiFile{}
job.requireSort = false // Sorting is done immediately below
@@ -246,6 +226,10 @@ func (job *SearchJob) isFileFiltered(file *apiFile) bool {
return false
}
if job.filtersRuntime.SizeMin >= 0 && file.Size < uint64(job.filtersRuntime.SizeMin) || job.filtersRuntime.SizeMax >= 0 && file.Size > uint64(job.filtersRuntime.SizeMax) {
return false
}
return true
}

5
webapi/Search Temp.go
View File

@@ -19,10 +19,11 @@ import (
)
func dispatchSearch(input SearchRequest) (job *SearchJob) {
Timeout, FileType, FileFormat, DateFrom, DateTo := input.Parse()
Timeout := input.Parse()
Filter := input.ToSearchFilter()
// create the search job
job = CreateSearchJob(Timeout, input.MaxResults, input.Sort, FileType, FileFormat, DateFrom, DateTo)
job = CreateSearchJob(Timeout, input.MaxResults, Filter)
// Create test data
// * Between 0-100 results

55
webapi/Search.go
View File

@@ -34,6 +34,8 @@ type SearchRequest struct {
TerminateID []uuid.UUID `json:"terminate"` // Optional: Previous search IDs to terminate. This is if the user makes a new search from the same tab. Same as first calling /search/terminate.
FileType int `json:"filetype"` // File type such as binary, text document etc. See core.TypeX. -1 = not used.
FileFormat int `json:"fileformat"` // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 = not used.
SizeMin int `json:"sizemin"` // Min file size in bytes. -1 = not used.
SizeMax int `json:"sizemax"` // Max file size in bytes. -1 = not used.
}
// Sort orders
@@ -109,6 +111,8 @@ Optional parameters:
&filetype=[File Type]
&fileformat=[File Format]
&from=[Date From]&to=[Date To]
&sizemin=[Minimum file size]
&sizemax=[Maximum file size]
&sort=[sort order]
Result: 200 with JSON structure SearchResult. Check the field status.
*/
@@ -124,14 +128,6 @@ func apiSearchResult(w http.ResponseWriter, r *http.Request) {
limit = 100
}
// filters and sort parameter
filterReset, _ := strconv.ParseBool(r.Form.Get("reset"))
fileType, _ := strconv.Atoi(r.Form.Get("filetype"))
fileFormat, _ := strconv.Atoi(r.Form.Get("fileformat"))
dateFrom, _ := time.Parse(apiDateFormat, r.Form.Get("from"))
dateTo, _ := time.Parse(apiDateFormat, r.Form.Get("to"))
sort, _ := strconv.Atoi(r.Form.Get("sort"))
// find the job ID
job := JobLookup(jobID)
if job == nil {
@@ -139,9 +135,19 @@ func apiSearchResult(w http.ResponseWriter, r *http.Request) {
return
}
// apply runtime filters, if any
if filterReset {
job.RuntimeFilter(sort, fileType, fileFormat, dateFrom, dateTo)
// filters and sort parameter
if filterReset, _ := strconv.ParseBool(r.Form.Get("reset")); filterReset {
fileType, _ := strconv.Atoi(r.Form.Get("filetype"))
fileFormat, _ := strconv.Atoi(r.Form.Get("fileformat"))
dateFrom := r.Form.Get("from")
dateTo := r.Form.Get("to")
sort, _ := strconv.Atoi(r.Form.Get("sort"))
sizeMin, _ := strconv.Atoi(r.Form.Get("sizemin"))
sizeMax, _ := strconv.Atoi(r.Form.Get("sizemax"))
filter := inputToSearchFilter(sort, fileType, fileFormat, dateFrom, dateTo, sizeMin, sizeMax)
job.RuntimeFilter(filter)
}
// query all results
@@ -293,18 +299,35 @@ func apiExplore(w http.ResponseWriter, r *http.Request) {
EncodeJSON(w, r, result)
}
func (input *SearchRequest) Parse() (Timeout time.Duration, FileType, FileFormat int, DateFrom, DateTo time.Time) {
func (input *SearchRequest) Parse() (Timeout time.Duration) {
if input.Timeout == 0 {
Timeout = time.Second * 20 // default timeout: 20 seconds
} else {
Timeout = time.Duration(input.Timeout) * time.Second
}
FileType = input.FileType
FileFormat = input.FileFormat
return
}
DateFrom, _ = time.Parse(apiDateFormat, input.DateFrom)
DateTo, _ = time.Parse(apiDateFormat, input.DateTo)
// ToSearchFilter converts the user input to a valid search filter
func (input *SearchRequest) ToSearchFilter() (output SearchFilter) {
return inputToSearchFilter(input.Sort, input.FileType, input.FileFormat, input.DateFrom, input.DateTo, input.SizeMin, input.SizeMax)
}
func inputToSearchFilter(Sort, FileType, FileFormat int, DateFrom, DateTo string, SizeMin, SizeMax int) (output SearchFilter) {
output.Sort = Sort
output.FileType = FileType
output.FileFormat = FileFormat
output.SizeMin = SizeMin
output.SizeMax = SizeMax
dateFrom, errFrom := time.Parse(apiDateFormat, DateFrom)
dateTo, errTo := time.Parse(apiDateFormat, DateTo)
if errFrom == nil && errTo == nil && dateFrom.Before(dateTo) {
output.DateFrom = dateFrom
output.DateTo = dateTo
output.IsDates = true
}
return
}

16
webapi/readme.md
View File

@@ -521,6 +521,8 @@ The following filters are supported:
### Submitting a Search Request
This starts a search request and returns an ID that can be used to collect the results asynchronously. Note that some of the filters described below (such as `filetype`) must be set to -1 if they are not used.
```
Request: POST /search with JSON SearchRequest
Response: 200 on success with JSON SearchRequestResponse
@@ -537,6 +539,8 @@ type SearchRequest struct {
TerminateID []uuid.UUID `json:"terminate"` // Optional: Previous search IDs to terminate. This is if the user makes a new search from the same tab. Same as first calling /search/terminate.
FileType int `json:"filetype"` // File type such as binary, text document etc. See core.TypeX. -1 = not used.
FileFormat int `json:"fileformat"` // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 = not used.
SizeMin int `json:"sizemin"` // Min file size in bytes. -1 = not used.
SizeMax int `json:"sizemax"` // Max file size in bytes. -1 = not used.
}
type SearchRequestResponse struct {
@@ -551,7 +555,12 @@ Example POST request to `http://127.0.0.1:112/search`:
{
"term": "Test Search",
"timeout": 10,
"maxresults": 1000
"maxresults": 1000,
"sort": 0,
"filetype": -1,
"fileformat": -1,
"sizemin": -1,
"sizemax": -1
}
```
@@ -567,7 +576,8 @@ Example response:
### Returning Search Results
This function returns search results. The default limit is 100.
If reset is set, all results will be filtered and sorted according to the settings. This means that the new first result will be returned again and internal result offset is set to 0.
If reset is set, all results will be filtered and sorted according to the provided parameters. This means that the new first result will be returned again and internal result offset is set to 0. Note that most filters must be set to -1 if they are not used (see the field comments in the `SearchRequest` structure in `/search` above).
```
Request: GET /search/result?id=[UUID]&limit=[max records]
@@ -576,6 +586,8 @@ Request: GET /search/result?id=[UUID]&limit=[max records]
&filetype=[File Type]
&fileformat=[File Format]
&from=[Date From]&to=[Date To]
&sizemin=[Minimum file size]
&sizemax=[Maximum file size]
&sort=[sort order]
Result: 200 with JSON structure SearchResult. Check the field status.
```