From 7c3fc8938fa0b5792b65306becf73621d3139645 Mon Sep 17 00:00:00 2001 From: Kleissner Date: Sun, 10 Oct 2021 03:03:35 +0200 Subject: [PATCH] webapi search: Add min/max file size as filter. --- webapi/Search Job.go | 42 ++++++++++----------------------- webapi/Search Temp.go | 5 ++-- webapi/Search.go | 55 ++++++++++++++++++++++++++++++------------- webapi/readme.md | 16 +++++++++++-- 4 files changed, 69 insertions(+), 49 deletions(-) diff --git a/webapi/Search Job.go b/webapi/Search Job.go index e0f250d..116f615 100644 --- a/webapi/Search Job.go +++ b/webapi/Search Job.go @@ -23,6 +23,8 @@ type SearchFilter struct { FileType int // File type such as binary, text document etc. See core.TypeX. -1 = not used. FileFormat int // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 = not used. Sort int // Sort order. See SortX. + SizeMin int // Min file size in bytes. -1 = not used. + SizeMax int // Max file size in bytes. -1 = not used. } // SearchJob is a collection of search jobs @@ -58,22 +60,13 @@ type SearchJob struct { // CreateSearchJob creates a new search job and adds it to the lookup list. // Timeout and MaxResults must be set and must not be 0. -func CreateSearchJob(Timeout time.Duration, MaxResults, Sort, FileType, FileFormat int, DateFrom, DateTo time.Time) (job *SearchJob) { +func CreateSearchJob(Timeout time.Duration, MaxResults int, Filter SearchFilter) (job *SearchJob) { job = &SearchJob{} job.id = uuid.New() job.timeout = Timeout job.maxResult = MaxResults - - job.filtersStart = SearchFilter{Sort: Sort, FileType: FileType, FileFormat: FileFormat} - - if !DateFrom.IsZero() && !DateTo.IsZero() && DateFrom.Before(DateTo) { - job.filtersStart.DateFrom = DateFrom - job.filtersStart.DateTo = DateTo - job.filtersStart.IsDates = true - } - - // initialize the runtime filters as the same - job.filtersRuntime = job.filtersStart + job.filtersStart = Filter + job.filtersRuntime = Filter // initialize the runtime filters as the same // add to the list of jobs allJobsMutex.Lock() @@ -189,30 +182,17 @@ func (job *SearchJob) PeekResult(Offset, Limit int) (Result []*apiFile) { return Result } -// RuntimeFilter allows to apply filters at runtime to search jobs that already started. -// To remove the filters, call this function without the filters set. Sort 0 = none, file type and format -1 = not used, dates 0 = not used. -func (job *SearchJob) RuntimeFilter(Sort, FileType, FileFormat int, DateFrom, DateTo time.Time) { +// RuntimeFilter allows to apply filters at runtime to search jobs that already started. To remove the filters, call this function without the filters set. +func (job *SearchJob) RuntimeFilter(Filter SearchFilter) { job.ResultSync.Lock() defer job.ResultSync.Unlock() + job.filtersRuntime = Filter + // FreezeFiles and current offset is reset job.FreezeFiles = nil job.currentOffset = 0 - job.filtersRuntime.Sort = Sort - job.filtersRuntime.FileType = FileType - job.filtersRuntime.FileFormat = FileFormat - - if !DateFrom.IsZero() && !DateTo.IsZero() { - job.filtersRuntime.DateFrom = DateFrom - job.filtersRuntime.DateTo = DateTo - job.filtersRuntime.IsDates = true - } else { - job.filtersRuntime.DateFrom = time.Time{} - job.filtersRuntime.DateTo = time.Time{} - job.filtersRuntime.IsDates = false - } - // files remain in AllFiles, but Files needs to be filtered based on the new filter job.Files = []*apiFile{} job.requireSort = false // Sorting is done immediately below @@ -246,6 +226,10 @@ func (job *SearchJob) isFileFiltered(file *apiFile) bool { return false } + if job.filtersRuntime.SizeMin >= 0 && file.Size < uint64(job.filtersRuntime.SizeMin) || job.filtersRuntime.SizeMax >= 0 && file.Size > uint64(job.filtersRuntime.SizeMax) { + return false + } + return true } diff --git a/webapi/Search Temp.go b/webapi/Search Temp.go index 8d469b5..ce0bf13 100644 --- a/webapi/Search Temp.go +++ b/webapi/Search Temp.go @@ -19,10 +19,11 @@ import ( ) func dispatchSearch(input SearchRequest) (job *SearchJob) { - Timeout, FileType, FileFormat, DateFrom, DateTo := input.Parse() + Timeout := input.Parse() + Filter := input.ToSearchFilter() // create the search job - job = CreateSearchJob(Timeout, input.MaxResults, input.Sort, FileType, FileFormat, DateFrom, DateTo) + job = CreateSearchJob(Timeout, input.MaxResults, Filter) // Create test data // * Between 0-100 results diff --git a/webapi/Search.go b/webapi/Search.go index c8ebf34..6bb4893 100644 --- a/webapi/Search.go +++ b/webapi/Search.go @@ -34,6 +34,8 @@ type SearchRequest struct { TerminateID []uuid.UUID `json:"terminate"` // Optional: Previous search IDs to terminate. This is if the user makes a new search from the same tab. Same as first calling /search/terminate. FileType int `json:"filetype"` // File type such as binary, text document etc. See core.TypeX. -1 = not used. FileFormat int `json:"fileformat"` // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 = not used. + SizeMin int `json:"sizemin"` // Min file size in bytes. -1 = not used. + SizeMax int `json:"sizemax"` // Max file size in bytes. -1 = not used. } // Sort orders @@ -109,6 +111,8 @@ Optional parameters: &filetype=[File Type] &fileformat=[File Format] &from=[Date From]&to=[Date To] + &sizemin=[Minimum file size] + &sizemax=[Maximum file size] &sort=[sort order] Result: 200 with JSON structure SearchResult. Check the field status. */ @@ -124,14 +128,6 @@ func apiSearchResult(w http.ResponseWriter, r *http.Request) { limit = 100 } - // filters and sort parameter - filterReset, _ := strconv.ParseBool(r.Form.Get("reset")) - fileType, _ := strconv.Atoi(r.Form.Get("filetype")) - fileFormat, _ := strconv.Atoi(r.Form.Get("fileformat")) - dateFrom, _ := time.Parse(apiDateFormat, r.Form.Get("from")) - dateTo, _ := time.Parse(apiDateFormat, r.Form.Get("to")) - sort, _ := strconv.Atoi(r.Form.Get("sort")) - // find the job ID job := JobLookup(jobID) if job == nil { @@ -139,9 +135,19 @@ func apiSearchResult(w http.ResponseWriter, r *http.Request) { return } - // apply runtime filters, if any - if filterReset { - job.RuntimeFilter(sort, fileType, fileFormat, dateFrom, dateTo) + // filters and sort parameter + if filterReset, _ := strconv.ParseBool(r.Form.Get("reset")); filterReset { + fileType, _ := strconv.Atoi(r.Form.Get("filetype")) + fileFormat, _ := strconv.Atoi(r.Form.Get("fileformat")) + dateFrom := r.Form.Get("from") + dateTo := r.Form.Get("to") + sort, _ := strconv.Atoi(r.Form.Get("sort")) + sizeMin, _ := strconv.Atoi(r.Form.Get("sizemin")) + sizeMax, _ := strconv.Atoi(r.Form.Get("sizemax")) + + filter := inputToSearchFilter(sort, fileType, fileFormat, dateFrom, dateTo, sizeMin, sizeMax) + + job.RuntimeFilter(filter) } // query all results @@ -293,18 +299,35 @@ func apiExplore(w http.ResponseWriter, r *http.Request) { EncodeJSON(w, r, result) } -func (input *SearchRequest) Parse() (Timeout time.Duration, FileType, FileFormat int, DateFrom, DateTo time.Time) { +func (input *SearchRequest) Parse() (Timeout time.Duration) { if input.Timeout == 0 { Timeout = time.Second * 20 // default timeout: 20 seconds } else { Timeout = time.Duration(input.Timeout) * time.Second } - FileType = input.FileType - FileFormat = input.FileFormat + return +} - DateFrom, _ = time.Parse(apiDateFormat, input.DateFrom) - DateTo, _ = time.Parse(apiDateFormat, input.DateTo) +// ToSearchFilter converts the user input to a valid search filter +func (input *SearchRequest) ToSearchFilter() (output SearchFilter) { + return inputToSearchFilter(input.Sort, input.FileType, input.FileFormat, input.DateFrom, input.DateTo, input.SizeMin, input.SizeMax) +} + +func inputToSearchFilter(Sort, FileType, FileFormat int, DateFrom, DateTo string, SizeMin, SizeMax int) (output SearchFilter) { + output.Sort = Sort + output.FileType = FileType + output.FileFormat = FileFormat + output.SizeMin = SizeMin + output.SizeMax = SizeMax + + dateFrom, errFrom := time.Parse(apiDateFormat, DateFrom) + dateTo, errTo := time.Parse(apiDateFormat, DateTo) + if errFrom == nil && errTo == nil && dateFrom.Before(dateTo) { + output.DateFrom = dateFrom + output.DateTo = dateTo + output.IsDates = true + } return } diff --git a/webapi/readme.md b/webapi/readme.md index 1d3f21b..3706ebe 100644 --- a/webapi/readme.md +++ b/webapi/readme.md @@ -521,6 +521,8 @@ The following filters are supported: ### Submitting a Search Request +This starts a search request and returns an ID that can be used to collect the results asynchronously. Note that some of the filters described below (such as `filetype`) must be set to -1 if they are not used. + ``` Request: POST /search with JSON SearchRequest Response: 200 on success with JSON SearchRequestResponse @@ -537,6 +539,8 @@ type SearchRequest struct { TerminateID []uuid.UUID `json:"terminate"` // Optional: Previous search IDs to terminate. This is if the user makes a new search from the same tab. Same as first calling /search/terminate. FileType int `json:"filetype"` // File type such as binary, text document etc. See core.TypeX. -1 = not used. FileFormat int `json:"fileformat"` // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 = not used. + SizeMin int `json:"sizemin"` // Min file size in bytes. -1 = not used. + SizeMax int `json:"sizemax"` // Max file size in bytes. -1 = not used. } type SearchRequestResponse struct { @@ -551,7 +555,12 @@ Example POST request to `http://127.0.0.1:112/search`: { "term": "Test Search", "timeout": 10, - "maxresults": 1000 + "maxresults": 1000, + "sort": 0, + "filetype": -1, + "fileformat": -1, + "sizemin": -1, + "sizemax": -1 } ``` @@ -567,7 +576,8 @@ Example response: ### Returning Search Results This function returns search results. The default limit is 100. -If reset is set, all results will be filtered and sorted according to the settings. This means that the new first result will be returned again and internal result offset is set to 0. + +If reset is set, all results will be filtered and sorted according to the provided parameters. This means that the new first result will be returned again and internal result offset is set to 0. Note that most filters must be set to -1 if they are not used (see the field comments in the `SearchRequest` structure in `/search` above). ``` Request: GET /search/result?id=[UUID]&limit=[max records] @@ -576,6 +586,8 @@ Request: GET /search/result?id=[UUID]&limit=[max records] &filetype=[File Type] &fileformat=[File Format] &from=[Date From]&to=[Date To] + &sizemin=[Minimum file size] + &sizemax=[Maximum file size] &sort=[sort order] Result: 200 with JSON structure SearchResult. Check the field status. ```