Peernet-Archive/core
1
0
Fork 0
mirror of https://github.com/PeernetOfficial/core.git synced 2026-07-17 02:47:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

webapi: Add file type and format detection via /file/format

Browse Source
This commit is contained in:
Kleissner
2021-09-26 23:40:22 +02:00
parent 3fbb1a824b
commit dc093f5f6d
5 changed files with 268 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
File Formats.go
View File

@@ -17,7 +17,7 @@ const (
TypeAudio // Audio
TypeDocument // Any document file, including office documents, PDFs, power point, spreadsheets
TypeExecutable // Any executable file, OS independent
TypeContainer // Container files like ZIP, RAR, TAR
TypeContainer // Container files like ZIP, RAR, TAR, ISO
TypeCompressed // Compressed files like GZ, BZ
TypeFolder // Virtual folder
TypeEbook // Ebook
@@ -43,6 +43,9 @@ const (
FormatCSV // CSV file
FormatFolder // Virtual folder
FormatExecutable // Executable file
FormatInstaller // Installer
FormatAPK // APK
FormatISO // ISO
)
// Future tags to be defined for audio/video: Artist, Album, Title, Length, Bitrate, Codec

1
webapi/API.go
View File

@@ -57,6 +57,7 @@ func Start(ListenAddresses []string, UseSSL bool, CertificateFile, CertificateKe
//Router.HandleFunc("/search/result/ws", apiSearchResultStream).Methods("GET")
Router.HandleFunc("/search/terminate", apiSearchTerminate).Methods("GET")
Router.HandleFunc("/explore", apiExplore).Methods("GET")
Router.HandleFunc("/file/format", apiFileFormat).Methods("GET")
for _, listen := range ListenAddresses {
go startWebServer(listen, UseSSL, CertificateFile, CertificateKey, Router, "API", TimeoutRead, TimeoutWrite)

221
webapi/File Detection.go Normal file
View File

@@ -0,0 +1,221 @@
/*
File Name: File Detection.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
*/
package webapi
import (
"net/http"
"os"
"path"
"strings"
"github.com/PeernetOfficial/core"
)
// PathToExtension translates a path to a file extension, if possible. It also returns the second file extension if there is one (relevant for files like "test.tar.gz").
func PathToExtension(Path string) (extension, extension2 string, valid bool) {
_, fileA := path.Split(Path)
parts := strings.Split(fileA, ".")
if len(parts) <= 1 {
return "", "", false
}
extension = parts[len(parts)-1]
extension = strings.ToLower(extension)
if len(parts) >= 3 {
extension2 = parts[len(parts)-2]
extension2 = strings.ToLower(extension2)
}
return extension, extension2, true
}
// FileTranslateExtension translates the extension to a File Type and File Format. If invalid, types are 0.
func FileTranslateExtension(extension string) (fileType, fileFormat uint16) {
switch extension {
case "txt", "log", "ini", "json", "md":
return core.TypeText, core.FormatText
case "csv", "tsv":
return core.TypeText, core.FormatCSV
case "html", "htm":
return core.TypeText, core.FormatHTML
case "doc", "docx", "rtf", "odt":
return core.TypeDocument, core.FormatWord
case "pdf":
return core.TypeDocument, core.FormatPDF
case "xls", "xlsx", "ods":
return core.TypeDocument, core.FormatExcel
case "gif", "jpg", "jpeg", "png", "svg", "bmp", "tif", "tiff", "jfif":
return core.TypePicture, core.FormatPicture
case "mp4", "flv", "avi", "mov", "mpg", "mpeg", "h264", "3g2", "3gp", "mkv", "wmv", "webm":
return core.TypeVideo, core.FormatVideo
case "mp3", "ogg", "flac":
return core.TypeAudio, core.FormatAudio
case "zip", "rar", "7z", "tar":
return core.TypeContainer, core.FormatContainer
case "ppt", "pptx", "odp":
return core.TypeDocument, core.FormatPowerpoint
case "epub", "mobi", "prc":
return core.TypeEbook, core.FormatEbook
case "gz", "bz", "bz2", "xz":
return core.TypeCompressed, core.FormatCompressed
case "sql":
return core.TypeText, core.FormatDatabase
case "eml", "mbox":
return core.TypeText, core.FormatEmail
case "exe", "sys", "dll", "cmd", "bat":
return core.TypeExecutable, core.FormatExecutable
case "msi":
return core.TypeExecutable, core.FormatInstaller
case "apk":
return core.TypeExecutable, core.FormatAPK
case "iso":
return core.TypeContainer, core.FormatISO
default:
return core.TypeBinary, core.FormatBinary
}
}
// HTTPContentTypeToCore translates the HTTP content type to the File Type and File Format used by the core package.
func HTTPContentTypeToCore(httpContentType string) (fileType, fileFormat uint16) {
switch httpContentType {
case "text/html", "application/xhtml+xml", "application/xml":
return core.TypeText, core.FormatHTML
case "text/plain":
return core.TypeText, core.FormatText
case "text/csv", "text/tsv", "text/tab-separated-values", "text/x-csv", "application/csv", "application/x-csv", "text/x-comma-separated-values":
return core.TypeText, core.FormatCSV
case "application/pdf", "application/x-pdf":
return core.TypeDocument, core.FormatPDF
case "application/msword", "application/rtf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.oasis.opendocument.text":
return core.TypeDocument, core.FormatWord
case "application/excel", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.oasis.opendocument.spreadsheet":
return core.TypeDocument, core.FormatExcel
case "application/vnd.ms-powerpoint", "application/vnd.openxmlformats-officedocument.presentationml.presentation", "application/vnd.oasis.opendocument.presentation":
return core.TypeDocument, core.FormatPowerpoint
case "image/png", "image/jpeg", "image/gif", "image/svg+xml", "image/tiff", "image/webp", "image/bmp", "image/x-bmp", "image/x-windows-bmp": // image/x-icon excluded
return core.TypePicture, core.FormatPicture
case "audio/aac", "audio/midi", "audio/ogg", "audio/x-wav", "audio/webm", "audio/3gpp", "audio/3gpp2", "audio/mpeg", "audio/vorbis":
return core.TypeAudio, core.FormatAudio
case "video/x-msvideo", "video/x-ms-wmv", "video/mpeg", "video/ogg", "video/webm", "video/3gpp", "video/3gpp2", "video/x-flv", "video/mp4":
return core.TypeVideo, core.FormatVideo
case "application/zip", "application/x-rar-compressed", "application/x-tar", "application/x-bzip", "application/x-bzip2", "application/x-7z-compressed":
return core.TypeContainer, core.FormatContainer
case "application/epub+zip", "application/x-mobipocket-ebook":
return core.TypeEbook, core.FormatEbook
default:
return core.TypeBinary, core.FormatBinary
}
}
// FileDataToHTTPContentType returns the HTTP content type based on the initial file data. It reads the first 512 bytes of the file.
func FileDataToHTTPContentType(Path string) (httpContentType string, err error) {
file, err := os.Open(Path)
if err != nil {
return "", err
}
// Read up to 512 bytes. This specific number comes from http.DetectContentType which specifies it as constant "sniffLen".
buff := make([]byte, 512)
if _, err := file.Read(buff); err != nil {
return "", err
}
if err := file.Close(); err != nil {
return "", err
}
httpContentType = http.DetectContentType(buff)
// sanitize it first
httpContentType = strings.ToLower(strings.TrimSpace(httpContentType))
if indexD := strings.IndexAny(httpContentType, ";"); indexD >= 0 {
httpContentType = httpContentType[:indexD]
}
return httpContentType, nil
}
// FileDetectType detects the File Type and File Format of a file. It uses the extension if available, otherwise the file data, for detection.
func FileDetectType(Path string) (fileType, fileFormat uint16, err error) {
// If a file extension is available, use that to detect the file type and file format.
// Otherwise, use the initial file data for detection.
if ext1, _, valid := PathToExtension(Path); valid {
fileType, fileFormat = FileTranslateExtension(ext1)
return fileType, fileFormat, nil
}
httpContentType, err := FileDataToHTTPContentType(Path)
if err != nil {
return core.TypeBinary, core.FormatBinary, err
}
fileType, fileFormat = HTTPContentTypeToCore(httpContentType)
return fileType, fileFormat, nil
}
type apiResponseFileFormat struct {
Status int `json:"status"` // Status: 0 = Success, 1 = Error reading file
FileType uint16 `json:"filetype"` // File Type.
FileFormat uint16 `json:"fileformat"` // File Format.
}
/*
apiFileFormat detects the file type and file format of the specified file.
It will primarily use the file extension for detection. If unavailable, it uses the first 512 bytes of the file data to detect the type.
Request: GET /file/format?path=[file path on disk]
Result: 200 with JSON structure apiResponseFileFormat
*/
func apiFileFormat(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
filePath := r.Form.Get("path")
if filePath == "" {
http.Error(w, "", http.StatusBadRequest)
return
}
fileType, fileFormat, err := FileDetectType(filePath)
if err != nil {
EncodeJSON(w, r, apiResponseFileFormat{Status: 1})
return
}
EncodeJSON(w, r, apiResponseFileFormat{Status: 0, FileType: fileType, FileFormat: fileFormat})
}

2
webapi/Profile.go
View File

@@ -75,7 +75,7 @@ func apiProfileRead(w http.ResponseWriter, r *http.Request) {
}
/*
apiProfileWrite writes a specific users profile field or blob.
apiProfileWrite writes profile fields or blobs.
For the index see core.ProfileFieldX and core.ProfileBlobX constants.
Request: POST /profile/write with JSON structure apiProfileData

45
webapi/readme.md
View File

@@ -35,18 +35,20 @@ These are the functions provided by the API:
/blockchain/self/read Read a block of the blockchain
/blockchain/self/add/file Add file to the blockchain
/blockchain/self/list/file List all files stored on the blockchain
/blockchain/self/delete/file Deletes files from the blockchain
/blockchain/self/delete/file Delete files from the blockchain
/profile/list List all users profile fields and blobs
/profile/read Reads a specific users profile field or blob
/profile/write Writes a specific users profile field or blob
/profile/delete Deletes profile fields or blobs
/profile/read Read a profile field or blob
/profile/write Write profile fields or blobs
/profile/delete Delete profile fields or blobs
/search Submit a search request
/search/result Return search results
/search/terminate Terminate a search
/explore List recently shared files
/file/format Detect file type and format
```
# API Documentation
@@ -420,6 +422,8 @@ Example response:
### Profile Write
This writes profile fields and blobs. It can write multiple fields and blobs at once.
```
Request: POST /profile/write with JSON structure apiProfileData
Response: 200 with JSON structure apiBlockchainBlockStatus
@@ -589,3 +593,36 @@ Result: 200 with JSON structure SearchResult. Check the field status.
Example request to list 20 recently shared files (all file types): `http://127.0.0.1:112/explore&limit=20`
Example request to list 10 recent documents: `http://127.0.0.1:112/explore?type=5&limit=10`
## Helper Functions
These helper functions are usually not needed, but can be useful in special cases.
### Detect file type and file format
This function detects the file type and file format of the specified file. It will primarily use the file extension for detection. If unavailable, it uses the first 512 bytes of the file data to detect the type.
```
Request: GET /file/format?path=[file path on disk]
Result: 200 with JSON structure apiResponseFileFormat
```
```go
type apiResponseFileFormat struct {
Status int `json:"status"` // Status: 0 = Success, 1 = Error reading file
FileType uint16 `json:"filetype"` // File Type.
FileFormat uint16 `json:"fileformat"` // File Format.
}
```
Example request: `http://127.0.0.1:112/file/format?path=test.txt`
Example response:
```json
{
"status": 0,
"filetype": 1,
"fileformat": 10
}
```