From dc093f5f6d25058d3b2d25bacefc56499be1f0fc Mon Sep 17 00:00:00 2001 From: Kleissner Date: Sun, 26 Sep 2021 23:40:22 +0200 Subject: [PATCH] webapi: Add file type and format detection via /file/format --- File Formats.go | 5 +- webapi/API.go | 1 + webapi/File Detection.go | 221 +++++++++++++++++++++++++++++++++++++++ webapi/Profile.go | 2 +- webapi/readme.md | 45 +++++++- 5 files changed, 268 insertions(+), 6 deletions(-) create mode 100644 webapi/File Detection.go diff --git a/File Formats.go b/File Formats.go index d762461..3a2d074 100644 --- a/File Formats.go +++ b/File Formats.go @@ -17,7 +17,7 @@ const ( TypeAudio // Audio TypeDocument // Any document file, including office documents, PDFs, power point, spreadsheets TypeExecutable // Any executable file, OS independent - TypeContainer // Container files like ZIP, RAR, TAR + TypeContainer // Container files like ZIP, RAR, TAR, ISO TypeCompressed // Compressed files like GZ, BZ TypeFolder // Virtual folder TypeEbook // Ebook @@ -43,6 +43,9 @@ const ( FormatCSV // CSV file FormatFolder // Virtual folder FormatExecutable // Executable file + FormatInstaller // Installer + FormatAPK // APK + FormatISO // ISO ) // Future tags to be defined for audio/video: Artist, Album, Title, Length, Bitrate, Codec diff --git a/webapi/API.go b/webapi/API.go index 114ea6f..f64a6af 100644 --- a/webapi/API.go +++ b/webapi/API.go @@ -57,6 +57,7 @@ func Start(ListenAddresses []string, UseSSL bool, CertificateFile, CertificateKe //Router.HandleFunc("/search/result/ws", apiSearchResultStream).Methods("GET") Router.HandleFunc("/search/terminate", apiSearchTerminate).Methods("GET") Router.HandleFunc("/explore", apiExplore).Methods("GET") + Router.HandleFunc("/file/format", apiFileFormat).Methods("GET") for _, listen := range ListenAddresses { go startWebServer(listen, UseSSL, CertificateFile, CertificateKey, Router, "API", TimeoutRead, TimeoutWrite) diff --git a/webapi/File Detection.go b/webapi/File Detection.go new file mode 100644 index 0000000..d72036e --- /dev/null +++ b/webapi/File Detection.go @@ -0,0 +1,221 @@ +/* +File Name: File Detection.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "net/http" + "os" + "path" + "strings" + + "github.com/PeernetOfficial/core" +) + +// PathToExtension translates a path to a file extension, if possible. It also returns the second file extension if there is one (relevant for files like "test.tar.gz"). +func PathToExtension(Path string) (extension, extension2 string, valid bool) { + _, fileA := path.Split(Path) + + parts := strings.Split(fileA, ".") + if len(parts) <= 1 { + return "", "", false + } + extension = parts[len(parts)-1] + extension = strings.ToLower(extension) + + if len(parts) >= 3 { + extension2 = parts[len(parts)-2] + extension2 = strings.ToLower(extension2) + } + + return extension, extension2, true +} + +// FileTranslateExtension translates the extension to a File Type and File Format. If invalid, types are 0. +func FileTranslateExtension(extension string) (fileType, fileFormat uint16) { + switch extension { + case "txt", "log", "ini", "json", "md": + return core.TypeText, core.FormatText + + case "csv", "tsv": + return core.TypeText, core.FormatCSV + + case "html", "htm": + return core.TypeText, core.FormatHTML + + case "doc", "docx", "rtf", "odt": + return core.TypeDocument, core.FormatWord + + case "pdf": + return core.TypeDocument, core.FormatPDF + + case "xls", "xlsx", "ods": + return core.TypeDocument, core.FormatExcel + + case "gif", "jpg", "jpeg", "png", "svg", "bmp", "tif", "tiff", "jfif": + return core.TypePicture, core.FormatPicture + + case "mp4", "flv", "avi", "mov", "mpg", "mpeg", "h264", "3g2", "3gp", "mkv", "wmv", "webm": + return core.TypeVideo, core.FormatVideo + + case "mp3", "ogg", "flac": + return core.TypeAudio, core.FormatAudio + + case "zip", "rar", "7z", "tar": + return core.TypeContainer, core.FormatContainer + + case "ppt", "pptx", "odp": + return core.TypeDocument, core.FormatPowerpoint + + case "epub", "mobi", "prc": + return core.TypeEbook, core.FormatEbook + + case "gz", "bz", "bz2", "xz": + return core.TypeCompressed, core.FormatCompressed + + case "sql": + return core.TypeText, core.FormatDatabase + + case "eml", "mbox": + return core.TypeText, core.FormatEmail + + case "exe", "sys", "dll", "cmd", "bat": + return core.TypeExecutable, core.FormatExecutable + + case "msi": + return core.TypeExecutable, core.FormatInstaller + + case "apk": + return core.TypeExecutable, core.FormatAPK + + case "iso": + return core.TypeContainer, core.FormatISO + + default: + return core.TypeBinary, core.FormatBinary + } +} + +// HTTPContentTypeToCore translates the HTTP content type to the File Type and File Format used by the core package. +func HTTPContentTypeToCore(httpContentType string) (fileType, fileFormat uint16) { + switch httpContentType { + case "text/html", "application/xhtml+xml", "application/xml": + return core.TypeText, core.FormatHTML + + case "text/plain": + return core.TypeText, core.FormatText + + case "text/csv", "text/tsv", "text/tab-separated-values", "text/x-csv", "application/csv", "application/x-csv", "text/x-comma-separated-values": + return core.TypeText, core.FormatCSV + + case "application/pdf", "application/x-pdf": + return core.TypeDocument, core.FormatPDF + + case "application/msword", "application/rtf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.oasis.opendocument.text": + return core.TypeDocument, core.FormatWord + + case "application/excel", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.oasis.opendocument.spreadsheet": + return core.TypeDocument, core.FormatExcel + + case "application/vnd.ms-powerpoint", "application/vnd.openxmlformats-officedocument.presentationml.presentation", "application/vnd.oasis.opendocument.presentation": + return core.TypeDocument, core.FormatPowerpoint + + case "image/png", "image/jpeg", "image/gif", "image/svg+xml", "image/tiff", "image/webp", "image/bmp", "image/x-bmp", "image/x-windows-bmp": // image/x-icon excluded + return core.TypePicture, core.FormatPicture + + case "audio/aac", "audio/midi", "audio/ogg", "audio/x-wav", "audio/webm", "audio/3gpp", "audio/3gpp2", "audio/mpeg", "audio/vorbis": + return core.TypeAudio, core.FormatAudio + + case "video/x-msvideo", "video/x-ms-wmv", "video/mpeg", "video/ogg", "video/webm", "video/3gpp", "video/3gpp2", "video/x-flv", "video/mp4": + return core.TypeVideo, core.FormatVideo + + case "application/zip", "application/x-rar-compressed", "application/x-tar", "application/x-bzip", "application/x-bzip2", "application/x-7z-compressed": + return core.TypeContainer, core.FormatContainer + + case "application/epub+zip", "application/x-mobipocket-ebook": + return core.TypeEbook, core.FormatEbook + + default: + return core.TypeBinary, core.FormatBinary + } +} + +// FileDataToHTTPContentType returns the HTTP content type based on the initial file data. It reads the first 512 bytes of the file. +func FileDataToHTTPContentType(Path string) (httpContentType string, err error) { + file, err := os.Open(Path) + if err != nil { + return "", err + } + + // Read up to 512 bytes. This specific number comes from http.DetectContentType which specifies it as constant "sniffLen". + buff := make([]byte, 512) + + if _, err := file.Read(buff); err != nil { + return "", err + } + + if err := file.Close(); err != nil { + return "", err + } + + httpContentType = http.DetectContentType(buff) + + // sanitize it first + httpContentType = strings.ToLower(strings.TrimSpace(httpContentType)) + if indexD := strings.IndexAny(httpContentType, ";"); indexD >= 0 { + httpContentType = httpContentType[:indexD] + } + + return httpContentType, nil +} + +// FileDetectType detects the File Type and File Format of a file. It uses the extension if available, otherwise the file data, for detection. +func FileDetectType(Path string) (fileType, fileFormat uint16, err error) { + // If a file extension is available, use that to detect the file type and file format. + // Otherwise, use the initial file data for detection. + if ext1, _, valid := PathToExtension(Path); valid { + fileType, fileFormat = FileTranslateExtension(ext1) + return fileType, fileFormat, nil + } + + httpContentType, err := FileDataToHTTPContentType(Path) + if err != nil { + return core.TypeBinary, core.FormatBinary, err + } + + fileType, fileFormat = HTTPContentTypeToCore(httpContentType) + return fileType, fileFormat, nil +} + +type apiResponseFileFormat struct { + Status int `json:"status"` // Status: 0 = Success, 1 = Error reading file + FileType uint16 `json:"filetype"` // File Type. + FileFormat uint16 `json:"fileformat"` // File Format. +} + +/* +apiFileFormat detects the file type and file format of the specified file. +It will primarily use the file extension for detection. If unavailable, it uses the first 512 bytes of the file data to detect the type. + +Request: GET /file/format?path=[file path on disk] +Result: 200 with JSON structure apiResponseFileFormat +*/ +func apiFileFormat(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + filePath := r.Form.Get("path") + if filePath == "" { + http.Error(w, "", http.StatusBadRequest) + return + } + + fileType, fileFormat, err := FileDetectType(filePath) + if err != nil { + EncodeJSON(w, r, apiResponseFileFormat{Status: 1}) + return + } + + EncodeJSON(w, r, apiResponseFileFormat{Status: 0, FileType: fileType, FileFormat: fileFormat}) +} diff --git a/webapi/Profile.go b/webapi/Profile.go index 607562f..33936a3 100644 --- a/webapi/Profile.go +++ b/webapi/Profile.go @@ -75,7 +75,7 @@ func apiProfileRead(w http.ResponseWriter, r *http.Request) { } /* -apiProfileWrite writes a specific users profile field or blob. +apiProfileWrite writes profile fields or blobs. For the index see core.ProfileFieldX and core.ProfileBlobX constants. Request: POST /profile/write with JSON structure apiProfileData diff --git a/webapi/readme.md b/webapi/readme.md index 78c45c4..9f63d9e 100644 --- a/webapi/readme.md +++ b/webapi/readme.md @@ -35,18 +35,20 @@ These are the functions provided by the API: /blockchain/self/read Read a block of the blockchain /blockchain/self/add/file Add file to the blockchain /blockchain/self/list/file List all files stored on the blockchain -/blockchain/self/delete/file Deletes files from the blockchain +/blockchain/self/delete/file Delete files from the blockchain /profile/list List all users profile fields and blobs -/profile/read Reads a specific users profile field or blob -/profile/write Writes a specific users profile field or blob -/profile/delete Deletes profile fields or blobs +/profile/read Read a profile field or blob +/profile/write Write profile fields or blobs +/profile/delete Delete profile fields or blobs /search Submit a search request /search/result Return search results /search/terminate Terminate a search /explore List recently shared files + +/file/format Detect file type and format ``` # API Documentation @@ -420,6 +422,8 @@ Example response: ### Profile Write +This writes profile fields and blobs. It can write multiple fields and blobs at once. + ``` Request: POST /profile/write with JSON structure apiProfileData Response: 200 with JSON structure apiBlockchainBlockStatus @@ -589,3 +593,36 @@ Result: 200 with JSON structure SearchResult. Check the field status. Example request to list 20 recently shared files (all file types): `http://127.0.0.1:112/explore&limit=20` Example request to list 10 recent documents: `http://127.0.0.1:112/explore?type=5&limit=10` + +## Helper Functions + +These helper functions are usually not needed, but can be useful in special cases. + +### Detect file type and file format + +This function detects the file type and file format of the specified file. It will primarily use the file extension for detection. If unavailable, it uses the first 512 bytes of the file data to detect the type. + +``` +Request: GET /file/format?path=[file path on disk] +Result: 200 with JSON structure apiResponseFileFormat +``` + +```go +type apiResponseFileFormat struct { + Status int `json:"status"` // Status: 0 = Success, 1 = Error reading file + FileType uint16 `json:"filetype"` // File Type. + FileFormat uint16 `json:"fileformat"` // File Format. +} +``` + +Example request: `http://127.0.0.1:112/file/format?path=test.txt` + +Example response: + +```json +{ + "status": 0, + "filetype": 1, + "fileformat": 10 +} +```