core/Block Records.go

/*
File Name:  Block Encoding.go
Copyright:  2021 Peernet s.r.o.
Author:     Peter Kleissner

This files defines the encoding of blocks and records within.

File records:
Offset  Size    Info
0       32      Hash blake3 of the file content
32      16      File ID
48      1       File Type (low-level)
49      2       File Format (high-level)
51      8       File Size
59      2       Count of Tags
61      ?       Tags

Each file tag provides additional optional information:
Offset  Size    Info
0       2       Type
2       4       Size of data that follows
6       ?       Data according to the tag type

Tag data record contains only raw data and may be referenced by Tags in File records.
This is a basic embedded way of compression when tags are repetitive in multiple files within the same block.

*/

package core

import (
	"encoding/binary"
	"errors"
	"math"
	"time"

	"github.com/PeernetOfficial/core/sanitize"
	"github.com/google/uuid"
)

// ---- Block record structures (decoded) ----

// RecordTypeX defines the type of the record
const (
	RecordTypeUsername      = 0 // Username. Arbitrary name defined by the user.
	RecordTypeTagData       = 1 // Tag data record to be referenced by one or multiple tags. Only valid in the context of the current block.
	RecordTypeFile          = 2 // File
	RecordTypeDelete        = 3 // Delete previous record by ID.
	RecordTypeCertificate   = 4 // Certificate to certify provided information in the blockchain issued by a trusted 3rd party.
	RecordTypeContentRating = 5 // Content rating (positive).
	RecordTypeContentReport = 6 // Content report (negative).
)

// BlockRecordUser specifies user information
type BlockRecordUser struct {
	Name          string // Arbitrary name of the user (original input).
	NameSanitized string // Sanitized version of the name.
}

// BlockRecordFile is the metadata of a file published on the blockchain
type BlockRecordFile struct {
	Hash        []byte               // Hash of the file data
	ID          uuid.UUID            // ID
	Type        uint8                // Type (low-level)
	Format      uint16               // Format (high-level)
	Size        uint64               // Size of the file data
	TagsRaw     []BlockRecordFileTag // Tags to provide additional metadata
	TagsDecoded []interface{}        // Decoded tags. See FileTagX structures.
}

// ---- Tag structures ----

// BlockRecordFileTag provides additional metadata about the file.
// New tags can be defines in the future without breaking support.
type BlockRecordFileTag struct {
	Type uint16 // Type (low-level). See TagTypeX constants.
	Data []byte // Actual data of the tag, decoded into FileTagX structures.

	// If top bit of Type is set, then Data must be 2, 4, or 8 bytes representing the distance number (positive or negative) of raw record in the block that will be used as data.
	// This is an embedded basic compression algorithm for repetitive tag. For example directory tags or album tags might be heavily repetitive among files.
}

// TagTypeName defines the type of the tag
const (
	TagTypeName        = 0 // Name of file
	TagTypeFolder      = 1 // Folder
	TagTypeDateCreated = 2 // Date when the file was originally created. This may differ from the date in the block record, which indicates when the file was shared.
	TagTypeDescription = 3 // Arbitrary description of the file. May contain hashtags.
)

// FileTagFolder specifies in which folder the file is stored.
// A corresponding TypeFolder file record matching the name may exist to provide additional details about the folder.
type FileTagFolder struct {
	Name string // Name of the folder
}

// FileTagName specifies the file name. Empty names are allowed, but not recommended!
type FileTagName struct {
	Name string // Name of the file
}

// FileTagDescription is an arbitrary of the description of the file. It may contain hashtags to help tagging and searching.
type FileTagDescription struct {
	Description string
}

// FileTagCreated is the date when the file was originally created
type FileTagCreated struct {
	Date time.Time // Created time
}

// ---- low-level encoding ----

// decodeBlockRecordUser decodes the username, if available. Otherwise return nil.
func decodeBlockRecordUser(recordsRaw []BlockRecordRaw) (user *BlockRecordUser) {
	for _, record := range recordsRaw {
		if record.Type == RecordTypeUsername {
			user = &BlockRecordUser{Name: string(record.Data), NameSanitized: sanitize.Username(string(record.Data))}
			// continue to seek for an overriding username record, even though that would be stupid within the same block
		}
	}
	return
}

// encodeBlockRecordUser encodes the username
func encodeBlockRecordUser(user BlockRecordUser) (recordsRaw []BlockRecordRaw, err error) {
	recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeUsername, Data: []byte(user.Name)})

	return recordsRaw, nil
}

// decodeBlockRecordFiles decodes only file records. Other records are ignored
func decodeBlockRecordFiles(recordsRaw []BlockRecordRaw) (files []BlockRecordFile, err error) {
	for i, record := range recordsRaw {
		switch record.Type {
		case RecordTypeFile:
			if len(record.Data) < 61 {
				return nil, errors.New("decodeBlockRecordFiles file record invalid size")
			}

			file := BlockRecordFile{}
			file.Hash = make([]byte, hashSize)
			copy(file.Hash, record.Data[0:0+hashSize])
			copy(file.ID[:], record.Data[32:32+16])
			file.Type = record.Data[48]
			file.Format = binary.LittleEndian.Uint16(record.Data[49 : 49+2])
			file.Size = binary.LittleEndian.Uint64(record.Data[51 : 51+8])

			countTags := binary.LittleEndian.Uint16(record.Data[59 : 59+2])

			index := 61

			for n := uint16(0); n < countTags; n++ {
				if index+6 > len(record.Data) {
					return nil, errors.New("decodeBlockRecordFiles file record tags invalid size")
				}

				tag := BlockRecordFileTag{}
				tag.Type = binary.LittleEndian.Uint16(record.Data[index:index+2]) & 0x7FFF
				tagSize := binary.LittleEndian.Uint32(record.Data[index+2 : index+2+4])
				isDataReference := record.Data[index+1]&0x80 != 0

				if index+6+int(tagSize) > len(record.Data) {
					return nil, errors.New("decodeBlockRecordFiles file record tag data invalid size")
				}

				if isDataReference { // reference to RecordTypeTagData record?
					var refRecordNumber int
					if tagSize == 2 {
						refRecordNumber = i + int(int16(binary.LittleEndian.Uint16(record.Data[index+6:index+6+2])))
					} else if tagSize == 4 {
						refRecordNumber = i + int(int32(binary.LittleEndian.Uint32(record.Data[index+6:index+6+4])))
					} else if tagSize == 8 {
						refRecordNumber = i + int(int64(binary.LittleEndian.Uint64(record.Data[index+6:index+6+8])))
					} else {
						return nil, errors.New("decodeBlockRecordFiles file record tag reference invalid size")
					}

					if refRecordNumber < 0 || refRecordNumber >= len(recordsRaw) {
						return nil, errors.New("decodeBlockRecordFiles file record tag reference not available")
					} else if recordsRaw[refRecordNumber].Type != RecordTypeTagData {
						return nil, errors.New("decodeBlockRecordFiles file record tag reference invalid")
					}

					tag.Data = recordsRaw[refRecordNumber].Data

				} else {
					tag.Data = record.Data[index+6 : index+6+int(tagSize)]
				}

				file.TagsRaw = append(file.TagsRaw, tag)

				if decoded, err := decodeFileTag(tag); err != nil {
					return nil, err
				} else if decoded != nil {
					file.TagsDecoded = append(file.TagsDecoded, decoded)
				}

				index += 6 + int(tagSize)
			}

			files = append(files, file)
		}
	}

	return files, err
}

// decodeFileTag decodes a file tag. If the tag type is not known, it returns nil.
func decodeFileTag(tag BlockRecordFileTag) (decoded interface{}, err error) {
	switch tag.Type {
	case TagTypeFolder:
		return FileTagFolder{Name: string(tag.Data)}, nil

	case TagTypeName:
		return FileTagName{Name: string(tag.Data)}, nil

	case TagTypeDescription:
		return FileTagDescription{Description: string(tag.Data)}, nil

	case TagTypeDateCreated:
		if len(tag.Data) != 8 {
			return nil, errors.New("decodeFileTag invalid date tag size")
		}

		timeB := int64(binary.LittleEndian.Uint64(tag.Data[0:8]))
		return FileTagCreated{Date: time.Unix(timeB, 0)}, nil

	}

	return nil, nil
}

// encodeFileTag encodes a file tag. If the tag type is not known, it returns nil.
func encodeFileTag(decoded interface{}) (tag BlockRecordFileTag, err error) {
	switch v := decoded.(type) {
	case FileTagFolder:
		return BlockRecordFileTag{Type: TagTypeFolder, Data: []byte(v.Name)}, nil

	case FileTagName:
		return BlockRecordFileTag{Type: TagTypeName, Data: []byte(v.Name)}, nil

	case FileTagDescription:
		return BlockRecordFileTag{Type: TagTypeDescription, Data: []byte(v.Description)}, nil

	case FileTagCreated:
		var tempDate [8]byte
		binary.LittleEndian.PutUint64(tempDate[0:8], uint64(v.Date.UTC().Unix()))
		return BlockRecordFileTag{Type: TagTypeDateCreated, Data: tempDate[:]}, nil

	}

	return tag, errors.New("encodeFileTag unknown tag type")
}

// encodeBlockRecordFiles encodes files into the block record data
// This function should be called grouped with all files in the same folder. The folder name is deduplicated; only unique folder records will be returned.
// Note that this function only stores the folder names as tags; it does not create separate TypeFolder file records.
func encodeBlockRecordFiles(files []BlockRecordFile) (recordsRaw []BlockRecordRaw, err error) {
	uniqueTagDataMap := make(map[string]struct{})
	duplicateTagDataMap := make(map[string]int) // list of tag data that appeared twice. Number in recordsRaw.

	// loop through all tags to encode them and create list of duplicates that will be replaced by references
	for n := range files {
		for m := range files[n].TagsDecoded {
			tag, err := encodeFileTag(files[n].TagsDecoded[m])
			if err != nil {
				return nil, err
			}
			files[n].TagsRaw = append(files[n].TagsRaw, tag)

			if len(tag.Data) > 4 {
				if _, ok := uniqueTagDataMap[string(tag.Data)]; !ok {
					uniqueTagDataMap[string(tag.Data)] = struct{}{}
				} else if _, ok := duplicateTagDataMap[string(tag.Data)]; !ok {
					recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeTagData, Data: tag.Data})
					duplicateTagDataMap[string(tag.Data)] = len(recordsRaw) - 1
				}
			}
		}
	}

	// then encode all files as records
	for n := range files {
		data := make([]byte, 61)

		if len(files[n].Hash) != hashSize {
			return nil, errors.New("encodeBlockRecords invalid file hash")
		}

		copy(data[0:32], files[n].Hash[0:32])
		copy(data[32:32+16], files[n].ID[:])

		data[48] = files[n].Type
		binary.LittleEndian.PutUint16(data[49:49+2], files[n].Format)
		binary.LittleEndian.PutUint64(data[51:51+8], files[n].Size)
		binary.LittleEndian.PutUint16(data[59:59+2], uint16(len(files[n].TagsRaw)))

		for _, tagRaw := range files[n].TagsRaw {
			if len(tagRaw.Data) > 4 {
				if refNumber, ok := duplicateTagDataMap[string(tagRaw.Data)]; ok {
					// In case the data is duplicated, use reference to the RecordTypeTagData instead
					tagRaw.Type |= 0x8000
					tagRaw.Data = intToBytes(-(len(recordsRaw) - refNumber))
				}
			}

			var tempTag [6]byte

			binary.LittleEndian.PutUint16(tempTag[0:2], tagRaw.Type)
			binary.LittleEndian.PutUint32(tempTag[2:2+4], uint32(len(tagRaw.Data)))

			data = append(data, tempTag[:]...)
			data = append(data, tagRaw.Data...)
		}

		recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeFile, Data: data})
	}

	return recordsRaw, nil
}

// intToBytes encodes int to little endian byte array as it fits to 16, 32 or 64 bit.
func intToBytes(number int) (buffer []byte) {
	buffer = make([]byte, 4)

	if number <= math.MaxInt16 && number >= math.MinInt16 {
		binary.LittleEndian.PutUint16(buffer[0:2], uint16(number))
		return buffer[0:2]
	} else if number <= math.MaxInt32 && number >= math.MinInt32 {
		binary.LittleEndian.PutUint32(buffer[0:4], uint32(number))
		return buffer[0:4]
	}

	binary.LittleEndian.PutUint64(buffer[0:8], uint64(number))
	return buffer[0:8]
}

// TagRawToText returns the tag text of the given tag, if available. Empty if not.
func (file *BlockRecordFile) TagRawToText(tagType uint16) string {
	for m := range file.TagsRaw {
		if file.TagsRaw[m].Type == tagType {
			return string(file.TagsRaw[m].Data)
		}
	}

	return ""
}

// ---- high-level decoding ----

// BlockDecoded contains the decoded records from a block
type BlockDecoded struct {
	Block
	RecordsDecoded []interface{} // Decoded records. See BlockRecordX structures.
}

// decodeBlockRecords decodes all raw records in the block and returns a high-level decoded structure
// Use decodeBlockRecordX instead for specific record decoding.
func decodeBlockRecords(block *Block) (decoded *BlockDecoded, err error) {
	decoded = &BlockDecoded{Block: *block}

	files, err := decodeBlockRecordFiles(block.RecordsRaw)
	if err != nil {
		return nil, err
	}

	for _, file := range files {
		decoded.RecordsDecoded = append(decoded.RecordsDecoded, file)
	}

	if user := decodeBlockRecordUser(block.RecordsRaw); user != nil {
		decoded.RecordsDecoded = append(decoded.RecordsDecoded, user)
	}

	return decoded, nil
}