ltsv2avro

README

go-ltsv2avro

LTSV log lines -> AVRO records

Documentation

Index

• Constants
• Variables
• type AvroConfig
• type Codec
• type Config
• type DecodeConfig
• type EncodeConfig
• type Label
• type LabelConfig
  • func (l LabelConfig) AddTag(tag string, m map[string]any) error
  • func (l LabelConfig) ToDeleteFunc() func(string, any) bool
• type Level
• type LtsvConfig
  • func (l LtsvConfig) AnyToBodyString(a any) (string, error)
  • func (l LtsvConfig) ToLtsvToMap() LtsvToMap
• type LtsvItem
• type LtsvRow
• type LtsvToMap
  • func (c LtsvToMap) RowsToMaps(ltsv iter.Seq2[LtsvRow, error]) iter.Seq2[map[string]any, error]
• type Separator
  • func (s Separator) ToStringToStrings() StringToStrings
• type StringToItems
  • func (c StringToItems) InputsToLtsvRows(i iter.Seq2[string, error]) iter.Seq2[LtsvRow, error]
• type StringToLevel
  • func (c StringToLevel) AnyToLevel(a any) Level
  • func (c StringToLevel) ToUpper() StringToLevel
• type StringToLevelMap
  • func (m StringToLevelMap) ToStringToLevel() StringToLevel
• type StringToStrings
  • func (c StringToStrings) ToStringToItems(sep string) StringToItems
• type StringToTimestamp
  • func (c StringToTimestamp) AnyToTimestamp(a any) (Timestamp, error)
  • func (c StringToTimestamp) Or(alt StringToTimestamp) StringToTimestamp
• type Timestamp
• type TimestampFormat
  • func (t TimestampFormat) ToStringToTimestamp() StringToTimestamp

Constants

const BlobSizeMaxDefault int = 1048576

const BlockLengthDefault int = 100

const LabelValueSeparatorDefault string = ":"

Variables

var (
	ErrInvalidTimestamp error = errors.New("invalid timestamp")
	ErrInvalidBody      error = errors.New("invalid body")

	ErrInvalidLtsv error = errors.New("invalid ltsv")

	ErrInvalidTags error = errors.New("invalid tag type")
)

Types

type AvroConfig

type AvroConfig struct {
	DecodeConfig
	EncodeConfig
}

var AvroConfigDefault AvroConfig = AvroConfig{
	DecodeConfig: DecodeConfigDefault,
	EncodeConfig: EncodeConfigDefault,
}

type Codec

type Codec string

const (
	CodecNull    Codec = "null"
	CodecDeflate Codec = "deflate"
	CodecSnappy  Codec = "snappy"
	CodecZstd    Codec = "zstandard"
	CodecBzip2   Codec = "bzip2"
	CodecXz      Codec = "xz"
)

type Config

type Config struct {
	AvroConfig
	LtsvConfig
}

var ConfigDefault Config = Config{
	AvroConfig: AvroConfigDefault,
	LtsvConfig: LtsvConfigDefault,
}

type DecodeConfig

type DecodeConfig struct{ BlobSizeMax int }

var DecodeConfigDefault DecodeConfig = DecodeConfig{
	BlobSizeMax: BlobSizeMaxDefault,
}

type EncodeConfig

type EncodeConfig struct {
	BlockLength int
	Codec
}

var EncodeConfigDefault EncodeConfig = EncodeConfig{
	BlockLength: BlockLengthDefault,
	Codec:       CodecNull,
}

type Label

type Label string

const (
	LabelTimestampDefault Label = "timestamp"

	LabelLevelDefault    Label = "level"
	LabelSeverityDefault Label = "severity"

	LabelMessageDefault Label = "message"
	LabelBodyDefault    Label = "body"

	LabelTagDefault Label = "tag"

	LabelAttributesDefault Label = "attributes"
)

type LabelConfig

type LabelConfig struct {
	TimestampLabel Label
	SeverityLabel  Label
	BodyLabel      Label

	AttributesLabel Label

	TagLabel Label
}

var LabelConfigDefault LabelConfig = LabelConfig{
	TimestampLabel: LabelTimestampDefault,
	SeverityLabel:  LabelSeverityDefault,
	BodyLabel:      LabelBodyDefault,

	AttributesLabel: LabelAttributesDefault,

	TagLabel: LabelTagDefault,
}

func (LabelConfig) AddTag

func (l LabelConfig) AddTag(tag string, m map[string]any) error

func (LabelConfig) ToDeleteFunc

func (l LabelConfig) ToDeleteFunc() func(string, any) bool

type Level

type Level string

const (
	LevelUnspecified Level = "LEVEL_UNSPECIFIED"
	LevelTrace       Level = "LEVEL_TRACE"
	LevelDebug       Level = "LEVEL_DEBUG"
	LevelInfo        Level = "LEVEL_INFO"
	LevelWarn        Level = "LEVEL_WARN"
	LevelError       Level = "LEVEL_ERROR"
	LevelFatal       Level = "LEVEL_FATAL"
)

type LtsvConfig

type LtsvConfig struct {
	LabelConfig
	StringToLevel
	StringToTimestamp
}

var LtsvConfigDefault LtsvConfig = LtsvConfig{
	LabelConfig:       LabelConfigDefault,
	StringToLevel:     StrToLevelDefault,
	StringToTimestamp: StrToTimeDefault,
}

func (LtsvConfig) AnyToBodyString

func (l LtsvConfig) AnyToBodyString(a any) (string, error)

func (LtsvConfig) ToLtsvToMap

func (l LtsvConfig) ToLtsvToMap() LtsvToMap

type LtsvItem

type LtsvItem struct {
	Label string
	Value string
}

type LtsvRow

type LtsvRow []LtsvItem

type LtsvToMap

type LtsvToMap func(
	input LtsvRow,
	output map[string]any,
	attr map[string]any,
) error

func (LtsvToMap) RowsToMaps

func (c LtsvToMap) RowsToMaps(
	ltsv iter.Seq2[LtsvRow, error],
) iter.Seq2[map[string]any, error]

type Separator

type Separator string

const SeparatorDefault Separator = "\t"

func (Separator) ToStringToStrings

func (s Separator) ToStringToStrings() StringToStrings

type StringToItems

type StringToItems func(input string, output []LtsvItem) ([]LtsvItem, error)

var StrToItemsDefault StringToItems = StrToStringsDefault.
	ToStringToItems(LabelValueSeparatorDefault)

func (StringToItems) InputsToLtsvRows

func (c StringToItems) InputsToLtsvRows(
	i iter.Seq2[string, error],
) iter.Seq2[LtsvRow, error]

type StringToLevel

type StringToLevel func(string) Level

var StrToLevelDefault StringToLevel = StrToLevelMap.
	ToStringToLevel().
	ToUpper()

func (StringToLevel) AnyToLevel

func (c StringToLevel) AnyToLevel(a any) Level

func (StringToLevel) ToUpper

func (c StringToLevel) ToUpper() StringToLevel

type StringToLevelMap

type StringToLevelMap map[string]Level

var StrToLevelMap StringToLevelMap = maps.Collect(
	func(yield func(string, Level) bool,
	) {
		yield("TRACE", LevelTrace)
		yield("DEBUG", LevelDebug)
		yield("INFO", LevelInfo)
		yield("WARN", LevelWarn)
		yield("ERROR", LevelError)
		yield("FATAL", LevelFatal)
	})

func (StringToLevelMap) ToStringToLevel

func (m StringToLevelMap) ToStringToLevel() StringToLevel

type StringToStrings

type StringToStrings func(string) []string

var StrToStringsDefault StringToStrings = SeparatorDefault.ToStringToStrings()

func (StringToStrings) ToStringToItems

func (c StringToStrings) ToStringToItems(sep string) StringToItems

type StringToTimestamp

type StringToTimestamp func(string) (Timestamp, error)

var StrToTimeDefault StringToTimestamp = TimestampFormatRFC3339Nano.
	ToStringToTimestamp().
	Or(TimestampFormatDateTime.ToStringToTimestamp())

func (StringToTimestamp) AnyToTimestamp

func (c StringToTimestamp) AnyToTimestamp(a any) (Timestamp, error)

func (StringToTimestamp) Or

func (c StringToTimestamp) Or(alt StringToTimestamp) StringToTimestamp

type Timestamp

type Timestamp time.Time

type TimestampFormat

type TimestampFormat string

const (
	TimestampFormatRFC3339Nano TimestampFormat = time.RFC3339Nano
	TimestampFormatDateTime    TimestampFormat = time.DateTime
)

func (TimestampFormat) ToStringToTimestamp

func (t TimestampFormat) ToStringToTimestamp() StringToTimestamp