tsm1

package
v1.5.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 25, 2018 License: MIT Imports: 46 Imported by: 90

Documentation

Overview

Package tsm1 provides a TSDB in the Time Structured Merge tree format.

Index

Constants

View Source 
const (
	// CompactionTempExtension is the extension used for temporary files created during compaction.
	CompactionTempExtension = "tmp"

	// TSMFileExtension is the extension used for TSM files.
	TSMFileExtension = "tsm"
)
View Source 
const (
	// BlockFloat64 designates a block encodes float64 values.
	BlockFloat64 = byte(0)

	// BlockInteger designates a block encodes int64 values.
	BlockInteger = byte(1)

	// BlockBoolean designates a block encodes boolean values.
	BlockBoolean = byte(2)

	// BlockString designates a block encodes string values.
	BlockString = byte(3)

	// BlockUnsigned designates a block encodes uint64 values.
	BlockUnsigned = byte(4)
)
View Source 
const (
	// The extension used to describe temporary snapshot files.
	TmpTSMFileExtension = "tmp"

	// The extension used to describe corrupt snapshot files.
	BadTSMFileExtension = "bad"
)
View Source 
const (
	// DefaultSegmentSize of 10MB is the size at which segment files will be rolled over.
	DefaultSegmentSize = 10 * 1024 * 1024

	// WALFileExtension is the file extension we expect for wal segments.
	WALFileExtension = "wal"

	// WALFilePrefix is the prefix on all wal segment files.
	WALFilePrefix = "_"
)
View Source 
const (
	// MagicNumber is written as the first 4 bytes of a data file to
	// identify the file as a tsm1 formatted file
	MagicNumber uint32 = 0x16D116D1

	// Version indicates the version of the TSM file format.
	Version byte = 1
)

Variables

View Source 
var (
	// ErrWALClosed is returned when attempting to write to a closed WAL file.
	ErrWALClosed = fmt.Errorf("WAL closed")

	// ErrWALCorrupt is returned when reading a corrupt WAL entry.
	ErrWALCorrupt = fmt.Errorf("corrupted WAL entry")
)
View Source 
var (
	//ErrNoValues is returned when TSMWriter.WriteIndex is called and there are no values to write.
	ErrNoValues = fmt.Errorf("no values written")

	// ErrTSMClosed is returned when performing an operation against a closed TSM file.
	ErrTSMClosed = fmt.Errorf("tsm file closed")

	// ErrMaxKeyLengthExceeded is returned when attempting to write a key that is too long.
	ErrMaxKeyLengthExceeded = fmt.Errorf("max key length exceeded")

	// ErrMaxBlocksExceeded is returned when attempting to write a block past the allowed number.
	ErrMaxBlocksExceeded = fmt.Errorf("max blocks exceeded")
)
View Source 
var ErrFileInUse = fmt.Errorf("file still in use")

ErrFileInUse is returned when attempting to remove or close a TSM file that is still being used.

View Source 
var (
	// ErrSnapshotInProgress is returned if a snapshot is attempted while one is already running.
	ErrSnapshotInProgress = fmt.Errorf("snapshot in progress")
)

Functions

func BlockCount added in v0.10.0 

func BlockCount(block []byte) int

BlockCount returns the number of timestamps encoded in block.

func BlockType added in v0.9.6 

func BlockType(block []byte) (byte, error)

BlockType returns the type of value encoded in a block or an error if the block type is unknown.

func BlockTypeToInfluxQLDataType added in v1.5.0 

func BlockTypeToInfluxQLDataType(typ byte) influxql.DataType

func CountTimestamps added in v0.10.0 

func CountTimestamps(b []byte) int

func Digest added in v1.5.0 

func Digest(dir string, w io.WriteCloser) error

Digest writes a digest of dir to w of a full shard dir.

func DigestWithOptions added in v1.5.0 

func DigestWithOptions(dir string, opts DigestOptions, w io.WriteCloser) error

DigestWithOptions writes a digest of dir to w using options to filter by time and key range.

func ErrCacheMemorySizeLimitExceeded added in v1.1.0 

func ErrCacheMemorySizeLimitExceeded(n, limit uint64) error

ErrCacheMemorySizeLimitExceeded returns an error indicating an operation could not be completed due to exceeding the cache-max-memory-size setting.

func MetricsGroupFromContext added in v1.5.0 

func MetricsGroupFromContext(ctx context.Context) *metrics.Group

MetricsGroupFromContext returns the tsm1 metrics.Group associated with the context or nil if no group has been assigned.

func NewContextWithMetricsGroup added in v1.5.0 

func NewContextWithMetricsGroup(ctx context.Context) context.Context

NewContextWithMetricsGroup creates a new context with a tsm1 metrics.Group for tracking various metrics when accessing TSM data.

func NewEngine 

func NewEngine(id uint64, idx tsdb.Index, database, path string, walPath string, sfile *tsdb.SeriesFile, opt tsdb.EngineOptions) tsdb.Engine

NewEngine returns a new instance of Engine.

func NewIndirectIndex 

func NewIndirectIndex() *indirectIndex

NewIndirectIndex returns a new indirect index.

func ParseTSMFileName added in v0.9.6 

func ParseTSMFileName(name string) (int, int, error)

ParseTSMFileName parses the generation and sequence from a TSM file name.

func SeriesAndFieldFromCompositeKey added in v1.0.0 

func SeriesAndFieldFromCompositeKey(key []byte) ([]byte, []byte)

SeriesAndFieldFromCompositeKey returns the series key and the field key extracted from the composite key.

func SeriesFieldKey 

func SeriesFieldKey(seriesKey, field string) string

SeriesFieldKey combine a series key and field name for a unique string to be hashed to a numeric ID.

func SeriesFieldKeyBytes added in v1.4.0 

func SeriesFieldKeyBytes(seriesKey, field string) []byte

func ZigZagDecode 

func ZigZagDecode(v uint64) int64

ZigZagDecode converts a previously zigzag encoded uint64 back to a int64.

func ZigZagEncode 

func ZigZagEncode(x int64) uint64

ZigZagEncode converts a int64 to a uint64 by zig zagging negative and positive values across even and odd numbers. Eg. [0,-1,1,-2] becomes [0, 1, 2, 3].

Types

type BatchDeleter added in v1.5.0 

type BatchDeleter interface {
	DeleteRange(keys [][]byte, min, max int64) error
	Commit() error
	Rollback() error
}

type BatchDeleters added in v1.5.0 

type BatchDeleters []BatchDeleter

func (BatchDeleters) Commit added in v1.5.0 

func (a BatchDeleters) Commit() error

func (BatchDeleters) DeleteRange added in v1.5.0 

func (a BatchDeleters) DeleteRange(keys [][]byte, min, max int64) error

func (BatchDeleters) Rollback added in v1.5.0 

func (a BatchDeleters) Rollback() error

type BitReader added in v0.13.0 

type BitReader struct {
	// contains filtered or unexported fields
}

BitReader reads bits from an io.Reader.

func NewBitReader added in v0.13.0 

func NewBitReader(data []byte) *BitReader

NewBitReader returns a new instance of BitReader that reads from data.

func (*BitReader) CanReadBitFast added in v0.13.0 

func (r *BitReader) CanReadBitFast() bool

CanReadBitFast returns true if calling ReadBitFast() is allowed. Fast bit reads are allowed when at least 2 values are in the buffer. This is because it is not required to refilled the buffer and the caller can inline the calls.

func (*BitReader) ReadBit added in v0.13.0 

func (r *BitReader) ReadBit() (bool, error)

ReadBit returns the next bit from the underlying data.

func (*BitReader) ReadBitFast added in v0.13.0 

func (r *BitReader) ReadBitFast() bool

ReadBitFast is an optimized bit read. IMPORTANT: Only allowed if CanReadFastBit() is true!

func (*BitReader) ReadBits added in v0.13.0 

func (r *BitReader) ReadBits(nbits uint) (uint64, error)

ReadBits reads nbits from the underlying data into a uint64. nbits must be from 1 to 64, inclusive.

func (*BitReader) Reset added in v0.13.0 

func (r *BitReader) Reset(data []byte)

Reset sets the underlying reader on b and reinitializes.

type BlockIterator added in v0.10.0 

type BlockIterator struct {
	// contains filtered or unexported fields
}

BlockIterator allows iterating over each block in a TSM file in order. It provides raw access to the block bytes without decoding them.

func (*BlockIterator) Err added in v1.4.0 

func (b *BlockIterator) Err() error

Err returns any errors encounter during iteration.

func (*BlockIterator) Next added in v0.10.0 

func (b *BlockIterator) Next() bool

Next returns true if there are more blocks to iterate through.

func (*BlockIterator) PeekNext added in v0.10.0 

func (b *BlockIterator) PeekNext() []byte

PeekNext returns the next key to be iterated or an empty string.

func (*BlockIterator) Read added in v0.10.0 

func (b *BlockIterator) Read() (key []byte, minTime int64, maxTime int64, typ byte, checksum uint32, buf []byte, err error)

Read reads information about the next block to be iterated.

type BooleanDecoder added in v0.11.0 

type BooleanDecoder struct {
	// contains filtered or unexported fields
}

BooleanDecoder decodes a series of booleans from an in-memory buffer.

func (*BooleanDecoder) Error added in v0.11.0 

func (e *BooleanDecoder) Error() error

Error returns the error encountered during decoding, if one occurred.

func (*BooleanDecoder) Next added in v0.11.0 

func (e *BooleanDecoder) Next() bool

Next returns whether there are any bits remaining in the decoder. It returns false if there was an error decoding. The error is available on the Error method.

func (*BooleanDecoder) Read added in v0.11.0 

func (e *BooleanDecoder) Read() bool

Read returns the next bit from the decoder.

func (*BooleanDecoder) SetBytes added in v0.13.0 

func (e *BooleanDecoder) SetBytes(b []byte)

SetBytes initializes the decoder with a new set of bytes to read from. This must be called before calling any other methods.

type BooleanEncoder added in v0.11.0 

type BooleanEncoder struct {
	// contains filtered or unexported fields
}

BooleanEncoder encodes a series of booleans to an in-memory buffer.

func NewBooleanEncoder added in v0.11.0 

func NewBooleanEncoder(sz int) BooleanEncoder

NewBooleanEncoder returns a new instance of BooleanEncoder.

func (*BooleanEncoder) Bytes added in v0.11.0 

func (e *BooleanEncoder) Bytes() ([]byte, error)

Bytes returns a new byte slice containing the encoded booleans from previous calls to Write.

func (*BooleanEncoder) Flush added in v1.2.3 

func (e *BooleanEncoder) Flush()

Flush is no-op

func (*BooleanEncoder) Reset added in v1.1.0 

func (e *BooleanEncoder) Reset()

Reset sets the encoder to its initial state.

func (*BooleanEncoder) Write added in v0.11.0 

func (e *BooleanEncoder) Write(b bool)

Write encodes b to the underlying buffer.

type BooleanValue added in v0.11.0 

type BooleanValue struct {
	// contains filtered or unexported fields
}

BooleanValue represents a boolean value.

func DecodeBooleanBlock added in v0.11.0 

func DecodeBooleanBlock(block []byte, a *[]BooleanValue) ([]BooleanValue, error)

DecodeBooleanBlock decodes the boolean block from the byte slice and appends the boolean values to a.

func (BooleanValue) Size added in v0.11.0 

func (v BooleanValue) Size() int

Size returns the number of bytes necessary to represent the value and its timestamp.

func (BooleanValue) String added in v0.11.0 

func (v BooleanValue) String() string

String returns the string representation of the value and its timestamp.

func (BooleanValue) UnixNano added in v0.11.0 

func (v BooleanValue) UnixNano() int64

UnixNano returns the timestamp of the value in nanoseconds since unix epoch.

func (BooleanValue) Value added in v0.11.0 

func (v BooleanValue) Value() interface{}

Value returns the underlying boolean value.

type BooleanValues added in v0.11.0 

type BooleanValues []BooleanValue

BooleanValues represents a slice of Boolean values.

func (BooleanValues) Deduplicate added in v0.11.0 

func (a BooleanValues) Deduplicate() BooleanValues

Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept. The returned Values are sorted if necessary.

func (BooleanValues) Encode added in v1.2.3 

func (a BooleanValues) Encode(buf []byte) ([]byte, error)

func (BooleanValues) Exclude added in v1.0.0 

func (a BooleanValues) Exclude(min, max int64) BooleanValues

Exclude returns the subset of values not in [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (BooleanValues) FindRange added in v1.4.0 

func (a BooleanValues) FindRange(min, max int64) (int, int)

FindRange returns the positions where min and max would be inserted into the array. If a[0].UnixNano() > max or a[len-1].UnixNano() < min then FindRange returns (-1, -1) indicating the array is outside the [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (BooleanValues) Include added in v1.0.0 

func (a BooleanValues) Include(min, max int64) BooleanValues

Include returns the subset values between min and max inclusive. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (BooleanValues) Len added in v0.11.0 

func (a BooleanValues) Len() int

Sort methods

func (BooleanValues) Less added in v0.11.0 

func (a BooleanValues) Less(i, j int) bool

func (BooleanValues) MaxTime added in v1.0.0 

func (a BooleanValues) MaxTime() int64

func (BooleanValues) Merge added in v1.0.0 

func (a BooleanValues) Merge(b BooleanValues) BooleanValues

Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.

func (BooleanValues) MinTime added in v1.0.0 

func (a BooleanValues) MinTime() int64

func (BooleanValues) Size added in v1.0.0 

func (a BooleanValues) Size() int

func (BooleanValues) Swap added in v0.11.0 

func (a BooleanValues) Swap(i, j int)

type Cache added in v0.9.6 

type Cache struct {
	// contains filtered or unexported fields
}

Cache maintains an in-memory store of Values for a set of keys.

func NewCache added in v0.9.6 

func NewCache(maxSize uint64, path string) *Cache

NewCache returns an instance of a cache which will use a maximum of maxSize bytes of memory. Only used for engine caches, never for snapshots.

func (*Cache) ApplyEntryFn added in v1.2.0 

func (c *Cache) ApplyEntryFn(f func(key []byte, entry *entry) error) error

ApplyEntryFn applies the function f to each entry in the Cache. ApplyEntryFn calls f on each entry in turn, within the same goroutine. It is safe for use by multiple goroutines.

func (*Cache) ClearSnapshot added in v0.9.6 

func (c *Cache) ClearSnapshot(success bool)

ClearSnapshot removes the snapshot cache from the list of flushing caches and adjusts the size.

func (*Cache) Count added in v1.4.0 

func (c *Cache) Count() int

func (*Cache) Deduplicate added in v0.10.0 

func (c *Cache) Deduplicate()

Deduplicate sorts the snapshot before returning it. The compactor and any queries coming in while it writes will need the values sorted.

func (*Cache) Delete added in v0.9.6 

func (c *Cache) Delete(keys [][]byte)

Delete removes all values for the given keys from the cache.

func (*Cache) DeleteRange added in v0.13.0 

func (c *Cache) DeleteRange(keys [][]byte, min, max int64)

DeleteRange removes the values for all keys containing points with timestamps between between min and max from the cache.

TODO(edd): Lock usage could possibly be optimised if necessary.

func (*Cache) Free added in v1.3.6 

func (c *Cache) Free()

Free releases the underlying store and memory held by the Cache.

func (*Cache) Keys added in v0.9.6 

func (c *Cache) Keys() [][]byte

Keys returns a sorted slice of all keys under management by the cache.

func (*Cache) MaxSize added in v0.9.6 

func (c *Cache) MaxSize() uint64

MaxSize returns the maximum number of bytes the cache may consume.

func (*Cache) SetMaxSize added in v0.13.0 

func (c *Cache) SetMaxSize(size uint64)

SetMaxSize updates the memory limit of the cache.

func (*Cache) Size added in v0.9.6 

func (c *Cache) Size() uint64

Size returns the number of point-calcuated bytes the cache currently uses.

func (*Cache) Snapshot added in v0.9.6 

func (c *Cache) Snapshot() (*Cache, error)

Snapshot takes a snapshot of the current cache, adds it to the slice of caches that are being flushed, and resets the current cache with new values.

func (*Cache) Split added in v1.4.0 

func (c *Cache) Split(n int) []*Cache

func (*Cache) Statistics added in v1.0.0 

func (c *Cache) Statistics(tags map[string]string) []models.Statistic

Statistics returns statistics for periodic monitoring.

func (*Cache) UpdateAge added in v0.11.0 

func (c *Cache) UpdateAge()

UpdateAge updates the age statistic based on the current time.

func (*Cache) UpdateCompactTime added in v0.11.0 

func (c *Cache) UpdateCompactTime(d time.Duration)

UpdateCompactTime updates WAL compaction time statistic based on d.

func (*Cache) Values added in v0.9.6 

func (c *Cache) Values(key []byte) Values

Values returns a copy of all values, deduped and sorted, for the given key.

func (*Cache) Write added in v0.9.6 

func (c *Cache) Write(key []byte, values []Value) error

Write writes the set of values for the key to the cache. This function is goroutine-safe. It returns an error if the cache will exceed its max size by adding the new values.

func (*Cache) WriteMulti added in v0.9.6 

func (c *Cache) WriteMulti(values map[string][]Value) error

WriteMulti writes the map of keys and associated values to the cache. This function is goroutine-safe. It returns an error if the cache will exceeded its max size by adding the new values. The write attempts to write as many values as possible. If one key fails, the others can still succeed and an error will be returned.

type CacheLoader added in v0.9.6 

type CacheLoader struct {
	Logger *zap.Logger
	// contains filtered or unexported fields
}

CacheLoader processes a set of WAL segment files, and loads a cache with the data contained within those files. Processing of the supplied files take place in the order they exist in the files slice.

func NewCacheLoader added in v0.9.6 

func NewCacheLoader(files []string) *CacheLoader

NewCacheLoader returns a new instance of a CacheLoader.

func (*CacheLoader) Load added in v0.9.6 

func (cl *CacheLoader) Load(cache *Cache) error

Load returns a cache loaded with the data contained within the segment files. If, during reading of a segment file, corruption is encountered, that segment file is truncated up to and including the last valid byte, and processing continues with the next segment file.

func (*CacheLoader) WithLogger added in v1.2.0 

func (cl *CacheLoader) WithLogger(log *zap.Logger)

WithLogger sets the logger on the CacheLoader.

type CacheStatistics added in v1.0.0 

type CacheStatistics struct {
	MemSizeBytes        int64
	DiskSizeBytes       int64
	SnapshotCount       int64
	CacheAgeMs          int64
	CachedBytes         int64
	WALCompactionTimeMs int64
	WriteOK             int64
	WriteErr            int64
	WriteDropped        int64
}

CacheStatistics hold statistics related to the cache.

type CompactionGroup added in v0.10.0 

type CompactionGroup []string

CompactionGroup represents a list of files eligible to be compacted together.

type CompactionPlanner added in v0.9.6 

type CompactionPlanner interface {
	Plan(lastWrite time.Time) []CompactionGroup
	PlanLevel(level int) []CompactionGroup
	PlanOptimize() []CompactionGroup
	Release(group []CompactionGroup)
	FullyCompacted() bool

	// ForceFull causes the planner to return a full compaction plan the next
	// time Plan() is called if there are files that could be compacted.
	ForceFull()
}

CompactionPlanner determines what TSM files and WAL segments to include in a given compaction run.

type Compactor added in v0.9.6 

type Compactor struct {
	Dir  string
	Size int

	FileStore interface {
		NextGeneration() int
		TSMReader(path string) *TSMReader
	}

	// RateLimit is the limit for disk writes for all concurrent compactions.
	RateLimit limiter.Rate
	// contains filtered or unexported fields
}

Compactor merges multiple TSM files into new files or writes a Cache into 1 or more TSM files.

func (*Compactor) Close added in v1.0.0 

func (c *Compactor) Close()

Close disables the Compactor.

func (*Compactor) CompactFast added in v0.10.0 

func (c *Compactor) CompactFast(tsmFiles []string) ([]string,