mirror of
https://github.com/grafana/loki.git
synced 2026-03-13 09:33:58 +08:00
4c81ebc7c3
This PR introduces an option for the column builder to limit the number of rows in a page. It can be set for logs object and index objects separately using `-dataobj-consumer.max-page-rows` and `-dataobj-index-builder.max-page-rows` respectively. Signed-off-by: Christian Haudum <christian.haudum@gmail.com>
71 lines
1.6 KiB
Go
71 lines
1.6 KiB
Go
package indexpointers
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/grafana/loki/v3/pkg/dataobj"
|
|
)
|
|
|
|
var indexPointerTestData = []IndexPointer{
|
|
{Path: "/path/to/index/object/1", StartTs: unixTime(10), EndTs: unixTime(20)},
|
|
{Path: "/path/to/index/object/2", StartTs: unixTime(12), EndTs: unixTime(17)},
|
|
{Path: "/path/to/index/object/3", StartTs: unixTime(13), EndTs: unixTime(18)},
|
|
}
|
|
|
|
func unixTime(sec int64) time.Time { return time.Unix(sec, 0) }
|
|
|
|
func TestRowReader(t *testing.T) {
|
|
dec := buildIndexPointersDecoder(t, 100, 0) // Many pages
|
|
r := NewRowReader(dec)
|
|
actual, err := readAllIndexPointers(context.Background(), r)
|
|
require.NoError(t, err)
|
|
require.Equal(t, indexPointerTestData, actual)
|
|
}
|
|
|
|
func buildIndexPointersDecoder(t *testing.T, pageSize, pageRows int) *Section {
|
|
t.Helper()
|
|
|
|
s := NewBuilder(nil, pageSize, pageRows)
|
|
for _, d := range indexPointerTestData {
|
|
s.Append(d.Path, d.StartTs, d.EndTs)
|
|
}
|
|
|
|
builder := dataobj.NewBuilder(nil)
|
|
require.NoError(t, builder.Append(s))
|
|
|
|
obj, closer, err := builder.Flush()
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() { closer.Close() })
|
|
|
|
sec, err := Open(t.Context(), obj.Sections()[0])
|
|
require.NoError(t, err)
|
|
return sec
|
|
}
|
|
|
|
func readAllIndexPointers(ctx context.Context, r *RowReader) ([]IndexPointer, error) {
|
|
var (
|
|
res []IndexPointer
|
|
buf = make([]IndexPointer, 128)
|
|
)
|
|
|
|
for {
|
|
n, err := r.Read(ctx, buf)
|
|
if n > 0 {
|
|
res = append(res, buf[:n]...)
|
|
}
|
|
if errors.Is(err, io.EOF) {
|
|
return res, nil
|
|
} else if err != nil {
|
|
return res, err
|
|
}
|
|
|
|
clear(buf)
|
|
}
|
|
}
|