mirror of
https://github.com/grafana/grafana.git
synced 2025-07-30 19:02:36 +08:00
Storage: Add ascending order support for NotOlderThan queries and introduce ResourceVersionMatch_Unset as default (#102505)
* Add support for ASC ordering and introduce ResourceVersionMatch_Unset as default Add SortAscending to continue token and add integration test for pagination. * Change protobuf order * Make backwards compatible * Update pkg/storage/unified/sql/backend.go Co-authored-by: Jean-Philippe Quéméner <JohnnyQQQQ@users.noreply.github.com> --------- Co-authored-by: Marco de Abreu <18629099+marcoabreu@users.noreply.github.com> Co-authored-by: Jean-Philippe Quéméner <JohnnyQQQQ@users.noreply.github.com>
This commit is contained in:
Marco de Abreu
@ -38,10 +38,12 @@ func toListRequest(k *resource.ResourceKey, opts storage.ListOptions) (*resource
|
||||
}
|
||||
|
||||
switch opts.ResourceVersionMatch {
|
||||
case "", metav1.ResourceVersionMatchNotOlderThan:
|
||||
req.VersionMatch = resource.ResourceVersionMatch_NotOlderThan
|
||||
case "":
|
||||
req.VersionMatchV2 = resource.ResourceVersionMatchV2_Unset
|
||||
case metav1.ResourceVersionMatchNotOlderThan:
|
||||
req.VersionMatchV2 = resource.ResourceVersionMatchV2_NotOlderThan
|
||||
case metav1.ResourceVersionMatchExact:
|
||||
req.VersionMatch = resource.ResourceVersionMatch_Exact
|
||||
req.VersionMatchV2 = resource.ResourceVersionMatchV2_Exact
|
||||
default:
|
||||
return nil, predicate, apierrors.NewBadRequest(
|
||||
fmt.Sprintf("unsupported version match: %v", opts.ResourceVersionMatch),
|
||||
|
||||
@ -9,6 +9,7 @@ import (
|
||||
type ContinueToken struct {
|
||||
StartOffset int64 `json:"o"`
|
||||
ResourceVersion int64 `json:"v"`
|
||||
SortAscending bool `json:"s"`
|
||||
}
|
||||
|
||||
func (c ContinueToken) String() string {
|
||||
|
||||
@ -10,6 +10,7 @@ func TestContinueToken(t *testing.T) {
|
||||
token := &ContinueToken{
|
||||
ResourceVersion: 100,
|
||||
StartOffset: 50,
|
||||
SortAscending: false,
|
||||
}
|
||||
assert.Equal(t, "eyJvIjo1MCwidiI6MTAwfQ==", token.String())
|
||||
assert.Equal(t, "eyJvIjo1MCwidiI6MTAwLCJzIjpmYWxzZX0=", token.String())
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -218,10 +218,19 @@ message ListOptions {
|
||||
}
|
||||
|
||||
enum ResourceVersionMatch {
|
||||
NotOlderThan = 0;
|
||||
Exact = 1;
|
||||
// Deprecated, use ResourceVersionMatch V2
|
||||
DEPRECATED_NotOlderThan = 0;
|
||||
DEPRECATED_Exact = 1;
|
||||
}
|
||||
|
||||
enum ResourceVersionMatchV2 {
|
||||
UNKNOWN = 0;
|
||||
Unset = 1;
|
||||
Exact = 2;
|
||||
NotOlderThan = 3;
|
||||
}
|
||||
|
||||
|
||||
message ListRequest {
|
||||
enum Source {
|
||||
STORE = 0; // the standard place
|
||||
@ -236,7 +245,8 @@ message ListRequest {
|
||||
int64 resource_version = 2;
|
||||
|
||||
// List options
|
||||
ResourceVersionMatch version_match = 3;
|
||||
// DEPRECATED - use version_match_v2
|
||||
optional ResourceVersionMatch version_match = 3;
|
||||
|
||||
// Maximum number of items to return
|
||||
// NOTE responses will also be limited by the response payload size
|
||||
@ -247,6 +257,8 @@ message ListRequest {
|
||||
|
||||
// Select values from history or trash
|
||||
Source source = 6;
|
||||
|
||||
ResourceVersionMatchV2 version_match_v2 = 7;
|
||||
}
|
||||
|
||||
message ListResponse {
|
||||
|
||||
@ -753,10 +753,11 @@ func (s *server) List(ctx context.Context, req *ListRequest) (*ListResponse, err
|
||||
rsp.Items = append(rsp.Items, item)
|
||||
if len(rsp.Items) >= int(req.Limit) || pageBytes >= maxPageBytes {
|
||||
t := iter.ContinueToken()
|
||||
if req.Source == ListRequest_HISTORY {
|
||||
// history lists in desc order, so the continue token takes the
|
||||
// final RV in the list, and then will start from there in the next page,
|
||||
// rather than the lists first RV
|
||||
if req.Source == ListRequest_HISTORY || req.Source == ListRequest_TRASH {
|
||||
// For history lists, we need to use the current RV in the continue token
|
||||
// to ensure consistent pagination. The order depends on VersionMatch:
|
||||
// - NotOlderThan: ascending order (oldest to newest)
|
||||
// - Unset: descending order (newest to oldest)
|
||||
t = iter.ContinueTokenWithCurrentRV()
|
||||
}
|
||||
if iter.Next() {
|
||||
|
||||
@ -544,9 +544,10 @@ func (b *backend) ListIterator(ctx context.Context, req *resource.ListRequest, c
|
||||
}
|
||||
|
||||
type listIter struct {
|
||||
rows db.Rows
|
||||
offset int64
|
||||
listRV int64
|
||||
rows db.Rows
|
||||
offset int64
|
||||
listRV int64
|
||||
sortAsc bool
|
||||
|
||||
// any error
|
||||
err error
|
||||
@ -561,11 +562,11 @@ type listIter struct {
|
||||
|
||||
// ContinueToken implements resource.ListIterator.
|
||||
func (l *listIter) ContinueToken() string {
|
||||
return resource.ContinueToken{ResourceVersion: l.listRV, StartOffset: l.offset}.String()
|
||||
return resource.ContinueToken{ResourceVersion: l.listRV, StartOffset: l.offset, SortAscending: l.sortAsc}.String()
|
||||
}
|
||||
|
||||
func (l *listIter) ContinueTokenWithCurrentRV() string {
|
||||
return resource.ContinueToken{ResourceVersion: l.rv, StartOffset: l.offset}.String()
|
||||
return resource.ContinueToken{ResourceVersion: l.rv, StartOffset: l.offset, SortAscending: l.sortAsc}.String()
|
||||
}
|
||||
|
||||
func (l *listIter) Error() error {
|
||||
@ -615,7 +616,7 @@ func (b *backend) listLatest(ctx context.Context, req *resource.ListRequest, cb
|
||||
return 0, fmt.Errorf("only works for the 'latest' resource version")
|
||||
}
|
||||
|
||||
iter := &listIter{}
|
||||
iter := &listIter{sortAsc: false}
|
||||
err := b.db.WithTx(ctx, ReadCommittedRO, func(ctx context.Context, tx db.Tx) error {
|
||||
var err error
|
||||
iter.listRV, err = fetchLatestRV(ctx, tx, b.dialect, req.Options.Key.Group, req.Options.Key.Resource)
|
||||
@ -650,7 +651,7 @@ func (b *backend) listLatest(ctx context.Context, req *resource.ListRequest, cb
|
||||
// listAtRevision fetches the resources from the resource_history table at a specific revision.
|
||||
func (b *backend) listAtRevision(ctx context.Context, req *resource.ListRequest, cb func(resource.ListIterator) error) (int64, error) {
|
||||
// Get the RV
|
||||
iter := &listIter{listRV: req.ResourceVersion}
|
||||
iter := &listIter{listRV: req.ResourceVersion, sortAsc: false}
|
||||
if req.NextPageToken != "" {
|
||||
continueToken, err := resource.GetContinueToken(req.NextPageToken)
|
||||
if err != nil {
|
||||
@ -700,14 +701,37 @@ func (b *backend) listAtRevision(ctx context.Context, req *resource.ListRequest,
|
||||
return iter.listRV, err
|
||||
}
|
||||
|
||||
// listLatest fetches the resources from the resource table.
|
||||
// getHistory fetches the resources from the resource table.
|
||||
func (b *backend) getHistory(ctx context.Context, req *resource.ListRequest, cb func(resource.ListIterator) error) (int64, error) {
|
||||
// Backwards compatibility for ResourceVersionMatch
|
||||
if req.VersionMatch != nil && req.GetVersionMatchV2() == resource.ResourceVersionMatchV2_UNKNOWN {
|
||||
switch req.GetVersionMatch() {
|
||||
case resource.ResourceVersionMatch_DEPRECATED_NotOlderThan:
|
||||
// This is not a typo. The old implementation actually did behave like Unset.
|
||||
req.VersionMatchV2 = resource.ResourceVersionMatchV2_Unset
|
||||
case resource.ResourceVersionMatch_DEPRECATED_Exact:
|
||||
req.VersionMatchV2 = resource.ResourceVersionMatchV2_Exact
|
||||
default:
|
||||
return 0, fmt.Errorf("unknown version match: %v", req.GetVersionMatch())
|
||||
}
|
||||
|
||||
// Log the migration for debugging purposes
|
||||
b.log.Debug("Old client request received, migrating from version_match to version_match_v2",
|
||||
"oldValue", req.GetVersionMatch(),
|
||||
"newValue", req.GetVersionMatchV2())
|
||||
}
|
||||
|
||||
listReq := sqlGetHistoryRequest{
|
||||
SQLTemplate: sqltemplate.New(b.dialect),
|
||||
Key: req.Options.Key,
|
||||
Trash: req.Source == resource.ListRequest_TRASH,
|
||||
}
|
||||
|
||||
// We are assuming that users want history in ascending order
|
||||
// when they are using NotOlderThan matching, and descending order
|
||||
// for Unset (default) and Exact matching.
|
||||
listReq.SortAscending = req.GetVersionMatchV2() == resource.ResourceVersionMatchV2_NotOlderThan
|
||||
|
||||
iter := &listIter{}
|
||||
if req.NextPageToken != "" {
|
||||
continueToken, err := resource.GetContinueToken(req.NextPageToken)
|
||||
@ -715,10 +739,12 @@ func (b *backend) getHistory(ctx context.Context, req *resource.ListRequest, cb
|
||||
return 0, fmt.Errorf("get continue token: %w", err)
|
||||
}
|
||||
listReq.StartRV = continueToken.ResourceVersion
|
||||
listReq.SortAscending = continueToken.SortAscending
|
||||
}
|
||||
iter.sortAsc = listReq.SortAscending
|
||||
|
||||
// Set ExactRV when using Exact matching
|
||||
if req.VersionMatch == resource.ResourceVersionMatch_Exact {
|
||||
if req.VersionMatchV2 == resource.ResourceVersionMatchV2_Exact {
|
||||
if req.ResourceVersion <= 0 {
|
||||
return 0, fmt.Errorf("expecting an explicit resource version query when using Exact matching")
|
||||
}
|
||||
@ -726,7 +752,7 @@ func (b *backend) getHistory(ctx context.Context, req *resource.ListRequest, cb
|
||||
}
|
||||
|
||||
// Set MinRV when using NotOlderThan matching to filter at the database level
|
||||
if req.ResourceVersion > 0 && req.VersionMatch == resource.ResourceVersionMatch_NotOlderThan {
|
||||
if req.ResourceVersion > 0 && req.VersionMatchV2 == resource.ResourceVersionMatchV2_NotOlderThan {
|
||||
listReq.MinRV = req.ResourceVersion
|
||||
}
|
||||
|
||||
|
||||
@ -391,7 +391,7 @@ func TestBackend_getHistory(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
versionMatch resource.ResourceVersionMatch
|
||||
versionMatch resource.ResourceVersionMatchV2
|
||||
resourceVersion int64
|
||||
expectedVersions []int64
|
||||
expectedListRv int64
|
||||
@ -400,30 +400,37 @@ func TestBackend_getHistory(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
name: "with ResourceVersionMatch_NotOlderThan",
|
||||
versionMatch: resource.ResourceVersionMatch_NotOlderThan,
|
||||
versionMatch: resource.ResourceVersionMatchV2_NotOlderThan,
|
||||
resourceVersion: rv2,
|
||||
expectedVersions: []int64{rv3, rv2},
|
||||
expectedVersions: []int64{rv2, rv3}, // Should be in ASC order due to NotOlderThan
|
||||
expectedListRv: rv3,
|
||||
expectedRowsCount: 2,
|
||||
},
|
||||
{
|
||||
name: "with ResourceVersionMatch_NotOlderThan and ResourceVersion=0",
|
||||
versionMatch: resource.ResourceVersionMatchV2_NotOlderThan,
|
||||
resourceVersion: 0,
|
||||
expectedVersions: []int64{rv1, rv2, rv3}, // Should be in ASC order due to NotOlderThan
|
||||
expectedListRv: rv3,
|
||||
expectedRowsCount: 3,
|
||||
},
|
||||
{
|
||||
name: "with ResourceVersionMatch_Exact",
|
||||
versionMatch: resource.ResourceVersionMatch_Exact,
|
||||
versionMatch: resource.ResourceVersionMatchV2_Exact,
|
||||
resourceVersion: rv2,
|
||||
expectedVersions: []int64{rv2},
|
||||
expectedListRv: rv3,
|
||||
expectedRowsCount: 1,
|
||||
},
|
||||
{
|
||||
name: "without version matcher",
|
||||
versionMatch: resource.ResourceVersionMatch_NotOlderThan,
|
||||
expectedVersions: []int64{rv3, rv2, rv1},
|
||||
name: "with ResourceVersionMatch_Unset (default)",
|
||||
expectedVersions: []int64{rv3, rv2, rv1}, // Should be in DESC order by default
|
||||
expectedListRv: rv3,
|
||||
expectedRowsCount: 3,
|
||||
},
|
||||
{
|
||||
name: "error with ResourceVersionMatch_Exact and ResourceVersion <= 0",
|
||||
versionMatch: resource.ResourceVersionMatch_Exact,
|
||||
versionMatch: resource.ResourceVersionMatchV2_Exact,
|
||||
resourceVersion: 0,
|
||||
expectedErr: "expecting an explicit resource version query when using Exact matching",
|
||||
},
|
||||
@ -439,7 +446,7 @@ func TestBackend_getHistory(t *testing.T) {
|
||||
req := &resource.ListRequest{
|
||||
Options: &resource.ListOptions{Key: key},
|
||||
ResourceVersion: tc.resourceVersion,
|
||||
VersionMatch: tc.versionMatch,
|
||||
VersionMatchV2: tc.versionMatch,
|
||||
Source: resource.ListRequest_HISTORY,
|
||||
}
|
||||
|
||||
@ -454,14 +461,18 @@ func TestBackend_getHistory(t *testing.T) {
|
||||
// Callback that tracks returned items
|
||||
callback := func(iter resource.ListIterator) error {
|
||||
count := 0
|
||||
var seenVersions []int64
|
||||
for iter.Next() {
|
||||
count++
|
||||
currentRV := iter.ResourceVersion()
|
||||
seenVersions = append(seenVersions, currentRV)
|
||||
expectedValue, ok := expectedValues[currentRV]
|
||||
require.True(t, ok, "Got unexpected RV: %d", currentRV)
|
||||
require.Equal(t, expectedValue, string(iter.Value()))
|
||||
}
|
||||
require.Equal(t, tc.expectedRowsCount, count)
|
||||
// Verify the order matches what we expect
|
||||
require.Equal(t, tc.expectedVersions, seenVersions, "Resource versions returned in incorrect order")
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -503,3 +514,153 @@ func TestBackend_getHistory(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBackend_getHistoryPagination tests the ordering behavior for ResourceVersionMatch_NotOlderThan
|
||||
// when using pagination, ensuring entries are returned in oldest-to-newest order.
|
||||
func TestBackend_getHistoryPagination(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Common setup
|
||||
key := &resource.ResourceKey{
|
||||
Namespace: "ns",
|
||||
Group: "gr",
|
||||
Resource: "rs",
|
||||
Name: "nm",
|
||||
}
|
||||
|
||||
// Create resource versions that will be returned in our test
|
||||
versions := make([]int64, 10)
|
||||
for i := range versions {
|
||||
versions[i] = int64(51 + i)
|
||||
}
|
||||
rv51, rv52, rv53, rv54, rv55, rv56, rv57, rv58, rv59, rv60 := versions[0], versions[1], versions[2], versions[3], versions[4], versions[5], versions[6], versions[7], versions[8], versions[9]
|
||||
|
||||
t.Run("pagination with NotOlderThan should return entries from oldest to newest", func(t *testing.T) {
|
||||
b, ctx := setupBackendTest(t)
|
||||
|
||||
// Define all pages we want to test
|
||||
pages := []struct {
|
||||
versions []int64
|
||||
token *resource.ContinueToken
|
||||
}{
|
||||
{
|
||||
versions: []int64{rv51, rv52, rv53, rv54},
|
||||
token: nil,
|
||||
},
|
||||
{
|
||||
versions: []int64{rv55, rv56, rv57, rv58},
|
||||
token: &resource.ContinueToken{
|
||||
ResourceVersion: rv54,
|
||||
StartOffset: 4,
|
||||
SortAscending: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
versions: []int64{rv59, rv60},
|
||||
token: &resource.ContinueToken{
|
||||
ResourceVersion: rv58,
|
||||
StartOffset: 8,
|
||||
SortAscending: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var allItems []int64
|
||||
initialRV := rv51
|
||||
|
||||
// Test each page
|
||||
for _, page := range pages {
|
||||
req := &resource.ListRequest{
|
||||
Options: &resource.ListOptions{Key: key},
|
||||
ResourceVersion: initialRV,
|
||||
VersionMatchV2: resource.ResourceVersionMatchV2_NotOlderThan,
|
||||
Source: resource.ListRequest_HISTORY,
|
||||
Limit: 4,
|
||||
}
|
||||
if page.token != nil {
|
||||
req.NextPageToken = page.token.String()
|
||||
}
|
||||
|
||||
items := make([]int64, 0)
|
||||
callback := func(iter resource.ListIterator) error {
|
||||
for iter.Next() {
|
||||
items = append(items, iter.ResourceVersion())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
b.SQLMock.ExpectBegin()
|
||||
historyRows := setupHistoryTest(b, page.versions, rv60)
|
||||
b.SQLMock.ExpectQuery("SELECT .* FROM resource_history").WillReturnRows(historyRows)
|
||||
b.SQLMock.ExpectCommit()
|
||||
|
||||
listRv, err := b.getHistory(ctx, req, callback)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, rv60, listRv, "Head version should be the latest resource version (rv60)")
|
||||
require.Equal(t, page.versions, items, "Items should be in ASC order")
|
||||
|
||||
allItems = append(allItems, items...)
|
||||
}
|
||||
|
||||
// Verify complete sequence
|
||||
expectedAllItems := []int64{rv51, rv52, rv53, rv54, rv55, rv56, rv57, rv58, rv59, rv60}
|
||||
require.Equal(t, expectedAllItems, allItems)
|
||||
})
|
||||
|
||||
t.Run("pagination with ResourceVersion=0 and NotOlderThan should return entries in ASC order", func(t *testing.T) {
|
||||
b, ctx := setupBackendTest(t)
|
||||
|
||||
req := &resource.ListRequest{
|
||||
Options: &resource.ListOptions{Key: key},
|
||||
ResourceVersion: 0,
|
||||
VersionMatchV2: resource.ResourceVersionMatchV2_NotOlderThan,
|
||||
Source: resource.ListRequest_HISTORY,
|
||||
Limit: 4,
|
||||
}
|
||||
|
||||
// First batch of items we expect, in ASC order (because of NotOlderThan flag)
|
||||
// Even with ResourceVersion=0, the order is ASC because we use SortAscending=true
|
||||
expectedVersions := []int64{rv51, rv52, rv53, rv54}
|
||||
items := make([]int64, 0)
|
||||
|
||||
callback := func(iter resource.ListIterator) error {
|
||||
for iter.Next() {
|
||||
items = append(items, iter.ResourceVersion())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
b.SQLMock.ExpectBegin()
|
||||
historyRows := setupHistoryTest(b, expectedVersions, rv60)
|
||||
b.SQLMock.ExpectQuery("SELECT .* FROM resource_history").WillReturnRows(historyRows)
|
||||
b.SQLMock.ExpectCommit()
|
||||
|
||||
listRv, err := b.getHistory(ctx, req, callback)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, rv60, listRv, "Head version should be the latest resource version (rv60)")
|
||||
require.Equal(t, expectedVersions, items, "Items should be in ASC order even with ResourceVersion=0")
|
||||
})
|
||||
}
|
||||
|
||||
// setupHistoryTest creates the necessary mock expectations for a history test
|
||||
func setupHistoryTest(b testBackend, resourceVersions []int64, latestRV int64) *sqlmock.Rows {
|
||||
// Expect fetch latest RV call - set to the highest resource version
|
||||
latestRVRows := sqlmock.NewRows([]string{"resource_version", "unix_timestamp"}).
|
||||
AddRow(latestRV, 0)
|
||||
b.SQLMock.ExpectQuery("SELECT .* FROM resource_version").WillReturnRows(latestRVRows)
|
||||
|
||||
// Create the mock rows for the history items
|
||||
cols := []string{"resource_version", "namespace", "name", "folder", "value"}
|
||||
historyRows := sqlmock.NewRows(cols)
|
||||
for _, rv := range resourceVersions {
|
||||
historyRows.AddRow(
|
||||
rv, // resource_version
|
||||
"ns", // namespace
|
||||
"nm", // name
|
||||
"folder", // folder
|
||||
[]byte(fmt.Sprintf("rv-%d", rv)), // value
|
||||
)
|
||||
}
|
||||
|
||||
return historyRows
|
||||
}
|
||||
|
||||
@ -16,12 +16,20 @@ WHERE 1 = 1
|
||||
AND {{ .Ident "action" }} = 3
|
||||
{{ end }}
|
||||
{{ if (gt .StartRV 0) }}
|
||||
{{ if .SortAscending }}
|
||||
AND {{ .Ident "resource_version" }} > {{ .Arg .StartRV }}
|
||||
{{ else }}
|
||||
AND {{ .Ident "resource_version" }} < {{ .Arg .StartRV }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
{{ if (gt .MinRV 0) }}
|
||||
AND {{ .Ident "resource_version" }} >= {{ .Arg .MinRV }}
|
||||
{{ end }}
|
||||
{{ if (gt .ExactRV 0) }}
|
||||
AND {{ .Ident "resource_version" }} = {{ .Arg .ExactRV }}
|
||||
{{ end }}
|
||||
{{ if .SortAscending }}
|
||||
ORDER BY resource_version ASC
|
||||
{{ else }}
|
||||
ORDER BY resource_version DESC
|
||||
{{ end }}
|
||||
|
||||
@ -243,11 +243,12 @@ func (r *sqlResourceHistoryDeleteRequest) Validate() error {
|
||||
|
||||
type sqlGetHistoryRequest struct {
|
||||
sqltemplate.SQLTemplate
|
||||
Key *resource.ResourceKey
|
||||
Trash bool // only deleted items
|
||||
StartRV int64 // from NextPageToken
|
||||
MinRV int64 // minimum resource version for NotOlderThan
|
||||
ExactRV int64 // exact resource version for Exact
|
||||
Key *resource.ResourceKey
|
||||
Trash bool // only deleted items
|
||||
StartRV int64 // from NextPageToken
|
||||
MinRV int64 // minimum resource version for NotOlderThan
|
||||
ExactRV int64 // exact resource version for Exact
|
||||
SortAscending bool // if true, sort by resource_version ASC, otherwise DESC
|
||||
}
|
||||
|
||||
func (r sqlGetHistoryRequest) Validate() error {
|
||||
|
||||
@ -443,6 +443,7 @@ func runTestIntegrationBackendList(t *testing.T, backend resource.StorageBackend
|
||||
continueToken := &resource.ContinueToken{
|
||||
ResourceVersion: rv8,
|
||||
StartOffset: 2,
|
||||
SortAscending: false,
|
||||
}
|
||||
res, err := server.List(ctx, &resource.ListRequest{
|
||||
NextPageToken: continueToken.String(),
|
||||
@ -492,42 +493,142 @@ func runTestIntegrationBackendListHistory(t *testing.T, backend resource.Storage
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, rvHistory5, rvHistory4)
|
||||
|
||||
t.Run("fetch first history page at revision with limit", func(t *testing.T) {
|
||||
res, err := server.List(ctx, &resource.ListRequest{
|
||||
Limit: 3,
|
||||
Source: resource.ListRequest_HISTORY,
|
||||
Options: &resource.ListOptions{
|
||||
Key: &resource.ResourceKey{
|
||||
Namespace: ns,
|
||||
Group: "group",
|
||||
Resource: "resource",
|
||||
Name: "item1",
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, res.Error)
|
||||
require.Len(t, res.Items, 3)
|
||||
t.Log(res.Items)
|
||||
// should be in desc order, so the newest RVs are returned first
|
||||
require.Equal(t, "item1 MODIFIED", string(res.Items[0].Value))
|
||||
require.Equal(t, rvHistory5, res.Items[0].ResourceVersion)
|
||||
require.Equal(t, "item1 MODIFIED", string(res.Items[1].Value))
|
||||
require.Equal(t, rvHistory4, res.Items[1].ResourceVersion)
|
||||
require.Equal(t, "item1 MODIFIED", string(res.Items[2].Value))
|
||||
require.Equal(t, rvHistory3, res.Items[2].ResourceVersion)
|
||||
t.Run("fetch history with different version matching", func(t *testing.T) {
|
||||
baseKey := &resource.ResourceKey{
|
||||
Namespace: ns,
|
||||
Group: "group",
|
||||
Resource: "resource",
|
||||
Name: "item1",
|
||||
}
|
||||
|
||||
continueToken, err := resource.GetContinueToken(res.NextPageToken)
|
||||
require.NoError(t, err)
|
||||
// should return the furthest back RV as the next page token
|
||||
require.Equal(t, rvHistory3, continueToken.ResourceVersion)
|
||||
tests := []struct {
|
||||
name string
|
||||
request *resource.ListRequest
|
||||
expectedVersions []int64
|
||||
expectedValues []string
|
||||
minExpectedHeadRV int64
|
||||
expectedContinueRV int64
|
||||
expectedSortAsc bool
|
||||
}{
|
||||
{
|
||||
name: "NotOlderThan with rv1 (ASC order)",
|
||||
request: &resource.ListRequest{
|
||||
Limit: 3,
|
||||
Source: resource.ListRequest_HISTORY,
|
||||
ResourceVersion: rv1,
|
||||
VersionMatchV2: resource.ResourceVersionMatchV2_NotOlderThan,
|
||||
Options: &resource.ListOptions{
|
||||
Key: baseKey,
|
||||
},
|
||||
},
|
||||
expectedVersions: []int64{rv1, rvHistory1, rvHistory2},
|
||||
expectedValues: []string{"item1 ADDED", "item1 MODIFIED", "item1 MODIFIED"},
|
||||
minExpectedHeadRV: rvHistory2,
|
||||
expectedContinueRV: rvHistory2,
|
||||
expectedSortAsc: true,
|
||||
},
|
||||
{
|
||||
name: "NotOlderThan with rv=0 (ASC order)",
|
||||
request: &resource.ListRequest{
|
||||
Limit: 3,
|
||||
Source: resource.ListRequest_HISTORY,
|
||||
ResourceVersion: 0,
|
||||
VersionMatchV2: resource.ResourceVersionMatchV2_NotOlderThan,
|
||||
Options: &resource.ListOptions{
|
||||
Key: baseKey,
|
||||
},
|
||||
},
|
||||
expectedVersions: []int64{rv1, rvHistory1, rvHistory2},
|
||||
expectedValues: []string{"item1 ADDED", "item1 MODIFIED", "item1 MODIFIED"},
|
||||
minExpectedHeadRV: rvHistory2,
|
||||
expectedContinueRV: rvHistory2,
|
||||
expectedSortAsc: true,
|
||||
},
|
||||
{
|
||||
name: "ResourceVersionMatch_Unset (DESC order)",
|
||||
request: &resource.ListRequest{
|
||||
Limit: 3,
|
||||
Source: resource.ListRequest_HISTORY,
|
||||
Options: &resource.ListOptions{
|
||||
Key: baseKey,
|
||||
},
|
||||
},
|
||||
expectedVersions: []int64{rvHistory5, rvHistory4, rvHistory3},
|
||||
expectedValues: []string{"item1 MODIFIED", "item1 MODIFIED", "item1 MODIFIED"},
|
||||
minExpectedHeadRV: rvHistory5,
|
||||
expectedContinueRV: rvHistory3,
|
||||
expectedSortAsc: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
res, err := server.List(ctx, tc.request)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, res.Error)
|
||||
require.Len(t, res.Items, 3)
|
||||
|
||||
// Check versions and values match expectations
|
||||
for i := 0; i < 3; i++ {
|
||||
require.Equal(t, tc.expectedVersions[i], res.Items[i].ResourceVersion)
|
||||
require.Equal(t, tc.expectedValues[i], string(res.Items[i].Value))
|
||||
}
|
||||
|
||||
// Check resource version in response
|
||||
require.GreaterOrEqual(t, res.ResourceVersion, tc.minExpectedHeadRV)
|
||||
|
||||
// Check continue token
|
||||
continueToken, err := resource.GetContinueToken(res.NextPageToken)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.expectedContinueRV, continueToken.ResourceVersion)
|
||||
require.Equal(t, tc.expectedSortAsc, continueToken.SortAscending)
|
||||
})
|
||||
}
|
||||
|
||||
// Test pagination for NotOlderThan (second page)
|
||||
t.Run("second page with NotOlderThan", func(t *testing.T) {
|
||||
// Get first page
|
||||
firstRequest := &resource.ListRequest{
|
||||
Limit: 3,
|
||||
Source: resource.ListRequest_HISTORY,
|
||||
ResourceVersion: rv1,
|
||||
VersionMatchV2: resource.ResourceVersionMatchV2_NotOlderThan,
|
||||
Options: &resource.ListOptions{Key: baseKey},
|
||||
}
|
||||
firstPageRes, err := server.List(ctx, firstRequest)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get continue token for second page
|
||||
continueToken, err := resource.GetContinueToken(firstPageRes.NextPageToken)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get second page
|
||||
secondPageRes, err := server.List(ctx, &resource.ListRequest{
|
||||
Limit: 3,
|
||||
Source: resource.ListRequest_HISTORY,
|
||||
ResourceVersion: rv1,
|
||||
VersionMatchV2: resource.ResourceVersionMatchV2_NotOlderThan,
|
||||
NextPageToken: continueToken.String(),
|
||||
Options: &resource.ListOptions{Key: baseKey},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, secondPageRes.Error)
|
||||
require.Len(t, secondPageRes.Items, 3)
|
||||
|
||||
// Second page should continue in ascending order
|
||||
expectedRVs := []int64{rvHistory3, rvHistory4, rvHistory5}
|
||||
for i, expectedRV := range expectedRVs {
|
||||
require.Equal(t, expectedRV, secondPageRes.Items[i].ResourceVersion)
|
||||
require.Equal(t, "item1 MODIFIED", string(secondPageRes.Items[i].Value))
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("fetch second page of history at revision", func(t *testing.T) {
|
||||
continueToken := &resource.ContinueToken{
|
||||
ResourceVersion: rvHistory3,
|
||||
StartOffset: 2,
|
||||
SortAscending: false,
|
||||
}
|
||||
res, err := server.List(ctx, &resource.ListRequest{
|
||||
NextPageToken: continueToken.String(),
|
||||
@ -551,6 +652,123 @@ func runTestIntegrationBackendListHistory(t *testing.T, backend resource.Storage
|
||||
require.Equal(t, "item1 MODIFIED", string(res.Items[1].Value))
|
||||
require.Equal(t, rvHistory1, res.Items[1].ResourceVersion)
|
||||
})
|
||||
|
||||
t.Run("paginated history with NotOlderThan returns items in ascending order", func(t *testing.T) {
|
||||
// Create 10 versions of a resource to test pagination
|
||||
ns2 := nsPrefix + "-ns2"
|
||||
resourceKey := &resource.ResourceKey{
|
||||
Namespace: ns2,
|
||||
Group: "group",
|
||||
Resource: "resource",
|
||||
Name: "paged-item",
|
||||
}
|
||||
|
||||
var resourceVersions []int64
|
||||
|
||||
// First create the initial resource
|
||||
initialRV, err := writeEvent(ctx, backend, "paged-item", resource.WatchEvent_ADDED, WithNamespace(ns2))
|
||||
require.NoError(t, err)
|
||||
resourceVersions = append(resourceVersions, initialRV)
|
||||
|
||||
// Create 9 more versions with modifications
|
||||
for i := 0; i < 9; i++ {
|
||||
rv, err := writeEvent(ctx, backend, "paged-item", resource.WatchEvent_MODIFIED, WithNamespace(ns2))
|
||||
require.NoError(t, err)
|
||||
resourceVersions = append(resourceVersions, rv)
|
||||
}
|
||||
|
||||
// Now we should have 10 versions total (1 ADDED + 9 MODIFIED)
|
||||
require.Len(t, resourceVersions, 10)
|
||||
|
||||
// Test pagination with limit of 3 and NotOlderThan, starting from the beginning
|
||||
pages := []struct {
|
||||
pageNumber int
|
||||
pageSize int
|
||||
startToken string
|
||||
}{
|
||||
{pageNumber: 1, pageSize: 3, startToken: ""},
|
||||
{pageNumber: 2, pageSize: 3, startToken: ""}, // Will be set in the test
|
||||
{pageNumber: 3, pageSize: 3, startToken: ""}, // Will be set in the test
|
||||
{pageNumber: 4, pageSize: 1, startToken: ""}, // Will be set in the test - last page with remaining item
|
||||
}
|
||||
|
||||
var allItems []*resource.ResourceWrapper
|
||||
|
||||
// Request first page with NotOlderThan and ResourceVersion=0 (should start from oldest)
|
||||
for i, page := range pages {
|
||||
req := &resource.ListRequest{
|
||||
Limit: int64(page.pageSize),
|
||||
Source: resource.ListRequest_HISTORY,
|
||||
ResourceVersion: 0,
|
||||
VersionMatchV2: resource.ResourceVersionMatchV2_NotOlderThan,
|
||||
Options: &resource.ListOptions{
|
||||
Key: resourceKey,
|
||||
},
|
||||
}
|
||||
|
||||
if i > 0 {
|
||||
// For subsequent pages, use the continue token from the previous page
|
||||
req.NextPageToken = pages[i].startToken
|
||||
}
|
||||
|
||||
res, err := server.List(ctx, req)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, res.Error)
|
||||
|
||||
// First 3 pages should have exactly pageSize items
|
||||
if i < 3 {
|
||||
require.Len(t, res.Items, page.pageSize, "Page %d should have %d items", i+1, page.pageSize)
|
||||
} else {
|
||||
// Last page should have 1 item (10 items total with 3+3+3+1 distribution)
|
||||
require.Len(t, res.Items, 1, "Last page should have 1 item")
|
||||
}
|
||||
|
||||
// Save continue token for next page if not the last page
|
||||
if i < len(pages)-1 {
|
||||
pages[i+1].startToken = res.NextPageToken
|
||||
require.NotEmpty(t, res.NextPageToken, "Should have continue token for page %d", i+1)
|
||||
} else {
|
||||
// Last page should not have a continue token
|
||||
require.Empty(t, res.NextPageToken, "Last page should not have continue token")
|
||||
}
|
||||
|
||||
// Add items to our collection
|
||||
allItems = append(allItems, res.Items...)
|
||||
|
||||
// Verify all items in current page are in ascending order
|
||||
for j := 1; j < len(res.Items); j++ {
|
||||
require.Less(t, res.Items[j-1].ResourceVersion, res.Items[j].ResourceVersion,
|
||||
"Items within page %d should be in ascending order", i+1)
|
||||
}
|
||||
|
||||
// For pages after the first, verify first item of current page is greater than last item of previous page
|
||||
if i > 0 && len(allItems) > page.pageSize {
|
||||
prevPageLastIdx := len(allItems) - len(res.Items) - 1
|
||||
currentPageFirstIdx := len(allItems) - len(res.Items)
|
||||
require.Greater(t, allItems[currentPageFirstIdx].ResourceVersion, allItems[prevPageLastIdx].ResourceVersion,
|
||||
"First item of page %d should have higher RV than last item of page %d", i+1, i)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify we got all 10 items
|
||||
require.Len(t, allItems, 10, "Should have retrieved all 10 items")
|
||||
|
||||
// Verify all items are in ascending order of resource version
|
||||
for i := 1; i < len(allItems); i++ {
|
||||
require.Less(t, allItems[i-1].ResourceVersion, allItems[i].ResourceVersion,
|
||||
"All items should be in ascending order of resource version")
|
||||
}
|
||||
|
||||
// Verify the first item is the initial ADDED event
|
||||
require.Equal(t, initialRV, allItems[0].ResourceVersion, "First item should be the initial ADDED event")
|
||||
require.Equal(t, "paged-item ADDED", string(allItems[0].Value))
|
||||
|
||||
// Verify all other items are MODIFIED events and correspond to our recorded resource versions
|
||||
for i := 1; i < len(allItems); i++ {
|
||||
require.Equal(t, "paged-item MODIFIED", string(allItems[i].Value))
|
||||
require.Equal(t, resourceVersions[i], allItems[i].ResourceVersion)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func runTestIntegrationBlobSupport(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
|
||||
|
||||
Reference in New Issue
Block a user