Unitstore: Fix the Folder migration (#96078)

* UniStore: Add folder to the SQL backend

This reverts commit aaa4f4a23e3ba0e02526f8beb05321761382259f.

* fix folder migration
This commit is contained in:
Georges Chaudy
2024-11-12 12:52:04 +01:00
committed by GitHub
parent 7f675cd218
commit c3571752b6
43 changed files with 235 additions and 72 deletions

View File

@ -200,6 +200,10 @@ func (r *rowsWrapper) ResourceVersion() int64 {
return r.row.RV return r.row.RV
} }
func (r *rowsWrapper) Folder() string {
return r.row.FolderUID
}
// Value implements resource.ListIterator. // Value implements resource.ListIterator.
func (r *rowsWrapper) Value() []byte { func (r *rowsWrapper) Value() []byte {
b, err := json.Marshal(r.row.Dash) b, err := json.Marshal(r.row.Dash)

View File

@ -128,8 +128,8 @@ func (a *dashboardSqlAccess) GetDashboard(ctx context.Context, orgId int64, uid
} }
// Read implements ResourceStoreServer. // Read implements ResourceStoreServer.
func (a *dashboardSqlAccess) ReadResource(ctx context.Context, req *resource.ReadRequest) *resource.ReadResponse { func (a *dashboardSqlAccess) ReadResource(ctx context.Context, req *resource.ReadRequest) *resource.BackendReadResponse {
rsp := &resource.ReadResponse{} rsp := &resource.BackendReadResponse{}
info, err := claims.ParseNamespace(req.Key.Namespace) info, err := claims.ParseNamespace(req.Key.Namespace)
if err == nil { if err == nil {
err = isDashboardKey(req.Key, true) err = isDashboardKey(req.Key, true)
@ -242,7 +242,7 @@ func (a *dashboardSqlAccess) WatchWriteEvents(ctx context.Context) (<-chan *reso
} }
// Simple wrapper for index implementation // Simple wrapper for index implementation
func (a *dashboardSqlAccess) Read(ctx context.Context, req *resource.ReadRequest) (*resource.ReadResponse, error) { func (a *dashboardSqlAccess) Read(ctx context.Context, req *resource.ReadRequest) (*resource.BackendReadResponse, error) {
return a.ReadResource(ctx, req), nil return a.ReadResource(ctx, req), nil
} }

View File

@ -138,7 +138,7 @@ func (s *cdkBackend) WriteEvent(ctx context.Context, event WriteEvent) (rv int64
return rv, err return rv, err
} }
func (s *cdkBackend) ReadResource(ctx context.Context, req *ReadRequest) *ReadResponse { func (s *cdkBackend) ReadResource(ctx context.Context, req *ReadRequest) *BackendReadResponse {
rv := req.ResourceVersion rv := req.ResourceVersion
path := s.getPath(req.Key, rv) path := s.getPath(req.Key, rv)
@ -166,7 +166,7 @@ func (s *cdkBackend) ReadResource(ctx context.Context, req *ReadRequest) *ReadRe
raw, err := s.bucket.ReadAll(ctx, path) raw, err := s.bucket.ReadAll(ctx, path)
if raw == nil && req.ResourceVersion > 0 { if raw == nil && req.ResourceVersion > 0 {
if req.ResourceVersion > s.rv.Load() { if req.ResourceVersion > s.rv.Load() {
return &ReadResponse{ return &BackendReadResponse{
Error: &ErrorResult{ Error: &ErrorResult{
Code: http.StatusGatewayTimeout, Code: http.StatusGatewayTimeout,
Reason: string(metav1.StatusReasonTimeout), // match etcd behavior Reason: string(metav1.StatusReasonTimeout), // match etcd behavior
@ -195,9 +195,11 @@ func (s *cdkBackend) ReadResource(ctx context.Context, req *ReadRequest) *ReadRe
raw = nil raw = nil
} }
if raw == nil { if raw == nil {
return &ReadResponse{Error: NewNotFoundError(req.Key)} return &BackendReadResponse{Error: NewNotFoundError(req.Key)}
} }
return &ReadResponse{ return &BackendReadResponse{
Key: req.Key,
Folder: "", // TODO: implement this
ResourceVersion: rv, ResourceVersion: rv,
Value: raw, Value: raw,
} }
@ -322,6 +324,10 @@ func (c *cdkListIterator) Namespace() string {
return c.currentKey // TODO (parse namespace from key) return c.currentKey // TODO (parse namespace from key)
} }
func (c *cdkListIterator) Folder() string {
return "" // TODO: implement this
}
var _ ListIterator = (*cdkListIterator)(nil) var _ ListIterator = (*cdkListIterator)(nil)
func buildTree(ctx context.Context, s *cdkBackend, key *ResourceKey) (*cdkListIterator, error) { func buildTree(ctx context.Context, s *cdkBackend, key *ResourceKey) (*cdkListIterator, error) {

View File

@ -24,6 +24,8 @@ type WriteEvent struct {
// WriteEvents after they include a resource version // WriteEvents after they include a resource version
type WrittenEvent struct { type WrittenEvent struct {
WriteEvent WriteEvent
// Metadata
Folder string
// The resource version // The resource version
ResourceVersion int64 ResourceVersion int64

View File

@ -50,10 +50,27 @@ type ListIterator interface {
// Used for fast(er) authz filtering // Used for fast(er) authz filtering
Name() string Name() string
// Folder of the current item
// Used for fast(er) authz filtering
Folder() string
// Value for the current item // Value for the current item
Value() []byte Value() []byte
} }
type BackendReadResponse struct {
// Metadata
Key *ResourceKey
Folder string
// The new resource version
ResourceVersion int64
// The properties
Value []byte
// Error details
Error *ErrorResult
}
// The StorageBackend is an internal abstraction that supports interacting with // The StorageBackend is an internal abstraction that supports interacting with
// the underlying raw storage medium. This interface is never exposed directly, // the underlying raw storage medium. This interface is never exposed directly,
// it is provided by concrete instances that actually write values. // it is provided by concrete instances that actually write values.
@ -64,7 +81,7 @@ type StorageBackend interface {
WriteEvent(context.Context, WriteEvent) (int64, error) WriteEvent(context.Context, WriteEvent) (int64, error)
// Read a resource from storage optionally at an explicit version // Read a resource from storage optionally at an explicit version
ReadResource(context.Context, *ReadRequest) *ReadResponse ReadResource(context.Context, *ReadRequest) *BackendReadResponse
// When the ResourceServer executes a List request, this iterator will // When the ResourceServer executes a List request, this iterator will
// query the backend for potential results. All results will be // query the backend for potential results. All results will be
@ -543,7 +560,11 @@ func (s *server) Read(ctx context.Context, req *ReadRequest) (*ReadResponse, err
rsp := s.backend.ReadResource(ctx, req) rsp := s.backend.ReadResource(ctx, req)
// TODO, check folder permissions etc // TODO, check folder permissions etc
return rsp, nil return &ReadResponse{
ResourceVersion: rsp.ResourceVersion,
Value: rsp.Value,
Error: rsp.Error,
}, nil
} }
func (s *server) List(ctx context.Context, req *ListRequest) (*ListResponse, error) { func (s *server) List(ctx context.Context, req *ListRequest) (*ListResponse, error) {

View File

@ -168,12 +168,15 @@ func (b *backend) create(ctx context.Context, event resource.WriteEvent) (int64,
var newVersion int64 var newVersion int64
guid := uuid.New().String() guid := uuid.New().String()
err := b.db.WithTx(ctx, ReadCommitted, func(ctx context.Context, tx db.Tx) error { err := b.db.WithTx(ctx, ReadCommitted, func(ctx context.Context, tx db.Tx) error {
// TODO: Set the Labels folder := ""
if event.Object != nil {
folder = event.Object.GetFolder()
}
// 1. Insert into resource // 1. Insert into resource
if _, err := dbutil.Exec(ctx, tx, sqlResourceInsert, sqlResourceRequest{ if _, err := dbutil.Exec(ctx, tx, sqlResourceInsert, sqlResourceRequest{
SQLTemplate: sqltemplate.New(b.dialect), SQLTemplate: sqltemplate.New(b.dialect),
WriteEvent: event, WriteEvent: event,
Folder: folder,
GUID: guid, GUID: guid,
}); err != nil { }); err != nil {
return fmt.Errorf("insert into resource: %w", err) return fmt.Errorf("insert into resource: %w", err)
@ -183,6 +186,7 @@ func (b *backend) create(ctx context.Context, event resource.WriteEvent) (int64,
if _, err := dbutil.Exec(ctx, tx, sqlResourceHistoryInsert, sqlResourceRequest{ if _, err := dbutil.Exec(ctx, tx, sqlResourceHistoryInsert, sqlResourceRequest{
SQLTemplate: sqltemplate.New(b.dialect), SQLTemplate: sqltemplate.New(b.dialect),
WriteEvent: event, WriteEvent: event,
Folder: folder,
GUID: guid, GUID: guid,
}); err != nil { }); err != nil {
return fmt.Errorf("insert into resource history: %w", err) return fmt.Errorf("insert into resource history: %w", err)
@ -225,12 +229,15 @@ func (b *backend) update(ctx context.Context, event resource.WriteEvent) (int64,
var newVersion int64 var newVersion int64
guid := uuid.New().String() guid := uuid.New().String()
err := b.db.WithTx(ctx, ReadCommitted, func(ctx context.Context, tx db.Tx) error { err := b.db.WithTx(ctx, ReadCommitted, func(ctx context.Context, tx db.Tx) error {
// TODO: Set the Labels folder := ""
if event.Object != nil {
folder = event.Object.GetFolder()
}
// 1. Update resource // 1. Update resource
_, err := dbutil.Exec(ctx, tx, sqlResourceUpdate, sqlResourceRequest{ _, err := dbutil.Exec(ctx, tx, sqlResourceUpdate, sqlResourceRequest{
SQLTemplate: sqltemplate.New(b.dialect), SQLTemplate: sqltemplate.New(b.dialect),
WriteEvent: event, WriteEvent: event,
Folder: folder,
GUID: guid, GUID: guid,
}) })
if err != nil { if err != nil {
@ -241,6 +248,7 @@ func (b *backend) update(ctx context.Context, event resource.WriteEvent) (int64,
if _, err := dbutil.Exec(ctx, tx, sqlResourceHistoryInsert, sqlResourceRequest{ if _, err := dbutil.Exec(ctx, tx, sqlResourceHistoryInsert, sqlResourceRequest{
SQLTemplate: sqltemplate.New(b.dialect), SQLTemplate: sqltemplate.New(b.dialect),
WriteEvent: event, WriteEvent: event,
Folder: folder,
GUID: guid, GUID: guid,
}); err != nil { }); err != nil {
return fmt.Errorf("insert into resource history: %w", err) return fmt.Errorf("insert into resource history: %w", err)
@ -285,8 +293,10 @@ func (b *backend) delete(ctx context.Context, event resource.WriteEvent) (int64,
guid := uuid.New().String() guid := uuid.New().String()
err := b.db.WithTx(ctx, ReadCommitted, func(ctx context.Context, tx db.Tx) error { err := b.db.WithTx(ctx, ReadCommitted, func(ctx context.Context, tx db.Tx) error {
// TODO: Set the Labels folder := ""
if event.Object != nil {
folder = event.Object.GetFolder()
}
// 1. delete from resource // 1. delete from resource
_, err := dbutil.Exec(ctx, tx, sqlResourceDelete, sqlResourceRequest{ _, err := dbutil.Exec(ctx, tx, sqlResourceDelete, sqlResourceRequest{
SQLTemplate: sqltemplate.New(b.dialect), SQLTemplate: sqltemplate.New(b.dialect),
@ -301,6 +311,7 @@ func (b *backend) delete(ctx context.Context, event resource.WriteEvent) (int64,
if _, err := dbutil.Exec(ctx, tx, sqlResourceHistoryInsert, sqlResourceRequest{ if _, err := dbutil.Exec(ctx, tx, sqlResourceHistoryInsert, sqlResourceRequest{
SQLTemplate: sqltemplate.New(b.dialect), SQLTemplate: sqltemplate.New(b.dialect),
WriteEvent: event, WriteEvent: event,
Folder: folder,
GUID: guid, GUID: guid,
}); err != nil { }); err != nil {
return fmt.Errorf("insert into resource history: %w", err) return fmt.Errorf("insert into resource history: %w", err)
@ -330,7 +341,7 @@ func (b *backend) delete(ctx context.Context, event resource.WriteEvent) (int64,
return newVersion, err return newVersion, err
} }
func (b *backend) ReadResource(ctx context.Context, req *resource.ReadRequest) *resource.ReadResponse { func (b *backend) ReadResource(ctx context.Context, req *resource.ReadRequest) *resource.BackendReadResponse {
_, span := b.tracer.Start(ctx, tracePrefix+".Read") _, span := b.tracer.Start(ctx, tracePrefix+".Read")
defer span.End() defer span.End()
@ -339,7 +350,7 @@ func (b *backend) ReadResource(ctx context.Context, req *resource.ReadRequest) *
readReq := &sqlResourceReadRequest{ readReq := &sqlResourceReadRequest{
SQLTemplate: sqltemplate.New(b.dialect), SQLTemplate: sqltemplate.New(b.dialect),
Request: req, Request: req,
readResponse: new(readResponse), Response: NewReadResponse(),
} }
sr := sqlResourceRead sr := sqlResourceRead
@ -348,21 +359,21 @@ func (b *backend) ReadResource(ctx context.Context, req *resource.ReadRequest) *
sr = sqlResourceHistoryRead sr = sqlResourceHistoryRead
} }
var res *readResponse var res *resource.BackendReadResponse
err := b.db.WithTx(ctx, ReadCommittedRO, func(ctx context.Context, tx db.Tx) error { err := b.db.WithTx(ctx, ReadCommittedRO, func(ctx context.Context, tx db.Tx) error {
var err error var err error
res, err = dbutil.QueryRow(ctx, tx, sr, readReq) res, err = dbutil.QueryRow(ctx, tx, sr, readReq)
return err return err
}) })
if errors.Is(err, sql.ErrNoRows) { if errors.Is(err, sql.ErrNoRows) {
return &resource.ReadResponse{ return &resource.BackendReadResponse{
Error: resource.NewNotFoundError(req.Key), Error: resource.NewNotFoundError(req.Key),
} }
} else if err != nil { } else if err != nil {
return &resource.ReadResponse{Error: resource.AsErrorResult(err)} return &resource.BackendReadResponse{Error: resource.AsErrorResult(err)}
} }
return &res.ReadResponse return res
} }
func (b *backend) ListIterator(ctx context.Context, req *resource.ListRequest, cb func(resource.ListIterator) error) (int64, error) { func (b *backend) ListIterator(ctx context.Context, req *resource.ListRequest, cb func(resource.ListIterator) error) (int64, error) {
@ -396,6 +407,7 @@ type listIter struct {
value []byte value []byte
namespace string namespace string
name string name string
folder string
} }
// ContinueToken implements resource.ListIterator. // ContinueToken implements resource.ListIterator.
@ -403,21 +415,22 @@ func (l *listIter) ContinueToken() string {
return ContinueToken{ResourceVersion: l.listRV, StartOffset: l.offset}.String() return ContinueToken{ResourceVersion: l.listRV, StartOffset: l.offset}.String()
} }
// Error implements resource.ListIterator.
func (l *listIter) Error() error { func (l *listIter) Error() error {
return l.err return l.err
} }
// Name implements resource.ListIterator.
func (l *listIter) Name() string { func (l *listIter) Name() string {
return l.name return l.name
} }
// Namespace implements resource.ListIterator.
func (l *listIter) Namespace() string { func (l *listIter) Namespace() string {
return l.namespace return l.namespace
} }
func (l *listIter) Folder() string {
return l.folder
}
// ResourceVersion implements resource.ListIterator. // ResourceVersion implements resource.ListIterator.
func (l *listIter) ResourceVersion() int64 { func (l *listIter) ResourceVersion() int64 {
return l.rv return l.rv
@ -432,7 +445,7 @@ func (l *listIter) Value() []byte {
func (l *listIter) Next() bool { func (l *listIter) Next() bool {
if l.rows.Next() { if l.rows.Next() {
l.offset++ l.offset++
l.err = l.rows.Scan(&l.rv, &l.namespace, &l.name, &l.value) l.err = l.rows.Scan(&l.rv, &l.namespace, &l.name, &l.folder, &l.value)
return true return true
} }
return false return false
@ -680,6 +693,7 @@ func (b *backend) poll(ctx context.Context, grp string, res string, since int64,
Type: resource.WatchEvent_Type(rec.Action), Type: resource.WatchEvent_Type(rec.Action),
PreviousRV: *prevRV, PreviousRV: *prevRV,
}, },
Folder: rec.Folder,
ResourceVersion: rec.ResourceVersion, ResourceVersion: rec.ResourceVersion,
// Timestamp: , // TODO: add timestamp // Timestamp: , // TODO: add timestamp
} }

View File

@ -8,7 +8,9 @@ import (
sqlmock "github.com/DATA-DOG/go-sqlmock" sqlmock "github.com/DATA-DOG/go-sqlmock"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/storage/unified/resource" "github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl" "github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl"
"github.com/grafana/grafana/pkg/storage/unified/sql/test" "github.com/grafana/grafana/pkg/storage/unified/sql/test"
@ -281,9 +283,14 @@ func TestResourceVersionAtomicInc(t *testing.T) {
func TestBackend_create(t *testing.T) { func TestBackend_create(t *testing.T) {
t.Parallel() t.Parallel()
meta, err := utils.MetaAccessor(&unstructured.Unstructured{
Object: map[string]any{},
})
require.NoError(t, err)
event := resource.WriteEvent{ event := resource.WriteEvent{
Type: resource.WatchEvent_ADDED, Type: resource.WatchEvent_ADDED,
Key: resKey, Key: resKey,
Object: meta,
} }
t.Run("happy path", func(t *testing.T) { t.Run("happy path", func(t *testing.T) {
@ -386,9 +393,15 @@ func TestBackend_create(t *testing.T) {
func TestBackend_update(t *testing.T) { func TestBackend_update(t *testing.T) {
t.Parallel() t.Parallel()
meta, err := utils.MetaAccessor(&unstructured.Unstructured{
Object: map[string]any{},
})
require.NoError(t, err)
meta.SetFolder("folderuid")
event := resource.WriteEvent{ event := resource.WriteEvent{
Type: resource.WatchEvent_MODIFIED, Type: resource.WatchEvent_MODIFIED,
Key: resKey, Key: resKey,
Object: meta,
} }
t.Run("happy path", func(t *testing.T) { t.Run("happy path", func(t *testing.T) {
@ -491,9 +504,14 @@ func TestBackend_update(t *testing.T) {
func TestBackend_delete(t *testing.T) { func TestBackend_delete(t *testing.T) {
t.Parallel() t.Parallel()
meta, err := utils.MetaAccessor(&unstructured.Unstructured{
Object: map[string]any{},
})
require.NoError(t, err)
event := resource.WriteEvent{ event := resource.WriteEvent{
Type: resource.WatchEvent_DELETED, Type: resource.WatchEvent_DELETED,
Key: resKey, Key: resKey,
Object: meta,
} }
t.Run("happy path", func(t *testing.T) { t.Run("happy path", func(t *testing.T) {

View File

@ -5,6 +5,7 @@ INSERT INTO {{ .Ident "resource_history" }}
{{ .Ident "resource" }}, {{ .Ident "resource" }},
{{ .Ident "namespace" }}, {{ .Ident "namespace" }},
{{ .Ident "name" }}, {{ .Ident "name" }},
{{ .Ident "folder" }},
{{ .Ident "previous_resource_version"}}, {{ .Ident "previous_resource_version"}},
{{ .Ident "value" }}, {{ .Ident "value" }},
@ -17,6 +18,7 @@ INSERT INTO {{ .Ident "resource_history" }}
{{ .Arg .WriteEvent.Key.Resource }}, {{ .Arg .WriteEvent.Key.Resource }},
{{ .Arg .WriteEvent.Key.Namespace }}, {{ .Arg .WriteEvent.Key.Namespace }},
{{ .Arg .WriteEvent.Key.Name }}, {{ .Arg .WriteEvent.Key.Name }},
{{ .Arg .Folder }},
{{ .Arg .WriteEvent.PreviousRV }}, {{ .Arg .WriteEvent.PreviousRV }},
{{ .Arg .WriteEvent.Value }}, {{ .Arg .WriteEvent.Value }},

View File

@ -2,6 +2,7 @@ SELECT
kv.{{ .Ident "resource_version" }}, kv.{{ .Ident "resource_version" }},
kv.{{ .Ident "namespace" }}, kv.{{ .Ident "namespace" }},
kv.{{ .Ident "name" }}, kv.{{ .Ident "name" }},
kv.{{ .Ident "folder" }},
kv.{{ .Ident "value" }} kv.{{ .Ident "value" }}
FROM {{ .Ident "resource_history" }} as kv FROM {{ .Ident "resource_history" }} as kv
INNER JOIN ( INNER JOIN (

View File

@ -4,6 +4,7 @@ SELECT
{{ .Ident "group" | .Into .Response.Key.Group }}, {{ .Ident "group" | .Into .Response.Key.Group }},
{{ .Ident "resource" | .Into .Response.Key.Resource }}, {{ .Ident "resource" | .Into .Response.Key.Resource }},
{{ .Ident "name" | .Into .Response.Key.Name }}, {{ .Ident "name" | .Into .Response.Key.Name }},
{{ .Ident "folder" | .Into .Response.Folder }},
{{ .Ident "value" | .Into .Response.Value }}, {{ .Ident "value" | .Into .Response.Value }},
{{ .Ident "action" | .Into .Response.Action }}, {{ .Ident "action" | .Into .Response.Action }},
{{ .Ident "previous_resource_version" | .Into .Response.PreviousRV }} {{ .Ident "previous_resource_version" | .Into .Response.PreviousRV }}

View File

@ -1,6 +1,11 @@
SELECT SELECT
{{ .Ident "resource_version" | .Into .ResourceVersion }}, {{ .Ident "namespace" | .Into .Response.Key.Namespace }},
{{ .Ident "value" | .Into .Value }} {{ .Ident "group" | .Into .Response.Key.Group }},
{{ .Ident "resource" | .Into .Response.Key.Resource }},
{{ .Ident "name" | .Into .Response.Key.Name }},
{{ .Ident "folder" | .Into .Response.Folder }},
{{ .Ident "resource_version" | .Into .Response.ResourceVersion }},
{{ .Ident "value" | .Into .Response.Value }}
FROM {{ .Ident "resource_history" }} FROM {{ .Ident "resource_history" }}

View File

@ -6,6 +6,7 @@ INSERT INTO {{ .Ident "resource" }}
{{ .Ident "resource" }}, {{ .Ident "resource" }},
{{ .Ident "namespace" }}, {{ .Ident "namespace" }},
{{ .Ident "name" }}, {{ .Ident "name" }},
{{ .Ident "folder" }},
{{ .Ident "previous_resource_version" }}, {{ .Ident "previous_resource_version" }},
{{ .Ident "value" }}, {{ .Ident "value" }},
@ -17,6 +18,7 @@ INSERT INTO {{ .Ident "resource" }}
{{ .Arg .WriteEvent.Key.Resource }}, {{ .Arg .WriteEvent.Key.Resource }},
{{ .Arg .WriteEvent.Key.Namespace }}, {{ .Arg .WriteEvent.Key.Namespace }},
{{ .Arg .WriteEvent.Key.Name }}, {{ .Arg .WriteEvent.Key.Name }},
{{ .Arg .Folder }},
{{ .Arg .WriteEvent.PreviousRV }}, {{ .Arg .WriteEvent.PreviousRV }},
{{ .Arg .WriteEvent.Value }}, {{ .Arg .WriteEvent.Value }},

View File

@ -2,6 +2,7 @@ SELECT
{{ .Ident "resource_version" }}, {{ .Ident "resource_version" }},
{{ .Ident "namespace" }}, {{ .Ident "namespace" }},
{{ .Ident "name" }}, {{ .Ident "name" }},
{{ .Ident "folder" }},
{{ .Ident "value" }} {{ .Ident "value" }}
FROM {{ .Ident "resource" }} FROM {{ .Ident "resource" }}
WHERE 1 = 1 WHERE 1 = 1

View File

@ -1,6 +1,11 @@
SELECT SELECT
{{ .Ident "resource_version" | .Into .ResourceVersion }}, {{ .Ident "namespace" | .Into .Response.Key.Namespace }},
{{ .Ident "value" | .Into .Value }} {{ .Ident "group" | .Into .Response.Key.Group }},
{{ .Ident "resource" | .Into .Response.Key.Resource }},
{{ .Ident "name" | .Into .Response.Key.Name }},
{{ .Ident "folder" | .Into .Response.Folder }},
{{ .Ident "resource_version" | .Into .Response.ResourceVersion }},
{{ .Ident "value" | .Into .Response.Value }}
FROM {{ .Ident "resource" }} FROM {{ .Ident "resource" }}
WHERE 1 = 1 WHERE 1 = 1
AND {{ .Ident "namespace" }} = {{ .Arg .Request.Key.Namespace }} AND {{ .Ident "namespace" }} = {{ .Arg .Request.Key.Namespace }}

View File

@ -2,6 +2,7 @@ UPDATE {{ .Ident "resource" }}
SET SET
{{ .Ident "guid" }} = {{ .Arg .GUID }}, {{ .Ident "guid" }} = {{ .Arg .GUID }},
{{ .Ident "value" }} = {{ .Arg .WriteEvent.Value }}, {{ .Ident "value" }} = {{ .Arg .WriteEvent.Value }},
{{ .Ident "folder" }} = {{ .Arg .Folder }},
{{ .Ident "action" }} = {{ .Arg .WriteEvent.Type }} {{ .Ident "action" }} = {{ .Arg .WriteEvent.Type }}
WHERE 1 = 1 WHERE 1 = 1
AND {{ .Ident "group" }} = {{ .Arg .WriteEvent.Key.Group }} AND {{ .Ident "group" }} = {{ .Arg .WriteEvent.Key.Group }}

View File

@ -113,5 +113,13 @@ func initResourceTables(mg *migrator.Migrator) string {
Cols: []string{"group", "resource"}, Type: migrator.IndexType, Cols: []string{"group", "resource"}, Type: migrator.IndexType,
})) }))
mg.AddMigration("Add column folder in resource_history", migrator.NewAddColumnMigration(resource_history_table, &migrator.Column{
Name: "folder", Type: migrator.DB_NVarchar, Length: 253, Nullable: false, Default: "",
}))
mg.AddMigration("Add column folder in resource", migrator.NewAddColumnMigration(resource_table, &migrator.Column{
Name: "folder", Type: migrator.DB_NVarchar, Length: 253, Nullable: false, Default: "",
}))
return marker return marker
} }

View File

@ -64,6 +64,7 @@ type sqlResourceRequest struct {
sqltemplate.SQLTemplate sqltemplate.SQLTemplate
GUID string GUID string
WriteEvent resource.WriteEvent WriteEvent resource.WriteEvent
Folder string
} }
func (r sqlResourceRequest) Validate() error { func (r sqlResourceRequest) Validate() error {
@ -76,6 +77,7 @@ type historyPollResponse struct {
PreviousRV *int64 PreviousRV *int64
Value []byte Value []byte
Action int Action int
Folder string
} }
func (r *historyPollResponse) Results() (*historyPollResponse, error) { func (r *historyPollResponse) Results() (*historyPollResponse, error) {
@ -108,6 +110,7 @@ func (r *sqlResourceHistoryPollRequest) Results() (*historyPollResponse, error)
Resource: r.Response.Key.Resource, Resource: r.Response.Key.Resource,
Name: r.Response.Key.Name, Name: r.Response.Key.Name,
}, },
Folder: r.Response.Folder,
ResourceVersion: r.Response.ResourceVersion, ResourceVersion: r.Response.ResourceVersion,
PreviousRV: prevRV, PreviousRV: prevRV,
Value: r.Response.Value, Value: r.Response.Value,
@ -116,33 +119,24 @@ func (r *sqlResourceHistoryPollRequest) Results() (*historyPollResponse, error)
} }
// sqlResourceReadRequest can be used to retrieve a row fromthe "resource" tables. // sqlResourceReadRequest can be used to retrieve a row fromthe "resource" tables.
func NewReadResponse() *resource.BackendReadResponse {
type readResponse struct { return &resource.BackendReadResponse{
resource.ReadResponse Key: &resource.ResourceKey{},
} }
func (r *readResponse) Results() (*readResponse, error) {
return r, nil
} }
type sqlResourceReadRequest struct { type sqlResourceReadRequest struct {
sqltemplate.SQLTemplate sqltemplate.SQLTemplate
Request *resource.ReadRequest Request *resource.ReadRequest
*readResponse Response *resource.BackendReadResponse
} }
func (r *sqlResourceReadRequest) Validate() error { func (r *sqlResourceReadRequest) Validate() error {
return nil // TODO return nil // TODO
} }
func (r *sqlResourceReadRequest) Results() (*readResponse, error) { func (r *sqlResourceReadRequest) Results() (*resource.BackendReadResponse, error) {
return &readResponse{ return r.Response, nil
ReadResponse: resource.ReadResponse{
Error: r.ReadResponse.Error,
ResourceVersion: r.ReadResponse.ResourceVersion,
Value: r.ReadResponse.Value,
},
}, nil
} }
// List // List
@ -157,6 +151,7 @@ func (r sqlResourceListRequest) Validate() error {
type historyListRequest struct { type historyListRequest struct {
ResourceVersion, Limit, Offset int64 ResourceVersion, Limit, Offset int64
Folder string
Options *resource.ListOptions Options *resource.ListOptions
} }
type sqlResourceHistoryListRequest struct { type sqlResourceHistoryListRequest struct {

View File

@ -43,6 +43,7 @@ func TestUnifiedStorageQueries(t *testing.T) {
Type: resource.WatchEvent_ADDED, Type: resource.WatchEvent_ADDED,
PreviousRV: 123, PreviousRV: 123,
}, },
Folder: "fldr",
}, },
}, },
}, },
@ -59,6 +60,7 @@ func TestUnifiedStorageQueries(t *testing.T) {
Name: "name", Name: "name",
}, },
}, },
Folder: "fldr",
}, },
}, },
}, },
@ -75,7 +77,7 @@ func TestUnifiedStorageQueries(t *testing.T) {
Name: "name", Name: "name",
}, },
}, },
readResponse: new(readResponse), Response: NewReadResponse(),
}, },
}, },
}, },
@ -143,9 +145,14 @@ func TestUnifiedStorageQueries(t *testing.T) {
SQLTemplate: mocks.NewTestingSQLTemplate(), SQLTemplate: mocks.NewTestingSQLTemplate(),
Request: &resource.ReadRequest{ Request: &resource.ReadRequest{
ResourceVersion: 123, ResourceVersion: 123,
Key: &resource.ResourceKey{}, Key: &resource.ResourceKey{
Namespace: "ns",
Group: "gp",
Resource: "rs",
Name: "nm",
}, },
readResponse: new(readResponse), },
Response: NewReadResponse(),
}, },
}, },
}, },
@ -173,6 +180,7 @@ func TestUnifiedStorageQueries(t *testing.T) {
}, },
PreviousRV: 1234, PreviousRV: 1234,
}, },
Folder: "fldr",
}, },
}, },
}, },

View File

@ -9,11 +9,13 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/credentials/insecure"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"github.com/grafana/authlib/claims" "github.com/grafana/authlib/claims"
"github.com/grafana/dskit/services" "github.com/grafana/dskit/services"
"github.com/grafana/grafana/pkg/apimachinery/identity" "github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/apimachinery/utils"
infraDB "github.com/grafana/grafana/pkg/infra/db" infraDB "github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/featuremgmt"
@ -63,9 +65,9 @@ func newServer(t *testing.T, cfg *setting.Cfg) (sql.Backend, resource.ResourceSe
} }
func TestIntegrationBackendHappyPath(t *testing.T) { func TestIntegrationBackendHappyPath(t *testing.T) {
if infraDB.IsTestDbSQLite() { // if infraDB.IsTestDbSQLite() {
t.Skip("TODO: test blocking, skipping to unblock Enterprise until we fix this") // t.Skip("TODO: test blocking, skipping to unblock Enterprise until we fix this")
} // }
if testing.Short() { if testing.Short() {
t.Skip("skipping integration test") t.Skip("skipping integration test")
} }
@ -116,6 +118,7 @@ func TestIntegrationBackendHappyPath(t *testing.T) {
require.Nil(t, resp.Error) require.Nil(t, resp.Error)
require.Equal(t, rv4, resp.ResourceVersion) require.Equal(t, rv4, resp.ResourceVersion)
require.Equal(t, "item2 MODIFIED", string(resp.Value)) require.Equal(t, "item2 MODIFIED", string(resp.Value))
require.Equal(t, "folderuid", resp.Folder)
}) })
t.Run("Read early version of item2", func(t *testing.T) { t.Run("Read early version of item2", func(t *testing.T) {
@ -151,10 +154,13 @@ func TestIntegrationBackendHappyPath(t *testing.T) {
require.Equal(t, "item1", event.Key.Name) require.Equal(t, "item1", event.Key.Name)
require.Equal(t, rv1, event.ResourceVersion) require.Equal(t, rv1, event.ResourceVersion)
require.Equal(t, resource.WatchEvent_ADDED, event.Type) require.Equal(t, resource.WatchEvent_ADDED, event.Type)
require.Equal(t, "folderuid", event.Folder)
event = <-stream event = <-stream
require.Equal(t, "item2", event.Key.Name) require.Equal(t, "item2", event.Key.Name)
require.Equal(t, rv2, event.ResourceVersion) require.Equal(t, rv2, event.ResourceVersion)
require.Equal(t, resource.WatchEvent_ADDED, event.Type) require.Equal(t, resource.WatchEvent_ADDED, event.Type)
require.Equal(t, "folderuid", event.Folder)
event = <-stream event = <-stream
require.Equal(t, "item3", event.Key.Name) require.Equal(t, "item3", event.Key.Name)
@ -418,6 +424,14 @@ func TestClientServer(t *testing.T) {
} }
func writeEvent(ctx context.Context, store sql.Backend, name string, action resource.WatchEvent_Type) (int64, error) { func writeEvent(ctx context.Context, store sql.Backend, name string, action resource.WatchEvent_Type) (int64, error) {
res := &unstructured.Unstructured{
Object: map[string]any{},
}
meta, err := utils.MetaAccessor(res)
if err != nil {
return 0, err
}
meta.SetFolder("folderuid")
return store.WriteEvent(ctx, resource.WriteEvent{ return store.WriteEvent(ctx, resource.WriteEvent{
Type: action, Type: action,
Value: []byte(name + " " + resource.WatchEvent_Type_name[int32(action)]), Value: []byte(name + " " + resource.WatchEvent_Type_name[int32(action)]),
@ -427,6 +441,7 @@ func writeEvent(ctx context.Context, store sql.Backend, name string, action reso
Resource: "resource", Resource: "resource",
Name: name, Name: name,
}, },
Object: meta,
}) })
} }

View File

@ -5,6 +5,7 @@ INSERT INTO `resource_history`
`resource`, `resource`,
`namespace`, `namespace`,
`name`, `name`,
`folder`,
`previous_resource_version`, `previous_resource_version`,
`value`, `value`,
`action` `action`
@ -15,6 +16,7 @@ INSERT INTO `resource_history`
'rr', 'rr',
'nn', 'nn',
'name', 'name',
'fldr',
1234, 1234,
'[]', '[]',
'UNKNOWN' 'UNKNOWN'

View File

@ -2,6 +2,7 @@ SELECT
kv.`resource_version`, kv.`resource_version`,
kv.`namespace`, kv.`namespace`,
kv.`name`, kv.`name`,
kv.`folder`,
kv.`value` kv.`value`
FROM `resource_history` as kv FROM `resource_history` as kv
INNER JOIN ( INNER JOIN (

View File

@ -4,6 +4,7 @@ SELECT
`group`, `group`,
`resource`, `resource`,
`name`, `name`,
`folder`,
`value`, `value`,
`action`, `action`,
`previous_resource_version` `previous_resource_version`

View File

@ -1,12 +1,17 @@
SELECT SELECT
`namespace`,
`group`,
`resource`,
`name`,
`folder`,
`resource_version`, `resource_version`,
`value` `value`
FROM `resource_history` FROM `resource_history`
WHERE 1 = 1 WHERE 1 = 1
AND `namespace` = '' AND `namespace` = 'ns'
AND `group` = '' AND `group` = 'gp'
AND `resource` = '' AND `resource` = 'rs'
AND `name` = '' AND `name` = 'nm'
AND `resource_version` <= 123 AND `resource_version` <= 123
ORDER BY `resource_version` DESC ORDER BY `resource_version` DESC
LIMIT 1 LIMIT 1

View File

@ -5,6 +5,7 @@ INSERT INTO `resource`
`resource`, `resource`,
`namespace`, `namespace`,
`name`, `name`,
`folder`,
`previous_resource_version`, `previous_resource_version`,
`value`, `value`,
`action` `action`
@ -15,6 +16,7 @@ INSERT INTO `resource`
'rr', 'rr',
'nn', 'nn',
'name', 'name',
'fldr',
123, 123,
'[]', '[]',
'ADDED' 'ADDED'

View File

@ -2,6 +2,7 @@ SELECT
`resource_version`, `resource_version`,
`namespace`, `namespace`,
`name`, `name`,
`folder`,
`value` `value`
FROM `resource` FROM `resource`
WHERE 1 = 1 WHERE 1 = 1

View File

@ -1,4 +1,9 @@
SELECT SELECT
`namespace`,
`group`,
`resource`,
`name`,
`folder`,
`resource_version`, `resource_version`,
`value` `value`
FROM `resource` FROM `resource`

View File

@ -2,6 +2,7 @@ UPDATE `resource`
SET SET
`guid` = '', `guid` = '',
`value` = '[]', `value` = '[]',
`folder` = 'fldr',
`action` = 'UNKNOWN' `action` = 'UNKNOWN'
WHERE 1 = 1 WHERE 1 = 1
AND `group` = 'gg' AND `group` = 'gg'

View File

@ -5,6 +5,7 @@ INSERT INTO "resource_history"
"resource", "resource",
"namespace", "namespace",
"name", "name",
"folder",
"previous_resource_version", "previous_resource_version",
"value", "value",
"action" "action"
@ -15,6 +16,7 @@ INSERT INTO "resource_history"
'rr', 'rr',
'nn', 'nn',
'name', 'name',
'fldr',
1234, 1234,
'[]', '[]',
'UNKNOWN' 'UNKNOWN'

View File

@ -2,6 +2,7 @@ SELECT
kv."resource_version", kv."resource_version",
kv."namespace", kv."namespace",
kv."name", kv."name",
kv."folder",
kv."value" kv."value"
FROM "resource_history" as kv FROM "resource_history" as kv
INNER JOIN ( INNER JOIN (

View File

@ -4,6 +4,7 @@ SELECT
"group", "group",
"resource", "resource",
"name", "name",
"folder",
"value", "value",
"action", "action",
"previous_resource_version" "previous_resource_version"

View File

@ -1,12 +1,17 @@
SELECT SELECT
"namespace",
"group",
"resource",
"name",
"folder",
"resource_version", "resource_version",
"value" "value"
FROM "resource_history" FROM "resource_history"
WHERE 1 = 1 WHERE 1 = 1
AND "namespace" = '' AND "namespace" = 'ns'
AND "group" = '' AND "group" = 'gp'
AND "resource" = '' AND "resource" = 'rs'
AND "name" = '' AND "name" = 'nm'
AND "resource_version" <= 123 AND "resource_version" <= 123
ORDER BY "resource_version" DESC ORDER BY "resource_version" DESC
LIMIT 1 LIMIT 1

View File

@ -5,6 +5,7 @@ INSERT INTO "resource"
"resource", "resource",
"namespace", "namespace",
"name", "name",
"folder",
"previous_resource_version", "previous_resource_version",
"value", "value",
"action" "action"
@ -15,6 +16,7 @@ INSERT INTO "resource"
'rr', 'rr',
'nn', 'nn',
'name', 'name',
'fldr',
123, 123,
'[]', '[]',
'ADDED' 'ADDED'

View File

@ -2,6 +2,7 @@ SELECT
"resource_version", "resource_version",
"namespace", "namespace",
"name", "name",
"folder",
"value" "value"
FROM "resource" FROM "resource"
WHERE 1 = 1 WHERE 1 = 1

View File

@ -1,4 +1,9 @@
SELECT SELECT
"namespace",
"group",
"resource",
"name",
"folder",
"resource_version", "resource_version",
"value" "value"
FROM "resource" FROM "resource"

View File

@ -2,6 +2,7 @@ UPDATE "resource"
SET SET
"guid" = '', "guid" = '',
"value" = '[]', "value" = '[]',
"folder" = 'fldr',
"action" = 'UNKNOWN' "action" = 'UNKNOWN'
WHERE 1 = 1 WHERE 1 = 1
AND "group" = 'gg' AND "group" = 'gg'

View File

@ -5,6 +5,7 @@ INSERT INTO "resource_history"
"resource", "resource",
"namespace", "namespace",
"name", "name",
"folder",
"previous_resource_version", "previous_resource_version",
"value", "value",
"action" "action"
@ -15,6 +16,7 @@ INSERT INTO "resource_history"
'rr', 'rr',
'nn', 'nn',
'name', 'name',
'fldr',
1234, 1234,
'[]', '[]',
'UNKNOWN' 'UNKNOWN'

View File

@ -2,6 +2,7 @@ SELECT
kv."resource_version", kv."resource_version",
kv."namespace", kv."namespace",
kv."name", kv."name",
kv."folder",
kv."value" kv."value"
FROM "resource_history" as kv FROM "resource_history" as kv
INNER JOIN ( INNER JOIN (

View File

@ -4,6 +4,7 @@ SELECT
"group", "group",
"resource", "resource",
"name", "name",
"folder",
"value", "value",
"action", "action",
"previous_resource_version" "previous_resource_version"

View File

@ -1,12 +1,17 @@
SELECT SELECT
"namespace",
"group",
"resource",
"name",
"folder",
"resource_version", "resource_version",
"value" "value"
FROM "resource_history" FROM "resource_history"
WHERE 1 = 1 WHERE 1 = 1
AND "namespace" = '' AND "namespace" = 'ns'
AND "group" = '' AND "group" = 'gp'
AND "resource" = '' AND "resource" = 'rs'
AND "name" = '' AND "name" = 'nm'
AND "resource_version" <= 123 AND "resource_version" <= 123
ORDER BY "resource_version" DESC ORDER BY "resource_version" DESC
LIMIT 1 LIMIT 1

View File

@ -5,6 +5,7 @@ INSERT INTO "resource"
"resource", "resource",
"namespace", "namespace",
"name", "name",
"folder",
"previous_resource_version", "previous_resource_version",
"value", "value",
"action" "action"
@ -15,6 +16,7 @@ INSERT INTO "resource"
'rr', 'rr',
'nn', 'nn',
'name', 'name',
'fldr',
123, 123,
'[]', '[]',
'ADDED' 'ADDED'

View File

@ -2,6 +2,7 @@ SELECT
"resource_version", "resource_version",
"namespace", "namespace",
"name", "name",
"folder",
"value" "value"
FROM "resource" FROM "resource"
WHERE 1 = 1 WHERE 1 = 1

View File

@ -1,4 +1,9 @@
SELECT SELECT
"namespace",
"group",
"resource",
"name",
"folder",
"resource_version", "resource_version",
"value" "value"
FROM "resource" FROM "resource"

View File

@ -2,6 +2,7 @@ UPDATE "resource"
SET SET
"guid" = '', "guid" = '',
"value" = '[]', "value" = '[]',
"folder" = 'fldr',
"action" = 'UNKNOWN' "action" = 'UNKNOWN'
WHERE 1 = 1 WHERE 1 = 1
AND "group" = 'gg' AND "group" = 'gg'