mirror of
https://github.com/grafana/grafana.git
synced 2025-08-01 05:11:50 +08:00
452 lines
13 KiB
Go
452 lines
13 KiB
Go
package legacy
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
|
|
"google.golang.org/grpc/metadata"
|
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
|
|
authlib "github.com/grafana/authlib/types"
|
|
dashboard "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
|
|
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
|
|
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
|
"github.com/grafana/grafana/pkg/infra/db"
|
|
"github.com/grafana/grafana/pkg/services/librarypanels"
|
|
"github.com/grafana/grafana/pkg/services/provisioning"
|
|
"github.com/grafana/grafana/pkg/services/search/sort"
|
|
"github.com/grafana/grafana/pkg/services/sqlstore"
|
|
"github.com/grafana/grafana/pkg/storage/legacysql"
|
|
"github.com/grafana/grafana/pkg/storage/unified/apistore"
|
|
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
|
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
|
)
|
|
|
|
type MigrateOptions struct {
|
|
Namespace string
|
|
Store resourcepb.BulkStoreClient
|
|
LargeObjects apistore.LargeObjectSupport
|
|
BlobStore resourcepb.BlobStoreClient
|
|
Resources []schema.GroupResource
|
|
WithHistory bool // only applies to dashboards
|
|
OnlyCount bool // just count the values
|
|
Progress func(count int, msg string)
|
|
}
|
|
|
|
// Read from legacy and write into unified storage
|
|
//
|
|
//go:generate mockery --name LegacyMigrator --structname MockLegacyMigrator --inpackage --filename legacy_migrator_mock.go --with-expecter
|
|
type LegacyMigrator interface {
|
|
Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error)
|
|
}
|
|
|
|
// This can migrate Folders, Dashboards and LibraryPanels
|
|
func ProvideLegacyMigrator(
|
|
sql db.DB, // direct access to tables
|
|
provisioning provisioning.ProvisioningService, // only needed for dashboard settings
|
|
libraryPanelSvc librarypanels.Service,
|
|
) LegacyMigrator {
|
|
dbp := legacysql.NewDatabaseProvider(sql)
|
|
return NewDashboardAccess(dbp, authlib.OrgNamespaceFormatter, nil, provisioning, libraryPanelSvc, sort.ProvideService())
|
|
}
|
|
|
|
type BlobStoreInfo struct {
|
|
Count int64
|
|
Size int64
|
|
}
|
|
|
|
// migrate function -- works for a single kind
|
|
type migratorFunc = func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
|
|
|
|
func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
|
info, err := authlib.ParseNamespace(opts.Namespace)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if opts.Progress == nil {
|
|
opts.Progress = func(count int, msg string) {} // noop
|
|
}
|
|
|
|
// Migrate everything
|
|
if len(opts.Resources) < 1 {
|
|
return nil, fmt.Errorf("missing resource selector")
|
|
}
|
|
|
|
migratorFuncs := []migratorFunc{}
|
|
settings := resource.BulkSettings{
|
|
RebuildCollection: true,
|
|
SkipValidation: true,
|
|
}
|
|
|
|
for _, res := range opts.Resources {
|
|
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
|
case "folder.grafana.app/folders":
|
|
migratorFuncs = append(migratorFuncs, a.migrateFolders)
|
|
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
|
Namespace: opts.Namespace,
|
|
Group: folders.GROUP,
|
|
Resource: folders.RESOURCE,
|
|
})
|
|
|
|
case "dashboard.grafana.app/librarypanels":
|
|
migratorFuncs = append(migratorFuncs, a.migratePanels)
|
|
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
|
Namespace: opts.Namespace,
|
|
Group: dashboard.GROUP,
|
|
Resource: dashboard.LIBRARY_PANEL_RESOURCE,
|
|
})
|
|
|
|
case "dashboard.grafana.app/dashboards":
|
|
migratorFuncs = append(migratorFuncs, a.migrateDashboards)
|
|
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
|
Namespace: opts.Namespace,
|
|
Group: dashboard.GROUP,
|
|
Resource: dashboard.DASHBOARD_RESOURCE,
|
|
})
|
|
default:
|
|
return nil, fmt.Errorf("unsupported resource: %s", res)
|
|
}
|
|
}
|
|
if opts.OnlyCount {
|
|
return a.countValues(ctx, opts)
|
|
}
|
|
|
|
ctx = metadata.NewOutgoingContext(ctx, settings.ToMD())
|
|
if md, ok := metadata.FromOutgoingContext(ctx); ok {
|
|
a.log.Debug("bulk grpc request metadata",
|
|
"metadata", md,
|
|
"collection", settings.Collection,
|
|
)
|
|
} else {
|
|
a.log.Debug("bulk grpc request, no metadata found",
|
|
"collection", settings.Collection,
|
|
)
|
|
}
|
|
|
|
stream, err := opts.Store.BulkProcess(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Now run each migration
|
|
blobStore := BlobStoreInfo{}
|
|
a.log.Info("start migrating legacy resources", "namespace", opts.Namespace, "orgId", info.OrgID, "stackId", info.StackID)
|
|
for _, m := range migratorFuncs {
|
|
blobs, err := m(ctx, info.OrgID, opts, stream)
|
|
if err != nil {
|
|
a.log.Error("error migrating legacy resources", "error", err, "namespace", opts.Namespace)
|
|
return nil, err
|
|
}
|
|
if blobs != nil {
|
|
blobStore.Count += blobs.Count
|
|
blobStore.Size += blobs.Size
|
|
}
|
|
}
|
|
a.log.Info("finished migrating legacy resources", "blobStore", blobStore)
|
|
return stream.CloseAndRecv()
|
|
}
|
|
|
|
func (a *dashboardSqlAccess) countValues(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
|
sql, err := a.sql(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
ns, err := authlib.ParseNamespace(opts.Namespace)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
orgId := ns.OrgID
|
|
rsp := &resourcepb.BulkResponse{}
|
|
err = sql.DB.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
|
|
for _, res := range opts.Resources {
|
|
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
|
case "folder.grafana.app/folders":
|
|
summary := &resourcepb.BulkResponse_Summary{}
|
|
summary.Group = folders.GROUP
|
|
summary.Group = folders.RESOURCE
|
|
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
|
|
" WHERE is_folder=TRUE AND org_id=?", orgId).Get(&summary.Count)
|
|
rsp.Summary = append(rsp.Summary, summary)
|
|
|
|
case "dashboard.grafana.app/librarypanels":
|
|
summary := &resourcepb.BulkResponse_Summary{}
|
|
summary.Group = dashboard.GROUP
|
|
summary.Resource = dashboard.LIBRARY_PANEL_RESOURCE
|
|
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("library_element")+
|
|
" WHERE org_id=?", orgId).Get(&summary.Count)
|
|
rsp.Summary = append(rsp.Summary, summary)
|
|
|
|
case "dashboard.grafana.app/dashboards":
|
|
summary := &resourcepb.BulkResponse_Summary{}
|
|
summary.Group = dashboard.GROUP
|
|
summary.Resource = dashboard.DASHBOARD_RESOURCE
|
|
rsp.Summary = append(rsp.Summary, summary)
|
|
|
|
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
|
|
" WHERE is_folder=FALSE AND org_id=?", orgId).Get(&summary.Count)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Also count history
|
|
_, err = sess.SQL(`SELECT COUNT(*)
|
|
FROM `+sql.Table("dashboard_version")+` as dv
|
|
JOIN `+sql.Table("dashboard")+` as dd
|
|
ON dd.id = dv.dashboard_id
|
|
WHERE org_id=?`, orgId).Get(&summary.History)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
return rsp, nil
|
|
}
|
|
|
|
func (a *dashboardSqlAccess) migrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
|
query := &DashboardQuery{
|
|
OrgID: orgId,
|
|
Limit: 100000000,
|
|
GetHistory: opts.WithHistory, // include history
|
|
Order: "ASC", // oldest first
|
|
}
|
|
|
|
blobs := &BlobStoreInfo{}
|
|
sql, err := a.sql(ctx)
|
|
if err != nil {
|
|
return blobs, err
|
|
}
|
|
|
|
opts.Progress(-1, "migrating dashboards...")
|
|
rows, err := a.getRows(ctx, sql, query)
|
|
if rows != nil {
|
|
defer func() {
|
|
_ = rows.Close()
|
|
}()
|
|
}
|
|
if err != nil {
|
|
return blobs, err
|
|
}
|
|
|
|
large := opts.LargeObjects
|
|
|
|
// Now send each dashboard
|
|
for i := 1; rows.Next(); i++ {
|
|
dash := rows.row.Dash
|
|
if dash.APIVersion == "" {
|
|
dash.APIVersion = fmt.Sprintf("%s/v0alpha1", dashboard.GROUP)
|
|
}
|
|
dash.SetNamespace(opts.Namespace)
|
|
dash.SetResourceVersion("") // it will be filled in by the backend
|
|
|
|
body, err := json.Marshal(dash)
|
|
if err != nil {
|
|
err = fmt.Errorf("error reading json from: %s // %w", rows.row.Dash.Name, err)
|
|
return blobs, err
|
|
}
|
|
|
|
req := &resourcepb.BulkRequest{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: opts.Namespace,
|
|
Group: dashboard.GROUP,
|
|
Resource: dashboard.DASHBOARD_RESOURCE,
|
|
Name: rows.Name(),
|
|
},
|
|
Value: body,
|
|
Folder: rows.row.FolderUID,
|
|
Action: resourcepb.BulkRequest_ADDED,
|
|
}
|
|
if dash.Generation > 1 {
|
|
req.Action = resourcepb.BulkRequest_MODIFIED
|
|
} else if dash.Generation < 0 {
|
|
req.Action = resourcepb.BulkRequest_DELETED
|
|
}
|
|
|
|
// With large object support
|
|
if large != nil && len(body) > large.Threshold() {
|
|
obj, err := utils.MetaAccessor(dash)
|
|
if err != nil {
|
|
return blobs, err
|
|
}
|
|
|
|
opts.Progress(i, fmt.Sprintf("[v:%d] %s Large object (%d)", dash.Generation, dash.Name, len(body)))
|
|
err = large.Deconstruct(ctx, req.Key, opts.BlobStore, obj, req.Value)
|
|
if err != nil {
|
|
return blobs, err
|
|
}
|
|
|
|
// The smaller version (most of spec removed)
|
|
req.Value, err = json.Marshal(dash)
|
|
if err != nil {
|
|
return blobs, err
|
|
}
|
|
blobs.Count++
|
|
blobs.Size += int64(len(body))
|
|
}
|
|
|
|
opts.Progress(i, fmt.Sprintf("[v:%2d] %s (size:%d / %d|%d)", dash.Generation, dash.Name, len(req.Value), i, rows.count))
|
|
|
|
err = stream.Send(req)
|
|
if err != nil {
|
|
if errors.Is(err, io.EOF) {
|
|
opts.Progress(i, fmt.Sprintf("stream EOF/cancelled. index=%d", i))
|
|
err = nil
|
|
}
|
|
return blobs, err
|
|
}
|
|
}
|
|
|
|
if len(rows.rejected) > 0 {
|
|
for _, row := range rows.rejected {
|
|
id := row.Dash.Labels[utils.LabelKeyDeprecatedInternalID]
|
|
a.log.Warn("rejected dashboard",
|
|
"dashboard", row.Dash.Name,
|
|
"uid", row.Dash.UID,
|
|
"id", id,
|
|
)
|
|
opts.Progress(-2, fmt.Sprintf("rejected: id:%s, uid:%s", id, row.Dash.Name))
|
|
}
|
|
}
|
|
|
|
if rows.Error() != nil {
|
|
return blobs, rows.Error()
|
|
}
|
|
|
|
opts.Progress(-2, fmt.Sprintf("finished dashboards... (%d)", rows.count))
|
|
return blobs, err
|
|
}
|
|
|
|
func (a *dashboardSqlAccess) migrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
|
query := &DashboardQuery{
|
|
OrgID: orgId,
|
|
Limit: 100000000,
|
|
GetFolders: true,
|
|
Order: "ASC",
|
|
}
|
|
|
|
sql, err := a.sql(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
opts.Progress(-1, "migrating folders...")
|
|
rows, err := a.getRows(ctx, sql, query)
|
|
if rows != nil {
|
|
defer func() {
|
|
_ = rows.Close()
|
|
}()
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Now send each dashboard
|
|
for i := 1; rows.Next(); i++ {
|
|
dash := rows.row.Dash
|
|
dash.APIVersion = "folder.grafana.app/v1beta1"
|
|
dash.Kind = "Folder"
|
|
dash.SetNamespace(opts.Namespace)
|
|
dash.SetResourceVersion("") // it will be filled in by the backend
|
|
|
|
spec := map[string]any{
|
|
"title": dash.Spec.Object["title"],
|
|
}
|
|
description := dash.Spec.Object["description"]
|
|
if description != nil {
|
|
spec["description"] = description
|
|
}
|
|
dash.Spec.Object = spec
|
|
|
|
body, err := json.Marshal(dash)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
req := &resourcepb.BulkRequest{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: opts.Namespace,
|
|
Group: "folder.grafana.app",
|
|
Resource: "folders",
|
|
Name: rows.Name(),
|
|
},
|
|
Value: body,
|
|
Folder: rows.row.FolderUID,
|
|
Action: resourcepb.BulkRequest_ADDED,
|
|
}
|
|
if dash.Generation > 1 {
|
|
req.Action = resourcepb.BulkRequest_MODIFIED
|
|
} else if dash.Generation < 0 {
|
|
req.Action = resourcepb.BulkRequest_DELETED
|
|
}
|
|
|
|
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", dash.Generation, dash.Name, len(req.Value)))
|
|
|
|
err = stream.Send(req)
|
|
if err != nil {
|
|
if errors.Is(err, io.EOF) {
|
|
err = nil
|
|
}
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if rows.Error() != nil {
|
|
return nil, rows.Error()
|
|
}
|
|
|
|
opts.Progress(-2, fmt.Sprintf("finished folders... (%d)", rows.count))
|
|
return nil, err
|
|
}
|
|
|
|
func (a *dashboardSqlAccess) migratePanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
|
opts.Progress(-1, "migrating library panels...")
|
|
panels, err := a.GetLibraryPanels(ctx, LibraryPanelQuery{
|
|
OrgID: orgId,
|
|
Limit: 1000000,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for i, panel := range panels.Items {
|
|
meta, err := utils.MetaAccessor(&panel)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
body, err := json.Marshal(panel)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
req := &resourcepb.BulkRequest{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: opts.Namespace,
|
|
Group: dashboard.GROUP,
|
|
Resource: dashboard.LIBRARY_PANEL_RESOURCE,
|
|
Name: panel.Name,
|
|
},
|
|
Value: body,
|
|
Folder: meta.GetFolder(),
|
|
Action: resourcepb.BulkRequest_ADDED,
|
|
}
|
|
if panel.Generation > 1 {
|
|
req.Action = resourcepb.BulkRequest_MODIFIED
|
|
}
|
|
|
|
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", i, meta.GetName(), len(req.Value)))
|
|
|
|
err = stream.Send(req)
|
|
if err != nil {
|
|
if errors.Is(err, io.EOF) {
|
|
err = nil
|
|
}
|
|
return nil, err
|
|
}
|
|
}
|
|
opts.Progress(-2, fmt.Sprintf("finished panels... (%d)", len(panels.Items)))
|
|
return nil, nil
|
|
}
|