mirror of
https://github.com/grafana/grafana.git
synced 2025-07-31 08:32:37 +08:00
Remote provisioning: consolidate resource operations (#102972)
* Move to new repository * Rename it to dual writer * Rename the function * Rename the methods * Rename to exportResource * Clean up logic in migrate and add TODOs * Add TODOs * Use generic client for unprovisioned * ForEachResource * More consolidation * Refactor more around client * Consolidate constants * ForEachFolder * More use of constants * Add FIXME notes * Use more constant * Remove Dashboard * Pass tree to folder manager * Replicate tree * Reduce export complexity * More refactoring * Use the ForEach for loading users * Limit in-memory folders * Isolate the object * Improve the export function * Move resources to resources package * Move delete operation * Move more logic * More consolidation * More renaming * Fix more issues * Ensure path exists when created a resource * Simply append error * Fix receiver lint issue * Fix cyclomatic complexity * Fix linting * Remove folder path creation
This commit is contained in:
Roberto Jiménez Sánchez
committed by
GitHub
GitHub
parent
5668ab9676
commit
bb344fcd83
@ -2,18 +2,15 @@ package provisioning
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apiserver/pkg/registry/rest"
|
||||
|
||||
"github.com/grafana/grafana-app-sdk/logging"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
|
||||
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
@ -54,7 +51,6 @@ func (*filesConnector) NewConnectOptions() (runtime.Object, bool, string) {
|
||||
}
|
||||
|
||||
// TODO: document the synchronous write and delete on the API Spec
|
||||
// TODO: Move dual write logic to `resources` package and keep this connector simple
|
||||
func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime.Object, responder rest.Responder) (http.Handler, error) {
|
||||
logger := logging.FromContext(ctx).With("logger", "files-connector", "repository_name", name)
|
||||
ctx = logging.Context(ctx, logger)
|
||||
@ -64,11 +60,23 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime.
|
||||
return nil, err
|
||||
}
|
||||
|
||||
reader, ok := repo.(repository.Reader)
|
||||
readWriter, ok := repo.(repository.ReaderWriter)
|
||||
if !ok {
|
||||
return nil, apierrors.NewBadRequest("repository does not support read")
|
||||
return nil, apierrors.NewBadRequest("repository does not support read-writing")
|
||||
}
|
||||
|
||||
parser, err := s.parsers.GetParser(ctx, readWriter)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get parser: %w", err)
|
||||
}
|
||||
|
||||
folderClient, err := parser.Clients().Folder()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get folder client: %w", err)
|
||||
}
|
||||
folders := resources.NewFolderManager(readWriter, folderClient, resources.NewEmptyFolderTree())
|
||||
dualReadWriter := resources.NewDualReadWriter(readWriter, parser, folders)
|
||||
|
||||
return withTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
query := r.URL.Query()
|
||||
ref := query.Get("ref")
|
||||
@ -89,30 +97,11 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime.
|
||||
|
||||
isDir := safepath.IsDir(filePath)
|
||||
if r.Method == http.MethodGet && isDir {
|
||||
// TODO: Implement folder navigation
|
||||
if len(filePath) > 0 {
|
||||
responder.Error(apierrors.NewBadRequest("folder navigation not yet supported"))
|
||||
return
|
||||
}
|
||||
|
||||
// TODO: Add pagination
|
||||
rsp, err := reader.ReadTree(ctx, ref)
|
||||
files, err := s.listFolderFiles(ctx, filePath, ref, readWriter)
|
||||
if err != nil {
|
||||
responder.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
files := &provisioning.FileList{}
|
||||
for _, v := range rsp {
|
||||
if !v.Blob {
|
||||
continue // folder item
|
||||
}
|
||||
files.Items = append(files.Items, provisioning.FileItem{
|
||||
Path: v.Path,
|
||||
Size: v.Size,
|
||||
Hash: v.Hash,
|
||||
})
|
||||
}
|
||||
responder.Object(http.StatusOK, files)
|
||||
return
|
||||
}
|
||||
@ -132,19 +121,60 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime.
|
||||
code := http.StatusOK
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
code, obj, err = s.doRead(ctx, reader, filePath, ref)
|
||||
resource, err := dualReadWriter.Read(ctx, filePath, ref)
|
||||
if err != nil {
|
||||
responder.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
obj = resource.AsResourceWrapper()
|
||||
code = http.StatusOK
|
||||
if len(resource.Errors) > 0 {
|
||||
code = http.StatusNotAcceptable
|
||||
}
|
||||
case http.MethodPost:
|
||||
obj, err = s.doWrite(ctx, false, repo, filePath, ref, message, r)
|
||||
if isDir {
|
||||
obj, err = dualReadWriter.CreateFolder(ctx, filePath, ref, message)
|
||||
} else {
|
||||
data, err := readBody(r, filesMaxBodySize)
|
||||
if err != nil {
|
||||
responder.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
resource, err := dualReadWriter.CreateResource(ctx, filePath, ref, message, data)
|
||||
if err != nil {
|
||||
responder.Error(err)
|
||||
return
|
||||
}
|
||||
obj = resource.AsResourceWrapper()
|
||||
}
|
||||
case http.MethodPut:
|
||||
// TODO: document in API specification
|
||||
if isDir {
|
||||
err = apierrors.NewMethodNotSupported(provisioning.RepositoryResourceInfo.GroupResource(), r.Method)
|
||||
} else {
|
||||
obj, err = s.doWrite(ctx, true, repo, filePath, ref, message, r)
|
||||
data, err := readBody(r, filesMaxBodySize)
|
||||
if err != nil {
|
||||
responder.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
resource, err := dualReadWriter.UpdateResource(ctx, filePath, ref, message, data)
|
||||
if err != nil {
|
||||
responder.Error(err)
|
||||
return
|
||||
}
|
||||
obj = resource.AsResourceWrapper()
|
||||
}
|
||||
case http.MethodDelete:
|
||||
// TODO: limit file size
|
||||
obj, err = s.doDelete(ctx, repo, filePath, ref, message)
|
||||
resource, err := dualReadWriter.Delete(ctx, filePath, ref, message)
|
||||
if err != nil {
|
||||
responder.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
obj = resource.AsResourceWrapper()
|
||||
default:
|
||||
err = apierrors.NewMethodNotSupported(provisioning.RepositoryResourceInfo.GroupResource(), r.Method)
|
||||
}
|
||||
@ -165,218 +195,32 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime.
|
||||
}), 30*time.Second), nil
|
||||
}
|
||||
|
||||
func (s *filesConnector) doRead(ctx context.Context, repo repository.Reader, path string, ref string) (int, *provisioning.ResourceWrapper, error) {
|
||||
info, err := repo.Read(ctx, path, ref)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
// listFolderFiles returns a list of files in a folder
|
||||
func (s *filesConnector) listFolderFiles(ctx context.Context, filePath string, ref string, readWriter repository.ReaderWriter) (*provisioning.FileList, error) {
|
||||
// TODO: Implement folder navigation
|
||||
if len(filePath) > 0 {
|
||||
return nil, apierrors.NewBadRequest("folder navigation not yet supported")
|
||||
}
|
||||
|
||||
parser, err := s.parsers.GetParser(ctx, repo)
|
||||
// TODO: Add pagination
|
||||
rsp, err := readWriter.ReadTree(ctx, ref)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
parsed, err := parser.Parse(ctx, info, true)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
|
||||
// GVR will exist for anything we can actually save
|
||||
// TODO: Add known error in parser for unsupported resource
|
||||
if parsed.GVR == nil {
|
||||
if parsed.GVK != nil {
|
||||
//nolint:govet
|
||||
parsed.Errors = append(parsed.Errors, fmt.Errorf("unknown resource for Kind: %s", parsed.GVK.Kind))
|
||||
} else {
|
||||
parsed.Errors = append(parsed.Errors, fmt.Errorf("unknown resource"))
|
||||
files := &provisioning.FileList{}
|
||||
for _, v := range rsp {
|
||||
if !v.Blob {
|
||||
continue // folder item
|
||||
}
|
||||
files.Items = append(files.Items, provisioning.FileItem{
|
||||
Path: v.Path,
|
||||
Size: v.Size,
|
||||
Hash: v.Hash,
|
||||
})
|
||||
}
|
||||
|
||||
code := http.StatusOK
|
||||
if len(parsed.Errors) > 0 {
|
||||
code = http.StatusNotAcceptable
|
||||
}
|
||||
return code, parsed.AsResourceWrapper(), nil
|
||||
}
|
||||
|
||||
func (s *filesConnector) doWrite(ctx context.Context, update bool, repo repository.Repository, path string, ref string, message string, req *http.Request) (*provisioning.ResourceWrapper, error) {
|
||||
if err := repository.IsWriteAllowed(repo.Config(), ref); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
writer, ok := repo.(repository.ReaderWriter)
|
||||
if !ok {
|
||||
return nil, apierrors.NewBadRequest("repository does not support read-writing")
|
||||
}
|
||||
|
||||
parser, err := s.parsers.GetParser(ctx, writer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() { _ = req.Body.Close() }()
|
||||
if safepath.IsDir(path) {
|
||||
return s.doCreateFolder(ctx, writer, path, ref, message, parser)
|
||||
}
|
||||
|
||||
data, err := readBody(req, filesMaxBodySize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
info := &repository.FileInfo{
|
||||
Data: data,
|
||||
Path: path,
|
||||
Ref: ref,
|
||||
}
|
||||
|
||||
// TODO: improve parser to parse out of reader
|
||||
parsed, err := parser.Parse(ctx, info, true)
|
||||
if err != nil {
|
||||
if errors.Is(err, resources.ErrUnableToReadResourceBytes) {
|
||||
return nil, apierrors.NewBadRequest("unable to read the request as a resource")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// GVR will exist for anything we can actually save
|
||||
// TODO: Add known error in parser for unsupported resource
|
||||
if parsed.GVR == nil {
|
||||
return nil, apierrors.NewBadRequest("The payload does not map to a known resource")
|
||||
}
|
||||
|
||||
// Do not write if any errors exist
|
||||
if len(parsed.Errors) > 0 {
|
||||
return parsed.AsResourceWrapper(), err
|
||||
}
|
||||
|
||||
data, err = parsed.ToSaveBytes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if update {
|
||||
err = writer.Update(ctx, path, ref, data, message)
|
||||
} else {
|
||||
err = writer.Create(ctx, path, ref, data, message)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Directly update the grafana database
|
||||
// Behaves the same running sync after writing
|
||||
if ref == "" {
|
||||
if parsed.Existing == nil {
|
||||
parsed.Upsert, err = parsed.Client.Create(ctx, parsed.Obj, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
parsed.Errors = append(parsed.Errors, err)
|
||||
}
|
||||
} else {
|
||||
parsed.Upsert, err = parsed.Client.Update(ctx, parsed.Obj, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
parsed.Errors = append(parsed.Errors, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return parsed.AsResourceWrapper(), err
|
||||
}
|
||||
|
||||
func (s *filesConnector) doCreateFolder(ctx context.Context, repo repository.Writer, path string, ref string, message string, parser *resources.Parser) (*provisioning.ResourceWrapper, error) {
|
||||
client, err := parser.Clients().Folder()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
manager := resources.NewFolderManager(repo, client)
|
||||
|
||||
// Now actually create the folder
|
||||
if err := repo.Create(ctx, path, ref, nil, message); err != nil {
|
||||
return nil, fmt.Errorf("failed to create folder: %w", err)
|
||||
}
|
||||
|
||||
cfg := repo.Config()
|
||||
wrap := &provisioning.ResourceWrapper{
|
||||
Path: path,
|
||||
Ref: ref,
|
||||
Repository: provisioning.ResourceRepositoryInfo{
|
||||
Type: cfg.Spec.Type,
|
||||
Namespace: cfg.Namespace,
|
||||
Name: cfg.Name,
|
||||
Title: cfg.Spec.Title,
|
||||
},
|
||||
Resource: provisioning.ResourceObjects{
|
||||
Action: provisioning.ResourceActionCreate,
|
||||
},
|
||||
}
|
||||
|
||||
if ref == "" {
|
||||
folderName, err := manager.EnsureFolderPathExist(ctx, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
current, err := manager.GetFolder(ctx, folderName)
|
||||
if err != nil && !apierrors.IsNotFound(err) {
|
||||
return nil, err // unable to check if the folder exists
|
||||
}
|
||||
wrap.Resource.Upsert = v0alpha1.Unstructured{
|
||||
Object: current.Object,
|
||||
}
|
||||
}
|
||||
|
||||
return wrap, nil
|
||||
}
|
||||
|
||||
// Deletes a file from the repository and the Grafana database.
|
||||
// If the path is a folder, it will return an error.
|
||||
// If the file is not parsable, it will return an error.
|
||||
func (s *filesConnector) doDelete(ctx context.Context, repo repository.Repository, path string, ref string, message string) (*provisioning.ResourceWrapper, error) {
|
||||
if err := repository.IsWriteAllowed(repo.Config(), ref); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Read the existing value
|
||||
access, ok := repo.(repository.ReaderWriter)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("repository is not read+writeable")
|
||||
}
|
||||
|
||||
file, err := access.Read(ctx, path, ref)
|
||||
if err != nil {
|
||||
return nil, err // unable to read value
|
||||
}
|
||||
|
||||
parser, err := s.parsers.GetParser(ctx, access)
|
||||
if err != nil {
|
||||
return nil, err // unable to read value
|
||||
}
|
||||
|
||||
// TODO: document in API specification
|
||||
// We can only delete parsable things
|
||||
parsed, err := parser.Parse(ctx, file, false)
|
||||
if err != nil {
|
||||
return nil, err // unable to read value
|
||||
}
|
||||
|
||||
parsed.Action = provisioning.ResourceActionDelete
|
||||
wrap := parsed.AsResourceWrapper()
|
||||
|
||||
// Now delete the file
|
||||
err = access.Delete(ctx, path, ref, message)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Delete the file in the grafana database
|
||||
if ref == "" {
|
||||
err = parsed.Client.Delete(ctx, parsed.Obj.GetName(), metav1.DeleteOptions{})
|
||||
if apierrors.IsNotFound(err) {
|
||||
err = nil // ignorable
|
||||
}
|
||||
}
|
||||
|
||||
return wrap, err
|
||||
return files, nil
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
Reference in New Issue
Block a user