package main import ( "bufio" "context" "fmt" "log" "os" "os/exec" "path/filepath" "strings" "dagger.io/dagger" "github.com/grafana/grafana/pkg/build/daggerbuild/backend" "github.com/grafana/grafana/pkg/build/daggerbuild/containers" "github.com/grafana/grafana/pkg/build/daggerbuild/pipelines" ) const ( proName = "enterprise2" // 1: The version (with a v prefix) // 2: The "edition". Options: 'oss', 'pro', 'enterprise'. // 3: The full name. 'grafana', 'grafana-enterprise', 'grafana-pro // 4: The 'ersion', or 'version' without the 'v'. // 5: The OS: 'windows', 'linux', 'darwin' // 6: The architecture: 'amd64', 'armv6', 'armv7', 'arm64'. // 7: -musl, sometimes. // 8: '.sha256', sometimes. tarGzFormat = "artifacts/downloads%[9]s/%[1]s/%[2]s/release/%[3]s-%[4]s.%[5]s-%[6]s%[7]s.tar.gz%[8]s" debFormat = "artifacts/downloads%[9]s/%[1]s/%[2]s/release/%[3]s_%[4]s_%[6]s.deb%[8]s" rpmFormat = "artifacts/downloads%[9]s/%[1]s/%[2]s/release/%[3]s-%[4]s-1.%[6]s.rpm%[8]s" exeFormat = "artifacts/downloads%[9]s/%[1]s/%[2]s/release/%[3]s_%[4]s_%[6]s.exe%[8]s" msiFormat = "artifacts/downloads%[9]s/%[1]s/%[2]s/release/%[3]s_%[4]s_%[6]s.msi%[8]s" tarGzMainFormat = "%[2]s/main/%[3]s-%[4]s.%[5]s-%[6]s%[7]s.tar.gz%[8]s" debMainFormat = "%[2]s/main/%[3]s_%[4]s_%[6]s.deb%[8]s" // 1: ersion // 2. name (grafana-oss | grafana-enterprise) // 3: '-ubuntu', if set // 4: arch // 5: '.sha256', if set dockerFormat = "artifacts/docker/%[1]s/%[2]s-%[1]s%[3]s-%[4]s.img%[5]s" // 1: ersion // 2. name (grafana-oss | grafana-enterprise) cdnFormat = "artifacts/static-assets/%[2]s/%[1]s/public" cdnMainFormat = "grafana/%s/public" // 1: ersion storybookFormat = "artifacts/storybook/v%[1]s" // 1: version // 2: package name (@grafana-ui-10.0.0.tgz) npmFormat = "artifacts/npm/v%[1]s/npm-artifacts" sha256Ext = ".sha256" grafana = "grafana" ) // One artifact and be copied to multiple different locations (like armv7 tar.gz packages should be copied to tar.gz and -musl.tar.gz) type HandlerFunc func(name string) []string var Handlers = map[string]HandlerFunc{ ".tar.gz": TarGZHandler, ".deb": DebHandler, ".rpm": RPMHandler, ".docker.tar.gz": DockerHandler, ".exe": EXEHandler, ".msi": MSIHandler, ".zip": ZipHandler, } func IsMain() bool { return os.Getenv("IS_MAIN") != "" } func NPMHandler(name string) []string { var ( version = strings.TrimPrefix(os.Getenv("DRONE_TAG"), "v") ) return []string{fmt.Sprintf(npmFormat, version)} } func ZipHandler(name string) []string { files := EXEHandler(strings.ReplaceAll(name, "zip", "exe")) for i, v := range files { files[i] = strings.ReplaceAll(v, "exe", "zip") } return files } func MSIHandler(name string) []string { files := EXEHandler(strings.ReplaceAll(name, "msi", "exe")) for i, v := range files { files[i] = strings.ReplaceAll(v, "exe", "msi") } return files } func RPMHandler(name string) []string { ext := filepath.Ext(name) // If we're copying a sha256 file and not a tar.gz then we want to add .sha256 to the template // or just give it emptystring if it's not the sha256 file sha256 := "" if ext == sha256Ext { sha256 = sha256Ext } n := filepath.Base(name) // Surprisingly still works even with 'gs://' urls opts := pipelines.TarOptsFromFileName(strings.ReplaceAll(strings.ReplaceAll(n, sha256Ext, ""), "rpm", "tar.gz")) // In grafana-build we just use "" to refer to "oss" edition := "oss" fullName := grafana if opts.Edition != "" { edition = opts.Edition fullName += "-" + opts.Edition } goos, arch := backend.OSAndArch(opts.Distro) arm := backend.ArchVersion(opts.Distro) if arch == "arm" { if arm == "7" { arch = "armhfp" } } if arch == "arm64" { arch = "aarch64" } if arch == "amd64" { arch = "x86_64" } enterprise2 := "" version := opts.Version ersion := strings.Replace(strings.TrimPrefix(version, "v"), "-", "~", 1) if edition == "pro" { // "pro" in this case is called "enterprise2" fullName = "grafana-enterprise2" edition = proName // and is in the 'downloads-enterprise2' folder instead of 'downloads' enterprise2 = "-enterprise2" // and has an period separator {version}.{arch} instead of {version}_{arch} } dst := fmt.Sprintf(rpmFormat, version, edition, fullName, ersion, goos, arch, edition, sha256, enterprise2) return []string{ dst, } } func EXEHandler(name string) []string { packages := DebHandler(strings.ReplaceAll(name, "exe", "deb")) for i, v := range packages { v = strings.ReplaceAll(v, "deb", "exe") v = strings.ReplaceAll(v, "amd64", "windows-amd64") v = strings.ReplaceAll(v, "_", "-") v = strings.ReplaceAll(v, "~", "-") v = strings.ReplaceAll(v, "-windows", ".windows") packages[i] = v } return packages } func DebHandler(name string) []string { ext := filepath.Ext(name) format := debFormat if IsMain() { format = debMainFormat } // If we're copying a sha256 file and not a tar.gz then we want to add .sha256 to the template // or just give it emptystring if it's not the sha256 file sha256 := "" if ext == sha256Ext { sha256 = sha256Ext } n := filepath.Base(name) // Surprisingly still works even with 'gs://' urls opts := pipelines.TarOptsFromFileName(strings.ReplaceAll(strings.ReplaceAll(n, sha256Ext, ""), "deb", "tar.gz")) // In grafana-build we just use "" to refer to "oss" edition := "oss" fullName := grafana version := opts.Version ersion := strings.TrimPrefix(version, "v") ersion = strings.Replace(ersion, "-", "~", 1) enterprise2 := "" if opts.Edition != "" { edition = opts.Edition fullName += "-" + opts.Edition if edition == "pro" { // "pro" in this case is called "enterprise2" fullName = "grafana-enterprise2" edition = proName // and is in the 'downloads-enterprise2' folder instead of 'downloads' enterprise2 = "-enterprise2" } if edition == "pro-rpi" { // "pro" in this case is called "enterprise2" fullName = "grafana-enterprise2-rpi" edition = proName // and is in the 'downloads-enterprise2' folder instead of 'downloads' enterprise2 = "-enterprise2" } if edition == "rpi" { edition = "oss" } if edition == "enterprise-rpi" { edition = "enterprise" } } names := []string{fullName} goos, arch := backend.OSAndArch(opts.Distro) if arch == "arm" { arch = "armhf" // If we're building for arm then we also copy the same thing, but with the name '-rpi'. for osme reason? names = []string{fullName} } dst := []string{} for _, n := range names { dst = append(dst, fmt.Sprintf(format, opts.Version, edition, n, ersion, goos, arch, edition, sha256, enterprise2)) } return dst } func TarGZHandler(name string) []string { ext := filepath.Ext(name) // If we're copying a sha256 file and not a tar.gz then we want to add .sha256 to the template // or just give it emptystring if it's not the sha256 file sha256 := "" if ext == sha256Ext { sha256 = sha256Ext } n := filepath.Base(name) // Surprisingly still works even with 'gs://' urls opts := pipelines.TarOptsFromFileName(strings.ReplaceAll(n, sha256Ext, "")) // In grafana-build we just use "" to refer to "oss" edition := "oss" fullName := grafana version := opts.Version ersion := strings.TrimPrefix(version, "v") enterprise2 := "" if opts.Edition != "" { edition = opts.Edition fullName += "-" + opts.Edition if edition == "pro" { enterprise2 = "-enterprise2" fullName = "grafana-enterprise2" edition = proName } } libc := []string{""} goos, arch := backend.OSAndArch(opts.Distro) if arch == "arm64" || arch == "arm" || arch == "amd64" && goos == "linux" { libc = []string{"", "-musl"} } arm := backend.ArchVersion(opts.Distro) if arch == "arm" { arch += "v" + arm // I guess we don't create an arm-6-musl? if arm == "6" { libc = []string{""} } } format := tarGzFormat if IsMain() { format = tarGzMainFormat } dst := []string{} for _, m := range libc { dst = append(dst, fmt.Sprintf(format, opts.Version, edition, fullName, ersion, goos, arch, m, sha256, enterprise2)) } return dst } func DockerHandler(name string) []string { ext := filepath.Ext(name) // If we're copying a sha256 file and not a tar.gz then we want to add .sha256 to the template // or just give it emptystring if it's not the sha256 file sha256 := "" if ext == sha256Ext { sha256 = sha256Ext } n := filepath.Base(name) // Surprisingly still works even with 'gs://' urls // try to get .ubuntu.docker.tar.gz.sha256 / .ubuntu.docker.tar.gz / docker.tar.gz to all just end in 'tar.gz' normalized := strings.ReplaceAll(n, sha256Ext, "") normalized = strings.ReplaceAll(normalized, ".ubuntu", "") normalized = strings.ReplaceAll(normalized, ".docker", "") opts := pipelines.TarOptsFromFileName(normalized) // In grafana-build we just use "" to refer to "oss" edition := "oss" fullName := grafana if opts.Edition != "" { edition = opts.Edition if edition == "pro" { edition = proName } } fullName += "-" + edition ubuntu := "" if strings.Contains(name, "ubuntu") { ubuntu = "-ubuntu" } _, arch := backend.OSAndArch(opts.Distro) if arch == "arm" { arch += "v" + backend.ArchVersion(opts.Distro) } return []string{ fmt.Sprintf(dockerFormat, strings.TrimPrefix(opts.Version, "v"), fullName, ubuntu, arch, sha256), } } func CDNHandler(name string) []string { if IsMain() { // This folder is is always ${dist}/${version}/${name}/${public} dist, err := filepath.Rel(".", filepath.Join(name, "../../../")) if err != nil { panic(err) } path, err := filepath.Rel(dist, name) if err != nil { panic(err) } s := strings.Split(path, string(os.PathSeparator)) return []string{fmt.Sprintf(cdnMainFormat, s[0])} } version := strings.TrimPrefix(os.Getenv("DRONE_TAG"), "v") return []string{fmt.Sprintf(cdnFormat, version, grafana)} } func StorybookHandler(name string) []string { version := strings.TrimPrefix(os.Getenv("DRONE_TAG"), "v") return []string{fmt.Sprintf(storybookFormat, version)} } // A hopefully temporary script that prints the gsutil commands that will move these artifacts into the location where they were expected previously. // Just pipe this into bash or exec or whatever to do the actual copying. // Run without redirecting stdout to verify the operations. func main() { prefix := os.Args[1] ctx := context.Background() client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stderr)) if err != nil { panic(err) } var ( scanner = bufio.NewScanner(os.Stdin) authenticator = containers.GCSAuth(client, &containers.GCPOpts{ ServiceAccountKeyBase64: os.Getenv("GCP_KEY_BASE64"), }) container = client.Container().From("google/cloud-sdk:alpine") ) // if c, err := authenticator.Authenticate(client, container); err == nil { container = c } else { panic(err) } for scanner.Scan() { var ( name = scanner.Text() ) handler, ext := getHandler(name, Handlers) destinations := handler(name) if ext == "" { for _, v := range destinations { dir := filepath.Join(prefix, filepath.Dir(v)) v := filepath.Join(prefix, v) log.Println("Creating dir", dir) if err := os.MkdirAll(dir, 0700); err != nil { panic(err) } log.Println("Copying", name, "to", v) //nolint:gosec cmd := exec.Command("cp", "-r", strings.TrimPrefix(name, "file://"), v) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { panic(err) } } continue } log.Println("File:", name, "to be copied as", destinations) for _, v := range destinations { dir := filepath.Join(prefix, filepath.Dir(v)) v := filepath.Join(prefix, v) log.Println("Creating directory", dir) if err := os.MkdirAll(dir, 0700); err != nil { panic(err) } log.Println("Copying", name, "to", dir, "as", v) //nolint:gosec cmd := exec.Command("cp", strings.TrimPrefix(name, "file://"), v) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { panic(err) } } } log.Println("Copying", prefix, "to gcs") dst := os.Getenv("DESTINATION") container = container.WithMountedDirectory("dist", client.Host().Directory(prefix)). WithExec([]string{"gcloud", "storage", "cp", "-r", "/dist/*", dst}) stdout, err := container.Stdout(ctx) if err != nil { panic(err) } stderr, err := container.Stdout(ctx) if err != nil { panic(err) } fmt.Fprint(os.Stdout, stdout) fmt.Fprint(os.Stderr, stderr) } func getHandler(name string, handlers map[string]HandlerFunc) (HandlerFunc, string) { ext := filepath.Ext(name) // sha256 extensions should be handled the same way what precedes the extension if ext == sha256Ext { ext = filepath.Ext(strings.ReplaceAll(name, sha256Ext, "")) } // tar.gz extensions can also have docker.tar.gz so we need to make sure we don't skip that if ext == ".gz" { ext = ".tar.gz" if filepath.Ext(strings.ReplaceAll(name, ".tar.gz", "")) == ".docker" || filepath.Ext(strings.ReplaceAll(name, ".tar.gz.sha256", "")) == ".docker" { ext = ".docker.tar.gz" } } handler := handlers[ext] // If there is no extension, then we are either dealing with public assets // or the storybook, which both require some extra handling: if ext != "" { return handler, ext } if filepath.Base(name) == "public" { return CDNHandler, "" } if filepath.Base(name) == "storybook" { return StorybookHandler, "" } if filepath.Base(name) == "npm-packages" { return NPMHandler, "" } panic("no handler found") }