feat: add support for robots.txt disabling search indexing (#2929)

* feat: add support for robots.txt

Can toggle disabling search engine indexing. Closes #2684

* fix: unexport ts const

controllers/robots.go

@@ -0,0 +1,28 @@
+package controllers
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/owncast/owncast/core/data"
+)
+
+// GetRobotsDotTxt returns the contents of our robots.txt.
+func GetRobotsDotTxt(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain")
+	contents := []string{
+		"User-agent: *",
+		"Disallow: /admin",
+		"Disallow: /api",
+	}
+
+	if data.GetDisableSearchIndexing() {
+		contents = append(contents, "Disallow: /")
+	}
+
+	txt := []byte(strings.Join(contents, "\n"))
+
+	if _, err := w.Write(txt); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+}