Add push to remote mirror repository (#15157)

* Added push mirror model.

* Integrated push mirror into queue.

* Moved methods into own file.

* Added basic implementation.

* Mirror wiki too.

* Removed duplicated method.

* Get url for different remotes.

* Added migration.

* Unified remote url access.

* Add/Remove push mirror remotes.

* Prevent hangs with missing credentials.

* Moved code between files.

* Changed sanitizer interface.

* Added push mirror backend methods.

* Only update the mirror remote.

* Limit refs on push.

* Added UI part.

* Added missing table.

* Delete mirror if repository gets removed.

* Changed signature. Handle object errors.

* Added upload method.

* Added "upload" unit tests.

* Added transfer adapter unit tests.

* Send correct headers.

* Added pushing of LFS objects.

* Added more logging.

* Simpler body handling.

* Process files in batches to reduce HTTP calls.

* Added created timestamp.

* Fixed invalid column name.

* Changed name to prevent xorm auto setting.

* Remove table header im empty.

* Strip exit code from error message.

* Added docs page about mirroring.

* Fixed date.

* Fixed merge errors.

* Moved test to integrations.

* Added push mirror test.

* Added test.

models/migrations/migrations.go

@@ -315,6 +315,8 @@ var migrations = []Migration{
 	NewMigration("Always save primary email on email address table", addPrimaryEmail2EmailAddress),
 	// v182 -> v183
 	NewMigration("Add issue resource index table", addIssueResourceIndexTable),
+	// v183 -> v184
+	NewMigration("Create PushMirror table", createPushMirrorTable),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v180.go

@@ -64,7 +64,7 @@ func removeCredentials(payload string) (string, error) {
 
 	opts.AuthPassword = ""
 	opts.AuthToken = ""
-	opts.CloneAddr = util.SanitizeURLCredentials(opts.CloneAddr, true)
+	opts.CloneAddr = util.NewStringURLSanitizer(opts.CloneAddr, true).Replace(opts.CloneAddr)
 
 	confBytes, err := json.Marshal(opts)
 	if err != nil {

models/migrations/v183.go

@@ -0,0 +1,39 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+	"time"
+
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+func createPushMirrorTable(x *xorm.Engine) error {
+	type PushMirror struct {
+		ID         int64 `xorm:"pk autoincr"`
+		RepoID     int64 `xorm:"INDEX"`
+		RemoteName string
+
+		Interval       time.Duration
+		CreatedUnix    timeutil.TimeStamp `xorm:"created"`
+		LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
+		LastError      string             `xorm:"text"`
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	if err := sess.Sync2(new(PushMirror)); err != nil {
+		return fmt.Errorf("Sync2: %v", err)
+	}
+
+	return sess.Commit()
+}