proc/native,Makefile: allow compiling on macOS without native backend

On macOS 10.14 Apple changed the command line tools so that system
headers now need to be manually installed.

Instead of adding one extra install step to the install procedure add a
build tag to allow compilation of delve without the native backend on
macOS. By default (i.e. when using `go get`) this is how delve will be
compiled on macOS, the make script is changed to enable compiling the
native backend if the required dependencies have been installed.

Insure that both configuration still build correctly on Travis CI and
change the documentation to describe how to compile the native backend
and that it isn't normally needed.

Fixes #1359

Documentation/installation/osx/install.md

@@ -1,9 +1,5 @@
 # Installation on OSX
 
-Please use the following steps to build and install Delve on OSX.
-
-## Manual install
-
 Ensure you have a proper compilation toolchain.
 
 This should be as simple as:
@@ -16,15 +12,15 @@ Now you can install delve using `go get`:
 $ go get -u github.com/derekparker/delve/cmd/dlv
 ```
 
-With this method you will not be able to use delve's native backend.
+With this method you will not be able to use delve's native backend, *but you don't need it anyway*: the native backend on macOS [has known problems](https://github.com/derekparker/delve/issues/1112) on recent issues of the OS and is not currently maintained.
 
-Alternatively, you can clone the repo into `$GOPATH/src/github.com/derekparker/delve` and run:
+## Compiling the native backend
 
-```
-$ make install
-```
-from that directory.
+Only do this if you have a valid reason to use the native backend.
+
+1. Run `xcode-select --install`
+2. On macOS 10.14 manually install the legacy include headers by running `/Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg`
+3. Clone the repo into `$GOPATH/src/github.com/derekparker/delve`
+4. Run `make install` in that directory
 
 The makefile will take care of creating and installing a self-signed certificate automatically.
-
-Note: If you are using Go 1.5 you must set `GO15VENDOREXPERIMENT=1` before continuing. The `GO15VENDOREXPERIMENT` env var simply opts into the [Go 1.5 Vendor Experiment](https://docs.google.com/document/d/1Bz5-UB7g2uPBdOx-rw5t9MxJwkfpx90cqG9AFL0JAYo/).

pkg/proc/native/exc_user_darwin.c

@@ -1,3 +1,5 @@
+//+build darwin,macnative
+
 /*
  * IDENTIFICATION:
  * stub generated Sun Feb 22 20:54:31 2015

pkg/proc/native/exec_darwin.c

@@ -1,3 +1,5 @@
+//+build darwin,macnative
+
 #include "exec_darwin.h"
 #include "stdio.h"
 

pkg/proc/native/exec_darwin.h

@@ -1,3 +1,5 @@
+//+build darwin,macnative
+
 #include "proc_darwin.h"
 
 #include <unistd.h>

pkg/proc/native/mach_exc_user_darwin.c

@@ -1,3 +1,5 @@
+//+build darwin,macnative
+
 /*
  * IDENTIFICATION:
  * stub generated Sat Feb 21 18:10:52 2015

pkg/proc/native/nonative_darwin.go

@@ -0,0 +1,124 @@
+//+build darwin,!macnative
+
+package native
+
+import (
+	"errors"
+	"sync"
+
+	"github.com/derekparker/delve/pkg/proc"
+)
+
+var ErrNativeBackendDisabled = errors.New("native backend disabled during compilation")
+
+// Launch returns ErrNativeBackendDisabled.
+func Launch(cmd []string, wd string, foreground bool) (*Process, error) {
+	return nil, ErrNativeBackendDisabled
+}
+
+// Attach returns ErrNativeBackendDisabled.
+func Attach(pid int) (*Process, error) {
+	return nil, ErrNativeBackendDisabled
+}
+
+// WaitStatus is a synonym for the platform-specific WaitStatus
+type WaitStatus struct{}
+
+// OSSpecificDetails holds information specific to the OSX/Darwin
+// operating system / kernel.
+type OSSpecificDetails struct{}
+
+// OSProcessDetails holds Darwin specific information.
+type OSProcessDetails struct{}
+
+func findExecutable(path string, pid int) string {
+	panic(ErrNativeBackendDisabled)
+}
+
+func killProcess(pid int) error {
+	panic(ErrNativeBackendDisabled)
+}
+
+func registers(thread *Thread, floatingPoint bool) (proc.Registers, error) {
+	panic(ErrNativeBackendDisabled)
+}
+
+func (dbp *Process) loadProcessInformation(wg *sync.WaitGroup) {
+	panic(ErrNativeBackendDisabled)
+}
+
+func (dbp *Process) requestManualStop() (err error) {
+	panic(ErrNativeBackendDisabled)
+}
+
+func (dbp *Process) resume() error {
+	panic(ErrNativeBackendDisabled)
+}
+
+func (dbp *Process) trapWait(pid int) (*Thread, error) {
+	panic(ErrNativeBackendDisabled)
+}
+
+func (dbp *Process) stop(trapthread *Thread) (err error) {
+	panic(ErrNativeBackendDisabled)
+}
+
+func (dbp *Process) updateThreadList() error {
+	panic(ErrNativeBackendDisabled)
+}
+
+func (dbp *Process) kill() (err error) {
+	panic(ErrNativeBackendDisabled)
+}
+
+func (dbp *Process) detach(kill bool) error {
+	panic(ErrNativeBackendDisabled)
+}
+
+// Blocked returns true if the thread is blocked
+func (t *Thread) Blocked() bool {
+	panic(ErrNativeBackendDisabled)
+}
+
+// SetPC sets the value of the PC register.
+func (thread *Thread) SetPC(pc uint64) error {
+	panic(ErrNativeBackendDisabled)
+}
+
+// SetSP sets the value of the SP register.
+func (thread *Thread) SetSP(sp uint64) error {
+	panic(ErrNativeBackendDisabled)
+}
+
+// SetDX sets the value of the DX register.
+func (thread *Thread) SetDX(dx uint64) error {
+	panic(ErrNativeBackendDisabled)
+}
+
+// ReadMemory reads len(buf) bytes at addr into buf.
+func (t *Thread) ReadMemory(buf []byte, addr uintptr) (int, error) {
+	panic(ErrNativeBackendDisabled)
+}
+
+// WriteMemory writes the contents of data at addr.
+func (t *Thread) WriteMemory(addr uintptr, data []byte) (int, error) {
+	panic(ErrNativeBackendDisabled)
+}
+
+func (t *Thread) resume() error {
+	panic(ErrNativeBackendDisabled)
+}
+
+func (t *Thread) singleStep() error {
+	panic(ErrNativeBackendDisabled)
+}
+
+func (t *Thread) restoreRegisters(sr proc.Registers) error {
+	panic(ErrNativeBackendDisabled)
+}
+
+// Stopped returns whether the thread is stopped at
+// the operating system level.
+func (t *Thread) Stopped() bool {
+	panic(ErrNativeBackendDisabled)
+}

pkg/proc/native/proc_darwin.c

@@ -1,3 +1,5 @@
+//+build darwin,macnative
+
 #include "proc_darwin.h"
 
 static const unsigned char info_plist[]

pkg/proc/native/proc_darwin.go

@@ -1,3 +1,5 @@
+//+build darwin,macnative
+
 package native
 
 // #include "proc_darwin.h"

pkg/proc/native/proc_darwin.h

@@ -1,3 +1,5 @@
+//+build darwin,macnative
+
 #include <sys/types.h>
 #include <libproc.h>
 #include <mach/mach.h>

pkg/proc/native/ptrace_darwin.go

@@ -1,3 +1,5 @@
+//+build darwin,macnative
+
 package native
 
 import sys "golang.org/x/sys/unix"

pkg/proc/native/registers_darwin_amd64.go

@@ -1,3 +1,5 @@
+//+build darwin,macnative
+
 package native
 
 // #include "threads_darwin.h"

pkg/proc/native/threads_darwin.c

@@ -1,3 +1,5 @@
+//+build darwin,macnative
+
 #include "threads_darwin.h"
 
 int

pkg/proc/native/threads_darwin.go

@@ -1,3 +1,5 @@
+//+build darwin,macnative
+
 package native
 
 // #include "threads_darwin.h"

pkg/proc/native/threads_darwin.h

@@ -1,3 +1,5 @@
+//+build darwin,macnative
+
 #include <stdlib.h>
 #include <sys/types.h>
 #include <mach/mach.h>

pkg/proc/proc_test.go

@@ -38,12 +38,7 @@ func init() {
 func TestMain(m *testing.M) {
 	flag.StringVar(&testBackend, "backend", "", "selects backend")
 	flag.Parse()
-	if testBackend == "" {
-		testBackend = os.Getenv("PROCTEST")
-		if testBackend == "" {
-			testBackend = "native"
-		}
-	}
+	protest.DefaultTestBackend(&testBackend)
 	os.Exit(protest.RunTestsWithFixtures(m))
 }
 
@@ -2036,14 +2031,23 @@ func TestIssue509(t *testing.T) {
 	cmd.Dir = nomaindir
 	assertNoError(cmd.Run(), t, "go build")
 	exepath := filepath.Join(nomaindir, "debug")
-	_, err := native.Launch([]string{exepath}, ".", false)
+	defer os.Remove(exepath)
+	var err error
+
+	switch testBackend {
+	case "native":
+		_, err = native.Launch([]string{exepath}, ".", false)
+	case "lldb":
+		_, err = gdbserial.LLDBLaunch([]string{exepath}, ".", false)
+	default:
+		t.Skip("test not valid for this backend")
+	}
 	if err == nil {
 		t.Fatalf("expected error but none was generated")
 	}
 	if err != proc.ErrNotExecutable {
 		t.Fatalf("expected error \"%v\" got \"%v\"", proc.ErrNotExecutable, err)
 	}
-	os.Remove(exepath)
 }
 
 func TestUnsupportedArch(t *testing.T) {
@@ -2070,7 +2074,17 @@ func TestUnsupportedArch(t *testing.T) {
 	}
 	defer os.Remove(outfile)
 
-	p, err := native.Launch([]string{outfile}, ".", false)
+	var p proc.Process
+
+	switch testBackend {
+	case "native":
+		p, err = native.Launch([]string{outfile}, ".", false)
+	case "lldb":
+		p, err = gdbserial.LLDBLaunch([]string{outfile}, ".", false)
+	default:
+		t.Skip("test not valid for this backend")
+	}
+
 	switch err {
 	case proc.ErrUnsupportedLinuxArch, proc.ErrUnsupportedWindowsArch, proc.ErrUnsupportedDarwinArch:
 		// all good

pkg/proc/test/support.go

@@ -266,3 +266,20 @@ func MustSupportFunctionCalls(t *testing.T, testBackend string) {
 		t.Skip("this backend does not support function calls")
 	}
 }
+
+// DefaultTestBackend changes the value of testBackend to be the default
+// test backend for the OS, if testBackend isn't already set.
+func DefaultTestBackend(testBackend *string) {
+	if *testBackend != "" {
+		return
+	}
+	*testBackend = os.Getenv("PROCTEST")
+	if *testBackend != "" {
+		return
+	}
+	if runtime.GOOS == "darwin" {
+		*testBackend = "lldb"
+	} else {
+		*testBackend = "native"
+	}
+}

pkg/terminal/command_test.go

@@ -29,12 +29,7 @@ var testBackend string
 func TestMain(m *testing.M) {
 	flag.StringVar(&testBackend, "backend", "", "selects backend")
 	flag.Parse()
-	if testBackend == "" {
-		testBackend = os.Getenv("PROCTEST")
-		if testBackend == "" {
-			testBackend = "native"
-		}
-	}
+	test.DefaultTestBackend(&testBackend)
 	os.Exit(test.RunTestsWithFixtures(m))
 }
 

scripts/make.go

@@ -34,9 +34,9 @@ func NewMakeCommands() *cobra.Command {
 		Use:   "build",
 		Short: "Build delve",
 		Run: func(cmd *cobra.Command, args []string) {
-			checkCertCmd(nil, nil)
-			execute("go", "build", buildFlags(), DelveMainPackagePath)
-			if runtime.GOOS == "darwin" && os.Getenv("CERT") != "" {
+			tagFlag := prepareMacnative()
+			execute("go", "build", tagFlag, buildFlags(), DelveMainPackagePath)
+			if runtime.GOOS == "darwin" && os.Getenv("CERT") != "" && canMacnative() {
 				codesign("./dlv")
 			}
 		},
@@ -46,9 +46,9 @@ func NewMakeCommands() *cobra.Command {
 		Use:   "install",
 		Short: "Installs delve",
 		Run: func(cmd *cobra.Command, args []string) {
-			checkCertCmd(nil, nil)
-			execute("go", "install", buildFlags(), DelveMainPackagePath)
-			if runtime.GOOS == "darwin" {
+			tagFlag := prepareMacnative()
+			execute("go", "install", tagFlag, buildFlags(), DelveMainPackagePath)
+			if runtime.GOOS == "darwin" && os.Getenv("CERT") != "" && canMacnative() {
 				codesign(installedExecutablePath())
 			}
 		},
@@ -183,6 +183,33 @@ func installedExecutablePath() string {
 	return filepath.Join(strings.TrimSpace(gopath[0]), "bin", "dlv")
 }
 
+// canMacnative returns true if we can build the native backend for macOS,
+// i.e. cgo enabled and the legacy SDK headers:
+// https://forums.developer.apple.com/thread/104296
+func canMacnative() bool {
+	if runtime.GOOS != "darwin" {
+		return false
+	}
+	if strings.TrimSpace(getoutput("go", "env", "CGO_ENABLED")) != "1" {
+		return false
+	}
+	_, err := os.Stat("/usr/include/sys/types.h")
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+// prepareMacnative checks if we can build the native backend for macOS and
+// if we can checks the certificate and then returns the -tags flag.
+func prepareMacnative() string {
+	if !canMacnative() {
+		return ""
+	}
+	checkCertCmd(nil, nil)
+	return "-tags=macnative"
+}
+
 func buildFlags() []string {
 	buildSHA, err := exec.Command("git", "rev-parse", "HEAD").CombinedOutput()
 	if err != nil {
@@ -214,6 +241,13 @@ func testCmd(cmd *cobra.Command, args []string) {
 	checkCertCmd(nil, nil)
 
 	if os.Getenv("TRAVIS") == "true" && runtime.GOOS == "darwin" {
+		fmt.Println("Building with native backend")
+		execute("go", "build", "-tags=macnative", buildFlags(), DelveMainPackagePath)
+
+		fmt.Println("\nBuilding without native backend")
+		execute("go", "build", buildFlags(), DelveMainPackagePath)
+
+		fmt.Println("\nTesting")
 		os.Setenv("PROCTEST", "lldb")
 		executeq("sudo", "-E", "go", "test", testFlags(), allPackages())
 		return

service/test/integration2_test.go

@@ -32,12 +32,7 @@ func TestMain(m *testing.M) {
 	var logOutput string
 	flag.StringVar(&logOutput, "log-output", "", "configures log output")
 	flag.Parse()
-	if testBackend == "" {
-		testBackend = os.Getenv("PROCTEST")
-		if testBackend == "" {
-			testBackend = "native"
-		}
-	}
+	protest.DefaultTestBackend(&testBackend)
 	logflags.Setup(logOutput != "", logOutput)
 	os.Exit(protest.RunTestsWithFixtures(m))
 }