Update vendor

vendor/golang.org/x/tools/go/ast/astutil/imports.go

@@ -275,9 +275,10 @@ func DeleteNamedImport(fset *token.FileSet, f *ast.File, name, path string) (del
 
 				// We deleted an entry but now there may be
 				// a blank line-sized hole where the import was.
-				if line-lastLine > 1 {
+				if line-lastLine > 1 || !gen.Rparen.IsValid() {
 					// There was a blank line immediately preceding the deleted import,
-					// so there's no need to close the hole.
+					// so there's no need to close the hole. The right parenthesis is
+					// invalid after AddImport to an import statement without parenthesis.
 					// Do nothing.
 				} else if line != fset.File(gen.Rparen).LineCount() {
 					// There was no blank line. Close the hole.

vendor/golang.org/x/tools/go/gcexportdata/gcexportdata.go

@@ -100,7 +100,7 @@ func Read(in io.Reader, fset *token.FileSet, imports map[string]*types.Package,
 // Write writes encoded type information for the specified package to out.
 // The FileSet provides file position information for named objects.
 func Write(out io.Writer, fset *token.FileSet, pkg *types.Package) error {
-	b, err := gcimporter.BExportData(fset, pkg)
+	b, err := gcimporter.IExportData(fset, pkg)
 	if err != nil {
 		return err
 	}

vendor/golang.org/x/tools/go/internal/gcimporter/bimport.go

@@ -332,7 +332,7 @@ func (p *importer) pos() token.Pos {
 	p.prevFile = file
 	p.prevLine = line
 
-	return p.fake.pos(file, line)
+	return p.fake.pos(file, line, 0)
 }
 
 // Synthesize a token.Pos
@@ -341,7 +341,9 @@ type fakeFileSet struct {
 	files map[string]*token.File
 }
 
-func (s *fakeFileSet) pos(file string, line int) token.Pos {
+func (s *fakeFileSet) pos(file string, line, column int) token.Pos {
+	// TODO(mdempsky): Make use of column.
+
 	// Since we don't know the set of needed file positions, we
 	// reserve maxlines positions per file.
 	const maxlines = 64 * 1024

vendor/golang.org/x/tools/go/internal/gcimporter/gcimporter.go

@@ -344,7 +344,7 @@ func (p *parser) expectKeyword(keyword string) {
 
 // PackageId = string_lit .
 //
-func (p *parser) parsePackageId() string {
+func (p *parser) parsePackageID() string {
 	id, err := strconv.Unquote(p.expect(scanner.String))
 	if err != nil {
 		p.error(err)
@@ -384,7 +384,7 @@ func (p *parser) parseDotIdent() string {
 //
 func (p *parser) parseQualifiedName() (id, name string) {
 	p.expect('@')
-	id = p.parsePackageId()
+	id = p.parsePackageID()
 	p.expect('.')
 	// Per rev f280b8a485fd (10/2/2013), qualified names may be used for anonymous fields.
 	if p.tok == '?' {
@@ -696,7 +696,7 @@ func (p *parser) parseInterfaceType(parent *types.Package) types.Type {
 
 	// Complete requires the type's embedded interfaces to be fully defined,
 	// but we do not define any
-	return types.NewInterface(methods, nil).Complete()
+	return newInterface(methods, nil).Complete()
 }
 
 // ChanType = ( "chan" [ "<-" ] | "<-" "chan" ) Type .
@@ -785,7 +785,7 @@ func (p *parser) parseType(parent *types.Package) types.Type {
 func (p *parser) parseImportDecl() {
 	p.expectKeyword("import")
 	name := p.parsePackageName()
-	p.getPkg(p.parsePackageId(), name)
+	p.getPkg(p.parsePackageID(), name)
 }
 
 // int_lit = [ "+" | "-" ] { "0" ... "9" } .

vendor/golang.org/x/tools/go/internal/gcimporter/iexport.go

@@ -6,8 +6,6 @@
 // This file was derived from $GOROOT/src/cmd/compile/internal/gc/iexport.go;
 // see that file for specification of the format.
 
-// +build go1.11
-
 package gcimporter
 
 import (
@@ -28,7 +26,10 @@ import (
 const iexportVersion = 0
 
 // IExportData returns the binary export data for pkg.
+//
 // If no file set is provided, position info will be missing.
+// The package path of the top-level package will not be recorded,
+// so that calls to IImportData can override with a provided package path.
 func IExportData(fset *token.FileSet, pkg *types.Package) (b []byte, err error) {
 	defer func() {
 		if e := recover(); e != nil {
@@ -48,6 +49,7 @@ func IExportData(fset *token.FileSet, pkg *types.Package) (b []byte, err error)
 		stringIndex: map[string]uint64{},
 		declIndex:   map[types.Object]uint64{},
 		typIndex:    map[types.Type]uint64{},
+		localpkg:    pkg,
 	}
 
 	for i, pt := range predeclared() {
@@ -73,7 +75,7 @@ func IExportData(fset *token.FileSet, pkg *types.Package) (b []byte, err error)
 	// Append indices to data0 section.
 	dataLen := uint64(p.data0.Len())
 	w := p.newWriter()
-	w.writeIndex(p.declIndex, pkg)
+	w.writeIndex(p.declIndex)
 	w.flush()
 
 	// Assemble header.
@@ -95,14 +97,14 @@ func IExportData(fset *token.FileSet, pkg *types.Package) (b []byte, err error)
 // we're writing out the main index, which is also read by
 // non-compiler tools and includes a complete package description
 // (i.e., name and height).
-func (w *exportWriter) writeIndex(index map[types.Object]uint64, localpkg *types.Package) {
+func (w *exportWriter) writeIndex(index map[types.Object]uint64) {
 	// Build a map from packages to objects from that package.
 	pkgObjs := map[*types.Package][]types.Object{}
 
 	// For the main index, make sure to include every package that
 	// we reference, even if we're not exporting (or reexporting)
 	// any symbols from it.
-	pkgObjs[localpkg] = nil
+	pkgObjs[w.p.localpkg] = nil
 	for pkg := range w.p.allPkgs {
 		pkgObjs[pkg] = nil
 	}
@@ -121,12 +123,12 @@ func (w *exportWriter) writeIndex(index map[types.Object]uint64, localpkg *types
 	}
 
 	sort.Slice(pkgs, func(i, j int) bool {
-		return pkgs[i].Path() < pkgs[j].Path()
+		return w.exportPath(pkgs[i]) < w.exportPath(pkgs[j])
 	})
 
 	w.uint64(uint64(len(pkgs)))
 	for _, pkg := range pkgs {
-		w.string(pkg.Path())
+		w.string(w.exportPath(pkg))
 		w.string(pkg.Name())
 		w.uint64(uint64(0)) // package height is not needed for go/types
 
@@ -143,6 +145,8 @@ type iexporter struct {
 	fset *token.FileSet
 	out  *bytes.Buffer
 
+	localpkg *types.Package
+
 	// allPkgs tracks all packages that have been referenced by
 	// the export data, so we can ensure to include them in the
 	// main index.
@@ -195,6 +199,13 @@ type exportWriter struct {
 	prevLine int64
 }
 
+func (w *exportWriter) exportPath(pkg *types.Package) string {
+	if pkg == w.p.localpkg {
+		return ""
+	}
+	return pkg.Path()
+}
+
 func (p *iexporter) doDecl(obj types.Object) {
 	w := p.newWriter()
 	w.setPkg(obj.Pkg(), false)
@@ -267,6 +278,11 @@ func (w *exportWriter) tag(tag byte) {
 }
 
 func (w *exportWriter) pos(pos token.Pos) {
+	if w.p.fset == nil {
+		w.int64(0)
+		return
+	}
+
 	p := w.p.fset.Position(pos)
 	file := p.Filename
 	line := int64(p.Line)
@@ -299,7 +315,7 @@ func (w *exportWriter) pkg(pkg *types.Package) {
 	// Ensure any referenced packages are declared in the main index.
 	w.p.allPkgs[pkg] = true
 
-	w.string(pkg.Path())
+	w.string(w.exportPath(pkg))
 }
 
 func (w *exportWriter) qualifiedIdent(obj types.Object) {
@@ -394,7 +410,7 @@ func (w *exportWriter) doTyp(t types.Type, pkg *types.Package) {
 			w.pos(f.Pos())
 			w.string(f.Name())
 			w.typ(f.Type(), pkg)
-			w.bool(f.Embedded())
+			w.bool(f.Anonymous())
 			w.string(t.Tag(i)) // note (or tag)
 		}
 

vendor/golang.org/x/tools/go/internal/gcimporter/iimport.go

@@ -63,8 +63,8 @@ const (
 // If the export data version is not recognized or the format is otherwise
 // compromised, an error is returned.
 func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (_ int, pkg *types.Package, err error) {
-	const currentVersion = 0
-	version := -1
+	const currentVersion = 1
+	version := int64(-1)
 	defer func() {
 		if e := recover(); e != nil {
 			if version > currentVersion {
@@ -77,9 +77,9 @@ func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []
 
 	r := &intReader{bytes.NewReader(data), path}
 
-	version = int(r.uint64())
+	version = int64(r.uint64())
 	switch version {
-	case currentVersion:
+	case currentVersion, 0:
 	default:
 		errorf("unknown iexport format version %d", version)
 	}
@@ -93,7 +93,8 @@ func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []
 	r.Seek(sLen+dLen, io.SeekCurrent)
 
 	p := iimporter{
-		ipath: path,
+		ipath:   path,
+		version: int(version),
 
 		stringData:  stringData,
 		stringCache: make(map[uint64]string),
@@ -142,20 +143,18 @@ func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []
 		p.pkgIndex[pkg] = nameIndex
 		pkgList[i] = pkg
 	}
-	var localpkg *types.Package
-	for _, pkg := range pkgList {
-		if pkg.Path() == path {
-			localpkg = pkg
-		}
+	if len(pkgList) == 0 {
+		errorf("no packages found for %s", path)
+		panic("unreachable")
 	}
-
-	names := make([]string, 0, len(p.pkgIndex[localpkg]))
-	for name := range p.pkgIndex[localpkg] {
+	p.ipkg = pkgList[0]
+	names := make([]string, 0, len(p.pkgIndex[p.ipkg]))
+	for name := range p.pkgIndex[p.ipkg] {
 		names = append(names, name)
 	}
 	sort.Strings(names)
 	for _, name := range names {
-		p.doDecl(localpkg, name)
+		p.doDecl(p.ipkg, name)
 	}
 
 	for _, typ := range p.interfaceList {
@@ -165,17 +164,19 @@ func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []
 	// record all referenced packages as imports
 	list := append(([]*types.Package)(nil), pkgList[1:]...)
 	sort.Sort(byPath(list))
-	localpkg.SetImports(list)
+	p.ipkg.SetImports(list)
 
 	// package was imported completely and without errors
-	localpkg.MarkComplete()
+	p.ipkg.MarkComplete()
 
 	consumed, _ := r.Seek(0, io.SeekCurrent)
-	return int(consumed), localpkg, nil
+	return int(consumed), p.ipkg, nil
 }
 
 type iimporter struct {
-	ipath string
+	ipath   string
+	ipkg    *types.Package
+	version int
 
 	stringData  []byte
 	stringCache map[uint64]string
@@ -226,6 +227,9 @@ func (p *iimporter) pkgAt(off uint64) *types.Package {
 		return pkg
 	}
 	path := p.stringAt(off)
+	if path == p.ipath {
+		return p.ipkg
+	}
 	errorf("missing package %q in %q", path, p.ipath)
 	return nil
 }
@@ -255,6 +259,7 @@ type importReader struct {
 	currPkg    *types.Package
 	prevFile   string
 	prevLine   int64
+	prevColumn int64
 }
 
 func (r *importReader) obj(name string) {
@@ -448,6 +453,19 @@ func (r *importReader) qualifiedIdent() (*types.Package, string) {
 }
 
 func (r *importReader) pos() token.Pos {
+	if r.p.version >= 1 {
+		r.posv1()
+	} else {
+		r.posv0()
+	}
+
+	if r.prevFile == "" && r.prevLine == 0 && r.prevColumn == 0 {
+		return token.NoPos
+	}
+	return r.p.fake.pos(r.prevFile, int(r.prevLine), int(r.prevColumn))
+}
+
+func (r *importReader) posv0() {
 	delta := r.int64()
 	if delta != deltaNewFile {
 		r.prevLine += delta
@@ -457,12 +475,18 @@ func (r *importReader) pos() token.Pos {
 		r.prevFile = r.string()
 		r.prevLine = l
 	}
+}
 
-	if r.prevFile == "" && r.prevLine == 0 {
-		return token.NoPos
+func (r *importReader) posv1() {
+	delta := r.int64()
+	r.prevColumn += delta >> 1
+	if delta&1 != 0 {
+		delta = r.int64()
+		r.prevLine += delta >> 1
+		if delta&1 != 0 {
+			r.prevFile = r.string()
+		}
 	}
-
-	return r.p.fake.pos(r.prevFile, int(r.prevLine))
 }
 
 func (r *importReader) typ() types.Type {

vendor/golang.org/x/tools/go/internal/packagesdriver/sizes.go

@@ -11,17 +11,15 @@ import (
 	"encoding/json"
 	"fmt"
 	"go/types"
-	"log"
-	"os"
 	"os/exec"
 	"strings"
-	"time"
+
+	"golang.org/x/tools/internal/gocommand"
 )
 
 var debug = false
 
-// GetSizes returns the sizes used by the underlying driver with the given parameters.
-func GetSizes(ctx context.Context, buildFlags, env []string, dir string, usesExportData bool) (types.Sizes, error) {
+func GetSizes(ctx context.Context, buildFlags, env []string, gocmdRunner *gocommand.Runner, dir string) (types.Sizes, error) {
 	// TODO(matloob): Clean this up. This code is mostly a copy of packages.findExternalDriver.
 	const toolPrefix = "GOPACKAGESDRIVER="
 	tool := ""
@@ -41,7 +39,7 @@ func GetSizes(ctx context.Context, buildFlags, env []string, dir string, usesExp
 	}
 
 	if tool == "off" {
-		return GetSizesGolist(ctx, buildFlags, env, dir, usesExportData)
+		return GetSizesGolist(ctx, buildFlags, env, gocmdRunner, dir)
 	}
 
 	req, err := json.Marshal(struct {
@@ -77,97 +75,43 @@ func GetSizes(ctx context.Context, buildFlags, env []string, dir string, usesExp
 	return response.Sizes, nil
 }
 
-func GetSizesGolist(ctx context.Context, buildFlags, env []string, dir string, usesExportData bool) (types.Sizes, error) {
-	args := []string{"list", "-f", "{{context.GOARCH}} {{context.Compiler}}"}
-	args = append(args, buildFlags...)
-	args = append(args, "--", "unsafe")
-	stdout, err := InvokeGo(ctx, env, dir, usesExportData, args...)
+func GetSizesGolist(ctx context.Context, buildFlags, env []string, gocmdRunner *gocommand.Runner, dir string) (types.Sizes, error) {
+	inv := gocommand.Invocation{
+		Verb:       "list",
+		Args:       []string{"-f", "{{context.GOARCH}} {{context.Compiler}}", "--", "unsafe"},
+		Env:        env,
+		BuildFlags: buildFlags,
+		WorkingDir: dir,
+	}
+	stdout, stderr, friendlyErr, rawErr := gocmdRunner.RunRaw(ctx, inv)
 	var goarch, compiler string
-	if err != nil {
-		if strings.Contains(err.Error(), "cannot find main module") {
+	if rawErr != nil {
+		if strings.Contains(rawErr.Error(), "cannot find main module") {
 			// User's running outside of a module. All bets are off. Get GOARCH and guess compiler is gc.
 			// TODO(matloob): Is this a problem in practice?
-			envout, enverr := InvokeGo(ctx, env, dir, usesExportData, "env", "GOARCH")
+			inv := gocommand.Invocation{
+				Verb:       "env",
+				Args:       []string{"GOARCH"},
+				Env:        env,
+				WorkingDir: dir,
+			}
+			envout, enverr := gocmdRunner.Run(ctx, inv)
 			if enverr != nil {
-				return nil, err
+				return nil, enverr
 			}
 			goarch = strings.TrimSpace(envout.String())
 			compiler = "gc"
 		} else {
-			return nil, err
+			return nil, friendlyErr
 		}
 	} else {
 		fields := strings.Fields(stdout.String())
 		if len(fields) < 2 {
-			return nil, fmt.Errorf("could not determine GOARCH and Go compiler")
+			return nil, fmt.Errorf("could not parse GOARCH and Go compiler in format \"<GOARCH> <compiler>\":\nstdout: <<%s>>\nstderr: <<%s>>",
+				stdout.String(), stderr.String())
 		}
 		goarch = fields[0]
 		compiler = fields[1]
 	}
 	return types.SizesFor(compiler, goarch), nil
 }
-
-// InvokeGo returns the stdout of a go command invocation.
-func InvokeGo(ctx context.Context, env []string, dir string, usesExportData bool, args ...string) (*bytes.Buffer, error) {
-	if debug {
-		defer func(start time.Time) { log.Printf("%s for %v", time.Since(start), cmdDebugStr(env, args...)) }(time.Now())
-	}
-	stdout := new(bytes.Buffer)
-	stderr := new(bytes.Buffer)
-	cmd := exec.CommandContext(ctx, "go", args...)
-	// On darwin the cwd gets resolved to the real path, which breaks anything that
-	// expects the working directory to keep the original path, including the
-	// go command when dealing with modules.
-	// The Go stdlib has a special feature where if the cwd and the PWD are the
-	// same node then it trusts the PWD, so by setting it in the env for the child
-	// process we fix up all the paths returned by the go command.
-	cmd.Env = append(append([]string{}, env...), "PWD="+dir)
-	cmd.Dir = dir
-	cmd.Stdout = stdout
-	cmd.Stderr = stderr
-	if err := cmd.Run(); err != nil {
-		exitErr, ok := err.(*exec.ExitError)
-		if !ok {
-			// Catastrophic error:
-			// - executable not found
-			// - context cancellation
-			return nil, fmt.Errorf("couldn't exec 'go %v': %s %T", args, err, err)
-		}
-
-		// Export mode entails a build.
-		// If that build fails, errors appear on stderr
-		// (despite the -e flag) and the Export field is blank.
-		// Do not fail in that case.
-		if !usesExportData {
-			return nil, fmt.Errorf("go %v: %s: %s", args, exitErr, stderr)
-		}
-	}
-
-	// As of writing, go list -export prints some non-fatal compilation
-	// errors to stderr, even with -e set. We would prefer that it put
-	// them in the Package.Error JSON (see https://golang.org/issue/26319).
-	// In the meantime, there's nowhere good to put them, but they can
-	// be useful for debugging. Print them if $GOPACKAGESPRINTGOLISTERRORS
-	// is set.
-	if len(stderr.Bytes()) != 0 && os.Getenv("GOPACKAGESPRINTGOLISTERRORS") != "" {
-		fmt.Fprintf(os.Stderr, "%s stderr: <<%s>>\n", cmdDebugStr(env, args...), stderr)
-	}
-
-	// debugging
-	if false {
-		fmt.Fprintf(os.Stderr, "%s stdout: <<%s>>\n", cmdDebugStr(env, args...), stdout)
-	}
-
-	return stdout, nil
-}
-
-func cmdDebugStr(envlist []string, args ...string) string {
-	env := make(map[string]string)
-	for _, kv := range envlist {
-		split := strings.Split(kv, "=")
-		k, v := split[0], split[1]
-		env[k] = v
-	}
-
-	return fmt.Sprintf("GOROOT=%v GOPATH=%v GO111MODULE=%v PWD=%v go %v", env["GOROOT"], env["GOPATH"], env["GO111MODULE"], env["PWD"], args)
-}

vendor/golang.org/x/tools/go/packages/doc.go

@@ -60,8 +60,7 @@ causes Load to run in LoadFiles mode, collecting minimal information.
 See the documentation for type Config for details.
 
 As noted earlier, the Config.Mode controls the amount of detail
-reported about the loaded packages, with each mode returning all the data of the
-previous mode with some extra added. See the documentation for type LoadMode
+reported about the loaded packages. See the documentation for type LoadMode
 for details.
 
 Most tools should pass their command-line arguments (after any flags)

vendor/golang.org/x/tools/go/packages/external.go

@@ -12,6 +12,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"os"
 	"os/exec"
 	"strings"
 )
@@ -76,15 +77,21 @@ func findExternalDriver(cfg *Config) driver {
 		}
 
 		buf := new(bytes.Buffer)
+		stderr := new(bytes.Buffer)
 		cmd := exec.CommandContext(cfg.Context, tool, words...)
 		cmd.Dir = cfg.Dir
 		cmd.Env = cfg.Env
 		cmd.Stdin = bytes.NewReader(req)
 		cmd.Stdout = buf
-		cmd.Stderr = new(bytes.Buffer)
+		cmd.Stderr = stderr
+
 		if err := cmd.Run(); err != nil {
 			return nil, fmt.Errorf("%v: %v: %s", tool, err, cmd.Stderr)
 		}
+		if len(stderr.Bytes()) != 0 && os.Getenv("GOPACKAGESPRINTDRIVERERRORS") != "" {
+			fmt.Fprintf(os.Stderr, "%s stderr: <<%s>>\n", cmdDebugStr(cmd, words...), stderr)
+		}
+
 		var response driverResponse
 		if err := json.Unmarshal(buf.Bytes(), &response); err != nil {
 			return nil, err

vendor/golang.org/x/tools/go/packages/golist.go

@@ -6,26 +6,25 @@ package packages
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"go/types"
-	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
 	"reflect"
-	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
-	"time"
 	"unicode"
 
 	"golang.org/x/tools/go/internal/packagesdriver"
-	"golang.org/x/tools/internal/gopathwalk"
-	"golang.org/x/tools/internal/semver"
+	"golang.org/x/tools/internal/gocommand"
+	"golang.org/x/xerrors"
 )
 
 // debug controls verbose logging.
@@ -44,16 +43,21 @@ type responseDeduper struct {
 	dr           *driverResponse
 }
 
-// init fills in r with a driverResponse.
-func (r *responseDeduper) init(dr *driverResponse) {
-	r.dr = dr
-	r.seenRoots = map[string]bool{}
-	r.seenPackages = map[string]*Package{}
+func newDeduper() *responseDeduper {
+	return &responseDeduper{
+		dr:           &driverResponse{},
+		seenRoots:    map[string]bool{},
+		seenPackages: map[string]*Package{},
+	}
+}
+
+// addAll fills in r with a driverResponse.
+func (r *responseDeduper) addAll(dr *driverResponse) {
 	for _, pkg := range dr.Packages {
-		r.seenPackages[pkg.ID] = pkg
+		r.addPackage(pkg)
 	}
 	for _, root := range dr.Roots {
-		r.seenRoots[root] = true
+		r.addRoot(root)
 	}
 }
 
@@ -73,25 +77,47 @@ func (r *responseDeduper) addRoot(id string) {
 	r.dr.Roots = append(r.dr.Roots, id)
 }
 
-// goInfo contains global information from the go tool.
-type goInfo struct {
-	rootDirs map[string]string
-	env      goEnv
+type golistState struct {
+	cfg *Config
+	ctx context.Context
+
+	envOnce    sync.Once
+	goEnvError error
+	goEnv      map[string]string
+
+	rootsOnce     sync.Once
+	rootDirsError error
+	rootDirs      map[string]string
+
+	// vendorDirs caches the (non)existence of vendor directories.
+	vendorDirs map[string]bool
 }
 
-type goEnv struct {
-	modulesOn bool
+// getEnv returns Go environment variables. Only specific variables are
+// populated -- computing all of them is slow.
+func (state *golistState) getEnv() (map[string]string, error) {
+	state.envOnce.Do(func() {
+		var b *bytes.Buffer
+		b, state.goEnvError = state.invokeGo("env", "-json", "GOMOD", "GOPATH")
+		if state.goEnvError != nil {
+			return
+		}
+
+		state.goEnv = make(map[string]string)
+		decoder := json.NewDecoder(b)
+		if state.goEnvError = decoder.Decode(&state.goEnv); state.goEnvError != nil {
+			return
+		}
+	})
+	return state.goEnv, state.goEnvError
 }
 
-func determineEnv(cfg *Config) goEnv {
-	buf, err := invokeGo(cfg, "env", "GOMOD")
+// mustGetEnv is a convenience function that can be used if getEnv has already succeeded.
+func (state *golistState) mustGetEnv() map[string]string {
+	env, err := state.getEnv()
 	if err != nil {
-		return goEnv{}
+		panic(fmt.Sprintf("mustGetEnv: %v", err))
 	}
-	gomod := bytes.TrimSpace(buf.Bytes())
-
-	env := goEnv{}
-	env.modulesOn = len(gomod) > 0
 	return env
 }
 
@@ -99,42 +125,38 @@ func determineEnv(cfg *Config) goEnv {
 // the build system package structure.
 // See driver for more details.
 func goListDriver(cfg *Config, patterns ...string) (*driverResponse, error) {
-	var sizes types.Sizes
+	// Make sure that any asynchronous go commands are killed when we return.
+	parentCtx := cfg.Context
+	if parentCtx == nil {
+		parentCtx = context.Background()
+	}
+	ctx, cancel := context.WithCancel(parentCtx)
+	defer cancel()
+
+	response := newDeduper()
+
+	// Fill in response.Sizes asynchronously if necessary.
 	var sizeserr error
 	var sizeswg sync.WaitGroup
 	if cfg.Mode&NeedTypesSizes != 0 || cfg.Mode&NeedTypes != 0 {
 		sizeswg.Add(1)
 		go func() {
-			sizes, sizeserr = getSizes(cfg)
+			var sizes types.Sizes
+			sizes, sizeserr = packagesdriver.GetSizesGolist(ctx, cfg.BuildFlags, cfg.Env, cfg.gocmdRunner, cfg.Dir)
+			// types.SizesFor always returns nil or a *types.StdSizes.
+			response.dr.Sizes, _ = sizes.(*types.StdSizes)
 			sizeswg.Done()
 		}()
 	}
 
-	// start fetching rootDirs
-	var info goInfo
-	var rootDirsReady, envReady = make(chan struct{}), make(chan struct{})
-	go func() {
-		info.rootDirs = determineRootDirs(cfg)
-		close(rootDirsReady)
-	}()
-	go func() {
-		info.env = determineEnv(cfg)
-		close(envReady)
-	}()
-	getGoInfo := func() *goInfo {
-		<-rootDirsReady
-		<-envReady
-		return &info
-	}
-
-	// always pass getGoInfo to golistDriver
-	golistDriver := func(cfg *Config, patterns ...string) (*driverResponse, error) {
-		return golistDriver(cfg, getGoInfo, patterns...)
+	state := &golistState{
+		cfg:        cfg,
+		ctx:        ctx,
+		vendorDirs: map[string]bool{},
 	}
 
 	// Determine files requested in contains patterns
 	var containFiles []string
-	var packagesNamed []string
 	restPatterns := make([]string, 0, len(patterns))
 	// Extract file= and other [querytype]= patterns. Report an error if querytype
 	// doesn't exist.
@@ -150,8 +172,6 @@ extractQueries:
 				containFiles = append(containFiles, value)
 			case "pattern":
 				restPatterns = append(restPatterns, value)
-			case "iamashamedtousethedisabledqueryname":
-				packagesNamed = append(packagesNamed, value)
 			case "": // not a reserved query
 				restPatterns = append(restPatterns, pattern)
 			default:
@@ -167,52 +187,34 @@ extractQueries:
 		}
 	}
 
-	response := &responseDeduper{}
-	var err error
-
 	// See if we have any patterns to pass through to go list. Zero initial
 	// patterns also requires a go list call, since it's the equivalent of
 	// ".".
 	if len(restPatterns) > 0 || len(patterns) == 0 {
-		dr, err := golistDriver(cfg, restPatterns...)
+		dr, err := state.createDriverResponse(restPatterns...)
 		if err != nil {
 			return nil, err
 		}
-		response.init(dr)
-	} else {
-		response.init(&driverResponse{})
+		response.addAll(dr)
 	}
 
-	sizeswg.Wait()
-	if sizeserr != nil {
-		return nil, sizeserr
-	}
-	// types.SizesFor always returns nil or a *types.StdSizes
-	response.dr.Sizes, _ = sizes.(*types.StdSizes)
-
-	var containsCandidates []string
-
 	if len(containFiles) != 0 {
-		if err := runContainsQueries(cfg, golistDriver, response, containFiles, getGoInfo); err != nil {
+		if err := state.runContainsQueries(response, containFiles); err != nil {
 			return nil, err
 		}
 	}
 
-	if len(packagesNamed) != 0 {
-		if err := runNamedQueries(cfg, golistDriver, response, packagesNamed); err != nil {
-			return nil, err
-		}
-	}
-
-	modifiedPkgs, needPkgs, err := processGolistOverlay(cfg, response, getGoInfo)
+	modifiedPkgs, needPkgs, err := state.processGolistOverlay(response)
 	if err != nil {
 		return nil, err
 	}
+
+	var containsCandidates []string
 	if len(containFiles) > 0 {
 		containsCandidates = append(containsCandidates, modifiedPkgs...)
 		containsCandidates = append(containsCandidates, needPkgs...)
 	}
-	if err := addNeededOverlayPackages(cfg, golistDriver, response, needPkgs, getGoInfo); err != nil {
+	if err := state.addNeededOverlayPackages(response, needPkgs); err != nil {
 		return nil, err
 	}
 	// Check candidate packages for containFiles.
@@ -241,36 +243,32 @@ extractQueries:
 		}
 	}
 
+	sizeswg.Wait()
+	if sizeserr != nil {
+		return nil, sizeserr
+	}
 	return response.dr, nil
 }
 
-func addNeededOverlayPackages(cfg *Config, driver driver, response *responseDeduper, pkgs []string, getGoInfo func() *goInfo) error {
+func (state *golistState) addNeededOverlayPackages(response *responseDeduper, pkgs []string) error {
 	if len(pkgs) == 0 {
 		return nil
 	}
-	drivercfg := *cfg
-	if getGoInfo().env.modulesOn {
-		drivercfg.BuildFlags = append(drivercfg.BuildFlags, "-mod=readonly")
-	}
-	dr, err := driver(&drivercfg, pkgs...)
-
+	dr, err := state.createDriverResponse(pkgs...)
 	if err != nil {
 		return err
 	}
 	for _, pkg := range dr.Packages {
 		response.addPackage(pkg)
 	}
-	_, needPkgs, err := processGolistOverlay(cfg, response, getGoInfo)
+	_, needPkgs, err := state.processGolistOverlay(response)
 	if err != nil {
 		return err
 	}
-	if err := addNeededOverlayPackages(cfg, driver, response, needPkgs, getGoInfo); err != nil {
-		return err
-	}
-	return nil
+	return state.addNeededOverlayPackages(response, needPkgs)
 }
 
-func runContainsQueries(cfg *Config, driver driver, response *responseDeduper, queries []string, goInfo func() *goInfo) error {
+func (state *golistState) runContainsQueries(response *responseDeduper, queries []string) error {
 	for _, query := range queries {
 		// TODO(matloob): Do only one query per directory.
 		fdir := filepath.Dir(query)
@@ -280,31 +278,17 @@ func runContainsQueries(cfg *Config, driver driver, response *responseDeduper, q
 		if err != nil {
 			return fmt.Errorf("could not determine absolute path of file= query path %q: %v", query, err)
 		}
-		dirResponse, err := driver(cfg, pattern)
-		if err != nil || (len(dirResponse.Packages) == 1 && len(dirResponse.Packages[0].Errors) == 1) {
-			// There was an error loading the package. Try to load the file as an ad-hoc package.
-			// Usually the error will appear in a returned package, but may not if we're in modules mode
-			// and the ad-hoc is located outside a module.
+		dirResponse, err := state.createDriverResponse(pattern)
+
+		// If there was an error loading the package, or the package is returned
+		// with errors, try to load the file as an ad-hoc package.
+		// Usually the error will appear in a returned package, but may not if we're
+		// in module mode and the ad-hoc is located outside a module.
+		if err != nil || len(dirResponse.Packages) == 1 && len(dirResponse.Packages[0].GoFiles) == 0 &&
+			len(dirResponse.Packages[0].Errors) == 1 {
 			var queryErr error
-			dirResponse, queryErr = driver(cfg, query)
-			if queryErr != nil {
-				// Return the original error if the attempt to fall back failed.
-				return err
-			}
-			// Special case to handle issue #33482:
-			// If this is a file= query for ad-hoc packages where the file only exists on an overlay,
-			// and exists outside of a module, add the file in for the package.
-			if len(dirResponse.Packages) == 1 && len(dirResponse.Packages) == 1 &&
-				dirResponse.Packages[0].ID == "command-line-arguments" && len(dirResponse.Packages[0].GoFiles) == 0 {
-				filename := filepath.Join(pattern, filepath.Base(query)) // avoid recomputing abspath
-				// TODO(matloob): check if the file is outside of a root dir?
-				for path := range cfg.Overlay {
-					if path == filename {
-						dirResponse.Packages[0].Errors = nil
-						dirResponse.Packages[0].GoFiles = []string{path}
-						dirResponse.Packages[0].CompiledGoFiles = []string{path}
-					}
-				}
+			if dirResponse, queryErr = state.adhocPackage(pattern, query); queryErr != nil {
+				return err // return the original error
 			}
 		}
 		isRoot := make(map[string]bool, len(dirResponse.Roots))
@@ -332,260 +316,47 @@ func runContainsQueries(cfg *Config, driver driver, response *responseDeduper, q
 	return nil
 }
 
-// modCacheRegexp splits a path in a module cache into module, module version, and package.
-var modCacheRegexp = regexp.MustCompile(`(.*)@([^/\\]*)(.*)`)
-
-func runNamedQueries(cfg *Config, driver driver, response *responseDeduper, queries []string) error {
-	// calling `go env` isn't free; bail out if there's nothing to do.
-	if len(queries) == 0 {
-		return nil
-	}
-	// Determine which directories are relevant to scan.
-	roots, modRoot, err := roots(cfg)
+// adhocPackage attempts to load or construct an ad-hoc package for a given
+// query, if the original call to the driver produced inadequate results.
+func (state *golistState) adhocPackage(pattern, query string) (*driverResponse, error) {
+	response, err := state.createDriverResponse(query)
 	if err != nil {
-		return err
+		return nil, err
 	}
-
-	// Scan the selected directories. Simple matches, from GOPATH/GOROOT
-	// or the local module, can simply be "go list"ed. Matches from the
-	// module cache need special treatment.
-	var matchesMu sync.Mutex
-	var simpleMatches, modCacheMatches []string
-	add := func(root gopathwalk.Root, dir string) {
-		// Walk calls this concurrently; protect the result slices.
-		matchesMu.Lock()
-		defer matchesMu.Unlock()
-
-		path := dir
-		if dir != root.Path {
-			path = dir[len(root.Path)+1:]
-		}
-		if pathMatchesQueries(path, queries) {
-			switch root.Type {
-			case gopathwalk.RootModuleCache:
-				modCacheMatches = append(modCacheMatches, path)
-			case gopathwalk.RootCurrentModule:
-				// We'd need to read go.mod to find the full
-				// import path. Relative's easier.
-				rel, err := filepath.Rel(cfg.Dir, dir)
-				if err != nil {
-					// This ought to be impossible, since
-					// we found dir in the current module.
-					panic(err)
-				}
-				simpleMatches = append(simpleMatches, "./"+rel)
-			case gopathwalk.RootGOPATH, gopathwalk.RootGOROOT:
-				simpleMatches = append(simpleMatches, path)
-			}
-		}
+	// If we get nothing back from `go list`,
+	// try to make this file into its own ad-hoc package.
+	// TODO(rstambler): Should this check against the original response?
+	if len(response.Packages) == 0 {
+		response.Packages = append(response.Packages, &Package{
+			ID:              "command-line-arguments",
+			PkgPath:         query,
+			GoFiles:         []string{query},
+			CompiledGoFiles: []string{query},
+			Imports:         make(map[string]*Package),
+		})
+		response.Roots = append(response.Roots, "command-line-arguments")
 	}
-
-	startWalk := time.Now()
-	gopathwalk.Walk(roots, add, gopathwalk.Options{ModulesEnabled: modRoot != "", Debug: debug})
-	cfg.Logf("%v for walk", time.Since(startWalk))
-
-	// Weird special case: the top-level package in a module will be in
-	// whatever directory the user checked the repository out into. It's
-	// more reasonable for that to not match the package name. So, if there
-	// are any Go files in the mod root, query it just to be safe.
-	if modRoot != "" {
-		rel, err := filepath.Rel(cfg.Dir, modRoot)
-		if err != nil {
-			panic(err) // See above.
-		}
-
-		files, err := ioutil.ReadDir(modRoot)
-		for _, f := range files {
-			if strings.HasSuffix(f.Name(), ".go") {
-				simpleMatches = append(simpleMatches, rel)
-				break
-			}
-		}
-	}
-
-	addResponse := func(r *driverResponse) {
-		for _, pkg := range r.Packages {
-			response.addPackage(pkg)
-			for _, name := range queries {
-				if pkg.Name == name {
-					response.addRoot(pkg.ID)
-					break
+	// Handle special cases.
+	if len(response.Packages) == 1 {
+		// golang/go#33482: If this is a file= query for ad-hoc packages where
+		// the file only exists on an overlay, and exists outside of a module,
+		// add the file to the package and remove the errors.
+		if response.Packages[0].ID == "command-line-arguments" ||
+			filepath.ToSlash(response.Packages[0].PkgPath) == filepath.ToSlash(query) {
+			if len(response.Packages[0].GoFiles) == 0 {
+				filename := filepath.Join(pattern, filepath.Base(query)) // avoid recomputing abspath
+				// TODO(matloob): check if the file is outside of a root dir?
+				for path := range state.cfg.Overlay {
+					if path == filename {
+						response.Packages[0].Errors = nil
+						response.Packages[0].GoFiles = []string{path}
+						response.Packages[0].CompiledGoFiles = []string{path}
+					}
 				}
 			}
 		}
 	}
-
-	if len(simpleMatches) != 0 {
-		resp, err := driver(cfg, simpleMatches...)
-		if err != nil {
-			return err
-		}
-		addResponse(resp)
-	}
-
-	// Module cache matches are tricky. We want to avoid downloading new
-	// versions of things, so we need to use the ones present in the cache.
-	// go list doesn't accept version specifiers, so we have to write out a
-	// temporary module, and do the list in that module.
-	if len(modCacheMatches) != 0 {
-		// Collect all the matches, deduplicating by major version
-		// and preferring the newest.
-		type modInfo struct {
-			mod   string
-			major string
-		}
-		mods := make(map[modInfo]string)
-		var imports []string
-		for _, modPath := range modCacheMatches {
-			matches := modCacheRegexp.FindStringSubmatch(modPath)
-			mod, ver := filepath.ToSlash(matches[1]), matches[2]
-			importPath := filepath.ToSlash(filepath.Join(matches[1], matches[3]))
-
-			major := semver.Major(ver)
-			if prevVer, ok := mods[modInfo{mod, major}]; !ok || semver.Compare(ver, prevVer) > 0 {
-				mods[modInfo{mod, major}] = ver
-			}
-
-			imports = append(imports, importPath)
-		}
-
-		// Build the temporary module.
-		var gomod bytes.Buffer
-		gomod.WriteString("module modquery\nrequire (\n")
-		for mod, version := range mods {
-			gomod.WriteString("\t" + mod.mod + " " + version + "\n")
-		}
-		gomod.WriteString(")\n")
-
-		tmpCfg := *cfg
-
-		// We're only trying to look at stuff in the module cache, so
-		// disable the network. This should speed things up, and has
-		// prevented errors in at least one case, #28518.
-		tmpCfg.Env = append(append([]string{"GOPROXY=off"}, cfg.Env...))
-
-		var err error
-		tmpCfg.Dir, err = ioutil.TempDir("", "gopackages-modquery")
-		if err != nil {
-			return err
-		}
-		defer os.RemoveAll(tmpCfg.Dir)
-
-		if err := ioutil.WriteFile(filepath.Join(tmpCfg.Dir, "go.mod"), gomod.Bytes(), 0777); err != nil {
-			return fmt.Errorf("writing go.mod for module cache query: %v", err)
-		}
-
-		// Run the query, using the import paths calculated from the matches above.
-		resp, err := driver(&tmpCfg, imports...)
-		if err != nil {
-			return fmt.Errorf("querying module cache matches: %v", err)
-		}
-		addResponse(resp)
-	}
-
-	return nil
-}
-
-func getSizes(cfg *Config) (types.Sizes, error) {
-	return packagesdriver.GetSizesGolist(cfg.Context, cfg.BuildFlags, cfg.Env, cfg.Dir, usesExportData(cfg))
-}
-
-// roots selects the appropriate paths to walk based on the passed-in configuration,
-// particularly the environment and the presence of a go.mod in cfg.Dir's parents.
-func roots(cfg *Config) ([]gopathwalk.Root, string, error) {
-	stdout, err := invokeGo(cfg, "env", "GOROOT", "GOPATH", "GOMOD")
-	if err != nil {
-		return nil, "", err
-	}
-
-	fields := strings.Split(stdout.String(), "\n")
-	if len(fields) != 4 || len(fields[3]) != 0 {
-		return nil, "", fmt.Errorf("go env returned unexpected output: %q", stdout.String())
-	}
-	goroot, gopath, gomod := fields[0], filepath.SplitList(fields[1]), fields[2]
-	var modDir string
-	if gomod != "" {
-		modDir = filepath.Dir(gomod)
-	}
-
-	var roots []gopathwalk.Root
-	// Always add GOROOT.
-	roots = append(roots, gopathwalk.Root{filepath.Join(goroot, "/src"), gopathwalk.RootGOROOT})
-	// If modules are enabled, scan the module dir.
-	if modDir != "" {
-		roots = append(roots, gopathwalk.Root{modDir, gopathwalk.RootCurrentModule})
-	}
-	// Add either GOPATH/src or GOPATH/pkg/mod, depending on module mode.
-	for _, p := range gopath {
-		if modDir != "" {
-			roots = append(roots, gopathwalk.Root{filepath.Join(p, "/pkg/mod"), gopathwalk.RootModuleCache})
-		} else {
-			roots = append(roots, gopathwalk.Root{filepath.Join(p, "/src"), gopathwalk.RootGOPATH})
-		}
-	}
-
-	return roots, modDir, nil
-}
-
-// These functions were copied from goimports. See further documentation there.
-
-// pathMatchesQueries is adapted from pkgIsCandidate.
-// TODO: is it reasonable to do Contains here, rather than an exact match on a path component?
-func pathMatchesQueries(path string, queries []string) bool {
-	lastTwo := lastTwoComponents(path)
-	for _, query := range queries {
-		if strings.Contains(lastTwo, query) {
-			return true
-		}
-		if hasHyphenOrUpperASCII(lastTwo) && !hasHyphenOrUpperASCII(query) {
-			lastTwo = lowerASCIIAndRemoveHyphen(lastTwo)
-			if strings.Contains(lastTwo, query) {
-				return true
-			}
-		}
-	}
-	return false
-}
-
-// lastTwoComponents returns at most the last two path components
-// of v, using either / or \ as the path separator.
-func lastTwoComponents(v string) string {
-	nslash := 0
-	for i := len(v) - 1; i >= 0; i-- {
-		if v[i] == '/' || v[i] == '\\' {
-			nslash++
-			if nslash == 2 {
-				return v[i:]
-			}
-		}
-	}
-	return v
-}
-
-func hasHyphenOrUpperASCII(s string) bool {
-	for i := 0; i < len(s); i++ {
-		b := s[i]
-		if b == '-' || ('A' <= b && b <= 'Z') {
-			return true
-		}
-	}
-	return false
-}
-
-func lowerASCIIAndRemoveHyphen(s string) (ret string) {
-	buf := make([]byte, 0, len(s))
-	for i := 0; i < len(s); i++ {
-		b := s[i]
-		switch {
-		case b == '-':
-			continue
-		case 'A' <= b && b <= 'Z':
-			buf = append(buf, b+('a'-'A'))
-		default:
-			buf = append(buf, b)
-		}
-	}
-	return string(buf)
+	return response, nil
 }
 
 // Fields must match go list;
@@ -610,6 +381,7 @@ type jsonPackage struct {
 	Imports         []string
 	ImportMap       map[string]string
 	Deps            []string
+	Module          *Module
 	TestGoFiles     []string
 	TestImports     []string
 	XTestGoFiles    []string
@@ -630,16 +402,15 @@ func otherFiles(p *jsonPackage) [][]string {
 	return [][]string{p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.FFiles, p.SFiles, p.SwigFiles, p.SwigCXXFiles, p.SysoFiles}
 }
 
-// golistDriver uses the "go list" command to expand the pattern
-// words and return metadata for the specified packages. dir may be
-// "" and env may be nil, as per os/exec.Command.
-func golistDriver(cfg *Config, rootsDirs func() *goInfo, words ...string) (*driverResponse, error) {
+// createDriverResponse uses the "go list" command to expand the pattern
+// words and return a response for the specified packages.
+func (state *golistState) createDriverResponse(words ...string) (*driverResponse, error) {
 	// go list uses the following identifiers in ImportPath and Imports:
 	//
 	// 	"p"			-- importable package or main (command)
-	//      "q.test"		-- q's test executable
+	// 	"q.test"		-- q's test executable
 	// 	"p [q.test]"		-- variant of p as built for q's test executable
-	//	"q_test [q.test]"	-- q's external test package
+	// 	"q_test [q.test]"	-- q's external test package
 	//
 	// The packages p that are built differently for a test q.test
 	// are q itself, plus any helpers used by the external test q_test,
@@ -647,11 +418,13 @@ func golistDriver(cfg *Config, rootsDirs func() *goInfo, words ...string) (*driv
 
 	// Run "go list" for complete
 	// information on the specified packages.
-	buf, err := invokeGo(cfg, golistargs(cfg, words)...)
+	buf, err := state.invokeGo("list", golistargs(state.cfg, words)...)
 	if err != nil {
 		return nil, err
 	}
 	seen := make(map[string]*jsonPackage)
+	pkgs := make(map[string]*Package)
+	additionalErrors := make(map[string][]Error)
 	// Decode the JSON and convert it to Package form.
 	var response driverResponse
 	for dec := json.NewDecoder(buf); dec.More(); {
@@ -682,18 +455,81 @@ func golistDriver(cfg *Config, rootsDirs func() *goInfo, words ...string) (*driv
 		// contained in a known module or GOPATH entry. This will allow the package to be
 		// properly "reclaimed" when overlays are processed.
 		if filepath.IsAbs(p.ImportPath) && p.Error != nil {
-			pkgPath, ok := getPkgPath(p.ImportPath, rootsDirs)
+			pkgPath, ok, err := state.getPkgPath(p.ImportPath)
+			if err != nil {
+				return nil, err
+			}
 			if ok {
 				p.ImportPath = pkgPath
 			}
 		}
 
 		if old, found := seen[p.ImportPath]; found {
-			if !reflect.DeepEqual(p, old) {
-				return nil, fmt.Errorf("internal error: go list gives conflicting information for package %v", p.ImportPath)
+			// If one version of the package has an error, and the other doesn't, assume
+			// that this is a case where go list is reporting a fake dependency variant
+			// of the imported package: When a package tries to invalidly import another
+			// package, go list emits a variant of the imported package (with the same
+			// import path, but with an error on it, and the package will have a
+			// DepError set on it). An example of when this can happen is for imports of
+			// main packages: main packages can not be imported, but they may be
+			// separately matched and listed by another pattern.
+			// See golang.org/issue/36188 for more details.
+
+			// The plan is that eventually, hopefully in Go 1.15, the error will be
+			// reported on the importing package rather than the duplicate "fake"
+			// version of the imported package. Once all supported versions of Go
+			// have the new behavior this logic can be deleted.
+			// TODO(matloob): delete the workaround logic once all supported versions of
+			// Go return the errors on the proper package.
+
+			// There should be exactly one version of a package that doesn't have an
+			// error.
+			if old.Error == nil && p.Error == nil {
+				if !reflect.DeepEqual(p, old) {
+					return nil, fmt.Errorf("internal error: go list gives conflicting information for package %v", p.ImportPath)
+				}
+				continue
 			}
-			// skip the duplicate
-			continue
+
+			// Determine if this package's error needs to be bubbled up.
+			// This is a hack, and we expect for go list to eventually set the error
+			// on the package.
+			if old.Error != nil {
+				var errkind string
+				if strings.Contains(old.Error.Err, "not an importable package") {
+					errkind = "not an importable package"
+				} else if strings.Contains(old.Error.Err, "use of internal package") && strings.Contains(old.Error.Err, "not allowed") {
+					errkind = "use of internal package not allowed"
+				}
+				if errkind != "" {
+					if len(old.Error.ImportStack) < 1 {
+						return nil, fmt.Errorf(`internal error: go list gave a %q error with empty import stack`, errkind)
+					}
+					importingPkg := old.Error.ImportStack[len(old.Error.ImportStack)-1]
+					if importingPkg == old.ImportPath {
+						// Using an older version of Go which put this package itself on top of import
+						// stack, instead of the importer. Look for importer in second from top
+						// position.
+						if len(old.Error.ImportStack) < 2 {
+							return nil, fmt.Errorf(`internal error: go list gave a %q error with an import stack without importing package`, errkind)
+						}
+						importingPkg = old.Error.ImportStack[len(old.Error.ImportStack)-2]
+					}
+					additionalErrors[importingPkg] = append(additionalErrors[importingPkg], Error{
+						Pos:  old.Error.Pos,
+						Msg:  old.Error.Err,
+						Kind: ListError,
+					})
+				}
+			}
+
+			// Make sure that if there's a version of the package without an error,
+			// that's the one reported to the user.
+			if old.Error == nil {
+				continue
+			}
+
+			// This package will replace the old one at the end of the loop.
 		}
 		seen[p.ImportPath] = p
 
@@ -703,6 +539,27 @@ func golistDriver(cfg *Config, rootsDirs func() *goInfo, words ...string) (*driv
 			GoFiles:         absJoin(p.Dir, p.GoFiles, p.CgoFiles),
 			CompiledGoFiles: absJoin(p.Dir, p.CompiledGoFiles),
 			OtherFiles:      absJoin(p.Dir, otherFiles(p)...),
+			forTest:         p.ForTest,
+			Module:          p.Module,
+		}
+
+		if (state.cfg.Mode&typecheckCgo) != 0 && len(p.CgoFiles) != 0 {
+			if len(p.CompiledGoFiles) > len(p.GoFiles) {
+				// We need the cgo definitions, which are in the first
+				// CompiledGoFile after the non-cgo ones. This is a hack but there
+				// isn't currently a better way to find it. We also need the pure
+				// Go files and unprocessed cgo files, all of which are already
+				// in pkg.GoFiles.
+				cgoTypes := p.CompiledGoFiles[len(p.GoFiles)]
+				pkg.CompiledGoFiles = append([]string{cgoTypes}, pkg.GoFiles...)
+			} else {
+				// golang/go#38990: go list silently fails to do cgo processing
+				pkg.CompiledGoFiles = nil
+				pkg.Errors = append(pkg.Errors, Error{
+					Msg:  "go list failed to return CompiledGoFiles; https://golang.org/issue/38990?",
+					Kind: ListError,
+				})
+			}
 		}
 
 		// Work around https://golang.org/issue/28749:
@@ -778,38 +635,86 @@ func golistDriver(cfg *Config, rootsDirs func() *goInfo, words ...string) (*driv
 			pkg.CompiledGoFiles = pkg.GoFiles
 		}
 
+		// Temporary work-around for golang/go#39986. Parse filenames out of
+		// error messages. This happens if there are unrecoverable syntax
+		// errors in the source, so we can't match on a specific error message.
+		if err := p.Error; err != nil && len(err.ImportStack) == 0 && len(pkg.CompiledGoFiles) == 0 {
+			if split := strings.Split(err.Pos, ":"); len(split) > 1 {
+				if filename := split[0]; filename != "" {
+					if !filepath.IsAbs(filename) {
+						filename = filepath.Join(state.cfg.Dir, filename)
+					}
+					if info, _ := os.Stat(filename); info != nil {
+						pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, filename)
+						pkg.GoFiles = append(pkg.GoFiles, filename)
+					}
+				}
+			}
+		}
+
 		if p.Error != nil {
+			msg := strings.TrimSpace(p.Error.Err) // Trim to work around golang.org/issue/32363.
+			// Address golang.org/issue/35964 by appending import stack to error message.
+			if msg == "import cycle not allowed" && len(p.Error.ImportStack) != 0 {
+				msg += fmt.Sprintf(": import stack: %v", p.Error.ImportStack)
+			}
 			pkg.Errors = append(pkg.Errors, Error{
-				Pos: p.Error.Pos,
-				Msg: strings.TrimSpace(p.Error.Err), // Trim to work around golang.org/issue/32363.
+				Pos:  p.Error.Pos,
+				Msg:  msg,
+				Kind: ListError,
 			})
 		}
 
+		pkgs[pkg.ID] = pkg
+	}
+
+	for id, errs := range additionalErrors {
+		if p, ok := pkgs[id]; ok {
+			p.Errors = append(p.Errors, errs...)
+		}
+	}
+	for _, pkg := range pkgs {
 		response.Packages = append(response.Packages, pkg)
 	}
+	sort.Slice(response.Packages, func(i, j int) bool { return response.Packages[i].ID < response.Packages[j].ID })
 
 	return &response, nil
 }
 
 // getPkgPath finds the package path of a directory if it's relative to a root directory.
-func getPkgPath(dir string, goInfo func() *goInfo) (string, bool) {
-	for rdir, rpath := range goInfo().rootDirs {
+func (state *golistState) getPkgPath(dir string) (string, bool, error) {
+	absDir, err := filepath.Abs(dir)
+	if err != nil {
+		return "", false, err
+	}
+	roots, err := state.determineRootDirs()
+	if err != nil {
+		return "", false, err
+	}
+
+	for rdir, rpath := range roots {
+		// Make sure that the directory is in the module,
+		// to avoid creating a path relative to another module.
+		if !strings.HasPrefix(absDir, rdir) {
+			continue
+		}
 		// TODO(matloob): This doesn't properly handle symlinks.
 		r, err := filepath.Rel(rdir, dir)
 		if err != nil {
 			continue
 		}
 		if rpath != "" {
-			// We choose only ore root even though the directory even it can belong in multiple modules
+			// We choose only one root even though the directory even it can belong in multiple modules
 			// or GOPATH entries. This is okay because we only need to work with absolute dirs when a
 			// file is missing from disk, for instance when gopls calls go/packages in an overlay.
 			// Once the file is saved, gopls, or the next invocation of the tool will get the correct
 			// result straight from golist.
 			// TODO(matloob): Implement module tiebreaking?
-			return path.Join(rpath, filepath.ToSlash(r)), true
+			return path.Join(rpath, filepath.ToSlash(r)), true, nil
 		}
+		return filepath.ToSlash(r), true, nil
 	}
-	return "", false
+	return "", false, nil
 }
 
 // absJoin absolutizes and flattens the lists of files.
@@ -828,8 +733,8 @@ func absJoin(dir string, fileses ...[]string) (res []string) {
 func golistargs(cfg *Config, words []string) []string {
 	const findFlags = NeedImports | NeedTypes | NeedSyntax | NeedTypesInfo
 	fullargs := []string{
-		"list", "-e", "-json",
-		fmt.Sprintf("-compiled=%t", cfg.Mode&(NeedCompiledGoFiles|NeedSyntax|NeedTypesInfo|NeedTypesSizes) != 0),
+		"-e", "-json",
+		fmt.Sprintf("-compiled=%t", cfg.Mode&(NeedCompiledGoFiles|NeedSyntax|NeedTypes|NeedTypesInfo|NeedTypesSizes) != 0),
 		fmt.Sprintf("-test=%t", cfg.Tests),
 		fmt.Sprintf("-export=%t", usesExportData(cfg)),
 		fmt.Sprintf("-deps=%t", cfg.Mode&NeedImports != 0),
@@ -844,25 +749,23 @@ func golistargs(cfg *Config, words []string) []string {
 }
 
 // invokeGo returns the stdout of a go command invocation.
-func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) {
-	stdout := new(bytes.Buffer)
-	stderr := new(bytes.Buffer)
-	cmd := exec.CommandContext(cfg.Context, "go", args...)
-	// On darwin the cwd gets resolved to the real path, which breaks anything that
-	// expects the working directory to keep the original path, including the
-	// go command when dealing with modules.
-	// The Go stdlib has a special feature where if the cwd and the PWD are the
-	// same node then it trusts the PWD, so by setting it in the env for the child
-	// process we fix up all the paths returned by the go command.
-	cmd.Env = append(append([]string{}, cfg.Env...), "PWD="+cfg.Dir)
-	cmd.Dir = cfg.Dir
-	cmd.Stdout = stdout
-	cmd.Stderr = stderr
-	defer func(start time.Time) {
-		cfg.Logf("%s for %v, stderr: <<%s>>\n", time.Since(start), cmdDebugStr(cmd, args...), stderr)
-	}(time.Now())
+func (state *golistState) invokeGo(verb string, args ...string) (*bytes.Buffer, error) {
+	cfg := state.cfg
 
-	if err := cmd.Run(); err != nil {
+	inv := gocommand.Invocation{
+		Verb:       verb,
+		Args:       args,
+		BuildFlags: cfg.BuildFlags,
+		Env:        cfg.Env,
+		Logf:       cfg.Logf,
+		WorkingDir: cfg.Dir,
+	}
+	gocmdRunner := cfg.gocmdRunner
+	if gocmdRunner == nil {
+		gocmdRunner = &gocommand.Runner{}
+	}
+	stdout, stderr, _, err := gocmdRunner.RunRaw(cfg.Context, inv)
+	if err != nil {
 		// Check for 'go' executable not being found.
 		if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound {
 			return nil, fmt.Errorf("'go list' driver requires 'go', but %s", exec.ErrNotFound)
@@ -872,7 +775,7 @@ func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) {
 		if !ok {
 			// Catastrophic error:
 			// - context cancellation
-			return nil, fmt.Errorf("couldn't exec 'go %v': %s %T", args, err, err)
+			return nil, xerrors.Errorf("couldn't run 'go': %w", err)
 		}
 
 		// Old go version?
@@ -896,10 +799,15 @@ func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) {
 			//    (the Graphic characters without spaces) and may also exclude the
 			//    characters !"#$%&'()*,:;<=>?[\]^`{|} and the Unicode replacement character U+FFFD.
 			return unicode.IsOneOf([]*unicode.RangeTable{unicode.L, unicode.M, unicode.N, unicode.P, unicode.S}, r) &&
-				strings.IndexRune("!\"#$%&'()*,:;<=>?[\\]^`{|}\uFFFD", r) == -1
+				!strings.ContainsRune("!\"#$%&'()*,:;<=>?[\\]^`{|}\uFFFD", r)
 		}
 		if len(stderr.String()) > 0 && strings.HasPrefix(stderr.String(), "# ") {
-			if strings.HasPrefix(strings.TrimLeftFunc(stderr.String()[len("# "):], isPkgPathRune), "\n") {
+			msg := stderr.String()[len("# "):]
+			if strings.HasPrefix(strings.TrimLeftFunc(msg, isPkgPathRune), "\n") {
+				return stdout, nil
+			}
+			// Treat pkg-config errors as a special case (golang.org/issue/36770).
+			if strings.HasPrefix(msg, "pkg-config") {
 				return stdout, nil
 			}
 		}
@@ -947,13 +855,24 @@ func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) {
 				// TODO(matloob): command-line-arguments isn't correct here.
 				"command-line-arguments", strings.Trim(stderr.String(), "\n"))
 			return bytes.NewBufferString(output), nil
+		}
 
+		// Another variation of the previous error
+		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "outside module root") {
+			output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
+				// TODO(matloob): command-line-arguments isn't correct here.
+				"command-line-arguments", strings.Trim(stderr.String(), "\n"))
+			return bytes.NewBufferString(output), nil
 		}
 
 		// Workaround for an instance of golang.org/issue/26755: go list -e  will return a non-zero exit
 		// status if there's a dependency on a package that doesn't exist. But it should return
 		// a zero exit status and set an error on that package.
 		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "no Go files in") {
+			// Don't clobber stdout if `go list` actually returned something.
+			if len(stdout.String()) > 0 {
+				return stdout, nil
+			}
 			// try to extract package name from string
 			stderrStr := stderr.String()
 			var importPath string
@@ -977,22 +896,6 @@ func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) {
 			return nil, fmt.Errorf("go %v: %s: %s", args, exitErr, stderr)
 		}
 	}
-
-	// As of writing, go list -export prints some non-fatal compilation
-	// errors to stderr, even with -e set. We would prefer that it put
-	// them in the Package.Error JSON (see https://golang.org/issue/26319).
-	// In the meantime, there's nowhere good to put them, but they can
-	// be useful for debugging. Print them if $GOPACKAGESPRINTGOLISTERRORS
-	// is set.
-	if len(stderr.Bytes()) != 0 && os.Getenv("GOPACKAGESPRINTGOLISTERRORS") != "" {
-		fmt.Fprintf(os.Stderr, "%s stderr: <<%s>>\n", cmdDebugStr(cmd, args...), stderr)
-	}
-
-	// debugging
-	if false {
-		fmt.Fprintf(os.Stderr, "%s stdout: <<%s>>\n", cmdDebugStr(cmd, args...), stdout)
-	}
-
 	return stdout, nil
 }
 

vendor/golang.org/x/tools/go/packages/golist_overlay.go

@@ -1,13 +1,14 @@
 package packages
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"go/parser"
 	"go/token"
-	"path"
+	"log"
+	"os"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 )
@@ -17,15 +18,20 @@ import (
 // sometimes incorrect.
 // TODO(matloob): Handle unsupported cases, including the following:
 // - determining the correct package to add given a new import path
-func processGolistOverlay(cfg *Config, response *responseDeduper, rootDirs func() *goInfo) (modifiedPkgs, needPkgs []string, err error) {
+func (state *golistState) processGolistOverlay(response *responseDeduper) (modifiedPkgs, needPkgs []string, err error) {
 	havePkgs := make(map[string]string) // importPath -> non-test package ID
 	needPkgsSet := make(map[string]bool)
 	modifiedPkgsSet := make(map[string]bool)
 
+	pkgOfDir := make(map[string][]*Package)
 	for _, pkg := range response.dr.Packages {
 		// This is an approximation of import path to id. This can be
 		// wrong for tests, vendored packages, and a number of other cases.
 		havePkgs[pkg.PkgPath] = pkg.ID
+		x := commonDir(pkg.GoFiles)
+		if x != "" {
+			pkgOfDir[x] = append(pkgOfDir[x], pkg)
+		}
 	}
 
 	// If no new imports are added, it is safe to avoid loading any needPkgs.
@@ -35,7 +41,23 @@ func processGolistOverlay(cfg *Config, response *responseDeduper, rootDirs func(
 	// potentially modifying the transitive set of dependencies).
 	var overlayAddsImports bool
 
-	for opath, contents := range cfg.Overlay {
+	// If both a package and its test package are created by the overlay, we
+	// need the real package first. Process all non-test files before test
+	// files, and make the whole process deterministic while we're at it.
+	var overlayFiles []string
+	for opath := range state.cfg.Overlay {
+		overlayFiles = append(overlayFiles, opath)
+	}
+	sort.Slice(overlayFiles, func(i, j int) bool {
+		iTest := strings.HasSuffix(overlayFiles[i], "_test.go")
+		jTest := strings.HasSuffix(overlayFiles[j], "_test.go")
+		if iTest != jTest {
+			return !iTest // non-tests are before tests.
+		}
+		return overlayFiles[i] < overlayFiles[j]
+	})
+	for _, opath := range overlayFiles {
+		contents := state.cfg.Overlay[opath]
 		base := filepath.Base(opath)
 		dir := filepath.Dir(opath)
 		var pkg *Package           // if opath belongs to both a package and its test variant, this will be the test variant
@@ -48,6 +70,9 @@ func processGolistOverlay(cfg *Config, response *responseDeduper, rootDirs func(
 			// to the overlay.
 			continue
 		}
+		// If all the overlay files belong to a different package, change the
+		// package name to that package.
+		maybeFixPackageName(pkgName, isTestFile, pkgOfDir[dir])
 	nextPackage:
 		for _, p := range response.dr.Packages {
 			if pkgName != p.Name && p.ID != "command-line-arguments" {
@@ -65,14 +90,8 @@ func processGolistOverlay(cfg *Config, response *responseDeduper, rootDirs func(
 					testVariantOf = p
 					continue nextPackage
 				}
+				// We must have already seen the package of which this is a test variant.
 				if pkg != nil && p != pkg && pkg.PkgPath == p.PkgPath {
-					// If we've already seen the test variant,
-					// make sure to label which package it is a test variant of.
-					if hasTestFiles(pkg) {
-						testVariantOf = p
-						continue nextPackage
-					}
-					// If we have already seen the package of which this is a test variant.
 					if hasTestFiles(p) {
 						testVariantOf = pkg
 					}
@@ -83,54 +102,68 @@ func processGolistOverlay(cfg *Config, response *responseDeduper, rootDirs func(
 				}
 			}
 		}
-		// The overlay could have included an entirely new package.
-		if pkg == nil {
+		// The overlay could have included an entirely new package or an
+		// ad-hoc package. An ad-hoc package is one that we have manually
+		// constructed from inadequate `go list` results for a file= query.
+		// It will have the ID command-line-arguments.
+		if pkg == nil || pkg.ID == "command-line-arguments" {
 			// Try to find the module or gopath dir the file is contained in.
 			// Then for modules, add the module opath to the beginning.
-			var pkgPath string
-			for rdir, rpath := range rootDirs().rootDirs {
-				// TODO(matloob): This doesn't properly handle symlinks.
-				r, err := filepath.Rel(rdir, dir)
-				if err != nil {
-					continue
-				}
-				pkgPath = filepath.ToSlash(r)
-				if rpath != "" {
-					pkgPath = path.Join(rpath, pkgPath)
-				}
-				// We only create one new package even it can belong in multiple modules or GOPATH entries.
-				// This is okay because tools (such as the LSP) that use overlays will recompute the overlay
-				// once the file is saved, and golist will do the right thing.
-				// TODO(matloob): Implement module tiebreaking?
+			pkgPath, ok, err := state.getPkgPath(dir)
+			if err != nil {
+				return nil, nil, err
+			}
+			if !ok {
 				break
 			}
-			if pkgPath == "" {
-				continue
-			}
+			var forTest string // only set for x tests
 			isXTest := strings.HasSuffix(pkgName, "_test")
 			if isXTest {
+				forTest = pkgPath
 				pkgPath += "_test"
 			}
 			id := pkgPath
-			if isTestFile && !isXTest {
-				id = fmt.Sprintf("%s [%s.test]", pkgPath, pkgPath)
-			}
-			// Try to reclaim a package with the same id if it exists in the response.
-			for _, p := range response.dr.Packages {
-				if reclaimPackage(p, id, opath, contents) {
-					pkg = p
-					break
+			if isTestFile {
+				if isXTest {
+					id = fmt.Sprintf("%s [%s.test]", pkgPath, forTest)
+				} else {
+					id = fmt.Sprintf("%s [%s.test]", pkgPath, pkgPath)
 				}
 			}
-			// Otherwise, create a new package
-			if pkg == nil {
-				pkg = &Package{PkgPath: pkgPath, ID: id, Name: pkgName, Imports: make(map[string]*Package)}
-				response.addPackage(pkg)
-				havePkgs[pkg.PkgPath] = id
-				// Add the production package's sources for a test variant.
-				if isTestFile && !isXTest && testVariantOf != nil {
-					pkg.GoFiles = append(pkg.GoFiles, testVariantOf.GoFiles...)
-					pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, testVariantOf.CompiledGoFiles...)
+			if pkg != nil {
+				// TODO(rstambler): We should change the package's path and ID
+				// here. The only issue is that this messes with the roots.
+			} else {
+				// Try to reclaim a package with the same ID, if it exists in the response.
+				for _, p := range response.dr.Packages {
+					if reclaimPackage(p, id, opath, contents) {
+						pkg = p
+						break
+					}
+				}
+				// Otherwise, create a new package.
+				if pkg == nil {
+					pkg = &Package{
+						PkgPath: pkgPath,
+						ID:      id,
+						Name:    pkgName,
+						Imports: make(map[string]*Package),
+					}
+					response.addPackage(pkg)
+					havePkgs[pkg.PkgPath] = id
+					// Add the production package's sources for a test variant.
+					if isTestFile && !isXTest && testVariantOf != nil {
+						pkg.GoFiles = append(pkg.GoFiles, testVariantOf.GoFiles...)
+						pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, testVariantOf.CompiledGoFiles...)
+						// Add the package under test and its imports to the test variant.
+						pkg.forTest = testVariantOf.PkgPath
+						for k, v := range testVariantOf.Imports {
+							pkg.Imports[k] = &Package{ID: v.ID}
+						}
+					}
+					if isXTest {
+						pkg.forTest = forTest
+					}
 				}
 			}
 		}
@@ -147,42 +180,47 @@ func processGolistOverlay(cfg *Config, response *responseDeduper, rootDirs func(
 			continue
 		}
 		for _, imp := range imports {
-			_, found := pkg.Imports[imp]
-			if !found {
-				overlayAddsImports = true
-				// TODO(matloob): Handle cases when the following block isn't correct.
-				// These include imports of vendored packages, etc.
-				id, ok := havePkgs[imp]
-				if !ok {
-					id = imp
-				}
-				pkg.Imports[imp] = &Package{ID: id}
-				// Add dependencies to the non-test variant version of this package as wel.
-				if testVariantOf != nil {
-					testVariantOf.Imports[imp] = &Package{ID: id}
+			// TODO(rstambler): If the package is an x test and the import has
+			// a test variant, make sure to replace it.
+			if _, found := pkg.Imports[imp]; found {
+				continue
+			}
+			overlayAddsImports = true
+			id, ok := havePkgs[imp]
+			if !ok {
+				var err error
+				id, err = state.resolveImport(dir, imp)
+				if err != nil {
+					return nil, nil, err
 				}
 			}
+			pkg.Imports[imp] = &Package{ID: id}
+			// Add dependencies to the non-test variant version of this package as well.
+			if testVariantOf != nil {
+				testVariantOf.Imports[imp] = &Package{ID: id}
+			}
 		}
-		continue
 	}
 
-	// toPkgPath tries to guess the package path given the id.
-	// This isn't always correct -- it's certainly wrong for
-	// vendored packages' paths.
-	toPkgPath := func(id string) string {
-		// TODO(matloob): Handle vendor paths.
-		i := strings.IndexByte(id, ' ')
-		if i >= 0 {
-			return id[:i]
+	// toPkgPath guesses the package path given the id.
+	toPkgPath := func(sourceDir, id string) (string, error) {
+		if i := strings.IndexByte(id, ' '); i >= 0 {
+			return state.resolveImport(sourceDir, id[:i])
 		}
-		return id
+		return state.resolveImport(sourceDir, id)
 	}
 
-	// Do another pass now that new packages have been created to determine the
-	// set of missing packages.
+	// Now that new packages have been created, do another pass to determine
+	// the new set of missing packages.
 	for _, pkg := range response.dr.Packages {
 		for _, imp := range pkg.Imports {
-			pkgPath := toPkgPath(imp.ID)
+			if len(pkg.GoFiles) == 0 {
+				return nil, nil, fmt.Errorf("cannot resolve imports for package %q with no Go files", pkg.PkgPath)
+			}
+			pkgPath, err := toPkgPath(filepath.Dir(pkg.GoFiles[0]), imp.ID)
+			if err != nil {
+				return nil, nil, err
+			}
 			if _, ok := havePkgs[pkgPath]; !ok {
 				needPkgsSet[pkgPath] = true
 			}
@@ -202,6 +240,52 @@ func processGolistOverlay(cfg *Config, response *responseDeduper, rootDirs func(
 	return modifiedPkgs, needPkgs, err
 }
 
+// resolveImport finds the the ID of a package given its import path.
+// In particular, it will find the right vendored copy when in GOPATH mode.
+func (state *golistState) resolveImport(sourceDir, importPath string) (string, error) {
+	env, err := state.getEnv()
+	if err != nil {
+		return "", err
+	}
+	if env["GOMOD"] != "" {
+		return importPath, nil
+	}
+
+	searchDir := sourceDir
+	for {
+		vendorDir := filepath.Join(searchDir, "vendor")
+		exists, ok := state.vendorDirs[vendorDir]
+		if !ok {
+			info, err := os.Stat(vendorDir)
+			exists = err == nil && info.IsDir()
+			state.vendorDirs[vendorDir] = exists
+		}
+
+		if exists {
+			vendoredPath := filepath.Join(vendorDir, importPath)
+			if info, err := os.Stat(vendoredPath); err == nil && info.IsDir() {
+				// We should probably check for .go files here, but shame on anyone who fools us.
+				path, ok, err := state.getPkgPath(vendoredPath)
+				if err != nil {
+					return "", err
+				}
+				if ok {
+					return path, nil
+				}
+			}
+		}
+
+		// We know we've hit the top of the filesystem when we Dir / and get /,
+		// or C:\ and get C:\, etc.
+		next := filepath.Dir(searchDir)
+		if next == searchDir {
+			break
+		}
+		searchDir = next
+	}
+	return importPath, nil
+}
+
 func hasTestFiles(p *Package) bool {
 	for _, f := range p.GoFiles {
 		if strings.HasSuffix(f, "_test.go") {
@@ -211,44 +295,69 @@ func hasTestFiles(p *Package) bool {
 	return false
 }
 
-// determineRootDirs returns a mapping from directories code can be contained in to the
-// corresponding import path prefixes of those directories.
-// Its result is used to try to determine the import path for a package containing
-// an overlay file.
-func determineRootDirs(cfg *Config) map[string]string {
-	// Assume modules first:
-	out, err := invokeGo(cfg, "list", "-m", "-json", "all")
+// determineRootDirs returns a mapping from absolute directories that could
+// contain code to their corresponding import path prefixes.
+func (state *golistState) determineRootDirs() (map[string]string, error) {
+	env, err := state.getEnv()
 	if err != nil {
-		return determineRootDirsGOPATH(cfg)
+		return nil, err
+	}
+	if env["GOMOD"] != "" {
+		state.rootsOnce.Do(func() {
+			state.rootDirs, state.rootDirsError = state.determineRootDirsModules()
+		})
+	} else {
+		state.rootsOnce.Do(func() {
+			state.rootDirs, state.rootDirsError = state.determineRootDirsGOPATH()
+		})
+	}
+	return state.rootDirs, state.rootDirsError
+}
+
+func (state *golistState) determineRootDirsModules() (map[string]string, error) {
+	// This will only return the root directory for the main module.
+	// For now we only support overlays in main modules.
+	// Editing files in the module cache isn't a great idea, so we don't
+	// plan to ever support that, but editing files in replaced modules
+	// is something we may want to support. To do that, we'll want to
+	// do a go list -m to determine the replaced module's module path and
+	// directory, and then a go list -m {{with .Replace}}{{.Dir}}{{end}} <replaced module's path>
+	// from the main module to determine if that module is actually a replacement.
+	// See bcmills's comment here: https://github.com/golang/go/issues/37629#issuecomment-594179751
+	// for more information.
+	out, err := state.invokeGo("list", "-m", "-json")
+	if err != nil {
+		return nil, err
 	}
 	m := map[string]string{}
 	type jsonMod struct{ Path, Dir string }
 	for dec := json.NewDecoder(out); dec.More(); {
 		mod := new(jsonMod)
 		if err := dec.Decode(mod); err != nil {
-			return m // Give up and return an empty map. Package won't be found for overlay.
+			return nil, err
 		}
 		if mod.Dir != "" && mod.Path != "" {
 			// This is a valid module; add it to the map.
-			m[mod.Dir] = mod.Path
+			absDir, err := filepath.Abs(mod.Dir)
+			if err != nil {
+				return nil, err
+			}
+			m[absDir] = mod.Path
 		}
 	}
-	return m
+	return m, nil
 }
 
-func determineRootDirsGOPATH(cfg *Config) map[string]string {
+func (state *golistState) determineRootDirsGOPATH() (map[string]string, error) {
 	m := map[string]string{}
-	out, err := invokeGo(cfg, "env", "GOPATH")
-	if err != nil {
-		// Could not determine root dir mapping. Everything is best-effort, so just return an empty map.
-		// When we try to find the import path for a directory, there will be no root-dir match and
-		// we'll give up.
-		return m
+	for _, dir := range filepath.SplitList(state.mustGetEnv()["GOPATH"]) {
+		absDir, err := filepath.Abs(dir)
+		if err != nil {
+			return nil, err
+		}
+		m[filepath.Join(absDir, "src")] = ""
 	}
-	for _, p := range filepath.SplitList(string(bytes.TrimSpace(out.Bytes()))) {
-		m[filepath.Join(p, "src")] = ""
-	}
-	return m
+	return m, nil
 }
 
 func extractImports(filename string, contents []byte) ([]string, error) {
@@ -308,3 +417,57 @@ func extractPackageName(filename string, contents []byte) (string, bool) {
 	}
 	return f.Name.Name, true
 }
+
+func commonDir(a []string) string {
+	seen := make(map[string]bool)
+	x := append([]string{}, a...)
+	for _, f := range x {
+		seen[filepath.Dir(f)] = true
+	}
+	if len(seen) > 1 {
+		log.Fatalf("commonDir saw %v for %v", seen, x)
+	}
+	for k := range seen {
+		// len(seen) == 1
+		return k
+	}
+	return "" // no files
+}
+
+// It is possible that the files in the disk directory dir have a different package
+// name from newName, which is deduced from the overlays. If they all have a different
+// package name, and they all have the same package name, then that name becomes
+// the package name.
+// It returns true if it changes the package name, false otherwise.
+func maybeFixPackageName(newName string, isTestFile bool, pkgsOfDir []*Package) {
+	names := make(map[string]int)
+	for _, p := range pkgsOfDir {
+		names[p.Name]++
+	}
+	if len(names) != 1 {
+		// some files are in different packages
+		return
+	}
+	var oldName string
+	for k := range names {
+		oldName = k
+	}
+	if newName == oldName {
+		return
+	}
+	// We might have a case where all of the package names in the directory are
+	// the same, but the overlay file is for an x test, which belongs to its
+	// own package. If the x test does not yet exist on disk, we may not yet
+	// have its package name on disk, but we should not rename the packages.
+	//
+	// We use a heuristic to determine if this file belongs to an x test:
+	// The test file should have a package name whose package name has a _test
+	// suffix or looks like "newName_test".
+	maybeXTest := strings.HasPrefix(oldName+"_test", newName) || strings.HasSuffix(newName, "_test")
+	if isTestFile && maybeXTest {
+		return
+	}
+	for _, p := range pkgsOfDir {
+		p.Name = newName
+	}
+}

vendor/golang.org/x/tools/go/packages/loadmode_string.go

@@ -0,0 +1,57 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packages
+
+import (
+	"fmt"
+	"strings"
+)
+
+var allModes = []LoadMode{
+	NeedName,
+	NeedFiles,
+	NeedCompiledGoFiles,
+	NeedImports,
+	NeedDeps,
+	NeedExportsFile,
+	NeedTypes,
+	NeedSyntax,
+	NeedTypesInfo,
+	NeedTypesSizes,
+}
+
+var modeStrings = []string{
+	"NeedName",
+	"NeedFiles",
+	"NeedCompiledGoFiles",
+	"NeedImports",
+	"NeedDeps",
+	"NeedExportsFile",
+	"NeedTypes",
+	"NeedSyntax",
+	"NeedTypesInfo",
+	"NeedTypesSizes",
+}
+
+func (mod LoadMode) String() string {
+	m := mod
+	if m == 0 {
+		return "LoadMode(0)"
+	}
+	var out []string
+	for i, x := range allModes {
+		if x > m {
+			break
+		}
+		if (m & x) != 0 {
+			out = append(out, modeStrings[i])
+			m = m ^ x
+		}
+	}
+	if m != 0 {
+		out = append(out, "Unknown")
+	}
+	return fmt.Sprintf("LoadMode(%s)", strings.Join(out, "|"))
+}

vendor/golang.org/x/tools/go/packages/packages.go

@@ -21,8 +21,12 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"time"
 
 	"golang.org/x/tools/go/gcexportdata"
+	"golang.org/x/tools/internal/gocommand"
+	"golang.org/x/tools/internal/packagesinternal"
+	"golang.org/x/tools/internal/typesinternal"
 )
 
 // A LoadMode controls the amount of detail to return when loading.
@@ -34,6 +38,9 @@ import (
 // Load may return more information than requested.
 type LoadMode int
 
+// TODO(matloob): When a V2 of go/packages is released, rename NeedExportsFile to
+// NeedExportFile to make it consistent with the Package field it's adding.
+
 const (
 	// NeedName adds Name and PkgPath.
 	NeedName LoadMode = 1 << iota
@@ -51,7 +58,7 @@ const (
 	// NeedDeps adds the fields requested by the LoadMode in the packages in Imports.
 	NeedDeps
 
-	// NeedExportsFile adds ExportsFile.
+	// NeedExportsFile adds ExportFile.
 	NeedExportsFile
 
 	// NeedTypes adds Types, Fset, and IllTyped.
@@ -65,6 +72,13 @@ const (
 
 	// NeedTypesSizes adds TypesSizes.
 	NeedTypesSizes
+
+	// typecheckCgo enables full support for type checking cgo. Requires Go 1.15+.
+	// Modifies CompiledGoFiles and Types, and has no effect on its own.
+	typecheckCgo
+
+	// NeedModule adds Module.
+	NeedModule
 )
 
 const (
@@ -123,6 +137,9 @@ type Config struct {
 	//
 	Env []string
 
+	// gocmdRunner guards go command calls from concurrency errors.
+	gocmdRunner *gocommand.Runner
+
 	// BuildFlags is a list of command-line flags to be passed through to
 	// the build system's query tool.
 	BuildFlags []string
@@ -160,7 +177,7 @@ type Config struct {
 	Tests bool
 
 	// Overlay provides a mapping of absolute file paths to file contents.
-	// If the file  with the given path already exists, the parser will use the
+	// If the file with the given path already exists, the parser will use the
 	// alternative file contents provided by the map.
 	//
 	// Overlays provide incomplete support for when a given file doesn't
@@ -174,6 +191,13 @@ type driver func(cfg *Config, patterns ...string) (*driverResponse, error)
 
 // driverResponse contains the results for a driver query.
 type driverResponse struct {
+	// NotHandled is returned if the request can't be handled by the current
+	// driver. If an external driver returns a response with NotHandled, the
+	// rest of the driverResponse is ignored, and go/packages will fallback
+	// to the next driver. If go/packages is extended in the future to support
+	// lists of multiple drivers, go/packages will fall back to the next driver.
+	NotHandled bool
+
 	// Sizes, if not nil, is the types.Sizes to use when type checking.
 	Sizes *types.StdSizes
 
@@ -215,14 +239,22 @@ func Load(cfg *Config, patterns ...string) ([]*Package, error) {
 	return l.refine(response.Roots, response.Packages...)
 }
 
-// defaultDriver is a driver that looks for an external driver binary, and if
-// it does not find it falls back to the built in go list driver.
+// defaultDriver is a driver that implements go/packages' fallback behavior.
+// It will try to request to an external driver, if one exists. If there's
+// no external driver, or the driver returns a response with NotHandled set,
+// defaultDriver will fall back to the go list driver.
 func defaultDriver(cfg *Config, patterns ...string) (*driverResponse, error) {
 	driver := findExternalDriver(cfg)
 	if driver == nil {
 		driver = goListDriver
 	}
-	return driver(cfg, patterns...)
+	response, err := driver(cfg, patterns...)
+	if err != nil {
+		return response, err
+	} else if response.NotHandled {
+		return goListDriver(cfg, patterns...)
+	}
+	return response, nil
 }
 
 // A Package describes a loaded Go package.
@@ -249,7 +281,7 @@ type Package struct {
 	GoFiles []string
 
 	// CompiledGoFiles lists the absolute file paths of the package's source
-	// files that were presented to the compiler.
+	// files that are suitable for type checking.
 	// This may differ from GoFiles if files are processed before compilation.
 	CompiledGoFiles []string
 
@@ -292,6 +324,44 @@ type Package struct {
 
 	// TypesSizes provides the effective size function for types in TypesInfo.
 	TypesSizes types.Sizes
+
+	// forTest is the package under test, if any.
+	forTest string
+
+	// module is the module information for the package if it exists.
+	Module *Module
+}
+
+// Module provides module information for a package.
+type Module struct {
+	Path      string       // module path
+	Version   string       // module version
+	Replace   *Module      // replaced by this module
+	Time      *time.Time   // time version was created
+	Main      bool         // is this the main module?
+	Indirect  bool         // is this module only an indirect dependency of main module?
+	Dir       string       // directory holding files for this module, if any
+	GoMod     string       // path to go.mod file used when loading this module, if any
+	GoVersion string       // go version used in module
+	Error     *ModuleError // error loading module
+}
+
+// ModuleError holds errors loading a module.
+type ModuleError struct {
+	Err string // the error itself
+}
+
+func init() {
+	packagesinternal.GetForTest = func(p interface{}) string {
+		return p.(*Package).forTest
+	}
+	packagesinternal.GetGoCmdRunner = func(config interface{}) *gocommand.Runner {
+		return config.(*Config).gocmdRunner
+	}
+	packagesinternal.SetGoCmdRunner = func(config interface{}, runner *gocommand.Runner) {
+		config.(*Config).gocmdRunner = runner
+	}
+	packagesinternal.TypecheckCgo = int(typecheckCgo)
 }
 
 // An Error describes a problem with a package's metadata, syntax, or types.
@@ -454,6 +524,9 @@ func newLoader(cfg *Config) *loader {
 	if ld.Config.Env == nil {
 		ld.Config.Env = os.Environ()
 	}
+	if ld.Config.gocmdRunner == nil {
+		ld.Config.gocmdRunner = &gocommand.Runner{}
+	}
 	if ld.Context == nil {
 		ld.Context = context.Background()
 	}
@@ -467,7 +540,7 @@ func newLoader(cfg *Config) *loader {
 	ld.requestedMode = ld.Mode
 	ld.Mode = impliedLoadMode(ld.Mode)
 
-	if ld.Mode&NeedTypes != 0 {
+	if ld.Mode&NeedTypes != 0 || ld.Mode&NeedSyntax != 0 {
 		if ld.Fset == nil {
 			ld.Fset = token.NewFileSet()
 		}
@@ -500,12 +573,23 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
 		if i, found := rootMap[pkg.ID]; found {
 			rootIndex = i
 		}
+
+		// Overlays can invalidate export data.
+		// TODO(matloob): make this check fine-grained based on dependencies on overlaid files
+		exportDataInvalid := len(ld.Overlay) > 0 || pkg.ExportFile == "" && pkg.PkgPath != "unsafe"
+		// This package needs type information if the caller requested types and the package is
+		// either a root, or it's a non-root and the user requested dependencies ...
+		needtypes := (ld.Mode&NeedTypes|NeedTypesInfo != 0 && (rootIndex >= 0 || ld.Mode&NeedDeps != 0))
+		// This package needs source if the call requested source (or types info, which implies source)
+		// and the package is either a root, or itas a non- root and the user requested dependencies...
+		needsrc := ((ld.Mode&(NeedSyntax|NeedTypesInfo) != 0 && (rootIndex >= 0 || ld.Mode&NeedDeps != 0)) ||
+			// ... or if we need types and the exportData is invalid. We fall back to (incompletely)
+			// typechecking packages from source if they fail to compile.
+			(ld.Mode&NeedTypes|NeedTypesInfo != 0 && exportDataInvalid)) && pkg.PkgPath != "unsafe"
 		lpkg := &loaderPackage{
 			Package:   pkg,
-			needtypes: (ld.Mode&(NeedTypes|NeedTypesInfo) != 0 && ld.Mode&NeedDeps != 0 && rootIndex < 0) || rootIndex >= 0,
-			needsrc: (ld.Mode&(NeedSyntax|NeedTypesInfo) != 0 && ld.Mode&NeedDeps != 0 && rootIndex < 0) || rootIndex >= 0 ||
-				len(ld.Overlay) > 0 || // Overlays can invalidate export data. TODO(matloob): make this check fine-grained based on dependencies on overlaid files
-				pkg.ExportFile == "" && pkg.PkgPath != "unsafe",
+			needtypes: needtypes,
+			needsrc:   needsrc,
 		}
 		ld.pkgs[lpkg.ID] = lpkg
 		if rootIndex >= 0 {
@@ -609,9 +693,9 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
 			}
 		}
 	}
-	// Load type data if needed, starting at
+	// Load type data and syntax if needed, starting at
 	// the initial packages (roots of the import DAG).
-	if ld.Mode&NeedTypes != 0 {
+	if ld.Mode&NeedTypes != 0 || ld.Mode&NeedSyntax != 0 {
 		var wg sync.WaitGroup
 		for _, lpkg := range initial {
 			wg.Add(1)
@@ -660,6 +744,9 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
 		if ld.requestedMode&NeedTypesSizes == 0 {
 			ld.pkgs[i].TypesSizes = nil
 		}
+		if ld.requestedMode&NeedModule == 0 {
+			ld.pkgs[i].Module = nil
+		}
 	}
 
 	return result, nil
@@ -713,7 +800,7 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) {
 	// which would then require that such created packages be explicitly
 	// inserted back into the Import graph as a final step after export data loading.
 	// The Diamond test exercises this case.
-	if !lpkg.needtypes {
+	if !lpkg.needtypes && !lpkg.needsrc {
 		return
 	}
 	if !lpkg.needsrc {
@@ -770,7 +857,7 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) {
 		lpkg.Errors = append(lpkg.Errors, errs...)
 	}
 
-	if len(lpkg.CompiledGoFiles) == 0 && lpkg.ExportFile != "" {
+	if ld.Config.Mode&NeedTypes != 0 && len(lpkg.CompiledGoFiles) == 0 && lpkg.ExportFile != "" {
 		// The config requested loading sources and types, but sources are missing.
 		// Add an error to the package and fall back to loading from export data.
 		appendError(Error{"-", fmt.Sprintf("sources missing for package %s", lpkg.ID), ParseError})
@@ -784,6 +871,9 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) {
 	}
 
 	lpkg.Syntax = files
+	if ld.Config.Mode&NeedTypes == 0 {
+		return
+	}
 
 	lpkg.TypesInfo = &types.Info{
 		Types:      make(map[ast.Expr]types.TypeAndValue),
@@ -832,6 +922,15 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) {
 		Error: appendError,
 		Sizes: ld.sizes,
 	}
+	if (ld.Mode & typecheckCgo) != 0 {
+		if !typesinternal.SetUsesCgo(tc) {
+			appendError(Error{
+				Msg:  "typecheckCgo requires Go 1.15+",
+				Kind: ListError,
+			})
+			return
+		}
+	}
 	types.NewChecker(tc, ld.Fset, lpkg.Types, lpkg.TypesInfo).Files(lpkg.Syntax)
 
 	lpkg.importErrors = nil // no longer needed