package goreadme import ( "context" "net/http" "path/filepath" "sort" "sync" "github.com/golang/gddo/doc" "github.com/hashicorp/go-multierror" "github.com/pkg/errors" ) // skipDirs contain directories that should not be scanned var skipDirs = map[string]bool{ "testdata": true, "internal": true, } // subpackagesFetcher fetches sub packages recursively. type subpackagesFetcher struct { client *http.Client importPath string recursive bool wg sync.WaitGroup mu sync.Mutex errors *multierror.Error packages []subPkg } func (f *subpackagesFetcher) Fetch(ctx context.Context, pkg *doc.Package) ([]subPkg, error) { for _, subDir := range pkg.Subdirectories { f.fetch(ctx, subDir) } f.wg.Wait() sort.Slice(f.packages, func(i, j int) bool { return f.packages[i].Path < f.packages[j].Path }) return f.packages, f.errors.ErrorOrNil() } // Concurrently fetches information for all sub directories. func (f *subpackagesFetcher) fetch(ctx context.Context, subDir string) { if skipDirs[filepath.Base(subDir)] { return } f.wg.Add(1) importPath := f.importPath + "/" + subDir go func() { defer f.wg.Done() sp, err := docGet(ctx, f.client, importPath, "") f.mu.Lock() defer f.mu.Unlock() if err != nil { f.errors = multierror.Append(f.errors, errors.Wrapf(err, "failed getting %s", importPath)) return } // Append to packages only if this directory is a go package. if sp.Name != "" { f.packages = append(f.packages, subPkg{Path: subDir, Package: sp}) } if f.recursive { for _, sd := range sp.Subdirectories { f.fetch(ctx, subDir+"/"+sd) } } }() }