schnutibox/vendor/github.com/bufbuild/buf/internal/pkg/git/cloner.go
Marvin Steadfast ae35d9ab41
Some checks failed
continuous-integration/drone/push Build is failing
uses buf for compiling proto files and implements the server service
2021-05-05 11:14:17 +02:00

259 lines
7.4 KiB
Go

// Copyright 2020-2021 Buf Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package git
import (
"bytes"
"context"
"errors"
"fmt"
"os/exec"
"strconv"
"strings"
"github.com/bufbuild/buf/internal/pkg/app"
"github.com/bufbuild/buf/internal/pkg/storage"
"github.com/bufbuild/buf/internal/pkg/storage/storageos"
"github.com/bufbuild/buf/internal/pkg/tmp"
"go.opencensus.io/trace"
"go.uber.org/multierr"
"go.uber.org/zap"
)
type cloner struct {
logger *zap.Logger
storageosProvider storageos.Provider
options ClonerOptions
}
func newCloner(
logger *zap.Logger,
storageosProvider storageos.Provider,
options ClonerOptions,
) *cloner {
return &cloner{
logger: logger,
storageosProvider: storageosProvider,
options: options,
}
}
func (c *cloner) CloneToBucket(
ctx context.Context,
envContainer app.EnvContainer,
url string,
depth uint32,
writeBucket storage.WriteBucket,
options CloneToBucketOptions,
) (retErr error) {
ctx, span := trace.StartSpan(ctx, "git_clone_to_bucket")
defer span.End()
var err error
switch {
case strings.HasPrefix(url, "http://"),
strings.HasPrefix(url, "https://"),
strings.HasPrefix(url, "ssh://"),
strings.HasPrefix(url, "git://"),
strings.HasPrefix(url, "file://"):
default:
return fmt.Errorf("invalid git url: %q", url)
}
if depth == 0 {
return errors.New("depth must be > 0")
}
depthArg := strconv.Itoa(int(depth))
args := []string{"clone", "--depth", depthArg}
if options.Name != nil {
if cloneBranch := options.Name.cloneBranch(); cloneBranch != "" {
args = append(args, "--branch", cloneBranch, "--single-branch")
}
}
tmpDir, err := tmp.NewDir()
if err != nil {
return err
}
defer func() {
retErr = multierr.Append(retErr, tmpDir.Close())
}()
args = append(args, url, tmpDir.AbsPath())
if strings.HasPrefix(url, "https://") {
extraArgs, err := c.getArgsForHTTPSCommand(envContainer)
if err != nil {
return err
}
args = append(args, extraArgs...)
}
if strings.HasPrefix(url, "ssh://") {
envContainer, err = c.getEnvContainerWithGitSSHCommand(envContainer)
if err != nil {
return err
}
}
buffer := bytes.NewBuffer(nil)
cmd := exec.CommandContext(ctx, "git", args...)
cmd.Env = app.Environ(envContainer)
cmd.Stderr = buffer
if err := cmd.Run(); err != nil {
// Suppress printing of temp path
return fmt.Errorf("%v\n%v", err, strings.Replace(buffer.String(), tmpDir.AbsPath(), "", -1))
}
if options.Name != nil && options.Name.checkout() != "" {
args = []string{
"checkout",
options.Name.checkout(),
}
buffer.Reset()
cmd = exec.CommandContext(ctx, "git", args...)
cmd.Env = app.Environ(envContainer)
cmd.Dir = tmpDir.AbsPath()
cmd.Stderr = buffer
if err := cmd.Run(); err != nil {
// Suppress printing of temp path
return fmt.Errorf("%v\n%v", err, strings.Replace(buffer.String(), tmpDir.AbsPath(), "", -1))
}
}
if options.RecurseSubmodules {
args = []string{
"submodule",
"update",
"--init",
"--recursive",
"--depth",
depthArg,
}
buffer.Reset()
cmd = exec.CommandContext(ctx, "git", args...)
cmd.Env = app.Environ(envContainer)
cmd.Dir = tmpDir.AbsPath()
cmd.Stderr = buffer
if err := cmd.Run(); err != nil {
// Suppress printing of temp path
return fmt.Errorf("%v\n%v", err, strings.Replace(buffer.String(), tmpDir.AbsPath(), "", -1))
}
}
// we do NOT want to read in symlinks
tmpReadWriteBucket, err := c.storageosProvider.NewReadWriteBucket(tmpDir.AbsPath())
if err != nil {
return err
}
var readBucket storage.ReadBucket = tmpReadWriteBucket
if options.Mapper != nil {
readBucket = storage.MapReadBucket(readBucket, options.Mapper)
}
ctx, span2 := trace.StartSpan(ctx, "git_clone_to_bucket_copy")
defer span2.End()
// do NOT copy external paths
_, err = storage.Copy(ctx, readBucket, writeBucket)
return err
}
func (c *cloner) getArgsForHTTPSCommand(envContainer app.EnvContainer) ([]string, error) {
if c.options.HTTPSUsernameEnvKey == "" || c.options.HTTPSPasswordEnvKey == "" {
return nil, nil
}
httpsUsernameSet := envContainer.Env(c.options.HTTPSUsernameEnvKey) != ""
httpsPasswordSet := envContainer.Env(c.options.HTTPSPasswordEnvKey) != ""
if !httpsUsernameSet {
if httpsPasswordSet {
return nil, fmt.Errorf("%s set but %s not set", c.options.HTTPSPasswordEnvKey, c.options.HTTPSUsernameEnvKey)
}
return nil, nil
}
c.logger.Debug("git_credential_helper_override")
return []string{
"--config",
fmt.Sprintf(
// TODO: is this OK for windows/other platforms?
// we might need an alternate flow where the binary has a sub-command to do this, and calls itself
//
// putting the variable name in this script, NOT the actual variable value
// we do not want to store the variable on disk, ever
// this is especially important if the program dies
// note that this means i.e. HTTPS_PASSWORD=foo invoke_program does not work as
// this variable needs to be in the actual global environment
// TODO this is a mess
"credential.helper=!f(){ echo username=${%s}; echo password=${%s}; };f",
c.options.HTTPSUsernameEnvKey,
c.options.HTTPSPasswordEnvKey,
),
}, nil
}
func (c *cloner) getEnvContainerWithGitSSHCommand(envContainer app.EnvContainer) (app.EnvContainer, error) {
gitSSHCommand, err := c.getGitSSHCommand(envContainer)
if err != nil {
return nil, err
}
if gitSSHCommand != "" {
c.logger.Debug("git_ssh_command_override")
return app.NewEnvContainerWithOverrides(
envContainer,
map[string]string{
"GIT_SSH_COMMAND": gitSSHCommand,
},
), nil
}
return envContainer, nil
}
func (c *cloner) getGitSSHCommand(envContainer app.EnvContainer) (string, error) {
sshKeyFilePath := envContainer.Env(c.options.SSHKeyFileEnvKey)
sshKnownHostsFiles := envContainer.Env(c.options.SSHKnownHostsFilesEnvKey)
if sshKeyFilePath == "" {
if sshKnownHostsFiles != "" {
return "", fmt.Errorf("%s set but %s not set", c.options.SSHKnownHostsFilesEnvKey, c.options.SSHKeyFileEnvKey)
}
return "", nil
}
if sshKnownHostsFilePaths := getSSHKnownHostsFilePaths(sshKnownHostsFiles); len(sshKnownHostsFilePaths) > 0 {
return fmt.Sprintf(
`ssh -q -i "%s" -o "IdentitiesOnly=yes" -o "UserKnownHostsFile=%s"`,
sshKeyFilePath,
strings.Join(sshKnownHostsFilePaths, " "),
), nil
}
// we want to set StrictHostKeyChecking=no because the SSH key file variable was set, so
// there is an ask to override the default ssh settings here
return fmt.Sprintf(
`ssh -q -i "%s" -o "IdentitiesOnly=yes" -o "UserKnownHostsFile=%s" -o "StrictHostKeyChecking=no"`,
sshKeyFilePath,
app.DevNullFilePath,
), nil
}
func getSSHKnownHostsFilePaths(sshKnownHostsFiles string) []string {
if sshKnownHostsFiles == "" {
return nil
}
var filePaths []string
for _, filePath := range strings.Split(sshKnownHostsFiles, ":") {
filePath = strings.TrimSpace(filePath)
if filePath != "" {
filePaths = append(filePaths, filePath)
}
}
return filePaths
}