source-controller/controllers/storage.go

/*
Copyright 2020 The Flux CD contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"crypto/sha1"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/go-git/go-git/v5/plumbing/format/gitignore"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/fluxcd/pkg/lockedfile"
	sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1"
)

const (
	excludeFile = ".sourceignore"
	excludeVCS  = ".git/,.gitignore,.gitmodules,.gitattributes"
	excludeExt  = "*.jpg,*.jpeg,*.gif,*.png,*.wmv,.*flv,.*tar.gz,*.zip"
)

// Storage manages artifacts
type Storage struct {
	// BasePath is the local directory path where the source artifacts are stored.
	BasePath string `json:"basePath"`

	// Hostname is the file server host name used to compose the artifacts URIs.
	Hostname string `json:"hostname"`

	// Timeout for artifacts operations
	Timeout time.Duration `json:"timeout"`
}

// NewStorage creates the storage helper for a given path and hostname
func NewStorage(basePath string, hostname string, timeout time.Duration) (*Storage, error) {
	if f, err := os.Stat(basePath); os.IsNotExist(err) || !f.IsDir() {
		return nil, fmt.Errorf("invalid dir path: %s", basePath)
	}

	return &Storage{
		BasePath: basePath,
		Hostname: hostname,
		Timeout:  timeout,
	}, nil
}

// ArtifactFor returns an artifact for the given Kubernetes object
func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, revision string) sourcev1.Artifact {
	kind = strings.ToLower(kind)
	path := sourcev1.ArtifactPath(kind, metadata.GetNamespace(), metadata.GetName(), fileName)
	localPath := filepath.Join(s.BasePath, path)
	url := fmt.Sprintf("http://%s/%s", s.Hostname, path)

	return sourcev1.Artifact{
		Path:           localPath,
		URL:            url,
		Revision:       revision,
		LastUpdateTime: metav1.Now(),
	}
}

// MkdirAll calls os.MkdirAll for the given artifact base dir
func (s *Storage) MkdirAll(artifact sourcev1.Artifact) error {
	dir := filepath.Dir(artifact.Path)
	return os.MkdirAll(dir, 0777)
}

// RemoveAll calls os.RemoveAll for the given artifact base dir
func (s *Storage) RemoveAll(artifact sourcev1.Artifact) error {
	dir := filepath.Dir(artifact.Path)
	return os.RemoveAll(dir)
}

// RemoveAllButCurrent removes all files for the given artifact base dir excluding the current one
func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error {
	dir := filepath.Dir(artifact.Path)
	var errors []string
	_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if path != artifact.Path && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink {
			if err := os.Remove(path); err != nil {
				errors = append(errors, info.Name())
			}
		}
		return nil
	})

	if len(errors) > 0 {
		return fmt.Errorf("faild to remove files: %s", strings.Join(errors, " "))
	}
	return nil
}

// ArtifactExist returns a boolean indicating whether the artifact file exists in storage
func (s *Storage) ArtifactExist(artifact sourcev1.Artifact) bool {
	if _, err := os.Stat(artifact.Path); os.IsNotExist(err) {
		return false
	}
	return true
}

// Archive creates a tar.gz to the artifact path from the given dir excluding any VCS specific
// files and directories, or any of the excludes defined in the excludeFiles.
func (s *Storage) Archive(artifact sourcev1.Artifact, dir string) error {
	if _, err := os.Stat(dir); err != nil {
		return err
	}

	ps, err := loadExcludePatterns(dir)
	if err != nil {
		return err
	}
	matcher := gitignore.NewMatcher(ps)

	gzFile, err := os.Create(artifact.Path)
	if err != nil {
		return err
	}
	defer gzFile.Close()

	gw := gzip.NewWriter(gzFile)
	defer gw.Close()

	tw := tar.NewWriter(gw)
	defer tw.Close()

	return filepath.Walk(dir, func(p string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Ignore anything that is not a file (directories, symlinks)
		if !fi.Mode().IsRegular() {
			return nil
		}

		// Ignore excluded extensions and files
		if matcher.Match(strings.Split(p, "/"), false) {
			return nil
		}

		header, err := tar.FileInfoHeader(fi, p)
		if err != nil {
			return err
		}
		// The name needs to be modified to maintain directory structure
		// as tar.FileInfoHeader only has access to the base name of the file.
		// Ref: https://golang.org/src/archive/tar/common.go?#L626
		relFilePath := p
		if filepath.IsAbs(dir) {
			relFilePath, err = filepath.Rel(dir, p)
			if err != nil {
				return err
			}
		}
		header.Name = relFilePath

		if err := tw.WriteHeader(header); err != nil {
			return err
		}

		f, err := os.Open(p)
		if err != nil {
			return err
		}
		if _, err := io.Copy(tw, f); err != nil {
			f.Close()
			return err
		}
		return f.Close()
	})
}

// WriteFile writes the given bytes to the artifact path if the checksum differs
func (s *Storage) WriteFile(artifact sourcev1.Artifact, data []byte) error {
	sum := s.Checksum(data)
	if file, err := os.Stat(artifact.Path); !os.IsNotExist(err) && !file.IsDir() {
		if fb, err := ioutil.ReadFile(artifact.Path); err == nil && sum == s.Checksum(fb) {
			return nil
		}
	}

	return ioutil.WriteFile(artifact.Path, data, 0644)
}

// Symlink creates or updates a symbolic link for the given artifact
// and returns the URL for the symlink
func (s *Storage) Symlink(artifact sourcev1.Artifact, linkName string) (string, error) {
	dir := filepath.Dir(artifact.Path)
	link := filepath.Join(dir, linkName)
	tmpLink := link + ".tmp"

	if err := os.Remove(tmpLink); err != nil && !os.IsNotExist(err) {
		return "", err
	}

	if err := os.Symlink(artifact.Path, tmpLink); err != nil {
		return "", err
	}

	if err := os.Rename(tmpLink, link); err != nil {
		return "", err
	}

	parts := strings.Split(artifact.URL, "/")
	url := strings.Replace(artifact.URL, parts[len(parts)-1], linkName, 1)
	return url, nil
}

// Checksum returns the SHA1 checksum for the given bytes as a string
func (s *Storage) Checksum(b []byte) string {
	return fmt.Sprintf("%x", sha1.Sum(b))
}

// Lock creates a file lock for the given artifact
func (s *Storage) Lock(artifact sourcev1.Artifact) (unlock func(), err error) {
	lockFile := artifact.Path + ".lock"
	mutex := lockedfile.MutexAt(lockFile)
	return mutex.Lock()
}

func loadExcludePatterns(dir string) ([]gitignore.Pattern, error) {
	path := strings.Split(dir, "/")
	var ps []gitignore.Pattern
	for _, p := range strings.Split(excludeVCS, ",") {
		ps = append(ps, gitignore.ParsePattern(p, path))
	}
	for _, p := range strings.Split(excludeExt, ",") {
		ps = append(ps, gitignore.ParsePattern(p, path))
	}
	if f, err := os.Open(filepath.Join(dir, excludeFile)); err == nil {
		defer f.Close()

		scanner := bufio.NewScanner(f)
		for scanner.Scan() {
			s := scanner.Text()
			if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 {
				ps = append(ps, gitignore.ParsePattern(s, path))
			}
		}
	} else if !os.IsNotExist(err) {
		return nil, err
	}
	return ps, nil
}