mirror of https://github.com/knative/func.git
Enable arm64 python buildpack (#2789)
* Enable Python buildpack build for arm64 Signed-off-by: Matej Vašek <mvasek@redhat.com> * Fix: force arm64 arch for Go buildpack Signed-off-by: Matej Vašek <mvasek@redhat.com> * Fix: ensure correct arch for sub-buildpacks Signed-off-by: Matej Vašek <mvasek@redhat.com> * Fixup linter issues Signed-off-by: Matej Vašek <mvasek@redhat.com> --------- Signed-off-by: Matej Vašek <mvasek@redhat.com>
This commit is contained in:
Matej Vašek
GitHub
parent
8e0b752edc
commit
33b9f15654
|
|
@ -16,9 +16,11 @@ import (
|
|||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
"golang.org/x/oauth2"
|
||||
"golang.org/x/term"
|
||||
|
||||
|
|
@ -106,6 +108,19 @@ func buildBuilderImage(ctx context.Context, variant, version, arch, builderTomlP
|
|||
if err != nil {
|
||||
return "", fmt.Errorf("cannot fix Go buildpack: %w", err)
|
||||
}
|
||||
if variant == "base" {
|
||||
err = fixupPythonBuildpackARM64(ctx, &builderConfig)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot fix Python buildpack: %w", err)
|
||||
}
|
||||
}
|
||||
// Sort by URI. This ensures locally build buildpacks (URI starting with 'file://') are last.
|
||||
// This is needed so locally build "sub buildpacks" are not overridden by upstream buildpacks (with bad arch).
|
||||
sort.Slice(builderConfig.Buildpacks, func(i, j int) bool {
|
||||
a := builderConfig.Buildpacks[i].URI
|
||||
b := builderConfig.Buildpacks[j].URI
|
||||
return a < b
|
||||
})
|
||||
}
|
||||
|
||||
err = updateJavaBuildpacks(ctx, &builderConfig, arch)
|
||||
|
|
@ -279,7 +294,7 @@ func buildBuilderImageMultiArch(ctx context.Context, variant string) error {
|
|||
"org.opencontainers.image.version": *release.Name,
|
||||
}).(v1.ImageIndex)
|
||||
for _, arch := range []string{"arm64", "amd64"} {
|
||||
if arch == "arm64" && variant != "tiny" {
|
||||
if arch == "arm64" && variant == "full" {
|
||||
_, _ = fmt.Fprintf(os.Stderr, "skipping arm64 build for variant: %q\n", variant)
|
||||
continue
|
||||
}
|
||||
|
|
@ -726,7 +741,10 @@ func downloadTarball(tarballUrl, destDir string) error {
|
|||
return fmt.Errorf("cannot read from tar reader: %w", err)
|
||||
}
|
||||
case tar.TypeSymlink:
|
||||
return fmt.Errorf("symlinks are not supported yet")
|
||||
err = os.Symlink(hdr.Linkname, dest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot create a symlink: %w", err)
|
||||
}
|
||||
case tar.TypeDir:
|
||||
err = os.MkdirAll(dest, 0755)
|
||||
if err != nil {
|
||||
|
|
@ -847,6 +865,18 @@ func fixupGoBuildpackARM64(ctx context.Context, config *builder.Config) error {
|
|||
return fmt.Errorf("cannot download Go buildpack source code: %w", err)
|
||||
}
|
||||
|
||||
// Set targets in Go package.toml to linux/arm64
|
||||
f, err := os.OpenFile(filepath.Join(goBuildpackSrcDir, "package.toml"), os.O_WRONLY|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot open package toml: %w", err)
|
||||
}
|
||||
targets := "\n[[targets]]\n arch = \"arm64\"\n os = \"linux\"\n"
|
||||
_, err = f.Write([]byte(targets))
|
||||
_ = f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannout update package toml: %w", err)
|
||||
}
|
||||
|
||||
cfgReader := buildpackage.NewConfigReader()
|
||||
packageConfig, err := cfgReader.Read(filepath.Join(goBuildpackSrcDir, "package.toml"))
|
||||
if err != nil {
|
||||
|
|
@ -1186,3 +1216,291 @@ func patchStack(stackTomlPath string) error {
|
|||
|
||||
return nil
|
||||
}
|
||||
|
||||
func fixupPythonBuildpackARM64(ctx context.Context, config *builder.Config) error {
|
||||
var (
|
||||
pythonBuildpackIndex int
|
||||
pythonBuildpackVersion string
|
||||
)
|
||||
for i, moduleConfig := range config.Buildpacks {
|
||||
uri := moduleConfig.URI
|
||||
if strings.Contains(uri, "buildpacks/python:") {
|
||||
pythonBuildpackIndex = i
|
||||
pythonBuildpackVersion = uri[strings.LastIndex(uri, ":")+1:]
|
||||
break
|
||||
}
|
||||
}
|
||||
if pythonBuildpackVersion == "" {
|
||||
return fmt.Errorf("python buildpack not found in the config")
|
||||
}
|
||||
|
||||
buildDir, err := os.MkdirTemp("", "build-dir-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot create temp dir: %w", err)
|
||||
}
|
||||
// sic! do not defer remove
|
||||
|
||||
pythonBuildpackSrcDir := filepath.Join(buildDir, "python")
|
||||
|
||||
pythonBuildpackRelease, err := getReleaseByVersion(ctx, "python", pythonBuildpackVersion)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot get Python release: %w", err)
|
||||
}
|
||||
|
||||
err = downloadTarball(*pythonBuildpackRelease.TarballURL, pythonBuildpackSrcDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot download Python buildpack source code: %w", err)
|
||||
}
|
||||
|
||||
// Set targets in Python package.toml to linux/arm64
|
||||
f, err := os.OpenFile(filepath.Join(pythonBuildpackSrcDir, "package.toml"), os.O_WRONLY|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot open package toml: %w", err)
|
||||
}
|
||||
targets := "\n[[targets]]\n arch = \"arm64\"\n os = \"linux\"\n"
|
||||
_, err = f.Write([]byte(targets))
|
||||
_ = f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannout update package toml: %w", err)
|
||||
}
|
||||
|
||||
cfgReader := buildpackage.NewConfigReader()
|
||||
packageConfig, err := cfgReader.Read(filepath.Join(pythonBuildpackSrcDir, "package.toml"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot read Python buildpack config: %w", err)
|
||||
}
|
||||
|
||||
buildBuildpack := func(name, version string) error {
|
||||
srcDir := filepath.Join(buildDir, name)
|
||||
cmd := exec.CommandContext(ctx, "./scripts/package.sh", "--version", version)
|
||||
cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
|
||||
cmd.Dir = srcDir
|
||||
cmd.Env = append(os.Environ(), "GOARCH=arm64")
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
return fmt.Errorf("build of buildpack %q failed: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type patchSourceFn = func(srcDir string) error
|
||||
// these buildpacks need rebuild since they are only amd64 in paketo upstream
|
||||
needsRebuild := map[string]patchSourceFn{
|
||||
"cpython": func(srcDir string) error {
|
||||
return fixupCPythonDistPkgRefs(filepath.Join(srcDir, "buildpack.toml"))
|
||||
},
|
||||
"miniconda": func(srcDir string) error {
|
||||
return fixupMinicondaDistPkgRefs(filepath.Join(srcDir, "buildpack.toml"))
|
||||
},
|
||||
"conda-env-update": nil,
|
||||
"pip": nil,
|
||||
"pip-install": nil,
|
||||
"pipenv": nil,
|
||||
"pipenv-install": nil,
|
||||
"poetry": nil,
|
||||
"poetry-install": nil,
|
||||
"poetry-run": nil,
|
||||
"python": nil,
|
||||
"python-start": nil,
|
||||
}
|
||||
|
||||
re := regexp.MustCompile(`^urn:cnb:registry:paketo-buildpacks/([\w-]+)@([\d.]+)$`)
|
||||
for i, dep := range packageConfig.Dependencies {
|
||||
m := re.FindStringSubmatch(dep.URI)
|
||||
if len(m) != 3 {
|
||||
return fmt.Errorf("cannot match buildpack name")
|
||||
}
|
||||
buildpackName := m[1]
|
||||
buildpackVersion := m[2]
|
||||
|
||||
patch, ok := needsRebuild[buildpackName]
|
||||
if !ok {
|
||||
// this dependency does not require rebuild for arm64
|
||||
continue
|
||||
}
|
||||
|
||||
var rel *github.RepositoryRelease
|
||||
rel, err = getReleaseByVersion(ctx, buildpackName, buildpackVersion)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot get release: %w", err)
|
||||
}
|
||||
|
||||
srcDir := filepath.Join(buildDir, buildpackName)
|
||||
|
||||
err = downloadTarball(*rel.TarballURL, srcDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot get tarball: %w", err)
|
||||
}
|
||||
if patch != nil {
|
||||
err = patch(srcDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot patch source code: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
err = buildBuildpack(buildpackName, buildpackVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
packageConfig.Dependencies[i].URI = "file://" + filepath.Join(srcDir, "build", "buildpackage.cnb")
|
||||
|
||||
}
|
||||
|
||||
bs, err := toml.Marshal(&packageConfig)
|
||||
err = os.WriteFile(filepath.Join(pythonBuildpackSrcDir, "package.toml"), bs, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot update package.toml: %w", err)
|
||||
}
|
||||
|
||||
err = buildBuildpack("python", pythonBuildpackVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
config.Buildpacks[pythonBuildpackIndex].URI = "file://" + filepath.Join(pythonBuildpackSrcDir, "build", "buildpackage.cnb")
|
||||
fmt.Println(pythonBuildpackSrcDir)
|
||||
return nil
|
||||
}
|
||||
|
||||
func fixupCPythonDistPkgRefs(buildpackToml string) error {
|
||||
tomlBytes, err := os.ReadFile(buildpackToml)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var config any
|
||||
err = toml.Unmarshal(tomlBytes, &config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
deps := config.(map[string]any)["metadata"].(map[string]any)["dependencies"].([]map[string]any)
|
||||
|
||||
// Since there are no cpython arm64 packages we set uri to source.
|
||||
// This will cause cpython compilation to be done during the build process.
|
||||
// This takes a while, but it's done only once per project.
|
||||
for _, dep := range deps {
|
||||
dep["checksum"] = dep["source-checksum"]
|
||||
dep["uri"] = dep["source"]
|
||||
}
|
||||
|
||||
bs, err := toml.Marshal(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.WriteFile(buildpackToml, bs, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func fixupMinicondaDistPkgRefs(buildpackToml string) error {
|
||||
tomlBytes, err := os.ReadFile(buildpackToml)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var config any
|
||||
err = toml.Unmarshal(tomlBytes, &config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
deps := config.(map[string]any)["metadata"].(map[string]any)["dependencies"].([]map[string]any)
|
||||
for _, dep := range deps {
|
||||
if dep["name"] == "Miniconda.sh" {
|
||||
newURI := strings.ReplaceAll(dep["uri"].(string), "x86_64", "aarch64")
|
||||
dep["uri"] = newURI
|
||||
|
||||
parts := strings.Split(newURI, "/")
|
||||
basename := parts[len(parts)-1]
|
||||
var sum string
|
||||
sum, err = getMinicondaHash(basename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot find hash for the dependency: %w", err)
|
||||
}
|
||||
dep["sha256"] = sum
|
||||
}
|
||||
}
|
||||
|
||||
bs, err := toml.Marshal(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.WriteFile(buildpackToml, bs, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getMinicondaHash(basename string) (string, error) {
|
||||
//nolint:bodyclose
|
||||
resp, err := http.Get("https://repo.anaconda.com/miniconda/")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func(Body io.ReadCloser) {
|
||||
_ = Body.Close()
|
||||
}(resp.Body)
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return "", fmt.Errorf("bad http code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
n, err := html.Parse(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
tbody := findNode(n, func(n *html.Node) bool {
|
||||
return n.Data == "tbody"
|
||||
})
|
||||
if tbody == nil {
|
||||
return "", fmt.Errorf("table body not found")
|
||||
}
|
||||
|
||||
for tr := range tbody.ChildNodes() {
|
||||
f, l := firstLastTD(tr)
|
||||
if f != l &&
|
||||
f != nil && f.FirstChild != nil && f.FirstChild.FirstChild != nil &&
|
||||
l != nil && l.FirstChild != nil {
|
||||
if f.FirstChild.FirstChild.Data == basename {
|
||||
return l.FirstChild.Data, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("hash not found")
|
||||
}
|
||||
|
||||
func findNode(node *html.Node, pred func(*html.Node) bool) *html.Node {
|
||||
if pred(node) {
|
||||
return node
|
||||
}
|
||||
for ch := range node.ChildNodes() {
|
||||
n := findNode(ch, pred)
|
||||
if n != nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func firstLastTD(tr *html.Node) (*html.Node, *html.Node) {
|
||||
var f, l *html.Node
|
||||
for ch := range tr.ChildNodes() {
|
||||
if ch.Data == "td" {
|
||||
if f == nil {
|
||||
f = ch
|
||||
}
|
||||
l = ch
|
||||
}
|
||||
}
|
||||
return f, l
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue