docker
/
model-runner
mirror of https://github.com/docker/model-runner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enable basic windows/arm64 support 

Signed-off-by: Piotr Stankiewicz <piotr.stankiewicz@docker.com>
This commit is contained in:
Piotr Stankiewicz 2025-04-23 14:00:42 +02:00 committed by Piotr
parent 7545f180cd
commit 978875e99c
2 changed files with 14 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
pkg/inference/backends/llamacpp/download_windows.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"net/http"
"path/filepath"
"runtime"
"github.com/docker/model-runner/pkg/logging"
)
@ -18,10 +19,12 @@ func (l *llamaCpp) ensureLatestLlamaCpp(ctx context.Context, log logging.Logger,
ShouldUseGPUVariantLock.Lock()
defer ShouldUseGPUVariantLock.Unlock()
if ShouldUseGPUVariant {
canUseCUDA11, err = hasCUDA11CapableGPU(ctx, nvGPUInfoBin)
if err != nil {
l.status = fmt.Sprintf("failed to check CUDA 11 capability: %v", err)
return fmt.Errorf("failed to check CUDA 11 capability: %w", err)
if runtime.GOARCH == "amd64" {
canUseCUDA11, err = hasCUDA11CapableGPU(ctx, nvGPUInfoBin)
if err != nil {
l.status = fmt.Sprintf("failed to check CUDA 11 capability: %v", err)
return fmt.Errorf("failed to check CUDA 11 capability: %w", err)
}
}
}
desiredVersion := "latest"
@ -29,6 +32,10 @@ func (l *llamaCpp) ensureLatestLlamaCpp(ctx context.Context, log logging.Logger,
if canUseCUDA11 {
desiredVariant = "cuda"
}
// TODO(p1-0tr): we should auto-detect if we can use opencl, but for now assume that we can
if runtime.GOARCH == "arm64" {
desiredVariant = "opencl"
}
l.status = fmt.Sprintf("looking for updates for %s variant", desiredVariant)
return l.downloadLatestLlamaCpp(ctx, log, httpClient, llamaCppPath, vendoredServerStoragePath, desiredVersion,
desiredVariant)

4
pkg/inference/backends/llamacpp/llamacpp.go
View File

@ -9,6 +9,7 @@ import (
"os/exec"
"path/filepath"
"runtime"
"slices"
"github.com/docker/model-runner/pkg/inference"
"github.com/docker/model-runner/pkg/inference/models"
@ -74,7 +75,8 @@ func (l *llamaCpp) Install(ctx context.Context, httpClient *http.Client) error {
// never support it on Intel Macs.
if runtime.GOOS == "linux" {
return errors.New("not implemented")
} else if (runtime.GOOS == "darwin" && runtime.GOARCH == "amd64") || (runtime.GOOS == "windows" && runtime.GOARCH == "arm64") {
} else if (runtime.GOOS == "darwin" && runtime.GOARCH == "amd64") ||
(runtime.GOOS == "windows" && !slices.Contains([]string{"amd64", "arm64"}, runtime.GOARCH)) {
return errors.New("platform not supported")
}