llama-swap/vulkan.Containerfile at main · 0rzech/llama-swap · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
ARG IMAGE_REGISTRY=docker.io
ARG IMAGE_BASE=${IMAGE_REGISTRY}/almalinux/10-minimal:latest

FROM ${IMAGE_BASE} AS builder

ARG LLAMA_SWAP_VERSION=154
ARG LLAMA_CPP_VERSION=6248

RUN <<EOF
    set -euo pipefail
    microdnf upgrade --assumeyes --setopt=install_weak_deps=0
    microdnf install --assumeyes --setopt=install_weak_deps=0 gzip tar
    microdnf clean all
    curl --proto '=https' --location --proto-redir '=https' --max-redirs 1 "https://github.com/mostlygeek/llama-swap/releases/download/v${LLAMA_SWAP_VERSION}/llama-swap_${LLAMA_SWAP_VERSION}_linux_amd64.tar.gz" --output 'llama-swap.tar.gz'
    mkdir --verbose 'llama-swap'
    tar --extract --verbose --file 'llama-swap.tar.gz' --directory 'llama-swap'
    rm --verbose llama-swap/LICENSE* llama-swap/README*
    curl --proto '=https' --location --proto-redir '=https' --max-redirs 1 "https://github.com/ggml-org/llama.cpp/releases/download/b${LLAMA_CPP_VERSION}/llama-b${LLAMA_CPP_VERSION}-bin-ubuntu-vulkan-x64.tar.gz" --output 'llama-cpp.tar.gz'
    mkdir --verbose 'llama-cpp'
    tar --extract --verbose --file 'llama-cpp.tar.gz' --strip-components 1 --directory 'llama-cpp'
    rm --verbose llama-cpp/LICENSE*
EOF

FROM ${IMAGE_BASE}

COPY start.sh /app/

ARG LLAMA_SWAP_VERSION=154
LABEL llama-swap-version="${LLAMA_SWAP_VERSION}"
COPY --from=builder /llama-swap/* /app/

ARG LLAMA_CPP_VERSION=6248
LABEL llama-cpp-version="${LLAMA_CPP_VERSION}"
LABEL llama-cpp-backend=vulkan
COPY --from=builder /llama-cpp/* /app/

RUN <<EOF
    set -euo pipefail
    microdnf upgrade --assumeyes --setopt=install_weak_deps=0
    microdnf install --assumeyes --setopt=install_weak_deps=0 libgomp shadow-utils vulkan-devel
    passwd --delete root
    usermod --expiredate 1 root
    useradd --user-group --create-home llama
    mkdir --parents '/home/llama/.cache/llama.cpp'
    chown --recursive llama:llama '/home/llama'
    microdnf remove --assumeyes shadow-utils
    microdnf clean all
EOF

WORKDIR /app

USER llama

HEALTHCHECK CMD ["/usr/bin/curl", "--silent", "--show-error", "--fail", "http://localhost:8080/health"]

ENTRYPOINT ["/app/start.sh"]
CMD ["-config", "/app/config.yaml"]

ARG CONFIG_VARIANT=example
COPY config.${CONFIG_VARIANT}.yaml /app/config.yaml