llama.cpp

Debian

cmake -B build
cmake --build build

Windows, with MinGW

P:\ath\to\llama.cpp> cmake -B build -G "MinGW Makefiles"
P:\ath\to\llama.cpp> cmake --build build --config Release

Built executables

Using the deprecated Makefile

cd llama.cpp
make -j

Trying a model

$ build/bin/llama-cli -m models/ggml-vocab-bert-bge.gguf --prompt "tell me a nice story" --predict 100
…
llama_model_load: error loading model: missing tensor 'token_embd.weight'
llama_model_load_from_file_impl: failed to load model
common_init_from_params: failed to load model 'models/ggml-vocab-bert-bge.gguf'
main: error: unable to load model

$ mkdir ~/llm-models
$ curl -L https://huggingface.co/aisuko/gpt2-117M-gguf/resolve/main/ggml-model-Q4_K_M.gguf                                -o ~/llm-models/ggml-model-Q4_K_M.gguf
$ curl -L https://huggingface.co/TheBloke/dolphin-2.2.1-mistral-7B-GGUF/resolve/main/dolphin-2.2.1-mistral-7b.Q4_K_M.gguf -o ~/llm-models/dolphin-2.2.1-mistral-7b.Q4_K_M.gguf

$ (cd ~/llm-models; stat --printf '%s\t%n\n' *)
4368450304      dolphin-2.2.1-mistral-7b.Q4_K_M.gguf
112858624       ggml-model-Q4_K_M.gguf

$ ./build/bin/llama-cli -m ~/llm-models/ggml-model-Q4_K_M.gguf                -p "Tell me about programmers and coffee" -n 200
$ ./build/bin/llama-cli -m ~/llm-models/dolphin-2.2.1-mistral-7b.Q4_K_M.gguf  -p "Tell me about programmers and coffee" -n 200

PS:> $progressPreference = 'SilentlyContinue'
PS:> invoke-webRequest https://huggingface.co/TheBloke/dolphin-2.2.1-mistral-7B-GGUF/resolve/main/dolphin-2.2.1-mistral-7b.Q4_K_M.gguf -outfile ~/llm-models/dolphin-2.2.1-mistral-7b.Q4_K_M.gguf

Testing the webserver

$ build/bin/llama-server -m ~/llm-models/dolphin-2.2.1-mistral-7b.Q4_K_M.gguf

$ build/bin/llama-server -m ~/llm-models/dolphin-2.2.1-mistral-7b.Q4_K_M.gguf --port 8888

grammars/

llama-cli \
  -hfr bartowski/Phi-3-medium-128k-instruct-GGUF \
  -hff Phi-3-medium-128k-instruct-Q8_0.gguf \
  -j '{
    "type": "array",
    "items": {
        "type": "object",
        "properties": {
            "name": {
                "type": "string",
                "minLength": 1,
                "maxLength": 100
            },
            "age": {
                "type": "integer",
                "minimum": 0,
                "maximum": 150
            }
        },
        "required": ["name", "age"],
        "additionalProperties": false
    },
    "minItems": 10,
    "maxItems": 100
  }' \
  -p 'Generate a {name, age}[] JSON array with famous actors of all ages.'

examples/json_schema_to_grammar.py name-age-schema.json

scripts

cmake/x64-windows-llvm.cmake

llama build number, build commit, build compiler and build target

Environment variables

models/7B/ggml-model-f16.gguf

ci/run.sh

examples/jeopardy

examples/llama.android

examples/quantize-stats

examples/rpc

examples/save-load-state

examples/sycl

examples/simple-cmake-pkg

examples/llama.swiftui

Windows: PrefetchVirtualMemory unavailable

Options

(Manual) installation

$ curl -L https://ollama.com/download/ollama-linux-amd64.tgz -o /tmp/ollama-linux-amd64.tgz

$ tar tf /tmp/ollama-linux-amd64.tgz | grep -v '/$'
./bin/ollama
./lib/ollama/libcudart.so.12
./lib/ollama/libcublasLt.so.12.4.5.8
./lib/ollama/libcudart.so.11.0
./lib/ollama/libcublas.so.11
./lib/ollama/libcublas.so.12
./lib/ollama/libcudart.so.11.3.109
./lib/ollama/libcublas.so.12.4.5.8
./lib/ollama/libcudart.so.12.4.127
./lib/ollama/libcublasLt.so.11.5.1.109
./lib/ollama/libcublasLt.so.11
./lib/ollama/runners/cuda_v11_avx/ollama_llama_server
./lib/ollama/runners/cuda_v11_avx/libggml_cuda_v11.so
./lib/ollama/runners/rocm_avx/ollama_llama_server
./lib/ollama/runners/rocm_avx/libggml_rocm.so
./lib/ollama/runners/cuda_v12_avx/ollama_llama_server
./lib/ollama/runners/cuda_v12_avx/libggml_cuda_v12.so
./lib/ollama/runners/cpu_avx2/ollama_llama_server
./lib/ollama/runners/cpu_avx/ollama_llama_server
./lib/ollama/libcublas.so.11.5.1.109
./lib/ollama/libcublasLt.so.12

$ sudo tar -C /usr -xzf /tmp/ollama-linux-amd64.tgz

Testing the installation

$ ollama serve

$ ollama -v
ollama version is 0.5.7

$ curl -s localhost:11434/api/version

Web API

$ curl -s http://localhost:11434/api/generate -d '{
   "model" : "llama2",
   "prompt": "tell me about programmers and coffe"
}' | jq

{
  "error": "model 'llama2' not found"
}

$ curl -s http://localhost:11434/api/pull -d '{
  "model": "llama2"
}' | jq

$ curl -s http://localhost:11434/api/tags | jq -r '.models[].name'

$ curl -s http://localhost:11434/api/generate -d '{
   "model" : "llama2",
   "prompt": "tell me about programmers and coffee"
}' | jq

{
  "error": "model requires more system memory (8.4 GiB) than is available (1.8 GiB)"
}

$ curl -s http://localhost:11434/api/generate -d '{
   "model" : "llama2",
   "prompt": "tell me about programmers and coffee"
}' | jq

{
  "model": "llama2",
  "created_at": "2025-02-01T14:27:46.867393127Z",
  "response": "\n",
  "done": false
}
{
  "model": "llama2",
  "created_at": "2025-02-01T14:27:46.947556123Z",
  "response": "Program",
  "done": false
}
{
  "model": "llama2",
  "created_at": "2025-02-01T14:27:46.996489966Z",
  "response": "mers",
  "done": false
}
{
  "model": "llama2",
  "created_at": "2025-02-01T14:27:47.044551309Z",
  "response": " and",
  "done": false
}

   …

{
  "model": "llama2",
  "created_at": "2025-02-01T14:28:57.985358597Z",
  "response": "!",
  "done": false
}
{
  "model": "llama2",
  "created_at": "2025-02-01T14:28:58.038683996Z",
  "response": "",
  "done": true,
  "done_reason": "stop",
  "context": [
    518,
    25580,
    …
    26529,
    29991
  ],
  "total_duration": 33946247078,
  "load_duration": 7990899,
  "prompt_eval_count": 29,
  "prompt_eval_duration": 558000000,
  "eval_count": 703,
  "eval_duration": 33378000000
}

Checking if GPU is used

$ ollama ps
NAME             ID              SIZE      PROCESSOR    UNTIL              
llama2:latest    78e26419b446    5.6 GB    100% GPU     4 minutes from now