MORPH
SIGN IN SIGN UP
abetlen / llama-cpp-python UNCLAIMED

Python bindings for llama.cpp

0 0 1 Python
Normal View History Raw
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
from __future__ import annotations
Update llama_cpp.py Make shared library code more robust with some platform specific functionality and more descriptive errors when failures occur
2023-04-02 21:50:13 -04:00
import os
Initial commit
2023-03-23 05:33:06 -04:00
import ctypes
import pathlib
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
import warnings
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
from typing import (
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
Callable,
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
Union,
NewType,
Optional,
TYPE_CHECKING,
)
Initial commit
2023-03-23 05:33:06 -04:00
feat: Expose libggml in internal APIs (#1761) * Expose libggml and refactor ctypes extension * Only expose libggml * Use ctypes_extensions module for libllama and libllava
2024-09-26 11:23:00 -04:00
from llama_cpp._ctypes_extensions import (
load_shared_library,
byref,
ctypes_function_for_shared_library,
)
Update llama.cpp
2023-04-11 11:59:03 -04:00
feat: Expose libggml in internal APIs (#1761) * Expose libggml and refactor ctypes extension * Only expose libggml * Use ctypes_extensions module for libllama and libllava
2024-09-26 11:23:00 -04:00
if TYPE_CHECKING:
from llama_cpp._ctypes_extensions import (
CtypesCData,
CtypesArray,
CtypesPointer,
CtypesVoidPointer,
CtypesRef,
CtypesPointerOrRef,
CtypesFuncPointer,
Update llama.cpp
2023-04-11 11:59:03 -04:00
)
Update llama_cpp.py Make shared library code more robust with some platform specific functionality and more descriptive errors when failures occur
2023-04-02 21:50:13 -04:00
# Specify the base name of the shared library to load
Update llama_cpp.py with PR requests lib_base_name and load_shared_library to _lib_base_name and _load_shared_library
2023-04-03 13:06:50 -04:00
_lib_base_name = "llama"
feat: Expose libggml in internal APIs (#1761) * Expose libggml and refactor ctypes extension * Only expose libggml * Use ctypes_extensions module for libllama and libllava
2024-09-26 11:23:00 -04:00
_override_base_path = os.environ.get("LLAMA_CPP_LIB_PATH")
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
_base_path = (
pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib"
if _override_base_path is None
else pathlib.Path(_override_base_path)
)
Update llama_cpp.py Make shared library code more robust with some platform specific functionality and more descriptive errors when failures occur
2023-04-02 21:50:13 -04:00
# Load the library
feat: Expose libggml in internal APIs (#1761) * Expose libggml and refactor ctypes extension * Only expose libggml * Use ctypes_extensions module for libllama and libllava
2024-09-26 11:23:00 -04:00
_lib = load_shared_library(_lib_base_name, _base_path)
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
ctypes_function = ctypes_function_for_shared_library(_lib)
feat: add support for KV cache quantization options (#1307) * add KV cache quantization options https://github.com/abetlen/llama-cpp-python/discussions/1220 https://github.com/abetlen/llama-cpp-python/issues/1305 * Add ggml_type * Use ggml_type instead of string for quantization * Add server support --------- Co-authored-by: Andrei Betlen <abetlen@gmail.com>
2024-04-01 22:19:28 +08:00
# from ggml.h
# // NOTE: always add types at the end of the enum to keep backward compatibility
# enum ggml_type {
# GGML_TYPE_F32 = 0,
# GGML_TYPE_F16 = 1,
# GGML_TYPE_Q4_0 = 2,
# GGML_TYPE_Q4_1 = 3,
# // GGML_TYPE_Q4_2 = 4, support has been removed
# // GGML_TYPE_Q4_3 = 5, support has been removed
# GGML_TYPE_Q5_0 = 6,
# GGML_TYPE_Q5_1 = 7,
# GGML_TYPE_Q8_0 = 8,
# GGML_TYPE_Q8_1 = 9,
# GGML_TYPE_Q2_K = 10,
# GGML_TYPE_Q3_K = 11,
# GGML_TYPE_Q4_K = 12,
# GGML_TYPE_Q5_K = 13,
# GGML_TYPE_Q6_K = 14,
# GGML_TYPE_Q8_K = 15,
# GGML_TYPE_IQ2_XXS = 16,
# GGML_TYPE_IQ2_XS = 17,
# GGML_TYPE_IQ3_XXS = 18,
# GGML_TYPE_IQ1_S = 19,
# GGML_TYPE_IQ4_NL = 20,
# GGML_TYPE_IQ3_S = 21,
# GGML_TYPE_IQ2_S = 22,
# GGML_TYPE_IQ4_XS = 23,
# GGML_TYPE_I8 = 24,
# GGML_TYPE_I16 = 25,
# GGML_TYPE_I32 = 26,
# GGML_TYPE_I64 = 27,
# GGML_TYPE_F64 = 28,
# GGML_TYPE_IQ1_M = 29,
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# GGML_TYPE_MXFP4 = 39,
# GGML_TYPE_NVFP4 = 40,
feat: add support for KV cache quantization options (#1307) * add KV cache quantization options https://github.com/abetlen/llama-cpp-python/discussions/1220 https://github.com/abetlen/llama-cpp-python/issues/1305 * Add ggml_type * Use ggml_type instead of string for quantization * Add server support --------- Co-authored-by: Andrei Betlen <abetlen@gmail.com>
2024-04-01 22:19:28 +08:00
# GGML_TYPE_COUNT,
# };
GGML_TYPE_F32 = 0
GGML_TYPE_F16 = 1
GGML_TYPE_Q4_0 = 2
GGML_TYPE_Q4_1 = 3
GGML_TYPE_Q5_0 = 6
GGML_TYPE_Q5_1 = 7
GGML_TYPE_Q8_0 = 8
GGML_TYPE_Q8_1 = 9
GGML_TYPE_Q2_K = 10
GGML_TYPE_Q3_K = 11
GGML_TYPE_Q4_K = 12
GGML_TYPE_Q5_K = 13
GGML_TYPE_Q6_K = 14
GGML_TYPE_Q8_K = 15
GGML_TYPE_IQ2_XXS = 16
GGML_TYPE_IQ2_XS = 17
GGML_TYPE_IQ3_XXS = 18
GGML_TYPE_IQ1_S = 19
GGML_TYPE_IQ4_NL = 20
GGML_TYPE_IQ3_S = 21
GGML_TYPE_IQ2_S = 22
GGML_TYPE_IQ4_XS = 23
GGML_TYPE_I8 = 24
GGML_TYPE_I16 = 25
GGML_TYPE_I32 = 26
GGML_TYPE_I64 = 27
GGML_TYPE_F64 = 28
GGML_TYPE_IQ1_M = 29
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
GGML_TYPE_MXFP4 = 39
GGML_TYPE_NVFP4 = 40
GGML_TYPE_COUNT = 41
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
Update llama.cpp
2024-01-18 21:21:49 -05:00
# from ggml-backend.h
# typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
ggml_backend_sched_eval_callback = ctypes.CFUNCTYPE(
ctypes.c_bool, ctypes.c_void_p, ctypes.c_bool, ctypes.c_void_p
)
Update llama.cpp
2024-01-18 21:21:49 -05:00
feat: Update llama.cpp
2024-03-02 22:20:04 -05:00
# // Abort callback
# // If not NULL, called before ggml computation
# // If it returns true, the computation is aborted
# typedef bool (*ggml_abort_callback)(void * data);
ggml_abort_callback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_void_p)
Update llama.cpp
2023-05-21 17:47:21 -04:00
# llama.h bindings
Update llama.cpp
2024-01-03 22:04:04 -05:00
_lib.llama_max_devices.argtypes = []
Update llama.cpp
2024-01-31 10:41:42 -05:00
_lib.llama_max_devices.restype = ctypes.c_size_t
Update llama.cpp
2024-01-03 22:04:04 -05:00
set llama_max_devices using library function
2023-12-22 15:19:28 -05:00
LLAMA_MAX_DEVICES = _lib.llama_max_devices()
Update llama.cpp
2023-06-06 16:23:55 -04:00
Update llama.cpp
2023-08-24 00:17:00 -04:00
# define LLAMA_DEFAULT_SEED 0xFFFFFFFF
Convert missed llama.cpp constants into standard python types
2023-09-13 21:11:52 -04:00
LLAMA_DEFAULT_SEED = 0xFFFFFFFF
Update llama.cpp
2023-08-24 00:17:00 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# define LLAMA_TOKEN_NULL -1
LLAMA_TOKEN_NULL = -1
Update llama.cpp
2024-01-08 14:51:29 -05:00
# define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla'
Update llama.cpp
2023-12-16 18:57:43 -05:00
LLAMA_FILE_MAGIC_GGLA = 0x67676C61
Update llama.cpp
2023-08-24 00:17:00 -04:00
# define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn'
Convert missed llama.cpp constants into standard python types
2023-09-13 21:11:52 -04:00
LLAMA_FILE_MAGIC_GGSN = 0x6767736E
Update llama.cpp
2023-05-21 17:47:21 -04:00
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
# define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq'
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
LLAMA_FILE_MAGIC_GGSQ = 0x67677371
Update llama.cpp
2023-08-24 00:17:00 -04:00
# define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN
Update llama.cpp
2023-05-21 17:47:21 -04:00
LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# define LLAMA_SESSION_VERSION 9
LLAMA_SESSION_VERSION = 9
Update llama.cpp
2023-05-01 10:44:28 -04:00
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
# define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
# define LLAMA_STATE_SEQ_VERSION 2
LLAMA_STATE_SEQ_VERSION = 2
Update llama.cpp
2023-07-05 01:00:14 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# struct llama_vocab;
llama_vocab_p = NewType("llama_vocab_p", int)
llama_vocab_p_ctypes = ctypes.c_void_p
Update llama.cpp
2023-06-26 08:50:38 -04:00
# struct llama_model;
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
llama_model_p = NewType("llama_model_p", int)
llama_model_p_ctypes = ctypes.c_void_p
Update llama.cpp
2023-06-26 08:50:38 -04:00
Update llama.cpp
2023-05-21 17:47:21 -04:00
# struct llama_context;
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
llama_context_p = NewType("llama_context_p", int)
llama_context_p_ctypes = ctypes.c_void_p
Update llama.cpp and re-organize low-level api
2023-03-24 14:58:42 -04:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# typedef struct llama_memory_i * llama_memory_t;
llama_memory_t = NewType("llama_memory_t", int)
llama_memory_t_ctypes = ctypes.c_void_p
Update llama.cpp and re-organize low-level api
2023-03-24 14:58:42 -04:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# struct llama_kv_cache; (DEPRECATED)
feat: Update llama.cpp
2025-04-11 03:28:20 -04:00
llama_kv_cache_p = NewType("llama_kv_cache_p", int)
llama_kv_cache_p_ctypes = ctypes.c_void_p
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# struct gguf_context;
gguf_context_p = NewType("gguf_context_p", int)
gguf_context_p_ctypes = ctypes.c_void_p
Update llama.cpp
2023-09-28 22:42:03 -04:00
# typedef int32_t llama_pos;
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
llama_pos = ctypes.c_int32
Update llama.cpp
2023-09-28 22:42:03 -04:00
# typedef int32_t llama_token;
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
llama_token = ctypes.c_int32
llama_token_p = ctypes.POINTER(llama_token)
Update llama.cpp
2023-09-28 22:42:03 -04:00
# typedef int32_t llama_seq_id;
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
llama_seq_id = ctypes.c_int32
Initial commit
2023-03-23 05:33:06 -04:00
Update llama.cpp
2023-08-24 00:17:00 -04:00
# enum llama_vocab_type {
feat: Update llama.cpp
2025-07-16 02:47:55 -04:00
# LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab
# LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback
# LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE
# LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece
# LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram
# LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization
# LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming
Update llama.cpp
2023-08-24 00:17:00 -04:00
# };
feat: Update llama.cpp
2024-03-15 12:58:45 -04:00
LLAMA_VOCAB_TYPE_NONE = 0
feat: Update llama.cpp
2024-03-28 12:06:46 -04:00
"""For models without vocab"""
feat: Update llama.cpp
2024-03-15 12:58:45 -04:00
LLAMA_VOCAB_TYPE_SPM = 1
feat: Update llama.cpp
2024-03-28 12:06:46 -04:00
"""LLaMA tokenizer based on byte-level BPE with byte fallback"""
feat: Update llama.cpp
2024-03-15 12:58:45 -04:00
LLAMA_VOCAB_TYPE_BPE = 2
feat: Update llama.cpp
2024-03-28 12:06:46 -04:00
"""GPT-2 tokenizer based on byte-level BPE"""
feat: Update llama.cpp
2024-03-15 12:58:45 -04:00
LLAMA_VOCAB_TYPE_WPM = 3
feat: Update llama.cpp
2024-03-28 12:06:46 -04:00
"""BERT tokenizer based on WordPiece"""
feat: Update llama.cpp
2024-07-01 21:28:11 -04:00
LLAMA_VOCAB_TYPE_UGM = 4
"""T5 tokenizer based on Unigram"""
feat: Update llama.cpp
2024-09-01 21:06:18 -04:00
LLAMA_VOCAB_TYPE_RWKV = 5
"""RWKV tokenizer based on greedy tokenization"""
feat: Update llama.cpp
2025-07-16 02:47:55 -04:00
LLAMA_VOCAB_TYPE_PLAMO2 = 6
"""PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming"""
Update llama.cpp
2023-08-24 00:17:00 -04:00
feat: Update llama.cpp
2025-07-15 07:35:40 -04:00
# NOTE: Deprecated and will be removed in the future. (already gone in llama.cpp)
feat: Update llama.cpp
2024-04-29 23:34:55 -04:00
# // pre-tokenization types
# enum llama_vocab_pre_type {
# LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0,
# LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1,
# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2,
# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3,
# LLAMA_VOCAB_PRE_TYPE_FALCON = 4,
# LLAMA_VOCAB_PRE_TYPE_MPT = 5,
# LLAMA_VOCAB_PRE_TYPE_STARCODER = 6,
# LLAMA_VOCAB_PRE_TYPE_GPT2 = 7,
feat: Update llama.cpp
2024-05-05 12:12:27 -04:00
# LLAMA_VOCAB_PRE_TYPE_REFACT = 8,
# LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9,
feat: Update llama.cpp
2024-05-22 02:40:31 -04:00
# LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10,
# LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11,
# LLAMA_VOCAB_PRE_TYPE_OLMO = 12,
# LLAMA_VOCAB_PRE_TYPE_DBRX = 13,
feat: Update llama.cpp
2024-05-27 10:51:57 -04:00
# LLAMA_VOCAB_PRE_TYPE_SMAUG = 14,
feat: Update llama.cpp
2024-06-17 11:56:03 -04:00
# LLAMA_VOCAB_PRE_TYPE_PORO = 15,
feat: Update llama.cpp
2024-07-09 00:15:22 -04:00
# LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16,
# LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17,
# LLAMA_VOCAB_PRE_TYPE_VIKING = 18,
# LLAMA_VOCAB_PRE_TYPE_JAIS = 19,
feat: Update llama.cpp
2024-07-22 15:20:12 -04:00
# LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20,
# LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21,
# LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22,
feat: Update llama.cpp
2024-08-15 14:46:46 -04:00
# LLAMA_VOCAB_PRE_TYPE_BLOOM = 23,
# LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24,
feat: Update llama.cpp
2024-08-16 14:43:25 -04:00
# LLAMA_VOCAB_PRE_TYPE_EXAONE = 25,
feat: Update llama.cpp
2024-09-29 15:23:24 -04:00
# LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26,
feat: Update llama.cpp
2024-12-06 02:33:55 -05:00
# LLAMA_VOCAB_PRE_TYPE_MINERVA = 27,
feat: Update llama.cpp
2025-01-08 16:33:53 -05:00
# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28,
feat: Update llama.cpp
2025-03-12 04:11:42 -04:00
# LLAMA_VOCAB_PRE_TYPE_GPT4O = 29,
feat: Update llama.cpp
2025-04-11 03:28:20 -04:00
# LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30,
# LLAMA_VOCAB_PRE_TYPE_TRILLION = 31,
# LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32,
feat: Update llama.cpp
2025-05-08 06:52:52 -04:00
# LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33,
# LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34,
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35,
feat: Update llama.cpp
2024-04-29 23:34:55 -04:00
# };
LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0
LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1
LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2
LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3
LLAMA_VOCAB_PRE_TYPE_FALCON = 4
LLAMA_VOCAB_PRE_TYPE_MPT = 5
LLAMA_VOCAB_PRE_TYPE_STARCODER = 6
LLAMA_VOCAB_PRE_TYPE_GPT2 = 7
feat: Update llama.cpp
2024-05-05 12:12:27 -04:00
LLAMA_VOCAB_PRE_TYPE_REFACT = 8
LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9
feat: Update llama.cpp
2024-05-22 02:40:31 -04:00
LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10
LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11
LLAMA_VOCAB_PRE_TYPE_OLMO = 12
LLAMA_VOCAB_PRE_TYPE_DBRX = 13
feat: Update llama.cpp
2024-05-27 10:51:57 -04:00
LLAMA_VOCAB_PRE_TYPE_SMAUG = 14
feat: Update llama.cpp
2024-06-17 11:56:03 -04:00
LLAMA_VOCAB_PRE_TYPE_PORO = 15
feat: Update llama.cpp
2025-05-08 06:52:52 -04:00
LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16
feat: Update llama.cpp
2024-07-09 00:15:22 -04:00
LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17
LLAMA_VOCAB_PRE_TYPE_VIKING = 18
LLAMA_VOCAB_PRE_TYPE_JAIS = 19
feat: Update llama.cpp
2024-07-22 15:20:12 -04:00
LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20
LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21
LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22
feat: Update llama.cpp
2024-08-15 14:46:46 -04:00
LLAMA_VOCAB_PRE_TYPE_BLOOM = 23
LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24
feat: Update llama.cpp
2024-08-16 14:43:25 -04:00
LLAMA_VOCAB_PRE_TYPE_EXAONE = 25
feat: Update llama.cpp
2024-09-29 15:23:24 -04:00
LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26
feat: Update llama.cpp
2024-12-06 02:33:55 -05:00
LLAMA_VOCAB_PRE_TYPE_MINERVA = 27
feat: Update llama.cpp
2025-01-08 16:33:53 -05:00
LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28
feat: Update llama.cpp
2025-03-12 04:11:42 -04:00
LLAMA_VOCAB_PRE_TYPE_GPT4O = 29
feat: Update llama.cpp
2025-04-11 03:28:20 -04:00
LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30
LLAMA_VOCAB_PRE_TYPE_TRILLION = 31
LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32
feat: Update llama.cpp
2025-05-08 06:52:52 -04:00
LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33
LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35
feat: Update llama.cpp
2024-04-29 23:34:55 -04:00
feat: Update llama.cpp
2024-02-25 20:52:14 -05:00
# // note: these values should be synchronized with ggml_rope
# // TODO: maybe move this enum to ggml.h (ggml_rope_type)
# enum llama_rope_type {
feat: Update llama.cpp
2024-12-19 01:55:12 -05:00
# LLAMA_ROPE_TYPE_NONE = -1,
# LLAMA_ROPE_TYPE_NORM = 0,
# LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX,
# LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE,
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE,
feat: Update llama.cpp
2024-12-19 01:55:12 -05:00
# LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION,
feat: Update llama.cpp
2024-02-25 20:52:14 -05:00
# };
LLAMA_ROPE_TYPE_NONE = -1
LLAMA_ROPE_TYPE_NORM = 0
feat: Update llama.cpp
2024-08-15 14:46:46 -04:00
LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX = 2
feat: Update llama.cpp
2024-12-19 01:55:12 -05:00
LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE = 8
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE = 40
feat: Update llama.cpp
2024-12-19 01:55:12 -05:00
LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION = 24
feat: Update llama.cpp
2024-02-25 20:52:14 -05:00
feat: Update llama.cpp
2024-06-07 02:02:12 -04:00
# enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file
Update llama.cpp
2023-08-24 00:17:00 -04:00
# LLAMA_TOKEN_TYPE_UNDEFINED = 0,
# LLAMA_TOKEN_TYPE_NORMAL = 1,
# LLAMA_TOKEN_TYPE_UNKNOWN = 2,
# LLAMA_TOKEN_TYPE_CONTROL = 3,
# LLAMA_TOKEN_TYPE_USER_DEFINED = 4,
# LLAMA_TOKEN_TYPE_UNUSED = 5,
# LLAMA_TOKEN_TYPE_BYTE = 6,
# };
Convert missed llama.cpp constants into standard python types
2023-09-13 21:11:52 -04:00
LLAMA_TOKEN_TYPE_UNDEFINED = 0
LLAMA_TOKEN_TYPE_NORMAL = 1
LLAMA_TOKEN_TYPE_UNKNOWN = 2
LLAMA_TOKEN_TYPE_CONTROL = 3
LLAMA_TOKEN_TYPE_USER_DEFINED = 4
LLAMA_TOKEN_TYPE_UNUSED = 5
LLAMA_TOKEN_TYPE_BYTE = 6
Update llama.cpp
2023-08-24 00:17:00 -04:00
Update llama.cpp
2023-09-28 22:42:03 -04:00
feat: Update llama.cpp
2024-06-07 02:02:12 -04:00
# enum llama_token_attr {
# LLAMA_TOKEN_ATTR_UNDEFINED = 0,
# LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0,
# LLAMA_TOKEN_ATTR_UNUSED = 1 << 1,
# LLAMA_TOKEN_ATTR_NORMAL = 1 << 2,
# LLAMA_TOKEN_ATTR_CONTROL = 1 << 3, // SPECIAL?
# LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4,
# LLAMA_TOKEN_ATTR_BYTE = 1 << 5,
# LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6,
# LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7,
# LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8,
# LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9,
# };
LLAMA_TOKEN_ATTR_UNDEFINED = 0
LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0
LLAMA_TOKEN_ATTR_UNUSED = 1 << 1
LLAMA_TOKEN_ATTR_NORMAL = 1 << 2
LLAMA_TOKEN_ATTR_CONTROL = 1 << 3
LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4
LLAMA_TOKEN_ATTR_BYTE = 1 << 5
LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6
LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7
LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8
LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9
Update llama.cpp
2023-09-28 22:42:03 -04:00
# // model file types
Update llama.cpp
2023-08-24 00:17:00 -04:00
# enum llama_ftype {
# LLAMA_FTYPE_ALL_F32 = 0,
Update llama.cpp
2023-09-28 22:42:03 -04:00
# LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
# // LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
Update llama.cpp
2023-09-28 22:42:03 -04:00
# // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed
# // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed
# LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q3_K_S = 11, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q3_K_M = 12, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q3_K_L = 13, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q4_K_S = 14, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q4_K_M = 15, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q5_K_S = 16, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q5_K_M = 17, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q6_K = 18, // except 1d tensors
Update llama.cpp
2024-01-08 14:51:29 -05:00
# LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19, // except 1d tensors
Update llama.cpp
2024-01-11 22:51:12 -05:00
# LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensors
feat: Update llama.cpp
2024-02-26 11:40:58 -05:00
# LLAMA_FTYPE_MOSTLY_IQ3_XS = 22, // except 1d tensors
Update llama.cpp
2024-01-30 09:48:09 -05:00
# LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors
feat: Update llama.cpp
2024-02-18 21:30:36 -05:00
# LLAMA_FTYPE_MOSTLY_IQ1_S = 24, // except 1d tensors
feat: Update llama.cpp
2024-02-21 11:05:58 -05:00
# LLAMA_FTYPE_MOSTLY_IQ4_NL = 25, // except 1d tensors
feat: Update llama.cpp
2024-02-24 23:47:29 -05:00
# LLAMA_FTYPE_MOSTLY_IQ3_S = 26, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_IQ3_M = 27, // except 1d tensors
feat: Update llama.cpp
2024-02-26 11:40:58 -05:00
# LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors
feat: Update llama.cpp
2024-02-27 12:22:17 -05:00
# LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors
feat: Update llama.cpp
2024-03-26 22:58:53 -04:00
# LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors
feat: Update llama.cpp
2024-05-08 08:42:22 -04:00
# LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors
feat: Update llama.cpp
2024-12-08 22:40:19 -05:00
# //LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // removed from gguf files, use Q4_0 and runtime repack
# //LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // removed from gguf files, use Q4_0 and runtime repack
# //LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // removed from gguf files, use Q4_0 and runtime repack
feat: Update llama.cpp
2024-09-06 04:12:29 -04:00
# LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors
feat: Update llama.cpp
2025-08-07 06:40:18 -07:00
# LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
#
Update llama.cpp
2023-08-24 00:17:00 -04:00
# LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file
# };
Convert missed llama.cpp constants into standard python types
2023-09-13 21:11:52 -04:00
LLAMA_FTYPE_ALL_F32 = 0
LLAMA_FTYPE_MOSTLY_F16 = 1
LLAMA_FTYPE_MOSTLY_Q4_0 = 2
LLAMA_FTYPE_MOSTLY_Q4_1 = 3
LLAMA_FTYPE_MOSTLY_Q8_0 = 7
LLAMA_FTYPE_MOSTLY_Q5_0 = 8
LLAMA_FTYPE_MOSTLY_Q5_1 = 9
LLAMA_FTYPE_MOSTLY_Q2_K = 10
LLAMA_FTYPE_MOSTLY_Q3_K_S = 11
LLAMA_FTYPE_MOSTLY_Q3_K_M = 12
LLAMA_FTYPE_MOSTLY_Q3_K_L = 13
LLAMA_FTYPE_MOSTLY_Q4_K_S = 14
LLAMA_FTYPE_MOSTLY_Q4_K_M = 15
LLAMA_FTYPE_MOSTLY_Q5_K_S = 16
LLAMA_FTYPE_MOSTLY_Q5_K_M = 17
LLAMA_FTYPE_MOSTLY_Q6_K = 18
Update llama.cpp
2024-01-11 22:51:12 -05:00
LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19
LLAMA_FTYPE_MOSTLY_IQ2_XS = 20
LLAMA_FTYPE_MOSTLY_Q2_K_S = 21
feat: Update llama.cpp
2024-02-26 11:40:58 -05:00
LLAMA_FTYPE_MOSTLY_IQ3_XS = 22
Update llama.cpp
2024-01-30 09:48:09 -05:00
LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23
feat: Update llama.cpp
2024-02-18 21:30:36 -05:00
LLAMA_FTYPE_MOSTLY_IQ1_S = 24
feat: Update llama.cpp
2024-02-21 11:05:58 -05:00
LLAMA_FTYPE_MOSTLY_IQ4_NL = 25
feat: Update llama.cpp
2024-02-24 23:47:29 -05:00
LLAMA_FTYPE_MOSTLY_IQ3_S = 26
LLAMA_FTYPE_MOSTLY_IQ3_M = 27
feat: Update llama.cpp
2024-02-26 11:40:58 -05:00
LLAMA_FTYPE_MOSTLY_IQ2_S = 28
LLAMA_FTYPE_MOSTLY_IQ2_M = 29
feat: Update llama.cpp
2024-02-27 12:22:17 -05:00
LLAMA_FTYPE_MOSTLY_IQ4_XS = 30
feat: Update llama.cpp
2024-05-08 08:42:22 -04:00
LLAMA_FTYPE_MOSTLY_IQ1_M = 31
LLAMA_FTYPE_MOSTLY_BF16 = 32
feat: Update llama.cpp
2024-12-08 22:40:19 -05:00
# LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33
# LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34
# LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35
feat: Update llama.cpp
2024-09-06 04:12:29 -04:00
LLAMA_FTYPE_MOSTLY_TQ1_0 = 36
LLAMA_FTYPE_MOSTLY_TQ2_0 = 37
feat: Update llama.cpp
2025-08-07 06:40:18 -07:00
LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
LLAMA_FTYPE_MOSTLY_NVFP4 = 39
Convert missed llama.cpp constants into standard python types
2023-09-13 21:11:52 -04:00
LLAMA_FTYPE_GUESSED = 1024
Update llama.cpp
2023-08-24 00:17:00 -04:00
Update llama.cpp Fix build examples Exclude examples directory Revert cmake changes Try actions/checkout@v4 Try to update submodules Revert Update llama.cpp Fix build examples Exclude examples directory Revert cmake changes Try actions/checkout@v4 Try to update submodules Revert
2023-11-02 13:40:20 -04:00
# enum llama_rope_scaling_type {
feat: Update llama.cpp
2024-02-25 16:53:58 -05:00
# LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1,
# LLAMA_ROPE_SCALING_TYPE_NONE = 0,
# LLAMA_ROPE_SCALING_TYPE_LINEAR = 1,
# LLAMA_ROPE_SCALING_TYPE_YARN = 2,
feat: Update llama.cpp
2024-12-06 02:33:55 -05:00
# LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3,
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE,
Update llama.cpp Fix build examples Exclude examples directory Revert cmake changes Try actions/checkout@v4 Try to update submodules Revert Update llama.cpp Fix build examples Exclude examples directory Revert cmake changes Try actions/checkout@v4 Try to update submodules Revert
2023-11-02 13:40:20 -04:00
# };
feat: Update llama.cpp
2024-02-25 16:53:58 -05:00
LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1
LLAMA_ROPE_SCALING_TYPE_NONE = 0
LLAMA_ROPE_SCALING_TYPE_LINEAR = 1
LLAMA_ROPE_SCALING_TYPE_YARN = 2
feat: Update llama.cpp
2024-12-06 02:33:55 -05:00
LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE
Black formatting
2023-03-24 14:35:41 -04:00
feat: Update llama.cpp
2024-02-15 15:17:30 -05:00
# enum llama_pooling_type {
feat: Update llama.cpp
2024-03-03 11:27:04 -05:00
# LLAMA_POOLING_TYPE_UNSPECIFIED = -1,
feat: Update llama.cpp
2024-02-25 16:53:58 -05:00
# LLAMA_POOLING_TYPE_NONE = 0,
# LLAMA_POOLING_TYPE_MEAN = 1,
# LLAMA_POOLING_TYPE_CLS = 2,
feat: Update llama_cpp.py bindings
2024-06-21 16:56:15 -04:00
# LLAMA_POOLING_TYPE_LAST = 3,
feat: Update llama.cpp
2024-09-29 15:23:24 -04:00
# LLAMA_POOLING_TYPE_RANK = 4, // used by reranking models to attach the classification head to the graph
feat: Update llama.cpp
2024-02-15 15:17:30 -05:00
# };
feat: Update llama.cpp
2024-03-03 11:27:04 -05:00
LLAMA_POOLING_TYPE_UNSPECIFIED = -1
feat: Update llama.cpp
2024-02-25 16:53:58 -05:00
LLAMA_POOLING_TYPE_NONE = 0
LLAMA_POOLING_TYPE_MEAN = 1
LLAMA_POOLING_TYPE_CLS = 2
fix(misc): Format
2024-07-09 12:20:17 -04:00
LLAMA_POOLING_TYPE_LAST = 3
feat: Update llama.cpp
2024-09-29 15:23:24 -04:00
LLAMA_POOLING_TYPE_RANK = 4
feat: Update llama.cpp
2024-02-15 15:17:30 -05:00
feat: Update llama.cpp
2024-07-05 20:44:46 -04:00
# enum llama_attention_type {
# LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1,
# LLAMA_ATTENTION_TYPE_CAUSAL = 0,
# LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1,
# };
LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1
LLAMA_ATTENTION_TYPE_CAUSAL = 0
LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1
feat: Update llama.cpp
2024-09-29 15:23:24 -04:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# enum llama_flash_attn_type {
# LLAMA_FLASH_ATTN_TYPE_AUTO = -1,
# LLAMA_FLASH_ATTN_TYPE_DISABLED = 0,
# LLAMA_FLASH_ATTN_TYPE_ENABLED = 1,
# };
LLAMA_FLASH_ATTN_TYPE_AUTO = -1
LLAMA_FLASH_ATTN_TYPE_DISABLED = 0
LLAMA_FLASH_ATTN_TYPE_ENABLED = 1
Update llama.cpp
2024-01-13 22:37:49 -05:00
# enum llama_split_mode {
feat: Update llama.cpp
2024-09-29 15:23:24 -04:00
# LLAMA_SPLIT_MODE_NONE = 0, // single GPU
# LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported
Update llama.cpp
2024-01-13 22:37:49 -05:00
# };
feat: Update llama.cpp
2024-02-25 16:53:58 -05:00
LLAMA_SPLIT_MODE_NONE = 0
LLAMA_SPLIT_MODE_LAYER = 1
LLAMA_SPLIT_MODE_ROW = 2
Update llama.cpp
2023-11-20 14:11:33 -05:00
Update llama.cpp
2024-01-15 10:12:10 -05:00
Update llama.cpp
2023-05-21 17:47:21 -04:00
# typedef struct llama_token_data {
# llama_token id; // token id
# float logit; // log-odds of the token
# float p; // probability of the token
# } llama_token_data;
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
class llama_token_data(ctypes.Structure):
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
"""Used to store token data
Update llama.cpp
2024-01-08 14:51:29 -05:00
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
Attributes:
id (llama_token): token id
logit (float): log-odds of the token
p (float): probability of the token"""
Update llama.cpp
2024-01-08 14:51:29 -05:00
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
if TYPE_CHECKING:
id: llama_token
logit: float
p: float
Initial commit
2023-03-23 05:33:06 -04:00
_fields_ = [
Update llama.cpp
2023-05-21 17:47:21 -04:00
("id", llama_token),
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("logit", ctypes.c_float),
("p", ctypes.c_float),
Initial commit
2023-03-23 05:33:06 -04:00
]
Black formatting
2023-03-24 14:35:41 -04:00
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
llama_token_data_p = ctypes.POINTER(llama_token_data)
Initial commit
2023-03-23 05:33:06 -04:00
Update llama.cpp
2023-05-01 10:44:28 -04:00
Update llama.cpp
2023-05-21 17:47:21 -04:00
# typedef struct llama_token_data_array {
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
# // TODO: consider SoA
# // NOTE: this pointer can be modified by the samplers
Update llama.cpp
2023-05-21 17:47:21 -04:00
# llama_token_data * data;
# size_t size;
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# int64_t selected; // this is the index in the data array (i.e. not the token id)
Update llama.cpp
2023-05-21 17:47:21 -04:00
# bool sorted;
# } llama_token_data_array;
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
class llama_token_data_array(ctypes.Structure):
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
"""Used to sample tokens given logits
Update llama.cpp
2024-01-08 14:51:29 -05:00
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
Attributes:
data (ctypes.Array[llama_token_data]): token data
size (int): size of the array
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
selected (int): index in the data array (i.e. not the token id)
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
sorted (bool): whether the array is sorted"""
Update llama.cpp
2024-01-08 14:51:29 -05:00
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
if TYPE_CHECKING:
data: CtypesArray[llama_token_data]
size: int
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
selected: int
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
sorted: bool
Update llama.cpp
2023-05-01 10:44:28 -04:00
_fields_ = [
("data", llama_token_data_p),
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("size", ctypes.c_size_t),
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
("selected", ctypes.c_int64),
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("sorted", ctypes.c_bool),
Update llama.cpp
2023-05-01 10:44:28 -04:00
]
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
llama_token_data_array_p = ctypes.POINTER(llama_token_data_array)
Update llama.cpp
2023-05-01 10:44:28 -04:00
feat: Update llama.cpp
2024-05-05 12:12:27 -04:00
# typedef bool (*llama_progress_callback)(float progress, void * user_data);
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
llama_progress_callback = ctypes.CFUNCTYPE(
ctypes.c_bool, ctypes.c_float, ctypes.c_void_p
)
Black formatting
2023-03-24 14:35:41 -04:00
Update llama.cpp
2023-08-24 00:17:00 -04:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Input data for llama_encode/llama_decode
Update llama.cpp
2023-09-28 22:42:03 -04:00
# // A llama_batch object can contain input about one or many sequences
# // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens
# //
# // - token : the token ids of the input (used when embd is NULL)
# // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL)
# // - pos : the positions of the respective token in the sequence
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // (if set to NULL, the token position will be tracked automatically by llama_encode/llama_decode)
Update llama.cpp
2023-09-28 22:42:03 -04:00
# // - seq_id : the sequence to which the respective token belongs
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
# // (if set to NULL, the sequence ID will be assumed to be 0)
Update llama.cpp
2024-03-06 01:32:00 -05:00
# // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // (if set to NULL:
# // - if embeddings: all tokens are output
# // - if not: only the last token is output
# // )
Update llama.cpp
2023-09-28 22:42:03 -04:00
# //
# typedef struct llama_batch {
# int32_t n_tokens;
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
Update llama.cpp
2023-10-19 02:55:08 -04:00
# llama_token * token;
# float * embd;
# llama_pos * pos;
Update llama.cpp
2023-10-31 21:29:35 -04:00
# int32_t * n_seq_id;
Update llama.cpp
2023-10-19 02:55:08 -04:00
# llama_seq_id ** seq_id;
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# int8_t * logits; // TODO: rename this to "output"
Update llama.cpp
2023-09-28 22:42:03 -04:00
# } llama_batch;
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
class llama_batch(ctypes.Structure):
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
"""Input data for llama_encode/llama_decode
Update llama.cpp
2023-11-23 16:26:00 -05:00
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
A llama_batch object can contain input about one or many sequences
Update llama.cpp
2023-11-23 16:26:00 -05:00
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens
Update llama.cpp
2023-11-23 16:26:00 -05:00
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
Attributes:
feat: Update llama.cpp
2024-03-08 20:58:50 -05:00
n_tokens (int): number of tokens
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
token (ctypes.Array[llama_token]): the token ids of the input (used when embd is NULL)
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
embd (ctypes.Array[ctypes.ctypes.c_float]): token embeddings (i.e. float vector of size n_embd) (used when token is NULL)
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
pos (ctypes.Array[ctypes.Array[llama_pos]]): the positions of the respective token in the sequence
Update llama.cpp
2024-01-08 14:51:29 -05:00
seq_id (ctypes.Array[ctypes.Array[llama_seq_id]]): the sequence to which the respective token belongs
feat: Update llama.cpp
2024-03-08 20:58:50 -05:00
logits (ctypes.Array[ctypes.ctypes.c_int8]): if zero, the logits for the respective token will not be output
Update llama.cpp
2024-01-08 14:51:29 -05:00
"""
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
if TYPE_CHECKING:
n_tokens: int
token: CtypesArray[llama_token]
embd: CtypesArray[ctypes.c_float]
pos: CtypesArray[CtypesArray[llama_pos]]
n_seq_id: CtypesArray[ctypes.c_int]
seq_id: CtypesArray[CtypesArray[llama_seq_id]]
logits: CtypesArray[ctypes.c_int8]
Update llama.cpp
2023-09-28 22:42:03 -04:00
_fields_ = [
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("n_tokens", ctypes.c_int32),
("token", ctypes.POINTER(llama_token)),
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
("embd", ctypes.POINTER(ctypes.c_float)),
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("pos", ctypes.POINTER(llama_pos)),
("n_seq_id", ctypes.POINTER(ctypes.c_int32)),
("seq_id", ctypes.POINTER(ctypes.POINTER(llama_seq_id))),
("logits", ctypes.POINTER(ctypes.c_int8)),
Update llama.cpp
2023-09-28 22:42:03 -04:00
]
Merge branch main into custom_rope
2023-07-15 15:11:01 -04:00
Update llama.cpp
2024-01-08 14:51:29 -05:00
Update llama.cpp
2023-12-11 10:21:35 -05:00
# enum llama_model_kv_override_type {
feat: Update llama.cpp
2024-02-25 16:53:58 -05:00
# LLAMA_KV_OVERRIDE_TYPE_INT,
# LLAMA_KV_OVERRIDE_TYPE_FLOAT,
# LLAMA_KV_OVERRIDE_TYPE_BOOL,
feat: Update llama.cpp
2024-04-27 23:41:54 -04:00
# LLAMA_KV_OVERRIDE_TYPE_STR,
Update llama.cpp
2023-12-11 10:21:35 -05:00
# };
feat: Update llama.cpp
2024-02-25 16:53:58 -05:00
LLAMA_KV_OVERRIDE_TYPE_INT = 0
LLAMA_KV_OVERRIDE_TYPE_FLOAT = 1
LLAMA_KV_OVERRIDE_TYPE_BOOL = 2
feat: Update llama.cpp
2024-04-27 23:41:54 -04:00
LLAMA_KV_OVERRIDE_TYPE_STR = 3
Update llama.cpp
2023-12-11 10:21:35 -05:00
Update llama.cpp
2024-01-08 14:51:29 -05:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# enum llama_model_meta_key {
# LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE,
# LLAMA_MODEL_META_KEY_SAMPLING_TOP_K,
# LLAMA_MODEL_META_KEY_SAMPLING_TOP_P,
# LLAMA_MODEL_META_KEY_SAMPLING_MIN_P,
# LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY,
# LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD,
# LLAMA_MODEL_META_KEY_SAMPLING_TEMP,
# LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N,
# LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT,
# LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT,
# LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU,
# LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA,
# };
LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE = 0
LLAMA_MODEL_META_KEY_SAMPLING_TOP_K = 1
LLAMA_MODEL_META_KEY_SAMPLING_TOP_P = 2
LLAMA_MODEL_META_KEY_SAMPLING_MIN_P = 3
LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY = 4
LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD = 5
LLAMA_MODEL_META_KEY_SAMPLING_TEMP = 6
LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N = 7
LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT = 8
LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT = 9
LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU = 10
LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA = 11
Update llama.cpp
2023-12-11 10:21:35 -05:00
# struct llama_model_kv_override {
# enum llama_model_kv_override_type tag;
feat: Update llama.cpp
2024-04-27 23:41:54 -04:00
# char key[128];
Update llama.cpp
2023-12-11 10:21:35 -05:00
# union {
feat: Update llama.cpp
2024-04-27 23:41:54 -04:00
# int64_t val_i64;
# double val_f64;
# bool val_bool;
# char val_str[128];
Update llama.cpp
2023-12-11 10:21:35 -05:00
# };
# };
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
class llama_model_kv_override_value(ctypes.Union):
Update llama.cpp
2023-12-11 10:21:35 -05:00
_fields_ = [
fix: adjust kv_override member names to match llama.cpp
2024-05-29 02:28:58 -04:00
("val_i64", ctypes.c_int64),
("val_f64", ctypes.c_double),
("val_bool", ctypes.c_bool),
("val_str", ctypes.c_char * 128),
Update llama.cpp
2023-12-11 10:21:35 -05:00
]
Update llama.cpp
2023-09-28 22:42:03 -04:00
feat: Update llama.cpp
2024-04-27 23:41:54 -04:00
if TYPE_CHECKING:
fix: adjust kv_override member names to match llama.cpp
2024-05-29 02:28:58 -04:00
val_i64: int
val_f64: float
val_bool: bool
val_str: bytes
feat: Update llama.cpp
2024-04-27 23:41:54 -04:00
Update llama.cpp
2024-01-08 14:51:29 -05:00
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
class llama_model_kv_override(ctypes.Structure):
fix: inccorect bindings for kv override. Based on #1011
2023-12-22 14:52:20 -05:00
_fields_ = [
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("tag", ctypes.c_int),
feat: Update llama.cpp
2024-04-27 23:41:54 -04:00
("key", ctypes.c_char * 128),
fix: inccorect bindings for kv override. Based on #1011
2023-12-22 14:52:20 -05:00
("value", llama_model_kv_override_value),
]
feat: Update llama.cpp
2024-04-27 23:41:54 -04:00
if TYPE_CHECKING:
tag: int
key: bytes
value: Union[int, float, bool, bytes]
Update llama.cpp
2024-01-08 14:51:29 -05:00
feat: Update llama.cpp
2025-04-11 03:28:20 -04:00
# struct llama_model_tensor_buft_override {
# const char * pattern;
# ggml_backend_buffer_type_t buft;
# };
Update llama.cpp
2023-09-28 22:42:03 -04:00
# struct llama_model_params {
feat: Update llama.cpp
2024-11-28 18:27:55 -05:00
# // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used)
# ggml_backend_dev_t * devices;
feat: Update llama.cpp
2025-04-11 03:28:20 -04:00
# // NULL-terminated list of buffer types to use for tensors that match a pattern
# const struct llama_model_tensor_buft_override * tensor_buft_overrides;
Update llama.cpp
2023-09-28 22:42:03 -04:00
# int32_t n_gpu_layers; // number of layers to store in VRAM
Update llama.cpp
2024-01-13 22:37:49 -05:00
# enum llama_split_mode split_mode; // how to split the model across multiple GPUs
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE
Update llama.cpp
2024-01-13 22:37:49 -05:00
# int32_t main_gpu;
Update llama.cpp
2024-01-31 10:41:42 -05:00
# // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()
Update llama.cpp
2024-01-13 22:37:49 -05:00
# const float * tensor_split;
Merge branch main into custom_rope
2023-07-15 15:11:01 -04:00
Update llama.cpp
2023-12-22 14:10:34 -05:00
# // Called with a progress value between 0.0 and 1.0. Pass NULL to disable.
# // If the provided progress_callback returns true, model loading continues.
# // If it returns false, model loading is immediately aborted.
Update llama.cpp
2023-06-20 11:25:10 -04:00
# llama_progress_callback progress_callback;
Update llama.cpp
2024-01-13 22:37:49 -05:00
Update llama.cpp
2023-06-20 11:25:10 -04:00
# // context pointer passed to the progress callback
# void * progress_callback_user_data;
Update llama.cpp
2023-05-21 17:47:21 -04:00
Update llama.cpp
2023-12-11 10:21:35 -05:00
# // override key-value pairs of the model meta data
# const struct llama_model_kv_override * kv_overrides;
Revert ctypes argtype change
2023-07-20 18:51:53 -04:00
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
Update llama.cpp
2023-06-20 11:25:10 -04:00
# // Keep the booleans together to avoid misalignment during copy-by-value.
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# bool vocab_only; // only load the vocabulary, no weights
# bool use_mmap; // use mmap if possible
# bool use_direct_io; // use direct io, takes precedence over use_mmap when supported
# bool use_mlock; // force system to keep model in RAM
# bool check_tensors; // validate model tensor data
feat: Update llama.cpp
2025-08-07 06:40:18 -07:00
# bool use_extra_bufts; // use extra buffer types (used for weight repacking)
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# bool no_host; // bypass host buffer allowing extra buffers to be used
# bool no_alloc; // only load metadata and simulate memory allocations
Update llama.cpp
2023-05-21 17:47:21 -04:00
# };
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
class llama_model_params(ctypes.Structure):
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
"""Parameters for llama_model
Update llama.cpp
2024-01-08 14:51:29 -05:00
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
Attributes:
feat: Update llama.cpp
2025-04-11 03:28:20 -04:00
devices (ctypes.Array[ggml_backend_dev_t]): NULL-terminated list of devices to use for offloading (if NULL, all available devices are used)
tensor_buft_overrides (ctypes.Array[llama_model_tensor_buft_override]): NULL-terminated list of buffer types to use for tensors that match a pattern
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
n_gpu_layers (int): number of layers to store in VRAM
Update llama.cpp
2024-01-13 22:37:49 -05:00
split_mode (int): how to split the model across multiple GPUs
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
main_gpu (int): the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()
Update llama.cpp
2023-12-22 14:10:34 -05:00
progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted.
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback
Update llama.cpp
2023-12-11 10:21:35 -05:00
kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
vocab_only (bool): only load the vocabulary, no weights
use_mmap (bool): use mmap if possible
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
use_direct_io (bool): use direct io, takes precedence over use_mmap when supported
feat: Update llama.cpp
2024-04-27 23:41:54 -04:00
use_mlock (bool): force system to keep model in RAM
feat: Update llama.cpp
2025-08-07 06:40:18 -07:00
check_tensors (bool): validate model tensor data
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
use_extra_bufts (bool): use extra buffer types (used for weight repacking)
no_host (bool): bypass host buffer allowing extra buffers to be used
no_alloc (bool): only load metadata and simulate memory allocations"""
Update llama.cpp
2024-01-08 14:51:29 -05:00
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
if TYPE_CHECKING:
feat: Update llama.cpp
2025-04-11 03:28:20 -04:00
devices: CtypesArray[ctypes.c_void_p] # NOTE: unused
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
tensor_buft_overrides: CtypesArray[
llama_model_tensor_buft_override
] # NOTE: unused
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
n_gpu_layers: int
split_mode: int
main_gpu: int
tensor_split: CtypesArray[ctypes.c_float]
progress_callback: Callable[[float, ctypes.c_void_p], bool]
progress_callback_user_data: ctypes.c_void_p
kv_overrides: CtypesArray[llama_model_kv_override]
vocab_only: bool
use_mmap: bool
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
use_direct_io: bool
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
use_mlock: bool
feat: Update llama.cpp
2024-04-27 23:41:54 -04:00
check_tensors: bool
feat: Update llama.cpp
2025-08-07 06:40:18 -07:00
use_extra_bufts: bool
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
no_host: bool
no_alloc: bool
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
Initial commit
2023-03-23 05:33:06 -04:00
_fields_ = [
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
("devices", ctypes.c_void_p), # NOTE: unnused
("tensor_buft_overrides", ctypes.c_void_p), # NOTE: unused
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("n_gpu_layers", ctypes.c_int32),
("split_mode", ctypes.c_int),
("main_gpu", ctypes.c_int32),
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
("tensor_split", ctypes.POINTER(ctypes.c_float)),
Update llama.cpp
2023-06-20 11:25:10 -04:00
("progress_callback", llama_progress_callback),
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("progress_callback_user_data", ctypes.c_void_p),
("kv_overrides", ctypes.POINTER(llama_model_kv_override)),
("vocab_only", ctypes.c_bool),
("use_mmap", ctypes.c_bool),
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
("use_direct_io", ctypes.c_bool),
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("use_mlock", ctypes.c_bool),
feat: Update llama.cpp
2024-04-27 23:41:54 -04:00
("check_tensors", ctypes.c_bool),
feat: Update llama.cpp
2025-08-07 06:40:18 -07:00
("use_extra_bufts", ctypes.c_bool),
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
("no_host", ctypes.c_bool),
("no_alloc", ctypes.c_bool),
]
# struct llama_sampler_seq_config {
# llama_seq_id seq_id;
# struct llama_sampler * sampler;
# };
class llama_sampler_seq_config(ctypes.Structure):
if TYPE_CHECKING:
seq_id: int
sampler: ctypes.c_void_p
_fields_ = [
("seq_id", llama_seq_id),
("sampler", ctypes.c_void_p),
Initial commit
2023-03-23 05:33:06 -04:00
]
Black formatting
2023-03-24 14:35:41 -04:00
feat: Update llama.cpp
2024-05-27 10:51:57 -04:00
# // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // https://github.com/ggml-org/llama.cpp/pull/7544
Update llama.cpp
2023-09-28 22:42:03 -04:00
# struct llama_context_params {
Update llama.cpp
2023-11-03 11:34:50 -04:00
# uint32_t n_ctx; // text context, 0 = from model
feat: Update llama.cpp
2024-03-13 15:57:35 -04:00
# uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode
# uint32_t n_ubatch; // physical maximum batch size
# uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models)
feat: Update llama.cpp
2024-08-30 00:33:58 -04:00
# int32_t n_threads; // number of threads to use for generation
# int32_t n_threads_batch; // number of threads to use for batch processing
feat: Update llama.cpp
2024-03-03 11:27:04 -05:00
# enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type`
# enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id
feat: Update llama.cpp
2024-07-05 20:44:46 -04:00
# enum llama_attention_type attention_type; // attention type to use for embeddings
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# enum llama_flash_attn_type flash_attn_type; // when to enable Flash Attention
Update llama.cpp
2023-09-28 22:42:03 -04:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // ref: https://github.com/ggml-org/llama.cpp/pull/2054
Update llama.cpp Fix build examples Exclude examples directory Revert cmake changes Try actions/checkout@v4 Try to update submodules Revert Update llama.cpp Fix build examples Exclude examples directory Revert cmake changes Try actions/checkout@v4 Try to update submodules Revert
2023-11-02 13:40:20 -04:00
# float rope_freq_base; // RoPE base frequency, 0 = from model
# float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model
Update llama.cpp
2023-11-26 15:38:22 -05:00
# float yarn_ext_factor; // YaRN extrapolation mix factor, negative = from model
Update llama.cpp Fix build examples Exclude examples directory Revert cmake changes Try actions/checkout@v4 Try to update submodules Revert Update llama.cpp Fix build examples Exclude examples directory Revert cmake changes Try actions/checkout@v4 Try to update submodules Revert
2023-11-02 13:40:20 -04:00
# float yarn_attn_factor; // YaRN magnitude scaling factor
# float yarn_beta_fast; // YaRN low correction dim
# float yarn_beta_slow; // YaRN high correction dim
# uint32_t yarn_orig_ctx; // YaRN original context size
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# float defrag_thold; // defragment the KV cache if holes/size > thold, <= 0 disabled (default)
Update llama.cpp
2023-09-28 22:42:03 -04:00
Update llama.cpp
2024-01-18 21:21:49 -05:00
# ggml_backend_sched_eval_callback cb_eval;
# void * cb_eval_user_data;
feat: Update llama.cpp
2024-05-27 10:51:57 -04:00
# enum ggml_type type_k; // data type for K cache [EXPERIMENTAL]
# enum ggml_type type_v; // data type for V cache [EXPERIMENTAL]
Update llama.cpp
2023-09-28 22:42:03 -04:00
feat: Update llama.cpp
2024-03-02 22:20:04 -05:00
# // Abort callback
# // if it returns true, execution of llama_decode() will be aborted
# // currently works only with CPU execution
# ggml_abort_callback abort_callback;
# void * abort_callback_data;
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value.
# bool embeddings; // if true, extract embeddings (together with logits)
# bool offload_kqv; // offload the KQV ops (including the KV cache) to GPU
# bool no_perf; // measure performance timings
# bool op_offload; // offload host tensor operations to device
# bool swa_full; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)
# // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases
# // ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573
feat: Update llama.cpp
2025-08-07 06:40:18 -07:00
# bool kv_unified; // use a unified buffer across the input sequences when computing the attention
# // try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix
# // ref: https://github.com/ggml-org/llama.cpp/pull/14363
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# struct llama_sampler_seq_config * samplers;
# size_t n_samplers;
Update llama.cpp
2023-09-28 22:42:03 -04:00
# };
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
class llama_context_params(ctypes.Structure):
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
"""Parameters for llama_context
Update llama.cpp
2024-01-08 14:51:29 -05:00
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
Attributes:
n_ctx (int): text context, 0 = from model
feat: Update llama.cpp
2024-03-13 15:57:35 -04:00
n_batch (int): logical maximum batch size that can be submitted to llama_decode
n_ubatch (int): physical maximum batch size
n_seq_max (int): max number of sequences (i.e. distinct states for recurrent models)
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
n_threads (int): number of threads to use for generation
n_threads_batch (int): number of threads to use for batch processing
rope_scaling_type (int): RoPE scaling type, from `enum llama_rope_scaling_type`
feat: Update llama.cpp
2024-03-03 11:27:04 -05:00
pooling_type (int): whether to pool (sum) embedding results by sequence id (ignored if no pooling layer)
feat: Update llama.cpp
2024-07-05 20:44:46 -04:00
attention_type (int): attention type to use for embeddings
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
flash_attn_type (int): when to enable flash attention
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
rope_freq_base (float): RoPE base frequency, 0 = from model
rope_freq_scale (float): RoPE frequency scaling factor, 0 = from model
yarn_ext_factor (float): YaRN extrapolation mix factor, negative = from model
yarn_attn_factor (float): YaRN magnitude scaling factor
yarn_beta_fast (float): YaRN low correction dim
yarn_beta_slow (float): YaRN high correction dim
yarn_orig_ctx (int): YaRN original context size
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
defrag_thold (float): defragment the KV cache if holes/size > thold, <= 0 disabled (default)
Update llama.cpp
2024-01-18 21:21:49 -05:00
cb_eval (ggml_backend_sched_eval_callback): callback for scheduling eval
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
cb_eval_user_data (ctypes.ctypes.c_void_p): user data for cb_eval
Update llama.cpp
2023-12-11 10:21:35 -05:00
type_k (int): data type for K cache
type_v (int): data type for V cache
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted
abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback
Update llama.cpp
2024-03-06 01:32:00 -05:00
embeddings (bool): if true, extract embeddings (together with logits)
Update llama.cpp
2024-01-08 14:51:29 -05:00
offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU
fix: error showing time spent in llama perf context print (#1898) * feat: Sync with llama.cpp Add `no_perf` field to `llama_context_params` to optionally disable performance timing measurements. * fix: Display performance metrics by default --------- Co-authored-by: Andrei <abetlen@gmail.com>
2025-01-29 11:57:37 +08:00
no_perf (bool): whether to measure performance timings
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
op_offload (bool): offload host tensor operations to device
swa_full (bool): use full-size SWA cache
feat: Update llama.cpp
2025-08-07 06:40:18 -07:00
kv_unified (bool): use a unified buffer across the input sequences when computing the attention
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
samplers (ctypes.POINTER(llama_sampler_seq_config)): backend sampler chain configuration
n_samplers (int): number of backend sampler chain configurations
Update llama.cpp
2024-01-08 14:51:29 -05:00
"""
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
if TYPE_CHECKING:
n_ctx: int
n_batch: int
n_ubatch: int
n_seq_max: int
n_threads: int
n_threads_batch: int
rope_scaling_type: int
pooling_type: int
feat: Update llama.cpp
2024-07-05 20:44:46 -04:00
attention_type: int
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
flash_attn_type: int
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
rope_freq_base: float
rope_freq_scale: float
yarn_ext_factor: float
yarn_attn_factor: float
yarn_beta_fast: float
yarn_beta_slow: float
yarn_orig_ctx: int
defrag_thold: float
cb_eval: Callable[[ctypes.c_void_p, bool], bool]
cb_eval_user_data: ctypes.c_void_p
type_k: int
type_v: int
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
abort_callback: Callable[[ctypes.c_void_p], bool]
abort_callback_data: ctypes.c_void_p
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
embeddings: bool
offload_kqv: bool
fix: error showing time spent in llama perf context print (#1898) * feat: Sync with llama.cpp Add `no_perf` field to `llama_context_params` to optionally disable performance timing measurements. * fix: Display performance metrics by default --------- Co-authored-by: Andrei <abetlen@gmail.com>
2025-01-29 11:57:37 +08:00
no_perf: bool
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
op_offload: bool
swa_full: bool
feat: Update llama.cpp
2025-08-07 06:40:18 -07:00
kv_unified: bool
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
samplers: ctypes.POINTER(llama_sampler_seq_config)
n_samplers: int
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
Update llama.cpp
2023-09-28 22:42:03 -04:00
_fields_ = [
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("n_ctx", ctypes.c_uint32),
("n_batch", ctypes.c_uint32),
feat: Update llama.cpp
2024-03-13 15:57:35 -04:00
("n_ubatch", ctypes.c_uint32),
("n_seq_max", ctypes.c_uint32),
feat: Update llama.cpp
2024-08-30 00:33:58 -04:00
("n_threads", ctypes.c_int32),
("n_threads_batch", ctypes.c_int32),
feat: Update llama.cpp
2024-03-03 11:27:04 -05:00
("rope_scaling_type", ctypes.c_int),
("pooling_type", ctypes.c_int),
feat: Update llama.cpp
2024-07-05 20:44:46 -04:00
("attention_type", ctypes.c_int),
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
("flash_attn_type", ctypes.c_int),
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("rope_freq_base", ctypes.c_float),
("rope_freq_scale", ctypes.c_float),
("yarn_ext_factor", ctypes.c_float),
("yarn_attn_factor", ctypes.c_float),
("yarn_beta_fast", ctypes.c_float),
("yarn_beta_slow", ctypes.c_float),
("yarn_orig_ctx", ctypes.c_uint32),
feat: Update llama.cpp
2024-02-27 12:22:17 -05:00
("defrag_thold", ctypes.c_float),
Update llama.cpp
2024-01-18 21:21:49 -05:00
("cb_eval", ggml_backend_sched_eval_callback),
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("cb_eval_user_data", ctypes.c_void_p),
("type_k", ctypes.c_int),
("type_v", ctypes.c_int),
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
("abort_callback", ggml_abort_callback),
("abort_callback_data", ctypes.c_void_p),
Update llama.cpp
2024-03-06 01:32:00 -05:00
("embeddings", ctypes.c_bool),
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("offload_kqv", ctypes.c_bool),
fix: error showing time spent in llama perf context print (#1898) * feat: Sync with llama.cpp Add `no_perf` field to `llama_context_params` to optionally disable performance timing measurements. * fix: Display performance metrics by default --------- Co-authored-by: Andrei <abetlen@gmail.com>
2025-01-29 11:57:37 +08:00
("no_perf", ctypes.c_bool),
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
("op_offload", ctypes.c_bool),
("swa_full", ctypes.c_bool),
feat: Update llama.cpp
2025-08-07 06:40:18 -07:00
("kv_unified", ctypes.c_bool),
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
("samplers", ctypes.POINTER(llama_sampler_seq_config)),
("n_samplers", ctypes.c_size_t),
Update llama.cpp
2023-09-28 22:42:03 -04:00
]
Initial commit
2023-03-23 05:33:06 -04:00
Update llama.cpp
2023-08-24 00:17:00 -04:00
# // Signature for logging events
# // Note that text includes the new line character at the end for most events.
# // If your logging mechanism cannot handle that, check if the last character is '\n' and strip it
# // if it exists.
# // It might not exist for progress report where '.' is output repeatedly.
# typedef void (*llama_log_callback)(enum llama_log_level level, const char * text, void * user_data);
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
llama_log_callback = ctypes.CFUNCTYPE(
None, ctypes.c_int, ctypes.c_char_p, ctypes.c_void_p
)
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Signature for logging events
Note that text includes the new line character at the end for most events.
If your logging mechanism cannot handle that, check if the last character is '\n' and strip it
if it exists.
It might not exist for progress report where '.' is output repeatedly."""
Initial commit
2023-03-23 05:33:06 -04:00
Black formatting
2023-03-24 14:35:41 -04:00
Update llama.cpp
2023-06-10 12:17:38 -04:00
# // model quantization parameters
# typedef struct llama_model_quantize_params {
feat: Update llama.cpp
2025-05-08 06:52:52 -04:00
# int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()
# enum llama_ftype ftype; // quantize to this llama_ftype
# enum ggml_type output_tensor_type; // output tensor type
# enum ggml_type token_embedding_type; // token embeddings tensor type
# bool allow_requantize; // allow quantizing non-f32/f16 tensors
# bool quantize_output_tensor; // quantize output.weight
# bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored
# bool pure; // quantize all tensors to the default type
# bool keep_split; // quantize to the same number of shards
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# bool dry_run; // calculate and show the final quantization size without performing quantization
feat: Update llama.cpp
2025-05-08 06:52:52 -04:00
# void * imatrix; // pointer to importance matrix data
# void * kv_overrides; // pointer to vector containing overrides
# void * tensor_types; // pointer to vector containing tensor types
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# void * prune_layers; // pointer to vector containing layer indices to prune
Update llama.cpp
2023-06-10 12:17:38 -04:00
# } llama_model_quantize_params;
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
class llama_model_quantize_params(ctypes.Structure):
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
"""Parameters for llama_model_quantize
Update llama.cpp
2024-01-08 14:51:29 -05:00
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
Attributes:
nthread (int): number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()
ftype (int): quantize to this llama_ftype
feat: Update llama.cpp
2024-03-22 23:43:29 -04:00
output_tensor_type (int): output tensor type
feat: Update llama.cpp
2024-08-06 20:21:18 -04:00
token_embedding_type (int): token embeddings tensor type
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
allow_requantize (bool): allow quantizing non-f32/f16 tensors
quantize_output_tensor (bool): quantize output.weight
only_copy (bool): only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored
feat: Update llama.cpp
2024-03-13 15:57:35 -04:00
pure (bool): quantize all tensors to the default type
feat: Update llama.cpp
2024-04-25 21:21:29 -04:00
keep_split (bool): quantize to the same number of shards
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
dry_run (bool): calculate and show the final quantization size without performing quantization
feat: Update llama.cpp
2024-03-22 23:43:29 -04:00
imatrix (ctypes.c_void_p): pointer to importance matrix data
feat: Update llama.cpp
2024-03-26 22:58:53 -04:00
kv_overrides (ctypes.c_void_p): pointer to vector containing overrides
feat: Update llama.cpp
2025-05-08 06:52:52 -04:00
tensor_types (ctypes.c_void_p): pointer to vector containing tensor types
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
prune_layers (ctypes.c_void_p): pointer to vector containing layer indices to prune
Update llama.cpp
2024-01-08 14:51:29 -05:00
"""
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
if TYPE_CHECKING:
nthread: int
ftype: int
output_tensor_type: int
token_embedding_type: int
allow_requantize: bool
quantize_output_tensor: bool
only_copy: bool
pure: bool
feat: Update llama.cpp
2024-04-25 21:21:29 -04:00
keep_split: bool
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
dry_run: bool
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
imatrix: ctypes.c_void_p
kv_overrides: ctypes.c_void_p
feat: Update llama.cpp
2025-05-08 06:52:52 -04:00
tensor_types: ctypes.c_void_p
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
prune_layers: ctypes.c_void_p
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
Update llama.cpp
2023-06-10 12:17:38 -04:00
_fields_ = [
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("nthread", ctypes.c_int32),
("ftype", ctypes.c_int),
feat: Update llama.cpp
2024-03-22 23:43:29 -04:00
("output_tensor_type", ctypes.c_int),
("token_embedding_type", ctypes.c_int),
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("allow_requantize", ctypes.c_bool),
("quantize_output_tensor", ctypes.c_bool),
("only_copy", ctypes.c_bool),
("pure", ctypes.c_bool),
feat: Update llama.cpp
2024-04-25 21:21:29 -04:00
("keep_split", ctypes.c_bool),
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
("dry_run", ctypes.c_bool),
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("imatrix", ctypes.c_void_p),
feat: Update llama.cpp
2024-03-26 22:58:53 -04:00
("kv_overrides", ctypes.c_void_p),
feat: Update llama.cpp
2025-05-08 06:52:52 -04:00
("tensor_types", ctypes.c_void_p),
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
("prune_layers", ctypes.c_void_p),
Update llama.cpp
2023-06-10 12:17:38 -04:00
]
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# typedef struct llama_logit_bias {
# llama_token token;
# float bias;
# } llama_logit_bias;
class llama_logit_bias(ctypes.Structure):
"""Used to store logit bias
Update llama.cpp
2023-07-24 13:04:34 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
Attributes:
token (llama_token): token id
bias (float): bias"""
feat: Update llama.cpp
2024-06-07 02:02:12 -04:00
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
if TYPE_CHECKING:
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
token: llama_token
bias: float
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
Update llama.cpp
2023-07-24 13:04:34 -04:00
_fields_ = [
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
("token", llama_token),
("bias", ctypes.c_float),
Update llama.cpp
2023-07-24 13:04:34 -04:00
]
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
llama_logit_bias_p = ctypes.POINTER(llama_logit_bias)
Update llama.cpp
2023-07-24 13:04:34 -04:00
Update llama.cpp
2023-07-06 17:57:56 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# typedef struct llama_sampler_chain_params {
# bool no_perf; // whether to measure performance timings
# } llama_sampler_chain_params;
class llama_sampler_chain_params(ctypes.Structure):
"""Parameters for llama_sampler_chain
Attributes:
no_perf (bool): whether to measure performance timings"""
Update llama.cpp
2023-07-06 17:57:56 -04:00
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
if TYPE_CHECKING:
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
no_perf: bool
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
Update llama.cpp
2023-07-06 17:57:56 -04:00
_fields_ = [
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
("no_perf", ctypes.c_bool),
Update llama.cpp
2023-07-06 17:57:56 -04:00
]
feat: Update llama.cpp
2024-02-19 04:11:34 -05:00
# // used in chat template
# typedef struct llama_chat_message {
# const char * role;
# const char * content;
# } llama_chat_message;
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
class llama_chat_message(ctypes.Structure):
feat: Update llama.cpp
2024-02-19 04:11:34 -05:00
_fields_ = [
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
("role", ctypes.c_char_p),
("content", ctypes.c_char_p),
feat: Update llama.cpp
2024-02-19 04:11:34 -05:00
]
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
# // lora adapter
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# struct llama_adapter_lora;
llama_adapter_lora_p = ctypes.c_void_p
llama_adapter_lora_p_ctypes = ctypes.POINTER(ctypes.c_void_p)
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
Update llama.cpp
2023-09-28 22:42:03 -04:00
# // Helpers for getting default parameters
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API struct llama_model_params llama_model_default_params(void);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_model_default_params",
[],
llama_model_params,
)
Update llama.cpp
2023-09-28 22:42:03 -04:00
def llama_model_default_params() -> llama_model_params:
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
"""Get default parameters for llama_model"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-09-28 22:42:03 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API struct llama_context_params llama_context_default_params(void);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_context_default_params",
[],
llama_context_params,
)
Initial commit
2023-03-23 05:33:06 -04:00
def llama_context_default_params() -> llama_context_params:
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
"""Get default parameters for llama_context"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Initial commit
2023-03-23 05:33:06 -04:00
Black formatting
2023-03-24 14:35:41 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API struct llama_sampler_chain_params llama_sampler_chain_default_params(void);
@ctypes_function(
"llama_sampler_chain_default_params",
[],
llama_sampler_chain_params,
)
def llama_sampler_chain_default_params() -> llama_sampler_chain_params:
"""Get default parameters for llama_sampler_chain"""
...
Update llama.cpp
2023-08-24 00:17:00 -04:00
# LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_model_quantize_default_params",
[],
llama_model_quantize_params,
)
Update llama.cpp
2023-06-10 12:17:38 -04:00
def llama_model_quantize_default_params() -> llama_model_quantize_params:
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
"""Get default parameters for llama_model_quantize"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-06-10 12:17:38 -04:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type);
@ctypes_function("llama_flash_attn_type_name", [ctypes.c_int], ctypes.c_char_p)
def llama_flash_attn_type_name(flash_attn_type: int, /) -> Optional[bytes]:
"""Get the flash attention type name."""
...
Update llama.cpp
2023-05-21 17:47:21 -04:00
# // Initialize the llama + ggml backend
Update llama.cpp
2023-06-29 01:08:15 -04:00
# // If numa is true, use NUMA optimizations
Update llama.cpp
2023-05-21 17:47:21 -04:00
# // Call once at the start of the program
feat: Update llama.cpp
2024-02-17 00:37:51 -05:00
# LLAMA_API void llama_backend_init(void);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_backend_init",
[],
None,
)
feat: Update llama.cpp
2024-02-17 00:37:51 -05:00
def llama_backend_init():
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Initialize the llama + ggml backend
Call once at the start of the program"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Merge branch main into custom_rope
2023-07-15 15:11:01 -04:00
feat: Update llama.cpp
2024-02-17 00:37:51 -05:00
# // numa strategies
# enum ggml_numa_strategy {
# GGML_NUMA_STRATEGY_DISABLED = 0,
# GGML_NUMA_STRATEGY_DISTRIBUTE = 1,
# GGML_NUMA_STRATEGY_ISOLATE = 2,
# GGML_NUMA_STRATEGY_NUMACTL = 3,
# GGML_NUMA_STRATEGY_MIRROR = 4,
# GGML_NUMA_STRATEGY_COUNT
# };
GGML_NUMA_STRATEGY_DISABLED = 0
GGML_NUMA_STRATEGY_DISTRIBUTE = 1
GGML_NUMA_STRATEGY_ISOLATE = 2
GGML_NUMA_STRATEGY_NUMACTL = 3
GGML_NUMA_STRATEGY_MIRROR = 4
GGML_NUMA_STRATEGY_COUNT = 5
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# // Call once at the end of the program - currently only used for MPI
# LLAMA_API void llama_backend_free(void);
@ctypes_function(
"llama_backend_free",
[],
None,
)
def llama_backend_free():
"""Call once at the end of the program - currently only used for MPI"""
...
feat: Update llama.cpp
2024-02-17 00:37:51 -05:00
# //optional:
# LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_numa_init",
[ctypes.c_int],
None,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_numa_init(numa: int, /): ...
feat: Update llama.cpp
2024-02-17 00:37:51 -05:00
feat: Update llama.cpp
2024-08-30 00:33:58 -04:00
# // Optional: an auto threadpool gets created in ggml if not passed explicitly
# LLAMA_API void llama_attach_threadpool(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# struct llama_context * ctx,
# ggml_threadpool_t threadpool,
# ggml_threadpool_t threadpool_batch);
# TODO: Add llama_attach_threadpool
feat: Update llama.cpp
2024-08-30 00:33:58 -04:00
# LLAMA_API void llama_detach_threadpool(struct llama_context * ctx);
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# TODO: Add llama_detach_threadpool
Update llama.cpp
2023-05-21 17:47:21 -04:00
Merge branch main into custom_rope
2023-07-15 15:11:01 -04:00
feat: Update llama.cpp
2025-01-08 16:33:53 -05:00
# DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file(
Update llama.cpp
2023-08-24 00:17:00 -04:00
# const char * path_model,
feat: Update llama.cpp
2025-01-08 16:33:53 -05:00
# struct llama_model_params params),
# "use llama_model_load_from_file instead");
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_load_model_from_file",
[ctypes.c_char_p, llama_model_params],
llama_model_p_ctypes,
)
Update llama.cpp
2023-06-26 08:50:38 -04:00
def llama_load_model_from_file(
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
path_model: bytes, params: llama_model_params, /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> Optional[llama_model_p]: ...
Update llama.cpp
2023-06-26 08:50:38 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# // Load the model from a file
# // If the file is split into multiple parts, the file name must follow this pattern: <name>-%05d-of-%05d.gguf
# // If the split file name does not follow this pattern, use llama_model_load_from_splits
feat: Update llama.cpp
2025-01-08 16:33:53 -05:00
# LLAMA_API struct llama_model * llama_model_load_from_file(
# const char * path_model,
# struct llama_model_params params);
@ctypes_function(
"llama_model_load_from_file",
[ctypes.c_char_p, llama_model_params],
llama_model_p_ctypes,
)
def llama_model_load_from_file(
path_model: bytes, params: llama_model_params, /
) -> Optional[llama_model_p]:
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
"""Load the model from a file
If the file is split into multiple parts, the file name must follow this pattern: <name>-%05d-of-%05d.gguf
If the split file name does not follow this pattern, use llama_model_load_from_splits"""
...
# // Load the model from multiple splits (support custom naming scheme)
# // The paths must be in the correct order
# LLAMA_API struct llama_model * llama_model_load_from_splits(
# const char ** paths,
# size_t n_paths,
# struct llama_model_params params);
@ctypes_function(
"llama_model_load_from_splits",
[ctypes.POINTER(ctypes.c_char_p), ctypes.c_size_t, llama_model_params],
llama_model_p_ctypes,
)
def llama_model_load_from_splits(
paths: List[bytes], n_paths: int, params: llama_model_params, /
) -> Optional[llama_model_p]:
"""Load the model from multiple splits (support custom naming scheme)
The paths must be in the correct order"""
feat: Update llama.cpp
2025-01-08 16:33:53 -05:00
...
feat: Update llama.cpp to ggerganov/llama.cpp@c0159f9c1f874da15e94f371d136f5920b4b5335 (#2161) * Update llama.cpp to c0159f9c1 * Add changelog entry for llama.cpp update
2026-03-25 15:21:44 -07:00
# // Load a model from an open FILE pointer
# LLAMA_API struct llama_model * llama_model_load_from_file_ptr(
# FILE * file,
# struct llama_model_params params);
@ctypes_function(
"llama_model_load_from_file_ptr",
[ctypes.c_void_p, llama_model_params],
llama_model_p_ctypes,
)
def llama_model_load_from_file_ptr(
file: ctypes.c_void_p, params: llama_model_params, /
) -> Optional[llama_model_p]:
"""Load a model from an open FILE pointer."""
...
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# LLAMA_API void llama_model_save_to_file(
# const struct llama_model * model,
# const char * path_model);
@ctypes_function(
"llama_model_save_to_file",
[llama_model_p_ctypes, ctypes.c_char_p],
None,
)
def llama_model_save_to_file(model: llama_model_p, path_model: bytes, /):
"""Save the model to a file"""
...
# DEPRECATED(LLAMA_API void llama_free_model(struct llama_model * model),
# "use llama_model_free instead");
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_free_model",
[llama_model_p_ctypes],
None,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_free_model(model: llama_model_p, /): ...
Update llama.cpp
2023-06-26 08:50:38 -04:00
feat: Update llama.cpp
2025-01-08 16:33:53 -05:00
# LLAMA_API void llama_model_free(struct llama_model * model);
@ctypes_function(
"llama_model_free",
[llama_model_p_ctypes],
None,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_model_free(model: llama_model_p, /): ...
feat: Update llama.cpp
2025-01-08 16:33:53 -05:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# typedef void (*llama_model_set_tensor_data_t)(struct ggml_tensor * tensor, void * userdata);
llama_model_set_tensor_data_t = ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_void_p)
# LLAMA_API struct llama_model * llama_model_init_from_user(
# struct gguf_context * metadata,
# llama_model_set_tensor_data_t set_tensor_data,
# void * set_tensor_data_ud,
# struct llama_model_params params);
@ctypes_function(
"llama_model_init_from_user",
[
gguf_context_p_ctypes,
llama_model_set_tensor_data_t,
ctypes.c_void_p,
llama_model_params,
],
llama_model_p_ctypes,
)
def llama_model_init_from_user(
metadata: gguf_context_p,
set_tensor_data: llama_model_set_tensor_data_t,
set_tensor_data_ud: ctypes.c_void_p,
params: llama_model_params,
/,
) -> Optional[llama_model_p]:
"""Initialize a model from user-provided metadata and tensor data."""
...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API struct llama_context * llama_init_from_model(
Update llama.cpp
2023-08-24 00:17:00 -04:00
# struct llama_model * model,
Update llama.cpp
2023-06-26 08:50:38 -04:00
# struct llama_context_params params);
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
@ctypes_function(
"llama_init_from_model",
[llama_model_p_ctypes, llama_context_params],
llama_context_p_ctypes,
)
def llama_init_from_model(
model: llama_model_p, params: llama_context_params, /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> Optional[llama_context_p]: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API struct llama_context * llama_new_context_with_model(
# struct llama_model * model,
# struct llama_context_params params),
# "use llama_init_from_model instead");
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_new_context_with_model",
[llama_model_p_ctypes, llama_context_params],
llama_context_p_ctypes,
)
Update llama.cpp
2023-06-26 08:50:38 -04:00
def llama_new_context_with_model(
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
model: llama_model_p, params: llama_context_params, /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> Optional[llama_context_p]: ...
Update llama.cpp
2023-06-26 08:50:38 -04:00
Update llama.cpp
2023-08-24 00:17:00 -04:00
# // Frees all allocated memory
# LLAMA_API void llama_free(struct llama_context * ctx);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_free",
[llama_context_p_ctypes],
None,
)
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
def llama_free(ctx: llama_context_p, /):
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Frees all allocated memory"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-08-24 00:17:00 -04:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# enum llama_params_fit_status {
# LLAMA_PARAMS_FIT_STATUS_SUCCESS = 0,
# LLAMA_PARAMS_FIT_STATUS_FAILURE = 1,
# LLAMA_PARAMS_FIT_STATUS_ERROR = 2,
# };
LLAMA_PARAMS_FIT_STATUS_SUCCESS = 0
LLAMA_PARAMS_FIT_STATUS_FAILURE = 1
LLAMA_PARAMS_FIT_STATUS_ERROR = 2
# LLAMA_API enum llama_params_fit_status llama_params_fit(
# const char * path_model,
# struct llama_model_params * mparams,
# struct llama_context_params * cparams,
# float * tensor_split,
# struct llama_model_tensor_buft_override * tensor_buft_overrides,
# size_t * margins,
# uint32_t n_ctx_min,
# enum ggml_log_level log_level);
@ctypes_function(
"llama_params_fit",
[
ctypes.c_char_p,
ctypes.POINTER(llama_model_params),
ctypes.POINTER(llama_context_params),
ctypes.POINTER(ctypes.c_float),
ctypes.c_void_p,
ctypes.POINTER(ctypes.c_size_t),
ctypes.c_uint32,
ctypes.c_int,
],
ctypes.c_int,
)
def llama_params_fit(
path_model: bytes,
mparams: CtypesPointerOrRef[llama_model_params],
cparams: CtypesPointerOrRef[llama_context_params],
tensor_split: Optional[CtypesPointer[ctypes.c_float]],
tensor_buft_overrides: ctypes.c_void_p,
margins: Optional[CtypesPointer[ctypes.c_size_t]],
n_ctx_min: int,
log_level: int,
/,
) -> int:
"""Fit model and context parameters for a model path."""
...
Update llama.cpp
2023-08-24 00:17:00 -04:00
# LLAMA_API int64_t llama_time_us(void);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_time_us",
[],
ctypes.c_int64,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_time_us() -> int: ...
Update llama.cpp
2023-05-21 17:47:21 -04:00
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
# LLAMA_API size_t llama_max_devices(void);
@ctypes_function("llama_max_devices", [], ctypes.c_size_t)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_max_devices() -> int: ...
Initial commit
2023-03-23 05:33:06 -04:00
Black formatting
2023-03-24 14:35:41 -04:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# LLAMA_API size_t llama_max_parallel_sequences(void);
@ctypes_function("llama_max_parallel_sequences", [], ctypes.c_size_t)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_max_parallel_sequences() -> int: ...
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# LLAMA_API size_t llama_max_tensor_buft_overrides(void);
@ctypes_function("llama_max_tensor_buft_overrides", [], ctypes.c_size_t)
def llama_max_tensor_buft_overrides() -> int:
"""Get the maximum number of tensor buffer type overrides."""
...
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
# LLAMA_API bool llama_supports_mmap (void);
@ctypes_function("llama_supports_mmap", [], ctypes.c_bool)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_supports_mmap() -> bool: ...
Update llama.cpp
2024-01-31 10:41:42 -05:00
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
# LLAMA_API bool llama_supports_mlock (void);
@ctypes_function("llama_supports_mlock", [], ctypes.c_bool)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_supports_mlock() -> bool: ...
Update llama.cpp
2024-01-31 10:41:42 -05:00
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
# LLAMA_API bool llama_supports_gpu_offload(void);
@ctypes_function("llama_supports_gpu_offload", [], ctypes.c_bool)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_supports_gpu_offload() -> bool: ...
Update llama.cpp
2023-08-24 00:17:00 -04:00
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
# LLAMA_API bool llama_supports_rpc (void);
@ctypes_function("llama_supports_rpc", [], ctypes.c_bool)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_supports_rpc() -> bool: ...
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
# LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx);
@ctypes_function("llama_n_ctx", [llama_context_p_ctypes], ctypes.c_uint32)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_n_ctx(ctx: llama_context_p, /) -> int: ...
Update llama.cpp
2023-08-24 00:17:00 -04:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# LLAMA_API uint32_t llama_n_ctx_seq (const struct llama_context * ctx);
@ctypes_function("llama_n_ctx_seq", [llama_context_p_ctypes], ctypes.c_uint32)
def llama_n_ctx_seq(ctx: llama_context_p, /) -> int:
"""Get the context size per sequence."""
...
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
# LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx);
@ctypes_function("llama_n_batch", [llama_context_p_ctypes], ctypes.c_uint32)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_n_batch(ctx: llama_context_p, /) -> int: ...
feat: Update llama.cpp
2024-03-13 15:57:35 -04:00
# LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx);
@ctypes_function("llama_n_ubatch", [llama_context_p_ctypes], ctypes.c_uint32)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_n_ubatch(ctx: llama_context_p, /) -> int: ...
feat: Update llama.cpp
2024-03-13 15:57:35 -04:00
# LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx);
@ctypes_function("llama_n_seq_max", [llama_context_p_ctypes], ctypes.c_uint32)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_n_seq_max(ctx: llama_context_p, /) -> int: ...
Update llama.cpp
2023-08-24 18:01:42 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead");
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_n_ctx_train(model: llama_model_p, /) -> int: ...
Update llama.cpp
2023-09-09 12:12:32 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead");
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_n_embd(model: llama_model_p, /) -> int: ...
Update llama.cpp
2023-08-24 00:17:00 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API int32_t llama_n_layer (const struct llama_model * model), "use llama_model_n_layer instead");
feat: Update llama.cpp
2024-03-18 10:26:36 -04:00
@ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_n_layer(model: llama_model_p, /) -> int: ...
feat: Update llama.cpp
2024-09-18 20:07:48 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API int32_t llama_n_head (const struct llama_model * model), "use llama_model_n_head instead");
feat: Update llama.cpp
2024-09-18 20:07:48 -04:00
@ctypes_function("llama_n_head", [llama_model_p_ctypes], ctypes.c_int32)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_n_head(model: llama_model_p, /) -> int: ...
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead");
@ctypes_function("llama_n_vocab", [llama_vocab_p_ctypes], ctypes.c_int32)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_n_vocab(model: llama_vocab_p, /) -> int: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx);
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
@ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ...
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx);
@ctypes_function("llama_get_memory", [llama_context_p_ctypes], llama_memory_t_ctypes)
def llama_get_memory(ctx: llama_context_p, /) -> Optional[llama_memory_t]:
"""Get the memory for the context"""
feat: Update llama.cpp
2025-04-11 03:28:20 -04:00
...
# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx);
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
@ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_pooling_type(ctx: llama_context_p, /) -> int: ...
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model);
@ctypes_function("llama_model_get_vocab", [llama_model_p_ctypes], llama_vocab_p_ctypes)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_model_get_vocab(model: llama_model_p, /) -> Optional[llama_vocab_p]: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API enum llama_rope_type llama_model_rope_type(const struct llama_model * model);
@ctypes_function("llama_model_rope_type", [llama_model_p_ctypes], ctypes.c_int)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_model_rope_type(model: llama_model_p, /) -> int: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API int32_t llama_model_n_ctx_train(const struct llama_model * model);
@ctypes_function("llama_model_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_model_n_ctx_train(model: llama_model_p, /) -> int: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API int32_t llama_model_n_embd (const struct llama_model * model);
@ctypes_function("llama_model_n_embd", [llama_model_p_ctypes], ctypes.c_int32)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_model_n_embd(model: llama_model_p, /) -> int: ...
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# LLAMA_API int32_t llama_model_n_embd_inp (const struct llama_model * model);
@ctypes_function("llama_model_n_embd_inp", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_embd_inp(model: llama_model_p, /) -> int:
"""Get the model input embedding size."""
...
# LLAMA_API int32_t llama_model_n_embd_out (const struct llama_model * model);
@ctypes_function("llama_model_n_embd_out", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_embd_out(model: llama_model_p, /) -> int:
"""Get the model output embedding size."""
...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API int32_t llama_model_n_layer (const struct llama_model * model);
@ctypes_function("llama_model_n_layer", [llama_model_p_ctypes], ctypes.c_int32)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_model_n_layer(model: llama_model_p, /) -> int: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API int32_t llama_model_n_head (const struct llama_model * model);
@ctypes_function("llama_model_n_head", [llama_model_p_ctypes], ctypes.c_int32)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_model_n_head(model: llama_model_p, /) -> int: ...
feat: Update llama.cpp
2024-03-18 10:26:36 -04:00
feat: Update llama.cpp
2025-03-12 04:11:42 -04:00
# LLAMA_API int32_t llama_model_n_head_kv (const struct llama_model * model);
@ctypes_function("llama_model_n_head_kv", [llama_model_p_ctypes], ctypes.c_int32)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_model_n_head_kv(model: llama_model_p, /) -> int: ...
feat: Update llama.cpp
2025-03-12 04:11:42 -04:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# LLAMA_API int32_t llama_model_n_swa (const struct llama_model * model);
@ctypes_function("llama_model_n_swa", [llama_model_p_ctypes], ctypes.c_int32)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_model_n_swa(model: llama_model_p, /) -> int: ...
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
Update llama.cpp
2023-10-03 15:23:35 -04:00
# // Get the model's RoPE frequency scaling factor
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model);
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
@ctypes_function(
"llama_model_rope_freq_scale_train", [llama_model_p_ctypes], ctypes.c_float
)
def llama_model_rope_freq_scale_train(model: llama_model_p, /) -> float: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Returns the number of classifier outputs (only valid for classifier models)
# // Undefined behavior for non-classifier models
# LLAMA_API uint32_t llama_model_n_cls_out(const struct llama_model * model);
@ctypes_function("llama_model_n_cls_out", [llama_model_p_ctypes], ctypes.c_uint32)
def llama_model_n_cls_out(model: llama_model_p, /) -> int:
"""Returns the number of classifier outputs (only valid for classifier models)"""
...
# // Returns label of classifier output by index (<n_cls_out). Returns nullptr if no label provided
# LLAMA_API const char * llama_model_cls_label(const struct llama_model * model, uint32_t i);
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
@ctypes_function(
"llama_model_cls_label", [llama_model_p_ctypes, ctypes.c_uint32], ctypes.c_char_p
)
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
def llama_model_cls_label(model: llama_model_p, i: int, /) -> Optional[bytes]:
"""Returns label of classifier output by index. Returns None if no label provided"""
...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model);
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
@ctypes_function("llama_vocab_type", [llama_vocab_p_ctypes], ctypes.c_int)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_vocab_type(vocab: llama_vocab_p, /) -> int: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API int32_t llama_vocab_n_tokens(const struct llama_vocab * vocab);
@ctypes_function("llama_vocab_n_tokens", [llama_vocab_p_ctypes], ctypes.c_int32)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_vocab_n_tokens(vocab: llama_vocab_p, /) -> int: ...
Update llama.cpp
2023-10-03 15:23:35 -04:00
Update llama.cpp
2023-11-20 14:11:33 -05:00
# // Functions to access the model's GGUF metadata scalar values
# // - The functions return the length of the string on success, or -1 on failure
# // - The output string is always null-terminated and cleared on failure
feat: Update llama.cpp
2024-12-19 01:55:12 -05:00
# // - When retrieving a string, an extra byte must be allocated to account for the null terminator
Update llama.cpp
2023-11-20 14:11:33 -05:00
# // - GGUF array values are not supported by these functions
# // Get metadata value as a string by key name
Update llama.cpp
2024-01-03 22:04:04 -05:00
# LLAMA_API int32_t llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_model_meta_val_str",
[
llama_model_p_ctypes,
ctypes.c_char_p,
ctypes.c_char_p,
ctypes.c_size_t,
],
ctypes.c_int32,
)
Update llama.cpp
2023-11-20 14:11:33 -05:00
def llama_model_meta_val_str(
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
model: llama_model_p,
key: Union[ctypes.c_char_p, bytes],
buf: bytes,
buf_size: int,
/,
Update llama.cpp
2023-11-20 14:11:33 -05:00
) -> int:
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Get metadata value as a string by key name"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-11-20 14:11:33 -05:00
# // Get the number of metadata key/value pairs
Update llama.cpp
2024-01-03 22:04:04 -05:00
# LLAMA_API int32_t llama_model_meta_count(const struct llama_model * model);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function("llama_model_meta_count", [llama_model_p_ctypes], ctypes.c_int32)
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
def llama_model_meta_count(model: llama_model_p, /) -> int:
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Get the number of metadata key/value pairs"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-11-20 14:11:33 -05:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# // Get sampling metadata key name. Returns nullptr if the key is invalid
# LLAMA_API const char * llama_model_meta_key_str(enum llama_model_meta_key key);
@ctypes_function("llama_model_meta_key_str", [ctypes.c_int], ctypes.c_char_p)
def llama_model_meta_key_str(key: int, /) -> Optional[bytes]:
"""Get sampling metadata key name. Returns None if the key is invalid."""
...
Update llama.cpp
2023-11-20 14:11:33 -05:00
# // Get metadata key name by index
Update llama.cpp
2024-01-03 22:04:04 -05:00
# LLAMA_API int32_t llama_model_meta_key_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_model_meta_key_by_index",
[
llama_model_p_ctypes,
ctypes.c_int32,
ctypes.c_char_p,
ctypes.c_size_t,
],
ctypes.c_int32,
)
Update llama.cpp
2023-11-20 14:11:33 -05:00
def llama_model_meta_key_by_index(
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
model: llama_model_p,
i: Union[ctypes.c_int, int],
buf: Union[bytes, CtypesArray[ctypes.c_char]],
buf_size: int,
/,
Update llama.cpp
2023-11-20 14:11:33 -05:00
) -> int:
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Get metadata key name by index"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-11-20 14:11:33 -05:00
# // Get metadata value as a string by index
Update llama.cpp
2024-01-03 22:04:04 -05:00
# LLAMA_API int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_model_meta_val_str_by_index",
[
llama_model_p_ctypes,
ctypes.c_int32,
ctypes.c_char_p,
ctypes.c_size_t,
],
ctypes.c_int32,
)
Update llama.cpp
2023-11-20 14:11:33 -05:00
def llama_model_meta_val_str_by_index(
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
model: llama_model_p,
i: Union[ctypes.c_int, int],
buf: Union[bytes, CtypesArray[ctypes.c_char]],
buf_size: int,
/,
Update llama.cpp
2023-11-20 14:11:33 -05:00
) -> int:
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Get metadata value as a string by index"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-11-20 14:11:33 -05:00
Update llama.cpp
2023-08-24 00:17:00 -04:00
# // Get a string describing the model type
Update llama.cpp
2024-01-03 22:04:04 -05:00
# LLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_model_desc",
[llama_model_p_ctypes, ctypes.c_char_p, ctypes.c_size_t],
ctypes.c_int32,
)
Update llama.cpp
2023-09-28 22:42:03 -04:00
def llama_model_desc(
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
model: llama_model_p,
buf: Union[bytes, CtypesArray[ctypes.c_char]],
buf_size: Union[ctypes.c_size_t, int],
/,
Update llama.cpp
2023-09-28 22:42:03 -04:00
) -> int:
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Get a string describing the model type"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-08-24 00:17:00 -04:00
Update llama.cpp
2023-08-25 14:35:53 -04:00
# // Returns the total size of all the tensors in the model in bytes
# LLAMA_API uint64_t llama_model_size(const struct llama_model * model);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function("llama_model_size", [llama_model_p_ctypes], ctypes.c_uint64)
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
def llama_model_size(model: llama_model_p, /) -> int:
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Returns the total size of all the tensors in the model in bytes"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-08-25 14:35:53 -04:00
Update llama.cpp
2023-08-24 00:17:00 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# // Get the default chat template. Returns nullptr if not available
feat: Update llama.cpp
2025-01-28 21:40:53 -05:00
# // If name is NULL, returns the default chat template
# LLAMA_API const char * llama_model_chat_template(const struct llama_model * model, const char * name);
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
@ctypes_function(
"llama_model_chat_template",
[llama_model_p_ctypes, ctypes.c_char_p],
ctypes.c_char_p,
)
def llama_model_chat_template(
model: llama_model_p, name: Optional[bytes], /
) -> Optional[bytes]:
feat: Update llama.cpp
2025-01-28 21:40:53 -05:00
"""Get the default chat template. Returns None if not available
If name is None, returns the default chat template"""
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
...
Update llama.cpp
2023-08-25 14:35:53 -04:00
# // Returns the total number of parameters in the model
# LLAMA_API uint64_t llama_model_n_params(const struct llama_model * model);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function("llama_model_n_params", [llama_model_p_ctypes], ctypes.c_uint64)
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
def llama_model_n_params(model: llama_model_p, /) -> int:
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Returns the total number of parameters in the model"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-08-25 14:35:53 -04:00
feat: Update llama.cpp
2024-07-04 12:23:47 -04:00
# // Returns true if the model contains an encoder that requires llama_encode() call
# LLAMA_API bool llama_model_has_encoder(const struct llama_model * model);
@ctypes_function("llama_model_has_encoder", [llama_model_p_ctypes], ctypes.c_bool)
def llama_model_has_encoder(model: llama_model_p, /) -> bool:
"""Returns true if the model contains an encoder that requires llama_encode() call"""
...
feat: Update llama.cpp
2024-08-10 18:33:33 -04:00
# // Returns true if the model contains a decoder that requires llama_decode() call
# LLAMA_API bool llama_model_has_decoder(const struct llama_model * model);
@ctypes_function("llama_model_has_decoder", [llama_model_p_ctypes], ctypes.c_bool)
def llama_model_has_decoder(model: llama_model_p, /) -> bool:
"""Returns true if the model contains a decoder that requires llama_decode() call"""
...
feat: Update llama.cpp
2024-07-04 12:23:47 -04:00
# // For encoder-decoder models, this function returns id of the token that must be provided
# // to the decoder to start generating output sequence. For other models, it returns -1.
# LLAMA_API llama_token llama_model_decoder_start_token(const struct llama_model * model);
fix(misc): Format
2024-07-09 12:20:17 -04:00
@ctypes_function(
"llama_model_decoder_start_token", [llama_model_p_ctypes], ctypes.c_int32
)
feat: Update llama.cpp
2024-07-04 12:23:47 -04:00
def llama_model_decoder_start_token(model: llama_model_p, /) -> int:
"""For encoder-decoder models, this function returns id of the token that must be provided
fix(misc): Format
2024-07-09 12:20:17 -04:00
to the decoder to start generating output sequence. For other models, it returns -1.
"""
feat: Update llama.cpp
2024-07-04 12:23:47 -04:00
...
feat: Update llama.cpp
2024-08-22 09:17:45 -04:00
# // Returns true if the model is recurrent (like Mamba, RWKV, etc.)
# LLAMA_API bool llama_model_is_recurrent(const struct llama_model * model);
@ctypes_function("llama_model_is_recurrent", [llama_model_p_ctypes], ctypes.c_bool)
def llama_model_is_recurrent(model: llama_model_p, /) -> bool:
"""Returns true if the model is recurrent (like Mamba, RWKV, etc.)"""
...
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# // Returns true if the model is hybrid (like Jamba, Granite, etc.)
# LLAMA_API bool llama_model_is_hybrid(const struct llama_model * model);
@ctypes_function("llama_model_is_hybrid", [llama_model_p_ctypes], ctypes.c_bool)
def llama_model_is_hybrid(model: llama_model_p, /) -> bool:
"""Returns true if the model is hybrid (like Jamba, Granite, etc.)"""
...
feat: Update llama.cpp
2025-08-07 06:40:18 -07:00
# // Returns true if the model is diffusion-based (like LLaDA, Dream, etc.)
# LLAMA_API bool llama_model_is_diffusion(const struct llama_model * model);
@ctypes_function("llama_model_is_diffusion", [llama_model_p_ctypes], ctypes.c_bool)
def llama_model_is_diffusion(model: llama_model_p, /) -> bool:
"""Returns true if the model is diffusion-based (like LLaDA, Dream, etc.)"""
...
Update llama.cpp
2023-06-10 12:17:38 -04:00
# // Returns 0 on success
Update llama.cpp
2024-01-03 22:04:04 -05:00
# LLAMA_API uint32_t llama_model_quantize(
Update llama.cpp
2023-05-21 17:47:21 -04:00
# const char * fname_inp,
# const char * fname_out,
Update llama.cpp
2023-06-10 12:17:38 -04:00
# const llama_model_quantize_params * params);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_model_quantize",
[
ctypes.c_char_p,
ctypes.c_char_p,
ctypes.POINTER(llama_model_quantize_params),
],
ctypes.c_uint32,
)
Update llama.cpp
2023-04-22 19:50:28 -04:00
def llama_model_quantize(
Update llama.cpp
2023-06-10 12:17:38 -04:00
fname_inp: bytes,
fname_out: bytes,
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
params: CtypesPointerOrRef[llama_model_quantize_params],
/,
Fix llama_cpp and Llama type signatures. Closes #221
2023-05-19 11:59:33 -04:00
) -> int:
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Returns 0 on success"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Initial commit
2023-03-23 05:33:06 -04:00
Black formatting
2023-03-24 14:35:41 -04:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# //
# // Adapters
# //
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
# // Load a LoRA adapter from file
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init(
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
# struct llama_model * model,
# const char * path_lora);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
"llama_adapter_lora_init",
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
[llama_model_p_ctypes, ctypes.c_char_p],
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
llama_adapter_lora_p_ctypes,
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
)
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
def llama_adapter_lora_init(
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
model: llama_model_p, path_lora: bytes, /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> Optional[llama_adapter_lora_p]: ...
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# // Get metadata value as a string by key name
# LLAMA_API int32_t llama_adapter_meta_val_str(const struct llama_adapter_lora * adapter, const char * key, char * buf, size_t buf_size);
@ctypes_function(
"llama_adapter_meta_val_str",
[
llama_adapter_lora_p_ctypes,
ctypes.c_char_p,
ctypes.c_char_p,
ctypes.c_size_t,
],
ctypes.c_int32,
)
def llama_adapter_meta_val_str(
adapter: llama_adapter_lora_p,
key: bytes,
buf: Union[bytes, CtypesArray[ctypes.c_char]],
buf_size: int,
/,
) -> int:
"""Get adapter metadata value as a string by key name."""
...
# // Get the number of metadata key/value pairs
# LLAMA_API int32_t llama_adapter_meta_count(const struct llama_adapter_lora * adapter);
@ctypes_function(
"llama_adapter_meta_count", [llama_adapter_lora_p_ctypes], ctypes.c_int32
)
def llama_adapter_meta_count(adapter: llama_adapter_lora_p, /) -> int:
"""Get the number of adapter metadata key/value pairs."""
...
# // Get metadata key name by index
# LLAMA_API int32_t llama_adapter_meta_key_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size);
@ctypes_function(
"llama_adapter_meta_key_by_index",
[
llama_adapter_lora_p_ctypes,
ctypes.c_int32,
ctypes.c_char_p,
ctypes.c_size_t,
],
ctypes.c_int32,
)
def llama_adapter_meta_key_by_index(
adapter: llama_adapter_lora_p,
i: int,
buf: Union[bytes, CtypesArray[ctypes.c_char]],
buf_size: int,
/,
) -> int:
"""Get adapter metadata key name by index."""
...
# // Get metadata value as a string by index
# LLAMA_API int32_t llama_adapter_meta_val_str_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size);
@ctypes_function(
"llama_adapter_meta_val_str_by_index",
[
llama_adapter_lora_p_ctypes,
ctypes.c_int32,
ctypes.c_char_p,
ctypes.c_size_t,
],
ctypes.c_int32,
)
def llama_adapter_meta_val_str_by_index(
adapter: llama_adapter_lora_p,
i: int,
buf: Union[bytes, CtypesArray[ctypes.c_char]],
buf_size: int,
/,
) -> int:
"""Get adapter metadata value as a string by index."""
...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# // Manually free a LoRA adapter
# // Note: loaded adapters will be free when the associated model is deleted
# LLAMA_API void llama_adapter_lora_free(struct llama_adapter_lora * adapter);
@ctypes_function(
"llama_adapter_lora_free",
[llama_adapter_lora_p_ctypes],
None,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_adapter_lora_free(adapter: llama_adapter_lora_p, /): ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# // Get the invocation tokens if the current lora is an alora
# LLAMA_API uint64_t llama_adapter_get_alora_n_invocation_tokens(const struct llama_adapter_lora * adapter);
@ctypes_function(
"llama_adapter_get_alora_n_invocation_tokens",
[llama_adapter_lora_p_ctypes],
ctypes.c_uint64,
)
def llama_adapter_get_alora_n_invocation_tokens(
adapter: llama_adapter_lora_p, /
) -> int:
"""Get the invocation token count if the current LoRA is an aLoRA."""
...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# LLAMA_API const llama_token * llama_adapter_get_alora_invocation_tokens (const struct llama_adapter_lora * adapter);
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
@ctypes_function(
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
"llama_adapter_get_alora_invocation_tokens",
[llama_adapter_lora_p_ctypes],
ctypes.POINTER(llama_token),
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
def llama_adapter_get_alora_invocation_tokens(
adapter: llama_adapter_lora_p, /
) -> Optional[CtypesPointer[llama_token]]:
"""Get the invocation tokens if the current LoRA is an aLoRA."""
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
...
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# // The following functions operate on a llama_context, hence the naming: llama_verb_...
# // Set LoRa adapters on the context. Will only modify if the adapters currently in context are different.
# LLAMA_API int32_t llama_set_adapters_lora(
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
# struct llama_context * ctx,
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# struct llama_adapter_lora ** adapters,
# size_t n_adapters,
# float * scales);
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
@ctypes_function(
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
"llama_set_adapters_lora",
[
llama_context_p_ctypes,
ctypes.POINTER(llama_adapter_lora_p_ctypes),
ctypes.c_size_t,
ctypes.POINTER(ctypes.c_float),
],
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
ctypes.c_int32,
)
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
def llama_set_adapters_lora(
ctx: llama_context_p,
adapters: Optional[CtypesArray[llama_adapter_lora_p_ctypes]],
n_adapters: int,
scales: Optional[CtypesArray[ctypes.c_float]],
/,
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
) -> int:
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
"""Set LoRA adapters on the context if they differ from the current adapters."""
feat: Update llama.cpp
2024-07-17 18:06:58 -04:00
...
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# Deprecated compatibility wrapper for the renamed llama_set_adapters_lora().
def llama_set_adapter_lora(
ctx: llama_context_p, adapter: llama_adapter_lora_p, scale: float, /
) -> int:
warnings.warn(
"llama_set_adapter_lora is deprecated; use llama_set_adapters_lora instead",
DeprecationWarning,
stacklevel=2,
)
adapters = (llama_adapter_lora_p_ctypes * 1)(adapter)
scales = (ctypes.c_float * 1)(scale)
return llama_set_adapters_lora(ctx, adapters, 1, scales)
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
feat: Update llama.cpp
2024-03-18 10:26:36 -04:00
# // Apply a loaded control vector to a llama_context, or if data is NULL, clear
# // the currently loaded vector.
# // n_embd should be the size of a single layer's control, and data should point
# // to an n_embd x n_layers buffer starting from layer 1.
# // il_start and il_end are the layer range the vector should apply to (both inclusive)
# // See llama_control_vector_load in common to load a control vector.
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# LLAMA_API int32_t llama_set_adapter_cvec(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# struct llama_context * ctx,
feat: Update llama.cpp
2024-03-18 10:26:36 -04:00
# const float * data,
# size_t len,
# int32_t n_embd,
# int32_t il_start,
# int32_t il_end);
@ctypes_function(
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
"llama_set_adapter_cvec",
feat: Update llama.cpp
2024-03-18 10:26:36 -04:00
[
llama_context_p_ctypes,
ctypes.POINTER(ctypes.c_float),
ctypes.c_size_t,
ctypes.c_int32,
ctypes.c_int32,
ctypes.c_int32,
],
ctypes.c_int32,
)
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
def llama_set_adapter_cvec(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
ctx: llama_context_p,
feat: Update llama.cpp
2024-03-18 10:26:36 -04:00
data: CtypesPointerOrRef[ctypes.c_float],
len: int,
n_embd: int,
il_start: int,
il_end: int,
/,
) -> int:
"""Apply a loaded control vector to a llama_context, or if data is NULL, clear
the currently loaded vector.
n_embd should be the size of a single layer's control, and data should point
to an n_embd x n_layers buffer starting from layer 1.
il_start and il_end are the layer range the vector should apply to (both inclusive)
See llama_control_vector_load in common to load a control vector."""
...
Update llama.cpp
2023-06-26 08:50:38 -04:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# Deprecated compatibility wrapper for the renamed llama_set_adapter_cvec().
def llama_apply_adapter_cvec(
ctx: llama_context_p,
data: CtypesPointerOrRef[ctypes.c_float],
len: int,
n_embd: int,
il_start: int,
il_end: int,
/,
) -> int:
warnings.warn(
"llama_apply_adapter_cvec is deprecated; use llama_set_adapter_cvec instead",
DeprecationWarning,
stacklevel=2,
)
return llama_set_adapter_cvec(ctx, data, len, n_embd, il_start, il_end)
Update llama.cpp
2023-09-28 22:42:03 -04:00
# //
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Memory
Update llama.cpp
2023-09-28 22:42:03 -04:00
# //
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Clear the memory contents
# // If data == true, the data buffers will also be cleared together with the metadata
# LLAMA_API void llama_memory_clear(
# llama_memory_t mem,
# bool data);
@ctypes_function(
"llama_memory_clear",
[llama_memory_t_ctypes, ctypes.c_bool],
None,
)
def llama_memory_clear(mem: llama_memory_t, data: bool, /):
"""Clear the memory contents
If data == true, the data buffers will also be cleared together with the metadata"""
...
Add bindings for LoRA adapters. Closes #88
2023-04-18 01:30:04 -04:00
Update llama.cpp
2023-11-23 16:26:00 -05:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Removes all tokens that belong to the specified sequence and have positions in [p0, p1)
# // Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails
# // seq_id < 0 : match any sequence
# // p0 < 0 : [0, p1]
# // p1 < 0 : [p0, inf)
# LLAMA_API bool llama_memory_seq_rm(
# llama_memory_t mem,
# llama_seq_id seq_id,
# llama_pos p0,
# llama_pos p1);
@ctypes_function(
"llama_memory_seq_rm",
[
llama_memory_t_ctypes,
llama_seq_id,
llama_pos,
llama_pos,
],
ctypes.c_bool,
)
def llama_memory_seq_rm(
mem: llama_memory_t,
seq_id: Union[llama_seq_id, int],
p0: Union[llama_pos, int],
p1: Union[llama_pos, int],
/,
) -> bool:
"""Removes all tokens that belong to the specified sequence and have positions in [p0, p1)
Update llama.cpp
2023-11-23 16:26:00 -05:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails
Update llama.cpp
2023-11-23 16:26:00 -05:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
seq_id < 0 : match any sequence
p0 < 0 : [0, p1]
p1 < 0 : [p0, inf)"""
...
Update llama.cpp
2023-11-23 16:26:00 -05:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Copy all tokens that belong to the specified sequence to another sequence
# // p0 < 0 : [0, p1]
# // p1 < 0 : [p0, inf)
# LLAMA_API void llama_memory_seq_cp(
# llama_memory_t mem,
# llama_seq_id seq_id_src,
# llama_seq_id seq_id_dst,
# llama_pos p0,
# llama_pos p1);
@ctypes_function(
"llama_memory_seq_cp",
[
llama_memory_t_ctypes,
llama_seq_id,
llama_seq_id,
llama_pos,
llama_pos,
],
None,
)
def llama_memory_seq_cp(
mem: llama_memory_t,
seq_id_src: Union[llama_seq_id, int],
seq_id_dst: Union[llama_seq_id, int],
p0: Union[llama_pos, int],
p1: Union[llama_pos, int],
/,
):
"""Copy all tokens that belong to the specified sequence to another sequence
p0 < 0 : [0, p1]
p1 < 0 : [p0, inf)"""
...
Update llama.cpp
2023-11-23 16:26:00 -05:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Removes all tokens that do not belong to the specified sequence
# LLAMA_API void llama_memory_seq_keep(
# llama_memory_t mem,
# llama_seq_id seq_id);
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
@ctypes_function("llama_memory_seq_keep", [llama_memory_t_ctypes, llama_seq_id], None)
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
def llama_memory_seq_keep(mem: llama_memory_t, seq_id: Union[llama_seq_id, int], /):
"""Removes all tokens that do not belong to the specified sequence"""
...
Update llama.cpp
2023-11-23 16:26:00 -05:00
feat: Add typechecking for ctypes structure attributes
2024-04-10 02:40:41 -04:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1)
# // p0 < 0 : [0, p1]
# // p1 < 0 : [p0, inf)
# LLAMA_API void llama_memory_seq_add(
# llama_memory_t mem,
# llama_seq_id seq_id,
# llama_pos p0,
# llama_pos p1,
# llama_pos delta);
@ctypes_function(
"llama_memory_seq_add",
[
llama_memory_t_ctypes,
llama_seq_id,
llama_pos,
llama_pos,
llama_pos,
],
None,
)
def llama_memory_seq_add(
mem: llama_memory_t,
seq_id: Union[llama_seq_id, int],
p0: Union[llama_pos, int],
p1: Union[llama_pos, int],
delta: Union[llama_pos, int],
/,
):
"""Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1)
p0 < 0 : [0, p1]
p1 < 0 : [p0, inf)"""
...
Update llama.cpp
2023-11-23 16:26:00 -05:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Integer division of the positions by factor of `d > 1`
# // p0 < 0 : [0, p1]
# // p1 < 0 : [p0, inf)
# LLAMA_API void llama_memory_seq_div(
# llama_memory_t mem,
# llama_seq_id seq_id,
# llama_pos p0,
# llama_pos p1,
# int d);
@ctypes_function(
"llama_memory_seq_div",
[
llama_memory_t_ctypes,
llama_seq_id,
llama_pos,
llama_pos,
ctypes.c_int,
],
None,
)
def llama_memory_seq_div(
mem: llama_memory_t,
seq_id: Union[llama_seq_id, int],
p0: Union[llama_pos, int],
p1: Union[llama_pos, int],
d: Union[ctypes.c_int, int],
/,
):
"""Integer division of the positions by factor of `d > 1`
p0 < 0 : [0, p1]
p1 < 0 : [p0, inf)"""
...
Fix ctypes definitions of `llama_kv_cache_view_update` and `llama_kv_cache_view_free`. (#1028)
2023-12-18 18:11:26 -05:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Returns the smallest position present in the memory for the specified sequence
# // This is typically non-zero only for SWA caches
# // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory
# // Return -1 if the sequence is empty
# LLAMA_API llama_pos llama_memory_seq_pos_min(
# llama_memory_t mem,
# llama_seq_id seq_id);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
"llama_memory_seq_pos_min", [llama_memory_t_ctypes, llama_seq_id], llama_pos
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
def llama_memory_seq_pos_min(
mem: llama_memory_t, seq_id: Union[llama_seq_id, int], /
) -> int:
"""Returns the smallest position present in the memory for the specified sequence
This is typically non-zero only for SWA caches
Return -1 if the sequence is empty"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-11-23 16:26:00 -05:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Returns the largest position present in the memory for the specified sequence
# // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory
# // Return -1 if the sequence is empty
# LLAMA_API llama_pos llama_memory_seq_pos_max(
# llama_memory_t mem,
# llama_seq_id seq_id);
@ctypes_function(
"llama_memory_seq_pos_max", [llama_memory_t_ctypes, llama_seq_id], llama_pos
)
def llama_memory_seq_pos_max(
mem: llama_memory_t, seq_id: Union[llama_seq_id, int], /
) -> int:
"""Returns the largest position present in the memory for the specified sequence
Return -1 if the sequence is empty"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-11-23 16:26:00 -05:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Check if the memory supports shifting
# LLAMA_API bool llama_memory_can_shift(llama_memory_t mem);
@ctypes_function("llama_memory_can_shift", [llama_memory_t_ctypes], ctypes.c_bool)
def llama_memory_can_shift(mem: llama_memory_t, /) -> bool:
"""Check if the memory supports shifting"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-11-23 16:26:00 -05:00
Update llama.cpp
2023-09-28 22:42:03 -04:00
# //
# // State / sessions
# //
Update llama_cpp and add kv_cache api support
2023-04-02 13:33:49 -04:00
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
# // Returns the *actual* size in bytes of the state
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // (logits, embedding and memory)
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
# // Only use when saving the state, not when restoring it, otherwise the size may be too small.
# LLAMA_API size_t llama_state_get_size(struct llama_context * ctx);
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
@ctypes_function("llama_state_get_size", [llama_context_p_ctypes], ctypes.c_size_t)
def llama_state_get_size(ctx: llama_context_p, /) -> int:
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
"""Returns the *actual* size in bytes of the state (logits, embedding and memory)"""
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
...
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
# LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx),
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
# "use llama_state_get_size instead");
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function("llama_get_state_size", [llama_context_p_ctypes], ctypes.c_size_t)
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
def llama_get_state_size(ctx: llama_context_p, /) -> int:
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
"""Returns the size in bytes of the state (DEPRECATED)"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-04-22 19:50:28 -04:00
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
# // Copies the state to the specified destination address.
# // Destination needs to have allocated enough memory.
# // Returns the number of bytes copied
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
# LLAMA_API size_t llama_state_get_data(
Update llama.cpp
2023-09-28 22:42:03 -04:00
# struct llama_context * ctx,
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
# uint8_t * dst,
# size_t size);
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
@ctypes_function(
"llama_state_get_data",
[
llama_context_p_ctypes,
ctypes.POINTER(ctypes.c_uint8),
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
ctypes.c_size_t,
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
],
ctypes.c_size_t,
)
def llama_state_get_data(
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
ctx: llama_context_p,
dst: CtypesArray[ctypes.c_uint8],
size: Union[ctypes.c_size_t, int],
/,
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
) -> int:
"""Copies the state to the specified destination address.
Destination needs to have allocated enough memory.
Returns the number of bytes copied"""
...
# LLAMA_API DEPRECATED(size_t llama_copy_state_data(
# struct llama_context * ctx,
# uint8_t * dst),
# "use llama_state_get_data instead");
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_copy_state_data",
[
llama_context_p_ctypes,
ctypes.POINTER(ctypes.c_uint8),
],
ctypes.c_size_t,
)
Fix: runtime type errors
2023-05-05 14:12:26 -04:00
def llama_copy_state_data(
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
ctx: llama_context_p, dst: CtypesArray[ctypes.c_uint8], /
Fix return type
2023-05-07 19:30:14 -04:00
) -> int:
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
"""Copies the state to the specified destination address (DEPRECATED)"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-04-22 19:50:28 -04:00
feat: Update llama.cpp
2024-03-01 12:57:16 -05:00
# // Set the state reading from the specified address
# // Returns the number of bytes read
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
# LLAMA_API size_t llama_state_set_data(
Update llama.cpp
2023-09-28 22:42:03 -04:00
# struct llama_context * ctx,
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
# const uint8_t * src,
# size_t size);
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
@ctypes_function(
"llama_state_set_data",
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
[llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t],
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
ctypes.c_size_t,
)
def llama_state_set_data(
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
ctx: llama_context_p,
src: CtypesArray[ctypes.c_uint8],
size: Union[ctypes.c_size_t, int],
/,
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
) -> int:
"""Set the state reading from the specified address
Returns the number of bytes read"""
...
# LLAMA_API DEPRECATED(size_t llama_set_state_data(
# struct llama_context * ctx,
# const uint8_t * src),
# "use llama_state_set_data instead");
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_set_state_data",
[llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8)],
ctypes.c_size_t,
)
Add types for all low-level api functions
2023-05-05 12:22:27 -04:00
def llama_set_state_data(
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
ctx: llama_context_p, src: CtypesArray[ctypes.c_uint8], /
Fix return type
2023-05-07 19:30:14 -04:00
) -> int:
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
"""Set the state reading from the specified address (DEPRECATED)"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-04-22 19:50:28 -04:00
Update llama.cpp
2023-04-28 15:32:43 -04:00
# Save/load session file
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
# LLAMA_API bool llama_state_load_file(
Update llama.cpp
2023-09-28 22:42:03 -04:00
# struct llama_context * ctx,
# const char * path_session,
# llama_token * tokens_out,
# size_t n_token_capacity,
# size_t * n_token_count_out);
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
@ctypes_function(
"llama_state_load_file",
[
llama_context_p_ctypes,
ctypes.c_char_p,
llama_token_p,
ctypes.c_size_t,
ctypes.POINTER(ctypes.c_size_t),
],
ctypes.c_bool,
)
def llama_state_load_file(
ctx: llama_context_p,
path_session: bytes,
tokens_out: CtypesArray[llama_token],
n_token_capacity: Union[ctypes.c_size_t, int],
n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t],
/,
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> bool: ...
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
# LLAMA_API DEPRECATED(bool llama_load_session_file(
# struct llama_context * ctx,
# const char * path_session,
# llama_token * tokens_out,
# size_t n_token_capacity,
# size_t * n_token_count_out),
# "use llama_state_load_file instead");
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_load_session_file",
[
llama_context_p_ctypes,
ctypes.c_char_p,
llama_token_p,
ctypes.c_size_t,
ctypes.POINTER(ctypes.c_size_t),
],
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
ctypes.c_bool,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
Update llama.cpp
2023-04-28 15:32:43 -04:00
def llama_load_session_file(
ctx: llama_context_p,
path_session: bytes,
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
tokens_out: CtypesArray[llama_token],
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
n_token_capacity: Union[ctypes.c_size_t, int],
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t],
/,
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> bool: ...
Update llama.cpp
2023-04-28 15:32:43 -04:00
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
# LLAMA_API bool llama_state_save_file(
Update llama.cpp
2023-09-28 22:42:03 -04:00
# struct llama_context * ctx,
# const char * path_session,
# const llama_token * tokens,
# size_t n_token_count);
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
@ctypes_function(
"llama_state_save_file",
[
llama_context_p_ctypes,
ctypes.c_char_p,
llama_token_p,
ctypes.c_size_t,
],
ctypes.c_bool,
)
def llama_state_save_file(
ctx: llama_context_p,
path_session: bytes,
tokens: CtypesArray[llama_token],
n_token_count: Union[ctypes.c_size_t, int],
/,
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> bool: ...
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
# LLAMA_API DEPRECATED(bool llama_save_session_file(
# struct llama_context * ctx,
# const char * path_session,
# const llama_token * tokens,
# size_t n_token_count),
# "use llama_state_save_file instead");
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_save_session_file",
[
llama_context_p_ctypes,
ctypes.c_char_p,
llama_token_p,
ctypes.c_size_t,
],
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
ctypes.c_bool,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
Update llama.cpp
2023-04-28 15:32:43 -04:00
def llama_save_session_file(
Add types for all low-level api functions
2023-05-05 12:22:27 -04:00
ctx: llama_context_p,
path_session: bytes,
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
tokens: CtypesArray[llama_token],
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
n_token_count: Union[ctypes.c_size_t, int],
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
/,
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> bool: ...
Update llama.cpp
2023-04-28 15:32:43 -04:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Get the exact size needed to copy the state of a single sequence
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
# LLAMA_API size_t llama_state_seq_get_size(
# struct llama_context * ctx,
# llama_seq_id seq_id);
@ctypes_function(
"llama_state_seq_get_size",
[llama_context_p_ctypes, llama_seq_id],
ctypes.c_size_t,
)
def llama_state_seq_get_size(ctx: llama_context_p, seq_id: llama_seq_id, /) -> int:
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
"""Get the exact size needed to copy the state of a single sequence"""
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
...
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Copy the state of a single sequence into the specified buffer
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
# LLAMA_API size_t llama_state_seq_get_data(
# struct llama_context * ctx,
# uint8_t * dst,
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
# size_t size,
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
# llama_seq_id seq_id);
@ctypes_function(
"llama_state_seq_get_data",
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
[
llama_context_p_ctypes,
ctypes.POINTER(ctypes.c_uint8),
ctypes.c_size_t,
llama_seq_id,
],
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
ctypes.c_size_t,
)
def llama_state_seq_get_data(
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
ctx: llama_context_p,
dst: CtypesArray[ctypes.c_uint8],
size: Union[ctypes.c_size_t, int],
seq_id: llama_seq_id,
/,
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
) -> int:
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
"""Copy the state of a single sequence into the specified buffer"""
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
...
# // Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence
# // Returns:
# // - Positive: Ok
# // - Zero: Failed to load
# LLAMA_API size_t llama_state_seq_set_data(
# struct llama_context * ctx,
# const uint8_t * src,
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
# size_t size,
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
# llama_seq_id dest_seq_id);
@ctypes_function(
"llama_state_seq_set_data",
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
[
llama_context_p_ctypes,
ctypes.POINTER(ctypes.c_uint8),
ctypes.c_size_t,
llama_seq_id,
],
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
ctypes.c_size_t,
)
def llama_state_seq_set_data(
feat: Update llama.cpp
2024-07-28 11:58:22 -04:00
ctx: llama_context_p,
src: CtypesArray[ctypes.c_uint8],
size: Union[ctypes.c_size_t, int],
dest_seq_id: llama_seq_id,
/,
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
) -> int:
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
"""Copy the sequence data into the specified sequence"""
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
...
# LLAMA_API size_t llama_state_seq_save_file(
# struct llama_context * ctx,
# const char * filepath,
# llama_seq_id seq_id,
# const llama_token * tokens,
# size_t n_token_count);
@ctypes_function(
"llama_state_seq_save_file",
[
llama_context_p_ctypes,
ctypes.c_char_p,
llama_seq_id,
llama_token_p,
ctypes.c_size_t,
],
ctypes.c_size_t,
)
def llama_state_seq_save_file(
ctx: llama_context_p,
filepath: bytes,
seq_id: llama_seq_id,
tokens: CtypesArray[llama_token],
n_token_count: Union[ctypes.c_size_t, int],
/,
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> int: ...
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
# LLAMA_API size_t llama_state_seq_load_file(
# struct llama_context * ctx,
# const char * filepath,
# llama_seq_id dest_seq_id,
# llama_token * tokens_out,
# size_t n_token_capacity,
# size_t * n_token_count_out);
@ctypes_function(
"llama_state_seq_load_file",
[
llama_context_p_ctypes,
ctypes.c_char_p,
llama_seq_id,
llama_token_p,
ctypes.c_size_t,
ctypes.POINTER(ctypes.c_size_t),
],
ctypes.c_size_t,
)
def llama_state_seq_load_file(
ctx: llama_context_p,
filepath: bytes,
dest_seq_id: llama_seq_id,
tokens_out: CtypesArray[llama_token],
n_token_capacity: Union[ctypes.c_size_t, int],
n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t],
/,
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> int: ...
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
Update llama.cpp
2023-09-28 22:42:03 -04:00
# //
# // Decoding
# //
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
# // Return batch for single sequence of tokens
# // The sequence ID will be fixed to 0
# // The position of the tokens will be tracked automatically by llama_decode
Update llama.cpp
2023-09-28 22:42:03 -04:00
# //
# // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it
# //
# LLAMA_API struct llama_batch llama_batch_get_one(
# llama_token * tokens,
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
# int32_t n_tokens);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_batch_get_one",
[
llama_token_p,
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
ctypes.c_int32,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
],
llama_batch,
)
Update llama.cpp
2023-09-28 22:42:03 -04:00
def llama_batch_get_one(
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
tokens: CtypesArray[llama_token],
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
n_tokens: Union[ctypes.c_int, int],
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
/,
Update llama.cpp
2023-09-28 22:42:03 -04:00
) -> llama_batch:
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
"""Return batch for single sequence of tokens
Update llama.cpp
2023-11-23 16:26:00 -05:00
NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it
"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-09-28 22:42:03 -04:00
Update llama.cpp
2023-10-19 02:55:08 -04:00
# // Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
# // Each token can be assigned up to n_seq_max sequence ids
Update llama.cpp
2023-09-28 22:42:03 -04:00
# // The batch has to be freed with llama_batch_free()
# // If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float)
# // Otherwise, llama_batch.token will be allocated to store n_tokens llama_token
# // The rest of the llama_batch members are allocated with size n_tokens
# // All members are left uninitialized
# LLAMA_API struct llama_batch llama_batch_init(
# int32_t n_tokens,
Update llama.cpp
2023-10-19 02:55:08 -04:00
# int32_t embd,
# int32_t n_seq_max);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_batch_init", [ctypes.c_int32, ctypes.c_int32, ctypes.c_int32], llama_batch
)
Update llama.cpp
2023-09-28 22:42:03 -04:00
def llama_batch_init(
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
n_tokens: Union[ctypes.c_int32, int],
embd: Union[ctypes.c_int32, int],
n_seq_max: Union[ctypes.c_int32, int],
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
/,
Update llama.cpp
2023-09-28 22:42:03 -04:00
) -> llama_batch:
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
Each token can be assigned up to n_seq_max sequence ids
The batch has to be freed with llama_batch_free()
If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float)
Otherwise, llama_batch.token will be allocated to store n_tokens llama_token
The rest of the llama_batch members are allocated with size n_tokens
All members are left uninitialized"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-09-28 22:42:03 -04:00
# // Frees a batch of tokens allocated with llama_batch_init()
# LLAMA_API void llama_batch_free(struct llama_batch batch);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function("llama_batch_free", [llama_batch], None)
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
def llama_batch_free(batch: llama_batch, /):
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Frees a batch of tokens allocated with llama_batch_init()"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-09-28 22:42:03 -04:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Process a batch of tokens.
# // In contrast to llama_decode() - this call does not use KV cache.
# // For encode-decoder contexts, processes the batch using the encoder.
# // Can store the encoder output internally for later use by the decoder's cross-attention layers.
feat: Update llama.cpp
2024-07-04 12:23:47 -04:00
# // 0 - success
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // < 0 - error. the memory state is restored to the state before this call
feat: Update llama.cpp
2024-07-04 12:23:47 -04:00
# LLAMA_API int32_t llama_encode(
# struct llama_context * ctx,
# struct llama_batch batch);
@ctypes_function("llama_encode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32)
def llama_encode(ctx: llama_context_p, batch: llama_batch, /) -> int:
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
"""Process a batch of tokens using the encoder.
feat: Update llama.cpp
2024-07-04 12:23:47 -04:00
0 - success
< 0 - error"""
...
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Process a batch of tokens.
# // Requires the context to have a memory.
# // For encode-decoder contexts, processes the batch using the decoder.
Update llama.cpp
2023-09-28 22:42:03 -04:00
# // Positive return values does not mean a fatal error, but rather a warning.
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context
# // To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max()
# // Upon other return values, the memory state is restored to the state before this call
# // 0 - success
# // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)
# // 2 - aborted (processed ubatches will remain in the context's memory)
# // -1 - invalid input batch
# // < -1 - fatal error (processed ubatches will remain in the context's memory)
Update llama.cpp
2024-01-03 22:04:04 -05:00
# LLAMA_API int32_t llama_decode(
Update llama.cpp
2023-09-28 22:42:03 -04:00
# struct llama_context * ctx,
# struct llama_batch batch);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function("llama_decode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32)
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
def llama_decode(ctx: llama_context_p, batch: llama_batch, /) -> int:
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
"""Process a batch of tokens.
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
0 - success
1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
2 - aborted (processed ubatches will remain in the context's memory)
-1 - invalid input batch
< -1 - fatal error (processed ubatches will remain in the context's memory)"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-09-28 22:42:03 -04:00
# // Set the number of threads used for decoding
# // n_threads is the number of threads used for generation (single token)
# // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)
feat: Update llama.cpp
2024-08-30 00:33:58 -04:00
# LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_set_n_threads",
[
llama_context_p_ctypes,
feat: Update llama.cpp
2024-08-30 00:33:58 -04:00
ctypes.c_int32,
ctypes.c_int32,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
],
None,
)
Update llama.cpp
2023-09-28 22:42:03 -04:00
def llama_set_n_threads(
ctx: llama_context_p,
feat: Update llama.cpp
2024-08-30 00:33:58 -04:00
n_threads: Union[ctypes.c_int32, int],
n_threads_batch: Union[ctypes.c_int32, int],
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
/,
Update llama.cpp
2023-09-28 22:42:03 -04:00
):
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Set the number of threads used for decoding
n_threads is the number of threads used for generation (single token)
Update llama.cpp
2023-11-23 16:26:00 -05:00
n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)
"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-09-28 22:42:03 -04:00
feat: Update llama.cpp
2024-03-10 23:45:05 -04:00
feat: Update llama.cpp
2024-05-24 01:43:36 -04:00
# // Get the number of threads used for generation of a single token.
feat: Update llama.cpp
2024-08-30 00:33:58 -04:00
# LLAMA_API int32_t llama_n_threads(struct llama_context * ctx);
@ctypes_function("llama_n_threads", [llama_context_p_ctypes], ctypes.c_int32)
feat: Update llama.cpp
2024-05-24 01:43:36 -04:00
def llama_n_threads(ctx: llama_context_p, /) -> int:
"""Get the number of threads used for generation of a single token"""
...
# // Get the number of threads used for prompt and batch processing (multiple token).
feat: Update llama.cpp
2024-08-30 00:33:58 -04:00
# LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx);
@ctypes_function("llama_n_threads_batch", [llama_context_p_ctypes], ctypes.c_int32)
feat: Update llama.cpp
2024-05-24 01:43:36 -04:00
def llama_n_threads_batch(ctx: llama_context_p, /) -> int:
"""Get the number of threads used for prompt and batch processing (multiple token)"""
...
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // Set whether the context outputs embeddings or not
# // TODO: rename to avoid confusion with llama_get_embeddings()
feat: Update llama_cpp.py bindings
2024-06-21 16:56:15 -04:00
# LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings);
@ctypes_function("llama_set_embeddings", [llama_context_p_ctypes, ctypes.c_bool], None)
def llama_set_embeddings(ctx: llama_context_p, embeddings: bool, /):
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
"""Set whether the context outputs embeddings or not"""
feat: Update llama_cpp.py bindings
2024-06-21 16:56:15 -04:00
...
feat: Update llama.cpp
2024-03-10 23:45:05 -04:00
# // Set whether to use causal attention or not
# // If set to true, the model will only attend to the past tokens
# LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn);
@ctypes_function("llama_set_causal_attn", [llama_context_p_ctypes, ctypes.c_bool], None)
def llama_set_causal_attn(ctx: llama_context_p, causal_attn: bool, /):
"""Set whether to use causal attention or not
If set to true, the model will only attend to the past tokens"""
...
feat: Update llama.cpp
2025-04-11 03:28:20 -04:00
# // Set whether the model is in warmup mode or not
# // If true, all model tensors are activated during llama_decode() to load and cache their weights.
# LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup);
@ctypes_function("llama_set_warmup", [llama_context_p_ctypes, ctypes.c_bool], None)
def llama_set_warmup(ctx: llama_context_p, warmup: bool, /):
"""Set whether the model is in warmup mode or not
If true, all model tensors are activated during llama_decode() to load and cache their weights."""
...
feat: Update llama.cpp
2024-03-02 22:20:04 -05:00
# // Set abort callback
# LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data);
@ctypes_function(
"llama_set_abort_callback",
[llama_context_p_ctypes, ggml_abort_callback, ctypes.c_void_p],
None,
)
def llama_set_abort_callback(
ctx: llama_context_p,
abort_callback: Callable[[ctypes.c_void_p], None],
abort_callback_data: ctypes.c_void_p,
/,
):
"""Set abort callback"""
...
Update llama.cpp
2023-08-24 00:17:00 -04:00
feat: Update llama.cpp
2024-03-13 15:57:35 -04:00
# // Wait until all computations are finished
# // This is automatically done when using one of the functions below to obtain the computation results
# // and is not necessary to call it explicitly in most cases
# LLAMA_API void llama_synchronize(struct llama_context * ctx);
@ctypes_function("llama_synchronize", [llama_context_p_ctypes], None)
def llama_synchronize(ctx: llama_context_p, /):
"""Wait until all computations are finished
This is automatically done when using one of the functions below to obtain the computation results
and is not necessary to call it explicitly in most cases"""
...
feat: Update llama.cpp
2024-03-02 22:20:04 -05:00
# // Token logits obtained from the last call to llama_decode()
feat: Update llama.cpp
2024-03-26 22:58:53 -04:00
# // The logits for which llama_batch.logits[i] != 0 are stored contiguously
# // in the order they have appeared in the batch.
# // Rows: number of tokens for which llama_batch.logits[i] != 0
Update llama.cpp
2023-09-28 22:42:03 -04:00
# // Cols: n_vocab
feat: Update llama.cpp
2025-08-07 06:40:18 -07:00
# // TODO: deprecate in favor of llama_get_logits_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522)
Update llama.cpp
2023-08-24 00:17:00 -04:00
# LLAMA_API float * llama_get_logits(struct llama_context * ctx);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_get_logits", [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float)
)
def llama_get_logits(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float]:
docs: Remove ref to llama_eval in llama_cpp.py docs (#1819) The llama_eval function was removed in 0.2.14 (0d37ce5) but the docs still reference it. Updated to match the llama.h docs from upstream.
2024-12-06 22:41:52 +13:00
"""Token logits obtained from the last call to llama_decode()
The logits for which llama_batch.logits[i] != 0 are stored contiguously
in the order they have appeared in the batch.
Rows: number of tokens for which llama_batch.logits[i] != 0
feat: Update llama.cpp
2024-03-08 20:58:50 -05:00
Cols: n_vocab
feat: Update llama.cpp
2024-03-13 15:57:35 -04:00
feat: Update llama.cpp
2024-03-08 20:58:50 -05:00
Returns:
Pointer to the logits buffer of shape (n_tokens, n_vocab)"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Initial commit
2023-03-23 05:33:06 -04:00
Black formatting
2023-03-24 14:35:41 -04:00
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
# // Logits for the ith token. For positive indices, Equivalent to:
feat: Update llama.cpp
2024-03-26 22:58:53 -04:00
# // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
# // Negative indicies can be used to access logits in reverse order, -1 is the last logit.
feat: Update llama.cpp
2024-03-26 22:58:53 -04:00
# // returns NULL for invalid ids.
Update llama.cpp
2023-09-28 22:42:03 -04:00
# LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_get_logits_ith",
[llama_context_p_ctypes, ctypes.c_int32],
ctypes.POINTER(ctypes.c_float),
)
Update llama.cpp
2023-09-28 22:42:03 -04:00
def llama_get_logits_ith(
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
ctx: llama_context_p, i: Union[ctypes.c_int32, int], /
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
) -> CtypesArray[ctypes.c_float]:
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Logits for the ith token. Equivalent to:
llama_get_logits(ctx) + i*n_vocab"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-09-28 22:42:03 -04:00
feat: Update llama.cpp
2024-03-26 22:58:53 -04:00
# // Get all output token embeddings.
# // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model,
# // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously
# // in the order they have appeared in the batch.
# // shape: [n_outputs*n_embd]
# // Otherwise, returns NULL.
feat: Update llama.cpp
2025-08-07 06:40:18 -07:00
# // TODO: deprecate in favor of llama_get_embeddings_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522)
Update llama.cpp
2023-08-24 00:17:00 -04:00
# LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_get_embeddings", [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float)
)
def llama_get_embeddings(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float]:
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Get the embeddings for the input
shape: [n_embd] (1-dimensional)"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Merge branch main into custom_rope
2023-07-15 15:11:01 -04:00
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
# // Get the embeddings for the ith token. For positive indices, Equivalent to:
feat: Update llama.cpp
2024-03-26 22:58:53 -04:00
# // llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd
feat: Update llama.cpp
2024-04-09 09:53:49 -04:00
# // Negative indicies can be used to access embeddings in reverse order, -1 is the last embedding.
Update llama.cpp
2024-03-06 01:32:00 -05:00
# // shape: [n_embd] (1-dimensional)
feat: Update llama.cpp
2024-03-26 22:58:53 -04:00
# // returns NULL for invalid ids.
Update llama.cpp
2024-02-13 12:24:00 -05:00
# LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_get_embeddings_ith",
[llama_context_p_ctypes, ctypes.c_int32],
ctypes.POINTER(ctypes.c_float),
)
Update llama.cpp
2024-02-13 12:24:00 -05:00
def llama_get_embeddings_ith(
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
ctx: llama_context_p, i: Union[ctypes.c_int32, int], /
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
) -> CtypesArray[ctypes.c_float]:
Update llama.cpp
2024-02-13 12:24:00 -05:00
"""Get the embeddings for the ith sequence
llama_get_embeddings(ctx) + i*n_embd"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2024-02-13 12:24:00 -05:00
Update llama.cpp
2024-03-06 01:32:00 -05:00
# // Get the embeddings for a sequence id
# // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[n_cls_out] with the rank(s) of the sequence
feat: Update llama.cpp
2024-09-29 15:23:24 -04:00
# // otherwise: float[n_embd] (1-dimensional)
Update llama.cpp
2024-03-06 01:32:00 -05:00
# LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id);
@ctypes_function(
"llama_get_embeddings_seq",
[llama_context_p_ctypes, llama_seq_id],
ctypes.POINTER(ctypes.c_float),
)
def llama_get_embeddings_seq(
ctx: llama_context_p, seq_id: Union[llama_seq_id, int], /
) -> CtypesArray[ctypes.c_float]:
"""Get the embeddings for a sequence id
Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE
shape: [n_embd] (1-dimensional)"""
...
feat: Update llama.cpp
2024-03-13 15:57:35 -04:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# // Get the backend sampled token for the ith token.
# // Returns LLAMA_TOKEN_NULL if no token was sampled.
# LLAMA_API llama_token llama_get_sampled_token_ith(struct llama_context * ctx, int32_t i);
@ctypes_function(
"llama_get_sampled_token_ith", [llama_context_p_ctypes, ctypes.c_int32], llama_token
)
def llama_get_sampled_token_ith(
ctx: llama_context_p, i: Union[ctypes.c_int32, int], /
) -> int:
"""Get the backend sampled token for the ith token."""
...
# // Get the backend sampled probabilities for the ith token
# LLAMA_API float * llama_get_sampled_probs_ith (struct llama_context * ctx, int32_t i);
@ctypes_function(
"llama_get_sampled_probs_ith",
[llama_context_p_ctypes, ctypes.c_int32],
ctypes.POINTER(ctypes.c_float),
)
def llama_get_sampled_probs_ith(
ctx: llama_context_p, i: Union[ctypes.c_int32, int], /
) -> Optional[CtypesPointer[ctypes.c_float]]:
"""Get the backend sampled probabilities for the ith token."""
...
# LLAMA_API uint32_t llama_get_sampled_probs_count_ith(struct llama_context * ctx, int32_t i);
@ctypes_function(
"llama_get_sampled_probs_count_ith",
[llama_context_p_ctypes, ctypes.c_int32],
ctypes.c_uint32,
)
def llama_get_sampled_probs_count_ith(
ctx: llama_context_p, i: Union[ctypes.c_int32, int], /
) -> int:
"""Get the backend sampled probability count for the ith token."""
...
# // Get the backend sampled logits for the ith token
# LLAMA_API float * llama_get_sampled_logits_ith (struct llama_context * ctx, int32_t i);
@ctypes_function(
"llama_get_sampled_logits_ith",
[llama_context_p_ctypes, ctypes.c_int32],
ctypes.POINTER(ctypes.c_float),
)
def llama_get_sampled_logits_ith(
ctx: llama_context_p, i: Union[ctypes.c_int32, int], /
) -> Optional[CtypesPointer[ctypes.c_float]]:
"""Get the backend sampled logits for the ith token."""
...
# LLAMA_API uint32_t llama_get_sampled_logits_count_ith(struct llama_context * ctx, int32_t i);
@ctypes_function(
"llama_get_sampled_logits_count_ith",
[llama_context_p_ctypes, ctypes.c_int32],
ctypes.c_uint32,
)
def llama_get_sampled_logits_count_ith(
ctx: llama_context_p, i: Union[ctypes.c_int32, int], /
) -> int:
"""Get the backend sampled logit count for the ith token."""
...
# // Get the backend sampled candidates for the ith token
# LLAMA_API llama_token * llama_get_sampled_candidates_ith (struct llama_context * ctx, int32_t i);
@ctypes_function(
"llama_get_sampled_candidates_ith",
[llama_context_p_ctypes, ctypes.c_int32],
ctypes.POINTER(llama_token),
)
def llama_get_sampled_candidates_ith(
ctx: llama_context_p, i: Union[ctypes.c_int32, int], /
) -> Optional[CtypesPointer[llama_token]]:
"""Get the backend sampled candidates for the ith token."""
...
# LLAMA_API uint32_t llama_get_sampled_candidates_count_ith(struct llama_context * ctx, int32_t i);
@ctypes_function(
"llama_get_sampled_candidates_count_ith",
[llama_context_p_ctypes, ctypes.c_int32],
ctypes.c_uint32,
)
def llama_get_sampled_candidates_count_ith(
ctx: llama_context_p, i: Union[ctypes.c_int32, int], /
) -> int:
"""Get the backend sampled candidate count for the ith token."""
...
Update llama.cpp
2023-08-24 00:17:00 -04:00
# //
# // Vocab
# //
Update llama.cpp and re-organize low-level api
2023-03-24 14:58:42 -04:00
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API const char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
"llama_vocab_get_text", [llama_vocab_p_ctypes, llama_token], ctypes.c_char_p
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
def llama_vocab_get_text(
vocab: llama_vocab_p, token: Union[llama_token, int], /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> bytes: ...
Initial commit
2023-03-23 05:33:06 -04:00
Black formatting
2023-03-24 14:35:41 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API float llama_vocab_get_score(const struct llama_vocab * vocab, llama_token token);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
"llama_vocab_get_score", [llama_vocab_p_ctypes, llama_token], ctypes.c_float
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
def llama_vocab_get_score(
vocab: llama_vocab_p, token: Union[llama_token, int], /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> float: ...
Update llama.cpp
2023-03-25 16:26:03 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API enum llama_token_attr llama_vocab_get_attr(const struct llama_vocab * vocab, llama_token token);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
"llama_vocab_get_attr", [llama_vocab_p_ctypes, llama_token], ctypes.c_int
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
def llama_vocab_get_attr(
vocab: llama_vocab_p, token: Union[llama_token, int], /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> int: ...
Merge branch main into custom_rope
2023-07-15 15:11:01 -04:00
feat: Update llama.cpp
2024-04-21 20:46:40 -04:00
# // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API bool llama_vocab_is_eog(const struct llama_vocab * vocab, llama_token token);
feat: Update llama.cpp
2024-04-21 20:46:40 -04:00
@ctypes_function(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
"llama_vocab_is_eog", [llama_vocab_p_ctypes, llama_token], ctypes.c_bool
feat: Update llama.cpp
2024-04-21 20:46:40 -04:00
)
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
def llama_vocab_is_eog(vocab: llama_vocab_p, token: Union[llama_token, int], /) -> bool:
feat: Update llama.cpp
2024-04-21 20:46:40 -04:00
"""Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)"""
...
feat: Update llama.cpp
2024-05-27 10:51:57 -04:00
# // Identify if Token Id is a control token or a render-able token
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API bool llama_vocab_is_control(const struct llama_vocab * vocab, llama_token token);
feat: Update llama.cpp
2024-05-27 10:51:57 -04:00
@ctypes_function(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
"llama_vocab_is_control", [llama_vocab_p_ctypes, llama_token], ctypes.c_bool
feat: Update llama.cpp
2024-05-27 10:51:57 -04:00
)
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
def llama_vocab_is_control(
vocab: llama_vocab_p, token: Union[llama_token, int], /
fix(misc): Format
2024-07-09 12:20:17 -04:00
) -> bool:
feat: Update llama.cpp
2024-05-27 10:51:57 -04:00
"""Identify if Token Id is a control token or a render-able token"""
...
Update llama.cpp
2023-08-24 00:17:00 -04:00
# // Special tokens
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API llama_token llama_vocab_bos(const struct llama_vocab * vocab); // beginning-of-sentence
@ctypes_function("llama_vocab_bos", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_bos(vocab: llama_vocab_p, /) -> llama_token:
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""beginning-of-sentence"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Merge branch main into custom_rope
2023-07-15 15:11:01 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API llama_token llama_vocab_eos(const struct llama_vocab * vocab); // end-of-sentence
@ctypes_function("llama_vocab_eos", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_eos(vocab: llama_vocab_p, /) -> llama_token:
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""end-of-sentence"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Merge branch main into custom_rope
2023-07-15 15:11:01 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API llama_token llama_vocab_eot(const struct llama_vocab * vocab); // end-of-turn
@ctypes_function("llama_vocab_eot", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_eot(vocab: llama_vocab_p, /) -> llama_token:
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
"""end-of-turn"""
...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API llama_token llama_vocab_sep(const struct llama_vocab * vocab); // sentence separator
@ctypes_function("llama_vocab_sep", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_sep(vocab: llama_vocab_p, /) -> llama_token:
"""sentence separator"""
feat: Update llama.cpp
2024-04-10 02:25:58 -04:00
...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API llama_token llama_vocab_nl (const struct llama_vocab * vocab); // next-line
@ctypes_function("llama_vocab_nl", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_nl(vocab: llama_vocab_p, /) -> llama_token:
"""next-line"""
feat: Update llama.cpp
2024-04-10 02:25:58 -04:00
...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API llama_token llama_vocab_pad(const struct llama_vocab * vocab); // padding
@ctypes_function("llama_vocab_pad", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_pad(vocab: llama_vocab_p, /) -> llama_token:
"""padding"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-08-24 00:17:00 -04:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
feat: Update llama.cpp
2025-08-07 06:40:18 -07:00
# LLAMA_API llama_token llama_vocab_mask(const struct llama_vocab * vocab); // mask
@ctypes_function("llama_vocab_mask", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_mask(vocab: llama_vocab_p, /) -> llama_token:
"""mask"""
...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_get_add_bos",
[llama_vocab_p_ctypes],
ctypes.c_bool,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_vocab_get_add_bos(vocab: llama_vocab_p, /) -> bool: ...
Update llama.cpp
2023-08-24 00:17:00 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_get_add_eos",
[llama_vocab_p_ctypes],
ctypes.c_bool,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_vocab_get_add_eos(vocab: llama_vocab_p, /) -> bool: ...
Update llama.cpp
2023-11-20 14:11:33 -05:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_get_add_sep",
[llama_vocab_p_ctypes],
ctypes.c_bool,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_vocab_get_add_sep(vocab: llama_vocab_p, /) -> bool: ...
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_fim_pre",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_vocab_fim_pre(vocab: llama_vocab_p, /) -> llama_token: ...
Update llama.cpp
2023-11-20 14:11:33 -05:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_fim_suf",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_vocab_fim_suf(vocab: llama_vocab_p, /) -> llama_token: ...
Update llama.cpp
2023-10-03 15:23:35 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_fim_mid",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_vocab_fim_mid(vocab: llama_vocab_p, /) -> llama_token: ...
Update llama.cpp
2023-10-03 15:23:35 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API llama_token llama_vocab_fim_pad(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_fim_pad",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_vocab_fim_pad(vocab: llama_vocab_p, /) -> llama_token: ...
Update llama.cpp
2023-10-03 15:23:35 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_fim_rep",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_vocab_fim_rep(vocab: llama_vocab_p, /) -> llama_token: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_fim_sep",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_vocab_fim_sep(vocab: llama_vocab_p, /) -> llama_token: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# DEPRECATED functions
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead");
@ctypes_function(
"llama_token_get_text",
[llama_vocab_p_ctypes, llama_token],
ctypes.c_char_p,
)
def llama_token_get_text(
vocab: llama_vocab_p, token: Union[llama_token, int], /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> bytes: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead");
@ctypes_function(
"llama_token_get_score",
[llama_vocab_p_ctypes, llama_token],
ctypes.c_float,
)
def llama_token_get_score(
vocab: llama_vocab_p, token: Union[llama_token, int], /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> float: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead");
@ctypes_function(
"llama_token_get_attr",
[llama_vocab_p_ctypes, llama_token],
ctypes.c_int,
)
def llama_token_get_attr(
vocab: llama_vocab_p, token: Union[llama_token, int], /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> int: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead");
@ctypes_function(
"llama_token_is_eog",
[llama_vocab_p_ctypes, llama_token],
ctypes.c_bool,
)
def llama_token_is_eog(
vocab: llama_vocab_p, token: Union[llama_token, int], /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> bool: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead");
@ctypes_function(
"llama_token_is_control",
[llama_vocab_p_ctypes, llama_token],
ctypes.c_bool,
)
def llama_token_is_control(
vocab: llama_vocab_p, token: Union[llama_token, int], /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> bool: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead");
@ctypes_function(
"llama_token_bos",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_token_bos(vocab: llama_vocab_p, /) -> int: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead");
@ctypes_function(
"llama_token_eos",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_token_eos(vocab: llama_vocab_p, /) -> int: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead");
@ctypes_function(
"llama_token_eot",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_token_eot(vocab: llama_vocab_p, /) -> int: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead");
@ctypes_function(
"llama_token_cls",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_token_cls(vocab: llama_vocab_p, /) -> int: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead");
@ctypes_function(
"llama_token_sep",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_token_sep(vocab: llama_vocab_p, /) -> int: ...
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead");
@ctypes_function(
"llama_token_nl",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_token_nl(vocab: llama_vocab_p, /) -> int: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead");
@ctypes_function(
"llama_token_pad",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_token_pad(vocab: llama_vocab_p, /) -> int: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead");
@ctypes_function(
"llama_add_bos_token",
[llama_vocab_p_ctypes],
ctypes.c_bool,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_add_bos_token(vocab: llama_vocab_p, /) -> bool: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead");
@ctypes_function(
"llama_add_eos_token",
[llama_vocab_p_ctypes],
ctypes.c_bool,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_add_eos_token(vocab: llama_vocab_p, /) -> bool: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead");
@ctypes_function(
"llama_token_fim_pre",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_token_fim_pre(vocab: llama_vocab_p, /) -> llama_token: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead");
@ctypes_function(
"llama_token_fim_suf",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_token_fim_suf(vocab: llama_vocab_p, /) -> llama_token: ...
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead");
@ctypes_function(
"llama_token_fim_mid",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_token_fim_mid(vocab: llama_vocab_p, /) -> llama_token: ...
Update llama.cpp
2023-10-03 15:23:35 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead");
@ctypes_function(
"llama_token_fim_pad",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_token_fim_pad(vocab: llama_vocab_p, /) -> llama_token: ...
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead");
@ctypes_function(
"llama_token_fim_rep",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_token_fim_rep(vocab: llama_vocab_p, /) -> llama_token: ...
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead");
@ctypes_function(
"llama_token_fim_sep",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_token_fim_sep(vocab: llama_vocab_p, /) -> llama_token: ...
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# // CLS is equivalent to BOS
# DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification
# "use llama_vocab_bos instead");
@ctypes_function(
"llama_vocab_cls",
[llama_vocab_p_ctypes],
llama_token,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_vocab_cls(vocab: llama_vocab_p, /) -> llama_token: ...
Update llama.cpp
2023-10-03 15:23:35 -04:00
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
Update llama.cpp
2023-08-24 00:17:00 -04:00
# //
# // Tokenization
# //
feat: Update llama.cpp
2024-09-29 15:23:24 -04:00
# // The API is thread-safe.
# //
Update llama.cpp
2023-08-24 00:17:00 -04:00
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
Update llama.cpp
2023-10-19 02:55:08 -04:00
# /// @details Convert the provided text into tokens.
# /// @param tokens The tokens pointer must be large enough to hold the resulting tokens.
feat: Update llama.cpp
2024-03-13 15:57:35 -04:00
# /// @return Returns the number of tokens on success, no more than n_tokens_max
Update llama.cpp
2023-10-19 02:55:08 -04:00
# /// @return Returns a negative number on failure - the number of tokens that would have been returned
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# /// @return Returns INT32_MIN on overflow (e.g., tokenization result size exceeds int32_t limit)
feat: Update llama.cpp
2024-07-05 20:44:46 -04:00
# /// @param add_special Allow to add BOS and EOS tokens if model is configured to do so.
feat: Update llama.cpp
2024-04-10 02:25:58 -04:00
# /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated
# /// as plaintext. Does not insert a leading space.
Update llama.cpp
2024-01-03 22:04:04 -05:00
# LLAMA_API int32_t llama_tokenize(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# const struct llama_vocab * vocab,
Update llama.cpp
2023-10-19 02:55:08 -04:00
# const char * text,
Update llama.cpp
2024-01-03 22:04:04 -05:00
# int32_t text_len,
Update llama.cpp
2023-10-19 02:55:08 -04:00
# llama_token * tokens,
feat: Update llama.cpp
2024-03-13 15:57:35 -04:00
# int32_t n_tokens_max,
feat: Update llama.cpp
2024-04-10 02:25:58 -04:00
# bool add_special,
# bool parse_special);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_tokenize",
[
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
llama_vocab_p_ctypes,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
ctypes.c_char_p,
ctypes.c_int32,
llama_token_p,
ctypes.c_int32,
ctypes.c_bool,
ctypes.c_bool,
],
ctypes.c_int32,
)
Update llama.cpp
2023-10-19 02:55:08 -04:00
def llama_tokenize(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
vocab: llama_vocab_p,
Update llama.cpp
2023-10-19 02:55:08 -04:00
text: bytes,
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
text_len: Union[ctypes.c_int, int],
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
tokens: CtypesArray[llama_token],
feat: Update llama.cpp
2024-03-13 15:57:35 -04:00
n_tokens_max: Union[ctypes.c_int, int],
feat: Update llama.cpp
2024-04-10 02:25:58 -04:00
add_special: Union[ctypes.c_bool, bool],
parse_special: Union[ctypes.c_bool, bool],
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
/,
Update llama.cpp
2023-10-19 02:55:08 -04:00
) -> int:
feat: Update llama.cpp
2024-03-13 15:57:35 -04:00
"""Convert the provided text into tokens.
feat: Update llama.cpp
2024-03-18 10:26:36 -04:00
feat: Update llama.cpp
2024-03-13 15:57:35 -04:00
Args:
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
vocab: The vocabulary to use for tokenization.
feat: Update llama.cpp
2024-03-13 15:57:35 -04:00
text: The text to tokenize.
text_len: The length of the text.
tokens: The tokens pointer must be large enough to hold the resulting tokens.
n_max_tokens: The maximum number of tokens to return.
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
add_special: Allow adding special tokens if the model is configured to do so.
feat: Update llama.cpp
2024-04-10 02:25:58 -04:00
parse_special: Allow parsing special tokens.
feat: Update llama.cpp
2024-03-18 10:26:36 -04:00
feat: Update llama.cpp
2024-03-13 15:57:35 -04:00
Returns:
Returns the number of tokens on success, no more than n_tokens_max
feat: Update llama.cpp
2024-03-18 10:26:36 -04:00
Returns a negative number on failure - the number of tokens that would have been returned
"""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-10-19 02:55:08 -04:00
Update llama.cpp
2023-08-27 12:59:20 -04:00
# // Token Id -> Piece.
# // Uses the vocabulary in the provided context.
# // Does not write null terminator to the buffer.
feat: Update llama.cpp
2024-07-05 20:44:46 -04:00
# // User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix')
feat: Update llama.cpp
2024-04-21 20:46:40 -04:00
# // @param special If true, special tokens are rendered in the output.
Update llama.cpp
2024-01-03 22:04:04 -05:00
# LLAMA_API int32_t llama_token_to_piece(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# const struct llama_vocab * vocab,
Update llama.cpp
2023-09-28 22:42:03 -04:00
# llama_token token,
# char * buf,
feat: Update llama.cpp
2024-04-21 20:46:40 -04:00
# int32_t length,
feat: Update llama.cpp
2024-07-05 20:44:46 -04:00
# int32_t lstrip,
feat: Update llama.cpp
2024-04-21 20:46:40 -04:00
# bool special);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_token_to_piece",
[
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
llama_vocab_p_ctypes,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
llama_token,
ctypes.c_char_p,
ctypes.c_int32,
feat: Update llama.cpp
2024-07-05 20:44:46 -04:00
ctypes.c_int32,
feat: Update llama.cpp
2024-04-21 20:46:40 -04:00
ctypes.c_bool,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
],
ctypes.c_int32,
)
Update llama.cpp
2023-08-27 12:59:20 -04:00
def llama_token_to_piece(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
vocab: llama_vocab_p,
Update llama.cpp
2023-11-05 16:57:10 -05:00
token: Union[llama_token, int],
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
buf: Union[ctypes.c_char_p, bytes, CtypesArray[ctypes.c_char]],
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
length: Union[ctypes.c_int, int],
feat: Update llama.cpp
2024-07-05 20:44:46 -04:00
lstrip: Union[ctypes.c_int, int],
feat: Update llama.cpp
2024-04-21 20:46:40 -04:00
special: Union[ctypes.c_bool, bool],
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
/,
Update llama.cpp
2023-08-24 00:17:00 -04:00
) -> int:
docs: Add docstrings from llama.cpp
2023-11-23 00:26:26 -05:00
"""Token Id -> Piece.
Uses the vocabulary in the provided context.
Does not write null terminator to the buffer.
Update llama.cpp
2023-11-23 16:26:00 -05:00
User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens.
feat: Update llama.cpp
2024-04-21 20:46:40 -04:00
Args:
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
vocab: The vocabulary to use for tokenization.
feat: Update llama.cpp
2024-04-21 20:46:40 -04:00
token: The token to convert.
buf: The buffer to write the token to.
length: The length of the buffer.
feat: Update llama.cpp
2024-07-05 20:44:46 -04:00
lstrip: The number of leading spaces to skip.
feat: Update llama.cpp
2024-04-21 20:46:40 -04:00
special: If true, special tokens are rendered in the output."""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp and re-organize low-level api
2023-03-24 14:58:42 -04:00
Black formatting
2023-03-24 14:59:29 -04:00
feat: Update llama.cpp
2024-07-05 20:44:46 -04:00
# /// @details Convert the provided tokens into text (inverse of llama_tokenize()).
# /// @param text The char pointer must be large enough to hold the resulting text.
# /// @return Returns the number of chars/bytes on success, no more than text_len_max.
# /// @return Returns a negative number on failure - the number of chars/bytes that would have been returned.
# /// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so.
# /// @param unparse_special If true, special tokens are rendered in the output.
# LLAMA_API int32_t llama_detokenize(
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# const struct llama_vocab * vocab,
feat: Update llama.cpp
2024-07-05 20:44:46 -04:00
# const llama_token * tokens,
# int32_t n_tokens,
# char * text,
# int32_t text_len_max,
# bool remove_special,
# bool unparse_special);
@ctypes_function(
"llama_detokenize",
[
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
llama_vocab_p_ctypes,
feat: Update llama.cpp
2024-07-05 20:44:46 -04:00
ctypes.POINTER(llama_token),
ctypes.c_int32,
ctypes.c_char_p,
ctypes.c_int32,
ctypes.c_bool,
ctypes.c_bool,
],
ctypes.c_int32,
)
def llama_detokenize(
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
vocab: llama_vocab_p,
feat: Update llama.cpp
2024-07-05 20:44:46 -04:00
tokens: CtypesArray[llama_token],
n_tokens: Union[ctypes.c_int, int],
text: bytes,
text_len_max: Union[ctypes.c_int, int],
remove_special: Union[ctypes.c_bool, bool],
unparse_special: Union[ctypes.c_bool, bool],
/,
) -> int:
"""Convert the provided tokens into text (inverse of llama_tokenize()).
Args:
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
vocab: The vocabulary to use for tokenization.
feat: Update llama.cpp
2024-07-05 20:44:46 -04:00
tokens: The tokens to convert.
n_tokens: The number of tokens.
text: The buffer to write the text to.
text_len_max: The length of the buffer.
remove_special: Allow to remove BOS and EOS tokens if model is configured to do so.
unparse_special: If true, special tokens are rendered in the output."""
...
feat: Update llama.cpp
2024-07-23 21:26:20 -04:00
# //
# // Chat templates
# //
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
feat: Update llama.cpp
2024-02-19 04:11:34 -05:00
# /// Apply chat template. Inspired by hf apply_chat_template() on python.
# /// Both "model" and "custom_template" are optional, but at least one is required. "custom_template" has higher precedence than "model"
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggml-org/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template
# /// @param tmpl A Jinja template to use for this chat. If this is nullptr, the model's default chat template will be used instead.
feat: Update llama.cpp
2024-02-19 04:11:34 -05:00
# /// @param chat Pointer to a list of multiple llama_chat_message
# /// @param n_msg Number of llama_chat_message in this chat
# /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message.
# /// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages)
# /// @param length The size of the allocated buffer
# /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template.
# LLAMA_API int32_t llama_chat_apply_template(
# const char * tmpl,
# const struct llama_chat_message * chat,
# size_t n_msg,
# bool add_ass,
# char * buf,
# int32_t length);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
"llama_chat_apply_template",
[
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
ctypes.c_char_p, # tmpl
ctypes.POINTER(llama_chat_message), # chat
ctypes.c_size_t, # n_msg
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
ctypes.c_bool, # add_ass (added)
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
ctypes.c_char_p, # buf
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
ctypes.c_int32, # length
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
],
ctypes.c_int32,
)
feat: Update llama.cpp
2024-02-19 04:11:34 -05:00
def llama_chat_apply_template(
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
tmpl: bytes,
chat: CtypesArray[llama_chat_message],
n_msg: int,
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
add_ass: bool, # Added parameter
buf: bytes,
length: int,
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
/,
feat: Update llama.cpp
2024-09-18 20:07:48 -04:00
) -> int:
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
"""Apply chat template.
Args:
tmpl: Template to use. If None, uses model's default
chat: Array of chat messages
n_msg: Number of messages
add_ass: Whether to end prompt with assistant token
buf: Output buffer
length: Buffer length
Returns:
Number of bytes written, or needed if buffer too small
"""
feat: Update llama.cpp
2024-09-18 20:07:48 -04:00
...
feat: Update llama.cpp
2024-02-19 04:11:34 -05:00
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
feat: Update llama.cpp
2024-12-06 02:33:55 -05:00
# // Get list of built-in chat templates
# LLAMA_API int32_t llama_chat_builtin_templates(const char ** output, size_t len);
@ctypes_function(
"llama_chat_builtin_templates",
[
ctypes.POINTER(ctypes.c_char_p),
ctypes.c_size_t,
],
ctypes.c_int32,
)
def llama_chat_builtin_templates(
output: CtypesArray[bytes],
len: Union[ctypes.c_size_t, int],
/,
) -> int:
"""Get list of built-in chat templates.
Args:
output: Output buffer to store template names.
len: Length of the output buffer.
Returns:
Number of templates available.
Returns a negative number on error.
"""
...
Update llama.cpp
2023-08-24 00:17:00 -04:00
# //
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# // Sampling API
# //
Update llama.cpp
2023-08-24 00:17:00 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# typedef void * llama_sampler_context_t;
llama_sampler_context_t = ctypes.c_void_p
Update llama.cpp
2023-08-24 00:17:00 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# struct llama_sampler_data {
# struct ggml_tensor * logits;
# struct ggml_tensor * probs;
# struct ggml_tensor * sampled;
# struct ggml_tensor * candidates;
# };
class llama_sampler_data(ctypes.Structure):
_fields_ = [
("logits", ctypes.c_void_p),
("probs", ctypes.c_void_p),
("sampled", ctypes.c_void_p),
("candidates", ctypes.c_void_p),
]
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# // user code can implement the interface below in order to create custom llama_sampler
# struct llama_sampler_i {
# const char * (*name) (const struct llama_sampler * smpl); // can be NULL
# void (*accept)( struct llama_sampler * smpl, llama_token token); // can be NULL
# void (*apply) ( struct llama_sampler * smpl, llama_token_data_array * cur_p); // required
# void (*reset) ( struct llama_sampler * smpl); // can be NULL
# struct llama_sampler * (*clone) (const struct llama_sampler * smpl); // can be NULL if ctx is NULL
# void (*free) ( struct llama_sampler * smpl); // can be NULL if ctx is NULL
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# // TODO: API for internal libllama usage for appending the sampling to an existing ggml_cgraph
# //void (*apply_ggml) (struct llama_sampler * smpl, ...);
# };
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
class llama_sampler_i(ctypes.Structure): ...
Update llama.cpp
2023-07-24 13:04:34 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# struct llama_sampler {
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# const struct llama_sampler_i * iface;
# llama_sampler_context_t ctx;
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# };
class llama_sampler(ctypes.Structure):
_fields_ = [
("iface", ctypes.POINTER(llama_sampler_i)),
("ctx", llama_sampler_context_t),
]
if TYPE_CHECKING:
llama_sampler_p = CtypesPointer[llama_sampler]
llama_sampler_p_ctypes = ctypes.POINTER(llama_sampler)
llama_sampler_i_name = ctypes.CFUNCTYPE(ctypes.c_char_p, llama_sampler_p_ctypes)
llama_sampler_i_accept = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes, llama_token)
llama_sampler_i_apply = ctypes.CFUNCTYPE(
None, llama_sampler_p_ctypes, llama_token_data_array_p
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
llama_sampler_i_reset = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes)
llama_sampler_i_clone = ctypes.CFUNCTYPE(llama_sampler_p_ctypes, llama_sampler_p_ctypes)
llama_sampler_i_free = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes)
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
llama_sampler_i_backend_init = ctypes.CFUNCTYPE(
ctypes.c_bool, llama_sampler_p_ctypes, ctypes.c_void_p
)
llama_sampler_i_backend_accept = ctypes.CFUNCTYPE(
None,
llama_sampler_p_ctypes,
ctypes.c_void_p,
ctypes.c_void_p,
ctypes.c_void_p,
)
llama_sampler_i_backend_apply = ctypes.CFUNCTYPE(
None,
llama_sampler_p_ctypes,
ctypes.c_void_p,
ctypes.c_void_p,
ctypes.POINTER(llama_sampler_data),
)
llama_sampler_i_backend_set_input = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes)
Update llama.cpp
2023-07-24 13:04:34 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
llama_sampler_i._fields_ = [
("name", llama_sampler_i_name),
("accept", llama_sampler_i_accept),
("apply", llama_sampler_i_apply),
("reset", llama_sampler_i_reset),
("clone", llama_sampler_i_clone),
("free", llama_sampler_i_free),
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
("backend_init", llama_sampler_i_backend_init),
("backend_accept", llama_sampler_i_backend_accept),
("backend_apply", llama_sampler_i_backend_apply),
("backend_set_input", llama_sampler_i_backend_set_input),
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
]
Update llama.cpp
2023-07-24 13:04:34 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# // attach a sampler to the context
# LLAMA_API bool llama_set_sampler(struct llama_context * ctx, llama_seq_id seq_id, struct llama_sampler * smpl);
@ctypes_function(
"llama_set_sampler",
[llama_context_p_ctypes, llama_seq_id, llama_sampler_p_ctypes],
ctypes.c_bool,
)
def llama_set_sampler(
ctx: llama_context_p, seq_id: Union[llama_seq_id, int], smpl: llama_sampler_p, /
) -> bool:
"""Attach a sampler to the context."""
...
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# // mirror of llama_sampler_i:
feat: Update llama.cpp
2025-03-12 04:11:42 -04:00
# LLAMA_API struct llama_sampler * llama_sampler_init (const struct llama_sampler_i * iface, llama_sampler_context_t ctx);
@ctypes_function(
"llama_sampler_init",
[ctypes.POINTER(llama_sampler_i), llama_sampler_context_t],
llama_sampler_p_ctypes,
)
def llama_sampler_init(
iface: ctypes.POINTER(llama_sampler_i), ctx: llama_sampler_context_t, /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> llama_sampler_p: ...
feat: Update llama.cpp
2025-03-12 04:11:42 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API const char * llama_sampler_name (const struct llama_sampler * smpl);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_sampler_name",
[llama_sampler_p_ctypes],
ctypes.c_char_p,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_name(smpl: llama_sampler_p, /) -> bytes: ...
Update llama.cpp
2023-08-25 14:35:53 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API void llama_sampler_accept( struct llama_sampler * smpl, llama_token token);
feat: Update llama.cpp
2024-07-23 21:26:20 -04:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_sampler_accept",
[llama_sampler_p_ctypes, llama_token],
feat: Update llama.cpp
2024-07-23 21:26:20 -04:00
None,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_accept(smpl: llama_sampler_p, token: Union[llama_token, int], /): ...
feat: Update llama.cpp
2024-07-23 21:26:20 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API void llama_sampler_apply ( struct llama_sampler * smpl, llama_token_data_array * cur_p);
feat: Update llama.cpp
2024-07-23 21:26:20 -04:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_sampler_apply",
[llama_sampler_p_ctypes, llama_token_data_array_p],
feat: Update llama.cpp
2024-07-23 21:26:20 -04:00
None,
)
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
def llama_sampler_apply(
smpl: llama_sampler_p, cur_p: CtypesArray[llama_token_data_array], /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
): ...
feat: Update llama.cpp
2024-07-23 21:26:20 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API void llama_sampler_reset ( struct llama_sampler * smpl);
feat: Update llama.cpp
2024-07-23 21:26:20 -04:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_sampler_reset",
[llama_sampler_p_ctypes],
feat: Update llama.cpp
2024-07-23 21:26:20 -04:00
None,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_reset(smpl: llama_sampler_p, /): ...
feat: Update llama.cpp
2024-07-23 21:26:20 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API struct llama_sampler * llama_sampler_clone (const struct llama_sampler * smpl);
@ctypes_function(
"llama_sampler_clone",
[llama_sampler_p_ctypes],
llama_sampler_p_ctypes,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: ...
Fix return types and import comments
2023-05-01 14:02:06 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add)
# LLAMA_API void llama_sampler_free ( struct llama_sampler * smpl);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_sampler_free",
[llama_sampler_p_ctypes],
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
None,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_free(smpl: llama_sampler_p, /): ...
Update llama.cpp
2023-09-28 22:42:03 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# // llama_sampler_chain
# // a type of llama_sampler that can chain multiple samplers one after another
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API struct llama_sampler * llama_sampler_chain_init(struct llama_sampler_chain_params params);
@ctypes_function(
"llama_sampler_chain_init",
[llama_sampler_chain_params],
llama_sampler_p_ctypes,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_chain_init(
params: llama_sampler_chain_params, /
) -> llama_sampler_p: ...
Black formatting
2023-03-24 14:35:41 -04:00
Initial commit
2023-03-23 05:33:06 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# // important: takes ownership of the sampler object and will free it when llama_sampler_free is called
# LLAMA_API void llama_sampler_chain_add( struct llama_sampler * chain, struct llama_sampler * smpl);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_sampler_chain_add",
[llama_sampler_p_ctypes, llama_sampler_p_ctypes],
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
None,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_chain_add(chain: llama_sampler_p, smpl: llama_sampler_p, /): ...
Update llama.cpp
2024-01-15 10:12:10 -05:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API struct llama_sampler * llama_sampler_chain_get(const struct llama_sampler * chain, int32_t i);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_sampler_chain_get",
[llama_sampler_p_ctypes, ctypes.c_int32],
llama_sampler_p_ctypes,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
def llama_sampler_chain_get(
chain: llama_sampler_p, i: Union[ctypes.c_int32, int], /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> llama_sampler_p: ...
Update llama.cpp
2023-05-01 10:44:28 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API int llama_sampler_chain_n (const struct llama_sampler * chain);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_sampler_chain_n",
[llama_sampler_p_ctypes],
ctypes.c_int,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_chain_n(chain: llama_sampler_p, /) -> int: ...
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed
# LLAMA_API struct llama_sampler * llama_sampler_chain_remove( struct llama_sampler * chain, int32_t i);
@ctypes_function(
"llama_sampler_chain_remove",
[llama_sampler_p_ctypes, ctypes.c_int32],
llama_sampler_p_ctypes,
)
def llama_sampler_chain_remove(
chain: llama_sampler_p, i: Union[ctypes.c_int32, int], /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> llama_sampler_p: ...
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# // available samplers:
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
# LLAMA_API struct llama_sampler * llama_sampler_init_greedy(void);
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
@ctypes_function("llama_sampler_init_greedy", [], llama_sampler_p_ctypes)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_init_greedy() -> llama_sampler_p: ...
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
# LLAMA_API struct llama_sampler * llama_sampler_init_dist (uint32_t seed);
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
@ctypes_function("llama_sampler_init_dist", [ctypes.c_uint32], llama_sampler_p_ctypes)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_init_dist(seed: int) -> llama_sampler_p: ...
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
feat: Update llama.cpp
2025-05-08 06:52:52 -04:00
# /// Setting k <= 0 makes this a noop
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k);
@ctypes_function("llama_sampler_init_top_k", [ctypes.c_int32], llama_sampler_p_ctypes)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_init_top_k(k: int) -> llama_sampler_p: ...
Update llama.cpp
2023-05-01 10:44:28 -04:00
Update llama.cpp
2023-10-31 21:29:35 -04:00
# /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API struct llama_sampler * llama_sampler_init_top_p (float p, size_t min_keep);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_sampler_init_top_p",
[ctypes.c_float, ctypes.c_size_t],
llama_sampler_p_ctypes,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_init_top_p(p: float, min_keep: int) -> llama_sampler_p: ...
Update llama.cpp
2023-05-01 10:44:28 -04:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# /// @details Minimum P sampling as described in https://github.com/ggml-org/llama.cpp/pull/3841
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API struct llama_sampler * llama_sampler_init_min_p (float p, size_t min_keep);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_sampler_init_min_p",
[ctypes.c_float, ctypes.c_size_t],
llama_sampler_p_ctypes,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_init_min_p(p: float, min_keep: int) -> llama_sampler_p: ...
Update llama.cpp
2023-10-31 21:29:35 -04:00
# /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API struct llama_sampler * llama_sampler_init_typical (float p, size_t min_keep);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_sampler_init_typical",
[ctypes.c_float, ctypes.c_size_t],
llama_sampler_p_ctypes,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_init_typical(p: float, min_keep: int) -> llama_sampler_p: ...
Update llama.cpp
2023-05-01 10:44:28 -04:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# /// #details Updates the logits l_i` = l_i/t. When t <= 0.0f, the maximum logit is kept at it's original value, the rest are set to -inf
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API struct llama_sampler * llama_sampler_init_temp (float t);
@ctypes_function("llama_sampler_init_temp", [ctypes.c_float], llama_sampler_p_ctypes)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_init_temp(t: float) -> llama_sampler_p: ...
Update llama.cpp
2024-01-26 11:45:48 -05:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772.
# LLAMA_API struct llama_sampler * llama_sampler_init_temp_ext (float t, float delta, float exponent);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_sampler_init_temp_ext",
[ctypes.c_float, ctypes.c_float, ctypes.c_float],
llama_sampler_p_ctypes,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
def llama_sampler_init_temp_ext(
t: float, delta: float, exponent: float
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> llama_sampler_p: ...
Update llama.cpp
2023-09-28 22:42:03 -04:00
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
# /// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335
# LLAMA_API struct llama_sampler * llama_sampler_init_xtc (float p, float t, size_t min_keep, uint32_t seed);
@ctypes_function(
"llama_sampler_init_xtc",
[ctypes.c_float, ctypes.c_float, ctypes.c_size_t, ctypes.c_uint32],
llama_sampler_p_ctypes,
)
def llama_sampler_init_xtc(
p: float, t: float, min_keep: int, seed: int, /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> llama_sampler_p: ...
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
feat: Update llama.cpp
2025-03-12 04:11:42 -04:00
# /// @details Top n sigma sampling as described in academic paper "Top-nσ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641
# LLAMA_API struct llama_sampler * llama_sampler_init_top_n_sigma(float n);
@ctypes_function(
"llama_sampler_init_top_n_sigma",
[ctypes.c_float],
llama_sampler_p_ctypes,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_init_top_n_sigma(n: float, /) -> llama_sampler_p: ...
feat: Update llama.cpp
2025-03-12 04:11:42 -04:00
Update llama.cpp
2023-10-31 21:29:35 -04:00
# /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API struct llama_sampler * llama_sampler_init_mirostat(
# int32_t n_vocab,
# uint32_t seed,
# float tau,
# float eta,
# int32_t m);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_sampler_init_mirostat",
[ctypes.c_int32, ctypes.c_uint32, ctypes.c_float, ctypes.c_float, ctypes.c_int32],
llama_sampler_p_ctypes,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
def llama_sampler_init_mirostat(
n_vocab: int, seed: int, tau: float, eta: float, m: int, /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> llama_sampler_p: ...
Update llama.cpp
2023-05-01 10:44:28 -04:00
Update llama.cpp
2023-10-31 21:29:35 -04:00
# /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API struct llama_sampler * llama_sampler_init_mirostat_v2(
# uint32_t seed,
# float tau,
# float eta);
@ctypes_function(
"llama_sampler_init_mirostat_v2",
[ctypes.c_uint32, ctypes.c_float, ctypes.c_float],
llama_sampler_p_ctypes,
)
def llama_sampler_init_mirostat_v2(
seed: int, tau: float, eta: float, /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> llama_sampler_p: ...
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
feat: Update llama.cpp
2025-04-11 03:28:20 -04:00
# /// @details Intializes a GBNF grammar, see grammars/README.md for details.
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API struct llama_sampler * llama_sampler_init_grammar(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# const struct llama_vocab * vocab,
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# const char * grammar_str,
# const char * grammar_root);
@ctypes_function(
"llama_sampler_init_grammar",
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
[llama_vocab_p_ctypes, ctypes.c_char_p, ctypes.c_char_p],
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
llama_sampler_p_ctypes,
)
def llama_sampler_init_grammar(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
vocab: llama_vocab_p, grammar_str: bytes, grammar_root: bytes, /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> llama_sampler_p: ...
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy(
# const struct llama_vocab * vocab,
# const char * grammar_str,
# const char * grammar_root,
# const char ** trigger_words,
# size_t num_trigger_words,
# const llama_token * trigger_tokens,
# size_t num_trigger_tokens),
# "use llama_sampler_init_grammar_lazy_patterns instead");
@ctypes_function(
"llama_sampler_init_grammar_lazy",
[
llama_vocab_p_ctypes,
ctypes.c_char_p,
ctypes.c_char_p,
ctypes.POINTER(ctypes.c_char_p),
ctypes.c_size_t,
ctypes.POINTER(llama_token),
ctypes.c_size_t,
],
llama_sampler_p_ctypes,
)
def llama_sampler_init_grammar_lazy(
vocab: llama_vocab_p,
grammar_str: bytes,
grammar_root: bytes,
trigger_words: CtypesArray[bytes],
num_trigger_words: int,
trigger_tokens: CtypesArray[llama_token],
num_trigger_tokens: int,
/,
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> llama_sampler_p: ...
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
feat: Update llama.cpp
2025-03-12 04:11:42 -04:00
# /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639
# LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy_patterns(
# const struct llama_vocab * vocab,
# const char * grammar_str,
# const char * grammar_root,
# const char ** trigger_patterns,
# size_t num_trigger_patterns,
# const llama_token * trigger_tokens,
# size_t num_trigger_tokens);
@ctypes_function(
"llama_sampler_init_grammar_lazy_patterns",
[
llama_vocab_p_ctypes,
ctypes.c_char_p,
ctypes.c_char_p,
ctypes.POINTER(ctypes.c_char_p),
ctypes.c_size_t,
ctypes.POINTER(llama_token),
ctypes.c_size_t,
],
llama_sampler_p_ctypes,
)
def llama_sampler_init_grammar_lazy_patterns(
vocab: llama_vocab_p,
grammar_str: bytes,
grammar_root: bytes,
trigger_patterns: CtypesArray[bytes],
num_trigger_patterns: int,
trigger_tokens: CtypesArray[llama_token],
num_trigger_tokens: int,
/,
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> llama_sampler_p: ...
feat: Update llama.cpp
2025-03-12 04:11:42 -04:00
feat: Update llama.cpp
2024-12-19 01:55:12 -05:00
# /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first.
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API struct llama_sampler * llama_sampler_init_penalties(
feat: Update llama.cpp
2024-12-19 01:55:12 -05:00
# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size)
# float penalty_repeat, // 1.0 = disabled
# float penalty_freq, // 0.0 = disabled
# float penalty_present); // 0.0 = disabled
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
@ctypes_function(
"llama_sampler_init_penalties",
feat: Update llama.cpp
2024-12-19 01:55:12 -05:00
[ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_float],
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
llama_sampler_p_ctypes,
)
def llama_sampler_init_penalties(
penalty_last_n: int,
penalty_repeat: float,
penalty_freq: float,
penalty_present: float,
misc: additional type annotations for low level api
2024-02-22 02:00:09 -05:00
/,
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> llama_sampler_p: ...
Update llama.cpp
2023-05-01 10:44:28 -04:00
feat: Update llama.cpp
2024-10-31 13:50:18 -04:00
# /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982
# LLAMA_API struct llama_sampler * llama_sampler_init_dry(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# const struct llama_vocab * vocab,
# int32_t n_ctx_train,
feat: Update llama.cpp
2024-10-31 13:50:18 -04:00
# float dry_multiplier,
# float dry_base,
# int32_t dry_allowed_length,
# int32_t dry_penalty_last_n,
# const char ** seq_breakers,
# size_t num_breakers);
@ctypes_function(
"llama_sampler_init_dry",
[
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
llama_vocab_p_ctypes,
ctypes.c_int32,
feat: Update llama.cpp
2024-10-31 13:50:18 -04:00
ctypes.c_float,
ctypes.c_float,
ctypes.c_int32,
ctypes.c_int32,
ctypes.POINTER(ctypes.c_char_p),
ctypes.c_size_t,
],
llama_sampler_p_ctypes,
)
def llama_sampler_init_dry(
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
vocab: llama_vocab_p,
n_ctx_train: int,
feat: Update llama.cpp
2024-10-31 13:50:18 -04:00
dry_multiplier: float,
dry_base: float,
dry_allowed_length: int,
dry_penalty_last_n: int,
feat: Update llama.cpp
2025-03-12 04:11:42 -04:00
seq_breakers,
feat: Update llama.cpp
2024-10-31 13:50:18 -04:00
num_breakers: int,
/,
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> llama_sampler_p: ...
feat: Update llama.cpp
2024-10-31 13:50:18 -04:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# LLAMA_API struct llama_sampler * llama_sampler_init_adaptive_p(
# float target,
# float decay,
# uint32_t seed);
@ctypes_function(
"llama_sampler_init_adaptive_p",
[ctypes.c_float, ctypes.c_float, ctypes.c_uint32],
llama_sampler_p_ctypes,
)
def llama_sampler_init_adaptive_p(
target: float, decay: float, seed: int, /
) -> llama_sampler_p:
"""Initialize an adaptive-p sampler."""
...
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias(
# int32_t n_vocab,
# int32_t n_logit_bias,
# const llama_logit_bias * logit_bias);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_sampler_init_logit_bias",
[ctypes.c_int32, ctypes.c_int32, llama_logit_bias_p],
llama_sampler_p_ctypes,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
def llama_sampler_init_logit_bias(
n_vocab: int, n_logit_bias: int, logit_bias: CtypesArray[llama_logit_bias], /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> llama_sampler_p: ...
Update llama.cpp
2023-05-01 10:44:28 -04:00
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
# // this sampler is meant to be used for fill-in-the-middle infilling
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
# LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_vocab * vocab);
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
@ctypes_function(
"llama_sampler_init_infill",
feat: Update llama.cpp
2025-01-28 21:38:05 -05:00
[llama_vocab_p_ctypes],
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
llama_sampler_p_ctypes,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_init_infill(vocab: llama_vocab_p, /) -> llama_sampler_p: ...
feat: Update llama.cpp
2024-10-22 09:33:48 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# // Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise
# LLAMA_API uint32_t llama_sampler_get_seed(const struct llama_sampler * smpl);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_sampler_get_seed",
[llama_sampler_p_ctypes],
ctypes.c_uint32,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: ...
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# /// @details Sample and accept a token from the idx-th output of the last evaluation
# LLAMA_API llama_token llama_sampler_sample(struct llama_sampler * smpl, struct llama_context * ctx, int32_t idx);
@ctypes_function(
"llama_sampler_sample",
[llama_sampler_p_ctypes, llama_context_p_ctypes, ctypes.c_int32],
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
llama_token,
)
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
def llama_sampler_sample(
smpl: llama_sampler_p, ctx: llama_context_p, idx: int, /
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
) -> int: ...
Update llama.cpp
2023-05-01 10:44:28 -04:00
Update llama.cpp
2023-09-09 12:12:32 -04:00
# //
feat: Update llama.cpp
2024-06-07 02:02:12 -04:00
# // Model split
Update llama.cpp
2023-09-09 12:12:32 -04:00
# //
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
feat: Update llama.cpp
2024-03-22 23:43:29 -04:00
# /// @details Build a split GGUF final path for this chunk.
# LLAMA_API int llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int split_no, int split_count);
@ctypes_function(
"llama_split_path",
[ctypes.c_char_p, ctypes.c_size_t, ctypes.c_char_p, ctypes.c_int, ctypes.c_int],
ctypes.c_int,
)
def llama_split_path(
split_path: bytes,
maxlen: Union[ctypes.c_size_t, int],
path_prefix: bytes,
split_no: Union[ctypes.c_int, int],
split_count: Union[ctypes.c_int, int],
/,
) -> int:
"""Build a split GGUF final path for this chunk."""
...
# /// @details Extract the path prefix from the split_path if and only if the split_no and split_count match.
# LLAMA_API int llama_split_prefix(char * split_prefix, size_t maxlen, const char * split_path, int split_no, int split_count);
@ctypes_function(
"llama_split_prefix",
[ctypes.c_char_p, ctypes.c_size_t, ctypes.c_char_p, ctypes.c_int, ctypes.c_int],
ctypes.c_int,
)
def llama_split_prefix(
split_prefix: bytes,
maxlen: Union[ctypes.c_size_t, int],
split_path: bytes,
split_no: Union[ctypes.c_int, int],
split_count: Union[ctypes.c_int, int],
/,
) -> int:
"""Extract the path prefix from the split_path if and only if the split_no and split_count match."""
...
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# // Print system information
# LLAMA_API const char * llama_print_system_info(void);
@ctypes_function("llama_print_system_info", [], ctypes.c_char_p)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_print_system_info() -> bytes: ...
Update llama.cpp and re-organize low-level api
2023-03-24 14:58:42 -04:00
Black formatting
2023-03-24 14:59:29 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# // Set callback for all future logging events.
# // If this is not called, or NULL is supplied, everything is output on stderr.
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# // The logger state is global so these functions are NOT thread safe.
# LLAMA_API void llama_log_get(ggml_log_callback * log_callback, void ** user_data);
@ctypes_function(
"llama_log_get",
[ctypes.POINTER(llama_log_callback), ctypes.POINTER(ctypes.c_void_p)],
None,
)
def llama_log_get(
log_callback: CtypesPointerOrRef[llama_log_callback],
user_data: CtypesPointerOrRef[ctypes.c_void_p],
/,
):
"""Get the current logging callback and user data."""
...
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_log_set",
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
[llama_log_callback, ctypes.c_void_p],
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
None,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
def llama_log_set(
log_callback: Optional[CtypesFuncPointer],
user_data: ctypes.c_void_p,
/,
):
"""Set callback for all future logging events.
If this is not called, or NULL is supplied, everything is output on stderr."""
feat(low-level-api): Improve API static type-safety and performance (#1205)
2024-02-21 16:25:38 -05:00
...
Update llama.cpp
2023-07-06 17:57:56 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# //
# // Performance utils
# //
# struct llama_perf_context_data {
# double t_start_ms;
# double t_load_ms;
# double t_p_eval_ms;
# double t_eval_ms;
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# int32_t n_p_eval;
# int32_t n_eval;
feat: Update llama.cpp
2025-08-07 06:40:18 -07:00
# int32_t n_reused; // number of times a ggml compute graph had been reused
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# };
class llama_perf_context_data(ctypes.Structure):
_fields_ = [
("t_start_ms", ctypes.c_double),
("t_load_ms", ctypes.c_double),
("t_p_eval_ms", ctypes.c_double),
("t_eval_ms", ctypes.c_double),
("n_p_eval", ctypes.c_int32),
("n_eval", ctypes.c_int32),
feat: Update llama.cpp
2025-08-07 06:40:18 -07:00
("n_reused", ctypes.c_int32),
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
]
# struct llama_perf_sampler_data {
# double t_sample_ms;
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# int32_t n_sample;
# };
class llama_perf_sampler_data(ctypes.Structure):
_fields_ = [
("t_sample_ms", ctypes.c_double),
("n_sample", ctypes.c_int32),
]
# LLAMA_API struct llama_perf_context_data llama_perf_context (const struct llama_context * ctx);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_perf_context",
[llama_context_p_ctypes],
llama_perf_context_data,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_perf_context(ctx: llama_context_p, /) -> llama_perf_context_data: ...
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API void llama_perf_context_print(const struct llama_context * ctx);
@ctypes_function(
"llama_perf_context_print",
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
[llama_context_p_ctypes],
None,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_perf_context_print(ctx: llama_context_p, /): ...
Initial commit
2023-03-23 05:33:06 -04:00
Black formatting
2023-03-24 14:35:41 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API void llama_perf_context_reset( struct llama_context * ctx);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_perf_context_reset",
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
[llama_context_p_ctypes],
None,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_perf_context_reset(ctx: llama_context_p, /): ...
Initial commit
2023-03-23 05:33:06 -04:00
Black formatting
2023-03-24 14:35:41 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# // NOTE: the following work only with samplers constructed via llama_sampler_chain_init
# LLAMA_API struct llama_perf_sampler_data llama_perf_sampler (const struct llama_sampler * chain);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_perf_sampler",
[llama_sampler_p_ctypes],
llama_perf_sampler_data,
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_perf_sampler(chain: llama_sampler_p, /) -> llama_perf_sampler_data: ...
Update llama.cpp and re-organize low-level api
2023-03-24 14:58:42 -04:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain);
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
@ctypes_function(
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
"llama_perf_sampler_print",
[llama_sampler_p_ctypes],
misc: use decorator to bind low level api functions, fixes docs
2024-02-23 03:39:38 -05:00
None,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_perf_sampler_print(chain: llama_sampler_p, /): ...
docs: Improve low-level docstrings
2023-11-27 19:03:02 -05:00
feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test
2024-09-18 20:00:19 -04:00
# LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain);
@ctypes_function(
"llama_perf_sampler_reset",
[llama_sampler_p_ctypes],
None,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_perf_sampler_reset(chain: llama_sampler_p, /): ...
Update llama.cpp
2023-08-24 00:17:00 -04:00
feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry
2026-03-22 18:58:55 -07:00
# // print a breakdown of per-device memory use via LLAMA_LOG:
@ctypes_function("llama_memory_breakdown_print", [llama_context_p_ctypes], None)
def llama_memory_breakdown_print(ctx: llama_context_p, /):
"""Print a breakdown of per-device memory use."""
...
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# //
# // training
# //
# // function that returns whether or not a given tensor contains trainable parameters
# typedef bool (*llama_opt_param_filter)(const struct ggml_tensor * tensor, void * userdata);
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
llama_opt_param_filter = ctypes.CFUNCTYPE(
ctypes.c_bool, ctypes.c_void_p, ctypes.c_void_p
)
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# // always returns true
# LLAMA_API bool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userdata);
@ctypes_function(
"llama_opt_param_filter_all",
[ctypes.c_void_p, ctypes.c_void_p],
ctypes.c_bool,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_opt_param_filter_all(
tensor: ctypes.c_void_p, userdata: ctypes.c_void_p, /
) -> bool: ...
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# struct llama_opt_params {
# uint32_t n_ctx_train; // assumed context size post training, use context size specified in llama_context if 0
# llama_opt_param_filter param_filter; // callback for determining which tensors contain trainable parameters
# void * param_filter_ud; // userdata for determining which tensors contain trainable parameters
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters
# void * get_opt_pars_ud; // userdata for calculating optimizer parameters
# };
class llama_opt_params(ctypes.Structure):
_fields_ = [
("n_ctx_train", ctypes.c_uint32),
("param_filter", llama_opt_param_filter),
("param_filter_ud", ctypes.c_void_p),
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
(
"get_opt_pars",
ctypes.c_void_p,
), # ggml_opt_get_optimizer_params - not implemented here
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
("get_opt_pars_ud", ctypes.c_void_p),
]
# LLAMA_API void llama_opt_init(struct llama_context * lctx, struct llama_model * model, struct llama_opt_params lopt_params);
@ctypes_function(
"llama_opt_init",
[llama_context_p_ctypes, llama_model_p_ctypes, llama_opt_params],
None,
)
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
def llama_opt_init(
lctx: llama_context_p, model: llama_model_p, lopt_params: llama_opt_params, /
): ...
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
# LLAMA_API void llama_opt_epoch(
# struct llama_context * lctx,
# ggml_opt_dataset_t dataset,
# ggml_opt_result_t result_train,
# ggml_opt_result_t result_eval,
# int64_t idata_split,
# ggml_opt_epoch_callback callback_train,
# ggml_opt_epoch_callback callback_eval);
@ctypes_function(
"llama_opt_epoch",
[
llama_context_p_ctypes,
ctypes.c_void_p, # ggml_opt_dataset_t
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
ctypes.c_void_p, # ggml_opt_result_t
feat: Update llama.cpp
2025-07-01 00:51:44 -04:00
ctypes.c_void_p, # ggml_opt_result_t
ctypes.c_int64,
ctypes.c_void_p, # ggml_opt_epoch_callback
ctypes.c_void_p, # ggml_opt_epoch_callback
],
None,
)
def llama_opt_epoch(
lctx: llama_context_p,
dataset: ctypes.c_void_p,
result_train: ctypes.c_void_p,
result_eval: ctypes.c_void_p,
idata_split: int,
callback_train: ctypes.c_void_p,
callback_eval: ctypes.c_void_p,
/,
misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup
2026-03-22 16:12:02 -07:00
): ...
Reference in New Issue Copy Permalink