在这个教程中，我们与Underdog Saluki 27B合作，这是Qwen3.8-27B的7.9GB构件，它的制造者们说它保持工具调用完好无损，每重量约2位，甚至从自己的120任务基准测试上的全54GB模型中脱落。 Saluki基于ISTA-DASLab的GSQ-RCO对同一型号的量化，所以我们把两个文件并存于股票llama.cpp. 我们先从文件本身开始：两个GGUF头的变速器 by-tensor diff，然后是字节级比较，显示是Saluki是ISTA的位数的remix还是一个改变的模型。 然后我们用一个GPU提供每个文件，读取其内存和速度，并在伯克利函数调用Laderboard的120项原始任务上得分，使用BFCL自己的检查器进行严格和宽松的解析。
导入 Os
导入重置
导入点
导入 sys
导入 Json
导入数学
导入副本
导入语句
导入 glob
导入时间
导入类型
导入随机
导入 hipil
导入结构
导入大麻脂
导入 tar 文件
导入平台
导入追溯
导入子进程
导入收藏
导入urlib。请求
导入urlib.error
成果 = 步 - > 单行结果，用于总表
声称 = [#] (步骤、索赔、持有)；持有的是真、假或无(本轮未决定)
现值 = ["1]
def 横幅( 标题):
打印("\n" + "=" * 78)
打印(标题)
打印 ("=" * 78)
def section(name):
fn) 包装：
f 运行( *a, ** kw):
现 [0] = name.split (") [0].
横幅( 名称)
t0 = 时间。 time ()
尝试：
输出 = fn(*a,**kw)
RESULTS[名] = 如果出现( 出，str) 其他"OK"。
返回出来
例外作为e:
RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
打印 (f)\n [!] {name} 未完成：{type (e). name: {e}>
回溯.print exc(限制=3)
返回无
终于：
打印(f)\n [{time.time() - t0: 1f} s] )
json.dump({"结果"):RESULTS,"声称"：声称}，开放(os.path.join(WORK,"结果.json"),"w"，缩进=1).
返回运行
返回包装
索赔(案文，持有):
声称。 附录((CURRENT[0]，文本，持有))
持有的返回
横幅 ("1. 平台、 针头和助手")
System, MACHINE = 平台。 system ()，平台。 机器 ()
ON CUDA = TEM = "Linux"和 shutil.which ("nvidia-smi") 不是无
维护 ON CUDA 或 (SYSTEM = "Darwin" 和 MACHINE = "arm64"), "需要 NVIDIA GPU 在 Linux(Colab T4)或 Apple-硅Mac上"
Work = os.environ.get ("SALUKI WORK") 或 ("/content/saluki"，如果 os.path.isdir ("/content") 或 os.path.abspath ("saluki work") ).
(“模型”、“llama”、“bfcl”、“runs”):
os.makedirs(os.path.join (WORK, d)，存在 ok=True) (英语).
{\fn方正粗倩简体\fs12\an8\1cHFFFF00\b0}笔记本下载的每件东西都由尺寸和SHA -256.
LLAMA TAG = "b11565"#llama.cpp 2026-10-12发布，承诺f9a95aeb2
LLAMA_ASSET = {"Linux": ("llama-b11565-bin-ubuntu-cuda-12.8-x64.tar.gz", 171559844,
"61b8cf4e49739afe4008b06a92390dff999e706e9e7e9749c015544a9f241627")，中国大陆古代名人。
"达尔文":("llama-b11565-bin-macos-arm64.tar.gz",12105509).
"19289548468babcd5131cecccf3a9bc7c67030cc7f0a978dafc0db1ba7891068")}[SYSTEM]
MODELS = { (简体中文)
"萨卢基":dict(repo="康威研究/下犬-萨卢基-27B-1.0"),rev="4f60eba09f4b6db63c72915d328d97e1d93edb8c".
file="Underdog-Saluki-27B-1.0-IQ2-mix.gguf"，体积=7898369152
sha256="4a673518b11b1c4445f9b9a9d3c40356f5ba39f6a6260dcb25f3fbd475d9efb")，是中国大陆的特产。
"ISTA":dict(repo=)"ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF",rev="d562806dbafae37109975e970aae91b43e73b440" (中文(简体) ).
file="Qwen3.8-27B-GSQ-RCO-IQ2 XS.gguf"，尺寸=8422841472.
sha256="f0ae5006da0ce6225935339e4e989369f94de95d2263cf969519f8420c9ae02c")，中国植物物种信息数据库。
}
# BFCL v4 的数据和 BFCL 自己的检查码，在一个大猩猩承诺。
GORILLA = ("https://raw.githubusercontent.com/ShishirPatil/gorilla/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/"
(bfcl eval/)) (中文(简体) ).
BFCL_FILES = {
"数据/BFCL v4 simple python.json":"82d63ba502eb2520c6b5d1d9a5c4b590e03ff261565175561f6228a367d1991".
"数据/BFCL v4 multiple.json":"aef168155ebd74b7ac2401198b201343bc7d16d7a3d7e0d4e6d8ee82c6969b2a".
"数据/BFCL v4 irrelance.json":"2b6ed4c2e992ccf5f1678a701851f944bef750ee026ed1db89efed5be01a6".
"data/BFCL_v4_parallel.json": "19f51a82eff42e5d62541aa500115a056eb78f437c2ba1f10415fd7c8e5dda84",
"数据/BFCL v4 exityl multiple.json":"8863ea84333239f55c5f016154cf0830853c89f693c6ea270396a2f121960579".
"数据/可能回答/BFCL v4 simple python.json":"90cd5bc653690ee8e459b5b3f3fc9458606f7f3fcbb5b51b7dc581f8c86dc".
"数据/可能答案/BFCL v4 multiple.json":"244e00ce9395df948bcafc7bee64e8f9c87ef70887587d83cae45b13699f3047".
"数据/可能回答/BFCL v4 并行.json":"8a6aa19c1addc6a5a2f7e40f9dbf30cc7e95815e7b830c90589ab318229e0f0".
"数据/可能回答/BFCL v4 并行复数.json":"5ebf24f458c1f16300c05505d83d6f0a1b68b79be273a033febd0d4f840507e3".
"eval checker/ast eval/ast checker.py":"2aae7a68461a8f76c0贝3894c8901b66b56967a1989d3ab066051ef3b97f1538" 互联网档案馆的存檔，存档日期2014-03-07.
"constants/enums.py": "2182becfa2a1d071ee1db30db593b4758c6bf866aa12d2d4b8daf09175ea518a",
"constants/type_mappings.py": "1702fb67afbe2c492608e58e2b7d02e46381f50166b47f3c952f76e34c7cd3bd",
"utils.py": "3703b9bb63f83581c60b8e0b82aac74c360c9ca781e99e7d4eb28b0cce2285dd",
"model_handler/utils.py": "f78fd3edce603b333dc9a88ee2c041dc547d51f71aa449ffebc044c4b1e353f3",
}
# 下犬座的五种任务，每个24个；我们自己绘制种子样本，因为任务列表没有公布。
KINDS = [("simple python","调用一个正确的函数"), ("复数", "从几个中选择正确的函数")
("无关","在没有工具适合时知道",("并行","相同的功能，几个呼叫")).
("并行多功能","几种不同的功能").
N_PER_KIND, SEED, CTX, PORT = 24, 2026, 16384, 8080
# llama-server自身默认温度为0.8和最高 k 40，因此每个请求都声明其采样器。
GREEDY = dict(温度=0，上 k=0，上 p=1.0,min p=0)
def sh(cmd, **kw):
返回子进程。 run( cmd，抓取输出= True，文本= True, ** kw)
def sha256 file(路径，块=1 + 22):
h = 大麻脂.sha256()
以开放( path, "rb") 作为 f:
b in iter( lambda: f.read( chunk), b") 中：
h. 更新(b)
返回 h. hexdigest( )
def figure(url, dest，大小=无，sha=无，安静=假):
"""可续下载带卷曲，然后大小和SHA-256检查。 返回已花费的下载秒数。”
t0 = 时间。 time ()
如果不是 (os.path. highs(dest) 和 (大小为 None 或 os.path.getsize(dest) ==)):
试验范围(5):
r = sh(["curl", "-fsSL", "-C", "-", "--retry", "5", "-o", dest, url])
如果 r. returncode = 0 和 (大小为 None 或 os.path.getsize(dest) = 大小):
换行
时间。 sleep( 5 * (尝试+1) )
os.path.getsize (dest) (中文(简体) ).
profile 大小为 None 或者有 == 大小，f" {os.path.basename(dest)}: {got} 字节，期望 {大小}"
若为sha:
profile sha256 field(dest) == sha, f"{os.path.basename(dest)}: SHA-256不匹配".
返回时间。时间() - t0
打印(f)平台：{SYSTEM}{MACHINE}, Python {platform.python version ()}, glibc {platform.libc ver () [1] 或 'n/a'})
如果系统="Linux":
print("os :", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0])
gpu = sh(["nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"]).stdout.strip()
打印 ("gpu:", gpu)
#CUDA 12.8建造的llama.cpp船没有CUDA运行时间；Colab的形象有作为pip轮。
导入导入lib.util
如果导入lib.util.find spec ("nvidia.cuda runtime") 是 None 或导入lib.util.find spec ("nvidia.cublas") 是 None:
subprocess.run([sys.executable, "-m", "pip", "install", "-q", "nvidia-cuda-runtime-cu12==12.9.79",
"nvidia-cublas-cu12 ==12.9.2.10"]，检查=True).
导入 nvidia.cuda 运行时间，nvidia.cublas
CUDA LIBS = [os.path.join(list(m. path )][0),"lib"表示 m in(nvidia.cuda runtime,nvidia.cublas)].
其他情况：
芯片=sh(["sysctl","-n","machdep.cpu.brand string"]). stdout.strip().
mem = int(sh(["sysctl", "-n", "hw.memsize"]).stdout) / 2**30
打印( f) gpu: {chip}, { mem: 0f} GB 统一内存(Metal)".
CUDA_LIBS = []
HW = gpu.split (",") [0] 如果 TET = "Linux" 其他芯片
print(f"work dir : {WORK} ({shutil.disk_usage(WORK).free / 1e9:.0f} GB free)")
笔记本触摸的每样东西都会被钉在安全的地方设置的单元格记录了llama.cpp的发布，既按大小记录了模型文件，又按14个BFCL文件，SHA-256，它为每个请求修正了采样器，因为llama-服务器否则会回到我们没有选择的默认状态。 它在Linux或Apple-硅Mac上探测平台，NVIDIA GPU，在Colab上它确保了CUDA12运行时库预建的llama. cpp需求可以导入。 cpp需求可以导入。
@section ("2. 下载 llama.cpp b11565 和两个2位文件，并检查每个字节与它的标点).
def 下载 step ():
全球 LLAMA DIR, SERVER BIN, SERVER ENV
名称、 大小、 sha = LLAMA ASSET
tar = os.path.join(WORK,"llama"，名称)
获取(f)"https://github.com/ggml-org/llama.cpp/发布/download/{LLAMA TAG}/{name}"，焦油，大小，sha).
以 tarfile.open(tar) 作为 t:
t.extractall(os.path.join(WORK,"llama")，过滤="数据")
SERVER BIN = glob.glob(os.path.join(WORK,"llama","**","llama-服务器")，递归=True) [0]
LLAMA DIR = os.path.dirname(SERVER BIN) 互联网档案馆的存檔，存档日期2013-12-21.
b 在glob.glob(os.path.join(LLAMA DIR,“llama-*”)中):
os.chmod(b, 0o755) (中文(简体) ).
SERVER ENV = dict(os.environ) 维基百科中的相关条目：维基语录链接：名人名言 - 文学作品 - 谚语 - 谚语 - 谚语 - 谚语 - 谚语 - 谚语
如果 CUDA LIBS: (笑声)
SERVER_ENV["LD_LIBRARY_PATH"] = ":".join(CUDA_LIBS + [LLAMA_DIR, SERVER_ENV.get("LD_LIBRARY_PATH", "")])
缺少 = [l for l in sh (['ldd, sERVER BIN], env=SERVER ENV].stdout.splitlines () if "not found" in l]
声明不缺失，f"未解决的库：{ missing}"
r=sh([SERVER BIN,"-版本"),env=SERVER ENV).
ver = re.search(r"version: .*", r.stdout + r.stderr).group(0)
打印(f)"llama.cpp:{name}(SHA-256已选中) - > {ver}")
在BFCL FILES.items(:
dest = os.path.join(WORK, "bfcl", rel.replace("/", "__"))
获取(GORILLA + rel, dest, sha=digest)
print(f)"BFCL v4: {len(BFCL FILES)}来自大猩猩@6ea5797的文件(5个任务文件，4个答案文件，5个检查器来源)，全部SHA-256选中) ).
用于标记，m in MODELS. items( ):
path = os.path.join(WORK, "模型", m["文件"])
= os.path.存在(path)和 os.path.getsize(path) = ["大小"]
url = f"https://huggingface.co/{m['repo']}/resolve/{m['rev']}/{m['file']}"
dl = 获取(url, path, m['size]) # 大小检查，下方为散列
t0 = 时间。 time ()
OK=sha256 file(路径)=["sha256"]
如果在 {dl: 0f} s( { m ['size'] / dl / 1e6: 0f} MB/s) 中有其他 f 下载，则如何 = “ 已经在磁盘上”?
打印( f) { tag: 7s}: { m ['file]} { m [' six']:,} 字节，{how}; SHA-256 { 如果还有“ DOES Not MATCH ”, “ 匹配 ”
f"the Hub's LFS pin ({time.time() - t0:.0f} s to hash)")
c. 与文件的字节相同，以 { m ['repo' } { m ['rev' [:7]], ok)
坚持好
返回 f"llama.cpp {LLAMA TAG}; Saluki 7.90 GB 和 ISTA IQ2 XS 8.42 GB 由 SHA-256 验证"
下载步骤( )
我们下载了llama.cpp b11565的CUDA架构，既包含2位文件，也包含BFCL数据，并对照它的标点检查每个字节。 Hugging Face的下载是可重复的，散列器是在固定的修订版中Hub自己的LFS文摘，所以默默替换的文件会停止笔记本而不是改变其结果。 BFCL 的任务文件，答覆文件，以及检查器代码都来自一个大猩猩的承诺。 手机打印下载时间有多长，这是Colab运行最慢的部分。
GGML_TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 10: ("Q2_K", 256, 84), 12: ("Q4_K", 256, 144),
16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98), 19: ("IQ1_S", 256, 50),
21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136), 29: ("IQ1_M", 256, 56),
30:(“BF16”,1,2”)}# 名称，每个区块的值，每个区块的字节(ggml-common.h)
def 读作 gguf(路径):
"""GGUF v3 标题：元数据，变位表(名称，暗号，类型，偏移)以及变位数据起始位置".
f=开(路径，“rb”)
u = lambda fmt: struct.unpack (" <" + fmt, f.read(struct.calcsize(fmt))) [0] (中文(简体) ).
s = 羊肉：f.read(u ("Q")))) 代码("utf-8","替换").
scalar = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}
def val(t): (中文(简体) ).
如果第8节：
返回 s( )
如果第9节：
et, n = u("I"), u("Q")
返回 [val(et) for in(n)]
返回 u( scalar[t] )
(原始内容存档于2017-03-26). Expressed f.read (4) == b"GGUF"
版本，n tensors,n kv=u("I"),u("Q"),u("Q")
kv = {}
为
本文由 AI图鉴自动抓取并翻译，版权归原作者所有；如需查看原文请点下方链接。
In this tutorial, we work with Underdog Saluki 27B, a 7.9 GB build of Qwen3.8-27B that its makers say keeps tool calling intact at about two bits per weight and even edges out the full 54 GB model on their own 120-task benchmark. Saluki is built on ISTA-DASLab’s GSQ-RCO quantization of the same model, so we put both files side by side in stock llama.cpp. We start with the files themselves: a tensor-by-tensor diff of the two GGUF headers, then a byte-level comparison that shows whether Saluki is a remix of ISTA’s bits or a changed model. We then serve each file on one GPU, read its memory and speed, and score both on the same seeded 120 tasks from the Berkeley Function Calling Leaderboard, using BFCL’s own checker with strict and lenient parsing.
import os
import re
import ast
import sys
import json
import math
import copy
import enum
import glob
import time
import types
import random
import shutil
import struct
import hashlib
import tarfile
import platform
import traceback
import subprocess
import collections
import urllib.request
import urllib.error
RESULTS = {} # step -> one-line result, for the summary table
CLAIMS = [] # (step, claim, held) rows; held is True, False or None (not decided in this run)
CURRENT = ["1"]
def banner(title):
print("\n" + "=" * 78)
print(title)
print("=" * 78)
def section(name):
def wrap(fn):
def run(*a, **kw):
CURRENT[0] = name.split(".")[0]
banner(name)
t0 = time.time()
try:
out = fn(*a, **kw)
RESULTS[name] = out if isinstance(out, str) else "ok"
return out
except Exception as e:
RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}")
traceback.print_exc(limit=3)
return None
finally:
print(f"\n[{time.time() - t0:.1f} s]")
json.dump({"results": RESULTS, "claims": CLAIMS}, open(os.path.join(WORK, "results.json"), "w"), indent=1)
return run
return wrap
def claim(text, held):
CLAIMS.append((CURRENT[0], text, held))
return held
banner("1. Platform, pins and helpers")
SYSTEM, MACHINE = platform.system(), platform.machine()
ON_CUDA = SYSTEM == "Linux" and shutil.which("nvidia-smi") is not None
assert ON_CUDA or (SYSTEM == "Darwin" and MACHINE == "arm64"), "needs an NVIDIA GPU on Linux (Colab T4) or an Apple-silicon Mac"
WORK = os.environ.get("SALUKI_WORK") or ("/content/saluki" if os.path.isdir("/content") else os.path.abspath("saluki_work"))
for d in ("models", "llama", "bfcl", "runs"):
os.makedirs(os.path.join(WORK, d), exist_ok=True)
# Everything the notebook downloads is pinned by size and SHA-256.
LLAMA_TAG = "b11565" # llama.cpp release of 2026-10-12, commit f9a95aeb2
LLAMA_ASSET = {"Linux": ("llama-b11565-bin-ubuntu-cuda-12.8-x64.tar.gz", 171559844,
"61b8cf4e49739afe4008b06a92390dff999e706e9e7e9749c015544a9f241627"),
"Darwin": ("llama-b11565-bin-macos-arm64.tar.gz", 12105509,
"19289548468babcd5131cecccf3a9bc7c67030cc7f0a978dafc0db1ba7891068")}[SYSTEM]
MODELS = {
"Saluki": dict(repo="ConwayResearch/Underdog-Saluki-27B-1.0", rev="4f60eba09f4b6db63c72915d328d97e1d93edb8c",
file="Underdog-Saluki-27B-1.0-IQ2-mix.gguf", size=7898369152,
sha256="4a673518b11b1c4445f9b9a9d3c40356f5ba39f6a6260dcb25f3fbfd475d9efb"),
"ISTA": dict(repo="ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", rev="d562806dbafae37109975e970aae91b43e73b440",
file="Qwen3.8-27B-GSQ-RCO-IQ2_XS.gguf", size=8422841472,
sha256="f0ae5006da0ce6225935339e4e989369f94de95d2263cf969519f8420c9ae02c"),
}
# BFCL v4 data and BFCL's own checker code, at one gorilla commit.
GORILLA = ("https://raw.githubusercontent.com/ShishirPatil/gorilla/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/"
"berkeley-function-call-leaderboard/bfcl_eval/")
BFCL_FILES = {
"data/BFCL_v4_simple_python.json": "82dd63ba502eb2520c6b5d1d9a5c4b590e03ff261565175561f6228a367d1991",
"data/BFCL_v4_multiple.json": "aef168155ebd74b7ac2401198b201343bc7d16d7a3d7e0d4e6d8ee82c6969b2a",
"data/BFCL_v4_irrelevance.json": "2b6ed4c2e992cdcf5f1678a701851f944bef7550ee026ed1ddb89efed5be01a6",
"data/BFCL_v4_parallel.json": "19f51a82eff42e5d62541aa500115a056eb78f437c2ba1f10415fd7c8e5dda84",
"data/BFCL_v4_parallel_multiple.json": "8863ea8433239f55c5f016154cf0830853c89f693c6ea270396a2fa121960579",
"data/possible_answer/BFCL_v4_simple_python.json": "90cd5bc653690ee8e459b5b3f3fc9458606f7f3fcbf795bb51b7dc581f8c86dc",
"data/possible_answer/BFCL_v4_multiple.json": "244e00ce9395df948bcafc7bee64e8f9c87ef70887587d83cae45b13699f3047",
"data/possible_answer/BFCL_v4_parallel.json": "8a6aa19c1adddc6a5a2f7e40f9dbf30cc7e95815e7b830c90589ab318229e0f0",
"data/possible_answer/BFCL_v4_parallel_multiple.json": "5ebf24f458c1f16300c05505d83d6f0a1b68b79be273a033febd0d4f840507e3",
"eval_checker/ast_eval/ast_checker.py": "2aae7a68461a8f76c0be3894c8901b66b56967a1989d3ab066051e3fb97f1538",
"constants/enums.py": "2182becfa2a1d071ee1db30db593b4758c6bf866aa12d2d4b8daf09175ea518a",
"constants/type_mappings.py": "1702fb67afbe2c492608e58e2b7d02e46381f50166b47f3c952f76e34c7cd3bd",
"utils.py": "3703b9bb63f83581c60b8e0b82aac74c360c9ca781e99e7d4eb28b0cce2285dd",
"model_handler/utils.py": "f78fd3edce603b333dc9a88ee2c041dc547d51f71aa449ffebc044c4b1e353f3",
}
# Underdog Bench's five kinds of tasks, 24 each; we draw our own seeded sample because its task list is not published.
KINDS = [("simple_python", "Call the one right function"), ("multiple", "Pick the right function from several"),
("irrelevance", "Know when no tool fits"), ("parallel", "Same function, several calls"),
("parallel_multiple", "Several different functions")]
N_PER_KIND, SEED, CTX, PORT = 24, 2026, 16384, 8080
# llama-server's own defaults are temperature 0.8 and top_k 40, so every request states its sampler.
GREEDY = dict(temperature=0, top_k=0, top_p=1.0, min_p=0)
def sh(cmd, **kw):
return subprocess.run(cmd, capture_output=True, text=True, **kw)
def sha256_file(path, chunk=1 << 22):
h = hashlib.sha256()
with open(path, "rb") as f:
for b in iter(lambda: f.read(chunk), b""):
h.update(b)
return h.hexdigest()
def fetch(url, dest, size=None, sha=None, quiet=False):
"""Resumable download with curl, then a size and SHA-256 check. Returns seconds spent downloading."""
t0 = time.time()
if not (os.path.exists(dest) and (size is None or os.path.getsize(dest) == size)):
for attempt in range(5):
r = sh(["curl", "-fsSL", "-C", "-", "--retry", "5", "-o", dest, url])
if r.returncode == 0 and (size is None or os.path.getsize(dest) == size):
break
time.sleep(5 * (attempt + 1))
got = os.path.getsize(dest)
assert size is None or got == size, f"{os.path.basename(dest)}: {got} bytes, expected {size}"
if sha:
assert sha256_file(dest) == sha, f"{os.path.basename(dest)}: SHA-256 mismatch"
return time.time() - t0
print(f"platform : {SYSTEM} {MACHINE}, Python {platform.python_version()}, glibc {platform.libc_ver()[1] or 'n/a'}")
if SYSTEM == "Linux":
print("os :", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0])
gpu = sh(["nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"]).stdout.strip()
print("gpu :", gpu)
# The CUDA 12.8 build of llama.cpp ships without the CUDA runtime; Colab's image has it as pip wheels.
import importlib.util
if importlib.util.find_spec("nvidia.cuda_runtime") is None or importlib.util.find_spec("nvidia.cublas") is None:
subprocess.run([sys.executable, "-m", "pip", "install", "-q", "nvidia-cuda-runtime-cu12==12.9.79",
"nvidia-cublas-cu12==12.9.2.10"], check=True)
import nvidia.cuda_runtime, nvidia.cublas
CUDA_LIBS = [os.path.join(list(m.__path__)[0], "lib") for m in (nvidia.cuda_runtime, nvidia.cublas)]
else:
chip = sh(["sysctl", "-n", "machdep.cpu.brand_string"]).stdout.strip()
mem = int(sh(["sysctl", "-n", "hw.memsize"]).stdout) / 2**30
print(f"gpu : {chip}, {mem:.0f} GB unified memory (Metal)")
CUDA_LIBS = []
HW = gpu.split(",")[0] if SYSTEM == "Linux" else chip
print(f"work dir : {WORK} ({shutil.disk_usage(WORK).free / 1e9:.0f} GB free)")
Everything the notebook touches is pinned before anything runs. The setup cell records the llama.cpp release, both model files and fourteen BFCL files by size and SHA-256, and it fixes the sampler for every request, because llama-server otherwise falls back to defaults we did not choose. It detects the platform, an NVIDIA GPU on Linux or an Apple-silicon Mac, and on Colab it ensures the CUDA 12 runtime libraries the prebuilt llama. cpp needs are importable.cpp needs are importable.
@section("2. Download llama.cpp b11565 and both 2-bit files, and check every byte against its pin")
def download_step():
global LLAMA_DIR, SERVER_BIN, SERVER_ENV
name, size, sha = LLAMA_ASSET
tar = os.path.join(WORK, "llama", name)
fetch(f"https://github.com/ggml-org/llama.cpp/releases/download/{LLAMA_TAG}/{name}", tar, size, sha)
with tarfile.open(tar) as t:
t.extractall(os.path.join(WORK, "llama"), filter="data")
SERVER_BIN = glob.glob(os.path.join(WORK, "llama", "**", "llama-server"), recursive=True)[0]
LLAMA_DIR = os.path.dirname(SERVER_BIN)
for b in glob.glob(os.path.join(LLAMA_DIR, "llama-*")):
os.chmod(b, 0o755)
SERVER_ENV = dict(os.environ)
if CUDA_LIBS:
SERVER_ENV["LD_LIBRARY_PATH"] = ":".join(CUDA_LIBS + [LLAMA_DIR, SERVER_ENV.get("LD_LIBRARY_PATH", "")])
missing = [l for l in sh(["ldd", SERVER_BIN], env=SERVER_ENV).stdout.splitlines() if "not found" in l]
assert not missing, f"unresolved libraries: {missing}"
r = sh([SERVER_BIN, "--version"], env=SERVER_ENV)
ver = re.search(r"version: .*", r.stdout + r.stderr).group(0)
print(f"llama.cpp: {name} (SHA-256 checked) -> {ver}")
for rel, digest in BFCL_FILES.items():
dest = os.path.join(WORK, "bfcl", rel.replace("/", "__"))
fetch(GORILLA + rel, dest, sha=digest)
print(f"BFCL v4: {len(BFCL_FILES)} files from gorilla@6ea5797 (5 task files, 4 answer files, 5 checker sources), all SHA-256 checked")
for tag, m in MODELS.items():
path = os.path.join(WORK, "models", m["file"])
had = os.path.exists(path) and os.path.getsize(path) == m["size"]
url = f"https://huggingface.co/{m['repo']}/resolve/{m['rev']}/{m['file']}"
dl = fetch(url, path, m["size"]) # size check now, hash below
t0 = time.time()
ok = sha256_file(path) == m["sha256"]
how = "already on disk" if had else f"downloaded in {dl:.0f} s ({m['size'] / dl / 1e6:.0f} MB/s)"
print(f"{tag:7s}: {m['file']} {m['size']:,} bytes, {how}; SHA-256 {'matches' if ok else 'DOES NOT MATCH'} "
f"the Hub's LFS pin ({time.time() - t0:.0f} s to hash)")
claim(f"{m['file']} is byte-identical to the file pinned at {m['repo']}@{m['rev'][:7]}", ok)
assert ok
return f"llama.cpp {LLAMA_TAG}; Saluki 7.90 GB and ISTA IQ2_XS 8.42 GB verified by SHA-256"
download_step()
We download the CUDA build of llama.cpp b11565, both 2-bit files and the BFCL data, and check every byte against its pin. The Hugging Face downloads are resumable, and the hashes are the Hub’s own LFS digests at a fixed revision, so a silently replaced file would stop the notebook rather than change its results. BFCL’s task files, answer files, and checker code all come from one gorilla commit. The cell prints how long the downloads took, which is the slowest part of the run on Colab.
GGML_TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 10: ("Q2_K", 256, 84), 12: ("Q4_K", 256, 144),
16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98), 19: ("IQ1_S", 256, 50),
21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136), 29: ("IQ1_M", 256, 56),
30: ("BF16", 1, 2)} # name, values per block, bytes per block (ggml-common.h)
def read_gguf(path):
"""GGUF v3 header: metadata, tensor table (name, dims, type, offset) and where the tensor data starts."""
f = open(path, "rb")
u = lambda fmt: struct.unpack("<" + fmt, f.read(struct.calcsize(fmt)))[0]
s = lambda: f.read(u("Q")).decode("utf-8", "replace")
scalar = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}
def val(t):
if t == 8:
return s()
if t == 9:
et, n = u("I"), u("Q")
return [val(et) for _ in range(n)]
return u(scalar[t])
assert f.read(4) == b"GGUF"
version, n_tensors, n_kv = u("I"), u("Q"), u("Q")
kv = {}
for _
Auto-fetched and translated by AI Atlas. All rights belong to the original author.