git-market/rasbt-LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-11 14:21:41 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

typo

Browse Source
This commit is contained in:
casinca 2026-01-14 09:59:05 +01:00
parent 34891ed7c4
commit 78f4492ab7
7 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ch05/07_gpt_to_llama/standalone-llama32.ipynb
View File

@ -358,7 +358,7 @@
" self.final_norm = nn.RMSNorm(cfg[\"emb_dim\"], eps=1e-5, dtype=cfg[\"dtype\"])\n",
" self.out_head = nn.Linear(cfg[\"emb_dim\"], cfg[\"vocab_size\"], bias=False, dtype=cfg[\"dtype\"])\n",
"\n",
" # Reusuable utilities\n",
" # Reusable utilities\n",
" cos, sin = compute_rope_params(\n",
" head_dim=cfg[\"emb_dim\"] // cfg[\"n_heads\"],\n",
" theta_base=cfg[\"rope_base\"],\n",

2
ch05/11_qwen3/standalone-qwen3-moe-plus-kvcache.ipynb
View File

@ -432,7 +432,7 @@
" self.final_norm = RMSNorm(cfg[\"emb_dim\"])\n",
" self.out_head = nn.Linear(cfg[\"emb_dim\"], cfg[\"vocab_size\"], bias=False, dtype=cfg[\"dtype\"])\n",
"\n",
" # Reusuable utilities\n",
" # Reusable utilities\n",
" if cfg[\"head_dim\"] is None:\n",
" head_dim = cfg[\"emb_dim\"] // cfg[\"n_heads\"]\n",
" else:\n",

2
ch05/11_qwen3/standalone-qwen3-moe.ipynb
View File

@ -422,7 +422,7 @@
" self.final_norm = RMSNorm(cfg[\"emb_dim\"])\n",
" self.out_head = nn.Linear(cfg[\"emb_dim\"], cfg[\"vocab_size\"], bias=False, dtype=cfg[\"dtype\"])\n",
"\n",
" # Reusuable utilities\n",
" # Reusable utilities\n",
" if cfg[\"head_dim\"] is None:\n",
" head_dim = cfg[\"emb_dim\"] // cfg[\"n_heads\"]\n",
" else:\n",

2
ch05/11_qwen3/standalone-qwen3.ipynb
View File

@ -388,7 +388,7 @@
" self.final_norm = RMSNorm(cfg[\"emb_dim\"])\n",
" self.out_head = nn.Linear(cfg[\"emb_dim\"], cfg[\"vocab_size\"], bias=False, dtype=cfg[\"dtype\"])\n",
"\n",
" # Reusuable utilities\n",
" # Reusable utilities\n",
" if cfg[\"head_dim\"] is None:\n",
" head_dim = cfg[\"emb_dim\"] // cfg[\"n_heads\"]\n",
" else:\n",

2
ch05/14_ch05_with_other_llms/ch05-llama32.ipynb
View File

@ -333,7 +333,7 @@
" # self.out_head.weight = self.tok_emb.weight\n",
" # torch.nn.init.normal_(self.out_head.weight, mean=0.0, std=0.02)\n",
"\n",
" # Reusuable utilities\n",
" # Reusable utilities\n",
" cos, sin = compute_rope_params(\n",
" head_dim=cfg[\"emb_dim\"] // cfg[\"n_heads\"],\n",
" theta_base=cfg[\"rope_base\"],\n",

2
ch05/14_ch05_with_other_llms/ch05-qwen3.ipynb
View File

@ -336,7 +336,7 @@
" # self.out_head.weight = self.tok_emb.weight\n",
" # torch.nn.init.normal_(self.out_head.weight, mean=0.0, std=0.02)\n",
"\n",
" # Reusuable utilities\n",
" # Reusable utilities\n",
" if cfg[\"head_dim\"] is None:\n",
" head_dim = cfg[\"emb_dim\"] // cfg[\"n_heads\"]\n",
" else:\n",

2
pkg/llms_from_scratch/kv_cache/llama3.py
View File

@ -65,7 +65,7 @@ class Llama3Model(nn.Module):
self.final_norm = nn.RMSNorm(cfg["emb_dim"], eps=1e-5, dtype=cfg["dtype"])
self.out_head = nn.Linear(cfg["emb_dim"], cfg["vocab_size"], bias=False, dtype=cfg["dtype"])
# Reusuable utilities
# Reusable utilities
cos, sin = compute_rope_params(
head_dim=cfg["emb_dim"] // cfg["n_heads"],
theta_base=cfg["rope_base"],