Make datesets and loaders compatible with multiprocessing (#118)

.gitignore

@@ -14,7 +14,8 @@ ch05/01_main-chapter-code/model.pth
 ch05/01_main-chapter-code/model_and_optimizer.pth
 ch05/03_bonus_pretraining_on_gutenberg/model_checkpoints
 
-# Preprocessing output folders
+# Datasets
+ch05/03_bonus_pretraining_on_gutenberg/gutenberg
 ch05/03_bonus_pretraining_on_gutenberg/gutenberg_preprocessed
 
 # Temporary OS-related files

appendix-D/01_main-chapter-code/appendix-D.ipynb

@@ -47,7 +47,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "torch version: 2.2.1\n"
+      "torch version: 2.2.2\n"
      ]
     }
    ],
@@ -130,7 +130,8 @@
     "    max_length=GPT_CONFIG_124M[\"context_length\"],\n",
     "    stride=GPT_CONFIG_124M[\"context_length\"],\n",
     "    drop_last=True,\n",
-    "    shuffle=True\n",
+    "    shuffle=True,\n",
+    "    num_workers=0\n",
     ")\n",
     "\n",
     "val_loader = create_dataloader_v1(\n",
@@ -139,7 +140,8 @@
     "    max_length=GPT_CONFIG_124M[\"context_length\"],\n",
     "    stride=GPT_CONFIG_124M[\"context_length\"],\n",
     "    drop_last=False,\n",
-    "    shuffle=False\n",
+    "    shuffle=False,\n",
+    "    num_workers=0\n",
     ")"
    ]
   },
@@ -500,7 +502,7 @@
     "\n",
     "\n",
     "def train_model(model, train_loader, val_loader, optimizer, device, n_epochs,\n",
-    "                eval_freq, eval_iter, start_context, warmup_steps=10,\n",
+    "                eval_freq, eval_iter, start_context, tokenizer, warmup_steps=10,\n",
     "                initial_lr=3e-05, min_lr=1e-6):\n",
     "\n",
     "    train_losses, val_losses, track_tokens_seen, track_lrs = [], [], [], []\n",
@@ -562,8 +564,7 @@
     "\n",
     "        # Generate and print a sample from the model to monitor progress\n",
     "        generate_and_print_sample(\n",
-    "            model, train_loader.dataset.tokenizer,\n",
-    "            device, start_context\n",
+    "            model, tokenizer, device, start_context\n",
     "        )\n",
     "\n",
     "    return train_losses, val_losses, track_tokens_seen, track_lrs"
@@ -625,18 +626,21 @@
     }
    ],
    "source": [
+    "import tiktoken\n",
+    "\n",
     "torch.manual_seed(123)\n",
     "model = GPTModel(GPT_CONFIG_124M)\n",
     "model.to(device)\n",
     "\n",
     "peak_lr = 5e-4\n",
     "optimizer = torch.optim.AdamW(model.parameters(), weight_decay=0.1)\n",
+    "tokenizer = tiktoken.get_encoding(\"gpt2\")\n",
     "\n",
     "n_epochs = 15\n",
     "train_losses, val_losses, tokens_seen, lrs = train_model(\n",
     "    model, train_loader, val_loader, optimizer, device, n_epochs=n_epochs,\n",
     "    eval_freq=5, eval_iter=1, start_context=\"Every effort moves you\",\n",
-    "    warmup_steps=10, initial_lr=1e-5, min_lr=1e-5\n",
+    "    tokenizer=tokenizer, warmup_steps=10, initial_lr=1e-5, min_lr=1e-5\n",
     ")"
    ]
   },
@@ -705,7 +709,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/var/folders/jg/tpqyh1fd5js5wsr1d138k3n40000gn/T/ipykernel_34986/3589549395.py:5: UserWarning: The figure layout has changed to tight\n",
+      "/var/folders/jg/tpqyh1fd5js5wsr1d138k3n40000gn/T/ipykernel_9436/3589549395.py:5: UserWarning: The figure layout has changed to tight\n",
       "  plt.tight_layout(); plt.savefig(\"3.pdf\")\n"
      ]
     },
@@ -755,7 +759,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.6"
+   "version": "3.11.4"
   }
  },
  "nbformat": 4,

appendix-D/01_main-chapter-code/previous_chapters.py

@@ -20,12 +20,11 @@ import matplotlib.pyplot as plt
 
 class GPTDatasetV1(Dataset):
     def __init__(self, txt, tokenizer, max_length, stride):
-        self.tokenizer = tokenizer
         self.input_ids = []
         self.target_ids = []
 
         # Tokenize the entire text
-        token_ids = self.tokenizer.encode(txt)
+        token_ids = tokenizer.encode(txt)
 
         # Use a sliding window to chunk the book into overlapping sequences of max_length
         for i in range(0, len(token_ids) - max_length, stride):
@@ -42,7 +41,7 @@ class GPTDatasetV1(Dataset):
 
 
 def create_dataloader_v1(txt, batch_size=4, max_length=256,
-                         stride=128, shuffle=True, drop_last=True):
+                         stride=128, shuffle=True, drop_last=True, num_workers=0):
     # Initialize the tokenizer
     tokenizer = tiktoken.get_encoding("gpt2")
 
@@ -51,7 +50,7 @@ def create_dataloader_v1(txt, batch_size=4, max_length=256,
 
     # Create dataloader
     dataloader = DataLoader(
-        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last)
+        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last, num_workers=0)
 
     return dataloader
 

ch02/01_main-chapter-code/ch02.ipynb

@@ -37,7 +37,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "torch version: 2.2.1\n",
+      "torch version: 2.2.2\n",
       "tiktoken version: 0.5.1\n"
      ]
     }
@@ -724,7 +724,7 @@
       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
       "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
       "Cell \u001b[0;32mIn[16], line 5\u001b[0m\n\u001b[1;32m      1\u001b[0m tokenizer \u001b[38;5;241m=\u001b[39m SimpleTokenizerV1(vocab)\n\u001b[1;32m      3\u001b[0m text \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mHello, do you like tea. Is this-- a test?\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m----> 5\u001b[0m \u001b[43mtokenizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n",
-      "Cell \u001b[0;32mIn[12], line 9\u001b[0m, in \u001b[0;36mSimpleTokenizerV1.encode\u001b[0;34m(self, text)\u001b[0m\n\u001b[1;32m      7\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m([,.?_!\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m()\u001b[39m\u001b[38;5;130;01m\\'\u001b[39;00m\u001b[38;5;124m]|--|\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124ms)\u001b[39m\u001b[38;5;124m'\u001b[39m, text)\n\u001b[1;32m      8\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m [item\u001b[38;5;241m.\u001b[39mstrip() \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m preprocessed \u001b[38;5;28;01mif\u001b[39;00m item\u001b[38;5;241m.\u001b[39mstrip()]\n\u001b[0;32m----> 9\u001b[0m ids \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstr_to_int[s] \u001b[38;5;28;01mfor\u001b[39;00m s \u001b[38;5;129;01min\u001b[39;00m preprocessed]\n\u001b[1;32m     10\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ids\n",
+      "Cell \u001b[0;32mIn[12], line 9\u001b[0m, in \u001b[0;36mSimpleTokenizerV1.encode\u001b[0;34m(self, text)\u001b[0m\n\u001b[1;32m      7\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m([,.?_!\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m()\u001b[39m\u001b[38;5;130;01m\\'\u001b[39;00m\u001b[38;5;124m]|--|\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124ms)\u001b[39m\u001b[38;5;124m'\u001b[39m, text)\n\u001b[1;32m      8\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m [item\u001b[38;5;241m.\u001b[39mstrip() \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m preprocessed \u001b[38;5;28;01mif\u001b[39;00m item\u001b[38;5;241m.\u001b[39mstrip()]\n\u001b[0;32m----> 9\u001b[0m ids \u001b[38;5;241m=\u001b[39m \u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstr_to_int\u001b[49m\u001b[43m[\u001b[49m\u001b[43ms\u001b[49m\u001b[43m]\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43ms\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mpreprocessed\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m     10\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ids\n",
       "Cell \u001b[0;32mIn[12], line 9\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m      7\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m([,.?_!\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m()\u001b[39m\u001b[38;5;130;01m\\'\u001b[39;00m\u001b[38;5;124m]|--|\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124ms)\u001b[39m\u001b[38;5;124m'\u001b[39m, text)\n\u001b[1;32m      8\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m [item\u001b[38;5;241m.\u001b[39mstrip() \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m preprocessed \u001b[38;5;28;01mif\u001b[39;00m item\u001b[38;5;241m.\u001b[39mstrip()]\n\u001b[0;32m----> 9\u001b[0m ids \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstr_to_int\u001b[49m\u001b[43m[\u001b[49m\u001b[43ms\u001b[49m\u001b[43m]\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m s \u001b[38;5;129;01min\u001b[39;00m preprocessed]\n\u001b[1;32m     10\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ids\n",
       "\u001b[0;31mKeyError\u001b[0m: 'Hello'"
      ]
@@ -957,7 +957,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 24,
    "id": "ede1d41f-934b-4bf4-8184-54394a257a94",
    "metadata": {},
    "outputs": [],
@@ -967,7 +967,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 25,
    "id": "48967a77-7d17-42bf-9e92-fc619d63a59e",
    "metadata": {},
    "outputs": [
@@ -988,7 +988,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 26,
    "id": "6ad3312f-a5f7-4efc-9d7d-8ea09d7b5128",
    "metadata": {},
    "outputs": [],
@@ -998,7 +998,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": 27,
    "id": "5ff2cd85-7cfb-4325-b390-219938589428",
    "metadata": {},
    "outputs": [
@@ -1020,7 +1020,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 28,
    "id": "d26a48bb-f82e-41a8-a955-a1c9cf9d50ab",
    "metadata": {},
    "outputs": [
@@ -1080,7 +1080,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 29,
    "id": "848d5ade-fd1f-46c3-9e31-1426e315c71b",
    "metadata": {},
    "outputs": [
@@ -1111,7 +1111,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": 30,
    "id": "e84424a7-646d-45b6-99e3-80d15fb761f2",
    "metadata": {},
    "outputs": [],
@@ -1121,7 +1121,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 31,
    "id": "dfbff852-a92f-48c8-a46d-143a0f109f40",
    "metadata": {},
    "outputs": [
@@ -1154,7 +1154,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": 32,
    "id": "d97b031e-ed55-409d-95f2-aeb38c6fe366",
    "metadata": {},
    "outputs": [
@@ -1179,7 +1179,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 37,
+   "execution_count": 33,
    "id": "f57bd746-dcbf-4433-8e24-ee213a8c34a1",
    "metadata": {},
    "outputs": [
@@ -1221,7 +1221,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 38,
+   "execution_count": 34,
    "id": "e1770134-e7f3-4725-a679-e04c3be48cac",
    "metadata": {},
    "outputs": [
@@ -1229,7 +1229,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "PyTorch version: 2.1.0\n"
+      "PyTorch version: 2.2.2\n"
      ]
     }
    ],
@@ -1258,7 +1258,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 39,
+   "execution_count": 35,
    "id": "74b41073-4c9f-46e2-a1bd-d38e4122b375",
    "metadata": {},
    "outputs": [],
@@ -1268,12 +1268,11 @@
     "\n",
     "class GPTDatasetV1(Dataset):\n",
     "    def __init__(self, txt, tokenizer, max_length, stride):\n",
-    "        self.tokenizer = tokenizer\n",
     "        self.input_ids = []\n",
     "        self.target_ids = []\n",
     "\n",
     "        # Tokenize the entire text\n",
-    "        token_ids = self.tokenizer.encode(txt, allowed_special={'<|endoftext|>'})\n",
+    "        token_ids = tokenizer.encode(txt, allowed_special={\"<|endoftext|>\"})\n",
     "\n",
     "        # Use a sliding window to chunk the book into overlapping sequences of max_length\n",
     "        for i in range(0, len(token_ids) - max_length, stride):\n",
@@ -1291,12 +1290,12 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 40,
+   "execution_count": 36,
    "id": "5eb30ebe-97b3-43c5-9ff1-a97d621b3c4e",
    "metadata": {},
    "outputs": [],
    "source": [
-    "def create_dataloader_v1(txt, batch_size=4, max_length=256, stride=128, shuffle=True, drop_last=True):\n",
+    "def create_dataloader_v1(txt, batch_size=4, max_length=256, stride=128, shuffle=True, drop_last=True, num_workers=0):\n",
     "\n",
     "    # Initialize the tokenizer\n",
     "    tokenizer = tiktoken.get_encoding(\"gpt2\")\n",
@@ -1306,7 +1305,12 @@
     "\n",
     "    # Create dataloader\n",
     "    dataloader = DataLoader(\n",
-    "        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last)\n",
+    "        dataset,\n",
+    "        batch_size=batch_size,\n",
+    "        shuffle=shuffle,\n",
+    "        drop_last=drop_last,\n",
+    "        num_workers=0\n",
+    "    )\n",
     "\n",
     "    return dataloader"
    ]
@@ -1321,7 +1325,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 41,
+   "execution_count": 37,
    "id": "df31d96c-6bfd-4564-a956-6192242d7579",
    "metadata": {},
    "outputs": [],
@@ -1332,7 +1336,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 42,
+   "execution_count": 38,
    "id": "9226d00c-ad9a-4949-a6e4-9afccfc7214f",
    "metadata": {},
    "outputs": [
@@ -1354,7 +1358,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 43,
+   "execution_count": 39,
    "id": "10deb4bc-4de1-4d20-921e-4b1c7a0e1a6d",
    "metadata": {},
    "outputs": [
@@ -1398,7 +1402,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 44,
+   "execution_count": 40,
    "id": "1916e7a6-f03d-4f09-91a6-d0bdbac5a58c",
    "metadata": {},
    "outputs": [
@@ -1473,7 +1477,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 46,
+   "execution_count": 41,
    "id": "15a6304c-9474-4470-b85d-3991a49fa653",
    "metadata": {},
    "outputs": [],
@@ -1491,7 +1495,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 47,
+   "execution_count": 42,
    "id": "93cb2cee-9aa6-4bb8-8977-c65661d16eda",
    "metadata": {},
    "outputs": [],
@@ -1513,7 +1517,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 49,
+   "execution_count": 43,
    "id": "a686eb61-e737-4351-8f1c-222913d47468",
    "metadata": {},
    "outputs": [
@@ -1554,7 +1558,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 50,
+   "execution_count": 44,
    "id": "e43600ba-f287-4746-8ddf-d0f71a9023ca",
    "metadata": {},
    "outputs": [
@@ -1581,7 +1585,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 51,
+   "execution_count": 45,
    "id": "50280ead-0363-44c8-8c35-bb885d92c8b7",
    "metadata": {},
    "outputs": [
@@ -1874,7 +1878,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.10"
+   "version": "3.11.4"
   }
  },
  "nbformat": 4,

ch02/01_main-chapter-code/dataloader.ipynb

@@ -31,7 +31,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
    "id": "0ed4b7db-3b47-4fd3-a4a6-5f4ed5dd166e",
    "metadata": {},
    "outputs": [],
@@ -43,12 +43,11 @@
     "\n",
     "class GPTDatasetV1(Dataset):\n",
     "    def __init__(self, txt, tokenizer, max_length, stride):\n",
-    "        self.tokenizer = tokenizer\n",
     "        self.input_ids = []\n",
     "        self.target_ids = []\n",
     "\n",
     "        # Tokenize the entire text\n",
-    "        token_ids = self.tokenizer.encode(txt, allowed_special={'<|endoftext|>'})\n",
+    "        token_ids = tokenizer.encode(txt, allowed_special={\"<|endoftext|>\"})\n",
     "\n",
     "        # Use a sliding window to chunk the book into overlapping sequences of max_length\n",
     "        for i in range(0, len(token_ids) - max_length, stride):\n",
@@ -65,7 +64,7 @@
     "\n",
     "\n",
     "def create_dataloader_v1(txt, batch_size=4, max_length=256, \n",
-    "                         stride=128, shuffle=True, drop_last=True):\n",
+    "                         stride=128, shuffle=True, drop_last=True, num_workers=0):\n",
     "    # Initialize the tokenizer\n",
     "    tokenizer = tiktoken.get_encoding(\"gpt2\")\n",
     "\n",
@@ -74,7 +73,7 @@
     "\n",
     "    # Create dataloader\n",
     "    dataloader = DataLoader(\n",
-    "        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last)\n",
+    "        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last, num_workers=0)\n",
     "\n",
     "    return dataloader\n",
     "\n",
@@ -99,7 +98,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
    "id": "664397bc-6daa-4b88-90aa-e8fc1fbd5846",
    "metadata": {},
    "outputs": [],
@@ -117,7 +116,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
    "id": "d3664332-e6bb-447e-8b96-203aafde8b24",
    "metadata": {},
    "outputs": [
@@ -150,7 +149,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.10"
+   "version": "3.11.4"
   }
  },
  "nbformat": 4,

ch04/01_main-chapter-code/gpt.py

@@ -13,13 +13,12 @@ from torch.utils.data import Dataset, DataLoader
 
 
 class GPTDatasetV1(Dataset):
-    def __init__(self, txt, tokenizer, max_length, stride):
-        self.tokenizer = tokenizer
+    def __init__(self, txt, tokenizer, max_length, stride, num_workers=0):
         self.input_ids = []
         self.target_ids = []
 
         # Tokenize the entire text
-        token_ids = self.tokenizer.encode(txt)
+        token_ids = tokenizer.encode(txt)
 
         # Use a sliding window to chunk the book into overlapping sequences of max_length
         for i in range(0, len(token_ids) - max_length, stride):
@@ -36,7 +35,7 @@ class GPTDatasetV1(Dataset):
 
 
 def create_dataloader_v1(txt, batch_size=4, max_length=256,
-                         stride=128, shuffle=True, drop_last=True):
+                         stride=128, shuffle=True, drop_last=True, num_workers=0):
     # Initialize the tokenizer
     tokenizer = tiktoken.get_encoding("gpt2")
 

ch04/01_main-chapter-code/previous_chapters.py

@@ -11,7 +11,6 @@ from torch.utils.data import Dataset, DataLoader
 
 class GPTDatasetV1(Dataset):
     def __init__(self, txt, tokenizer, max_length, stride):
-        self.tokenizer = tokenizer
         self.input_ids = []
         self.target_ids = []
 
@@ -33,7 +32,7 @@ class GPTDatasetV1(Dataset):
 
 
 def create_dataloader_v1(txt, batch_size=4, max_length=256,
-                         stride=128, shuffle=True, drop_last=True):
+                         stride=128, shuffle=True, drop_last=True, num_workers=0):
     # Initialize the tokenizer
     tokenizer = tiktoken.get_encoding("gpt2")
 
@@ -42,7 +41,7 @@ def create_dataloader_v1(txt, batch_size=4, max_length=256,
 
     # Create dataloader
     dataloader = DataLoader(
-        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last)
+        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last, num_workers=0)
 
     return dataloader
 

ch05/01_main-chapter-code/exercise-solutions.ipynb

@@ -473,7 +473,8 @@
     "    max_length=GPT_CONFIG_124M[\"context_length\"],\n",
     "    stride=GPT_CONFIG_124M[\"context_length\"],\n",
     "    drop_last=True,\n",
-    "    shuffle=True\n",
+    "    shuffle=True,\n",
+    "    num_workers=0\n",
     ")\n",
     "\n",
     "val_loader = create_dataloader_v1(\n",
@@ -482,7 +483,8 @@
     "    max_length=GPT_CONFIG_124M[\"context_length\"],\n",
     "    stride=GPT_CONFIG_124M[\"context_length\"],\n",
     "    drop_last=False,\n",
-    "    shuffle=False\n",
+    "    shuffle=False,\n",
+    "    num_workers=0\n",
     ")"
    ]
   },
@@ -697,7 +699,8 @@
     "    max_length=GPT_CONFIG_124M[\"context_length\"],\n",
     "    stride=GPT_CONFIG_124M[\"context_length\"],\n",
     "    drop_last=True,\n",
-    "    shuffle=True\n",
+    "    shuffle=True,\n",
+    "    num_workers=0\n",
     ")\n",
     "\n",
     "val_loader = create_dataloader_v1(\n",
@@ -706,7 +709,8 @@
     "    max_length=GPT_CONFIG_124M[\"context_length\"],\n",
     "    stride=GPT_CONFIG_124M[\"context_length\"],\n",
     "    drop_last=False,\n",
-    "    shuffle=False\n",
+    "    shuffle=False,\n",
+    "    num_workers=0\n",
     ")"
    ]
   },
@@ -945,7 +949,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.11.4"
   }
  },
  "nbformat": 4,

ch05/01_main-chapter-code/gpt_train.py

@@ -7,6 +7,8 @@ import matplotlib.pyplot as plt
 import os
 import torch
 import urllib.request
+import tiktoken
+
 
 # Import from local files
 from previous_chapters import GPTModel, create_dataloader_v1, generate_text_simple
@@ -69,7 +71,7 @@ def generate_and_print_sample(model, tokenizer, device, start_context):
 
 
 def train_model_simple(model, train_loader, val_loader, optimizer, device, num_epochs,
-                       eval_freq, eval_iter, start_context):
+                       eval_freq, eval_iter, start_context, tokenizer):
     # Initialize lists to track losses and tokens seen
     train_losses, val_losses, track_tokens_seen = [], [], []
     tokens_seen = 0
@@ -99,7 +101,7 @@ def train_model_simple(model, train_loader, val_loader, optimizer, device, num_e
 
         # Print a sample text after each epoch
         generate_and_print_sample(
-            model, train_loader.dataset.tokenizer, device, start_context
+            model, tokenizer, device, start_context
         )
 
     return train_losses, val_losses, track_tokens_seen
@@ -169,7 +171,8 @@ def main(gpt_config, settings):
         max_length=gpt_config["context_length"],
         stride=gpt_config["context_length"],
         drop_last=True,
-        shuffle=True
+        shuffle=True,
+        num_workers=0
     )
 
     val_loader = create_dataloader_v1(
@@ -178,17 +181,20 @@ def main(gpt_config, settings):
         max_length=gpt_config["context_length"],
         stride=gpt_config["context_length"],
         drop_last=False,
-        shuffle=False
+        shuffle=False,
+        num_workers=0
     )
 
     ##############################
     # Train model
     ##############################
 
+    tokenizer = tiktoken.get_encoding("gpt2")
+
     train_losses, val_losses, tokens_seen = train_model_simple(
         model, train_loader, val_loader, optimizer, device,
         num_epochs=settings["num_epochs"], eval_freq=5, eval_iter=1,
-        start_context="Every effort moves you",
+        start_context="Every effort moves you", tokenizer=tokenizer
     )
 
     return train_losses, val_losses, tokens_seen, model

ch05/01_main-chapter-code/previous_chapters.py

@@ -14,12 +14,11 @@ from torch.utils.data import Dataset, DataLoader
 
 class GPTDatasetV1(Dataset):
     def __init__(self, txt, tokenizer, max_length, stride):
-        self.tokenizer = tokenizer
         self.input_ids = []
         self.target_ids = []
 
         # Tokenize the entire text
-        token_ids = self.tokenizer.encode(txt)
+        token_ids = tokenizer.encode(txt)
 
         # Use a sliding window to chunk the book into overlapping sequences of max_length
         for i in range(0, len(token_ids) - max_length, stride):
@@ -36,7 +35,7 @@ class GPTDatasetV1(Dataset):
 
 
 def create_dataloader_v1(txt, batch_size=4, max_length=256,
-                         stride=128, shuffle=True, drop_last=True):
+                         stride=128, shuffle=True, drop_last=True, num_workers=0):
     # Initialize the tokenizer
     tokenizer = tiktoken.get_encoding("gpt2")
 
@@ -45,7 +44,7 @@ def create_dataloader_v1(txt, batch_size=4, max_length=256,
 
     # Create dataloader
     dataloader = DataLoader(
-        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last)
+        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last, num_workers=0)
 
     return dataloader
 

ch05/02_alternative_weight_loading/previous_chapters.py

@@ -14,12 +14,11 @@ from torch.utils.data import Dataset, DataLoader
 
 class GPTDatasetV1(Dataset):
     def __init__(self, txt, tokenizer, max_length, stride):
-        self.tokenizer = tokenizer
         self.input_ids = []
         self.target_ids = []
 
         # Tokenize the entire text
-        token_ids = self.tokenizer.encode(txt)
+        token_ids = tokenizer.encode(txt)
 
         # Use a sliding window to chunk the book into overlapping sequences of max_length
         for i in range(0, len(token_ids) - max_length, stride):
@@ -36,7 +35,7 @@ class GPTDatasetV1(Dataset):
 
 
 def create_dataloader_v1(txt, batch_size=4, max_length=256,
-                         stride=128, shuffle=True, drop_last=True):
+                         stride=128, shuffle=True, drop_last=True, num_workers=0):
     # Initialize the tokenizer
     tokenizer = tiktoken.get_encoding("gpt2")
 
@@ -45,7 +44,7 @@ def create_dataloader_v1(txt, batch_size=4, max_length=256,
 
     # Create dataloader
     dataloader = DataLoader(
-        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last)
+        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last, num_workers=0)
 
     return dataloader
 

ch05/03_bonus_pretraining_on_gutenberg/pretraining_simple.py

@@ -15,6 +15,7 @@ import argparse
 import os
 from pathlib import Path
 import time
+import tiktoken
 import torch
 from previous_chapters import (
     create_dataloader_v1,
@@ -32,7 +33,7 @@ def read_text_file(file_path):
     return text_data
 
 
-def create_dataloaders(text_data, train_ratio, batch_size, max_length, stride):
+def create_dataloaders(text_data, train_ratio, batch_size, max_length, stride, num_workers=0):
     split_idx = int(train_ratio * len(text_data))
     train_loader = create_dataloader_v1(
         text_data[:split_idx],
@@ -40,7 +41,8 @@ def create_dataloaders(text_data, train_ratio, batch_size, max_length, stride):
         max_length=max_length,
         stride=stride,
         drop_last=True,
-        shuffle=True
+        shuffle=True,
+        num_workers=num_workers
     )
     val_loader = create_dataloader_v1(
         text_data[split_idx:],
@@ -48,7 +50,8 @@ def create_dataloaders(text_data, train_ratio, batch_size, max_length, stride):
         max_length=max_length,
         stride=stride,
         drop_last=False,
-        shuffle=False
+        shuffle=False,
+        num_workers=num_workers
     )
     return train_loader, val_loader
 
@@ -78,7 +81,7 @@ def print_eta(start_time, book_start_time, index, total_files):
 
 def train_model_simple(model, optimizer, device, n_epochs,
                        eval_freq, eval_iter, print_sample_iter, start_context,
-                       output_dir, save_ckpt_freq,
+                       output_dir, save_ckpt_freq, tokenizer,
                        batch_size=1024, train_ratio=0.90):
 
     train_losses, val_losses, track_tokens_seen = [], [], []
@@ -101,7 +104,8 @@ def train_model_simple(model, optimizer, device, n_epochs,
                     train_ratio=train_ratio,
                     batch_size=batch_size,
                     max_length=GPT_CONFIG_124M["context_length"],
-                    stride=GPT_CONFIG_124M["context_length"]
+                    stride=GPT_CONFIG_124M["context_length"],
+                    num_workers=0
                 )
                 print("Training ...")
                 model.train()
@@ -126,7 +130,7 @@ def train_model_simple(model, optimizer, device, n_epochs,
                     # Generate text passage
                     if global_step % print_sample_iter == 0:
                         generate_and_print_sample(
-                            model, train_loader.dataset.tokenizer, device, start_context
+                            model, tokenizer, device, start_context
                         )
 
                 if global_step % save_ckpt_freq:
@@ -196,6 +200,7 @@ if __name__ == "__main__":
     model = GPTModel(GPT_CONFIG_124M)
     model.to(device)
     optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=0.1)
+    tokenizer = tiktoken.get_encoding("gpt2")
 
     data_dir = args.data_dir
     all_files = [os.path.join(path, name) for path, subdirs, files
@@ -221,6 +226,7 @@ if __name__ == "__main__":
         output_dir=output_dir,
         save_ckpt_freq=args.save_ckpt_freq,
         start_context="Every effort moves you",
+        tokenizer=tokenizer
     )
 
     epochs_tensor = torch.linspace(0, args.n_epochs, len(train_losses))

ch05/03_bonus_pretraining_on_gutenberg/previous_chapters.py

@@ -21,11 +21,10 @@ import matplotlib.pyplot as plt
 
 class GPTDatasetV1(Dataset):
     def __init__(self, txt, tokenizer, max_length, stride):
-        self.tokenizer = tokenizer
         self.input_ids = []
         self.target_ids = []
 
-        token_ids = self.tokenizer.encode(txt, allowed_special={'<|endoftext|>'})
+        token_ids = tokenizer.encode(txt, allowed_special={'<|endoftext|>'})
 
         for i in range(0, len(token_ids) - max_length, stride):
             input_chunk = token_ids[i:i + max_length]
@@ -41,11 +40,11 @@ class GPTDatasetV1(Dataset):
 
 
 def create_dataloader_v1(txt, batch_size=4, max_length=256,
-                         stride=128, shuffle=True, drop_last=True):
+                         stride=128, shuffle=True, drop_last=True, num_workers=0):
     tokenizer = tiktoken.get_encoding("gpt2")
     dataset = GPTDatasetV1(txt, tokenizer, max_length, stride)
     dataloader = DataLoader(
-        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last)
+        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last, num_workers=0)
 
     return dataloader
 

ch05/05_bonus_hparam_tuning/hparam_search.py

@@ -6,6 +6,7 @@
 import itertools
 import math
 import os
+import tiktoken
 import torch
 from previous_chapters import GPTModel, create_dataloader_v1
 
@@ -58,7 +59,7 @@ def evaluate_model(model, train_loader, val_loader, device, eval_iter):
 
 def train_model(model, train_loader, val_loader, optimizer, device,
                 n_epochs, eval_freq, eval_iter,
-                encoded_start_context, warmup_iters=10,
+                encoded_start_context, tokenizer, warmup_iters=10,
                 initial_lr=3e-05, min_lr=1e-6):
     global_step = 0
 
@@ -120,6 +121,7 @@ if __name__ == "__main__":
     with open(os.path.join(script_dir, "the-verdict.txt"), "r", encoding="utf-8") as file:
         text_data = file.read()
 
+    tokenizer = tiktoken.get_encoding("gpt2")
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
     train_ratio = 0.95
@@ -155,7 +157,8 @@ if __name__ == "__main__":
                 max_length=GPT_CONFIG_124M["context_length"],
                 stride=GPT_CONFIG_124M["context_length"],
                 drop_last=True,
-                shuffle=True
+                shuffle=True,
+                num_workers=0
             )
 
             val_loader = create_dataloader_v1(
@@ -164,7 +167,8 @@ if __name__ == "__main__":
                 max_length=GPT_CONFIG_124M["context_length"],
                 stride=GPT_CONFIG_124M["context_length"],
                 drop_last=False,
-                shuffle=False
+                shuffle=False,
+                num_workers=0
             )
 
             model = GPTModel(GPT_CONFIG_124M)
@@ -176,7 +180,7 @@ if __name__ == "__main__":
                 weight_decay=HPARAM_CONFIG["weight_decay"]
             )
 
-            encoded_start_context = train_loader.dataset.tokenizer.encode("Nevertheless")
+            encoded_start_context = tokenizer.encode("Nevertheless")
             encoded_tensor = torch.tensor(encoded_start_context).unsqueeze(0)
 
             train_loss, val_loss = train_model(
@@ -184,6 +188,7 @@ if __name__ == "__main__":
                 n_epochs=HPARAM_CONFIG["n_epochs"],
                 eval_freq=5, eval_iter=1,
                 encoded_start_context=encoded_tensor,
+                tokenizer=tokenizer,
                 warmup_iters=HPARAM_CONFIG["warmup_iters"],
                 initial_lr=HPARAM_CONFIG["initial_lr"],
                 min_lr=HPARAM_CONFIG["min_lr"]

ch05/05_bonus_hparam_tuning/previous_chapters.py

@@ -19,12 +19,11 @@ from torch.utils.data import Dataset, DataLoader
 
 class GPTDatasetV1(Dataset):
     def __init__(self, txt, tokenizer, max_length, stride):
-        self.tokenizer = tokenizer
         self.input_ids = []
         self.target_ids = []
 
         # Tokenize the entire text
-        token_ids = self.tokenizer.encode(txt)
+        token_ids = tokenizer.encode(txt)
 
         # Use a sliding window to chunk the book into overlapping sequences of max_length
         for i in range(0, len(token_ids) - max_length, stride):
@@ -46,11 +45,11 @@ def create_dataloader_v1(txt, batch_size=4, max_length=256,
     tokenizer = tiktoken.get_encoding("gpt2")
 
     # Create dataset
-    dataset = GPTDatasetV1(txt, tokenizer, max_length, stride)
+    dataset = GPTDatasetV1(txt, tokenizer, max_length, stride, num_workers=0)
 
     # Create dataloader
     dataloader = DataLoader(
-        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last)
+        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last, num_workers=0)
 
     return dataloader