Address copilot comments

Author: venriq

main.py

@@ -27,16 +27,15 @@ def main():
         print(f"✗ Download error: {ve}")
         return
 
-    file_name = file_url.split("/")[-1]
-    file_path = os.path.join(local_folder, file_name)
+    file_path = os.path.join(local_folder, file_url.split("/")[-1])
 
     try:
         with open(file_path, "r", encoding="utf-8") as f:
             raw_text = f.read()
 
         tokens = tokenize(raw_text)
         print(f"✓ Tokenization complete. Total tokens: {len(tokens)}")
-        print(f"First 10 tokens: {tokens[:30]}")
+        print(f"First 30 tokens: {tokens[:30]}")
 
     except FileNotFoundError:
         print(f"✗ File not found: {file_path}")

settings/development.py

@@ -2,5 +2,5 @@
 
 settings = {
     "file_url": "https://raw.githubusercontent.com/rasbt/LLMs-from-scratch/main/ch02/01_main-chapter-code/the-verdict.txt",
-    "local_folder": f"{os.path.join(os.path.expanduser("~"), "data")}",
+    "local_folder": os.path.join(os.path.expanduser("~"), "data"),
 }

utils/downloader.py

@@ -4,17 +4,10 @@
 from settings.development import settings
 
 
-def assert_local_folder_exists(local_folder: str):
-    """Ensure the download folder exists."""
-
-    if not os.path.exists(local_folder):
-        os.makedirs(local_folder, exist_ok=True)
-
-
-def download(file_url, local_folder) -> bool:
+def download(file_url: str, local_folder: str) -> bool:
     """Download a file from a predefined URL if it does not exist locally."""
 
-    assert_local_folder_exists(local_folder)
+    os.makedirs(local_folder, exist_ok=True)
 
     file_name = file_url.split("/")[-1]
     file_path = os.path.join(local_folder, file_name)

utils/tokenizer.py

@@ -1,7 +1,7 @@
 import re
 
 
-def tokenize(text):
+def tokenize(text: str) -> list[str]:
     """Tokenizes the input text into words.
 
     Args:
@@ -11,7 +11,7 @@ def tokenize(text):
         list: A list of tokens (words).
     """
 
-    result = re.split(r'([,.:;?_!"()\']|--|\s)', text)
+    result = re.split(r"([,.:;?_!\"()']|--|\s)", text)
     result = [token.strip() for token in result if token.strip()]
     return result