Add the main file

Author: venriq

main.py

@@ -0,0 +1,48 @@
+import os
+import sys
+from pathlib import Path
+
+from settings.development import settings
+
+# Add project root to path for imports
+PROJECT_ROOT = Path(__file__).parent
+sys.path.insert(0, str(PROJECT_ROOT))
+
+from utils.downloader import download
+from utils.tokenizer import tokenize
+
+
+def main():
+    """Main entry point for the application."""
+
+    file_url = settings["file_url"]
+    local_folder = settings["local_folder"]
+
+    try:
+        if download(file_url, local_folder):
+            print("✓ File downloaded successfully.")
+        else:
+            print("✓ File already exists.")
+    except ValueError as ve:
+        print(f"✗ Download error: {ve}")
+        return
+
+    file_name = file_url.split("/")[-1]
+    file_path = os.path.join(local_folder, file_name)
+
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            raw_text = f.read()
+
+        tokens = tokenize(raw_text)
+        print(f"✓ Tokenization complete. Total tokens: {len(tokens)}")
+        print(f"First 10 tokens: {tokens[:30]}")
+
+    except FileNotFoundError:
+        print(f"✗ File not found: {file_path}")
+    except Exception as e:
+        print(f"✗ Error during tokenization: {e}")
+
+
+if __name__ == "__main__":
+    main()