MLP

Author: Kaweees

.justfile

@@ -22,6 +22,8 @@ build:
 run:
   @echo "Running..."
   @zig build run -Doptimize=ReleaseFast
+  @dot -Tpng assets/img/mlp.dot -o assets/img/mlp.png
+  @dot -Tpng assets/img/perceptron.dot -o assets/img/perceptron.png
 
 # Test the project
 test:

README.md

@@ -35,9 +35,13 @@ A scalar-valued automatic differentiation (autograd) engine for deep learning wr
 ## Preview
 
 <p align="center">
-  <img src="assets/img/train.png"
+  <img src="assets/img/mlp.png"
   width = "80%"
-  alt = "Training a model on MNIST dataset"
+  alt = "MLP"
+  />
+  <img src="assets/img/perceptron.png"
+  width = "80%"
+  alt = "Perceptron"
   />
 </p>
 

assets/img/mlp.png

@@ -6,8 +6,6 @@ const std = @import("std");
 const kiwigrad = @import("kiwigrad");
 const zbench = @import("zbench");
 
-const print = std.debug.print;
-
 pub fn main() !void {
     const stdout_file = std.io.getStdOut().writer();
     var bw = std.io.bufferedWriter(stdout_file);
@@ -19,43 +17,53 @@ pub fn main() !void {
     const ValueType = kiwigrad.engine.Value(f64);
     const NeuronType = kiwigrad.nn.Neuron(f64);
     const LayerType = kiwigrad.nn.Layer(f64);
-    // const MLPType = kiwigrad.nn.MLP;
+    const MLPType = kiwigrad.nn.MLP(f64);
 
     // Initialize allocators and components
     ValueType.init(alloc);
     NeuronType.init(alloc);
     LayerType.init(alloc);
+    MLPType.init(alloc);
     defer {
         ValueType.deinit();
         NeuronType.deinit();
         LayerType.deinit();
-        // MLPType.deinit();
+        MLPType.deinit();
     }
 
+    var sizes = [_]usize{ 3, 2, 1 };
+
     // Initialize the neuron
-    const neuron = NeuronType.new(3);
+    const mlp = MLPType.new(sizes.len - 1, sizes[0..]);
 
-    // Create sample input data
-    var input_data = [_]*ValueType{
-        ValueType.new(1.0),
-        ValueType.new(2.0),
-        ValueType.new(3.0),
+    const inputs = [_][3]*ValueType{
+        [_]*ValueType{ ValueType.new(2), ValueType.new(3), ValueType.new(-1) },
+        [_]*ValueType{ ValueType.new(3), ValueType.new(-1), ValueType.new(0.5) },
+        [_]*ValueType{ ValueType.new(0.5), ValueType.new(1), ValueType.new(1) },
+        [_]*ValueType{ ValueType.new(1), ValueType.new(2), ValueType.new(3) },
     };
 
-    // Forward pass through the layer
-    const output = neuron.forward(input_data[0..]);
+    mlp.draw_graph("assets/img/mlp", stdout);
+
+    for (inputs) |in| {
+        // Forward pass through the layer
+        const output = mlp.forward(@constCast(&in));
+        stdout.print("{d:7.4} ", .{output[0].data}) catch unreachable;
+        for (output) |o| {
+            _ = o.draw_graph("assets/img/perceptron", stdout);
+        }
+    }
 
-    // outputs now contains 2 ValueType pointers (one for each neuron)
-    print("Layer output: {d:.4}\n", .{output.data});
+    // // outputs now contains 2 ValueType pointers (one for each neuron)
+    // print("Layer output: {d:.4}\n", .{output.data});
 
-    print("output.data: {d:.4}\n", .{output.data});
-    print("output.grad: {d:.4}\n", .{output.grad});
+    // print("output.data: {d:.4}\n", .{output.data});
+    // print("output.grad: {d:.4}\n", .{output.grad});
 
-    output.backwardPass(alloc);
+    // output.backwardPass(alloc);
 
-    print("output.data: {d:.4}\n", .{output.data});
-    print("output.grad: {d:.4}\n", .{output.grad});
+    // print("output.data: {d:.4}\n", .{output.data});
+    // print("output.grad: {d:.4}\n", .{output.grad});
 
-    output.draw_graph("assets/img/train", stdout);
     try bw.flush(); // Don't forget to flush!
 }

assets/img/perceptron.png

@@ -18,7 +18,7 @@ pub const UnaryType = enum {
     pub fn toString(self: UnaryType) []const u8 {
         return switch (self) {
             .tanh => "tanh",
-            .exp => "exp",
+            .exp => "^",
             .relu => "ReLU",
             .softmax => "Softmax",
         };
@@ -58,6 +58,7 @@ pub const BinaryType = enum {
 /// - Addition
 /// - Subtraction
 /// - Multiplication
+/// - Exponentiation
 /// - Division
 /// - Rectified Linear Unit (ReLU)
 /// - Softmax
@@ -175,6 +176,16 @@ pub fn Value(comptime T: type) type {
             self.expr.binary.prev[1].grad += self.grad * self.expr.binary.prev[0].data;
         }
 
+        /// Exponentiate a value
+        pub inline fn exp(self: *Self) *Self {
+            return unary(std.math.exp(self.data), .exp, exp_back, self);
+        }
+
+        /// Backpropagation function for exponentiation
+        fn exp_back(self: *Self) void {
+            self.expr.unary.prev[0].grad += self.grad * std.math.exp(self.data);
+        }
+
         /// Subtract two values
         pub inline fn sub(self: *Self, other: *Self) *Self {
             return binary(self.data - other.data, .sub, sub_back, self, other);