{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "code",
"source": [
"from graphviz import Digraph\n",
"\n",
"def trace(root):\n",
" #Builds a set of all nodes and edges in a graph\n",
" nodes, edges = set(), set()\n",
" def build(v):\n",
" if v not in nodes:\n",
" nodes.add(v)\n",
" for child in v._prev:\n",
" edges.add((child, v))\n",
" build(child)\n",
" build(root)\n",
" return nodes, edges\n",
"\n",
"def draw_dot(root):\n",
" dot = Digraph(format='svg', graph_attr={'rankdir': 'LR'}) #LR == Left to Right\n",
"\n",
" nodes, edges = trace(root)\n",
" for n in nodes:\n",
" uid = str(id(n))\n",
" #For any value in the graph, create a rectangular ('record') node for it\n",
" dot.node(name = uid, label = \"{ %s | data %.4f | grad %.4f }\" % ( n.label, n.data, n.grad), shape='record')\n",
" if n._op:\n",
" #If this value is a result of some operation, then create an op node for it\n",
" dot.node(name = uid + n._op, label=n._op)\n",
" #and connect this node to it\n",
" dot.edge(uid + n._op, uid)\n",
"\n",
" for n1, n2 in edges:\n",
" #Connect n1 to the node of n2\n",
" dot.edge(str(id(n1)), str(id(n2)) + n2._op)\n",
"\n",
" return dot"
],
"metadata": {
"id": "T0rN8d146jvF"
},
"execution_count": 1,
"outputs": []
},
{
"cell_type": "code",
"source": [
"import math"
],
"metadata": {
"id": "JlYxBvFK0AjA"
},
"execution_count": 2,
"outputs": []
},
{
"cell_type": "code",
"source": [
"class Value:\n",
"\n",
" def __init__(self, data, _children=(), _op='', label=''):\n",
" self.data = data\n",
" self.grad = 0.0\n",
" self._backward = lambda: None #Its an empty function by default. This is what will do that gradient calculation at each of the operations.\n",
" self._prev = set(_children)\n",
" self._op = _op\n",
" self.label = label\n",
"\n",
"\n",
" def __repr__(self):\n",
" return f\"Value(data={self.data})\"\n",
"\n",
" def __add__(self, other):\n",
" other = other if isinstance(other, Value) else Value(other)\n",
" out = Value(self.data + other.data, (self, other), '+')\n",
"\n",
" def backward():\n",
" self.grad += 1.0 * out.grad\n",
" other.grad += 1.0 * out.grad\n",
"\n",
" out._backward = backward\n",
" return out\n",
"\n",
" def __mul__(self, other):\n",
" other = other if isinstance(other, Value) else Value(other)\n",
" out = Value(self.data * other.data, (self, other), '*')\n",
"\n",
" def backward():\n",
" self.grad += other.data * out.grad\n",
" other.grad += self.data * out.grad\n",
" out._backward = backward\n",
" return out\n",
"\n",
" def __rmul__(self, other): #other * self\n",
" return self * other\n",
"\n",
" def __truediv__(self, other): #self/other\n",
" return self * other**-1\n",
"\n",
" def __neg__(self):\n",
" return self * -1\n",
"\n",
" def __sub__(self, other): #self - other\n",
" return self + (-other)\n",
"\n",
" def __pow__(self, other):\n",
" assert isinstance(other, (int, float)), \"only supporting int/float powers for now\"\n",
" out = Value(self.data ** other, (self, ), f\"**{other}\")\n",
"\n",
" def backward():\n",
" self.grad += (other * (self.data ** (other - 1))) * out.grad\n",
"\n",
" out._backward = backward\n",
" return out\n",
"\n",
" def tanh(self):\n",
" x = self.data\n",
" t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)\n",
" out = Value(t, (self, ), 'tanh')\n",
"\n",
" def backward():\n",
" self.grad += 1 - (t**2) * out.grad\n",
"\n",
" out._backward = backward\n",
" return out\n",
"\n",
" def exp(self):\n",
" x = self.data\n",
" out = Value(math.exp(x), (self, ), 'exp') #We merged t and out, into just out\n",
"\n",
" def backward():\n",
" self.grad += out.data * out.grad\n",
"\n",
" out._backward = backward\n",
" return out\n",
"\n",
" def backward(self):\n",
"\n",
" topo = []\n",
" visited = set()\n",
" def build_topo(v):\n",
" if v not in visited:\n",
" visited.add(v)\n",
" for child in v._prev:\n",
" build_topo(child)\n",
" topo.append(v)\n",
"\n",
" build_topo(self)\n",
"\n",
" self.grad = 1.0\n",
" for node in reversed(topo):\n",
" node._backward()"
],
"metadata": {
"id": "tA0zbyEwFbD5"
},
"execution_count": 3,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"---------------"
],
"metadata": {
"id": "m9hy05zbxhLP"
}
},
{
"cell_type": "code",
"source": [
"import random"
],
"metadata": {
"id": "gu3tnJu1Wti5"
},
"execution_count": 5,
"outputs": []
},
{
"cell_type": "code",
"source": [
"class Neuron:\n",
"\tdef __init__(self, nin):\n",
"\t\tself.w = [ Value(random.uniform(-1,1)) for _ in range(nin) ]\n",
"\t\tself.b = Value(random.uniform(-1,1))\n",
"\n",
"\tdef __call__(self, x):\n",
"\t\t# (w*x)+b\n",
"\t\tact = sum( (wi*xi for wi,xi in zip(self.w, x)), self.b )\n",
"\t\tout = act.tanh()\n",
"\t\treturn out\n",
"\n",
"class Layer:\n",
"\tdef __init__(self, nin, nout):\n",
"\t\tself.neurons = [Neuron(nin) for _ in range(nout)]\n",
"\n",
"\tdef __call__(self, x):\n",
"\t\touts = [n(x) for n in self.neurons]\n",
"\t\treturn outs\n",
"\n",
"class MLP:\n",
"\tdef __init__(self, nin, nouts):\n",
"\t\tsz = [nin] + nouts\n",
"\t\tself.layers = [ Layer(sz[i], sz[i+1]) for i in range(len(nouts)) ]\n",
"\n",
"\tdef __call__(self, x):\n",
"\t\tfor layer in self.layers:\n",
"\t\t\tx = layer(x)\n",
"\t\treturn x"
],
"metadata": {
"id": "fjmlJl8RWp5K"
},
"execution_count": 8,
"outputs": []
},
{
"cell_type": "code",
"source": [
"x = [2.0, 3.0, -1.0]\n",
"n = MLP(3, [4, 4, 1])\n",
"n(x)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "VbhAb3CkWxoa",
"outputId": "8fbf7da2-1643-444c-e9d8-0255c6bdf537"
},
"execution_count": 9,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"[Value(data=0.6731685486278488)]"
]
},
"metadata": {},
"execution_count": 9
}
]
},
{
"cell_type": "markdown",
"source": [
"Now in the final output, we see it in the form of a list right. \\\n",
"\\\n",
"So just for making this look better, we are adding this one line in the return statement of the layer method where if it is just one value left i.e. if it is the last value, then just return that value directly instead of putting it inside a list."
],
"metadata": {
"id": "C3VxtKpWXCqU"
}
},
{
"cell_type": "code",
"source": [
"class Neuron:\n",
"\tdef __init__(self, nin):\n",
"\t\tself.w = [ Value(random.uniform(-1,1)) for _ in range(nin) ]\n",
"\t\tself.b = Value(random.uniform(-1,1))\n",
"\n",
"\tdef __call__(self, x):\n",
"\t\t# (w*x)+b\n",
"\t\tact = sum( (wi*xi for wi,xi in zip(self.w, x)), self.b )\n",
"\t\tout = act.tanh()\n",
"\t\treturn out\n",
"\n",
"class Layer:\n",
"\tdef __init__(self, nin, nout):\n",
"\t\tself.neurons = [Neuron(nin) for _ in range(nout)]\n",
"\n",
"\tdef __call__(self, x):\n",
"\t\touts = [n(x) for n in self.neurons]\n",
"\t\treturn outs[0] if len(outs)==1 else outs #The New added line for making the output better\n",
"\n",
"class MLP:\n",
"\tdef __init__(self, nin, nouts):\n",
"\t\tsz = [nin] + nouts\n",
"\t\tself.layers = [ Layer(sz[i], sz[i+1]) for i in range(len(nouts)) ]\n",
"\n",
"\tdef __call__(self, x):\n",
"\t\tfor layer in self.layers:\n",
"\t\t\tx = layer(x)\n",
"\t\treturn x"
],
"metadata": {
"id": "aCXXYNg_W680"
},
"execution_count": 10,
"outputs": []
},
{
"cell_type": "code",
"source": [
"x = [2.0, 3.0, -1.0]\n",
"n = MLP(3, [4, 4, 1])\n",
"n(x)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "aG9pKV_RXsO8",
"outputId": "196fd2b5-c838-4ba9-c4b8-e8ff3e773761"
},
"execution_count": 11,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"Value(data=-0.597277746687069)"
]
},
"metadata": {},
"execution_count": 11
}
]
},
{
"cell_type": "markdown",
"source": [
"Like that!\\\n",
"\\\n",
"Now finally, lets make the graph of this"
],
"metadata": {
"id": "ydnLxp7kXuuJ"
}
},
{
"cell_type": "code",
"source": [
"draw_dot(n(x))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000
},
"id": "6sPjflG3Xt8J",
"outputId": "1bca9b07-e07b-41b4-8d3f-bfa6676cba42"
},
"execution_count": 12,
"outputs": [
{
"output_type": "execute_result",
"data": {
"image/svg+xml": "\n\n\n\n\n",
"text/plain": [
""
]
},
"metadata": {},
"execution_count": 12
}
]
},
{
"cell_type": "markdown",
"source": [
"**THAT'S AN ENTIRE MLP THAT WE'VE DEFINED NOW!**"
],
"metadata": {
"id": "o2rQ1fX9YM40"
}
}
]
}