From 5048fe80a50594cf552a2aac86d198af01f5278c Mon Sep 17 00:00:00 2001
From: Grant Sanderson <grantsanderson7@gmail.com>
Date: Mon, 2 Oct 2017 18:19:38 -0700
Subject: [PATCH] Beginning part2 of nn project

---
 animation/simple_animations.py |   6 +-
 nn/part1.py                    |  27 ++-
 nn/part2.py                    | 403 +++++++++++++++++++++++++++++++++
 3 files changed, 421 insertions(+), 15 deletions(-)
 create mode 100644 nn/part2.py

diff --git a/animation/simple_animations.py b/animation/simple_animations.py
index 0f7a9299..7bdf054b 100644
--- a/animation/simple_animations.py
+++ b/animation/simple_animations.py
@@ -330,9 +330,9 @@ class LaggedStart(Animation):
             anim.update(alpha)
         return self
 
-    def clean_up(self, *args, **kwargs):
-        for anim in self.subanimations:
-            anim.clean_up(*args, **kwargs)
+    # def clean_up(self, *args, **kwargs):
+    #     for anim in self.subanimations:
+    #         anim.clean_up(*args, **kwargs)
 
 class DelayByOrder(Animation):
     """
diff --git a/nn/part1.py b/nn/part1.py
index 489a25a2..dbdfb0a4 100644
--- a/nn/part1.py
+++ b/nn/part1.py
@@ -127,6 +127,7 @@ class NetworkMobject(VGroup):
         "max_shown_neurons" : 16,
         "brace_for_large_layers" : True,
         "average_shown_activation_of_large_layer" : True,
+        "include_output_labels" : False,
     }
     def __init__(self, neural_network, **kwargs):
         VGroup.__init__(self, **kwargs)
@@ -143,6 +144,8 @@ class NetworkMobject(VGroup):
         layers.arrange_submobjects(RIGHT, buff = self.layer_to_layer_buff)
         self.layers = layers
         self.add(self.layers)
+        if self.include_output_labels:
+            self.add_output_labels()
 
     def get_layer(self, size):
         layer = VGroup()
@@ -255,18 +258,6 @@ class NetworkMobject(VGroup):
             submobject_mode = "lagged_start"
         )]
 
-class MNistNetworkMobject(NetworkMobject):
-    CONFIG = {
-        "neuron_to_neuron_buff" : SMALL_BUFF,
-        "layer_to_layer_buff" : 1.5,
-        "edge_stroke_width" : 1,
-    }
-
-    def __init__(self, **kwargs):
-        network = get_pretrained_network()
-        NetworkMobject.__init__(self, network, **kwargs)
-        self.add_output_labels()
-
     def add_output_labels(self):
         self.output_labels = VGroup()
         for n, neuron in enumerate(self.layers[-1].neurons):
@@ -277,6 +268,18 @@ class MNistNetworkMobject(NetworkMobject):
             self.output_labels.add(label)
         self.add(self.output_labels)
 
+class MNistNetworkMobject(NetworkMobject):
+    CONFIG = {
+        "neuron_to_neuron_buff" : SMALL_BUFF,
+        "layer_to_layer_buff" : 1.5,
+        "edge_stroke_width" : 1,
+        "include_output_labels" : True,
+    }
+
+    def __init__(self, **kwargs):
+        network = get_pretrained_network()
+        NetworkMobject.__init__(self, network, **kwargs)
+
 class NetworkScene(Scene):
     CONFIG = {
         "layer_sizes" : [8, 6, 6, 4],
diff --git a/nn/part2.py b/nn/part2.py
new file mode 100644
index 00000000..c6f0e1ae
--- /dev/null
+++ b/nn/part2.py
@@ -0,0 +1,403 @@
+import sys
+import os.path
+import cv2
+
+from helpers import *
+
+from mobject.tex_mobject import TexMobject
+from mobject import Mobject, Group
+from mobject.image_mobject import ImageMobject
+from mobject.vectorized_mobject import *
+
+from animation.animation import Animation
+from animation.transform import *
+from animation.simple_animations import *
+from animation.playground import *
+from animation.continual_animation import *
+from topics.geometry import *
+from topics.characters import *
+from topics.functions import *
+from topics.fractals import *
+from topics.number_line import *
+from topics.combinatorics import *
+from topics.numerals import *
+from topics.three_dimensions import *
+from topics.objects import *
+from topics.probability import *
+from topics.complex_numbers import *
+from topics.graph_scene import *
+from topics.common_scenes import *
+from scene import Scene
+from scene.reconfigurable_scene import ReconfigurableScene
+from scene.zoomed_scene import *
+from camera import Camera
+from mobject.svg_mobject import *
+from mobject.tex_mobject import *
+
+from nn.network import *
+from nn.part1 import *
+
+def get_training_image_group(train_in, train_out):
+    image = MNistMobject(train_in)
+    image.scale_to_fit_height(1)
+    arrow = Vector(RIGHT, color = BLUE, buff = 0)
+    output = np.argmax(train_out)
+    output_tex = TexMobject(str(output)).scale(1.5)
+    result = Group(image, arrow, output_tex)
+    result.arrange_submobjects(RIGHT)
+    result.to_edge(UP)
+    return result
+
+########
+
+class ShowLastVideo(TeacherStudentsScene):
+    def construct(self):
+        frame = ScreenRectangle()
+        frame.scale_to_fit_height(4.5)
+        frame.to_corner(UP+LEFT)
+        title = TextMobject("But what \\emph{is} a Neural Network")
+        title.move_to(frame)
+        title.to_edge(UP)
+        frame.next_to(title, DOWN)
+
+        assumption_words = TextMobject(
+            "I assume you've\\\\ watched this"
+        )
+        assumption_words.move_to(frame)
+        assumption_words.to_edge(RIGHT)
+        arrow = Arrow(RIGHT, LEFT, color = BLUE)
+        arrow.next_to(assumption_words, LEFT)
+
+
+        self.play(
+            ShowCreation(frame),
+            self.teacher.change, "raise_right_hand"
+        )
+        self.play(
+            Write(title),
+            self.get_student_changes(*["thinking"]*3)
+        )
+        self.play(
+            Animation(title),
+            GrowArrow(arrow),
+            FadeIn(assumption_words)
+        )
+        self.dither(5)
+
+class PreviewLearning(NetworkScene):
+    CONFIG = {
+        "layer_sizes" : DEFAULT_LAYER_SIZES,
+        "network_mob_config" : {
+            "neuron_to_neuron_buff" : SMALL_BUFF,
+            "layer_to_layer_buff" : 2,
+            "edge_stroke_width" : 1,
+            "neuron_stroke_color" : WHITE,
+            "neuron_stroke_width" : 2,
+            "neuron_fill_color" : WHITE,
+            "average_shown_activation_of_large_layer" : False,
+            "edge_propogation_color" : GREEN,
+            "edge_propogation_time" : 2,
+            "include_output_labels" : True,
+        },
+        "n_examples" : 15,
+        "max_stroke_width" : 3,
+        "stroke_width_exp" : 3,
+        "eta" : 5.0,
+    }
+    def construct(self):
+        self.initialize_network()
+        self.add_training_words()
+        self.show_training()
+
+    def initialize_network(self):
+        self.network_mob.scale(0.7)
+        self.network_mob.to_edge(DOWN)
+        self.color_network_edges()
+
+    def add_training_words(self):
+        words = TextMobject("Training in \\\\ progress $\\dots$")
+        words.scale(1.5)
+        words.to_corner(UP+LEFT)
+
+        self.add(words)
+
+    def show_training(self):
+        training_data, validation_data, test_data = load_data_wrapper()
+        for train_in, train_out in training_data[:self.n_examples]:
+            image = get_training_image_group(train_in, train_out)
+            self.activate_network(train_in, FadeIn(image))
+            self.backprop_one_example(
+                train_in, train_out, FadeOut(image)
+            )
+
+    def activate_network(self, train_in, *added_anims):
+        network_mob = self.network_mob
+        layers = network_mob.layers
+        activations = self.network.get_activation_of_all_layers(train_in)
+        active_layers = [
+            self.network_mob.get_active_layer(i, vect)
+            for i, vect in enumerate(activations)
+        ]
+        all_edges = VGroup(*it.chain(*network_mob.edge_groups))
+        edge_animation = LaggedStart(
+            ShowCreationThenDestruction, 
+            all_edges.copy().set_fill(YELLOW),
+            run_time = 1.5,
+            lag_ratio = 0.3,
+            remover = True,
+        )
+        layer_animation = Transform(
+            VGroup(*layers), VGroup(*active_layers),
+            run_time = 1.5,
+            submobject_mode = "lagged_start",
+            rate_func = None,
+        )
+
+        self.play(edge_animation, layer_animation, *added_anims)
+
+    def backprop_one_example(self, train_in, train_out, *added_outro_anims):
+        network_mob = self.network_mob
+        nabla_b, nabla_w = self.network.backprop(train_in, train_out)
+        neuron_groups = VGroup(*[
+            layer.neurons
+            for layer in network_mob.layers[1:]
+        ])
+        delta_neuron_groups = neuron_groups.copy()
+        edge_groups = network_mob.edge_groups
+        delta_edge_groups = VGroup(*[
+            edge_group.copy()
+            for edge_group in edge_groups
+        ])
+        tups = zip(
+            it.count(), nabla_b, nabla_w, 
+            delta_neuron_groups, neuron_groups,
+            delta_edge_groups, edge_groups
+        )
+        for i, nb, nw, delta_neurons, neurons, delta_edges, edges in reversed(tups):
+            shown_nw = self.get_adjusted_first_matrix(nw)
+            if np.max(shown_nw) == 0:
+                shown_nw = (2*np.random.random(shown_nw.shape)-1)**5
+            max_b = np.max(np.abs(nb))
+            max_w = np.max(np.abs(shown_nw))
+            for neuron, b in zip(delta_neurons, nb):
+                color = RED_E if b > 0 else GREEN_E
+                # neuron.set_fill(color, abs(b)/max_b)
+                neuron.set_stroke(color, 3)
+            for edge, w in zip(delta_edges.split(), shown_nw.T.flatten()):
+                edge.set_stroke(
+                    RED_E if w > 0 else GREEN_E,
+                    3*abs(w)/max_w
+                )
+                edge.rotate_in_place(np.pi)
+            if i == 0:
+                delta_edges.submobjects = [
+                    delta_edges[j]
+                    for j in np.argsort(shown_nw.T.flatten())
+                ]
+            network = self.network
+            network.weights[i] -= self.eta*nw
+            network.biases[i] -= self.eta*nb
+
+        reversed_delta_edges = VGroup(*reversed(delta_edge_groups))
+        reversed_delta_neurons = VGroup(*reversed(delta_neuron_groups))
+        edge_groups.save_state()
+        self.play(
+            ShowCreation(
+                reversed_delta_edges,
+                run_time = 2,
+                submobject_mode = "lagged_start",
+                lag_factor = 6,
+            ),
+            FadeIn(
+                reversed_delta_neurons,
+                run_time = 2,
+                submobject_mode = "lagged_start",
+                lag_factor = 4,
+            )
+        )
+        self.color_network_edges()
+        self.play(*it.chain(
+            [ReplacementTransform(
+                edge_groups.saved_state, edge_groups,
+            )],
+            map(FadeOut, [reversed_delta_edges, reversed_delta_neurons]),
+            added_outro_anims,
+        ))
+
+    #####
+
+    def get_adjusted_first_matrix(self, matrix):
+        n = self.network_mob.max_shown_neurons
+        if matrix.shape[1] > n:
+            half = matrix.shape[1]/2
+            return matrix[:,half-n/2:half+n/2]
+        else:
+            return matrix
+
+    def color_network_edges(self):
+        layers = self.network_mob.layers
+        weight_matrices = self.network.weights
+        for layer, matrix in zip(layers[1:], weight_matrices):
+            matrix = self.get_adjusted_first_matrix(matrix)
+            matrix_max = np.max(matrix)
+            for neuron, row in zip(layer.neurons, matrix):
+                for edge, w in zip(neuron.edges_in, row):
+                    color = GREEN if w > 0 else RED
+                    msw = self.max_stroke_width
+                    swe = self.stroke_width_exp
+                    sw = msw*(abs(w)/matrix_max)**swe
+                    sw = min(sw, msw)
+                    edge.set_stroke(color, sw)
+
+class TrainingVsTestData(Scene):
+    CONFIG = {
+        "n_examples" : 10,
+        "n_new_examples_shown" : 10,
+    }
+    def construct(self):
+        self.initialize_data()
+        self.introduce_all_data()
+        self.subdivide_into_training_and_testing()
+        self.scroll_through_much_data()
+
+    def initialize_data(self):
+        training_data, validation_data, test_data = load_data_wrapper()
+        self.data = training_data
+        self.curr_index = 0
+
+    def get_examples(self):
+        ci = self.curr_index
+        self.curr_index += self.n_examples
+        group = Group(*it.starmap(
+            get_training_image_group,
+            self.data[ci:ci+self.n_examples]
+        ))
+        group.arrange_submobjects(DOWN)
+        group.scale(0.5)
+        return group
+
+    def introduce_all_data(self):
+        training_examples, test_examples = [
+            self.get_examples() for x in range(2)
+        ]
+
+        self.play(
+            LaggedStart(FadeIn, training_examples),
+            LaggedStart(FadeIn, test_examples),
+        )
+
+        self.training_examples = training_examples
+        self.test_examples = test_examples
+
+    def subdivide_into_training_and_testing(self):
+        training_examples = self.training_examples
+        test_examples = self.test_examples
+        for examples in training_examples, test_examples:
+            examples.generate_target()
+        training_examples.target.shift(2*LEFT)
+        test_examples.target.shift(2*RIGHT)
+
+        train_brace = Brace(training_examples.target, LEFT)
+        train_words = train_brace.get_text("Train on \\\\ these")
+        test_brace = Brace(test_examples.target, RIGHT)
+        test_words = test_brace.get_text("Test on \\\\ these")
+
+        bools = [True]*(len(test_examples)-1) + [False]
+        random.shuffle(bools)
+        marks = VGroup()
+        for is_correct, test_example in zip(bools, test_examples.target):
+            if is_correct:
+                mark = TexMobject("\\checkmark")
+                mark.highlight(GREEN)
+            else:
+                mark = TexMobject("\\times")
+                mark.highlight(RED)
+            mark.next_to(test_example, LEFT)
+            marks.add(mark)
+
+        self.play(
+            MoveToTarget(training_examples),
+            GrowFromCenter(train_brace),
+            FadeIn(train_words)
+        )
+        self.dither()
+        self.play(
+            MoveToTarget(test_examples),
+            GrowFromCenter(test_brace),
+            FadeIn(test_words)
+        )
+        self.play(Write(marks))
+        self.dither()
+
+    def scroll_through_much_data(self):
+        training_examples = self.training_examples
+        colors = color_gradient([BLUE, YELLOW], self.n_new_examples_shown)
+        for color in colors:
+            new_examples = self.get_examples()
+            new_examples.move_to(training_examples)
+            for train_ex, new_ex in zip(training_examples, new_examples):
+                self.remove(train_ex)
+                self.add(new_ex)
+                new_ex[0][0].highlight(color)
+                self.dither(1./10)
+            training_examples = new_examples
+
+class NotSciFi(TeacherStudentsScene):
+    def construct(self):
+        students = self.students
+        self.student_says(
+            "Machines learning?!?",
+            student_index = 0,
+            target_mode = "confused",
+        )
+        bubble = students[0].bubble
+        students[0].bubble = None
+        self.student_says(
+            "Run!", student_index = 2,
+            target_mode = "pleading",
+            bubble_kwargs = {"direction" : LEFT}
+        )
+        self.dither()
+        students[0].bubble = bubble
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+