diff --git a/dlnd_face_generation.html b/dlnd_face_generation.html index 5910596..af5860b 100644 --- a/dlnd_face_generation.html +++ b/dlnd_face_generation.html @@ -11789,7 +11789,7 @@ div#notebook {
#data_dir = './data'
@@ -11843,7 +11843,7 @@ Found celeba Data
show_n_images = 25
@@ -11870,13 +11870,13 @@ Found celeba Data
-Out[3]:
+Out[4]:
@@ -12449,7 +12449,7 @@ FmBaZwAAAABJRU5ErkJggg==
show_n_images = 25
@@ -12471,13 +12471,13 @@ FmBaZwAAAABJRU5ErkJggg==
-Out[4]:
+Out[5]:
@@ -13602,7 +13602,7 @@ P7E6eb8E6wAAAABJRU5ErkJggg==
"""
@@ -13664,7 +13664,7 @@ Default GPU Device: /gpu:0
import problem_unittests as tests
@@ -13710,7 +13710,7 @@ Object was never used (type <class 'tensorflow.python.framework.ops.Opera
<tf.Operation 'assert_rank_2/Assert/Assert' type=Assert>
If you want to mark it as used call its "mark_used()" method.
It was originally created here:
-['File "/usr/local/lib/python3.5/runpy.py", line 193, in _run_module_as_main\n "__main__", mod_spec)', 'File "/usr/local/lib/python3.5/runpy.py", line 85, in _run_code\n exec(code, run_globals)', 'File "/usr/local/lib/python3.5/site-packages/ipykernel_launcher.py", line 16, in <module>\n app.launch_new_instance()', 'File "/usr/local/lib/python3.5/site-packages/traitlets/config/application.py", line 658, in launch_instance\n app.start()', 'File "/usr/local/lib/python3.5/site-packages/ipykernel/kernelapp.py", line 477, in start\n ioloop.IOLoop.instance().start()', 'File "/usr/local/lib/python3.5/site-packages/zmq/eventloop/ioloop.py", line 177, in start\n super(ZMQIOLoop, self).start()', 'File "/usr/local/lib/python3.5/site-packages/tornado/ioloop.py", line 888, in start\n handler_func(fd_obj, events)', 'File "/usr/local/lib/python3.5/site-packages/tornado/stack_context.py", line 277, in null_wrapper\n return fn(*args, **kwargs)', 'File "/usr/local/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events\n self._handle_recv()', 'File "/usr/local/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv\n self._run_callback(callback, msg)', 'File "/usr/local/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback\n callback(*args, **kwargs)', 'File "/usr/local/lib/python3.5/site-packages/tornado/stack_context.py", line 277, in null_wrapper\n return fn(*args, **kwargs)', 'File "/usr/local/lib/python3.5/site-packages/ipykernel/kernelbase.py", line 283, in dispatcher\n return self.dispatch_shell(stream, msg)', 'File "/usr/local/lib/python3.5/site-packages/ipykernel/kernelbase.py", line 235, in dispatch_shell\n handler(stream, idents, msg)', 'File "/usr/local/lib/python3.5/site-packages/ipykernel/kernelbase.py", line 399, in execute_request\n user_expressions, allow_stdin)', 'File "/usr/local/lib/python3.5/site-packages/ipykernel/ipkernel.py", line 196, in do_execute\n res = shell.run_cell(code, store_history=store_history, silent=silent)', 'File "/usr/local/lib/python3.5/site-packages/ipykernel/zmqshell.py", line 533, in run_cell\n return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)', 'File "/usr/local/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2698, in run_cell\n interactivity=interactivity, compiler=compiler, result=result)', 'File "/usr/local/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2808, in run_ast_nodes\n if self.run_code(code, result):', 'File "/usr/local/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2862, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)', 'File "<ipython-input-6-60e8f0e9d266>", line 22, in <module>\n tests.test_model_inputs(model_inputs)', 'File "/output/problem_unittests.py", line 12, in func_wrapper\n result = func(*args)', 'File "/output/problem_unittests.py", line 68, in test_model_inputs\n _check_input(learn_rate, [], \'Learning Rate\')', 'File "/output/problem_unittests.py", line 34, in _check_input\n _assert_tensor_shape(tensor, shape, \'Real Input\')', 'File "/output/problem_unittests.py", line 20, in _assert_tensor_shape\n assert tf.assert_rank(tensor, len(shape), message=\'{} has wrong rank\'.format(display_name))', 'File "/usr/local/lib/python3.5/site-packages/tensorflow/python/ops/check_ops.py", line 617, in assert_rank\n dynamic_condition, data, summarize)', 'File "/usr/local/lib/python3.5/site-packages/tensorflow/python/ops/check_ops.py", line 571, in _assert_rank_condition\n return control_flow_ops.Assert(condition, data, summarize=summarize)', 'File "/usr/local/lib/python3.5/site-packages/tensorflow/python/util/tf_should_use.py", line 170, in wrapped\n return _add_should_use_warning(fn(*args, **kwargs))', 'File "/usr/local/lib/python3.5/site-packages/tensorflow/python/util/tf_should_use.py", line 139, in _add_should_use_warning\n wrapped = TFShouldUseWarningWrapper(x)', 'File "/usr/local/lib/python3.5/site-packages/tensorflow/python/util/tf_should_use.py", line 96, in __init__\n stack = [s.strip() for s in traceback.format_stack()]']
+['File "/usr/local/lib/python3.5/runpy.py", line 193, in _run_module_as_main\n "__main__", mod_spec)', 'File "/usr/local/lib/python3.5/runpy.py", line 85, in _run_code\n exec(code, run_globals)', 'File "/usr/local/lib/python3.5/site-packages/ipykernel_launcher.py", line 16, in <module>\n app.launch_new_instance()', 'File "/usr/local/lib/python3.5/site-packages/traitlets/config/application.py", line 658, in launch_instance\n app.start()', 'File "/usr/local/lib/python3.5/site-packages/ipykernel/kernelapp.py", line 477, in start\n ioloop.IOLoop.instance().start()', 'File "/usr/local/lib/python3.5/site-packages/zmq/eventloop/ioloop.py", line 177, in start\n super(ZMQIOLoop, self).start()', 'File "/usr/local/lib/python3.5/site-packages/tornado/ioloop.py", line 888, in start\n handler_func(fd_obj, events)', 'File "/usr/local/lib/python3.5/site-packages/tornado/stack_context.py", line 277, in null_wrapper\n return fn(*args, **kwargs)', 'File "/usr/local/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events\n self._handle_recv()', 'File "/usr/local/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv\n self._run_callback(callback, msg)', 'File "/usr/local/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback\n callback(*args, **kwargs)', 'File "/usr/local/lib/python3.5/site-packages/tornado/stack_context.py", line 277, in null_wrapper\n return fn(*args, **kwargs)', 'File "/usr/local/lib/python3.5/site-packages/ipykernel/kernelbase.py", line 283, in dispatcher\n return self.dispatch_shell(stream, msg)', 'File "/usr/local/lib/python3.5/site-packages/ipykernel/kernelbase.py", line 235, in dispatch_shell\n handler(stream, idents, msg)', 'File "/usr/local/lib/python3.5/site-packages/ipykernel/kernelbase.py", line 399, in execute_request\n user_expressions, allow_stdin)', 'File "/usr/local/lib/python3.5/site-packages/ipykernel/ipkernel.py", line 196, in do_execute\n res = shell.run_cell(code, store_history=store_history, silent=silent)', 'File "/usr/local/lib/python3.5/site-packages/ipykernel/zmqshell.py", line 533, in run_cell\n return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)', 'File "/usr/local/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2698, in run_cell\n interactivity=interactivity, compiler=compiler, result=result)', 'File "/usr/local/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2808, in run_ast_nodes\n if self.run_code(code, result):', 'File "/usr/local/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2862, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)', 'File "<ipython-input-7-60e8f0e9d266>", line 22, in <module>\n tests.test_model_inputs(model_inputs)', 'File "/output/problem_unittests.py", line 12, in func_wrapper\n result = func(*args)', 'File "/output/problem_unittests.py", line 68, in test_model_inputs\n _check_input(learn_rate, [], \'Learning Rate\')', 'File "/output/problem_unittests.py", line 34, in _check_input\n _assert_tensor_shape(tensor, shape, \'Real Input\')', 'File "/output/problem_unittests.py", line 20, in _assert_tensor_shape\n assert tf.assert_rank(tensor, len(shape), message=\'{} has wrong rank\'.format(display_name))', 'File "/usr/local/lib/python3.5/site-packages/tensorflow/python/ops/check_ops.py", line 617, in assert_rank\n dynamic_condition, data, summarize)', 'File "/usr/local/lib/python3.5/site-packages/tensorflow/python/ops/check_ops.py", line 571, in _assert_rank_condition\n return control_flow_ops.Assert(condition, data, summarize=summarize)', 'File "/usr/local/lib/python3.5/site-packages/tensorflow/python/util/tf_should_use.py", line 170, in wrapped\n return _add_should_use_warning(fn(*args, **kwargs))', 'File "/usr/local/lib/python3.5/site-packages/tensorflow/python/util/tf_should_use.py", line 139, in _add_should_use_warning\n wrapped = TFShouldUseWarningWrapper(x)', 'File "/usr/local/lib/python3.5/site-packages/tensorflow/python/util/tf_should_use.py", line 96, in __init__\n stack = [s.strip() for s in traceback.format_stack()]']
==================================
Tests Passed
@@ -13731,12 +13731,20 @@ Tests Passed
alpha = 0.1 # for leaky relu
+# leaky relu slope
+alpha = 0.2
+
+# print loss every
print_every = 10
-show_every = 100 #change to 100
+
+# show image sample from G every
+show_every = 100
+
+# Smoothing
+smooth = 0.1
def discriminator(images, reuse=False):
@@ -13767,35 +13775,105 @@ Tests Passed
"""
with tf.variable_scope('discriminator', reuse=reuse):
- # Input layer is 28x28x3
- x1 = tf.layers.conv2d(images, 64, 5, strides=1, padding='valid')
- relu1 = tf.maximum(alpha * x1, x1)
+ # Weights Initializer
+ xavier_init = tf.contrib.layers.xavier_initializer()
+
+
+ ############# CONV BLOCK 1 ################
+ # conv2d -> relu
+
+ # Input layer is 28x28x3 (no batch norm here)
+ conv1 = tf.layers.conv2d(images, 64, 5,
+ strides=1,
+ padding='valid',
+ kernel_initializer=xavier_init)
+ relu1 = tf.maximum(alpha * conv1, conv1)
#print(relu1)
# 24x24x64
- x2 = tf.layers.conv2d(relu1, 128, 5, strides=1, padding='valid')
- bn2 = tf.layers.batch_normalization(x2, training=True)
- relu2 = tf.maximum(alpha * bn2, bn2)
+ ############# END CONV BLOCK 1 ############
+
+
+
+
+ ############# CONV BLOCK 2 ################
+ # conv2d -> batch_norm -> drop -> relu
+
+ conv2 = tf.layers.conv2d(relu1, 128, 5,
+ strides=1,
+ padding='valid',
+ use_bias=False,
+ kernel_initializer=xavier_init)
+
+ bn2 = tf.layers.batch_normalization(conv2, training=True)
+ drop2 = tf.layers.dropout(bn2, training=True)
+ relu2 = tf.maximum(alpha * drop2, drop2)
#print(relu2)
# 20x20x128
- x3 = tf.layers.conv2d(relu2, 256, 5, strides=1, padding='valid')
- bn3 = tf.layers.batch_normalization(x3, training=True)
- relu3 = tf.maximum(alpha * bn3, bn3)
+ ############# END CONV BLOCK 2 ############
+
+
+
+
+
+
+ ############# CONV BLOCK 3 ##############
+ # conv2d -> batch_norm -> drop -> relu
+
+ conv3 = tf.layers.conv2d(relu2, 256, 5,
+ strides=1,
+ padding='valid',
+ use_bias=False,
+ kernel_initializer=xavier_init)
+
+ bn3 = tf.layers.batch_normalization(conv3, training=True)
+ drop3 = tf.layers.dropout(bn3, training=True)
+ relu3 = tf.maximum(alpha * drop3, drop3)
#print(relu3)
# 16x16x256
- x4 = tf.layers.conv2d(relu3, 512, 5, strides=2, padding='valid')
- bn4 = tf.layers.batch_normalization(x4, training=True)
- relu4 = tf.maximum(alpha * bn4, bn4)
+ ############# END CONV BLOCK 3 ##############
+
+
+
+
+
+
+
+
+ ############# CONV BLOCK 4 ##################
+ # conv2d -> batch_norm -> drop -> relu
+
+
+ conv4 = tf.layers.conv2d(relu3, 512, 5,
+ strides=2,
+ padding='valid',
+ use_bias=False,
+ kernel_initializer=xavier_init)
+
+ bn4 = tf.layers.batch_normalization(conv4, training=True)
+ drop4 = tf.layers.dropout(bn4, training=True)
+ relu4 = tf.maximum(alpha * drop4, drop4)
#print(relu4)
# 6x6x512
+
+ ############# END CONV BLOCK 4 ##############
+
+
+
+
+
+ ############# Out Layer ##############
# Flatten it
flat = tf.reshape(relu4, (-1, 6*6*512))
- logits = tf.layers.dense(flat, 1)
+ logits = tf.layers.dense(flat, 1, kernel_initializer=xavier_init)
#print(flat)
out = tf.sigmoid(logits)
+
+
+ ############# End Out Layer ##############
@@ -13842,7 +13920,7 @@ Tests Passed
def generator(z, out_channel_dim, is_train=True):
@@ -13856,43 +13934,109 @@ Tests Passed
with tf.variable_scope('generator', reuse=not is_train):
- # First fully connected layer
- fc = tf.layers.dense(z, 7*7*1024)
+
+ # Weights Initializer
+ xavier_init = tf.contrib.layers.xavier_initializer()
+
+ ############# Fully Connected ################
+
+ fc = tf.layers.dense(z, 7*7*1024, kernel_initializer=xavier_init)
fc = tf.reshape(fc, (-1, 7, 7, 1024))
- fc = tf.maximum(alpha * fc, fc)
+ drop = tf.layers.dropout(fc, training=is_train)
+ relu = tf.maximum(alpha * drop, drop)
#print(fc)
# 7x7x1024 now
- # Reshape it to start the convolutional stack
- #x1 = tf.layers.batch_normalization(x1, training=is_train)
+ ############# END Fully connected #############
+
+
+
- # Deconv1
- dconv1 = tf.layers.conv2d_transpose(fc, 512, 3, strides=2, padding='same')
- dconv1 = tf.layers.batch_normalization(dconv1, training=is_train)
- dconv1 = tf.maximum(alpha * dconv1, dconv1)
+ ############# DCONV BLOCK 1 ###################
+ # conv2d_transpose -> batch_norm -> drop -> relu
+
+ dconv1 = tf.layers.conv2d_transpose(relu, 512, 3,
+ strides=2,
+ padding='same',
+ use_bias=False,
+ kernel_initializer=xavier_init)
+
+ bn1 = tf.layers.batch_normalization(dconv1, training=is_train)
+ drop1 = tf.layers.dropout(bn1, training=is_train)
+ relu1 = tf.maximum(alpha * drop1, drop1)
#print(dconv1)
# 14x14x512 now
- dconv2 = tf.layers.conv2d_transpose(dconv1, 256, 3, strides=2, padding='same')
- dconv2 = tf.layers.batch_normalization(dconv2, training=is_train)
- dconv2 = tf.maximum(alpha * dconv2, dconv2)
+ ############# END DCONV BLOCK 1 ################
+
+
+
+
+
+
+ ############# DCONV BLOCK 2 ###################
+ # conv2d_transpose -> batch_norm -> drop -> relu
+
+
+ dconv2 = tf.layers.conv2d_transpose(relu1, 256, 3,
+ strides=2,
+ padding='same',
+ use_bias=False,
+ kernel_initializer=xavier_init)
+
+ bn2 = tf.layers.batch_normalization(dconv2, training=is_train)
+ drop2 = tf.layers.dropout(bn2, training=is_train)
+ relu2 = tf.maximum(alpha * drop2, drop2)
#print(dconv2)
# 28x28x256 now
- dconv3 = tf.layers.conv2d_transpose(dconv2, 128, 5, strides=1, padding='same')
- dconv3 = tf.layers.batch_normalization(dconv3, training=is_train)
- dconv3 = tf.maximum(alpha * dconv3, dconv3)
+ ############# END DCONV BLOCK 2 ###################
+
+
+
+
+
+
+
+
+ ############# DCONV BLOCK 3 ###################
+ # conv2d_transpose -> batch_norm -> drop -> relu
+
+ dconv3 = tf.layers.conv2d_transpose(relu2, 128, 5,
+ strides=1,
+ padding='same',
+ use_bias=False,
+ kernel_initializer=xavier_init)
+
+ bn3 = tf.layers.batch_normalization(dconv3, training=is_train)
+ drop3 = tf.layers.dropout(bn3,training=is_train)
+ relu3 = tf.maximum(alpha * drop3, drop3)
#print(dconv3)
# 28x28x128 now
- # Output layer
- logits = tf.layers.conv2d_transpose(dconv3, out_channel_dim, 5, strides=1, padding='same')
+ ############# END DCONV BLOCK 3 ###################
+
+
+
+
+
+
+
+ ############# Out Layer ##############
+
+ logits = tf.layers.conv2d_transpose(relu3, out_channel_dim, 5,
+ strides=1,
+ padding='same',
+ kernel_initializer=xavier_init)
+
#print(logits)
# 28x28x(channels) now
out = tf.tanh(logits)
+
+ ############# Out Layer ##############
return out
@@ -13941,7 +14085,7 @@ Tests Passed
def model_loss(input_real, input_z, out_channel_dim):
@@ -13957,9 +14101,13 @@ Tests Passed
d_model_fake, d_logits_fake = discriminator(g_model, reuse=True)
d_loss_real = tf.reduce_mean(
- tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logits_real, labels=tf.ones_like(d_model_real)))
+ tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logits_real,
+ labels=tf.ones_like(d_model_real) * (1 - smooth)))
+
d_loss_fake = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logits_fake, labels=tf.zeros_like(d_model_fake)))
+
+
g_loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logits_fake, labels=tf.ones_like(d_model_fake)))
@@ -14008,7 +14156,7 @@ Tests Passed
def model_opt(d_loss, g_loss, learning_rate, beta1):
@@ -14075,7 +14223,7 @@ Tests Passed
"""
@@ -14127,7 +14275,7 @@ Tests Passed
from IPython.core.debugger import Tracer
@@ -14255,13 +14403,13 @@ Tests Passed
batch_size = 32
+batch_size = 16
z_dim = 100
-learning_rate = 0.001
-beta1 = 0.5
+learning_rate = 0.0008
+beta1 = 0.3
"""
@@ -14289,16 +14437,16 @@ Tests Passed
@@ -14312,249 +14460,111 @@ Epoch 1/2-Step 100... Discriminator Loss: 2.0016... Generator Loss: 0.2221
@@ -14567,16 +14577,16 @@ QmaizQAAAABJRU5ErkJggg==
Run your GANs on CelebA. It will take around 20 minutes on the average GPU to run one epoch. You can run the whole epoch or stop when it starts to generate realistic faces.
+batch_size = 64
-z_dim = 100
-learning_rate = 0.0002
-beta1 = 0.5
+
+
+
-"""
-DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
-"""
-epochs = 1
-celeba_dataset = helper.Dataset('celeba', glob(os.path.join(data_dir, 'img_align_celeba/*.jpg')))
-with tf.Graph().as_default():
- train(epochs, batch_size, z_dim, learning_rate, beta1, celeba_dataset.get_batches,
- celeba_dataset.shape, celeba_dataset.image_mode)
-
+
+
+
Run your GANs on CelebA. It will take around 20 minutes on the average GPU to run one epoch. You can run the whole epoch or stop when it starts to generate realistic faces.
+ +batch_size = 16
+z_dim = 100
+learning_rate = 0.0003
+beta1 = 0.5
+
+
+"""
+DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
+"""
+epochs = 1
+
+celeba_dataset = helper.Dataset('celeba', glob(os.path.join(data_dir, 'img_align_celeba/*.jpg')))
+with tf.Graph().as_default():
+ train(epochs, batch_size, z_dim, learning_rate, beta1, celeba_dataset.get_batches,
+ celeba_dataset.shape, celeba_dataset.image_mode)
+