Merge pull request #1 from anonymousICLR2019submitter/doc

Add full documentation

Author: anonymousICLR2019submitter

.gitignore

@@ -0,0 +1,16 @@
+### OSX ###
+*.DS_Store
+.AppleDouble
+.LSOverride
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Sphinx documentation
+docs/_build/
+
+# Jupyter Notebook
+.ipynb_checkpoints

deepobs/cifar10/cifar10_3c3d.py

@@ -20,7 +20,34 @@
 
 
 class set_up:
+    """Class providing the functionality for a vanilla CNN architecture on `CIFAR-10`.
+
+    It consists of three convolutional layers with ReLU activations, each followed by max-pooling, followed by two fully-connected layer with ReLU activations and a 10-unit output layer with softmax. The model uses cross-entroy loss. A weight decay is used on the weights (but not the biases) which defaults to ``0.002``. The weight matrices are initialized using the `Xavier-Initializer` and the biases are initialized to ``0``.
+
+    Basis data augmentation (random crop, left-right flip, lighting augmentation) is done on the training images.
+
+    A suggested training settings is for ``100`` epochs with a batch size of ``128``.
+
+    Args:
+        batch_size (int): Batch size of the data points. Defaults to ``128``.
+        weight_decay (float): Weight decay factor. In this model weight decay is applied to the weights, but not the biases. Defaults to ``0.002``.
+
+    Attributes:
+        data_loading (deepobs.data_loading): Data loading class for `CIFAR-10`, :class:`.cifar10_input.data_loading`.
+        losses (tf.Tensor): Tensor of size ``batch_size`` containing the individual losses per data point.
+        accuracy (tf.Tensor): Tensor containing the accuracy of the model.
+        train_init_op (tf.Operation): A TensorFlow operation to be performed before starting every training epoch.
+        train_eval_init_op (tf.Operation): A TensorFlow operation to be performed before starting every training eval epoch.
+        test_init_op (tf.Operation): A TensorFlow operation to be performed before starting every test evaluation phase.
+    """
     def __init__(self, batch_size=128, weight_decay=0.002):
+        """Initializes the problem set_up class.
+
+        Args:
+            batch_size (int): Batch size of the data points. Defaults to ``128``.
+            weight_decay (float): Weight decay factor. In this model weight decay is applied to the weights, but not the biases. Defaults to ``0.002``.
+
+        """
         self.data_loading = cifar10_input.data_loading(batch_size=batch_size)
         self.losses, self.accuracy = self.set_up(weight_decay)
 
@@ -30,50 +57,65 @@ def __init__(self, batch_size=128, weight_decay=0.002):
         self.test_init_op = tf.group([self.data_loading.test_init_op])
 
     def get(self):
+        """Returns the losses and the accuray of the model.
+
+        Returns:
+            tupel: Tupel consisting of the losses and the accuracy.
+
+        """
         return self.losses, self.accuracy
 
     def set_up(self, weight_decay):
+        """Sets up the test problem.
+
+        Args:
+            weight_decay (float): Weight decay factor. In this model weight decay is applied to the weights, but not the biases.
+
+        Returns:
+            tupel: Tupel consisting of the losses and the accuracy.
+
+        """
         X, y, phase = self.data_loading.load()
         print "X", X.get_shape()
 
         W_conv1 = self._conv_filter("W_conv1", [5, 5, 3, 64])
-        b_conv1 = self._bias_variable("b_conv1", [64], init_val=0.0)
-        h_conv1 = tf.nn.relu(self._conv2d(X, W_conv1, padding="VALID") + b_conv1)
+        b_conv1 = self.bias_variable("b_conv1", [64], init_val=0.0)
+        h_conv1 = tf.nn.relu(self.conv2d(X, W_conv1, padding="VALID") + b_conv1)
         print "h_conv1", h_conv1.get_shape()
 
-        h_pool1 = self._max_pool_3x3(h_conv1)
+        h_pool1 = self.max_pool_3x3(h_conv1)
         print "h_pool1", h_pool1.get_shape()
 
         W_conv2 = self._conv_filter("W_conv2", [3, 3, 64, 96])
-        b_conv2 = self._bias_variable("b_conv2", [96], init_val=0.0)
-        h_conv2 = tf.nn.relu(self._conv2d(h_pool1, W_conv2, padding="VALID") + b_conv2)
+        b_conv2 = self.bias_variable("b_conv2", [96], init_val=0.0)
+        h_conv2 = tf.nn.relu(self.conv2d(h_pool1, W_conv2, padding="VALID") + b_conv2)
         print "h_conv2", h_conv2.get_shape()
 
-        h_pool2 = self._max_pool_3x3(h_conv2)
+        h_pool2 = self.max_pool_3x3(h_conv2)
         print "h_pool2", h_pool2.get_shape()
 
         W_conv3 = self._conv_filter("W_conv3", [3, 3, 96, 128])
-        b_conv3 = self._bias_variable("b_conv3", [128], init_val=0.0)
-        h_conv3 = tf.nn.relu(self._conv2d(h_pool2, W_conv3, padding="SAME") + b_conv3)
+        b_conv3 = self.bias_variable("b_conv3", [128], init_val=0.0)
+        h_conv3 = tf.nn.relu(self.conv2d(h_pool2, W_conv3, padding="SAME") + b_conv3)
         print "h_conv3", h_conv3.get_shape()
 
-        h_pool3 = self._max_pool_3x3(h_conv3)
+        h_pool3 = self.max_pool_3x3(h_conv3)
         print "h_pool3", h_pool3.get_shape()
 
         dim = 1152  # Shape of h_pool3 is [batch_size, 3, 3, 128]
         h_pool3_flat = tf.reshape(h_pool3, tf.stack([-1, dim]))
         print "h_pool3_flat", h_pool3_flat.get_shape()
 
-        W_fc1 = self._weight_matrix("W_fc1", [dim, 512])
-        b_fc1 = self._bias_variable("b_fc1", [512], init_val=0.0)
+        W_fc1 = self.weight_matrix("W_fc1", [dim, 512])
+        b_fc1 = self.bias_variable("b_fc1", [512], init_val=0.0)
         h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1)
 
-        W_fc2 = self._weight_matrix("W_fc2", [512, 256])
-        b_fc2 = self._bias_variable("b_fc2", [256], init_val=0.0)
+        W_fc2 = self.weight_matrix("W_fc2", [512, 256])
+        b_fc2 = self.bias_variable("b_fc2", [256], init_val=0.0)
         h_fc2 = tf.nn.relu(tf.matmul(h_fc1, W_fc2) + b_fc2)
 
-        W_fc3 = self._weight_matrix("W_fc3", [256, 10])
-        b_fc3 = self._bias_variable("b_fc3", [10], init_val=0.0)
+        W_fc3 = self.weight_matrix("W_fc3", [256, 10])
+        b_fc3 = self.bias_variable("b_fc3", [10], init_val=0.0)
         linear_outputs = tf.matmul(h_fc2, W_fc3) + b_fc3
 
         losses = tf.nn.softmax_cross_entropy_with_logits_v2(
@@ -91,20 +133,72 @@ def set_up(self, weight_decay):
 
         return losses, accuracy
 
-    def _weight_matrix(self, name, shape):
+    def weight_matrix(self, name, shape):
+        """Creates a weight matrix, initialized by the `Xavier-initializer`.
+
+        Args:
+            name (str): Name of the weight variable.
+            shape (list): Dimensionality of the weight variable.
+
+        Returns:
+            tf.Variable: Weight variable.
+
+        """
         init = tf.contrib.layers.xavier_initializer()
         return tf.get_variable(name, shape, initializer=init)
 
     def _conv_filter(self, name, shape):
+        """Creates a convolutional filter matrix, initialized by the `Xavier-initializer`.
+
+        Args:
+            name (str): Name of the filter variable.
+            shape (list): Dimensionality of the filter variable.
+
+        Returns:
+            tf.Variable: Filter variable.
+
+        """
         init = tf.contrib.layers.xavier_initializer_conv2d()
         return tf.get_variable(name, shape, initializer=init)
 
-    def _bias_variable(self, name, shape, init_val):
+    def bias_variable(self, name, shape, init_val):
+        """Creates a bias variable of given shape and initialized to a given value.
+
+        Args:
+            name (str): Name of the bias variable.
+            shape (list): Dimensionality of the bias variable.
+            init_val (float): Initial value of the bias variable.
+
+        Returns:
+            tf.Variable: Bias variable.
+
+        """
         init = tf.constant_initializer(init_val)
         return tf.get_variable(name, shape, initializer=init)
 
-    def _conv2d(self, x, W, stride=1, padding="VALID"):
+    def conv2d(self, x, W, stride=1, padding="VALID"):
+        """Creates a two dimensional convolutional layer on top of a given input.
+
+        Args:
+            x (tf.Variable): Input to the layer.
+            W (tf.Variable): Weight variable of the convolutional layer.
+            stride (int): Stride of the convolution. Defaults to ``1``.
+            padding (str): Padding of the convolutional layers. Can be ``SAME`` or ``VALID``. Defaults to ``VALID``.
+
+        Returns:
+            tf.Variable: Output after the convolutional layer.
+
+        """
         return tf.nn.conv2d(x, W, strides=[1, stride, stride, 1], padding=padding)
 
-    def _max_pool_3x3(self, x):
+    def max_pool_3x3(self, x):
+        """Creates a ``3`` by ``3`` max pool layer on top of a given input.
+
+        Args:
+            x (tf.Variable): Input to the layer.
+
+        Returns:
+            tf.Variable: Output after the max pool layer.
+
+        """
         return tf.nn.max_pool(x, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding="SAME")

deepobs/cifar10/cifar10_input.py

@@ -9,7 +9,37 @@
 
 
 class data_loading:
+    """Class providing the data loading functionality for the CIFAR-10 data set.
+
+    Args:
+        batch_size (int): Batch size of the input-output pairs. No default value is given.
+        data_augmentation (bool): Switch to turn basic data augmentation on or off while training. Defaults to ``true``.
+
+    Attributes:
+        batch_size (int): Batch size of the input-output pairs.
+        data_augmentation (bool): Switch to turn basic data augmentation on or off while training.
+        train_eval_size (int): Number of data points to evaluate during the `train eval` phase. Currently set to ``10000`` the size of the test set.
+        D_train (tf.data.Dataset): The training data set.
+        D_train_eval (tf.data.Dataset): The training evaluation data set. It is the same data as `D_train` but we go through it separately.
+        D_test (tf.data.Dataset): The test data set.
+        phase (tf.Variable): Variable to describe which phase we are currently in. Can be "train", "train_eval" or "test". The phase variable can determine the behaviour of the network, for example deactivate dropout during evaluation.
+        iterator (tf.data.Iterator): A single iterator for all three data sets. We us the initialization operators (see below) to switch this iterator to the data sets.
+        X (tf.Tensor): Tensor holding the CIFAR-10 images. It has dimension `batch_size` x ``32`` (image size) x ``32`` (image size) x ``3`` (rgb).
+        y (tf.Tensor): Label of the CIFAR-10 images. It has dimension `batch_size` x ``10`` (number of classes).
+        train_init_op (tf.Operation): A TensorFlow operation to be performed before starting every training epoch. It sets the `phase` variable to "train" and initializes the iterator to the training data set.
+        train_eval_init_op (tf.Operation): A TensorFlow operation to be performed before starting every training eval phase. It sets the `phase` variable to "train_eval" and initializes the iterator to the training eval data set.
+        test_init_op (tf.Operation): A TensorFlow operation to be performed before starting every test evaluation phase. It sets the `phase` variable to "test" and initializes the iterator to the test data set.
+
+    """
+
     def __init__(self, batch_size, data_augmentation=True):
+        """Initializes the data loading class.
+
+        Args:
+            batch_size (int): Batch size of the input-output pairs. No default value is given.
+            data_augmentation (bool): Switch to turn basic data augmentation on or off while training. Defaults to ``true``.
+
+        """
         self.train_eval_size = 10000  # The size of the test set
         self.batch_size = batch_size
         self.data_augmentation = data_augmentation
@@ -33,15 +63,29 @@ def __init__(self, batch_size, data_augmentation=True):
             self.D_test), tf.assign(self.phase, "test")], name="test_init_op")
 
     def load(self):
+        """Returns the data (`X` and `y` ) and the phase variable.
+
+        Returns:
+            tupel: Tupel consisting of the data points (`X`), (`y`) and the phase variable (`phase`).
+
+        """
         return self.X, self.y, self.phase
 
     def train_dataset(self, batch_size, data_augmentation=True):
-        """Create a ``tf.data.Dataset`` for the CIFAR-10 training data."""
+        """Creates the training data set.
 
+        Args:
+            batch_size (int): Batch size of the input-output pairs.
+            data_augmentation (bool): Switch to turn basic data augmentation on or off while training. Defaults to ``true``.
+
+        Returns:
+            tf.data.Dataset: The training data set.
+
+        """
         pattern = os.path.join(dataset_utils.get_data_dir(),
                                "cifar-10", "data_batch_*.bin")
         if data_augmentation:
-            D = self._make_dataset(
+            D = self.make_dataset(
                 binaries_fname_pattern=pattern,
                 batch_size=batch_size,
                 crop_size=32,
@@ -56,7 +100,7 @@ def train_dataset(self, batch_size, data_augmentation=True):
                 num_prefetched_batches=3,
                 num_preprocessing_threads=8)
         else:
-            D = self._make_dataset(
+            D = self.make_dataset(
                 binaries_fname_pattern=pattern,
                 batch_size=batch_size, crop_size=32,
                 per_image_standardization=True,
@@ -72,12 +116,20 @@ def train_dataset(self, batch_size, data_augmentation=True):
         return D
 
     def train_eval_dataset(self, batch_size, data_augmentation=True):
-        """Create a ``tf.data.Dataset`` for the CIFAR-10 training evaluation data."""
+        """Creates the train eval data set.
+
+        Args:
+            batch_size (int): Batch size of the input-output pairs.
+            data_augmentation (bool): Switch to turn basic data augmentation on or off while evaluating the training data set. Defaults to ``true``.
+
+        Returns:
+            tf.data.Dataset: The train eval data set.
 
+        """
         pattern = os.path.join(dataset_utils.get_data_dir(),
                                "cifar-10", "data_batch_*.bin")
         if data_augmentation:
-            D = self._make_dataset(
+            D = self.make_dataset(
                 binaries_fname_pattern=pattern,
                 batch_size=batch_size,
                 crop_size=32,
@@ -93,7 +145,7 @@ def train_eval_dataset(self, batch_size, data_augmentation=True):
                 num_preprocessing_threads=8,
                 data_set_size=self.train_eval_size)
         else:
-            D = self._make_dataset(
+            D = self.make_dataset(
                 binaries_fname_pattern=pattern,
                 batch_size=batch_size,
                 crop_size=32,
@@ -111,11 +163,18 @@ def train_eval_dataset(self, batch_size, data_augmentation=True):
         return D
 
     def test_dataset(self, batch_size):
-        """Create a ``tf.data.Dataset`` for the CIFAR-10 test data."""
+        """Creates the test data set.
 
+        Args:
+            batch_size (int): Batch size of the input-output pairs.
+
+        Returns:
+            tf.data.Dataset: The test data set.
+
+        """
         pattern = os.path.join(dataset_utils.get_data_dir(),
                                "cifar-10", "test_batch.bin")
-        return self._make_dataset(
+        return self.make_dataset(
             binaries_fname_pattern=pattern,
             batch_size=batch_size,
             crop_size=32,
@@ -130,23 +189,29 @@ def test_dataset(self, batch_size):
             num_prefetched_batches=3,
             num_preprocessing_threads=4)
 
-    def _make_dataset(self,
-                      binaries_fname_pattern,
-                      batch_size,
-                      crop_size=32,
-                      per_image_standardization=True,
-                      random_crop=False,
-                      pad_before_random_crop=0,
-                      random_flip_left_right=False,
-                      lighting_augmentation=False,
-                      one_hot=True,
-                      shuffle=True,
-                      shuffle_buffer_size=10000,
-                      num_prefetched_batches=3,
-                      num_preprocessing_threads=8,
-                      data_set_size=-1):
-        """Produce CIFAR dataset."""
+    def make_dataset(self, binaries_fname_pattern, batch_size, crop_size=32, per_image_standardization=True, random_crop=False, pad_before_random_crop=0, random_flip_left_right=False, lighting_augmentation=False, one_hot=True, shuffle=True, shuffle_buffer_size=10000, num_prefetched_batches=3, num_preprocessing_threads=8, data_set_size=-1):
+        """Creates a data set from a pattern of the images and label files.
+
+        Args:
+            binaries_fname_pattern (str): Pattern of the ``,bin`` files containing the images and labels.
+            batch_size (int): Batch size of the input-output pairs.
+            crop_size (int): Crop size of each image. Defaults to ``32``.
+            per_image_standardization (bool): Switch to standardize each image to have zero mean and unit norm. Defaults to ``True``.
+            random_crop (bool): Switch if random crops should be used. Defaults to ``False``.
+            pad_before_random_crop (int): Defines the added padding before a random crop is applied. Defaults to ``0``.
+            random_flip_left_right (bool): Switch to randomly flip the images horizontally. Defaults to ``False``.
+            lighting_augmentation (bool): Switch to use random brightness, saturation and contrast on each image. Defaults to ``False``.
+            one_hot (bool): Switch to turn on or off one-hot encoding of the labels. Defaults to ``True``.
+            shuffle (bool):  Switch to turn on or off shuffling of the data set. Defaults to ``True``.
+            shuffle_buffer_size (int): Size of the shuffle buffer. Defaults to ``10000`` the size of the `test` and `train eval` data set, meaning that they will be completely shuffled.
+            num_prefetched_batches (int): Number of prefeteched batches, defaults to ``3``.
+            num_preprocessing_threads (int): The number of elements to process in parallel while applying the image transformations. Defaults to ``8``.
+            data_set_size (int): Size of the data set to extract from the images and label files. Defaults to ``-1`` meaning that the full data set is used.
+
+        Returns:
+            tf.data.Dataset: Data set object created from the images and label files.
 
+        """
         # Set number of bytes to read
         label_bytes = 1
         label_offset = 0