Add files via upload

Author: hyunjoonbok

tensorflow/How to Use Multi GPUs in Training in TensorFlow .ipynb

@@ -0,0 +1,145 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# How to Use Multi GPUs in Training in TensorFlow "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This is a simple codeset to utilize mutiple GPUs (if available) for a example MNIST problem in tensorflow"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Bascially, we would only have to add two additional lines of code in \"Network\" part to utilize Multiple GPUs"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "strategy = tf.distribute.MirroredStrategy()\n",
+    "with strategy.scope():"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Import Package\n",
+    "import os\n",
+    "import numpy as np\n",
+    "import tensorflow as tf\n",
+    "from tensorflow.keras import layers, models, losses, optimizers, datasets, utils\n",
+    "\n",
+    "# Data Prepare\n",
+    "(train_x, train_y), (test_x, test_y) = datasets.mnist.load_data()\n",
+    "train_x, test_x = np.expand_dims(train_x/255., -1), np.expand_dims(test_x/255., -1)\n",
+    "print(\"Train Data's Shape : \", train_x.shape, train_y.shape)\n",
+    "print(\"Test Data's Shape : \", test_x.shape, test_y.shape)\n",
+    "\n",
+    "# Build Network\n",
+    "strategy = tf.distribute.MirroredStrategy() # New Lines \n",
+    "with strategy.scope(): # New Lines\n",
+    "    cnn = models.Sequential()\n",
+    "    cnn.add(layers.Conv2D(16, 3, activation='relu', input_shape=(28, 28, 1,)))\n",
+    "    cnn.add(layers.MaxPool2D())\n",
+    "    cnn.add(layers.Conv2D(32, 3, activation='relu'))\n",
+    "    cnn.add(layers.MaxPool2D())\n",
+    "    cnn.add(layers.Flatten())\n",
+    "    cnn.add(layers.Dense(10, activation='softmax'))\n",
+    "\n",
+    "    cnn.compile(optimizer=optimizers.Adam(), loss=losses.sparse_categorical_crossentropy, metrics=['accuracy'])                \n",
+    "print(\"Network Built!\")\n",
+    "\n",
+    "# Training Network with Multi GPUs\n",
+    "epochs=10\n",
+    "batch_size_each_gpu = 4096\n",
+    "batch_size = batch_size_each_gpu*len(gpus)  \n",
+    "\n",
+    "## ================= ##\n",
+    "# Single GPU code \n",
+    "# epochs=10\n",
+    "# batch_size = 4096\n",
+    "## ================= ##\n",
+    "\n",
+    "history = cnn.fit(train_x, train_y, epochs=10, batch_size=batch_size, validation_data=(test_x, test_y))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Adding a single GPU codeset below for comparison"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Import Package\n",
+    "import os\n",
+    "import numpy as np\n",
+    "import tensorflow as tf\n",
+    "from tensorflow.keras import layers, models, losses, optimizers, datasets, utils\n",
+    "\n",
+    "# Data Prepare\n",
+    "(train_x, train_y), (test_x, test_y) = datasets.mnist.load_data()\n",
+    "train_x, test_x = np.expand_dims(train_x/255., -1), np.expand_dims(test_x/255., -1)\n",
+    "print(\"Train Data's Shape : \", train_x.shape, train_y.shape)\n",
+    "print(\"Test Data's Shape : \", test_x.shape, test_y.shape)\n",
+    "\n",
+    "# Build Network\n",
+    "cnn = models.Sequential()\n",
+    "cnn.add(layers.Conv2D(16, 3, activation='relu', input_shape=(28, 28, 1,)))\n",
+    "cnn.add(layers.MaxPool2D())\n",
+    "cnn.add(layers.Conv2D(32, 3, activation='relu'))\n",
+    "cnn.add(layers.MaxPool2D())\n",
+    "cnn.add(layers.Flatten())\n",
+    "cnn.add(layers.Dense(10, activation='softmax'))\n",
+    "\n",
+    "cnn.compile(optimizer=optimizers.Adam(), loss=losses.sparse_categorical_crossentropy, metrics=['accuracy'])                \n",
+    "print(\"Network Built!\")\n",
+    "\n",
+    "# Training Network\n",
+    "epochs=10\n",
+    "batch_size = 4096\n",
+    "history = cnn.fit(train_x, train_y, epochs=10, batch_size=batch_size, validation_data=(test_x, test_y))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}