{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist=input_data.read_data_sets('MNIST_data/',one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 初始化参数\n",
    "def weight_variable(shape):\n",
    "    initial = tf.truncated_normal(shape,stddev=0.1)\n",
    "    return tf.Variable(initial)\n",
    "\n",
    "def bias_variable(shape):\n",
    "    '''\n",
    "    bias的初始化会有比较大的影响，如果采用truncated_nomal初始化的话\n",
    "    模型貌似是不能收敛的\n",
    "    '''\n",
    "    initial = tf.constant(0.1,shape=shape)\n",
    "    return tf.Variable(initial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义卷积层和池化层\n",
    "def conv2d(x,W):\n",
    "    '''\n",
    "     x表示输入图像，具体定义：[batch, in_height, in_width, in_channels]`\n",
    "     W表示自定义的卷积核，具体定义如下：[filter_height, filter_width, in_channels, out_channels]\n",
    "     strids表示步长的维度信息，定义如下：[batch, height, width, channels]维度上的步长\n",
    "    '''\n",
    "    return tf.nn.conv2d(x,W,strides=[1,1,1,1],padding='SAME')\n",
    "\n",
    "def max_pool_2x2(x):\n",
    "    '''\n",
    "    x：[batch, in_height, in_width, in_channels]4个维度的\n",
    "    ksize表示pool所有核的维度信息，4个参数对应于输入x的四个维度的pool\n",
    "    stride：表示步长的维度信息\n",
    "    '''\n",
    "    return tf.nn.max_pool(x,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "input_layer\n",
    "'''\n",
    "x_input=tf.placeholder(shape=(None,784),dtype=tf.float32)\n",
    "y_input=tf.placeholder(shape=(None,10),dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "first layer conv\n",
    "这里比较奇怪，相当于我们要先将卷积核定义好，然后再输入\n",
    "''' \n",
    "# 第一层卷积核size为5*5，输入图像channel为1，输出feature-map的channel为32\n",
    "W_conv1=weight_variable([5,5,1,32])\n",
    "b_conv1=bias_variable([32])\n",
    "\n",
    "# 把输入变回28*28的图像\n",
    "x_image=tf.reshape(x_input,shape=[-1,28,28,1])\n",
    "\n",
    "# 做卷积\n",
    "h_conv1 = tf.nn.relu(conv2d(x_image,W_conv1)+b_conv1)\n",
    "h_pool1 = max_pool_2x2(h_conv1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "second layer conv\n",
    "'''\n",
    "W_conv2 = weight_variable([5,5,32,64])\n",
    "b_conv2 = bias_variable([64])\n",
    "\n",
    "h_conv2 = tf.nn.relu(conv2d(h_pool1,W_conv2)+b_conv2)\n",
    "h_pool2 = max_pool_2x2(h_conv2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Dense layer\n",
    "'''\n",
    "W_fc1=weight_variable([7*7*64,1024])\n",
    "b_fc1=bias_variable([1024])\n",
    "\n",
    "h_pool2_flat=tf.reshape(h_pool2,shape=[-1,7*7*64])\n",
    "h_fc1=tf.nn.relu(tf.matmul(h_pool2_flat,W_fc1)+b_fc1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Drop layer\n",
    "'''\n",
    "keep_prob=tf.placeholder(dtype=tf.float32)\n",
    "h_fc1_drop=tf.nn.dropout(h_fc1,keep_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "output_layer\n",
    "'''\n",
    "W_fc2=weight_variable([1024,10])\n",
    "b_fc2=weight_variable([10])\n",
    "\n",
    "y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop,W_fc2)+b_fc2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "define loss\n",
    "cross entropy\n",
    "\n",
    "为了规避掉softmax+crossentropy分开计算有数值上的问题，\n",
    "这里可以是用tf.nn.softmax_cross_entropy_with_logits来作为最后一层的激活函数，\n",
    "\n",
    "最后取个均值，就可以做最后的loss了\n",
    "'''\n",
    "loss=-tf.reduce_sum(y_input*tf.log(y_conv))\n",
    "trainer=tf.train.AdamOptimizer(learning_rate=1e-4).minimize(loss)\n",
    "\n",
    "# 计算每个变量的梯度\n",
    "grads = tf.gradients(loss, tf.trainable_variables())\n",
    "grads = list(zip(grads, tf.trainable_variables()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "evaluate model\n",
    "'''\n",
    "correct_pred=tf.equal(tf.argmax(y_conv,1),tf.argmax(y_input,1))\n",
    "# 计算均值，cast之后的correct_pred相当于是【0,0,0,1,1,1,0】这样的数组，\n",
    "# 算出它的均值就是对应的accuracy了\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_pred,'float'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Summary name Variable:0 is illegal; using Variable_0 instead.\n",
      "INFO:tensorflow:Summary name Variable_1:0 is illegal; using Variable_1_0 instead.\n",
      "INFO:tensorflow:Summary name Variable_2:0 is illegal; using Variable_2_0 instead.\n",
      "INFO:tensorflow:Summary name Variable_3:0 is illegal; using Variable_3_0 instead.\n",
      "INFO:tensorflow:Summary name Variable_4:0 is illegal; using Variable_4_0 instead.\n",
      "INFO:tensorflow:Summary name Variable_5:0 is illegal; using Variable_5_0 instead.\n",
      "INFO:tensorflow:Summary name Variable_6:0 is illegal; using Variable_6_0 instead.\n",
      "INFO:tensorflow:Summary name Variable_7:0 is illegal; using Variable_7_0 instead.\n",
      "INFO:tensorflow:Summary name Variable:0/gradient is illegal; using Variable_0/gradient instead.\n",
      "INFO:tensorflow:Summary name Variable_1:0/gradient is illegal; using Variable_1_0/gradient instead.\n",
      "INFO:tensorflow:Summary name Variable_2:0/gradient is illegal; using Variable_2_0/gradient instead.\n",
      "INFO:tensorflow:Summary name Variable_3:0/gradient is illegal; using Variable_3_0/gradient instead.\n",
      "INFO:tensorflow:Summary name Variable_4:0/gradient is illegal; using Variable_4_0/gradient instead.\n",
      "INFO:tensorflow:Summary name Variable_5:0/gradient is illegal; using Variable_5_0/gradient instead.\n",
      "INFO:tensorflow:Summary name Variable_6:0/gradient is illegal; using Variable_6_0/gradient instead.\n",
      "INFO:tensorflow:Summary name Variable_7:0/gradient is illegal; using Variable_7_0/gradient instead.\n"
     ]
    }
   ],
   "source": [
    "for var in tf.trainable_variables():\n",
    "    tf.summary.histogram(var.name, var)\n",
    "# Summarize all gradients\n",
    "for grad, var in grads:\n",
    "    tf.summary.histogram(var.name + '/gradient', grad)\n",
    "\n",
    "merged_summary_op = tf.summary.merge_all()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.12\n",
      "1 0.84\n",
      "2 0.94\n",
      "3 0.9\n",
      "4 0.86\n",
      "5 0.92\n",
      "6 0.98\n",
      "7 0.96\n",
      "8 0.96\n",
      "9 0.96\n",
      "10 0.98\n",
      "11 0.96\n",
      "12 1.0\n",
      "13 0.94\n",
      "14 0.94\n",
      "15 0.98\n",
      "16 1.0\n",
      "17 0.98\n",
      "18 0.92\n",
      "19 0.98\n",
      "20 0.98\n",
      "21 0.98\n",
      "22 1.0\n",
      "23 0.96\n",
      "24 1.0\n",
      "25 1.0\n",
      "26 0.96\n",
      "27 0.98\n",
      "28 1.0\n",
      "29 0.96\n",
      "30 0.98\n",
      "31 1.0\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-18-ef5bba5d1423>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     20\u001b[0m         _,summary_=sess.run([trainer,merged_summary_op],feed_dict={x_input:batch_x,\n\u001b[0;32m     21\u001b[0m                                     \u001b[0my_input\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mbatch_y\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 22\u001b[1;33m                                     keep_prob:0.5})\n\u001b[0m\u001b[0;32m     23\u001b[0m         \u001b[0msummary_writer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd_summary\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msummary_\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Python\\PY3.6\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    903\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    904\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 905\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    906\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    907\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Python\\PY3.6\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1135\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1136\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1137\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1138\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1139\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Python\\PY3.6\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1353\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1354\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[1;32m-> 1355\u001b[1;33m                            options, run_metadata)\n\u001b[0m\u001b[0;32m   1356\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1357\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Python\\PY3.6\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1359\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1360\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1361\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1362\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1363\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Python\\PY3.6\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1338\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1339\u001b[0m           return tf_session.TF_Run(session, options, feed_dict, fetch_list,\n\u001b[1;32m-> 1340\u001b[1;33m                                    target_list, status, run_metadata)\n\u001b[0m\u001b[0;32m   1341\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1342\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "'''\n",
    "train\n",
    "'''\n",
    "init=tf.global_variables_initializer()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    summary_writer = tf.summary.FileWriter('./logs/example',\n",
    "                                            graph=tf.get_default_graph())\n",
    "    for i in range(20000):\n",
    "        batch_x,batch_y=mnist.train.next_batch(50)\n",
    "        \n",
    "        if i%100==0:\n",
    "            result=sess.run(accuracy,feed_dict={x_input:batch_x,\n",
    "                                                y_input:batch_y,\n",
    "                                                keep_prob:1.0})\n",
    "            print(int(i/100),result)\n",
    "#         trainer.run(feed_dict={x_input:batch_x,\n",
    "#                                     y_input:batch_y,\n",
    "#                                     keep_prob:0.5})    \n",
    "        _,summary_=sess.run([trainer,merged_summary_op],feed_dict={x_input:batch_x,\n",
    "                                    y_input:batch_y,\n",
    "                                    keep_prob:0.5})\n",
    "        summary_writer.add_summary(summary_,i)\n",
    "\n",
    "    batch_test_x=mnist.test.images\n",
    "    batch_test_y=mnist.test.labels\n",
    "    all_test=sess.run(accuracy,feed_dict={x_input:batch_test_x,\n",
    "                                          y_input:batch_test_y,\n",
    "                                          keep_prob:1.0})\n",
    "    print(all_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
