From 2c91fa6eb51dd526e127ac94521c205f4dcfbebc Mon Sep 17 00:00:00 2001 From: Malar Kannan Date: Mon, 23 Oct 2017 11:05:35 +0530 Subject: [PATCH] updated to third week --- SecondSunday/mnist_phone_number.py | 39 +++-- ThirdSaturday/Mnist_tf.py | 71 +++++++++ ThirdSaturday/Mnist_tf_relu.py | 85 +++++++++++ ThirdSaturday/Notes.md | 27 ++++ ThirdSaturday/keras_mnist.py | 17 +++ ThirdSunday/FaceEyes.ipynb | 224 +++++++++++++++++++++++++++++ ThirdSunday/Faces.py | 121 ++++++++++++++++ ThirdSunday/Mnist.py | 73 ++++++++++ 8 files changed, 643 insertions(+), 14 deletions(-) create mode 100644 ThirdSaturday/Mnist_tf.py create mode 100644 ThirdSaturday/Mnist_tf_relu.py create mode 100644 ThirdSaturday/Notes.md create mode 100644 ThirdSaturday/keras_mnist.py create mode 100644 ThirdSunday/FaceEyes.ipynb create mode 100644 ThirdSunday/Faces.py create mode 100644 ThirdSunday/Mnist.py diff --git a/SecondSunday/mnist_phone_number.py b/SecondSunday/mnist_phone_number.py index deb4e7b..5c72f9a 100644 --- a/SecondSunday/mnist_phone_number.py +++ b/SecondSunday/mnist_phone_number.py @@ -1,16 +1,18 @@ %matplotlib inline from tensorflow.examples.tutorials.mnist import input_data import numpy as np +import pandas as pd +from sklearn.model_selection import train_test_split import matplotlib.pyplot as plt mnist = input_data.read_data_sets('./SecondSunday/mnist_data', one_hot=True) label_number = mnist.train.labels.argmax(axis=1) number_imgs = {str(i):mnist.train.images[np.argwhere(label_number == i).squeeze()] for i in range(10)} -DATA_COUNT = 100 -phone_number_digits = np.random.randint(10**9,10**10,size=(DATA_COUNT,10)) +DATA_COUNT = 10240 +# phone_number_digits = np.random.randint(10**9,10**10,size=(DATA_COUNT,10)) phone_number_digits = np.random.randint(10,size=(DATA_COUNT,10)) phone_number_digits.astype(str) -phone_number_strings = phone_number_digits.astype(str) +phone_number_strings = pd.DataFrame(phone_number_digits.astype(str).T).apply(lambda x: ''.join(x)).values def pick_img(num): rand_idx = np.random.randint(number_imgs[num].shape[0]) @@ -27,20 +29,29 @@ def create_phone_images(phone_array): return np.array(phone_number_images).reshape(-1,28*280) phone_number_imgs = create_phone_images(phone_number_strings) +train_imgs,test_imgs,train_digits,test_digits = train_test_split(phone_number_imgs,phone_number_digits) from keras.models import Sequential from keras.layers import Dense, Activation -model = Sequential([ - Dense(32, input_shape=(7840,)), - Activation('relu'), - Dense(10), - Activation('linear'), -]) +# model = Sequential([ +# Dense(32, input_shape=(7840,)), +# Activation('relu'), +# # Dense(24, input_shape=(32,)), +# # Activation('relu'), +# Dense(10), +# Activation('linear'), +# ]) -model.compile(optimizer='rmsprop', - loss='categorical_crossentropy', - metrics=['accuracy']) +# model.compile(optimizer='sgd', +# loss='mean_squared_error', +# metrics=['accuracy']) +# +# model.fit(train_imgs, train_digits, +# batch_size=128, +# epochs=100, +# validation_data=(test_imgs, test_digits)) -model.fit() -# plt.imshow(phone_number_imgs[np.random.randint(phone_number_imgs.shape[0])]) +# img_idx = np.random.randint(phone_number_imgs.shape[0]) +# print(phone_number_strings[img_idx]) +# plt.imshow(phone_number_imgs[img_idx].reshape(28,280)) diff --git a/ThirdSaturday/Mnist_tf.py b/ThirdSaturday/Mnist_tf.py new file mode 100644 index 0000000..e811791 --- /dev/null +++ b/ThirdSaturday/Mnist_tf.py @@ -0,0 +1,71 @@ + +# coding: utf-8 + +# In[40]: + +from tensorflow.examples.tutorials.mnist import input_data + + +# In[41]: + +mnist = input_data.read_data_sets('./mnist_data', one_hot=True) + + +# In[42]: + +xtrain,xtest = mnist.train,mnist.test +import tensorflow as tf +# mnist.train. + + +# In[43]: + +learning_rate = tf.constant(0.01,name='learning_rate') +xtrain.images.shape[1] + + +# In[44]: + +x = tf.placeholder(tf.float32, [None, 784]) +y = tf.placeholder(tf.float32, [None, 10]) + + +# In[45]: + +W1 = tf.Variable(tf.zeros([784, 512]),name='layer_1_weights') +b1 = tf.Variable(tf.zeros([512]),name='bias_1_weights') +W2 = tf.Variable(tf.zeros([512, 128]),name='layer_2_weights') +b2 = tf.Variable(tf.zeros([128]),name='bias_2_weights') +W_o = tf.Variable(tf.zeros([128, 10]),name='layer_output_weights') +b_o = tf.Variable(tf.zeros([10]),name='bias_output_weights') + + +# In[46]: + +layer_1 = tf.nn.relu(tf.add(tf.matmul(x,W1),b1)) +layer_2 = tf.nn.relu(tf.add(tf.matmul(layer_1,W2),b2)) +output_layer = tf.nn.softmax(tf.add(tf.matmul(layer_2,W_o),b_o)) + + +# In[47]: + +cross_entropy = tf.reduce_mean(-tf.reduce_sum(output_layer * tf.log(y), reduction_indices=[1])) + + +# In[48]: + +train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy) + + +# In[39]: + +with tf.Session() as s: + tf.global_variables_initializer() + [_,val] = s.run([train_step,cross_entropy],feed_dict={x:xtrain.images,y:xtrain.labels}) + + + +# In[ ]: + + + diff --git a/ThirdSaturday/Mnist_tf_relu.py b/ThirdSaturday/Mnist_tf_relu.py new file mode 100644 index 0000000..7480725 --- /dev/null +++ b/ThirdSaturday/Mnist_tf_relu.py @@ -0,0 +1,85 @@ + +# coding: utf-8 + +# In[1]: + +from tensorflow.examples.tutorials.mnist import input_data + + +# In[2]: + +mnist = input_data.read_data_sets('./mnist_data', one_hot=True) + + +# In[3]: + +xtrain,xtest = mnist.train,mnist.test +import tensorflow as tf +import math +# mnist.train. + + +# In[ ]: + + + + +# In[28]: + +learning_rate = tf.constant(0.01,name='learning_rate') +beta = tf.constant(0.01,name='regularization_beta') + + +# In[5]: + +x = tf.placeholder(tf.float32, [None, xtrain.images.shape[1]]) +y = tf.placeholder(tf.float32, [None, 10]) + + +# In[6]: + +W1 = tf.Variable(tf.random_normal([784, 512],stddev=2.0/28.0),name='layer_1_weights') +b1 = tf.Variable(tf.random_normal([512]),name='bias_1_weights') +W2 = tf.Variable(tf.random_normal([512, 128],stddev=2.0/math.sqrt(512)),name='layer_2_weights') +b2 = tf.Variable(tf.random_normal([128]),name='bias_2_weights') +W_o = tf.Variable(tf.random_normal([128, 10],stddev=2.0/math.sqrt(128)),name='layer_output_weights') +b_o = tf.Variable(tf.random_normal([10]),name='bias_output_weights') + + +# In[20]: + +layer_1 = tf.nn.relu(tf.add(tf.matmul(x,W1),b1)) +layer_2 = tf.nn.relu(tf.add(tf.matmul(layer_1,W2),b2)) +#y_ = tf.nn.softmax(tf.add(tf.matmul(layer_2,W_o),b_o))+1e-6 +y_ = tf.add(tf.matmul(layer_2,W_o),b_o) + + +# In[38]: + +#cross_entropy = tf.reduce_mean(-tf.reduce_sum(y * tf.log(y_))) +cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y,logits=y_)) +total_loss = cross_entropy+beta*(tf.nn.l2_loss(W1)+tf.nn.l2_loss(W2)+tf.nn.l2_loss(W_o)+tf.nn.l2_loss(b1)+tf.nn.l2_loss(b2)+tf.nn.l2_loss(b_o)) +correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(tf.nn.softmax(y_),1)) +accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) + + +# In[39]: + +train_step = tf.train.GradientDescentOptimizer(0.1).minimize(total_loss) + + +# In[40]: + +with tf.Session() as s: + tf.global_variables_initializer().run() + for i in range(20000): + batch_xs, batch_ys = xtrain.next_batch(100) + [_] = s.run([train_step],feed_dict={x:batch_xs,y:batch_ys}) + if i%1000 == 0: + print(s.run(accuracy, feed_dict={x: xtest.images, y: xtest.labels})) + + +# In[ ]: + + + diff --git a/ThirdSaturday/Notes.md b/ThirdSaturday/Notes.md new file mode 100644 index 0000000..1904b73 --- /dev/null +++ b/ThirdSaturday/Notes.md @@ -0,0 +1,27 @@ +Parameters: + variables that are learnt by the model through training. + + +HyperParameters: + variables that are empirical and have to be assigned manually. + + +Protocol: + Train,Test,Validation/Dev Set + update HyperParameters and try training with the devset accuracy. + pick the best params. + Depending on the datasize and the nature of the problem(no of classes to be classified to) decide the test datasize + + Error rate : (Bayes Error rate) lower possible error rate for any classifier of a random outcome. + (accuracy of the model shouldn't be more than this, + +Regularization: + if it is it means the model is overfitting to the training datas) + if the model is overfitting, use regularization to control it. + It is a technique to limit the expressiveness of the model. + eg. + 1. L2 regularizer -> Loss' = Loss + lambda*Sum(wi^2) // lambda is the regularization param + makes |wi| =~= 0. + controls the degree of non-linearity of the model, without having to redesign the model + 2. Dropout regularizer -> switching off some neurons + forcing the model learn from other features(neurons) diff --git a/ThirdSaturday/keras_mnist.py b/ThirdSaturday/keras_mnist.py new file mode 100644 index 0000000..ab7aa73 --- /dev/null +++ b/ThirdSaturday/keras_mnist.py @@ -0,0 +1,17 @@ +from keras.layers import Dense, Activation +from keras.optimizers import RMSprop +from keras.models import Sequential +from keras import losses + +from keras.models import Sequential +model = Sequential([Dense(units=64, input_dim=784), + Activation('relu'), + Dense(units=10), + Activation('softmax')]) + +model = Sequential([Dense(units=64, input_dim=784), + Activation('relu'), + Dense(units=10), + Activation('softmax')]) +model.compile(optimizer=RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0), loss=losses.,metrics=['accuracy']) +model.fit(xtrain.images,xtrain.labels,batch_size=10,epochs=10,validation_data=(xtest.images,xtest.labes)) diff --git a/ThirdSunday/FaceEyes.ipynb b/ThirdSunday/FaceEyes.ipynb new file mode 100644 index 0000000..0b02402 --- /dev/null +++ b/ThirdSunday/FaceEyes.ipynb @@ -0,0 +1,224 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 65, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import tensorflow as tf\n", + "import pandas as pd\n", + "import numpy as np\n", + "import math\n", + "import matplotlib.pyplot as plt\n", + "import msgpack as msg\n", + "import msgpack_numpy as m\n", + "from sklearn.model_selection import train_test_split\n", + "from skimage.transform import resize\n", + "# from sklearn.color import rgb2gray\n", + "m.patch()" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "def load_face_files():\n", + " all_data = [msg.load(open('./face_images/face_images{}.bin'.format(i),'rb')) for i in range(1,6)]\n", + " images = np.vstack([i[b'images'] for i in all_data])\n", + " gray_images = np.dot(images,np.array([0.2125,0.7154,0.0721]))/255.0\n", + "# print(gray_images.shape)\n", + "# scaled_gray_images = resize(,(32,32))/255.0\n", + "# import pdb;pdb.set_trace()\n", + " coords = np.vstack([i[b'co-ords'] for i in all_data])\n", + " coords_norm = coords/255.0\n", + " return gray_images,coords_norm\n", + "\n", + "images,coords = load_face_files()\n", + "\n", + "def plot_image(idx,n=1):\n", + " im = images[idx]\n", + " part_coords = np.split(coords[idx],3)\n", + " plt.figure(figsize=(16,16),dpi=80)\n", + " plt.subplot(n,4,1)\n", + " plt.imshow(im,'gray')\n", + " for i in range(2,5):\n", + " [x,y,w,h] = part_coords[i-2]#[:4]\n", + " # print([x,y,w,h],all([i<0 for i in [x,y,w,h]]))\n", + " if not all([j<0 for j in [x,y,w,h]]):\n", + " plt.subplot(n,4,i)\n", + " plt.imshow(im[y:y+h,x:x+w],'gray')\n", + "\n", + "\n", + "def get_head_images(c):\n", + " h_idx = []\n", + " for (idx,i) in enumerate(c):\n", + " head_coords = np.split(i,3)[0]\n", + " if not any([j<0 for j in head_coords]):\n", + " h_idx.append(idx)\n", + " return h_idx\n", + "\n", + "# plot_image(958)\n", + "\n", + "head_idxs = get_head_images(coords)\n", + "head_images = images[head_idxs].reshape(-1,images.shape[1]*images.shape[2])\n", + "head_coords = coords[head_idxs,:4].astype(np.float32)\n", + "tr_head_imgs,te_head_imgs,tr_head_crds,te_head_crds = train_test_split(head_images,head_coords,test_size=0.33)" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true, + "scrolled": true + }, + "outputs": [], + "source": [ + "def create_model(input_dim,output_dim):\n", + " g = tf.Graph()\n", + " with g.as_default():\n", + " learning_rate = tf.constant(0.1,name='learning_rate')\n", + " beta = tf.constant(0.00001,name='regularization_beta')\n", + " error_upper_bound = tf.constant(1.1,name='upper_bound')\n", + " error_lower_bound = tf.constant(0.9,name='lower_bound')\n", + " x = tf.placeholder(tf.float32, [None,input_dim])\n", + " y = tf.placeholder(tf.float32, [None,output_dim])\n", + " W1 = tf.Variable(tf.random_normal([input_dim, 512],stddev=2.0/math.sqrt(input_dim)),name='layer_1_weights')\n", + " b1 = tf.Variable(tf.random_normal([512]),name='bias_1_weights')\n", + " W2 = tf.Variable(tf.random_normal([512, 128],stddev=2.0/math.sqrt(512)),name='layer_2_weights')\n", + " b2 = tf.Variable(tf.random_normal([128]),name='bias_2_weights')\n", + " W_o = tf.Variable(tf.random_normal([128, output_dim],stddev=2.0/math.sqrt(128)),name='layer_output_weights')\n", + " b_o = tf.Variable(tf.random_normal([output_dim]),name='bias_output_weights')\n", + " layer_1 = tf.nn.relu(tf.add(tf.matmul(x,W1),b1))\n", + " layer_2 = tf.nn.relu(tf.add(tf.matmul(layer_1,W2),b2))\n", + " y_ = tf.nn.sigmoid(tf.add(tf.matmul(layer_2,W_o),b_o))\n", + " distance = tf.losses.mean_squared_error(labels = y,predictions = y_)\n", + " regularized_loss = distance+beta*(tf.nn.l2_loss(W1)+tf.nn.l2_loss(W2)+tf.nn.l2_loss(W_o)+tf.nn.l2_loss(b1)+tf.nn.l2_loss(b2)+tf.nn.l2_loss(b_o))\n", + " ratio = tf.div(y,y_)\n", + " accuracy = distance#tf.reduce_mean(tf.cast((ratio < error_upper_bound) & (ratio > error_lower_bound), tf.float32))\n", + " tf.summary.scalar('distance', distance)\n", + " tf.summary.histogram('Weights1', W1)\n", + " tf.summary.histogram('Bias1', b1)\n", + " tf.summary.histogram('Weights2', W2)\n", + " tf.summary.histogram('Bias2', b2)\n", + " tf.summary.histogram('Weights_output', W_o)\n", + " tf.summary.histogram('Bias_output', b_o)\n", + " merged_summary = tf.summary.merge_all()\n", + " train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(regularized_loss)\n", + " return (g,x,y,y_,train_step,accuracy,merged_summary)\n", + "\n", + "(g,x,y,y_,train_step,accuracy,merged_summary) = create_model(tr_head_imgs.shape[1],tr_head_crds.shape[1])" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "distance on validation set 0.0457288585603\n", + "distance on validation set 0.0384670570493\n", + "distance on validation set 0.0463402196765\n", + "distance on validation set 0.0418722033501\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"distance on validation set {}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0macc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m#,'saved to ',save_path)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtrain_step\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maccuracy\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmerged_summary\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(g, x, y, y_, train_step, accuracy, merged_summary)\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m20000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0mbatch_xs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbatch_ys\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtr_head_imgs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtr_head_crds\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m128\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m \u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtrain_step\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mbatch_xs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mbatch_ys\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 15\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0;36m100\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0mt_batch_xs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mt_batch_ys\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtr_head_imgs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtr_head_crds\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 765\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 766\u001b[0m result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 767\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 768\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 769\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 963\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 964\u001b[0m results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 965\u001b[0;31m feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m 966\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 967\u001b[0m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1013\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1014\u001b[0m return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1015\u001b[0;31m target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m 1016\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1017\u001b[0m return self._do_call(_prun_fn, self._session, handle, feed_dict,\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m 1020\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1021\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1022\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1023\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1024\u001b[0m \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m 1002\u001b[0m return tf_session.TF_Run(session, options,\n\u001b[1;32m 1003\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1004\u001b[0;31m status, run_metadata)\n\u001b[0m\u001b[1;32m 1005\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1006\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "def batch_data(data_x,data_y,size=128):\n", + " batch_idxs = np.random.randint(0,data_x.shape[0],size=size)\n", + " return (data_x[batch_idxs],data_y[batch_idxs])\n", + "\n", + "def train_model(g,x,y,y_,train_step,accuracy,merged_summary):\n", + " with g.as_default():\n", + " with tf.Session() as s:\n", + " train_writer = tf.summary.FileWriter('./tensor_log',s.graph)\n", + " tf.global_variables_initializer().run()\n", + "# saver = tf.train.Saver()\n", + " # saver.restore(s, \"/tmp/model.ckpt\")\n", + " for i in range(20000):\n", + " batch_xs,batch_ys = batch_data(tr_head_imgs,tr_head_crds,128)\n", + " s.run([train_step],feed_dict={x:batch_xs,y:batch_ys})\n", + " if i%100 == 0:\n", + " t_batch_xs,t_batch_ys = batch_data(tr_head_imgs,tr_head_crds,32)\n", + " [acc]= s.run([accuracy], feed_dict={x: t_batch_xs,y: t_batch_ys})\n", + " # train_writer.add_summary(summary,i)\n", + "# save_path = saver.save(s, \"/tmp/model.ckpt\")\n", + "# print(y_vals,t_batch_ys)\n", + " print(\"distance on validation set {}\".format(acc))#,'saved to ',save_path)\n", + "\n", + "train_model(g,x,y,y_,train_step,accuracy,merged_summary)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "hide_input": false, + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/ThirdSunday/Faces.py b/ThirdSunday/Faces.py new file mode 100644 index 0000000..70f893a --- /dev/null +++ b/ThirdSunday/Faces.py @@ -0,0 +1,121 @@ +#%matplotlib inline +import tensorflow as tf +import pandas as pd +import numpy as np +import math +import matplotlib.pyplot as plt +import msgpack as msg +import msgpack_numpy as m +from skimage.transform import resize +from sklearn.model_selection import train_test_split +# from sklearn.color import rgb2gray +m.patch() + + + +def load_face_files(): + all_data = [msg.load(open('./face_images/face_images{}.bin'.format(i),'rb')) for i in range(1,6)] + images = np.vstack([i[b'images'] for i in all_data]) + gray_images = np.dot(images,np.array([0.2125,0.7154,0.0721])) + coords = np.vstack([i[b'co-ords'] for i in all_data]) + return gray_images,coords + +images,coords = load_face_files() + +def plot_image(idx,n=1): + im = images[idx] + part_coords = np.split(coords[idx],3) + plt.figure(figsize=(16,16),dpi=80) + plt.subplot(n,4,1) + plt.imshow(im,'gray') + for i in range(2,5): + [x,y,w,h] = part_coords[i-2]#[:4] + # print([x,y,w,h],all([i<0 for i in [x,y,w,h]])) + if not all([j<0 for j in [x,y,w,h]]): + plt.subplot(n,4,i) + plt.imshow(im[y:y+h,x:x+w],'gray') + + +def get_head_images(c): + h_idx = [] + for (idx,i) in enumerate(c): + head_coords = np.split(i,3)[0] + if not any([j<0 for j in head_coords]): + h_idx.append(idx) + return h_idx + +# plot_image(958) + +head_idxs = get_head_images(coords) +head_images = images[head_idxs].reshape(-1,images.shape[1]*images.shape[2]) +head_coords = coords[head_idxs,:4].astype(np.float32) +tr_head_imgs,te_head_imgs,tr_head_crds,te_head_crds = train_test_split(head_images,head_coords,test_size=0.33) + + +def create_model(input_dim,output_dim): + g = tf.Graph() + with g.as_default(): + learning_rate = tf.constant(0.01,name='learning_rate') + beta = tf.constant(0.001,name='regularization_beta') + error_upper_bound = tf.constant(1.1,name='upper_bound') + error_lower_bound = tf.constant(0.9,name='lower_bound') + x = tf.placeholder(tf.float32, [None,input_dim]) + y = tf.placeholder(tf.float32, [None,output_dim]) + W1 = tf.Variable(tf.random_normal([input_dim, 512],stddev=2.0/math.sqrt(input_dim)),name='layer_1_weights') + b1 = tf.Variable(tf.random_normal([512]),name='bias_1_weights') + W2 = tf.Variable(tf.random_normal([512, 128],stddev=2.0/math.sqrt(512)),name='layer_2_weights') + b2 = tf.Variable(tf.random_normal([128]),name='bias_2_weights') + W_o = tf.Variable(tf.random_normal([128, output_dim],stddev=2.0/math.sqrt(128)),name='layer_output_weights') + b_o = tf.Variable(tf.random_normal([output_dim]),name='bias_output_weights') + layer_1 = tf.nn.relu(tf.add(tf.matmul(x,W1),b1)) + layer_2 = tf.nn.sigmoid(tf.add(tf.matmul(layer_1,W2),b2)) + #y_ = tf.nn.softmax(tf.add(tf.matmul(layer_2,W_o),b_o))+1e-6 + y_ = tf.add(tf.matmul(layer_2,W_o),b_o)#tf.nn.relu() + + #cross_entropy = tf.reduce_mean(-tf.reduce_sum(y * tf.log(y_))) + # cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y,logits=y_)) + # distance = tf.reduce_sum(tf.square(tf.subtract(y,y_))) + distance = tf.losses.mean_squared_error(labels = y,predictions = y_) + regularized_loss = distance+beta*(tf.nn.l2_loss(W1)+tf.nn.l2_loss(W2)+tf.nn.l2_loss(W_o)+tf.nn.l2_loss(b1)+tf.nn.l2_loss(b2)+tf.nn.l2_loss(b_o)) + # correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(tf.nn.softmax(y_),1)) + ratio = tf.div(y,y_) + accuracy = tf.reduce_mean(tf.cast((ratio < error_upper_bound) & (ratio > error_lower_bound), tf.float32)) + tf.summary.scalar('distance', distance) + tf.summary.histogram('Weights1', W1) + tf.summary.histogram('Bias1', b1) + tf.summary.histogram('Weights2', W2) + tf.summary.histogram('Bias2', b2) + tf.summary.histogram('Weights_output', W_o) + tf.summary.histogram('Bias_output', b_o) + merged_summary = tf.summary.merge_all() + train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(regularized_loss) + return (g,x,y,y_,train_step,accuracy,merged_summary) + +(g,x,y,y_,train_step,accuracy,merged_summary) = create_model(tr_head_imgs.shape[1],tr_head_crds.shape[1]) + +def batch_data(data_x,data_y,size=128): + batch_idxs = np.random.randint(0,data_x.shape[0],size=size) + return (data_x[batch_idxs],data_y[batch_idxs]) + +def train_model(g,x,y,y_,train_step,accuracy,merged_summary): + with g.as_default(): + with tf.Session() as s: + train_writer = tf.summary.FileWriter('./tensor_log',s.graph) + tf.global_variables_initializer().run() + saver = tf.train.Saver() + # saver.restore(s, "/tmp/model.ckpt") + for i in range(20000): + batch_xs,batch_ys = batch_data(tr_head_imgs,tr_head_crds,10) + s.run([train_step],feed_dict={x:batch_xs,y:batch_ys}) + if i%100 == 0: + t_batch_xs,t_batch_ys = batch_data(tr_head_imgs,tr_head_crds,5) + [summary,acc,y_vals]= s.run([merged_summary,accuracy,y_], feed_dict={x: t_batch_xs,y: t_batch_ys}) + train_writer.add_summary(summary,i) + save_path = saver.save(s, "/tmp/model.ckpt") + print(y_vals,t_batch_ys) + print("Accuracy on validation set {}".format(acc))#,'saved to ',save_path) + + +bb = np.array([[100,200,300,400], [100,200,300,400], [100,200,300,400], [100,200,300,400]]) +bb[:,1] +train_model(g,x,y,y_,train_step,accuracy,merged_summary) diff --git a/ThirdSunday/Mnist.py b/ThirdSunday/Mnist.py new file mode 100644 index 0000000..6613fbf --- /dev/null +++ b/ThirdSunday/Mnist.py @@ -0,0 +1,73 @@ + +# coding: utf-8 + +# In[1]: + +from tensorflow.examples.tutorials.mnist import input_data +import tensorflow as tf +import math + + +# In[2]: + +mnist = input_data.read_data_sets('./mnist_data', one_hot=True) + + +# In[3]: + +learning_rate = tf.constant(0.01,name='learning_rate') +beta = tf.constant(0.001,name='regularization_beta') +x = tf.placeholder(tf.float32, [None, mnist.train.images.shape[1]]) +y = tf.placeholder(tf.float32, [None, 10]) +W1 = tf.Variable(tf.random_normal([784, 512],stddev=2.0/28.0),name='layer_1_weights') +b1 = tf.Variable(tf.random_normal([512]),name='bias_1_weights') +W2 = tf.Variable(tf.random_normal([512, 128],stddev=2.0/math.sqrt(512)),name='layer_2_weights') +b2 = tf.Variable(tf.random_normal([128]),name='bias_2_weights') +W_o = tf.Variable(tf.random_normal([128, 10],stddev=2.0/math.sqrt(128)),name='layer_output_weights') +b_o = tf.Variable(tf.random_normal([10]),name='bias_output_weights') +layer_1 = tf.nn.relu(tf.add(tf.matmul(x,W1),b1)) +layer_2 = tf.nn.relu(tf.add(tf.matmul(layer_1,W2),b2)) +#y_ = tf.nn.softmax(tf.add(tf.matmul(layer_2,W_o),b_o))+1e-6 +y_ = tf.add(tf.matmul(layer_2,W_o),b_o) + + +# In[4]: + +#cross_entropy = tf.reduce_mean(-tf.reduce_sum(y * tf.log(y_))) +cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y,logits=y_)) +total_loss = cross_entropy+beta*(tf.nn.l2_loss(W1)+tf.nn.l2_loss(W2)+tf.nn.l2_loss(W_o)+tf.nn.l2_loss(b1)+tf.nn.l2_loss(b2)+tf.nn.l2_loss(b_o)) +correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(tf.nn.softmax(y_),1)) +accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) +tf.summary.scalar('cross_entropy', cross_entropy) +tf.summary.histogram('Weights1', W1) +tf.summary.histogram('Bias1', b1) +tf.summary.histogram('Weights2', W2) +tf.summary.histogram('Bias2', b2) +tf.summary.histogram('Weights_output', W_o) +tf.summary.histogram('Bias_output', b_o) +train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss) +#saver = tf.train.Saver() + + +# In[6]: + +with tf.Session() as s: +# merged = tf.summary.merge_all() +# train_writer = tf.summary.FileWriter('./train',s.graph) + tf.global_variables_initializer().run() +# saver.restore(s, "/tmp/model.ckpt") +# print("Model restored.") + for i in range(20000): + batch_xs, batch_ys = mnist.train.next_batch(128) + s.run([train_step],feed_dict={x:batch_xs,y:batch_ys}) + if i%1000 == 0: + [acc]= s.run([accuracy], feed_dict={x: mnist.test.images, y: mnist.test.labels}) + train_writer.add_summary(summary,i) +# save_path = saver.save(s, "/tmp/model.ckpt") + print("Accuracy on validation set {}".format(acc))#,'saved to ',save_path) + + +# In[ ]: + + +