diff --git a/Siamese.ipynb b/Siamese.ipynb new file mode 100644 index 0000000..02ea0ee --- /dev/null +++ b/Siamese.ipynb @@ -0,0 +1,446 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAC7CAYAAAB1qmWGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEYJJREFUeJzt3XuQ1fV5x/HPw7KwKAgoihvEQgze\nEzHuoFXTghqrTuKljlYbHYxk0KBTNGkStBpjJipN1ZjUxIqVQhqrpPFujC2DF3Q04EKRqwJVrEsQ\nVDRQVFh2n/6xh87q97twds/vXH5f3q8ZZ895zvec3/M7++zDz9/ta+4uAED+9ap2AgCAbNDQASAR\nNHQASAQNHQASQUMHgETQ0AEgETR0AEgEDR0AElFSQzez08zsNTNbbWZTskoKqDZqG3lkPb1S1Mzq\nJK2U9GVJLZJelnShuy/PLj2g8qht5FXvEt47RtJqd39dkszsAUlnSeqy6PtYX2/QniUsEujax9qi\nbb7VMvgoahs1pdjaLqWhD5P0VqfnLZKO3dkbGrSnjrWTS1gk0LV5Pierj6K2UVOKre1SGnpRzGyi\npImS1KA9yr04oGKobdSaUg6KrpU0vNPzAwqxT3D3ae7e5O5N9epbwuKAiqG2kUulNPSXJY0ys5Fm\n1kfSBZIeyyYtoKqobeRSj3e5uPt2M7tS0n9IqpM03d2XZZYZUCXUNvKqpH3o7v6kpCczygWoGdQ2\n8ogrRQEgETR0AEgEDR0AEkFDB4BE0NABIBE0dABIBA0dABJBQweARNDQASARNHQASAQNHQASQUMH\ngETQ0AEgETR0AEhE2aegQ6hu0MAg9tqdnw1ir4775+j7r9twTBBb8rWDo2Pblq/sZnbArn381THR\neL/fLQxi3nR4EHvjzPiE2l86aUkQe/7pzxedV+NLbdF4w+Pzi/6MPGMLHQASQUMHgETQ0AEgETR0\nAEhESQdFzWyNpM2S2iRtd/emLJJKXfvIA4LYkrF3B7FWj7//R/stCGJHnXN8dOxwDor2yO5Y23VD\n9onG22b1C2IPjLo9OnZ9W30QG9jr2SB2YO89ik9s/Nyih2646MNo/A8/6xPELrt5cnTsPve8VPTy\nak0WZ7mMc/d3M/gcoNZQ28gVdrkAQCJKbegu6T/NbIGZTcwiIaBGUNvInVJ3uZzo7mvNbD9Js83s\nVXf/xA6vwh/DRElqUDf2mwHVRW0jd0raQnf3tYWfGyQ9LCm4fMzdp7l7k7s31atvKYsDKobaRh71\neAvdzPaU1MvdNxcenyrph5llloDew8OzWSRp5LTVFc4E3bG71vbKnx4Yjb926L2RaPz/SParC2O/\n+CC8LcXCzfFltWwZ1GV+n1Zn7UHst4c8XnRes677h+jYy1dcGcR6vbCo6LyqqZRdLkMlPWxmOz7n\n39z9qUyyAqqL2kYu9bihu/vrko7KMBegJlDbyCtOWwSARNDQASAR3A89I//z/fDS+2NOWx4d++PG\n5zNffv/j34nG37o+zGvI4u3Rsf0e3T3uGQ3J/zTcozTr+PD2Ex3CNvHUR/GDolO/Mz6IDVgWudj2\nnY3R9/d6/60ucgh5r/BI58G3TYqOXX7+Pwaxg+r7R8d+dN2mIDbwkqHRsdvfXr+zFCuOLXQASAQN\nHQASQUMHgETQ0AEgETR0AEgEZ7lkZPFl4VH0Vo/PQF4Ozx51X/yFyOUxD29pjA6dvvnsINb76XAy\nDeRf68BwwofRfeLtoF3hTCvf+ZdLo2OHP/xiECvbX0F7+Mmfu/r30aGH9Qkv51981k+jY5/7/G+C\n2AmnxM+eGfgrznIBAJQBDR0AEkFDB4BE0NABIBEcFO2m+mfjBxTrLXLD5TL5r23hfaDXtO4bHXvO\nnuEl1uf33xAde/6/TgtiXxl2TDezQx60NVjRY7/w4iVB7MCbwoOftWzUFfOC2BOnxP+Wz+v/XhD7\n4Mwt0bEDf1VaXlljCx0AEkFDB4BE0NABIBE0dABIxC4buplNN7MNZra0U2xvM5ttZqsKPweXN00g\ne9Q2UlPMWS4zJN0p6ZedYlMkzXH3qWY2pfD8e9mnV10fnT0miH298d+jY2OX+Zd66f+Rcy6Pxved\n0zeI9f1jfFnXjA3/zV5y3s+KzqHlmnCCDEk64JZ8neXQhRnaTWv7kGuWFT22bsGAMmZSPX/3cnir\nC0k6b9y9QeyKI+ZGxz6h2vr3fpdb6O4+V9Knz307S9LMwuOZkuLfDFDDqG2kpqf70Ie6+7rC47cl\nxednAvKH2kZulXxQ1N1dityOrcDMJppZs5k1t2prqYsDKobaRt70tKGvN7NGSSr8jF96KMndp7l7\nk7s31Svc9wvUGGobudXTS/8fkzRe0tTCz0czy6gK6o44JBr/0e3hpfBNfbZ19SlFLy92P/Lrnjk3\niB323Vej72/bFM5K3pVDVh0cxOaf2RAdO6bvx0Hsd9/8cXTsqQ3fDWIjbo7fO9235mrrNana7vWF\nQ6PxsYNmB7GVreHvX5KGLG7NNKdaMfi5+N+BxlU2jywVc9ri/ZJeknSImbWY2QR1FPuXzWyVpFMK\nz4FcobaRml1uobv7hV28dHLGuQAVRW0jNVwpCgCJoKEDQCJo6ACQCCa4kNTexWznXZ/RUpxL3zwt\nGt/8V/2C2MEt84NYFrOlty1fGcQmzYjfUqD5sjuCWGNdmKskLZwQjj33ofHRsf7Kip2liDJaNX5Q\nNH5B/3eC2ImLL46O3evJlzPNCeXDFjoAJIKGDgCJoKEDQCJo6ACQCA6KZuTa9U1BbNM39omObWtZ\nVe50dmrEg+9G49effVwQm7o/B8Ty7OrTfxuNxy7z7/PzeL1K/51hRignttABIBE0dABIBA0dABJB\nQweARHBQdCfqrfh7nC/+Ymxim+oe/OySWTTcu1d7EOvOd/CHG+Px/ZmVs+bc/d6fBbGGJ8KrlZEv\nbKEDQCJo6ACQCBo6ACSChg4AiShmTtHpZrbBzJZ2iv3AzNaa2aLCf2eUN00ge9Q2UlPMWS4zJN0p\n6Zefiv/E3W/NPKMqeO2be0TjrZ7FHclrz5q/jF/i/Zt9w7McWj1+lkvsu/nMDfHlhefO1IwZSqi2\n6wYNDGIDerVUIRNUyy630N19rqSNFcgFqChqG6kpZR/6lWa2uPC/rYMzywioPmobudTThn6XpIMk\njZa0TtJtXQ00s4lm1mxmza3a2sPFARVDbSO3etTQ3X29u7e5e7ukeySN2cnYae7e5O5N9erb0zyB\niqC2kWc9uvTfzBrdfV3h6TmSlu5sfK277kuPVzuFkvUefkA0vvmYzwSxf/r6L0pe3vytDUHMtm0v\n+XOrLc+13TLhiCD2tQHPRMcu3DKizNnUvq1n/LHosR+29yljJtnZZUM3s/sljZU0xMxaJN0gaayZ\njZbkktZIuqyMOQJlQW0jNbts6O5+YSR8bxlyASqK2kZquFIUABJBQweARNDQASARTHCRiOU37h+N\nLzv1zpI+98H/HRKN3/W35wWxhhVMkIDatP2kY4LYA0d39bcRnoL68N+fHB05UL8vJa3MsYUOAImg\noQNAImjoAJAIGjoAJIKDojlU/2xjELul8cGyLGvG2uOj8YbHOQCK2hM7+ClJGydvCWKH1sfvvzNp\n7QlBbNCshdGx3o3cKoEtdABIBA0dABJBQweARNDQASARNHQASARnuUiqs/i89PUWn/E+ZtNfH1f0\n2Bt/GN6hdVy/j4t+fyyvVm/rYnTx6xDjJ60t6f2onL3WhDWwZvuHVcikMqx32L4+uHpzdGzzFx8I\nYrM/6hcdu/L6cKKQPq3N3cyuOthCB4BE0NABIBE0dABIxC4bupkNN7NnzGy5mS0zs8mF+N5mNtvM\nVhV+Di5/ukB2qG2kxtx3fvGqmTVKanT3hWY2QNICSWdLukTSRnefamZTJA129+/t7LP2sr39WIvf\nV7ia3rwxfnn7wgl3FP0Z3TtQWZpyLevIOZcHsVHj45c816J5PkebfKMVO353qO0zl78Xjbd7uC33\n1F8cGR27vaVyB8bbTxwdxN6YFB977mGLgtjN+xVfr+OuCOtdkvo9Unu3tSi2tne5he7u69x9YeHx\nZkkrJA2TdJakmYVhM9XxhwDkBrWN1HRrH7qZjZB0tKR5koa6+7rCS29LGpppZkAFUdtIQdEN3cz6\nS3pQ0lXuvqnza96x3ya678bMJppZs5k1t2prSckC5UBtIxVFNXQzq1dHwd/n7g8VwusL+yB37Ivc\nEHuvu09z9yZ3b6qPzNUHVBO1jZQUc5aLSbpX0gp3v73TS49JGl94PF7So9mnB5QPtY3UFHPp/wmS\nLpa0xMx2HFa+VtJUSb82swmS3pR0fnlSLL/Pzno3Gp9/UUMQG9O3+Ev0K2n+1jBXSZr29p8Hsfcn\n7R8de+gbq4NYec7TqRnJ13ZXJg16I4itf2Kv6NjmjQeWO53/N3XktCA2uk/xdyhZsC1esRfPnxDE\nDnr61ejYPNf8Lr8pd39BUleny9TeeVpAkahtpIYrRQEgETR0AEgEDR0AEsH90CW1LV8ZjX//W98I\nYm99NX7v9JWn351pTt01aXr8MubhN70Yib5f3mRQM2bc+pVofMPkuUHsxn1fiX9IV/GyCFvS9i4O\nU76yLYxdNOtvomNHTnkpiOX54GdX2EIHgETQ0AEgETR0AEgEDR0AEkFDB4BE7HKCiyzV6iQAWdh0\n4XFBrP6S9dGxTx0xK4iduvSCINY+Y7/o+z1ybePgRfGJDNpWrIrGU9TdCS6ylLfarvvcyCA27pHF\n0bHfGly5Gjr0uUuDWJ8le0THHnBL7AyuNGU2wQUAIB9o6ACQCBo6ACSChg4AieCgKJLBQVGkioOi\nALCboaEDQCJo6ACQiGImiR5uZs+Y2XIzW2ZmkwvxH5jZWjNbVPjvjPKnC2SH2kZqirkf+nZJ33b3\nhWY2QNICM5tdeO0n7n5r+dIDyoraRlKKmSR6naR1hcebzWyFpGHlTgwoN2obqenWPnQzGyHpaEnz\nCqErzWyxmU03s8EZ5wZUDLWNFBTd0M2sv6QHJV3l7psk3SXpIEmj1bGVc1sX75toZs1m1tyqrRmk\nDGSL2kYqimroZlavjoK/z90fkiR3X+/ube7eLukeSWNi73X3ae7e5O5N9eqbVd5AJqhtpKSYs1xM\n0r2SVrj77Z3ijZ2GnSNpafbpAeVDbSM1xZzlcoKkiyUtMbNFhdi1ki40s9GSXNIaSZeVJUOgfKht\nJKWYs1xekBS7h8CT2acDVA61jdRwpSgAJIKGDgCJoKEDQCJo6ACQCBo6ACSChg4AiaChA0AiaOgA\nkAgaOgAkwty9cgsze0fSm4WnQyS9W7GFVw7rVT1/4u77VmPBnWo7D99TT6W6bnlYr6Jqu6IN/RML\nNmt296aqLLyMWK/dW8rfU6rrltJ6scsFABJBQweARFSzoU+r4rLLifXavaX8PaW6bsmsV9X2oQMA\nssUuFwBIRMUbupmdZmavmdlqM5tS6eVnqTAj/AYzW9optreZzTazVYWfuZsx3syGm9kzZrbczJaZ\n2eRCPPfrVk6p1DZ1nb9126GiDd3M6iT9XNLpkg5Xx1Rfh1cyh4zNkHTap2JTJM1x91GS5hSe5812\nSd9298MlHSfpisLvKYV1K4vEanuGqOtcqvQW+hhJq939dXffJukBSWdVOIfMuPtcSRs/FT5L0szC\n45mSzq5oUhlw93XuvrDweLOkFZKGKYF1K6Nkapu6zt+67VDphj5M0ludnrcUYikZ6u7rCo/fljS0\nmsmUysxGSDpa0jwltm4ZS722k/rdp1rXHBQtI+84hSi3pxGZWX9JD0q6yt03dX4t7+uGnsv77z7l\nuq50Q18raXin5wcUYilZb2aNklT4uaHK+fSImdWro+jvc/eHCuEk1q1MUq/tJH73qdd1pRv6y5JG\nmdlIM+sj6QJJj1U4h3J7TNL4wuPxkh6tYi49YmYm6V5JK9z99k4v5X7dyij12s797353qOuKX1hk\nZmdIukNSnaTp7n5TRRPIkJndL2msOu7Wtl7SDZIekfRrSQeq4+5757v7pw8w1TQzO1HS85KWSGov\nhK9Vx/7GXK9bOaVS29R1/tZtB64UBYBEcFAUABJBQweARNDQASARNHQASAQNHQASQUMHgETQ0AEg\nETR0AEjE/wGH11+/+f1hxAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "'''Train a Siamese MLP on pairs of digits from the MNIST dataset.\n", + "\n", + "It follows Hadsell-et-al.'06 [1] by computing the Euclidean distance on the\n", + "output of the shared network and by optimizing the contrastive loss (see paper\n", + "for mode details).\n", + "\n", + "[1] \"Dimensionality Reduction by Learning an Invariant Mapping\"\n", + " http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf\n", + "\n", + "Gets to 97.2% test accuracy after 20 epochs.\n", + "2 seconds per epoch on a Titan X Maxwell GPU\n", + "'''\n", + "from __future__ import absolute_import\n", + "from __future__ import print_function\n", + "import numpy as np\n", + "\n", + "import random\n", + "from keras.datasets import mnist\n", + "from keras.models import Model\n", + "from keras.layers import Dense, Dropout, Input, Lambda\n", + "from keras.optimizers import RMSprop\n", + "from keras import backend as K\n", + "\n", + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "\n", + "num_classes = 10\n", + "\n", + "\n", + "def euclidean_distance(vects):\n", + " x, y = vects\n", + " return K.sqrt(K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), K.epsilon()))\n", + "\n", + "\n", + "def eucl_dist_output_shape(shapes):\n", + " shape1, shape2 = shapes\n", + " return (shape1[0], 1)\n", + "\n", + "\n", + "def contrastive_loss(y_true, y_pred):\n", + " '''Contrastive loss from Hadsell-et-al.'06\n", + " http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf\n", + " '''\n", + " margin = 1\n", + " return K.mean(y_true * K.square(y_pred) +\n", + " (1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))\n", + "\n", + "\n", + "def create_pairs(x, digit_indices):\n", + " '''Positive and negative pair creation.\n", + " Alternates between positive and negative pairs.\n", + " '''\n", + " pairs = []\n", + " labels = []\n", + " n = min([len(digit_indices[d]) for d in range(num_classes)]) - 1\n", + " for d in range(num_classes):\n", + " for i in range(n):\n", + " z1, z2 = digit_indices[d][i], digit_indices[d][i + 1]\n", + " pairs += [[x[z1], x[z2]]]\n", + " inc = random.randrange(1, num_classes)\n", + " dn = (d + inc) % num_classes\n", + " z1, z2 = digit_indices[d][i], digit_indices[dn][i]\n", + " pairs += [[x[z1], x[z2]]]\n", + " labels += [1, 0]\n", + " return np.array(pairs), np.array(labels)\n", + "\n", + "\n", + "def create_base_network(input_dim):\n", + " '''Base network to be shared (eq. to feature extraction).\n", + " '''\n", + " input = Input(shape=(input_dim,))\n", + " x = Dense(128, activation='relu')(input)\n", + " x = Dropout(0.1)(x)\n", + " x = Dense(128, activation='relu')(x)\n", + " x = Dropout(0.1)(x)\n", + " x = Dense(128, activation='relu')(x)\n", + " return Model(input, x)\n", + "\n", + "\n", + "def compute_accuracy(y_true, y_pred):\n", + " '''Compute classification accuracy with a fixed threshold on distances.\n", + " '''\n", + " pred = y_pred.ravel() < 0.5\n", + " return np.mean(pred == y_true)\n", + "\n", + "\n", + "def accuracy(y_true, y_pred):\n", + " '''Compute classification accuracy with a fixed threshold on distances.\n", + " '''\n", + " return K.mean(K.equal(y_true, K.cast(y_pred < 0.5, y_true.dtype)))\n", + "\n", + "\n", + "# the data, shuffled and split between train and test sets\n", + "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", + "x_train = x_train.reshape(60000, 784)\n", + "x_test = x_test.reshape(10000, 784)\n", + "x_train = x_train.astype('float32')\n", + "x_test = x_test.astype('float32')\n", + "x_train /= 255\n", + "x_test /= 255\n", + "input_dim = 784\n", + "epochs = 20\n", + "\n", + "# create training+test positive and negative pairs\n", + "digit_indices = [np.where(y_train == i)[0] for i in range(num_classes)]\n", + "tr_pairs, tr_y = create_pairs(x_train, digit_indices)\n", + "\n", + "digit_indices = [np.where(y_test == i)[0] for i in range(num_classes)]\n", + "te_pairs, te_y = create_pairs(x_test, digit_indices)\n", + "def show_nth(n):\n", + " plt.subplot(1,2,1)\n", + " plt.imshow(te_pairs[n][0].reshape(28,28))\n", + " print(te_y[n])\n", + " plt.subplot(1,2,2)\n", + " plt.imshow(te_pairs[n][1].reshape(28,28))\n", + "show_nth(0)\n", + "# # network definition\n", + "# base_network = create_base_network(input_dim)\n", + "\n", + "# input_a = Input(shape=(input_dim,))\n", + "# input_b = Input(shape=(input_dim,))\n", + "\n", + "# # because we re-use the same instance `base_network`,\n", + "# # the weights of the network\n", + "# # will be shared across the two branches\n", + "# processed_a = base_network(input_a)\n", + "# processed_b = base_network(input_b)\n", + "\n", + "# distance = Lambda(euclidean_distance,\n", + "# output_shape=eucl_dist_output_shape)([processed_a, processed_b])\n", + "\n", + "# model = Model([input_a, input_b], distance)\n", + "\n", + "# # train\n", + "# rms = RMSprop()\n", + "# model.compile(loss=contrastive_loss, optimizer=rms, metrics=[accuracy])\n", + "# model.fit([tr_pairs[:, 0], tr_pairs[:, 1]], tr_y,\n", + "# batch_size=128,\n", + "# epochs=epochs,\n", + "# validation_data=([te_pairs[:, 0], te_pairs[:, 1]], te_y))\n", + "\n", + "# # compute final accuracy on training and test sets\n", + "# y_pred = model.predict([tr_pairs[:, 0], tr_pairs[:, 1]])\n", + "# tr_acc = compute_accuracy(tr_y, y_pred)\n", + "# y_pred = model.predict([te_pairs[:, 0], te_pairs[:, 1]])\n", + "# te_acc = compute_accuracy(te_y, y_pred)\n", + "\n", + "# print('* Accuracy on training set: %0.2f%%' % (100 * tr_acc))\n", + "# print('* Accuracy on test set: %0.2f%%' % (100 * te_acc))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAC7CAYAAAB1qmWGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEYpJREFUeJzt3X+QVfV5x/HPAyzLL2tB44oILk0U\nxEQXWaBGneCvDnEiaKsZmWrRJEOSqtFIrIxOJG3shBkTNf6IFQUhrcpgFcGOk0AorWZUZEXkt1H5\noTD8SkyVGkWWffrHXjor3++Fy95z7+758n7NOHvvc7/3nuewDw/H+z3ne8zdBQDIvy4dnQAAIBs0\ndABIBA0dABJBQweARNDQASARNHQASAQNHQASQUMHgESU1dDNbKyZvWlmb5vZlKySAjoatY08svZe\nKWpmXSX9TtJFkrZIWiZpgruvzS49oPqobeRVtzLeO0rS2+6+QZLMbI6k8ZKKFn13q/Ue6l3GJoHi\nPtFH+tT3WAYfRW2jUym1tstp6AMkvdfm+RZJow/2hh7qrdF2QRmbBIpb6ouz+ihqG51KqbVdTkMv\niZlNkjRJknqoV6U3B1QNtY3OppxJ0a2SBrZ5fmIh9hnuPt3dG929sUa1ZWwOqBpqG7lUTkNfJulk\nMxtsZt0lXSlpQTZpAR2K2kYutfsrF3dvNrPrJf1aUldJM919TWaZAR2E2kZelfUdurs/L+n5jHIB\nOg1qG3nElaIAkAgaOgAkgoYOAImgoQNAImjoAJAIGjoAJIKGDgCJoKEDQCJo6ACQCBo6ACSChg4A\niaChA0AiaOgAkAgaOgAkgoYOAImgoQNAImjoAJAIGjoAJIKGDgCJKOueoma2SdJuSfskNbt7YxZJ\npa7b8XVB7IOz64PY1os8+v6N46YHsb2+Lzr27BVXBrFd7/WNjh02bXsQa970bnRs6qht5FFZDb3g\nPHf/fQafA3Q21DZyha9cACAR5TZ0l7TQzF4zs0lZJAR0EtQ2cqfcr1zOcfetZnacpEVmtt7dX2g7\noPCXYZIk9VCvMjcHVA21jdwp6wjd3bcWfu6UNE/SqMiY6e7e6O6NNaotZ3NA1VDbyKN2H6GbWW9J\nXdx9d+HxX0n6p8wyyxmrDf9Cb/jHM6NjH7j80SD2lZ5/Knlbez38d7hFLdGxLzY8EQYb4p/bcMw3\ngtigK0pOKxnUNvKqnK9c6iTNM7P9n/OEu/8qk6yAjkVtI5fa3dDdfYOkMzLMBegUqG3kFactAkAi\naOgAkIgsrhSFpHdvGRHEVl3984ps69rNFwSxGSctKvtzV3x5ZhAbp5Flfy6OHPvGhCcCdLtjRxB7\nbsiC6PtrrGsQO5xlLY65vSY61jZtDWJ/uGRYdGy/Z1cHsZbdu6NjOxuO0AEgETR0AEgEDR0AEkFD\nB4BE0NABIBGc5XKY/Kz49SYzv3F/5ts6/bHvReODf7w8iA2957ro2PXjH8w0Jxx5Ysta7B4XXz9i\n6k/CM6Viy1rEF6qQ9kbu6XI4y1qc+cNromPPOD48dp1f/0B07Mg/vyGI1d3/UnRsZ8MROgAkgoYO\nAImgoQNAImjoAJAIJkUPIjYB6ne+Hx07InJ/g2ITP/P+97ggNvOacUGsfumr8bxawkuhh3z/jejY\nrz773SD243+ZHh3bWBt+7oWr45c8/+aLR0XjSM+eMV8KYv95b3xCMWbJx32C2B13hmvvS1LNnyKz\nokV8eFJ4PNq9yG0F/uEH4WTtBy3N0bF9tsWXGsgDjtABIBE0dABIBA0dABJBQweARByyoZvZTDPb\naWar28T6mdkiM3ur8LNvZdMEskdtIzWlnOUyS9IDkn7ZJjZF0mJ3n2ZmUwrPb80+vY61c2TvILZs\naDhbLsUX5v+g5dPo2Klzw4X5619++TCz+yzfsyee18KmIHbVr78THbvmkvDMhVv6vRMd+8iTE4PY\n4AnxM206sVk6Qms7ptiyFj956OGSP2PCOxcHsQ+nDgxifZeUV++SdPQXBgexhqfi9Xpq9/DYdej8\n70fHnvLvS8tLrAMd8gjd3V+QdOC5euMlzS48ni3p0ozzAiqO2kZq2vsdep27bys83i6pLqN8gI5G\nbSO3yp4UdXeXVPRqADObZGZNZta0V/GvBYDOiNpG3rS3oe8ws/6SVPi5s9hAd5/u7o3u3lijyOWU\nQOdCbSO32nvp/wJJEyVNK/ycn1lGnUiXC/8QxIqtzRxbx/naDeHl/JJU/8PyJ4TKccp340sK3H/O\naUHs5n7ro2P/dtiyIPaSupeXWOdwRNR2zB9v/zgajy1rcfH6v46O7fqDPwtjr4fr92fhf0aE34ZN\nPW5uye8fuDDLbDqHUk5bfFLSy5KGmNkWM/umWov9IjN7S9KFhedArlDbSM0hj9DdfUKRly7IOBeg\nqqhtpIYrRQEgETR0AEgEDR0AEsENLiR1G3BCND55yG/K+twNT50cjddpV1mfWykz518YxG6+Nn6W\nC/Jt45zTg9ia4Y9Fx25pDs9+6XJ7fIkbf31leYlFWG38lNAv3LQ2iHUpcox67eZwWqTns/GzvfKM\nI3QASAQNHQASQUMHgETQ0AEgEUyKSvrjOYOi8cv7lH7V96T3xgSxAUXWZo7fazxfvthzSxB79S/O\nj45t3rCpwtngcP3dsHBCsNiyFpubw8v59Ur2k59SfAL0zXvj67TPH/RgEIvvgbT5riFBrJfyu+55\nMRyhA0AiaOgAkAgaOgAkgoYOAIlgUlTSrjOt7M94Z9qpQazn9vSuRNvva73DteLvbjw+OrYPk6I4\nQNfTwklKSVp3w9FBbP0l4eRnMUs+7hONH/XSxiC2r+RPzQ+O0AEgETR0AEgEDR0AEkFDB4BElHJP\n0ZlmttPMVreJ/cjMtprZisJ/F1c2TSB71DZSU8pZLrMkPSDplwfE73H3n2aeUQfY1yt+wXCxtZVj\nUlxbWZJqrGs0vternEhlzFLitV3M0xsbgtgtx6yKjh1e+1EQO3flJ2Vtf1SvZ6Lx83qGn1vscv6Y\nyW9cHo2fuGPNYXxKfh2yY7n7C5Ler0IuQFVR20hNOd+hX29mKwv/2xq/fQmQT9Q2cqm9Df0hSZ+X\n1CBpm6SfFRtoZpPMrMnMmvZqTzs3B1QNtY3caldDd/cd7r7P3VskPSJp1EHGTnf3RndvrFH83oBA\nZ0FtI8/adem/mfV3922Fp5dJWn2w8Z3d6advisaLrQ99JNnr8QukU/2zSa22izn+qq1BbNyzl0XH\n/sfQ8L4AxSZQy3XurTcEsZYJ4TITkvRiwxNB7LhHemWeU54csqGb2ZOSxkg61sy2SJoqaYyZNUhy\nSZskfbuCOQIVQW0jNYds6O4+IRKeUYFcgKqitpEarhQFgETQ0AEgETR0AEgEN7hAu2xu/jSI9dwV\nxtA5tezeHQYviMQknX/Z3wexnSNKPxbsuy5cJ+Lox1+Jjt31r+H5/Osb5kTHzvigPoj1WrMtHCip\n+SD5pYQjdABIBA0dABJBQweARNDQASARTIri/33r0oUljx3/2C1BbNCSl7JMB51Er3lLg1j9vMps\na/35jwaxYstMPPjmV4LYCe+tzTynPOEIHQASQUMHgETQ0AEgETR0AEgEDR0AEsFZLpI+uuOEaLzp\nsfCO94218Rs+vPvUl4LYoCsqcxOAShnZc2MQe3WPRcfW3/VGEEvzlheohK6nDSnyymtBJLbMhCTV\n3dcjw4zSwBE6ACSChg4AiaChA0AiDtnQzWygmS0xs7VmtsbMbizE+5nZIjN7q/Czb+XTBbJDbSM1\npUyKNkua7O7LzewoSa+Z2SJJ10ha7O7TzGyKpCmSbq1cqpXT5b9fj8avu/f6ILbs1vujYxeNfiiI\nXXPe96Jjuy5ZfhjZZW/jnNOj8bN7hBNSX349dttNqd9Hv8s0pw6SfG13Vhumdi957BWvfysaP76D\n/x51Roc8Qnf3be6+vPB4t6R1kgZIGi9pdmHYbEmXVipJoBKobaTmsL5DN7N6ScMlLZVU5+77bw+y\nXVJdppkBVURtIwUlN3Qz6yPpaUk3ufuHbV9zd5cU3meq9X2TzKzJzJr2Kry9FNDRqG2koqSGbmY1\nai34x939mUJ4h5n1L7zeX9LO2Hvdfbq7N7p7Y41qs8gZyAy1jZSUcpaLSZohaZ27393mpQWSJhYe\nT5Q0P/v0gMqhtpGaUs5yOVvS1ZJWmdmKQuw2SdMkzTWzb0raLOnrlUmx4/T/r/eDWOP5V0XHNo38\ntyC2ZUz80uSTlpSX1+H46G9GB7G5o++Ljn15T3iU2e/OpC+vPmJru5r8rDOC2ILRvygyOqw3W8xZ\no6U6ZEN3999Kii/oIV2QbTpA9VDbSA1XigJAImjoAJAIGjoAJIL10A+iZeX6IDbg9vg6zvPm9Qti\nC665Kzp27LE3B7GTrwvvrF6MjTgtiO046+jo2Icn/zyIndo9/u/40OcmBbFTXnm15LyAmJ0jewex\nwd3ik+0tkVX1u30SvQwAERyhA0AiaOgAkAgaOgAkgoYOAImgoQNAIjjL5TDtW/NmND577HlB7OHp\n4Yy9JP3qa3cHsbnnjghic544P/r+RyeFN9kYXhvfVszYtZdH40Mf2h3ESv9UIO6TY8OzVGJns0jS\nve8PC2LHPPJy5jmliiN0AEgEDR0AEkFDB4BE0NABIBFMimakecOmIFY74XPRsd8ZfmMQq7l1exB7\n7Ybwsn1JGvrcdSXnNfiZcPKpdsnK6NiWvZ+W/LlAqa66tPQbAMycf2EQqxeToqXiCB0AEkFDB4BE\n0NABIBGl3CR6oJktMbO1ZrbGzG4sxH9kZlvNbEXhv4srny6QHWobqSllUrRZ0mR3X25mR0l6zcwW\nFV67x91/Wrn0gIqitpGUUm4SvU3StsLj3Wa2TtKASieWgn27dkXjNQsj8YVhaJxGRt9/isq76QS3\nC2hFbVfH0xsbgtgtx6zqgEzSd1jfoZtZvaThkvbfXud6M1tpZjPNrG/GuQFVQ20jBSU3dDPrI+lp\nSTe5+4eSHpL0eUkNaj3K+VmR900ysyYza9qrPRmkDGSL2kYqSmroZlaj1oJ/3N2fkSR33+Hu+9y9\nRdIjkkbF3uvu09290d0ba1SbVd5AJqhtpKSUs1xM0gxJ69z97jbx/m2GXSZpdfbpAZVDbSM1pZzl\ncrakqyWtMrMVhdhtkiaYWYNa59g2Sfp2RTIEKofargJf3C+I3Xbi6OjYuqZ9lU4naaWc5fJbSRZ5\n6fns0wGqh9pGarhSFAASQUMHgETQ0AEgETR0AEgEN7gAUFF1970UxFbfFx/bs8xlLY50HKEDQCJo\n6ACQCBo6ACSChg4AiTD36q2ObWa7JG0uPD1W0u+rtvHqYb86zknu/rmO2HCb2s7Dn1N7pbpvediv\nkmq7qg39Mxs2a3L3xg7ZeAWxX0e2lP+cUt23lPaLr1wAIBE0dABIREc29OkduO1KYr+ObCn/OaW6\nb8nsV4d9hw4AyBZfuQBAIqre0M1srJm9aWZvm9mUam8/S4U7wu80s9VtYv3MbJGZvVX4mbs7xpvZ\nQDNbYmZrzWyNmd1YiOd+3yopldqmrvO3b/tVtaGbWVdJD0r6qqRhar3V17Bq5pCxWZLGHhCbImmx\nu58saXHhed40S5rs7sMk/aWk6wq/pxT2rSISq+1Zoq5zqdpH6KMkve3uG9z9U0lzJI2vcg6ZcfcX\nJL1/QHi8pNmFx7MlXVrVpDLg7tvcfXnh8W5J6yQNUAL7VkHJ1DZ1nb9926/aDX2ApPfaPN9SiKWk\nzt23FR5vl1TXkcmUy8zqJQ2XtFSJ7VvGUq/tpH73qdY1k6IV5K2nEOX2NCIz6yPpaUk3ufuHbV/L\n+76h/fL+u0+5rqvd0LdKGtjm+YmFWEp2mFl/SSr83NnB+bSLmdWotegfd/dnCuEk9q1CUq/tJH73\nqdd1tRv6Mkknm9lgM+su6UpJC6qcQ6UtkDSx8HiipPkdmEu7mJlJmiFpnbvf3eal3O9bBaVe27n/\n3R8JdV31C4vM7GJJ90rqKmmmu/9zVRPIkJk9KWmMWldr2yFpqqRnJc2VNEitq+993d0PnGDq1Mzs\nHEkvSlolqaUQvk2t3zfmet8qKZXapq7zt2/7caUoACSCSVEASAQNHQASQUMHgETQ0AEgETR0AEgE\nDR0AEkFDB4BE0NABIBH/BzEpOveQwDIbAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "show_nth(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.0\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAAkCAYAAACZmsEQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAACeFJREFUeJztnFuMZEUZx39f1Tl9mZ5ZmGF1g4ou\nKDESHwwQJMQQEw23F9QHA8a4URJe5E0fMDzIg4mK8cWgJGtCIj5A8MG4CTG6Eg1PXtaE5WKyslyM\nrMiwl5mdnek+l6rPh6ruHYZdZ2d2prunt35J55yuPtX1r6p/fXVOnz4lqkoikUgkdj5m1AISiUQi\nsTWkgJ5IJBITQgroiUQiMSGkgJ5IJBITQgroiUQiMSGkgJ5IJBITwkUFdBG5Q0SOiMhREXlwq0Ql\nEqMmeTuxE1k3oIvI4yIyLyIvrUqbE5GDwAHgHeAW4F4RuW77pCYSW0vydmLSkPUeLBKRW4EzwBOq\n+smY9ggwDXwU+CMwCywAqOr3t1NwIrFVJG8nJo1svQNU9TkR2bsm+W7gR0AD+AXwJ+B7wKfX5heR\n+4H7AUwrv2Hvx3IaUnPKdchwtE1JT3MWqzYAXmWwdc5grSKieC+oCiLhvRElNw5FaBiHoHiEnstx\nXjCiWFGcCl6FpnUo4Lyh9gYRxZUWyRT1AgLGeLwXRGLdnZA3aqraAmCM4p0BUUQI+SqBhgJxYqwM\nSHwrIDVoroiTcITV8LkXkJhHBUzcryUco3EL4AEXRWV6Nq8JZYCGz53EtFiGxLxWz+bv79tYh/58\nXhvIfEirJZShgGVQN6nCd2gek2JeUVAT35tVx5pQDYnH2gJ8FnSZEnwOGh0oDsQHvaYGWzg0M6EZ\nVZGiAmPxDYupHDgP1oAIaoPWXvcUpVuRtR48Hxfj7dW+llZ+w+V7L2NPfpoMT42h0IzFeooq+s2r\noNFb6kJ/m9hWqhL6IXrfZg5VIbOelq1waijqDCNKWWSY3JNZjxWPoBQuw1X2bL/GMSOiuNoOfGat\nYsSTGQ+AU4P3oczaWYzxmOgdE8cTgPeCL+27fVUL0vCDsQM68FD4cjk7DqLXbCy37mXBx9E8ZsXg\nW8HPErX2txrbRCuDbdY4F00Vi8GG8SBZqJfrj+3aIjboM0bPtjGAF0zmQ1otIQbErTUep4IRcLUB\nBZOFdvEqfcujLvy4IYM+ZND+YezE8TMYpzHdC+KD32PVBuNnMAYiWddTdQy2gOWFN4+r6vtYh3UD\n+nnYQzxrAf4b369LvqvFJ362j72dEyEgi2M2W+HZ+Y8zvzRNp1ly8vQUWeax1lMUGa1WRa+X46Ph\nQu1DkG63S3Lr+PBlCxzvdvoxlMVui1Ze46JZK2fJjGeuswJAJy850Z3idLeFqtBuVJS1JbNhIPXK\nnHazxMbOKqqMqWbJdKPkrcVdNPMKAGuCAfZMneE/Z3bRqzJ63QZZ7ui0Soo6lHtqfgYEZq5YJjOe\nss5YPtWmtavg8ukVTp7u0GjU5NbRK3PqyjJ32TKnlqYwJg7MaCDnDHnumGoVeB/SlleaIEoWB1Nd\nh4nQe6GuLFOdAu+FoshxvYxGp0Q1DPC6suSNmu5ii6xd452gziALeQjGHnS2Qlcs0p8wpqswkAsL\nDY89keN21aGT40Rkliy2EKpZBx6kMogHU4Q+EQ+urUgt+KbHFKEu+RnBFtCeV1b2hH0EXANaJxXb\ng7oNjSUlXwn9YyolW3EcOvToJqz8Hjbs7Xymzecf/xIdWzLXWOZ03aL2ln+c2oOqcGJhGps5vDNn\nT1hqgxil0awplhuIDX0nBj6we4HSWdp5hRGlqDO6VUblLL1uA0SxVsnz0Oa7WgV19ML8O7todUqq\nMsPVBi1N6I9+AKtMCDy5J2s6xHhmOj0WFjp0ZnoURU4WJ5S6shjryXPHmeMdpuZWEIGyyLCZp5HX\n1M6gKpRlmHCmOz0y6+k0SpbLRugfUZa6TTqtchAYiyocv9xt4J3FV+FEqdGusNZjTPC8qmBM0N5u\nlhRVjjGeurY4F4J3XVtarYqZdo9emSOiLHebcVILgdnYkMcYT7XYZGr3Ct3lJngha1WDydUYpejm\nYSJRBnq6y83QhvFExxcWyT2NdkWx2EJ6JpzA5D4E7VJozVvqaUXjHIQo+elwwmdCCME1IeuGgJ4t\nh/DmmmF85EuKawmmVA4/9q1/XYh5NxvQAY4BV6mqSphOPxTT3oWq7gf2A4jI0jO3PnrkvV/1zEXI\ngMMXlfuC2A0c3/5izvLq1nzN0HVvEZvV/ZEtKn9db6/19a9u2X8OX2+O17fqi9bnUvPHOLCt3t5s\nQH8b+DdwrYjcBMwD9wBfWSffEVW9cZNljgwROZR0D48R696Mt5Ovh8hO1Q3br32zf1s8AHwVeIBw\nev1+4GlVfXmrhCUSIyJ5O7FjWfcMXUSeBD4L7BaRN4HvAj8AngbuI/zi8WVVPbmNOhOJLSd5OzFp\nXMi/XO49z0ef20R5+zeRZxxIuofLUHRvobdTOw+Xnaobtln7uv9DTyQSicTOIK3lkkgkEhNCCuiJ\nRCIxIQwloI/7Qkci8oaIvCgiz4vIoZg2JyIHReSVuJ2N6SIiP4l1eUFErh+izvOuPbIRnSKyLx7/\niojsG5Huh0XkWGzz50XkrlWffSfqPiIit69KHzsfjaOm1SRvj0T36Lytqtv6Ijw8/ipwDeFx6sPA\nddtd7gY1vgHsXpP2CPBg3H8Q+GHcvwv4LeGh1JuBvwxR563A9cBLm9UJzAGvxe1s3J8dge6HgW+f\n49jrokeawNXRO3YcfTSOms6hMXl7+LpH5u1hnKHfBBxV1ddUtQSeIqyXMe7cTVjLg7j9wqr0JzTw\nZ+ByEblyGIJU9Tlg7V/oNqrzduCgqp5U1VPAQeCOEeg+H3cDT6lqoaqvA0cJHhpHH42jpgsheXt7\ndZ+Pbff2MAL6BwlP3vV5M6aNEwr8XkT+LmHRJYA9qvpW3F+9pse41WejOsdJ/wPxkvnx/uU0O0N3\nn3HUtJbk7dEwEm+nm6KBz6jq9cCdwDclLKs6QMP10tj/v3On6Iw8Rlii9lPAW8CPRytnYkneHj4j\n8/YwAvox4KpV78+5iNcoUdVjcTsP/JpwCfR2/3Izbufj4eNWn43qHAv9qvq2qjpV9cDPCW3O/9E3\nFrrXMI6a3kXy9qXl7WEE9L8RFjq6WkQahIWODgyh3AtCRDoiMtPfB24DXiJo7N8l3wf8Ju4fAL4W\n77TfDCyuuiwcBRvV+TvgNhGZjZeCt8W0obLmt9kvEtocgu57RKQpIlcD1wJ/ZTx9NI6aBiRvX4Le\n3s47wKvu7t4F/JNwJ/ehYZS5AW3XEO4qHwZe7usDrgCeBV4B/gDMxXQBfhrr8iJw4xC1Pkm4hKsI\nv7PdtxmdwDcIN2SOAl8fke5fRl0vRPNeuer4h6LuI8Cd4+yjcdSUvH3pejs9+p9IJBITQropmkgk\nEhNCCuiJRCIxIaSAnkgkEhNCCuiJRCIxIaSAnkgkEhNCCuiJRCIxIaSAnkgkEhPC/wCGQslVMqHG\n+wAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "'''Train a Siamese MLP on pairs of digits from the MNIST dataset.\n", + "\n", + "It follows Hadsell-et-al.'06 [1] by computing the Euclidean distance on the\n", + "output of the shared network and by optimizing the contrastive loss (see paper\n", + "for mode details).\n", + "\n", + "[1] \"Dimensionality Reduction by Learning an Invariant Mapping\"\n", + " http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf\n", + "\n", + "Gets to 97.2% test accuracy after 20 epochs.\n", + "2 seconds per epoch on a Titan X Maxwell GPU\n", + "'''\n", + "from __future__ import absolute_import\n", + "from __future__ import print_function\n", + "import numpy as np\n", + "\n", + "# import random\n", + "# from keras.datasets import mnist\n", + "from speech_data import speech_model_data\n", + "from keras.models import Model\n", + "from keras.layers import Input, Dense, Dropout, SimpleRNN, LSTM, Lambda\n", + "# Dense, Dropout, Input, Lambda, LSTM, SimpleRNN\n", + "from keras.optimizers import RMSprop, SGD\n", + "from keras.callbacks import TensorBoard\n", + "from keras import backend as K\n", + "\n", + "\n", + "def euclidean_distance(vects):\n", + " x, y = vects\n", + " return K.sqrt(K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True),\n", + " K.epsilon()))\n", + "\n", + "\n", + "def eucl_dist_output_shape(shapes):\n", + " shape1, shape2 = shapes\n", + " return (shape1[0], 1)\n", + "\n", + "\n", + "def contrastive_loss(y_true, y_pred):\n", + " '''Contrastive loss from Hadsell-et-al.'06\n", + " http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf\n", + " '''\n", + " margin = 1\n", + " # print(y_true, y_pred)\n", + " return K.mean(y_true * K.square(y_pred) +\n", + " (1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))\n", + "\n", + "\n", + "def create_base_rnn_network(input_dim):\n", + " '''Base network to be shared (eq. to feature extraction).\n", + " '''\n", + " inp = Input(shape=input_dim)\n", + " # d1 = Dense(1024, activation='sigmoid')(inp)\n", + " # # d2 = Dense(2, activation='sigmoid')(d1)\n", + " ls1 = LSTM(1024, return_sequences=True)(inp)\n", + " ls2 = LSTM(512, return_sequences=True)(ls1)\n", + " ls3 = LSTM(32)(ls2) # , return_sequences=True\n", + " # sr2 = SimpleRNN(128, return_sequences=True)(sr1)\n", + " # sr3 = SimpleRNN(32)(sr2)\n", + " # x = Dense(128, activation='relu')(sr1)\n", + " return Model(inp, ls3)\n", + "\n", + "def create_base_network(input_dim):\n", + " '''Base network to be shared (eq. to feature extraction).\n", + " '''\n", + " input = Input(shape=input_dim)\n", + " x = Dense(128, activation='relu')(input)\n", + " x = Dropout(0.1)(x)\n", + " x = Dense(128, activation='relu')(x)\n", + " x = Dropout(0.1)(x)\n", + " x = Dense(128, activation='relu')(x)\n", + " return Model(input, x)\n", + "\n", + "def compute_accuracy(y_true, y_pred):\n", + " '''Compute classification accuracy with a fixed threshold on distances.\n", + " '''\n", + " pred = y_pred.ravel() < 0.5\n", + " return np.mean(pred == y_true)\n", + "\n", + "\n", + "def accuracy(y_true, y_pred):\n", + " '''Compute classification accuracy with a fixed threshold on distances.\n", + " '''\n", + " return K.mean(K.equal(y_true, K.cast(y_pred < 0.5, y_true.dtype)))\n", + "\n", + "\n", + "# the data, shuffled and split between train and test sets\n", + "tr_pairs, te_pairs, tr_y, te_y = speech_model_data()\n", + "\n", + "# y_train.shape,y_test.shape\n", + "# x_train.shape,x_test.shape\n", + "# x_train = x_train.reshape(60000, 784)\n", + "# x_test = x_test.reshape(10000, 784)\n", + "# x_train = x_train.astype('float32')\n", + "# x_test = x_test.astype('float32')\n", + "# x_train /= 255\n", + "# x_test /= 255\n", + "\n", + "# input_dim = (tr_pairs.shape[2], tr_pairs.shape[3])\n", + "# epochs = 20\n", + "\n", + "# # network definition\n", + "# base_network = create_base_rnn_network(input_dim)\n", + "# input_a = Input(shape=input_dim)\n", + "# input_b = Input(shape=input_dim)\n", + "\n", + "# # because we re-use the same instance `base_network`,\n", + "# # the weights of the network\n", + "# # will be shared across the two branches\n", + "# processed_a = base_network(input_a)\n", + "# processed_b = base_network(input_b)\n", + "\n", + "# distance = Lambda(euclidean_distance,\n", + "# output_shape=eucl_dist_output_shape)(\n", + "# [processed_a, processed_b]\n", + "# )\n", + "\n", + "# model = Model([input_a, input_b], distance)\n", + "\n", + "# tb_cb = TensorBoard(log_dir='./siamese_logs', histogram_freq=1, batch_size=32,\n", + "# write_graph=True, write_grads=True, write_images=True,\n", + "# embeddings_freq=0, embeddings_layer_names=None,\n", + "# embeddings_metadata=None)\n", + "# # train\n", + "# rms = RMSprop(lr=0.00001) # lr=0.001)\n", + "# sgd = SGD(lr=0.001)\n", + "# model.compile(loss=contrastive_loss, optimizer=rms, metrics=[accuracy])\n", + "# model.fit([tr_pairs[:, 0], tr_pairs[:, 1]], tr_y,\n", + "# batch_size=128,\n", + "# epochs=epochs,\n", + "# validation_data=([te_pairs[:, 0], te_pairs[:, 1]], te_y),\n", + "# callbacks=[tb_cb])\n", + "\n", + "# # compute final accuracy on training and test sets\n", + "# y_pred = model.predict([tr_pairs[:, 0], tr_pairs[:, 1]])\n", + "# tr_acc = compute_accuracy(tr_y, y_pred)\n", + "# y_pred = model.predict([te_pairs[:, 0], te_pairs[:, 1]])\n", + "# te_acc = compute_accuracy(te_y, y_pred)\n", + "\n", + "# print('* Accuracy on training set: %0.2f%%' % (100 * tr_acc))\n", + "# print('* Accuracy on test set: %0.2f%%' % (100 * te_acc))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.0\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2UAAAG6CAYAAACIge6AAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzsvXuYleV5qH9/s2aYgQEHBxwQHB0U\nhCiWMRAxokFJPMQkmtRkG9McTOqOTZNt293spmlyxaSpjWnTdue02/izxpo0xsacjHVvjfEQDxFF\nheABFARFURAQ5DjMrPl+f8ws6xEYc38zHy/PfV29GsfxZn0wrGc97/scsjzPCYIgCIIgCIIgCIaG\nuqF+AUEQBEEQBEEQBPsykZQFQRAEQRAEQRAMIZGUBUEQBEEQBEEQDCGRlAVBEARBEARBEAwhkZQF\nQRAEQRAEQRAMIZGUBUEQBEEQBEEQDCGRlAVBEARBEARBEAwhkZQFQRAEQRAEQRAMIZGUBUEQBEEQ\nBEEQDCH1Q/0CdkXWODZnRIcnbPFUL7BJ9u2wfdtkIXDoCNe3wdUFQWnYeO+6PM8PMFSTsyw3/zY/\nDdfneX6aqAwGmSwbm8MhnrAj81w1Vru6ylHdqq/aXVF9ACMa3Ljbg/sau6sNqg+AvICfnSBp8lVP\nkK9fr/zg2PERhiZGljop46AO+NsFnm+Lp3qB22Xf5bnre3cBb5Sfl33Xyr4gKAtfzB63VNuA8y0Z\n8EUYK+qCoWBYB0zwYuSMFXdprhqL2o9VfdX3qDr4oJvkAUw7ZL7q28j+qu+xNx+p+gD/c0GQPn86\nS1PZ8RGGJkaWOymrAKNFXxG/vTfKvk/JSdS3/tn1ASO//17Vt+Uu5SIhCJImo+xv2MGgs7MbVq7R\ndItudRMowP+h9T7HAVDXtNMVAlX5oUfg3gGM/81jqg/gmQcP1Z1B4gzzVKnEx3I/g52U2QkUwBLZ\nZyeOIz8hC2GLeHkJwELZFwRBsC8wogGOGKfpmjr9WvId01pV38jjn1V927bI5fh7Ac/82E+gGk58\nXncGadPTUB3ql1A6yp2U7QSeFH1vE101lsm+78u+kbIPP3DvuMsN2kGQIhlQQCdIsDczEjje0x3R\n8pAn6+e+leILBHp63P6q3o3Nqg+gOs59jV3mlQLALL9ksyr/uQTpk4t9iKnEx3InZSOATrHH6q4C\n+qs2yj63MhCekX1Afb18ulFEr18QJEYq5RmByAig09NNYqUn6+e+LW5StuMZ+RCvyU9QKrgxchxr\nVd+jO2aoPoDennh3CgZIrzcAPpX4WO5nqOulbqRXS9073j8R02+i7D5rO2kEtjwp94CZJapBEAT7\nCj3AOk93OI94shrTZZ9d7j6r3B+DAG5/6gTV1zDWLzXs3jJcdwaJk8mD7RKg3O9GO+roXSImUld7\nqhcYL/vsnjK7vBIYP9VtEn7mX6JBOAh2RyrlGYGIXOJflUevA9iXby3vdss/Nm8cpfqK4KCJq1Tf\nk8snqz6AprHP6c4gbboq3o1yKvGx3ElZUy9107Zqut53F3BTVvZx7v57Lxs3yVdbBbzGIAiC5GkB\nxC06hSRl8sHlppWusGl8+Rdlns51qu/Ow45TfQAbo+QlGCBrxKQsFUqdlNU39LD/uPWa79nRBSRl\ndnngXT92feed5foooKcsCILdkkrNfCCS01fCKDFcHr0OwC3eyH4ANnrTJgG/2gU/uZ2Nu/dsPrNV\nH/gLroP0MYsXU4mPu32GLMsuA94JrM3zfPqLvv4/gE8CVeA/8zz/i/6vfxb4w/6vX5Dn+fX9Xz8N\n+Dp9g+4vzfP84t392j3d9axfM2bAD/WaFDD0wgyIAIyUk6gHXB1AxU7KCuh7C4LUSKU8IzWGMkay\nFRBXlNz9Dv/DOpPdJKql0w3kejwDhtGl+uzBIWdzleoDuJTzdGeQNnX0aq5U4uOeJJaXA98Crqh9\nIcuyk4AzgRl5nndlWdbW//UjgPcDRwITgBuzLDu8/z/7NnAyfRXw92RZdk2e57uev9tTR+8z4u1W\npz9lie/LPwZ2T5mdNAKblshHi1H1EATB3svlDFWMXAN8zXuQIy70R+LfMO1M1bdpgRt/xs/xFynb\ny6PH4d422suoASpFfNgIkqaAeeh7Pbt958jz/NdZlnW87MufAC7O87yr/3tq81rPBH7Y//UVWZYt\nA47p/3fL8jx/DCDLsh/2f++uI8Bq4It78hh7yAcLyKPtEfa3yz5zz1s/rdOfUn0brp6o+oIgRVIp\nz0iNIY2RYwCxuMK+kSmEg9zD1e1dBSyPbnR1q2hXfUuZqvoAdtoPHSRPLqZlqcTH1/sMhwMnZFl2\nEbAD+HSe5/cAE3npUPcn+78GsOplX3/VOoksyz4OfByAMQe7SU+H6KohjiMG/Ne4UvZRwJJIcc9O\nEKRKKuUZ+wiDEyP3O1itNGjHnfIH9D29SOtB7s6unTvkxcygJ2X3c7Tquw13xH4QDDWpxMfXm5TV\nA63AscCbgP/IskyZa57n+SXAJQDZjJk5J4rv6M80ea6iuEu+Knu3u7gT/OlXeslmEATB0DI4MXLi\nrNzclbmNAm6N5LUsHZUVqm99c/kDkH1TVsSkxEKGxARJk6mjPtLg9SZlTwI/yfM8B+7OsqyXvo/W\nT8FL3j0O6v8au/j6a7+4hp2Mnbj6db7EV1Kd6F9u2lORrm16n+orgoOmPqr6uqZG2UOQJs+KrlTK\nM/YRBiVG2myjgAXAcs5z3+PuMJIDDnla9QGMlqdX2dMcn7x1iuoDaOos/2qBoFzsrHp3W6nEx9f7\nDD8DTgJu7m9SHkZfId81wA+yLPtH+pqYpwB30/f7NSXLskn0BZr3Ax/Y/S+TqQ2zmzeJR4r9XHut\nnES929UVMXHyyTvkN/SYvhgEQVoMTozsRu0b3kIBi5Q7ZN9Kt0hpfZM44bmfCeO8w2SANnnQBwf5\nNxQ79oIl3EHJqMYahZezJyPxrwROBMZmWfYkcCFwGXBZlmUPADuBj/SfCD6YZdl/0Nec3AN8Ms/z\nar/nU8D19I37vSzP8wd392vbI/Hbx/n18qP/4BHVt+iDx6o+PuXqAH9a4vi4wg6C3ZFKzXxqDGWM\ntPeUdRTRhHyQqztk7hLVt2qNWxpYBPY0R9b5c+9aOr2dssG+wZYG780rlfi4J9MXz3mNf/XB1/j+\ni4CLXuXr18HA1tJX6quMHL15IP/JLnn8O9M01wtOfUiFXJc9q4AegS2yb2UMRg2C3ZFK0EmNoYyR\nNAFiWFuDvJgZ9PJFewdY7zJx7U4N+bfRHjdf17FV9UEx+96CxBE/+qUSH0tdgpllOcMad3rCIp72\nZ7JvtJxEFTHbRG7cjvUmQRAEr4MKauWCPVAC0CcUj8I7qAV4w5z7VF8R2DdlvUv8RHTCXH/HXZA2\n2xA/3ydCqZOynu3DePbBgzVfw7uf11w1uq/dzxXal3mz/NLAGYe5w00WLZVLNoMgUUr9hh0MPr2o\nlQuFLACWd2WOxS2T6ymgqXm9fD1oJ6LmxM4a+msMkqdCr+pLIT6W+hkqw7tpmeYNoNrwqQKWFE+W\nfaX+E+lj6abDXaF98xYECZJKeUYgI8aMu199NdrvhnxTdsNTp6q+6RMXqz7wl3Db0xcPmen25QGs\nwx+YEqRNj/jmlUp8LHUKUN3RwIZlYiL16QIGSjwj90PJr7G1w50CBfDeytWq75LJf6L6giAIgoFT\nxPLoh0e+UfW9YaJbJmcnPOAnZUfgPvMCZqo+gJ32xuwgeTL5piwFSp2UVRq7GdnhzXTftExeegy0\nHuuuktkgl2ZUKv6SyFksUH1XTx70dTxBMCiYm3tS2cMSiPSg3kQdzlJP1s8N8qCPLvnD/wT8g8vN\n8mqBlfJegSISKDsRDdLHvNJIJT6W+hnqsl5GNXp1yu868heaq4b9Znn7iServhH4ZQo98slie8U/\nnQ2CMmAnZSmUZwQidajDnCaz3JPV2OHqOlih+uoLSCbspMy+KTuwgET0etyy0iB96sUe1lTiY6mT\nsu7Njerm+e9P9idLvXnina5wpasbYY/YB1bhDV8BWM0E1RcEQbBP0Ii6nLmLYZ6shrxCxZ5EaA8O\nAX+1wGh5GMnZ1atUH8BtlbfoziBt6qJ88RWUOimrjOqm5XivtG3bluGaq8Zvlp/kCv1BUDrbcX8f\nd3YV8EEgCBIjlfKMQKQOdZKePTUQQK5218sN94ayOzvJ+/fKB1QfxPTFYOCYf/dSiY/lfoYcqj1e\nqdyOJ1s11wvYv4NyqUcRb5Rn456yfWPjBaovCIJgn0AuX7QXMwPwTldn71IropqkUf59vI0TVN9M\nO1MG1tCmO4O06U6i4NCl1ElZb29G1w7xFmVkt+eqca78QyUHsG3Iy6iDIBgSUqmZD2RKHcXBvnyz\nk6iN5vbtfuyesj/jn1RfEbeD1/EO3RmkTS6O+kglPpb67Tx/vp4dN3q3W8ecdavmqnH3QXNd4UJX\nV8S4X/ukstdMvIMgUVIpzwhE5B+KIvqr7OqPG/7sTNX35n+6SfUBDJcTR/twtYiJk3ZbQ5A+vXJS\nlkJ8LPUzZKOq1J/4vOazR+kCaj0/oPeU2ZMSwQ84BxzytOoLgrLw7FC/gCBtugFva0whh3hMl32n\nyVneXoB98/aXfEX1BUHgUOqkrLF+B5PHeHtT3sNPNVeNReOPdYWnubpVm/yJk6Na3HFadv19EKRI\nKuUZgUgV9SDvXzjfk9Xwpl4D8PaJ16k+u/KjCG7krapvKo+oPoDlTNadQdr0Uqe5UomPpU7KuvNh\nrOry3jCXNk7VXDVmXHiX6lt0lJvk7Vi3v+oDqLS4UXZvmH4VBENNKuUZgYg8En82d3uyfh5lhuo7\nDncNzdW8V/UBVORM9AzcHav/jj99sZ3YNxoMjCfw5jykEh9L/QyN2Q6mNno3ZbMKmDi0gJmucLyr\nY6G5M72PIw5zF1naO1iCoCw8PtQvIEibXtSerfWM8WQ15JuylWYWCoxjjeoDWCf/Ptr749bKI/ah\noNLXINjHKHVStr1nBPevP1rznTXmas1V49Efu6eA3OjqiqgoWHzW76m+RffKJaBBkCCplGcEIjlq\n0lNE1ULTBzeovvnMVn1H4B4ygr9XbATbVd9ijlJ9AGOKGBITJI05Ej+V+FjqpGx4/Tamj7l/qF/G\nrhGbrAG1FAXwm6yBnfKpHaNz1xcEQbAvsANY4ukmnOlP5Zva4vYvfZTvqj57Bxj45Ytf5TOq7734\nB9TLoqcsGCAr2TnUL6F0lDop29E7nIe2HqH5Pt28UnO9gDzC3i71YJrsAx7C+zMBoMcvsQyC1Ejl\nJDAQkZdHr2aCJ+vH7jWyb/OKmFDcKH/Y/AxfVX03c6LqAz8RDfYFvAP5VOJjqZOyhrpuDmz2Tu5W\n6NdQ0HCxN7IfoHvhfqqPD7o6gKlPe31+AAdMfUL1BUFZsEfil/oNOxh8GlD7kIsY1vAL3qX6ZjNf\n9RWxm20xbon/cg5Tfe/AnWAJxfSpBWnTICfyKcTHUj9DRq964vQefqa5alw+5qOq7+F3vlH1Tblw\nkeoDOFBePFlEo3UQlIG9fU9ZlmWnAV8HKsCleZ5f/LJ/fy7w98BT/V/6Vp7nlw7qi9yXyVBvyoro\nr+pgheqbxErVtxR/KvMIeZfnf3K66nuO0aoPYBSbdWeQNnv75O0i4mOpkzIbewEj+IuU7QqAR++V\nB5EAI2a6Tcd7+1/MIBgMMqDBfMfezXtNlmUV4NvAycCTwD1Zll2T5/nLP7lflef5p8RXFuwpOer0\nRXuIBvi3b6vl0sDNjFR94Ccon+NvVd/1nKr6ALYxQncGaaPvKbMzml3EyKLiY6mTsgz3A/ta2jRX\njXo7oXD3MnPWzO+7QvxBH3FTFgS7J8ugfhCTMuAYYFme54/1/frZD4EzoYDrlOD1IY/EL2Ks+S1y\n/9LZXKX6isA+aLSnJX5+9T+oPoCzJ1yuO4O06RFTED0+wu5iZCHxsdRJ2XC2q+UU9+ON169hv/nW\n/eVW1TeygJICu3yxjbWqLwgChYnwkmuOJ+FVr1LOyrLsLcAjwJ/leR5bZAeL4agTdjvxpx3beyjt\n0sD1jFV9RXARn1N9p064XvVB7CkLBk7OXj3krZD4WOqkzGaCnEwA/ObH81xhpzseftK4laoPoHWV\neDQLjG5/TvUFQYpkGTS4n3vGZlm24EX/fEme55cM0PEL4Mo8z7uyLDsf+DdAflMMXothI7Zz0MwH\nNd9tvEVz1bArKxbSqfrswSHgl4GexM2q70K+pPogyheDgaOWL/rxEX73GDng+FjqpGwbw9Vr++ur\nfh21XW7IlvKfHCxpP0T1/YIzVF8QlAd3v5DMujzPZ+3i3z8FtL/onw/ivxqWAcjz/MWj6y4F/s57\necHuydQbiiL6rmexYPffNADsHVs3c5LqA+iSE9FTcW+21jFG9QEcWMA6hSBtFuJWhhXArmJkIfGx\n1EnZCLZzFIs135cqF2quGmf1uKNlD53hnXpCMSOO7YDzAX6g+oKgLHxFdBVSM79r7gGmZFk2ib5g\n837gAy99TdmBeZ4/3f+PZwAPD+or3OfJ1RL6Rro0V41vrL9A9XWMWan6iug1t7FbBo4p4Hbws1y8\n+28KghfRRaPmSiU+ljop28kwdZllEaeARSxnNiliTK093GQ7w1VfEKRIIdOldkGe5z1Zln0KuJ6+\nkb+X5Xn+YJZlfw0syPP8GuCCLMvOoK8legNw7uC9wqBvGJY3stfu/wLoftLdvfm1MZ9WffbNG8BO\n8cMm+LvUHuYI1QewZi9IboNy0S2ue04lPpY6KatQLf3ui7PmuNMN7QXXV3G26gN/r8thLFN9QRA4\n5Hl+Hbx002ye51940f/+LPDZwX5dQR89eT1ru8SlvW4uAUBTxwbV9zkuUn1PvKQCycEeAGYny/bN\nG8BUHtGdQdosNkfHDgFFxMdSJ2UZuVpOUUSCN0Y+wbp+q9v3dnGz/3np5LW3q76/a4sVR0Ga/NyU\n9V2LBMELVLsa2LRyvOY7cao7UAJgWcthqs9efFzE9EV7zcu3+WPVdx7+fnd7KmaQPnX0erJE4mOp\nk7Je6tSJPnYCBTBMrsE/tdlt6C1i4mS3fJpa9tvQICgFGSV/xw4Gne3AA55u+dTJnqyfcfLKk0fk\nSg278gNgwaaZqu/sFnc3WxHj61cVcOMYpI06mTWR+FjqR+ihwnpxSlARvUt2wHlanmBknyoCrG45\nQPU1slP1BUEQ7AvUtfQw4rRnNd9MeVIiwHWcrvpOxx2utbGAGHlEi7tffRlusvyxFVeqPoDRk/x+\nxCBt7DLfFCh1UtZAj7pYeDjbNVcNex/JdWvcAHb+uO+oviIwh7kEQbIkchIYeOS9GV07vNKFcc3u\nISP4lRB2EmVOgKsxi3tV3/n8i+r7yaS3qz6Iipdg4KhJWSLxsdSP0MUwVoqDL8ayTnPV6OR+1Tdu\nnFuLPqyAWyh7+mIRJZZBEASpU1/fw7gxXsywy/HBH1JhJ2WTCxg0Ze8+u1s+/DVXDdW4jRN0Z5A2\nOeXfyzvYlDop2/FEMw/8jzdpvso3/avSSzlP9dknbEU0305c4k7T2jjNLx8JgiQp9Tt2MNhU6FGT\nniJK+W548EzV95kjv6r67uQ41Qf+Ye3hLFV9C9jVzvjXx2Es151B2uh7EROIj6V+hBEHb2b6N2/V\nfEU0op7ELarvyqc+sPtvGgDnT3TLHgDY5OqKKB8JguRIZLpUYJKpQxuO407NVeMXR75V9V2PO6FY\nHTbQj13KZyeORexs3S4OZQv2DXrMgJZIfCx1UraTRlYwSfPZb+bgv/nOmLhQ9dmTqgBumu325q3m\nQNUXBEGwL9BVHcaKTR2ab3mLP33xfZt+pPrOa3HHuY9ii+oDf7jJ2bjTF4vo//oeH9adQdr0ppBF\nyZQ6KaunR+0DO4HbNFcNu3xx0VePVX0nfuZjqg/8HSx2z0EQJEkijcyBR6VSZXSL9/65AHeUO8BP\nW35f9amn68BDHKH6AE7C3fe2Rb7ZKqKPu4jJnUHa/IStniyR+FjyR8jV6SwL6dRcNa596l2qb+Sn\nvPHGANsKWANwDe4zF3GbFwTJkUjQCTz6fiS8GPmJtf+muWrc0zZd9dl9b0WU8tmtEjPlXvMiWjkm\n8LTuDNKmgW5Plkh8LPUjVKlX34Db5BsegLkTb1F99qndwwWcAn7iUTdwL57ye6ovCIJgX6DaW2Hj\nVi9GXts2T3PVsJMe+xBv/wIqNez9S5sZqfraWaX6AH7Gu3VnkDZdBUw+3dspdVKWyTdl9ih3gNE8\np/qe/ezBqm/YVwpYzNzk6oqYEBkESRIl+MGLGFG3jc5mrw95I/trrhpT5cmBdhJVRPliEUmPyVIO\n151jWK87g7Spp8cVJhAfd5uUZVl2GfBOYG2e59Nf9u/+HPgacECe5+uyLMuArwOnA9uAc/M8v6//\nez8CfL7/P/2bPM93e93SSx3bxIk+Iwtobt0pTw489CsPqr5C+rX0v0ex1T0Igr2ToY2RmTo9sIgP\n6/bBpZ04rmaC6gMYLh80zpf3lI3AHdYFxZREBmlTxOTTvZ09uSm7HPgWcMWLv5hlWTtwCvDEi778\ndmBK///NBv4ZmJ1lWStwITALyIF7syy7Js/zXb5bV6iqb+i3yAsdAe6X+9SOxp2+WLEzKNCTMnNB\neBAkSyI18wlyOUMUI+0JxZ/b+hXNVeOi5s+qvndwneorIv50yR827SSqiH10cbgaDCmJxMfdPkKe\n57/OsqzjVf7VPwF/Afz8RV87E7giz/McuCvLstFZlh0InAj8Ms/zDQBZlv0SOA24cle/dpU6tR69\niOmLP+U9qs++2SqiZPPxKQeoviN4SPUFQZIkEnRSYyhjpM3y5kN158FyKZ/do9bBStUH/sh5e39c\nEWtoni7gxjFImyZ2eLJE4uPreoQsy84EnsrzfFFfNcYLTISXvAM/2f+11/r6LmlkJ5PEN0xzyWaN\nyXKj4nIOU30X8TnVB/Cf1Xeovusr/v64ICgHfzfULyAYAgYrRjbQrY43X1HArdFSeTDHYSxXfWtp\nU33gl4Haa2iOXbVI9QEsbD9adwZp003DUL+E0jHgpCzLshHAX9FXlqGTZdnHgY8D1B98IGsYp7nX\nM0Zz1fjNHe60qvFzHlN9H+Z7qg9g1CZxjCkwpjUahINgj0igkTl1BjNGNh7cplZX2D3S4A+AsKtJ\nihjKYd+U6ZOj/TVljGr3e/aDtNFLXhOIj6/npuwwYBJQOwE8CLgvy7JjgKfgJd2eB/V/7Sn6yjNe\n/PVbXk2e5/klwCUA2YyZ+WNPeTdHKyZ2aK4aU+a4J052gHgX16g+gBWt41VfTF8Mgj0gkfKMfYBB\ni5F1R3fm8zcdo73waov/qcbuX7KHAxQxDMu+HZyw9RnVd9PsN6s+gGF06c4gbTJyU5ZEfBzwI+R5\nvhj+674/y7KVwKz+yVLXAJ/KsuyH9DUxb8rz/Oksy64H/jbLstrYpFOA3Xb/Vhp6aBnvnbLN5m7N\nVWM5k1Xf/Ko7Zam+4veUrRVvL6GYmv4gCIKhYDBjZGNlJ5NaVmqvvaeAo+ZTuV71rZMrXopYA2An\nKNua3T0081b9RvUB/KT97bozSJucbPfftI+xJyPxr6TvBG9slmVPAhfmef6vr/Ht19E36ncZfeN+\nPwqQ5/mGLMu+DNzT/31/XWto3hXV5xrYcPVuy+r3mOrZfsAx6/kBOiorVF8RQXbOhvtU3/Wt0VMW\nBLslkZPA1BjKGNmdN7C6yxuwMKrRL0GzR87bMXdsAfu1xrJO9f2Kt6m+d7Veq/ogbsqCgRM3Za9k\nT6YvnrObf9/xov+dA598je+7DLhsIC+uMrqbke/2ru2LGPRh18vbt1BFjL5d1nqQ6rN3ugRBkiQS\ndFJjKGMkGVTqvWqIIibh2gnFGnkwxwSeVn0A/5N/VH1f509U33XN/q1WEZ+vgrTppc6TJRIfS/0I\nvdU6tm3xlkcf1bhYc9VYJpcv2pOligiydjP4ONaqviAIgmDgjK36t0ZdFTdeHM4jqm87w1UfwHlc\nqvpOXnu76iti0MdNnX6fWpA29UXs0d3LKXVSlndV6F62n+ZbPOYozVXD3mJvNzEXUS9vL6ReJq8B\nCIJkicPo4MXkUO3xfihWN/q7pm7jBNXXyf2qb6Q8KRH84SGL2qaovqfb/D/nYezUnUEwIBKIj6VO\nyugGxKFD2/Bu3WospFP12eWQKwvYO2OP+13PWNUXBEGwL2BXk2xs9Mvdu+SDxqr8saWIRcp2BY29\nPPq0H92q+gCufZ+7HihIH7V8MRFKnZTVj97J/mc+ofnGyM23ACdys+qzT9iKKF/8bl9vukYRI4mD\nIDkSqZkPPPI8U2/KirjtsKceP8QbVF8R5fM/4AOq7wtbv6z67nrfDNUHxVTlBGmj9iEmEh9L/Qj1\ndKub7KfKtegAX+YLqm8m96q+wzY9rvoAvrrii6rvW51/qPqCIEkSCTqBR1bXy7Amb+rdWnmIBsBo\nnlN99k1ZEXsyz+Yq1de0VdUxutk/CLXbGoJ9gZi++HISeISh5a3cqPoa5ZPKe1v8E7FKp7v7LJZH\nB0EQDJxKpcroFu8Dtj2+HmAzo1Tf0SxUfXvD1MANbe6esmkL/cPaxZ1+z36QNrGl7JWUOimro1f9\nwG4HB4DrcXdslX3RJsA7Ntyk+u5sPU71BUGSZCTRyBx41JGrB3kf3XCl5qrx962fUn12uXsHK1Uf\nwEMcofrqq+5B6GOd41UfFNMqEaRNE9s9WSLxsdRJWS916nAOe+kk+CWR9jTHo/DXAIgtDEAsnQyC\nPSKR8ozAo5pX2NjlDefY3NKguWrYPWV2ZcUn+GfVB3ACt6m+/Z7odn3mBLV+7pk0XXcGaRN7yl5J\nqR9hx84RPPC4N92wfeyTmqvGxmZ3WpU9OGRLAbeD20e6J5Ux6CMIgmDgVHvq2bTOi0H7bXM//APM\nn3KM6vsw31N9xzBf9QFsxP1c8OCkQ1XfERseU31QTOlrkDbdeIP8UqHUSVn9sJ2MPWSV5puPGxwA\nHqq6ZQpTK0tVXxEJz7AdbuBe3xwj8YNgjyj1O3Yw2Axr6OLAiSs13+McoLlqtOPFcIBxGzapvj9o\n/YHqA/gp71F9PXJd1v2t7gSdllbcAAAgAElEQVRLgA5W6M4gbRrtKqkE4mOpH6GJLiazXPMV8aax\n4dqJqu+hM90kbzF+8217sxtk44QtCIJg4DSxg6l4B3mH3PSs5qqxcJ7YNwI80eomjgs5WvUBLOVw\n1fc2eaBYe5f3uarG/Y3uztYgffaGITuDTamTsuFsV3ui7ma25nqBg1yd/UN6uBiwa6xlnOqzRyYH\nQZIk0sgceDTQzYFir/Q98/y+IHsAhB1/JrNM9YG/E3Xaanda4uMT/BvRUWzWnUHaVBAH2CQSH0ud\nlO1kmDr44kRu0VwvsMTVnTXzatVnj9gHmL1hkerb2OrW3wdBkiTSyBx4VKiyv1iibpcaAlzKearv\nguo3VF+lx51sCHB/o1u+eHzXfarvkIX+jeiozkjKgoHRYH4+TSQ+lvoR7JH4RfSUyQdiehJ1P35J\nQU+rO9FxjXzyGQRBsC+Q4/Yb2QMqwL816qm4x+HrKv7amLH2BwP5k9r3O89yhRRz4xikTZc8vTwF\nSp2UZeTq9WYRt0Yc6+ruxN3ZNbuAyVJju9arvjWNbaovCJIkkZPAwCOnju3i2pgxuO/tgNoXDnAv\ns1RfEc+sr6LpcXVHc78rxB9GEqRPHb2eLJH4WOpH2M5wdQljEW++nOvqqg+7b2xF1Hl3NQ5TfYUk\ny0GQIqV+xw4GG3uXZ8X+9I8/bOo47lR9R2x9WPUBLG52e/M2T3JX23TRqPqC4PWQk7nCBOJjqR9h\ne/dwFq/x3tAXj/MnEfJFVzeTe1Wf2kjZT+sdO1RfZY7/GoMgCFKnlzp24h2SmT3cNexhU51b5Z7m\n5hbVB/5hqO37BheoPoBP8m3dGaRNZt6UJUKpk7JRDZt50zhvmfJhRdQ8T3N141ij+i7mM6oP4PL6\nP1Z9a4jyxSDYLYlMlwo8eqhXe3JnPPqo5qoxYoo7Ev/qZrcf6ih+q/oARuA+86iqm5R9pnKx6gPY\nGbdvwQCpI/dkicTHUidl3dSr42+rRTzuB13d/YvdwRwf5nuqD2DD7CbVN0Ec6RwEQbCvUKXCZrzS\ntqemtGquGvYtjx0vivhcYN5eAjxXcQewFJFAxc6pYKDo5YsJUOqkrC/x9UrbiqiXb7p9g+ozm7ah\nmDfK1lvd8sXtc91nDoIkSaSROfDYvnMEix6fqfkqh/il5GUfXjW6y1spUGNxo9sqceSKx1Tf1yd9\nXPUBnMr1ujMI9phE4mOpH2F7VzOLlnsLnzce5o/7HdWyRfXZNf1FlGz+aO47Vd86xqq+IEiSRIJO\n4NE0bDuTDvEm/Y1fu0lz1VjV5sa0DlaqvjWN/kqW1UxQfc9OcidYntd1qeoDWNo4VXcGadNLnSdL\nJD6W+hGaGrcy+bAFmm9qATsRnv36warvmD/5F9VnBweAU6vuidi2StyUBUEQDJR6etypwnd5qhon\nnuH1hQMs5vdU30m4rw/8fW+N1S7V13yHP2Bh1LxYHh0MjCIG0e3tlDop66o2smzTYZpvVEsBbxrv\nFBsV8W/KjuAh1Qcwv+LdXgKsx1/eGQRJEm0bwYvoG4k/3BMW8FZs9oWDX75oD9EAqFTcD5u/rpyg\n+t4y9zbVB6i9jcG+QdVesp5AfCx1UlZfqTK2xTsFLGLc78jx7g+VvbPLDogAJ6+4XfWtnNSh+oIg\nSRIpzwg8GthJO6s03/fnuJMNAY7jDtU3Crdl4M6K2/MG/g3A27bepPoa3Yu3PvwZMUGw5yQSH0v9\nCBm5+ua2kg7NVaOtea3qu3TNearvO9v+VPUB7JAn2C9glisMgtLw/aF+AUHCDGcHR+AtPx5rn1wD\n7ZueUX2LW96g+o6Rb94AruJs1Tes2T2sPa7nbtUH/sTJIH1i+uIrKXVSVqXC5qp3Jf7Wyo2aq8Zj\nXz1S9X3iM/+o+rr9vm0WNs9QfZ3cr/qCIEkSOQkMPHqpY7tYvnja6ls1V41nJ4xUfddzquoroqds\nMu5gDjtZvrPlGNUHxUy3DtJGTcoSiY+lfoQGdjKh4u0kKWS54fu7Vd1o3PG8De7EfgCOfXSR6lsz\nyy+xDIIgSJ1e6tRenu5mTfUC83ETgAu6vqH67PH1ADdzkuqze8MnFzCVeZu8zidIn7gpeyWlTsqq\n1KtTjMZtKODaqMf9LbR3utw+6Y2qD+D49fepvsX4QTEIysENniqRk8DAo45edTmzXRoI/q3R6kZ3\nonARE+CO4req75y1P1d9uF0XADw4/VBfGiRNPeKlRiLxsdSPMJztHIW3g+XiVr+/igvdTP/0b/6n\n6jv+ATeBAsjl994YixoEe0gC06UCj5yMLrECxBwaUsMua7NH4r9lg99ftbTV3dnVLRf5LJw+3RXi\nV/kE6VOHO708hfhY6qTMZhbezrMa8755repbzmTV1+0PnKRhq+sbwTZXGARBsA/QQ726UqRawKea\nj/Bvqu+6te6EyEVtU1QfoPb5AWwf2aD6iki+YyR+EPzulDops+vlT17rjnIH+KM2d9nzqbiLmQtB\nTso6WegKgyBFEinPCDw2M5Jf4+2wGoY/K/0C3B6w/9c2V/UdzlLVB/CcvDx6XcVdIPddPqr6AOZw\np+4M0qbLvF1NJD6W+hGa2KE2uD4/xj1tAtR6foB75fHwDY+qOgD+Ztafq744YQvSRSyNSiToBB5t\nPMsn+D+ar/WmHZqrxglz3EXFKxs7VN+4Lr/Bav9Gt5Tv0FXuWoH3t1+l+iDKF4OBM5ztniyR+Fjq\nR1jXfQCXPHW+5vuriRdprhp2UvYhrlB9/2+We6oI8Pk7/kH1XTtnnuoLgrLwd0P9AoKkaaCbCXgT\nihfN80v57KnHR219QPUtb/YHVKzBnSh8V7u8hmarO0EZ4KFmf0hMkDbd0bryCkqdlLU1rOEDE/9J\n89mTDQFOwD0FnM9s1feXXV9VfQC5/N67kkmuMAhSJYFG5sBjB40sxRsq8bFVV2quGre3uxOA72x+\ns+qb98BvVB/AFdPd6o8OVqq+Rr9KlRHN8QE7GBh19LrCBOJjqZOyHOgRf5dnM19z1bATCrsBt/kJ\n+Yce+OWU41VfEU3HQRAEqbOVkepB3jPtLZqrxlr51miq3AO2o4AzwZV0qL5VuBO7VrZ2qD6IKcrB\nwKmyZqhfQukodVLWt6dsf803Yatblw2wuvlA1VfEUkebk5e4A1MWTfNLZoIgORKpmQ88hrFTPdQa\ns8nf5bm6xd0r9vtL/q/qKyL+2Ls32/aCD6/bY3l0MEAq5k1ZIvGx1I9gj/u9rvntmqvGVbxf9ZmD\nTQDumVL+fST23pkgKA/ipJ1Egk7gUUcvjeLExPoCLjuWcrjqs5OoGdf407D++Axv+ArAIWufVX3m\n21KNZ+b4t6xB2qi3q4nEx1I/QjcNrMY7Zevkfs1VYxIrVN/PeI/q+8SP3B0xALS5uuvmnu4KgyAI\n9hHMDzZZARtZjj7HXXli94bPONzPUPRJhPaK1QJ6ymKKcjBQemPQxysodVJWoapONxxbXa+5ahxT\ncfvUzsYdVfv87/trAIZv6VZ9p3Od6guCsvCnpiwjiUbmwMMuX7zvHH+CnrlHDeDyhX+s+ra+oU71\nAfwtf6X63nb6japv/KN+mWoQDCmJxMdSJ2VbqyNYsGmm5vv3lg9orhrHyQsT7Ybe49fcp/oAvfRh\nwVzvzzgIysWTniqR8oygvLzxmod159NnuD1lz3aOVH0H/GiL6gP4yvs+q/pGb3WTqOcP9Q9rx7BO\ndwZpU6HHkyUSH3f7CFmWXQa8E1ib5/n0/q/9PfAuYCewHPhonucb+//dZ4E/BKrABXmeX9//9dOA\nr9OXy16a5/nFu/u196ts5oSWX72e53pViriR+Qv+XvXZkwjP2fBz1QewYW6T6quPqU1BEOylDGWM\nzIGqeDy86Ax/6MW5fFf1mc8LIM/kAPzpi0ub3T+XCTyt+gBW4w49C9KnmyeG+iWUjj3JKy8HvgUv\n2Wr8S+CzeZ73ZFn2VeCzwGeyLDsCeD9wJDABuDHLslqX77eBk+k7Or4ny7Jr8jzf5VSLYXTRIfZs\njTC3h/fz0/VuD9hPxri+x6aPV33gLwONkfhBsIckcBKYIJczRDGygR51Mt/UrX5/1UPNR6g+u/Su\nu4BcwuyFB5ixwv1zuWeSPwBsb5gQGZQLfY1CAvFxt4+Q5/mvsyzreNnXbnjRP94FvLf/f58J/DDP\n8y5gRZZly4Bj+v/dsjzPHwPIsuyH/d+7y4CzidH8gjP24DH2jMks11w1vjDmr1WfPVa2iObbSdWV\nqu/Oir/UOwjKwSJPlUh5RmoMZYzspU6NGU3XaKoXWH/OWNW3aIo8ffEOPxHtmOMOAJPPQXnT2gdc\nIfBsm1tWGqSPujw6kfhoPMLH4IXpFBPpC0A1nuz/GvCS65AnYfcbL/fjeU7FGwd1IjdrrhqdSx9R\nfZ+f6jYIvxW3QRhgv0fdQR/Dp8UEniAIkqWwGLmDJpYy1XiNAFx7zjzNVcMc1gUFVFYUMMn9Tuao\nvk+slacoFzB9cXVblC8GAyPKF1/J75SUZVn2OaAH+Hfn5UCWZR8HPg4w4uAxbGS0pdaDA8C/TnWH\nh6yQa9FHbN2h+sAv9xjHWlcYBCmSyHSpfYmiY2TLwaPUJOVo3PH1AFdxtuqz91oun36Y6gM42l6/\nI6+h+fqEj7tC+tpNgmAgbGGDJ0skPr7upCzLsnPpa25+a57nef+Xn4KXjA88qP9r7OLrLyHP80uA\nSwBGzTo8N2uzv8pfaq4a9u3bW7hN9TUV0a7V6upiv0kQBKkxGDFy8qyW/CgWa6954lrxQ1I/q9rc\nicKHyW0IH3vgStUH8P9N/6ArXOLqPjDhB64Qf3J0kD7NbB3ql1A6XldS1j8l6i+AuXmev7j27Brg\nB1mW/SN9TcxTgLvpy2GnZFk2ib5A835gt1dMLWzkXXhF7kW8adinduPkZtmTK7erPoAlbYeovuX4\nJ5VBkByJ1MzvCwxWjOyhnvWM8V74ak9VY12b21N2FL9VfUV8Lvx3/kD1/ffp31d9819oY/SIipdg\noOS7/5Y9J5H4uCcj8a8ETgTGZln2JHAhfZOkGoFfZlkGcFee53+U5/mDWZb9B33NyT3AJ/M8r/Z7\nPgVcT98F42V5nj+4u1+7hwbWMu51PdirMZu7NVcNuzTjEQ7f/TcNgA9NuWL33zRApi15XPVdN03V\nBUG6JBB0UmMoY+Q6xnIp52nP8u7On2quGmvl2jtz+BfAUbO9m8YaH0aOu3K+805ucoXAT9rerjuD\ntNF7yhKIj3syffGcV/nyv+7i+y8CLnqVr18HA1sU1ktGF8MG8p/sErM/rSjsUbqtS/yeMta7uihf\nDIJgb2UoY+R+PK8Oc2pd4MeL82d9R/XdT6fqm7fiN6oP4LJJHarvrukzVF9PAc03RfTsB2mjTl9M\nhATyyj2nsYBGVPuNaCpLVd/WSXWqD6B5i/sXaRg7VV8QJEkijcyBRw8V97CxgAo0O0bO4U7Vhz8R\nn5snnaT6PnaH2/f2+JwDVB/Aetwy1SB96unxZInEx1InZc/TwvWcqvmm4o6vBzhJHvSxgkmqb1uj\nu/cMoPn5LapvNM+pviBIkkRq5gOP52jlKt6v+b455jOaq8Y6s+cNf0LkU6fIk6uAySxTfXYSdQtu\n0gjQwUrdGaRNFfHSIJH4WOpH6Opu4tGnvB0s1Yl+Gn0/R6u+inlyADRWCxhT6+Zkat9gEAQe/QMr\nvk7fGeSleZ5f/BrfdxZwNfCmPM8XDOJL3Kexd3nao9fBPww1D2oB3lbALs81ckw7ZMGzqu8j9f+h\n+gAWdbpLvYP0qac61C/hd6KI+FjqpGz/hg2cPPGq3X/jHlJET9ltnKD6PiQ3CO/3mLvoGcBcLQFQ\n2cv/YgbBoDDIJ4FZllWAbwMn07fM+J4sy67J8/yhl33fKOBPgPmD9+oC6CvJ72CFJ2z2VDW2MVz1\nzcTN+Y9c+JjqA6h0uoerO96g6ljf7N8O7u0fsIOhQJy/mEh8LHVSVqGq1qN30ai5aoyRp15sZH/V\nt8OdGwJAkxwgtuGXWAZB8DtzDLAsz/PHALIs+yFwJn2TA1/Ml4GvAv9rcF9eMIydTDLLxuRhaAAb\n29zD0Ak8rfq2vsHvux7BdtXXtEnVsaY5qlOCoSc3yxcHn0LiY6mTsp0MU3eLnY87BQr8kfjXcbrq\n+1LPl1VfEcTUpiDYQ9wK7LFZlr342uGS/sXENSYCL14//yQw+8WCLMveCLTnef6fWZZFUjYEVM0f\nCvHSrcboWRtV305xIjPAmka/ZnME23b/TQNB3qU2tcsdKAawsrFDdwZpk9nTF/0OpV3FyELiY6mT\nsmHspP0lz/y7Ybpq2Issj8LdmfJIy6GqD+DIFrfcw26KDoIk8csz1uV5Puv1/sdZltUB/wicq72i\nYEBk5Or02vveJ5dBAPvjJmV2xcto+fWBP/Ti+UMbVN9+q/22hp72BEbfBYNM5qr8jOZ1x8jXGx9L\nnZTtoIllTNZ8o7r8G5lxje4M4e1yKd+R8/16efs0Yrh9qhgEgcFT8JJShYP6v1ZjFDAduKV/QfJ4\n4Josy86IYR+DQ0Yvw8RVLxNYrblq/JR3q76z8frMAVbKE4/B76Pb71Y3idowr0n1gV+yGaTPXr6n\nrJD4WOqkzO4pu7rxvZqrht0PpU7SArZ2FrCnbLH7FymWRwfBHjD4I3/vAaZkWTaJvmDzfuADtX+Z\n5/km+K/lRFmW3QJ8OhKywaOHBnU/1Pgr5eYlYOo57vTFp3Ebpd+48GHVB7Cw011wbXMvr/uC/DWJ\nNoRgoPSYMxkSiY+lTsoa6GYcazTfEa/ov/vdsU/E7JPKhxqPUH0Abzr4AdWnNqoHQaoMctDJ87wn\ny7JPAdfTdz9+WZ7nD2ZZ9tfAgjzPrxm8VxO8GtsYwQJmesJD/9Vz9bOUw1XfYXK5exGDPvSDRneY\nYyGtHOaNbbBvUI94A5xIfCx1UtZLnVo/PmuDm0wALG319qiBXy8/d9Xdqg/g5+2nqL7GeDMPglKS\n5/l1wHUv+9oXXuN7TxyM1xT8F410MZnlnvAuT1XjLbNvU332XsunG/0Rxfaal9ff+fnqFNFHt1q+\nwQzSp1rAz+FgUkR8LHVSVqGH0Tyn+TJ3ej0AE1rdmy1z2iTA8xPuV30AZ86/QfUtmX2I6guCZIle\n+uBF5GR0mdMIC/hcbX9YNwebAMxd7R9c3j/BLV/skrf5jN7ql6lubo42hGBg6DtqE4iPpU7Kmuhi\nKmI9egHjftdPGaP67BLLFZUO1Qcw44lHVd+22bGnLAiCYKB00egOqpjjqWq0iS0I4O8pe3bCSNUH\ncDkfVX1HNLt9b0X0f1XsGssgeXr37j1lhVDqpKyHCusRk54pnqrG/JeuJfiduZmTVN//WfLnqg8A\nea2L3ZcXBEky+I3MQckZxWZO4mZP+G1P9QIXubrlHKb6luK2IACcIg/smr3Vvc1r2qDqALinfbov\nDZKmt/wj8QedUj/CThpZQYcndC94ABg2yS2lWCPXyxdyeLXQ1dXPla+wgyBFEgk6gUcdvW45n7+m\njJ1yn7R9y3MCv1Z9AIs5SvVtbG5RfdVmv86riD61IG3qzfLFROJjqR9hPza5I+Llumzwyw3fxo2q\nrxDktS72os0gCIJ9gR7q3WqSHZ6qxkZGq769YfS6/RrtHavbGv2WAXs9UBDsi5Q6KdtGs7pP4y1z\n3SlQACvNmzz8kfg3TX+z6gOYt/g3qu8h/LH9QVAO3L8rKTQyBx5VKu749YM9VY2q/EOrDjbBf30A\nv+YE1XfhXX+n+tbN9VsGog0hGCh6T1kC8bHUSdkItjITbw/pxFv9QupZc909qSPYpvpmblqk+orA\n3jsTBEmSSHlG4NFFI8vMHiu3Sg6ABfKi4g9xheorgqPtGn+5rHSLvUcNGMF23RmkTR29niyR+Fjq\nR6jQ65YBNHuqGnYPmH1rdOzaApKyAnrzgiAIgoFRTw9j8Xa9dE/TVC8wlaWqz04ofiv3f4Ff8bJD\n/uxy5MLHXCFwa+cxujNIm50UMHFmL6fUSVk93YxjrSnUUUtHgHfzU9V305QCyhc/6pZkFXFqFwTl\n4FlPlchJYODRQz3rxJ6yhut2/z0DZeY5bjXJciarvlncq/oAbpFj2vrmVtWHu0YNgHae8KVB0qhD\nihKJj6V+hDpyhtGl+XYUMBL/p7xb9XXiLnue3TVf9QGw2NWNbC9/43YQBEHZaGK7O2yqgJ4yu/rj\naXkZ9clrb1d9AAvb3EmEduvF9+eepfpg7xjAEpSL7dw11C+hdJQ6KbObmJvES7caF036vOqzx8o2\nLxZrdmvIY/b3r8Yo3SDYLYmcBAYePTSw1iyh3+qpalzF+1XfX3Kx6nuqTb6FAtrMCh/Qk+UP/ujH\nrhB47H3jdWeQNsPNGQqJxMdSP0JOpu44+fmkUzRXDXuRpT1if9Qs//Rq4gr31G5+xV3AHQTlwT2F\nzxOYLhV4ZORuCdAmT1Xj61yg+tQkFL8FAWAca1yhvW/Unb0CQMXcORXsE4iro4E04mPpkzJzXO1s\n/FI+dUcM8EPOVn1v3XSr6gOwb5zbJskBLAiCYB+gb9DHOk/oVgYCfrnhUqaqvhO5WfUB6iofgGk8\nrvr0JC8IAoVSJ2XPM4obeZvmO/kOv3b88DnuZCk7cWxYour6kHvzpm6NcY5BsDvyDKqlfscOBpu+\nkfji4AuvMOUFVsi7PO21Ma2r/I3Zw9rF20tgw5Qm1de6wn/m7bE8OhggveLVfCrxsdSPYI/E3zDH\nfWMDf/StfcJ2ZJs/+tb+qdEnSwVBaRBLfRMJOoFH34RisdKggEO89lmrVN+MJfIhXgElT3PX3u0K\n7bLSAiaRT+uRb/OC5Gny5vglEx9L/Qgj2KZOI2yd758O3TnbnSy1TT5t+tKkv1B9ABcu+TvVd38R\n83mDoBTcNNQvIEiYDLmXp4BBHzaPTztA9Y3bKq6t6GdNs/saD3nUfY2Pz3FfXxC8HnY2PjfUL6F0\nlDopq6PKKLZ4wkmeqsbPeI/q+xDfU30fWfUfqg/gsmnnqL7DWK76giBF8gx6KnWisYDJrMGg8iwH\n8B3O13yfP/0fNFeNRnMQCVAvD5SwEyjwE71n54xUfUXsBh0ZI/GDIcSPjzAUMbLUSVk9VUYjZtIF\nnALauznsaY5zkcso8BujzQmbQRAE+wpjWO8e5Plt1zx0jltN8vtL/q/q6z5Q1QGwfWSD6jtggXg4\nDRzQ5PoAaPGVQdoMc89rkqDUSVkvdWrz6OOT/BOxA+WeMrt88fb2N6o+gOMvvU/13XXeDNUXBOXB\n67PIs4xqvfmWHRFxb6dCj3tw+QZPVeMoFqu+R6cdpPqGYTa29GHfRF0x68Oq76sPfFH1ATza7v65\nBOnTNczrh/XjIwxFjCx1UtZDPevEkfNFlMnZO1Om4k5zfBj3lBKA81zdKtpdYRCUhkWqrVpJYBFL\noDGKLbyF2zyhn58wruquPOmquJUVOxmm+gCeY7Tqu3jDF1Xfs9PdckiA0WzUnUHa2LvtUoiPpU7K\numlQk54DrvSv7Gef446wP1xOyvRJVQDyvJTnj3JPUoMgCPYFKlTdD8MF7K96qOIeDHZudQ86bms+\nXvWB39bwXKs7OXr0Jv+z0OqWGB4SDIxe+8NkApQ6KaujynBzJ8nznqqG+vqAqvxH8vwUt7YdYL9r\nulXfc53uqWIQlAev4T8no1rE/O5gr6WbetaIB5fT6v2x5nYlxKhmN+E5eaHfSPehzktU39e5QPU1\nFPBZKHrKgqEklfhY8qQsZwTbPeHpnqqGOh0SuJeZqq+x4tejHNnq7j6zS0CDoDy4SVlPAkEn8NhK\nM/M5RvPN7fEHQx2DW01i7/KsdLolVABv40bVt5CjVd+8ym9UH0QcDwZOt3ijnEp8LHVSBriZ73c9\nVY2NX3Bveey+N7u2HeDWud6HgCAIguD1kVPnTq/1WrhfwE4ojuNO1beaCaoP/F6Zkza4SdTjE/xS\nQ7tkM0ifSqxleQWlTsoq9DCG9Z6wgMlSFbkI334zn1zAcJPxd2xSfQ/OOVT1BUGq2OXNwd7NSDZz\ngjnowy2CAOC4aW4StZijVN+8G/xbo5WndKi+bc3u/qWnC0hEY9BHMNSkEB9L/QTPsT9Xc5bmm7va\nL804godU32J+T/XtDcSbeRDsnlRq5gMPe5fno6f7Y83X0Kb6xrBO9T11SqvqA1hJh+prXuHeKMxu\nc4elACxtPUR3BsGekkp8LHVStpH9+Rnv0XzfbPuM5qqxnMm602T8o+6tFlDyn5ogCIJ9g6eYyOe5\nSPP94ob/prlqrD/FrYm0D/GKKF9UK3yADdPc6YtF9H91FbBaIEibnGyoX0LpKPXH6wms5gtc6AkL\nmDjUwUrV9x3OV30nTPm16gNoXRhjTINgsEnlJDDwmMhT/A2f84QFbCcZdYrba3QDp6q+/77w+6oP\n4IrOD6k+ewF3O6tUH8BqDtSdQbCnpBIfS52U9cjjfov48xomb9u0k7w7OU71AZz0hltUXxEnlUFQ\nDjYM9QsIEqaOqjpg4Z4/n665ajyEu6fMjpGPdY5XfQAf5XLV98b5D6u+DbPdmzeARnbqziBtMvKh\nfgmlo9RJWZUKmxnlCd3SdgB38hWwlMNVn33CBtD8I7e+vfJBfyRxEKRICieBgUcP9axnrOZ708IH\nNFcNe+S8uQIAivk71SV/Ljh6ipuU7b/Br3ZZ2TpCdwZp04s7wCaF+LjbpCzLssuAdwJr8zyf3v+1\nVuAqoANYCfy3PM+fy7IsA75O30awbcC5eZ7f1//ffAT4fL/2b/I8/7fd/drNbGW2uePEjzcsPKPT\nl4ocXb3fl8rJrT3BMghSJJU9LKkxlDGy1z64LOBi1z5otA9CT7vpVtUHsGjeFNWX2VMxC1h90NG6\nwpcGSWNWmqUSH/fkpuxy4FvAFS/62l8Cv8rz/OIsy/6y/58/A7wdmNL/f7OBfwZm9weoC4FZQA7c\nm2XZNXme73JsVN8pYAIgU6wAACAASURBVAHvHiJtrFV9f8APVN/mihiw+9lvvRu56+U1AEEQBIPI\n5QxRjKxQLf302nFyjNxiJqEA01xdEdw0682qb95PC1gDMGmS7gzSZieRyL+c3SZleZ7/Osuyjpd9\n+UzgxP7//W/ALfQFnDOBK/I8z4G7siwbnWXZgf3f+8s8zzcAZFn2S+A04Mpd/drrGMulnLeHj7J7\nPta4y1/udWEvTFxFu+o7drU/+pa3ujp7UlUQpEhfI3OpK873SYYyRlapsJHRynMAfH+et4Kmxk55\nKt9buVH1FbFI+Xt8WPX9D76h+uTLxiAYclKJj6/3Ccblef50//9+Bl6YxjERXjLW58n+r73W119B\nlmUfBz4OcMDBTXyGr77Ol/gqFDB9cQTbSu1bMsHfHTLt8sdV38ZzvQ8VQVAu3JUUKdTM7yMMSoxs\nO9hNeD74nR+rPoAHzz9U9T3CVNV32gN++eKnp39N9Y3eKq+2eYOrA+iIW49ggNiD8lKIj79zWpnn\neZ5lmTZCJc/zS4BLADpmjcnXmg1MBZQpTHvUTVCWTnEDTiG3UPIb+tQN7u9hEARBWSgyRk6d1ZxX\nzPJvu3cJuJ+jVd8JuGtetk5xhw0AXIE7Ev8v1n5L9T02yZ84GQQDpSovgk+B15uUrcmy7MA8z5/u\nL72oFY0/BS+pvzuo/2tP8V+lHLWv37K7X2QnDe649AIGffzynONVn73rY+yGLaoPAHcQFCtmR4AI\nUuUZzZTKHpZ9hEGJkfVUGc0u284GhnupBfTtGzV5Wl6hsrHRr9Swe8N5wtVNaPPel2qsaj5Idwap\n4y2PTiU+vt6k7BrgI8DF/f//5y/6+qeyLPshfU3Mm/qD0vXA32ZZtn//950CfHZ3v8h+bHbrx8/x\nVDVOXnu76ru67b2qb3XrL1QfwMQJ7qCPsdXoKQuCICkGJUZuYaS6i3LGE49qrhrzVrhDJa6dNE/1\nHcdS1Qf4vS3yWrGmAqZsdvQ86UuDpBkWM95ewZ6MxL+SvhO8sVmWPUnfhKiLgf/IsuwPgceB/9b/\n7dfRN+p3GX3jfj8KkOf5hizLvgzc0/99f11raN4VFXrY35ws5Q6BAuCPp/+D6uvEHWE/cUUB776t\nrq6nsvefbgTBq9OtmXJIYuRvagxljBzBVmZxr/cwsz1VjW9N+kPVZy+jnsDTu/+mIeaAHXJ5ylZX\nB7C4vYBGtSBptle8PsRU4uOeTF98rfulV8zg658o9cnX8FwGXDagV2fvHfD3KDNqnjt9UR9v3Ozq\nAOx+XnvvTBCUB3NJaxrTpVJjKGNkPT2MMfsy7vBUNdrPWLX7bxoAx3Gn6lN///ppZKcrdCs2C/lc\nMI41vjRImgbx0DKV+FjqJ3iO/bkar5xvxgNf0Vw1vrT1i6rvO80fV31FLAO1m8HHT5MnSwVBEOwD\n9O3yHKv5Dp3j9xodxW9V3y84Q/VNLaB88Sj5BPimKfKesjv8PWUbJ8QU5WBg9FDAzIO9nFInZcPZ\nzu+Zb26zPFWNSo/rs0fiPzOtRfUBjG53k6htzXLBfBCUBu+mLJVG5sCjQq+7K7OAHo9L+e+q70ub\nvqz6FrZMV32AO6AMmLdATqLGuLo+ZfSGBwOjXnzDSSU+ljopG8kWt1ShgHG/m1vchOIwlqs+86ax\nxqdW/6vq2zyl1D+GQVAaUgg6gYc96GPaen89yRlco/qWt7i7N6dW/Zuy/R4zy7JAXueEPBATgPGt\nUfESDIwG+VIjhfi4b30aLqC5tXWV2TMCi9uPUn1/vOkS1QfoqwUOaIkr7CAIgoHSxA4ON8vvxLWg\nNdTXB1zPqapvTWXc7r9pgLxlzN2qzxsc3s8kWwg7iuhfD5Im91cE7vXsW0mZm+8AcH77/1Z9J3Kz\n6tvZ5P/UN8zpVX3Pto1UfUFQHrwDh1TKMwKPjFwtAZL3PAOwmVGqb6xcJme3DACsbnVHFA9rdQeH\nHHCNfxC65owDdGeQNt113o7FVOJjqZOynQxj1Uv2bP5uTFzjT734EheqPvsUcH6jP+P4qDa3iXl9\nEQXuQVAK4hY4KI5e6tjGCE/4E09V45E/mar61uDebLXjTocEGC4nevtvcCty8FvN3cOBYJ8gIx/q\nl1A69qmk7NiWRZqrhn0KeAK/Vn3tm/xpWg12bltAKUUQpEZurwgJ9nqqVNiIOPVujqeqYScom3Er\nK9Skth97NLd989Y1d5jqA9hewO9jkDbm9MVU4mOpk7KMnIp5+lJAc6td+vA5/kb1vaflZ6oP4MyF\nN6i+lZM6VF8QlAe3ATOFPSyBR99IfLHSYL6nqjF2lltueDQLVd9yDlN94H8umLZAHsBSQMfAkmnu\nAJYgfexeyRTiY6mfoIGdbmnB6Z6qxgJ5zv5sOSraO2IA7pv7Bt0ZBEEQDDHyNDSAK/iw6ruAb6i+\nxUU0m8tsmOVOeR61SS6HBIbpIyKD1Mlw5xOkQKmTsmF0u0nZHZ6qxknHu4M5fltxA8TYqr87pH2L\nWxK5fWSD6guCFEmlkTnwqKeHNtZ6wgIOLlfSofoe4gjVdxZXqz6A+qrbX7Wu4vZdNzaJPzM1J+4w\nkiB96sSeslTiY6mTsh4qamnG+Lv8PRrDt7j7SK5vcQd9zNl0n+oDyOwy0HZ5p0sQJEgqQSfwGGZX\nkxRwcHn6lOtUXwcrVJ+dQAFUelznzkqj6qv0+DcUlcYY9BEMHanEx1InZd3yoI8jZ/nboxvkGvzF\np7g3ZdkTqq6PCa6uwT+0C4IgSJ4KPYzGGytdRAXaBLmZWx+iUZEDGjC6slH1TXtA7ilzczwAuqb4\nw0OCtMn9DXx7PaVOynTcAzYA7vnCdNV3Ad9UfYs6p6g+gBkLH1V993VGj1qQKg+rthSmSwUefYM+\nxmq+Qw/0p/UuYKbqs4do2NMhAW7jBNX3vrZrVR8/cnUA66d4P4fBvkEPm2Xf3h8fS52UZeQMM+uU\n/fYq3rTKna52e/sbVd/UrW4CBeinqW9c635wDYIg2BeoUu+OxHfnSQBwHHeqvpmb3NU2Df6aMia9\nYaUrlEPk7Z90P2cAHN3lTsUM0qcpj0EfL6fUSVkjXRzGMk94qKeq8VS7uz9krbwYs6mAnOza2fNU\nnz3iOAjKg7fUr69mvtRv2cEg00OFdeZIfH8oH2Pl09DVLQeovh+0/IHqA3gPP1V9+3W55YuzC+g1\nX9lykO4M0qY7W6O5UomPpX6Cbhp4WmxgOmTxs5qrhl36MEq+zv155ymqD+DMa9w9ZY+eEW/mQarY\nSdneX54RlJgC9pQtO8PdAzaJlarPTqDAH7M/rc1NyqoFfPKL96ZgoJg9ZanEx1InZVUqbGaUJ2zx\nVDVa17pHi6vavMEmACdwm+oDsNe6jGGdKwyCINgH0NfGFIA9wv4tVTem7fewP/23fYr8Z+J2SdBU\nwD66aRPkYSRB8jTF4O1XUOqkrJc6Nyn7c0/1AltdnX1TNmXFk6oPYMmkQ1TfhKo9Yz8I0iSFk8DA\no0rF7Sk72FPV2B93EuGKSofq+z/TP6n6AD7N11TflFY5jhcwvf7BCQX0hwRJs73B/blOIT6WOikb\nwTZmssATftVT1dj6ozrVZ58qskme2gRMWyifiLl7MYMgSXKyJKZLBR729EXcYcIAVOQMoF72feeB\nP1V9AN+ffpbqm1IvJ2UFnIN2dK30pUHSNIqDPlKJj6VOyrYxgoUcrfkOGef2QhVBpzz04q7OGaoP\n/OlXq1rGq74gKA/+iPEgqJGRu0nPXZ6qxpo57vCqdWYSCtw+XbxpLAr7BvM/ZR9wf2OnLw2SZlu2\nZKhfQukodVJWRy+N5vx1v0qBrkZ3YeJSDld99jhigAZ5ouOoWVtcYRAkSCrTpQKPrTRzr7gH7IOn\n/lhz1Vgvl0Isxx0ccgQPqT6A2fbEFHvZcwFlqnbrRZA+5oFSKvGx1E+wneFqOd9pTbdqrhqtq9xB\nH43t4l424IBVfsKz9Si3ZHPshkjKgiAIBkozW5nJvZqv250zBcBouafM9h37gFv5AXD7dHcP2JSt\nbvnikve5feFAEh+Ig8HFnL6YCqX+W1RHrzty/leeqsaST7pvbmvkPWWPtvvj5qc8KgeIKX6ACIJy\nII+yTqBmPvBoZAdTWar5Gr6tqV6g66/cahK1zxx4cLo/oOJyPqr6jp7itjWspEP1AbSxVncGwUBI\nIT6WOilbx1gu5TzN94nF/6a5anThBpyjuV/12aUj4DcdqxM2gyBRUtnDEnhUqXenLzZ7qhr2XrFb\nOEn1FbGSxW4baL7VG4gAcPgpXiJf4xGm6s4gbXrEdR6pxMdSJ2XD2c5RLPaEszxVjalb3QarXzW/\nTfV1yAER0NcAtOFtdQ+CINhX6KHera4oYCrfOvlgcJx8I6N+xuhnof3RSt4rNra63hUCaytuWWmQ\nPvZk1hQodVLWwibewXVD/TJ2SdMG1/cvzeervj/b8M+qD4BNru6Qtc+6wiBIkFROAgOPjNwdhlUA\nY3ETgBO5WfUdstqPP7dMkCc6yhWWCyr+CXURN45B2tThjsRPIT6WOinTF2MWsA+rez/X90d8R/Wt\nbm1VfQAT17uZ6PPHNqi+ICgP3aothT0sgUcjXRzGck/Y5qlq3CyXG66Q+6EumPBN1Qd+b8uSaW7f\n9bzLf6P6AB48N5ZHBwMlV20pxMdSJ2U7GcYTiOOg7vBUNda+x0167hf3sgH8z0cLuCmT33uHb3E/\nuAZBEOwL9O0pE2vbCji4HGEO6wImyDWWB6z1p/+OaXNvB6dd7g4M4hRXBzHoIxg4DXZdbgKUOikb\nxWZO4hZPWMDk9WVMVn2rzCQU6C7g5LPhedcXy6ODdPGWR6eyhyXw2EGTOmBhhtwjDfAhrlB9+8sj\n8fEfmRFtbiLKBFdXxDOPbo7VNsHAqIgtZanEx1I/wdMcyEX8leab1/IuzVXDLlNoF6fRANzW8mbV\nBzBvq1v6sJJJqi8IyoOdlO395RmBRzcNrDY/sRdwiPc1Pq363ou74Loyxx82YK+20Zc9FzBbq4jP\nGkHabKn8VnOlEh9LnZSN5xn+F1/zhE2eqoY9PWYyy1TfbZyg+gDm/VSuR/+kqwuCINgXaGYrs5lv\nCnX+jP+t+sbJGcW0hXJpIDC8U74pczflwBTZB5y0we9TC9JmVFQvvoJSJ2U683zl0VX33fIXFfc2\nb28YN28H2SBIlRROAgOPnMxtbncLNQC4l5mq71SuV324e5kB2N45whXaB8p2OSTwXGsBp95B0vTU\nu5NjU4iPpU7K7IDzzNwWzVVj/Fp3PrzdIHwel6o+gOf/yJ2WGA3CQRAEA6eLRpaLfc3HT79Pc9Xo\nolH1Xcp5qu/cc7+r+sDvDZ/2qHyb5w9l5qEJR/jSIGl2sGSoX8LvRJZlpwFfByrApXmeX/yyf/9H\n9NWCVembavHxPM8f2pWz1ElZPT3qjpPxfysv2AI2/JV7OtTBCtV3PaeqPoCTKreoPnXtQRCUCq/5\nXb8V2QOKCDqBh95H8bCnqjFujlsJYVdWzLjDn3qxc46biDLd1dm7RgGO7irgyjFImhG5u6dsMONj\nlmUV4NvAycCTwD1Zll3zsvj3gzzP/6X/+88A/hE4bVfeUidlOvKiZ4C7ma36RrFZ9Z3UdYvqA2he\n6/1FAtjWPlz1BUGKDPZ0qaKCTuCxkRauwSt5/9ixV2quGp1yfaB9C1VEocYae2KKuwUA/N3RVOv3\n/tKxYHDJM29P2RBMXzwGWJbn+WMAWZb9EDgTeCE+5nn+4lnlzezBYrZSJ2XdNLhTjHZ4qhqHs1T1\n3ckc1WcnUADc6urGnuOWbAZBoFBI0Ak8mtjBVB7xhAV8IlgrJyjf5VzV1/EetzoF4Fe8TfW98/ib\nVF8R66F+WznKlwZJs33vLl+cyEu7cJ+EV97SZFn2SeB/AsPYg8kWpU7K6uilEbER8BxPVWOhvOx5\nMyNV32Xt/kN/bK57mnpz5STVFwTl4QbVJjcyj82ybMGL/vmSPM8vedE/FxJ0Ao9hdLtrVBbs/lsG\nyqpp7s3WF/iy6pvxQAFLu+xyQ/kgtIgy1d/7h8W+NEia4fLpQAGDPnYXI3dLnuffBr6dZdkHgM8D\nH9nV95c6KeuikRV0eMIb7Hc2GDPHveWxR+yfvsn9UAjAr1zdUed6uyqCIFUK2MOyLs/z37mQaaBB\nJ/DouykTqzXcanwAPv7g91TfwiMPV31FTF986/QbXeHprk6seH2BrorcRxckTy5+3i1oT9muYuRT\n8JJa6oP6v/Za/BD45939gqVOyhroZpxZ8N3pqWq084Tqm88xqm9yi7v3DODITY+pvp3ydK4gCBQK\nCTqBh748Wq6SA/jV+cepvu3I4+YLSERXMskVuiG3kESUPynAGQTl5R5gSpZlk+iLi/9/e+8eXmV1\n531/VnaSnYOQmEDAQJAAEUSoqLH4eCi0HodafWy1HkYrba3tTJ06Pbw92F6defpO5207M0/rXO2M\nOjod2049to44pYPjiXpoUVQsiiCnKBI5RkJIyGlnvX/s7AyKAQLfldxZ+X2ui0sI2w93bjZ73b+1\nfocrgKv2fYFzrs57nzuK/zBw0GP5RAdlKbqpFHZfDHFkX33eFqmvs1QboJywWv1pDszT6mY06od3\nGkaMDPIcliCLjqGjjRJWCHcbr+2+V+bKUagsQQBO2/aS1KecvZ2juk7dmUPMR/XKsY26TrPGyCC/\nS+sbzPXRe9/tnLsBWEK2O/G/eu9fcc59B1juvV8E3OCcOwfoAt7mELJIjigoc859EbiObHH3SuCT\nwDFkd0wrgeeBa7z3nc65NPAz4BRgJ3C5977hQP48vLamrFSnyvFIqbaE4lLul/q8uAkUgBM36Nr+\neW0dnWEkh+H7oBJq0RlJhF4ji2jXNpsKsF6oOwo/V6Ut2Dq1+WWpD2AnlVqheqh3gGeh9uP1TiNu\n/DBv2Om9XwwsftfXvr3Pzwd8fnzYQZlzbgLwBWCm936vc+5esjupC4Afeu/vds7dAnyabErLp4G3\nvffTnHNXAN8HLj/Qn9FNPjuUH24B9nDVLefvT18q9c3oDnAKdZ5WV948fB9cDWOwGIo5ZSEWnZHC\nYKyR6uHRIVB3IjyLJ6W+ps9rZ40CnM4zWmGdVic+vARga+lYvdSImq68t2WuoVgfQ3Ck6Yv5QLFz\nrgsoAd4i230rl+JyJ/DXZBeci3t/DnA/8GPnnPPe99tCOUMeLYw6wkvchwDdF9ekp0t9DcrGJkBX\ngHKtAnHdwbbPVmiFhpEYdMMRh2AOi3HkBF0jU2QoZ5fuagP0hVpw2eKDv2gAzFgr3mjU9tYC4OoZ\nfyf1LX70Y1LfhpvGS30A4zoCDHwzoiZfPDw6hvXxsL8D7/1m59zfA28Ae8l+nD8P7PLe5/pcvkm2\nrTLs0165Ny2mmWz6xo59vc6564HrAcZOKqKEvYd7ifsjzigAmN6hnVPWmS6U+raV6QOeCR/STuEu\npFPqMwzDGGoGY42smFRK5Tt/+8go06lyqGvK/qtOW9R8wS/0XZnPmqE9zVOflE25XVsLD8DZeqUR\nN3n26LcfR5K+eDTZnb1aYBdwH3DBkV5Q7wyA2wAm1Y/10pOj3+hUOVq+LDzJAzahnekinWGTQzxr\ns7NOG4gaRqwMcqMP4wgYjDXyuPpRvpaGI1UGZSXvk/pWMVPqu2CuPii7hAe0wmatbvN1+s3aDmwd\nNwZGR1q4oUQc6+ORnPWdA2z03m8HcM79GjgDKHfO5ffuBO7bQjnXXvlN51w+2T25A7ZW7CGPNmX7\nW+2cZwBteiWwiIukvssa/1PqAw7ytzZwimnTCg0jQgLNYTHCEXyNlNdRBJhfJW3WBXw+8xOpD31J\nGS+K5+/MOF6bsjlhuTbbBaBLXfdmRE+hMHU4lvXxSIKyN4DTnHMlZFMzzgaWA48Dl5LtLnUt8GDv\n6xf1/vr3vb//2IFy5SE7p6waYWtZ8dBjgPIPCfP5gYtYJPXRqtUB8lSKo5vatULDMIyhJ/ga2U0+\nOxmju+IA68WL4t3Qo1Labo5nvviC1Acws2aVVijeCGWK2Ad0FuXppUbU+DxdTVksHElN2TLn3P3A\nC0A38CLZlIrfAHc75/6m92t39P4vdwA/d86tI1v9fsXB/owSWqln+eFe4v4EyJdfw3FS3y7Kpb4g\nU4PE97F7+G9uGEZwYukuNVIYjDUyQ0q6ZrywQN/XfLqyZX8IZuuV8rIB9bOL8LEqx9ZZAeYpGFHT\n5XTpi7Gsj0fUqsR7/1fAX73ryxuA97/Ha9uBywbiz8Nrm0Cs1KlynLlWu8v2V3X/R+r7TPUvpD4A\ndbp8QYBg2TAMY6gJvUZ2i4Oyk+97VebK0XKZNsU/X90usfvgLxkoz1Mv9Z3b9JTUR4BGiaOG8UxG\nY2hIYSdl7ybR/SPVCw7z3tS5AvE5btUKQ6QvflbsC9CS2DBiJIaWv4aOQrq0pzIBBgDfznVS3wf4\nndR32rKXpD6AmXXa9MV28WzQIm3MCEAqRHRrRM4Bs7MHTAzrY6K/gzx6KFE2gdA3WeLB67Sflh2I\nB4vVanWA/KRs++eP0goNIzHodo9jKWQ2dOTRo80mCXCC8kV+KPUt2/+Q8cjQNjwG9M2risQdj0OU\nNVSkrTbcGBj5wjg+lvUx0UFZN/nsUBYxF23QuXpZIe6ytJgFUt/Vq38l9QEwS6tLZ7TduQzDMEYC\n2eHRb+uE6oYSwHqmSn1j1Bepb0RIg3g3tGKxNq20S53tQpiZqEbcdOWLZz1EQKKDMk8eeynWCQPs\niKXEuXcLWCz1EaL2Vlwk3DZPOPbAMBKFbtGJZSfQ0JFt9HG0Thhgf0yd/SFvHBKg1nzyJeKjrVKt\nLhPgya+8Q9uJ2oiflNfVlMWyPiY6KJMToA3sXJZJfdJh2cDSWeJUD2De0melPvUwUMNIDr+X2mJY\ndAwdPeqNywDp7ur3bKE6cgzwXCCvbREfKBT9m9YHwEXWtMEYGHldWl8M62OigzJHD8Xs1QnF5Vqg\nzx1fLu7aNJ8npD4A5ml15dgOm2EYxkDJkKIFYXfDAL0aZqJteiFPX9yq1YE+ZXNsq7gZSYCUzQie\nh43Bxg31BSSPRAdl3RSwTZl/97ROlSN9mbDIGjifJVJf3WMBOk6K18SZtfo2zIYRG7HMYTF0dFFA\nI9U6YZD6qslSXyW62UZAkOYmp20SB1HXaHUsFPtA35XZiB/h4Wos62PCg7J8dlCpEwZo9zun+WWp\n7/Gy+VJfkO6L4pqyohBt+w3DMCJnFC3M53GdMEA2yVTWS32Pco7UN+U0/SzPl2rqpL4TvyJul3iL\nVgfQOilPLzWipqfQUl7fTaKDsk4K2cQknTBAo5cnys6U+o5Wp/IF2AVEPDNle5W1xDdiRd0SP9Ef\n2cYwZ/f5BXKnNL2SAM2wAjwXyAdcf0SrE5fCA9A2xxp2GQOjx+nKf2JZHxP9HZTQxkm8qBPO0Kly\ndFAo9cmPX8u0OoDnZmh74qsXbcNIDtbowxg+jL5XXHkPPHnlWVKfdAQAMKFKn7Op7sqszk5Be5AH\nwNhG3QaUMTLIt0Yf+5HooCzb7rdcJwxQU1Z/0fNSX6c6fyTA6JCZHdrC7Y60NrA1DMMYCTi89lRG\nu5wBMPdK7bGMfI0UJuPkeIbTpb4Zs1+X+l44T1/LEcMDsTG4tBasG+pLSByJDsqygzGF6XxFOlUO\n9SmP+sP8o2t/K/UBlLZq84BL6tulPsOIkVjmsBg65N0Xz9CpclSKO0OdvE3cGCpAHd0m9VDURq3u\n5B8EaK51pV5pxE2psE9eLOtjooOyVkqkLeIv4z9lrhzqdu6FaLs5Bvkb/oNW57TZkIYRJbF0lzJ0\neJx8OLOaX3KV1LeySjvt+do37pX6AG2DMoA5Wl17gPTFVaUBOqkZUdNWqBuyHsv6mOigrJQ26pXJ\n1AH+vsZu0+ZRn1QlrKEDNswdL/UBTCnbIvXtHqcvLjeMZKCv0TGMHNsZy+1cJ/NdXfErmSuHujGH\nOjvlhXp9MDGd17RC8bzsInWNGtA2TzjE3BgR9GAdO99NooOy3YxmCefLfJfl60/KVlcdK/WNokXq\nm7JYG0ABoNvcAGB0hT24GsahEEN3KUNHBU1czt06YYBuveogaqf4FOri5Q9LfQDr67XDo9Wbq1Nu\n1T8X5M8TNzcxosfhpb4Y1sdEfweltHI6z+iE6gGM6IOoB7hE6rugaKnUB+hTKUq1PsMwjJFAIZ3U\nsEknFM6hziFt1gUUo2ujDbD7JH2mhjp9ccoicRDVrdWB/u/FiJ885fToSEh0UNZDHm0IZ1/cpVPl\nWPfVaVLfVqqkvv/vQ38p9QF8Y+2PpL6HSi+U+gwjOehO52MpZDYSjLYnB4A2aCQ7KifppNW14eL0\nRfFjBqDfoDbiJ0/YOTaW9THRQVk7adZwnFIoZyba9vAfFufff+bhX0h9gLzN/mXb9GmlhhEbsSw6\nho4O0qxHuDHY9JjO1csKcWrFfJ6Q+grb9enzo0rFAYo6iAowMFseiBrRkydMX4xlfUx0UFbGbs5n\niU6oa+TYh7r1rbKGDuAzVQGCMnEKfuuNVuxpxIqlZxjhUD+IbF6oH2x5HGukvsUskPqqS8X95oEX\nOUnqu6xMvHH5kFYHsPWicXqpETVdw+DUe7BJdFDWTBmL+bDMd2G1fhewWjxARD3TZe2ciVIfQN3S\nN6W+HWlx+2DDSAzbpbYYWv4aOtQ1ZRM2NclcOf69ZqbUNxttS/wQTBcHovIasC+JfYaRAGJYHxMd\nlBXRrv1wW6RT5bhnzuVSXweFUl/dCm0ABcgXiD3i7lyGkRx0QVn2VCTRH9nGIFNAF8cINwZX12i7\nCQNMY73U1yleI8dv1OfynV8rzPAB2K3VheiyWVm9Qy81oiZf+DAZy/qY6O8gjx5t8eg8nSrHVrRH\n9uqi6BAnZSVztEfOjSFafhlGItgw1BdgREw3+WwTrkFnPvyCzJXjxfO0NWXqkgFxsgsAqVpxe3h1\n3Bhg3vixjdqsd8DV9AAAIABJREFUACN+Cm0a0n4kOih7fW8tn37llzLfp17Wt1/cNU/b7vd5TpH6\n/hTd/csx4QfaFJdHvmpBmWEcjFgKmQ0dbzCJG7lZ5vvoG7+VuXI0UCv1/W3rTVLfV+t+LPUBNHKM\n1Df21bVSH2VaHcD2BUfppUbUdBfoNvhjWR8THZSNL27kkyd8Wyf8vU6V4zpul/rmskzqm/Hw61If\ngPp9/4mme7VCw0gIC8W+GBYdQ8d4tvBl/l4nPFunyiFt1gVcUvqA1MctWh3A9M+JgyhtbxMIML50\n7LY9eqkRNfniUpgY1sdEB2UpMtr0xQDf7Ti2Sn3q1Iz2M6Q6AIrEowXerijSCg0jMQSYw2EYvaTp\nYCrrdEJxLAHQUqutGV7JbKlvxkn6jctMvrijcJO4i2uIJTdAm30jcsRZvjGQ6KAsQ4oWZROIAFPs\n/5EvSH1zeFHqK1oh1QHQJa7Nk9cIGEZi0D3lxpKeYejookBaUxZiD6GByVLfWfxO6mufK9UB8Ez6\ndKnv3JqnpD5+rtUBbP+spS8aA6M7bemL7ybRQdlodnMOj+iEAaJydfqidFg2BCnoLRCPPpt5dYDt\nWcMwjMhpJ80apuuE4uUH9N0SV4hngE35ib6Obv5nxUGUOGOTz4t9wNhGS180Bka+NfrYj0QHZR4n\n/0BXU4m2DewmcVL/K/VTpD6AEx7WdpRrLBsr9RlGclC2xI9jDouho5Auacfe7TP0px2n8LzUpx4e\n/dFZAZqblGm7HtddGWC0jZjXq20dNwZGZ8HbMlcs62Oig7ImKrgb3RywC9bqq1v3UiL1LUObS/Fn\na++U+gAQz3oe12qtdA3j4MQxh8XQUUQ704Q1ZWNv1Z92PPHZqVLffB6X+n6x4GNSH+hH29RUaIOy\nhlL9PDr1eCAjfjpoFdriWB8T/R2MZwtf5/s6obYnBwATNmrbw19U+5DUF6Sgt1SrK9LeQsMwjBFD\nSpmXX6FT5WgTb1yqN0Kvvu9XUh/Ac5fNkvrUa+SMRn1zk/K6XXKnETdFdAz1JSSORAdlGVLsQjgH\n7JP6D6KXauukPml9AEC7PjVDurkBNNVY90UjVnSdE2IpZDa0SN8T9TpVjmrxdGb1v4F2dbt54O/5\nitR3c82NUt/4FfpWiVupkjuNuOm29XE/Eh2UtTCKx/mgzHfaypdkrhyZD2lvoXQEANBUpw94KpZr\nW3RJA2/DSBRbpLYYFh0jwQSYX/V4rW4NB7iU+6W+EKl81/AzqW98oziICtC+Pt/6mxsDxkttMayP\niQ7KSmijnuU6oXa8CaCfU9ZItdRX8UCAHsc/0eqmzNM+uBqGYYwUpOmLAU6NanhD6kuJZ9tUBahr\neJKzpL7V1drAccZyfdaQYRhHTqKDsm7ypcWjrWeIBzqin7GlLNoGgtQIoM3YZEP1eK3QMBKDbsPB\n46LoLmXocHhtUPZvOlWO8q9qa43ULfFP3KQfyVJZs1Pqm7FNHEQFGClWjG7mlDEyyEM3FD2W9THR\nQZma0qW6N0CO4vO0H0RTWS/1sU2rA2COVlfbZCdlhmEYA0cclIk33ACeFXcU/nP+Ser7cc2npT7Q\nd1/cUKXduKyeq19zd3G03GnETQZrDvNuEh2U9ZCnnVO2UafK8QynS30N1Ep982Y/K/UB8LBWt6ZC\nn9NvGMlAt8PtI2n5a+joIUWL8tijTKfKsYDFUp96TtkXmm6T+gAeqThT6puyWhxEaTNKASg/Tzdz\nyhgZKFORY1kfE/0dtFHCcmE7qE+NvkvmyjFNfLKlbveLds5zlpVaXU2HdlfRMGIlhkJmQ0cPedo1\n4w86VY7ffUhbX1XCXqlvVcUUqQ+gkE6pr2mGtmFXRUpfa76TMXKnETfd4pOyGNbHRAdlKTKUC//S\nXr9SP3Fe3S1RmooCQU4HuUira0hP1goNIzGE2BUxjCz5dFGlzFE/Q6fKcQ6PSn3S7Bn0ARTAaxwn\n9Z3eLM54CdAoUf7sYkSPG+oLSCCJDsoK6ZR2bjp2+XaZK8fiem0qRQOTpb4Qc2f4sVYXYlE0jNiI\nZQ6LoaODItYzVeab97I+3b1h3mSpr41isU+cnQIcI57NtrNMm1e6pkw8DxW0abTGiKCLN2WuWNbH\nRAdlaTq06YGrdaocJ9WvkPqUJ4MArXP0HSdLt2kbpqhbHBtGjHgcmZ7hv+gYOvLIaLveXaZT5VBn\nk/yUT0p9d/IJqQ/gIXE6yYebHpP6xqEPvpdVnCh3GnGTr64pi2B9PKKgzDlXDtwOzCI7Be5TwBrg\nHmAy0AB83Hv/tnPOATeTnYTSBiz03r9wIH87Raxi5pFc4js4t/4pmSuHcrh1CDaltS37AWbcpG3P\nq/w7NoxkYZ1FRzKh18geUtqaMm1PDgDaFmpPor7ID6W+VwOsP+p5o3+s0LbFVAfKADuolDuNuOlO\n9rnQkHCkd+Rm4L+895c65wqBEuAm4FHv/fecc18Hvg58DfgTsg1364C5wD/3/rdfitnLbGVXiZ/q\nVDk++P3H9VIhM1YHGBIpTh0/nWe0QsOIEQ/d3cN/J3CEEXSNVM/yZJZOlUOdCfHBjHbNHd3YJfUB\nrKnRpgdO69A2FNuR1gdQcwOcvhlxU6o85Y9kfTzsoMw5VwZ8AFgI4L3vBDqdcxcD83tfdifwBNkF\n52LgZ957D/zBOVfunDvGe//WYV/9QNFuXgH6HbFKtEMnh0OfgZJWfScowzCMoWQw1sgS2jiJF3UX\nPUmnynF0wmcRNdVoOxsCtDBK6luV1p7mTWWd1Aewi3K504ibDPbs926O5KSsFtgO/NQ5dyLwPHAj\nMG6fRWQL9G3jTYB3TFR8s/drgxeUBWCOckFEH+RRqtUB8jllRSGakRhGZHjvyHRbuscwIvga2U4R\n65imu+Lbl+pcvRx1kzZVrniP9mRrWdlsqQ/gHi6X+r618R+kvhC01Fr3RWNgKLsvxrI+Hsl3kA+c\nDPyF936Zc+5msmkYfXjvvXPOD0TqnLseuB5gzKQi7e6LfkNM3phjk3qrMkBzE+4T+y4R+wwjQrKL\nzvBPzxhBBF8jKyeVUKJMAfqoTpWjBu0cyr1HFUh9e8SnWgBz0DYAe6H2eKlvdvOrUh8EmLFqRE8P\nzTJXLOvjkQRlbwJveu+X9f76frILztZcyoVz7hjoG6KyGdi368TE3q+9A+/9bcBtABPrx/k1CHOz\n9/xW5+pFPUF8G1VSHx/R6gDE3X5p1643hmEYSSD4GjmufqJfifCkZ8ldOlcvmRnaB6XRW7UnZd3V\n+ge5ct6W+qZ3rJH6CtZKdQCU1yc7TdVIHjbbbn8OO6Lw3m9xzm1yzk333q8BzgZW9f64Fvhe738f\n7P1fFgE3OOfuJlu83HywerLsnDLhLluAk82xL++R+ipniWvK9Gss6rK3Iu1sUcOIE08UO4EjhcFY\nI8to5nyW6C5a2+QPgMV8WOq7gTukvlFo13CATtJSX0daOzB7R73+dNBqyoyB0q2sKYtkfTzSMOUv\ngH/v7Sq1AfgkkAfc65z7NPA68PHe1y4m2+p3Hdl2vwcdNlJIJ5PZeISXuA8hTmS0Mx31OwetWl0Q\nAjRgMYzY8N7R3TX8F50RRtA1cjejeYRzZBd7wSZ9TdkNm7RB1As12oU8RDChbjlfsU3bEKHiDX2D\nhXR9p9xpxI10Tlkk6+MRBWXe+xXAe7VpOPs9XuuBzw/Ij9PuOOnSV/tor9D6dqpnfez3NyHga2Kf\njTcxDCNCQq+RJbRyCssP8+reA233egA212gXyZmt2nqofyr9c6kPshvKStZWTZT61ldNlfogQOdo\nI3o6A3QBHe4kulVJPt1UskMnDLDgtJVqu4eoW+kSIHecDrFvOJzmGcaQ4+jJJPoj2xhk0nRSS8NQ\nX8YBKVY2IgGKxOvFX5T+o1YI/C3flPrUDcUu2Kg/EW2qDdBJzYiatPRhMo71MdHfQTcpdnH0UF/G\nAVF3HJJ20gLUfUMAuFrsE6eAGoZhjAS6KNAOjxZnfgBU3KVNldt+5VFSn7rmDfQdJ1vQfs/lFfo6\nOuu+aAwUj35w+3An0UHZXkpYwRyZb16AifNVzU1SX2WZOAUgwOmg/KTMsh4M4+B4IIJCZkNLhoS/\nJ8Q1w+mMdgG6J6WdKQYwm5VSXwO1Ul9nmbYRCQTI8jGip1N5CBHJ+pjooKyIdmaySicMkCb3RNmZ\nUp/6w3z3h7UzXQBGL9bubmyfo90FNIzkINyR9i6KRcfQkSGlfRgO0H0x6V2vb+FzAZyflfqWMVfq\n+0KHPmWzMW0du4yBka/8cIhkfUx0UOZxdCBsBfuYTpWj4zptq9pxTdpuJO7XUl2W57W6sU/rUykM\nwzBiRz425j6dqo8va3XLUtoAJcQJj7r74qXcL/WtS+sbfVj6ojFQui1Naj8SHZQV0EU1BxzTMjAu\n0alyXNiojfSeqj5Z6jvjoy9IfQBOOKUACLM7axix4YFuN9RXYSQIj5OmL3bdJFP1UbDs4K8ZCPPn\nPiX1rSrTL0C3ik/K2iiW+kIEUOqGLkb85NGjk0WyPiY6KNtLMSuZLfOdvFbbShdg82XayujpvCb1\nba3Qd9EY36w9zdtSZZ0+jFgRz+EIUSNqDFvaKWIN02W+c5dqAx4AxE35/r7sL6W+uYijRuA6bpf6\n1EHUVNZLfQCrmCl3GnHTzWa1cNiT6KCsiwIalVXCAWZ2VTdpG328UTFW6jt20XapD5AvsuPXBhgg\nZxiGETl59Gg79goTU/oQZ6ikxE9ehfLOVZASF9Kd1viS1PdK9RSpD/Qpm0b8pJQnZZGQ6KBMvuAE\nmNm1Y662SYW6y9KxswMEZeL0xa4QbfsNIzY8UewEGjo6SLOOaUN9GQfmUa3ucx+/VerbkaqU+gCe\n4INS34Ky30p9x9Ao9RnG4SDdEIlkfUx0UNajbvRxhk6VYz36glkpPwngnKXV7Sqz7otGrFgTGyMc\npbRSz3KdMMDGJZcFcArZyRi5cwGLpb5lpe+X+jqVz1W9VFrTBmOAdKp3+CMg0UFZdk7ZSTrhN3Wq\nHKfdpE0r6KqR6oI0N2GFVjd2uT24GsZBiWQn0NDRTT67KNcJz9Op+hBnQqS6tamB01kj9QF0pLRz\nwJ7kLKmviq1SH8Bke8A2BoidlO1PooOyKrbyRX6oE4YYoyG+g8+UaXfE5r2sH5gtzrDUD6M2jBjx\ngHZEoDHMKaWVU5QnZfpeWGw5T9vIqSSj7fJXvEf/j2pdmTal9JzMI1Lf8lS91AeQSfbjpJFIOnWq\nSNbHRP8rKqRLO4PlSzpVDnXB7ArmSH3zqgIEZUu1uvZva32GYRgjgW5S7OJonTDAE4G0BAEYv1Xc\n/bda3/23nLelvn9MfUHqU889A33bfiN+emgf6ktIHIkOyjKkpKkZ43+i7/I388sbpL6asp9KfbRq\ndQDCKQUAFL2s9RlGlHgQN3UzhjmePPYqH4b1ByjsEQ9n9kXa5lXjG/XPBa9Ua5uHfKX1H6S+ogDP\nBV3ajE1jBFCgXM8iWR8THZTtZjRLOF/mm1F3m8yVw4k/z9dU6GbOAJy6MkDE82WxL4I8YMMYFOzf\nirEPTVRwN5fLfBc8Kk6DAIrnatMNu3WzsgHIF494CcGm0olSX922N6U+gJaqYXAjjUSRSYlrVyJY\nHxMdlJXRrO1iFGB3aHuttnPgTsTtecWnWoC+7kA/MsUwDAHOuQuAm4EUcLv3/nvv+v0vAdeRXQ63\nA5/y3r8+6Bc6QqlkJ5/g5zJf+40yVR9Ttm3RCsXrz3PzxO2E0TfSkHeItFMtwzhiQqyPiQ7Kuihg\nq7B1U908/e5QebO2c+DUsvVSX4AmS8g731pQZhgHZ5C7SznnUmSHapwLvAk855xb5L1ftc/LXgTq\nvfdtzrk/A34AwqMb44B0UsgmdC17i37we5krR5e4lrtgktZ36kZ9Nskvaj8m9Z3O01Lfv1RfLfUB\nfCxAnZphHDKRrI+JDsr2UMoznC7znbntBZkrR0GF1rezTHtSVrdRH4hu/7H2dHBUq7XEN4yDMvgt\nf98PrPPebwBwzt0NXAz0LTre+8f3ef0fAP3TntEvBXQxTrnzJk4NBP0cyrHPi9eLOq0OoIHJeqmQ\nj/CQ3GndF42BI+6+GMH6mOh/RaPYwwd4UicUB1CA/A6uYqbUd9rntHPUAMY+nPxF0TCMI2YCvKP9\n7ZvA3AO8/tPAb4NekfEOHD0Us1cnXK1T5Ri7TLxeiB+8VtccqxWCtmt0AF9jgPlAaZttYwyQLvSH\nBmLGOOf2nTlym/c+15wiyPqY6KCsmTJ+wwKZ77Sv6QMUbtLq5lYt0woDzJ3hDbEvguJMwwiOfifw\nQAvOgHDOXU22d988yZUZh0QLo3mEs2W+eefpR6g0zdU2gKh4TNtGe2qzvgTykbJzpL4StM1SKtkh\n9QGklacexoggX9kuMcxJ2Q7v/RH3pB3I+pjooOxo3tbO0xDnogOo+3IUiz982abVAfoasFKxzzCM\nQ+FgC85meEfB0sTer70D59w5wDeBed572y4fRI5iD2fwjE4YYIOs4mXxLCJxk4qGMm1nQ8iO81FS\nKD6FWsFJUh/ANNbJnUbcdCtP+QefIOtjooOytO9gWoew8UWINDlx04spa8WdqnQ14H1svkibBzph\nRZPUZxhRMvg5888Bdc65WrKLzRXAVfu+wDl3EnArcIH3PsQWkHEA8ujhKFp0Qv0cZf17VvwuW8Ec\nrRB9UPYo2pO3qYgbihEmJdKImy7lP+ZI1sdEB2W7XDkPpHWpGVdX/0rmyuHFLeedOt0wwBiACY3i\nIMrGmxjGoTGIi473vts5dwOwhGwLiH/13r/inPsOsNx7vwj4O+Ao4D7nHMAb3vuLBu8qRzZ7KWal\ncO7JmW/om2HJ26+LfaOUQW0vWxkn9X2Qxw/+ogEgrUPsRR2IGvHj8FphBOtjooOyUvZoW8EGSJNz\n4syM587Qzkw5dXGA4dHqDTFxCqhhGBq894vhncMivfff3ufn2i18Y0CU0Eo9z+uEIc46xZ/vrVfm\nSX3KkQI5dlEu9amvsSPAoLLJbJQ7jbjpQftvebAJsT4mOiiTc0sApzglcme1eAXTHTT+D9pGUHZS\nZhiHgge6hvoijCTRylEsO2DDr4Fx6mkBNvHEm6Glv+6R+sqv3CX1AewUR6K/4wNSXzlvS30Q5sTR\niJtuZQgSyfqY6KAs3d3NlG3CGivdHOo+NswaL/VNpkHqQ3ySB7D6am0L4aogE64NIwkI/wF6UDar\nMoY/hXRq26Uv1an60CZ/0HWZ1qechZrj/Wi7KE8T14D9hD+X+gA+yBNypxE3KXX3xQjWx0QHZV35\nKTZX6SqPJ3ToG0rUNmkbc7gNUl2QmrIZjeIWwgGu0TAMI3aKaGc6a3RCfc8Lebp7wVta3/yyJ7RC\nYAnnS30nsULq6wyQvqiepWbET4GNUdiPRAdl7aRZxzSZb8Lx+hksbaXanNhf1l918BcNgM9M+oXU\nB7C5Stt9sdD+YRrRIhycO/jdpYyEkyGPFkbphCHGk6gzVLTLT5BgYq74pExdo/YF/lHqM4whJ5L1\nMdFBWalvo75j+cFfeKgE2AXcldZ+WM4XpwDcVXWx1Adw5doHpb7tdUdJfYZhGCOBkkw7c5qFdWD6\nAxR5o49XqrSDMkPUV0kDZWCcOMU/RKMPqykzBkoKbX1oDCQ6KGtxo3g8farMd+EfHpO5cpQv0BYJ\nF6e1w6OvXK0NoAD5bsTYRuFpgmHESiQ7gYaOXaky/qPsLJnvstX/KXP1IQ7KTmjX5vivrdMPjy5H\n+1yg9hWjfc4wjCEnkvUx0UFZMXuZzUqdcIFOlePx9HypbyarpL6jq8TDqIGWsgKpryMVYnvWMJKA\npS8a4UiR0Z5QBGiGpU6J/O+6M6U+abOBXvZSLPWtYqbUtyPAHBp13ZsRP53s1MkiWR8THZSlyHB0\nRrhDdJ9OlWPuPG2dmrqVrgvwJh29Utt3tKs2gj6mhmEYg0wxe5mjfBj+tU6VY8NCbYfiuRltvdbW\nlHbQM8AD/G+pr1bclfkt+bBRuJT75U4jbtIh2oMPc5IdlHV6RjcKH9gDjGAZ06RNvWup0NZXdQU4\nhGqcM1bqm9S0XeozjGiJYCfQ0NFFAY3CB+zxH2+WuXJM2aTN1lhbo003XM9UqQ/0Qc/pPCP1STOQ\nelG+D42RQRe7tcII1sdEB2VdhSm21OiClPFz9AtOY4W2FZS6VW1jmTaAAsiQkvrU99AwkoNwDEck\n6RmGjh7yaFOmyr2qU+V4/WrtGiT9fgkTTFQq07KA27lO6lN3cwT4Nv+v3GnEjXx4dATrY6KDMhAH\nAAHmYXVQKPWpA57XmC71AUxlndSnvoeGYRgjAUcPaeVIkQCHHdISBODhlHYG2Ck8L/WBfiD1dF6T\n+hqYLPUBpGJ4IjYGFYcf6ktIHIkOyvLoYVRGWMQ8Q6fKMWWbNjXjwarzpD5116YQlLB3qC/BMJJP\nJDuBho7S7nbqm4R5+cfrVDm+nvqe1PdFfij1qQOo4YC6cQjASt4ndxpxs1dZUxTJ+pjooKyTQjam\nJsl8Jz6/VubK0XqDdni0epBliMGY6tQHOykzDMMYOJ35+bxRcbTMd+yr+vref6r6stS3+3ht999N\nqRqpD/QdHSvZIfWpa9QAZvNHudOIm2LbkN+PRAdlxW0dnLhCGEhp07yDoA54QgzGVJ++pTMdUp9h\nRIkHrFGpsQ895LGXEp0wRHlvmVb3u5RuLhuE2bhU16llxI9qIZqbSN+HxoigB+GhRiTrY6KDsr0l\naV6aIzwpq9KflC1P10t96jll5a365iYtpdoOkTanzIgX4SrhIcBIJWMYU+g7qekQBhUBOhQzW6ub\nWaNdI5egrVEDfW24mrYAAdRwKJUwkoX0RDmS9THRQVlRpoOZzcJASh+fMG+Zdk7Z6rnHSn0NpZOl\nPoDTtr0k9flEvwsNwzCSyU5Xyc/SF8p8f1Z1p8zVh7jBVk2zto77/rJLpT7QD1IuoU3q29ShT9lc\nmRZH30b07GXNUF9C4kj047DbCwXKnbszha4cW7U69Q7bZPHQSdDPPivYpvUZRrREUMhs6BjbtZM/\naxQGUkt1qj6u1OrU68V1ZbdrhcB6pkl96rKG6Wn9w/D5LJE7jbi5Q31SEsH6eMRBmXMuBSwHNnvv\nL3TO1QJ3A5XA88A13vtO51wa+BlwCtnqrsu99w0HcncdlWLLGcI5ZVcGOCq7T6vbyjipTz0vBWB9\nmfY0r7zM0h6MWBF+5kTSXWqkEXKNxKHdWj1N6OpFvYm3s05bpPa+AIOU1R2FF7NA6usQz0MFKBaf\n5hnxk0ePThbJ+qj4OL+R7MjJ0b2//j7wQ+/93c65W4BPA//c+9+3vffTnHNX9L7u8gNeXCZDZbPw\noSZAu9+X5tZJfeqTsuWcIvUB1IvnuqiHgRpGcgiwEWQMN4KtkXvzC3mlaqLsQk+o2yBz5ciI83HU\n60WI9vDqDJXZ4sBxpbrQD9iFrguoMTLIWB3ifhzRx6VzbiLwYeC7wJeccw74EHBV70vuBP6a7IJz\nce/PAe4Hfuycc977fqfHdaXyaSwTtvut0rf7PbFR2zzkseoxUt85rY9JfQArS2dJfeph1IYRJZHs\nBI4kQq+Rxe2dnLBaGEi9qlPlKDpD66vp1taU7SpbL/WBPugZhXBeK7AJfU3ZOHUthxE9BepGWBGs\nj0e6h/Uj4KvAqN5fVwK7vPe5W/MmMKH35xMg23vWe9/tnGvuff07BnA4564HrgeYNBEmNQkDqU/q\nVH00anWV1dp5JOtLp0h9oD/NU7f7NQzDSAhh18hqkH4ch3ioWS72iYO8ma36SLS8VDuKZgUnSX3X\n8HOpD6CytUnuNOImX5i9GAuH/TTsnLsQ2Oa9f945N191Qd7724DbAOrnOO+Ui8RPha4cN2p1LX1r\nt4ZqddQILEc8BiCjbXFsGFESyRyWkcJgrJGnnOx8u3AkVpE6gAL4nFb3RJm2Y1eITI3qVu1p3s5S\n/Sw1NepROUb8ZPKEdYiRrI9HckRxBnCRc24BUEQ2X/5moNw5l9+7EzgR2Nz7+s1ADfCmcy6f7EjJ\nA3ah2J1/FP9dNecILvFdF/zAUzJXjhLxiJO99dr5IeM69K0Nr912r9TnS6U6w4iTSOawjCCCr5Fu\nLxQpOxRXCl05xEvQcVXazoHSbJxenq44WerbI96s7aBQ6gNIZzrkTiNuHP1mZg+cSNbHww7KvPff\nAL4B0LsL+BXv/Z865+4DLiXbXepa4MHe/2VR769/3/v7jx0oVx5gVGYP85t1gVSBtidHFnEdfyPC\nbU9gU1qfOz6j+XWpz0lthmEYQ89grJGkgVrhRYd4rq7S6jrFnQN3VOhPeM5c8YLU1zSnSOprkL5p\nshS2R3BMYQwqeZa+uB8hinm+BtztnPsb4EXgjt6v3wH83Dm3DmgCrjiYSD6nLMR3e5lWd3mr9hSq\nSNuHBIBX5mjr1E7YqO/4ZRhREkEhs6FbI2kD6Zxi/QQVHqw6T+rrFJ/y3I9+ePQ91QulvmeZK/UV\n0in1ATxZGmIQrBEzLXnaIesxrI+SMMV7/wTwRO/PNwDvf4/XtDPQEKYTeOOILy8s39bq0n+r9aEM\nanuZOUkcRP1BqzOMKImku9RIJNga6UB6cBRg4/KipoelvuUV2u6/9yxdKPUBMEmrKxe3Dv8hX5T6\nAO5pXCh3GnEzWnm4Gsn6mOy2dx5tOoW+58X/TJ4R8UbFWKnv2Hp9vrx7VK40DMMwBkoe2Wo1FdoD\nGQBcu9ZXpW69rs0MzCJ+sjpt6UtS32fn3Sr1AdmzXcMYCBEEUWqSHZTlo81H12ZRZBEXRh+7XBxE\nBVhwusT3sUB8gm0YURJJdylDSD5QIfSF2LgUNw6sLhWvkfryKrq0ZW8UiJthfWj177VCSPrTpJFE\nlA0FIlkfk/3PqAvtIhHi4V+8s/hcvTY149S/1ecvFqiDWwvKDMMwBsyu9Gh+Xacb3PXRZb+VuXL8\nyy1XS30AYjUhAAAgAElEQVRn84jUN+Vpbft6gAJxx8lXLtHWcR/XrK/j3lVmLfGNgdFdJGyJHwnJ\nDsqKgdlCn747POJmiZz6sjiIukarA7TDSkE+DNQwoiSSlr+GjrKe3SxoFQZSV+pUOT5z3y+kvrsu\nu1jqm7LtwYO/aKAcr9V9R1y8PqdMvxP6jcYfyZ1G3OSra8oiWB+THZR1AMrugeLWvAC0in3ibonq\nHTaATWjb7F+wdKnUZxhREkkhs6GjI6+QhtJjZL4Zy7XjTgD5e/bKTdogavsl+hOesWv3SH3/kvmM\n1Pd2qlzqA+iyeaPGAPF5ShlRrI/JDsqKAGU2X5nQlWOe2CcOHE94QJ+m0HaJdsC1+rTRMAxjJFDU\n0cmMjcJAKkRNmfr0TdznY2y7NoACaKrTFnPvZIzUNy4jvolAgTX6MAaIiyCIUpPooGxvSSGvzJko\n851QGmAe1tNinzrIqxf7gMk0aIUhgmXDiBFbxIx98PnQJWz0UXCJztWH+j2rPtgSNyIBeKtOu9NY\nk9Fe5G9SC6Q+gCvTAdJAjbhRnpRBFOtjooMycHQLC5j+6zfqiAcu+I449U48vHNpzX7jcI6YGvHw\nuJ1zxC0sDSMxCE8xIukuZehoSR3Fo2WnyHwXnBkglVxdgyyea7n7mgKtEDhho3YDuKlWe/LWEKDl\npA8xWsCIG+u+uB+JDsqKOzs4cZOuyOrEz4sLtkDfSEM5lw2Yt/FZrRC4ufZ6qe/Gp2+T+gzDMEYC\nKbqpVO7k/Vin6kPZrAtgoVY3eqv+SW5D7XipTx1EzWWZ1AfQUqYPbo24yeRHcLQlJtFBWabQsbta\nd4mjPxwgjFY3+hB3Iny9VjuMGvSNPqjT6gwjSiLpLmXoyMNTqNzJUwdQIB950i5uKNFSqm/0kRL/\nQ61kh9T3fIC6hpmpVXKnETc9COs5I1kfEx2UdZNiR0qX2jb6D/p5JNJGJOgLhEOkKah32bZUWVGZ\nESvNOlUk3aUMHRlStDBKJ1TPoAR4SqsrEteArZmh616Zo0ZcqPYEH5T6OiiU+kI5jbjxyvzFSNbH\nRAdlhZluapqFgVSIIua3tLqS1napL8QuoLrRR3mr8MHVMAxjhNBKKc8yV+Y78+UXZK4+1NkkL2p1\n02rXa4XA4rS2kcZ01kh9ywOclGXkxYNG7PihvoAEkuigLJNytJSlZb6KcdqAB5AflxbdpfVNv+41\nrRDYhXbGSZHFZIZxcCLZCTR0jKaZ81miE4Zoia9uACF+akl192iFwPz0E1LfCuZIffISBGBMRtyl\nzIiefOUDdCTrY6KDsmbKeYhzZb5rt94rc/WhfhOIM/lSAd6l8h2xCPKADcMwBptu8tmhnGG1M8DY\nGHXNsLhZ7/rSKVohUM4uqa9RPMxzJvr6r62pcXKnETfd6qGDEZDooOwoWjiL3+mEq3WqPtSDjwPM\nFUs67cI5O4YRLZG0/DV05NFDCW06oXo9A9gm9onXi8kdDVohcE/6cqlvKtoUyx/yl1IfoH1WM0YI\nwgTGSNbHRAdleXjSdOqEukzI/0Gd7iG+RvWOHcBO5c4skIrgyNkwBgU7VTb2Qd59McDUGHlHR/FT\nS0N6slYIzGal1DccmmjspWSoL8EYZvSop0dHsD4mOigr6MwwYVOTTihuNw/A82KfeE6Zsgg8R5V4\n67NA+FdsGIYxUsgjwyhadMKzdao+Fml1rQu1D3LHBCike43pUt/3+brUdx23S32Adl6eMSKQ1pRF\nQqKDstbCIp6rmSbznfr9l2WuPi4S+97Q6lIz9G/6Noqlvi5LXzSMgxNJIbOhYwNTuJLbZL5lzJe5\n+hB3X7w/fanUd+2KALXm2r4cfE8clN2P9h6CfgyAET9dbNTJIlkfEx2UOcRDGE/TqfpQvwnERdFV\nAQop1YMnZ5bZ0EkjVoQdXyNZdAwdtTRwJ9fqhAEaFFOl1akHKf/XnHlSH8Ac8cTsneLuJj/jE1If\nwKXcL3caceMQdj6NZH1MeFDmtd0DQ7T7Vc9gEQdl1epBasCz4u6LRzeFeBIwDMOIG0ePtqZst07V\nh7gl/lvibiQf4SGpD2CNOH2xEe2Aa2nKay/dNqfMGDDC4dGRkOigDLw25zREZ0PxIEt1iu0y3q8V\nBqClrGCoL8EwAiFsBxVJdylDRzf50sZLU1JbZK4+Rmt1/5v/kPrUczcB6juWS32fS98i9Z3OM1If\nwBirKTMGSL7y0CWS9THhQZnT7r6E6Cx1vNgnrq8K8eGrXsQK2yP4l2QYhjHIpOlkMg06YYhGH+Ke\nEqNa92iFpVod6AdST04La2+A9ehq9XOMyuhP34y4yVOmL0ZCooMy9S4g5wWIytQpkeJ0SHWqB8Db\n4qCsvFTftt8wkoG4psyaVRn74HFklBuXT+tUfYjHvBQ1a30NpeJp1EBbqbYZ1ue4Veq7neukPoCW\n1Ci504ibHoT/mCNZHxMdlKXpYCrrdEL1EEuAMq1uS61WuCxAS/yrWu+S+tpKxUUHhhErERQyG1qU\nQVnT1frP4oqPi2uGtT00ggzMPnbbdqnv11V/IvUVKweO91KonClrjAiccng0RLE+JjooU7N97lFy\n59hvalMpxi/UbgOeX7dE6gNYXnqy1DeN9VKfYSQHa2JjhGMnlfyMq2S+bzz2I5mrD/U/AXE2yfIA\nxeZT069LfVvFLSw71ceX6JuRGPHTpZ4BFQGJDso8Tvrhcexq7e4VIB/2/FKdtv1ibaZB6gNYkTpJ\n6rOgzDAOgUha/ho6StmjrRueoVP18QOx71Wtboy4xT7AzjJtxssHeFLqC5FBE6KjoxE3ecp8w0jW\nx0QHZRlStKDLU948Qz+leMLCJqnvxMXaurfXF4yV+gDq0XaWktZEGEasRNJdytDRTjGrhN2m5m16\nVubK4cVjXpy4pixEK/fx92kv8qHLPiL1/ZHZUh+EaSpmxI0nTymLYn1MdFDWTpo1HCfznbxUvMUG\n+jll4gXn2BX608FxdVpn0ctSnWEYxoiggC7tLMoAO83uZrFQPOv5wm2PaYWAeNazfAxAOfrmWjNW\na1M2jfgpsuz+/Uh0UFZEB9N5TebbMk/clQMYv1EcRWkPoYKMAUjVioU1Yp9hxEgk3aUMHWnamSZs\nhrX5jADZJA9os0nULex3V+rnZI5Oabfs/5G/kPo2BVh05894Qu404qa7SNhwJpL1MdFBmZrxi8UB\nFOhnnIiDqP++6UytEJibWSb1FTRFcOZsGIYxyDggJXwSqWwVB1AAp2h1r8/VpuQfe1/ya82v4B6p\nrzFAy8lUDAU9xiAj7r4YAYkOygrppIZNMl/TggDtfh8Wn7+KPyvPbXxKKwT9boS+EZRhxEckhcyG\njk4KaWCyzDe1O0AK2jit7tibxUHUaVodIM94OWHjBqlvXa1+eHTFy5aLZgyM/L1CWSTrY6KDsr0U\ns1JYkPqhp38vc/WxUewbLfYtFvuAzddpU1wmLA6wO2sYMRLBomPocHjpfKiWsgAbl2u1D+tNN2qv\nseKBAMGEtoM9uydpUyxnskrqA9h9vD4N1IibTLF4QYtgfUx0UFZEO9NZoxOKi28BuETsU49tEO9S\nAkxYKw6idM3DDMMwRgw95NFGscwXotYo/7oGqW8vJVIfSwMEZeI1bVSzNsV/R4X+6bUjZSkvxsDw\nMRSBiUl0UNZOmnXojtkn3BWg3e+NWp8Tn2x1ia8PoED9eW6f5YZxcCJp+WvoyJBiF0fLfBc+re9E\n2HSG9mRrwj9oNwU3/yhAc5O/1F6jE6+5o9ijFQKjWvVOI25SPUJZJOtjooOyFD3agYTiVroAbqlY\neI1WV/C01gfoTxwjOHI2jOBE0l3K0NFFAduEuXK7T9OnoI1qFp9EiWvAJmwLkD4vfta4r+pCqe/8\nzBKpzzCGnEjWx0QHZXn0UIKwZaa6lTvoA4qHtbr2q7U+gJWls6S+U7fZoDLDMIyBkqaDyTTIfKPX\nBthqVmdCqNfx+8Q+gEat7rLu/9QKxTVvgL4TtRE9znrD7Efig7JiYVDWpc9SoCDh8URa3JoXYGa+\ntkjYJ/pdaBgJIZLuUoaObE2ZrsbqqRkny1w5aoVBI8CE+8QnWwEyaPi52Fen1TXNCdDQZZs9YRsD\npFDoimR9TPTjcDf57GSMzJcu03Wp6uMMrW58vnaW2hsV2pkuAGlhty+AFqspM6IlwGxEw+iljRJW\nMEfmK+dtmSuHcmMVkDfRCNE1cPQk7Ynjhjnjpb5MgEe/ziphqYkxIujKtzrEd5PooCxFRlpTVtms\nf0DqLMrTCsVzylZwklYIrGeq1HcVv5T6DCNKItkJNHSoOxRfuDZAo4867anMf886U+o7964AszzF\nTSynNG6R+q6qvkPqA7iz+dNypxE3+coasEjWx0QHZZuYyP/Dd2W+/+i4SubKUbBJ2T4G+d9IGn3+\n4vtZJvVlSEl9hhElkXSXMnSkyFDOLpnvqTp9+uJ6YQdlgMtb75X6nrtSWyMNcOoD2rqGF6q1x4OF\nAZ4Lni87Ue404qYt9ZpOFsn6eNghgHOuBvgZ2UlYHrjNe3+zc64CuAeYDDQAH/fev+2cc8DNwAKg\nDVjovX/hQH/GGHbyCXlytpbVs46V+mY8/brUF4Jf8qdS3yf4mdRnGMnBBqOPVAZjjcynm0p2yq65\nM8B8kmu3aYOo1jJtdsr0jHAWao4yre7k5a9Kfel6fSnH7I6VcqcRN8VefKgRAUdyLtMNfNl7/4Jz\nbhTwvHPuv4GFwKPe++85574OfB34GvAnZMtV64C5wD/3/rdf1EXML1TppxSn1Oel4oLeEFzOPUN9\nCYYxMomg5e8IIvgaqT4pa2CyzJWjSxznrUzPlvpOe+wlqQ+Qd5z85/prpb514hIEgFXpmXKnETd7\n3TqtMIL18bCDMu/9W8BbvT9vcc69CkwALgbm977sTuAJsgvOxcDPvPce+INzrtw5d0yvp5+L62Yc\nWw/3Evdjeod+R2xdWvvhpu5EeH6TepAaLKo4T+qrZ7nUZxjR4of6AoxDZTDWSEePNEV9k7oYClhV\npt1pVDcOWf0hbbYLwIyXtRkvH+BJqW8Tk6Q+CJMSacRNnnpBi2B9lIQAzrnJwEnAMmDcPovIFrKp\nG5BdjDbt87+92fu1dyw4zrnrgesBxkwqZhfliksEoDEt7qIBVNPvenlYqLslvqXuHAKsQrsjNpmN\nUp9hJAdLXzTCrZHjJ+XTITyWmY0+Ba1BPFisU9pHGxZ0LJb6AHn6Yk1m08FfNAAeSn1E6gO4iEVy\npxE3PYgb5UXAEQdlzrmjgF8Bf+m9351Ni8/ivffOuQHFrt7724DbAE6sz/dnCXeI2iiWuXKMatW2\n9BzbrPU9W33A7JfDQt08RJl+YxiGkSRCrpHH1xd7ZaMkZWZKjkp2SH3qIC/Vra9r6Rqt9f0mtUDq\nC1GCUCgelWPEj4vhaEvMEQVlzrkCsovNv3vvf9375a25lAvn3DHAtt6vb+adjWIn9n6tXzyODuGu\nWHWrtq0swKbSiVJfYak24DklQGrgEs6X+v63dV80DCNCQq+R6rprpSvHGHFQVk2j1NdYqp0BBkjn\nq4bgEc6RO+fzuNxpxI2dlO3PkXRfdMAdwKve+/+7z28tAq4Fvtf73wf3+foNzrm7yRYvNx8oVx6y\nrdKV6Yu7SnWuHDVo0woqNrVLfRtq9AuOOtAL0fHLMIwjxzl3AdmOgCngdu/99971+x8AfgS8D7jC\ne3//4F9lMhmMNdKTx15hBsgDXCJz5fgkP5X6Whgl9U2mQeoD/Zq2kvdJfY0cI/UB7BH/vRjxM9yD\nshDr45GclJ0BXAOsdM6t6P3aTWQXmnudc58GXgc+3vt7i8m2+l1Htt3vJ4/gzz4sStgrd/6KS6W+\nq6q0g5QbA9SUTUc4WwL9ImsYxpHjnEsBPwHOJVvf9JxzbpH3ftU+L3uDbDfBrwz+FSae4GtkN/ns\nEJ7K/A3fkrlyqDs6qtPdSzq0jUMAMmlt9ketuO56XN/hrA7laAZjZCDvXj6IhFofj6T74lOA6+e3\nz36P13vg8wP5MzKkpA/sewOkZsxhxcFfNACeTJ8l9YXI816JtiWx+h4ahiHh/cA67/0GgN4TnIuB\nvkXHe9/Q+3s2cOZdDMYa6fDSGt8QddfqbBL1Nf4w/UWpD2AB2uYhx4hTNp9acq7UB9B2vv75yoib\nYX5SFmR9FDdg15IiwyhaZD51gTDAguaHpb6dZdq2TSEWWbUzRHG5YcSHB7qUwjHOuX1zkW/rbSKR\n4726Aeo7BxmHjcNLN96UTUNyqE/KlM8EAMpmYjnUdW+3c53UV3e+fjbbcD71MIaG/nasDg/5+ggH\nXiODrI+JDsrUhJiHtb5MO+NEnQKwi6OlPtCnj4R4EDCM+PCgffDZ4b2vVwqNwUW9cbmYD8tcOebw\notSnTNcE+EDTs1IfQEtZgdQ3M7Xq4C8aACHm0VltuDFQvDQsk6+PMARrZKKDMo8jI7zEZzhd5sqh\nnBEDsIDfSH0hdq+UfydgNWWGkVAG3A3QGFyyJ2W69EX1CQ/A+zLa2Wc7UpVSX2NFhdQHSBuUgf65\nIERQpk5TNeJnmI9RCLI+Jjoo66JA2iXoI63/KXPlWF56stSnHvasHm4N+k5QM9HuAhpGnARJzzgQ\nzwF1zrlasovNFcBVg3kBxoHxaDfJlAFejraUttZoW9+sbQ3HsUbqA1iMdq6YevP3IfTDo8/hEbnT\niJsOadZVHOtjooOyvRSxipky365SfSrf6Twt9Y3r0HZF2pHW7ioC7ETrtJMywzgUgqRn9P+ned/t\nnLsBWEK25e+/eu9fcc59B1juvV/knDsVeAA4GviIc+7/eO9PGLSLHOFkyJefyqhRN4ZS15RVrNCO\noQFYMEfb6EM5rxXgL9b/i9QHUDh1WJ96GEOAQ9kfKo71MdFBWTcF0l2xfDIyVw51EFW6U9vErKVa\n/0GpnlN2tLhGzTAMDd77xfDOVnLe+2/v8/PnyKZtGENAD07aeEmdBQHwFf5O6lN3UV47R//2rdv0\nptT365o/kfp+NPVzUh9Yow9j4GgbfQw+IdbHRAdlo2iRTolXN6gAaEhPlvqqqrVBXohTqCf5gNR3\nPkukPsOIk0FPzzASjnrj8jr0JygV27QnUc9VTZP6lgVoKNpQo02JnC5OsbyVz0p9YKNtjIHj5bbh\nvz4mOihL08E01st8xeiHRI5r3S71pcUp/WNb92iFQE2NtqB3sngwpmEYxkjgbY7mfi6V+T7CIpkr\nx+oqbYdidTOSGzbeIfUBrK3Vnr5NbtaevJ1dpq//quENudOIm2He6CMIiQ7K8umWtogfFSBA2VSq\n/fCta9R++K6u0y6IoN+1s5oyI16UpwRx7AQaOtTdF0PM8qxkh9Q3ubVJ6nuptk7qAzhx2Vqp77m5\ns6S+dICH4RDjd4y4yQzvRh9BSHRQlm2Jr5thpQ6gAOo2aoMotLOjqenQt6n9TvrbB3/RAPga35P6\nDCNerG7D+B8K6ZS2IlcHUAAnLtYGKA8uOE/qm8syqQ9g+9yjpL4ScZaPemMVrKbMGDhOnMAYw/qY\n6KBMX8Ss7QIFUFirzTdUt8SfmdG3m7+U+6U+9fdsGMlhw1BfgGEcMuPQ1jQDbF+gDVCUzwQA47c1\nS32gT9ncKh4D8HaAjp3KrCZjZNAz7Ft96El0UOaAlLBjotKV49hN2pqyY9Na3x+qTpT6QB/oqefY\nGEacxJGeYegoZi/vQzecWVmfluOzzdqarStbH5T6tlSL01OAGUtfl/qWzdM2I3mEc6Q+gM9yi9xp\nxI0nT2qLYX1MdFDmcXQKhyY+wtkyV46L2x/WCsWbdsdV6dMUNqVqDv6iARCiK6ZhxMfgzmExkk+a\nDqYKm2E1MFnmCsXaam0Zgjo1EOBf510p9X1q7V1S3yN1+qDMMIaWONbHRAdlaubyrNy5pU67y7aG\n6VLfB5r03/OolDYlq7FsrNRnGIYxEkjRLd3Umok+3b1AXNacKdPVmQNMeEzbOATgUzXaIOq+ugul\nvhA1ZSXslTuNuMmTDo+Og0QHZRlS0s58IT6Ixjdqj7Yy1eukvjUV+u6L6sDRWuIb8aJMR44jPcPQ\nkU3x1+0OTw3Q1ry1TpmiFGC0jb75orbpKkhPQwEWcZHUB3AV/y53GnGjbQ4Tx/qY6KCsm3y2USXz\ntaGvXTp1xctS34SN4l27Wv0u4CPV2tSH2fxR6jMMwxgJOHqk7c1DtMRPpbW13Pni2vAtNfqaMvVm\n7czWV6W+htLJUh+Eqdk34sbafOxPooOyHvKkgdRZPClz9VEq9uli0CwBDqFmV+sKywFqmrdIfYYR\nJ3HkzBvJRdleP8exi7XNq5oWFEl9FcvFx1rAnfUfl/oaxR2KlbPtcuyxeaPGAMlImyjEsT4mPChz\ndFAo84UoYh5bqT0paxd3h89M0qaOgH4eidWUGfFi6YtGSLSzPJ9gvsyVY/0CbaA3v3Wp1BdiIsu1\ny+6V+prmagPRTWibdUF2Zp5hDIQ86ZyyONbHhAdlKfYKT8pG0SJz5dgyS5v6MK5Jm/bwx9KpUh/o\nC3pth82IF+0pgWHsS4YUu4QzpybTIHPlUJ++7SrVrrmpUn3a3dhte6S+56mX+tSz3gDp+9AYGXQH\nGFY/3El0UJZHj3T3JUTr9fFPi3vYi4uOQ3zPD4mLhC/hAanPMOIkjvQMQ0cnhdLUtjmskLlyzNio\nndnVWq3N/ii9K0AHOPGTVdWcrVKfOsgDqKZR7jTiplB6shXH+pj4oEx5uhWkEFVcU7a06v1S3zOc\nLvUBnMSLUt8qZkp9hpEcnhK64kjPMHSoNy5DrBcnd2ubVKS6xUGUvs8HbNPqTrx9rdT3N9d9S+oD\nmNCobypmxE2BdDmLY31MdFCWHR6tqykr6dAPiVRvLM7r0M4Vm1eqn1O2dpZ2eGdd45tSn2EYxkgg\nRYZy3pb5xgRIJ3qh7nipT92JkDlaHQCtWt1zs2ZJfSFKOV6o1v49G/HTVmDjkN5NooOyFBnph8ea\ntHa+FsDJ1eIFQjuXOcgMFvkHeqLfhYaRFOJIzzB05NPNGHbKfCFqytTt4ZuqtU0viu7Sd19Ur7tr\nZmmfXULUf13OPXKnETfa5jBxrI+JfxxWphwOi0JU8d/ID+pv0ArRz7K5pMpqyoxYUaYvGsY7yaNH\nOky5sllcIw3wf7W6ivPEQZR6DA3Aw1rd1dW/kvrOPuMhqQ/ghqV3yJ1G3OTrD2yHPYkOyhxeGpTN\nZJXM1cejYt88re6ra3+sFQLy0jxdR2fDiJg4cuaN5FIgTrsD+Ju//7LU961b/0Hqa79aqgOgSP1k\nJT55u4T/0AoBP1uuNGJH2gQ0jvUx0UHZXoqlTSDOav69zNWHdkwZqGtlvy32gQVRhjFkDP/0DEOJ\nJ1+4SxaiLuhbG7VBlJqiawJIPyr23azVpb6r/xxxK+VKI3a005WIYX1MdFCWpoPJ6AoBC0I0BxJ3\nX5R3lRWXvAG0zhO3JF4ZoCWxYRhG5GTI521hWv56pslcOU7+mngROk+rE094ySL+ll/4rjZY/jc+\nKfUB/NmUO+VOI3LSQ30BySPRQVkBXVTzlsznA7S+deo31UKx7z6xDyjFgijDGHziSM8wdKiHR1+S\nCVDfe6PYd4vYF6AZlrD3CgAnL9ZGeWcteFLqA0Dbtd8YCXQoZXGsj4kOygrppIZNMl9LWYHMlWN0\nk/hNoA6ixIsDAOpacPswN4xDII5Fx9DRTT47GSPz/S51lsyV48LWx7RC8Xqx8y6tD6BSHeiJfbMX\nBMg1tNnRxkCxOWX7keigLM/3SGeLlS7Rn/D8cbHWN06rY1yIGSzi2WwWlBmGYQycVkpZxlyZ758a\ntU05AFCXlE3R6ipv0voA+IzYJw7KGpisFQKM1iuNyNFWwkRBooMy1wOF7cJAapJOlUN9A8fp1tcs\nIeY5qu9jot+FhpEU4pjDYugoop3prNEJl+lUObrEzgLtRBZ9sy6As8U+cQZN8UW6ze4+xAeixghA\n2hI/jvVxZD0OS/NXs8xcKBaqa9SeF/sA6gM4DcMwjAGRpoOprBvqyzggBd/X+t78nNYnz/wAJi4U\nCxdodePYphUChMjKMeLmwaG+gOSR6KAsk3K0lOmilPVz9Z2lTv22eJtN3SEyxMaBOnfcctEN4xCI\nI2fe0JFHDyXKvtIhxp0s1+omfkTr40yxD+j6W62v4LtaX5CZrcP/kMIYbLxaNvzXx0QHZZ489lIi\n8ykHUefYKR4e/bT4Emf58VohMGX1Fqmv6xipzjCSw78pZXGkZxg6PI5OCnVCcb0WwHdv1/q++Xmt\njwADswu+JBaer9WtWzlVKwRO3RQiD9SImk6lLI71MdFBWScFNFIt861husyV47cZbavab4qHPftK\nbQAFwK+1uoI/aH2GYRjGYSA+1QL45m+0vmUf1vrmVml9gD6V76danXLgeB+z9EojcoqH+gKSR6KD\nsh7yaBOelI1jq8yV41y1sF2rcyH+hm8V+6xrk2EcAnGkZxg61GtkkE644g7Fc+dpfagbhwB+kdbn\nxKd5o7QdFrI8rVcakbNHKYtjfUx0UOaAlPQ4Upjm0cu4y7S+N3+g9U3crPUB4pQsrEDYiBfpBkYc\n6RmGjuycskqdMMBn8U5xo49KcckAvxD7ACeuKeObWt3xTwWoKbMGYMZAkW5exLE+JjooU1MV4KSM\nUq1u4hlaX9txWh9AibgTFNoMUMMwjBFBhhQtjBrqyzgg6kHKT8zX+uarBz0DLBH7xOPjnuQDWiFw\n7JJfyZ1G5DQP9QUkj0QHZW2UsIKTZL4bVt8hc+VoE88PKRGnPZRcqfUB8hRLOykzjEMhjvQMQ0eG\nFG9TrhOG6NVwg1Y3X7zmhqiF8ndpfd1/pfWNuzHABrWu/N8YKRQoZXGsj4kOytSdpTbPqJC5ctzT\nqu1h/yVx/j23iH2grzvQ/7UYhmFETz7d2plT4swPADaKfb8U+wI0N3EXaX0FO7W+TIjZB+paPyN+\nxMCJSLYAAAWsSURBVE3jYmDQgzLn3AXAzWQnotzuvf9ef6/tEndfXMZcmSvHl678rdS36B+kOi4K\n0dlQPeA6xIOAYURHHDnzRv8MZH0EcHhp3XW7ut088JujtL6PqR/kQtRCqZuHiFvi/zt/qhUCFzQu\nlTuNyJEebMWxPg5qUOacSwE/Idu08E3gOefcIu/9e1adHsUeTucZ2Z8fZHdInHp3kXq+ibhAOAjX\nDPUFGEYgHlLK4kjPMN6bga6PkF3TdnG07BqKGmWqPi5oz9MKb+zR+kKsP/8m9i3U6r7+xgFj/cPD\nGn0YA0XYODaW9XGwT8reD6zz3m8AcM7dDVwM7z1evoM0DUyW/eGF2kl1WcQ7Ylvqy6S+8QsDVFKq\nR5xYTZlhGMaA1kfItcQXDvsJ8ERQulYbRO3+ibQQhdGPBniQWyj2fVWr65Cnu2AZL8bACXBOMtwZ\n7KBsArBpn1+/Cf3nFHaQZh3TZH/4bP4oc/VxvFY3/j5xEKXrk/I/vCb2BZhjaRjxEUd6htEvA1of\nIbtGrheukRtqx8tcOaY8vUXqG71BHES9odUB+gBFvPkbYmZrsjsUGPETx/qYuH9Gzrnrget7f7nn\nn92X1xzi/zoG2BHmqvrnzwf7Dzw8huTeDBPs3vSP3Zv+OdR7c6zuj3xrCfz1GJ3P/m6HI+9eI+9y\nnz6UNfKQ3q/ipoHDhRH4OXfIDcpG4L05ZOze9E8M6yMMwd/vYAdlm4GafX49sfdrfXjvbwNuG6jY\nObfce29Zze+B3Zv+sXvTP3Zv+mco7o33/oLB/POMQeeg6yMc3hpp/5b7x+5N/9i96R+7N/1j6+Ph\nI67APSjPAXXOuVrnXCFwBeKZ3oZhGIYxDLH10TAMYwQzqCdl3vtu59wNZOfdp4B/9d6/MpjXYBiG\nYRhJw9ZHwzCMkc2g15R57xcD6hHJcBgpjyMIuzf9Y/emf+ze9I/dG0OOrY9Dgt2b/rF70z92b/rH\n7s1h4rz3Q30NhmEYhmEYhmEYI5bBrikzDMMwDMMwDMMw9iHxQZlz7gLn3Brn3Drn3Nff4/fTzrl7\nen9/mXNucu/Xz3XOPe+cW9n73w8N9rWHxu5N/9i96R+7N/1j98YYbth7tn/s3vSP3Zv+sXvTP3Zv\nAuO9T+wPssXO64EpQCHwEjDzXa/5c+CW3p9fAdzT+/OTgOren88CNg/192P3xu7NUP+we2P3xn7E\n88Pes3Zv7N7YvbF7E8+PIb+Ag7wB/hewZJ9ffwP4xrteswT4X70/zyc77M296zWO7LTE9FB/T3Zv\n7N7YvUnmD7s39mO4/bD3rN0buzd2b+zexPMj6emLE4BN+/z6zd6vvedrvPfdQDNQ+a7XfAx4wXvf\nEeg6hwK7N/1j96Z/7N70j90bY7hh79n+sXvTP3Zv+sfuTf/YvQnMoLfEH2yccycA3wfOG+prSRp2\nb/rH7k3/2L3pH7s3xnDD3rP9Y/emf+ze9I/dm/6xe3Ngkn5Sthmo2efXE3u/9p6vcc7lA2XAzt5f\nTwQeAD7hvV8f/GoHF7s3/WP3pn/s3vSP3RtjuGHv2f6xe9M/dm/6x+5N/9i9CUzSg7LngDrnXK1z\nrpBs0eCid71mEXBt788vBR7z3nvnXDnwG+Dr3vunB+2KBw+7N/1j96Z/7N70j90bY7hh79n+sXvT\nP3Zv+sfuTf/YvQnNUBe1HewHsAB4jWzHl2/2fu07wEW9Py8C7gPWAc8CU3q//i2gFVixz4+qof5+\n7N7YvRnqH3Zv7N7Yj3h+2HvW7o3dG7s3dm/i+OF6b5ZhGIZhGIZhGIYxBCQ9fdEwDMMwDMMwDCNq\nLCgzDMMwDMMwDMMYQiwoMwzDMAzDMAzDGEIsKDMMwzAMwzAMwxhCLCgzDMMwDMMwDMMYQiwoMwzD\nMAzDMAzDGEIsKDMMwzAMwzAMwxhCLCgzDMMwDMMwDMMYQv5/GPet38G9+HcAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def plot_spec(ims):\n", + " timebins, freqbins = np.shape(ims)\n", + " # import pdb;pdb.set_trace()\n", + "# plt.figure(figsize=(15, 7.5))\n", + " plt.imshow(np.transpose(ims), origin=\"lower\", aspect=\"auto\", cmap=\"jet\", interpolation=\"none\")\n", + " plt.colorbar()\n", + " xlocs = np.float32(np.linspace(0, timebins-1, 5))\n", + " plt.xticks(xlocs, [\"%.02f\" % l for l in ((xlocs*15/timebins)+(0.5*2**10))/22100])\n", + " ylocs = np.int16(np.round(np.linspace(0, freqbins-1, 10)))\n", + "# plt.yticks(ylocs, [\"%.02f\" % freq[i] for i in ylocs])\n", + " \n", + "def show_nth(n):\n", + " plt.figure(figsize=(15,7.5))\n", + " plt.subplot(1,2,1)\n", + " plot_spec(te_pairs[n][0].reshape(15,1654))\n", + " print(te_y[n])\n", + " plt.subplot(1,2,2)\n", + " plot_spec(te_pairs[n][1].reshape(15,1654))\n", + "show_nth(0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/mnist_siamese.py b/mnist_siamese.py index a6aec8b..88308bb 100644 --- a/mnist_siamese.py +++ b/mnist_siamese.py @@ -21,6 +21,9 @@ from keras.layers import Dense, Dropout, Input, Lambda from keras.optimizers import RMSprop from keras import backend as K +%matplotlib inline +import matplotlib.pyplot as plt + num_classes = 10 @@ -104,8 +107,6 @@ tr_pairs, tr_y = create_pairs(x_train, digit_indices) digit_indices = [np.where(y_test == i)[0] for i in range(num_classes)] te_pairs, te_y = create_pairs(x_test, digit_indices) -tr_pairs.shape - # network definition base_network = create_base_network(input_dim) diff --git a/speech_data.py b/speech_data.py index f82b330..546c5ec 100644 --- a/speech_data.py +++ b/speech_data.py @@ -2,6 +2,7 @@ import pandas as pd import numpy as np from spectro_gen import generate_aiff_spectrogram from sklearn.model_selection import train_test_split +import itertools import pickle,gc def sunflower_data(): @@ -18,43 +19,43 @@ def sunflower_data(): return np.lib.pad(spgr,[(0, max_samples-orig), (0,0)],'median') pad_sun = sunflowers['file'].apply(append_zeros).values x_data = np.vstack(pad_sun).reshape((sample_count,max_samples,sample_size,)) - # x_data.shape - # y_data.shape - # train_test_split(x_data,y_data,test_size=0.33)[].shape - # len(train_test_split(x_data,y_data,test_size=0.33)) - # sunflowers.loc[:,'file'][0] - # generate_aiff_spectrogram('outputs/sunflowers-Alex-150-normal-589.aiff') - # sunflowers[sunflowers['variant'] == 'phoneme'] - # sunflowers[sunflowers['variant'] == 'normal'] - # for s in sunflowers.values: - # print(s) return train_test_split(x_data,y_data,test_size=0.33) +def get_siamese_pairs(groupF1,groupF2): + group1 = [r for (i,r) in groupF1.iterrows()] + group2 = [r for (i,r) in groupF2.iterrows()] + f = [(g1,g2) for g2 in group2 for g1 in group1] + t = [i for i in itertools.combinations(group1,2)]+[i for i in itertools.combinations(group2,2)] + return (t,f) def sunflower_pairs_data(): audio_samples = pd.read_csv('./outputs/audio.csv',names=['word','voice','rate','variant','file']) - sunflowers = audio_samples.loc[audio_samples['word'] == 'sunflowers'].reset_index(drop=True) - sunflowers.loc[:,'file'] = sunflowers.loc[:,'file'].apply(lambda x:'outputs/'+x).apply(generate_aiff_spectrogram) - y_data = sunflowers['variant'].apply(lambda x:x=='normal').values - max_samples = sunflowers['file'].apply(lambda x:x.shape[0]).max() - sample_size = sunflowers['file'][0].shape[1] - sunflowers_pos = sunflowers[sunflowers['variant'] == 'normal'].reset_index(drop=True) - sunflowers_neg = sunflowers[sunflowers['variant'] == 'phoneme'].reset_index(drop=True) + audio_samples = audio_samples.loc[audio_samples['word'] == 'sunflowers'].reset_index(drop=True) + audio_samples.loc[:,'spectrogram'] = audio_samples.loc[:,'file'].apply(lambda x:'outputs/audio/'+x).apply(generate_aiff_spectrogram) + max_samples = audio_samples['spectrogram'].apply(lambda x:x.shape[0]).max() + sample_size = audio_samples['spectrogram'][0].shape[1] + same_data,diff_data = [],[] + for (w,g) in audio_samples.groupby(audio_samples['word']): + sample_norm = g.loc[audio_samples['variant'] == 'normal'] + sample_phon = g.loc[audio_samples['variant'] == 'phoneme'] + same , diff = get_siamese_pairs(sample_norm,sample_phon) + same_data.extend(same) + diff_data.extend(diff) + Y = np.hstack([np.ones(len(same_data)),np.zeros(len(diff_data))]) + X_sample_pairs = same_data+diff_data def append_zeros(spgr): - return np.lib.pad(spgr,[(0, max_samples-spgr.shape[0]), (0,0)],'median') - def create_data(sf): - sample_count = sf['file'].shape[0] - pad_sun = sf['file'].apply(append_zeros).values - x_data = np.vstack(pad_sun).reshape((sample_count,max_samples,sample_size)) - return x_data - x_data_pos = create_data(sunflowers_pos) - x_data_neg = create_data(sunflowers_neg) - x_pos_train, x_pos_test, x_neg_train, x_neg_test =train_test_split(x_data_pos,x_data_neg,test_size=0.33) - tr_y = np.array(x_pos_train.shape[0]*[[1,0]]) - te_y = np.array(x_pos_test.shape[0]*[[1,0]]) - tr_pairs = np.array([x_pos_train,x_neg_train]).reshape(x_pos_train.shape[0],2,max_samples,sample_size) - te_pairs = np.array([x_pos_test,x_neg_test]).reshape(x_pos_test.shape[0],2,max_samples,sample_size) - return tr_pairs,te_pairs,tr_y,te_y + sample = np.lib.pad(spgr,[(0, max_samples-spgr.shape[0]), (0,0)],'median') + return np.expand_dims(sample,axis=0) + def create_X(sp): + # sample_count = sp[0]['file'].shape[0] + l_sample = append_zeros(sp[0]['spectrogram']) + r_sample = append_zeros(sp[1]['spectrogram'])#.apply(append_zeros).values + # x_data = np.vstack(pad_sun).reshape((sample_count,max_samples,sample_size)) + return np.expand_dims(np.vstack([l_sample,r_sample]),axis=0) + X_list = (create_X(sp) for sp in X_sample_pairs) + X = np.vstack(X_list) + tr_pairs,te_pairs,tr_y,te_y = train_test_split(X,Y,test_size=0.1) + return train_test_split(X,Y,test_size=0.1) def create_spectrogram_data(audio_group='audio'): audio_samples = pd.read_csv('./outputs/'+audio_group+'.csv',names=['word','voice','rate','variant','file']) @@ -64,58 +65,71 @@ def create_spectrogram_data(audio_group='audio'): def create_speech_pairs_data(audio_group='audio'): audio_samples = pd.read_pickle('outputs/spectrogram.pkl') - y_data = audio_samples['variant'].apply(lambda x:x=='normal').values max_samples = audio_samples['spectrogram'].apply(lambda x:x.shape[0]).max() sample_size = audio_samples['spectrogram'][0].shape[1] - pickle.dump((max_samples,sample_size),open('./spectrogram_vars.pkl','wb')) - audio_samples_pos = audio_samples[audio_samples['variant'] == 'normal'].reset_index(drop=True) - audio_samples_neg = audio_samples[audio_samples['variant'] == 'phoneme'].reset_index(drop=True) - def append_zeros(spgr): - return np.lib.pad(spgr,[(0, max_samples-spgr.shape[0]), (0,0)],'median') - def create_data(sf): - sample_count = sf['spectrogram'].shape[0] - pad_sun = sf['spectrogram'].apply(append_zeros).values - print('appended zeros') - x_data = np.vstack(pad_sun).reshape((sample_count,max_samples,sample_size)) - print('reshaped') - return x_data - print('creating speech pair data') - x_data_pos = create_data(audio_samples_pos) - x_data_neg = create_data(audio_samples_neg) - np.save('outputs/x_data_pos.npy',x_data_pos) - np.save('outputs/x_data_neg.npy',x_data_neg) - print('pickled speech pairs') -def create_speech_model_data(): - (max_samples,sample_size) = pickle.load(open('./spectrogram_vars.pkl','rb')) - x_data_pos = np.load('outputs/x_data_pos.npy') - x_data_neg = np.load('outputs/x_data_neg.npy') - x_pos_train, x_pos_test, x_neg_train, x_neg_test =train_test_split(x_data_pos,x_data_neg,test_size=0.33) - del x_data_pos - del x_data_neg + def append_zeros(spgr): + sample = np.lib.pad(spgr,[(0, max_samples-spgr.shape[0]), (0,0)],'median') + return sample + def create_X(sp): + l_sample = append_zeros(sp[0]['spectrogram']) + r_sample = append_zeros(sp[1]['spectrogram']) + return np.asarray([l_sample,r_sample]) + + print('generating siamese speech pairs') + same_data,diff_data = [],[] + for (w,g) in audio_samples.groupby(audio_samples['word']): + sample_norm = g.loc[audio_samples['variant'] == 'normal']#.reset_index(drop=True) + sample_phon = g.loc[audio_samples['variant'] == 'phoneme']#.reset_index(drop=True) + same , diff = get_siamese_pairs(sample_norm,sample_phon) + same_data.extend([create_X(s) for s in same[:10]]) + diff_data.extend([create_X(d) for d in diff[:10]]) + print('creating all speech pairs') + Y = np.hstack([np.ones(len(same_data)),np.zeros(len(diff_data))]) + print('casting as array speech pairs') + X = np.asarray(same_data+diff_data) + print('pickling X/Y') + np.save('outputs/X.npy',X) + np.save('outputs/Y.npy',Y) + del X gc.collect() - print('split train and test') - tr_y = np.array(x_pos_train.shape[0]*[[1,0]]) - te_y = np.array(x_pos_test.shape[0]*[[1,0]]) - tr_pairs = np.array([x_pos_train,x_neg_train]).reshape(x_pos_train.shape[0],2,max_samples,sample_size) - te_pairs = np.array([x_pos_test,x_neg_test]).reshape(x_pos_test.shape[0],2,max_samples,sample_size) - print('reshaped to input dim') + print('train/test splitting speech pairs') + tr_pairs,te_pairs,tr_y,te_y = train_test_split(X,Y,test_size=0.1) + print('pickling train/test') np.save('outputs/tr_pairs.npy',tr_pairs) np.save('outputs/te_pairs.npy',te_pairs) np.save('outputs/tr_y.npy',tr_y) np.save('outputs/te_y.npy',te_y) - print('pickled speech model data') - # return tr_pairs,te_pairs,tr_y,te_y + +# def create_speech_model_data(): +# (max_samples,sample_size) = pickle.load(open('./spectrogram_vars.pkl','rb')) +# x_data_pos = np.load('outputs/x_data_pos.npy') +# x_data_neg = np.load('outputs/x_data_neg.npy') +# x_pos_train, x_pos_test, x_neg_train, x_neg_test =train_test_split(x_data_pos,x_data_neg,test_size=0.1) +# del x_data_pos +# del x_data_neg +# gc.collect() +# print('split train and test') +# tr_y = np.array(x_pos_train.shape[0]*[1]) +# te_y = np.array(x_pos_test.shape[0]*[[1,0]]) +# tr_pairs = np.array([x_pos_train,x_neg_train]).reshape(x_pos_train.shape[0],2,max_samples,sample_size) +# te_pairs = np.array([x_pos_test,x_neg_test]).reshape(x_pos_test.shape[0],2,max_samples,sample_size) +# print('reshaped to input dim') +# np.save('outputs/tr_pairs.npy',tr_pairs) +# np.save('outputs/te_pairs.npy',te_pairs) +# np.save('outputs/tr_y.npy',tr_y) +# np.save('outputs/te_y.npy',te_y) +# print('pickled speech model data') def speech_model_data(): - tr_pairs = np.load('outputs/tr_pairs.npy') - te_pairs = np.load('outputs/te_pairs.npy') + tr_pairs = np.load('outputs/tr_pairs.npy').astype(np.float32)/255.0 + te_pairs = np.load('outputs/te_pairs.npy').astype(np.float32)/255.0 tr_y = np.load('outputs/tr_y.npy') te_y = np.load('outputs/te_y.npy') return tr_pairs,te_pairs,tr_y,te_y if __name__ == '__main__': + # sunflower_pairs_data() #create_spectrogram_data() - # create_speech_pairs_data() - # create_speech_model_data() - print(speech_model_data()) + create_speech_pairs_data() + # print(speech_model_data()) diff --git a/speech_siamese.py b/speech_siamese.py index aaf3256..39bfe6e 100644 --- a/speech_siamese.py +++ b/speech_siamese.py @@ -14,17 +14,21 @@ from __future__ import absolute_import from __future__ import print_function import numpy as np -import random +# import random # from keras.datasets import mnist from speech_data import speech_model_data from keras.models import Model -from keras.layers import Dense, Dropout, Input, Lambda, LSTM, SimpleRNN -from keras.optimizers import RMSprop +from keras.layers import Input, Dense, Dropout, SimpleRNN, LSTM, Lambda +# Dense, Dropout, Input, Lambda, LSTM, SimpleRNN +from keras.optimizers import RMSprop, SGD +from keras.callbacks import TensorBoard from keras import backend as K + def euclidean_distance(vects): x, y = vects - return K.sqrt(K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), K.epsilon())) + return K.sqrt(K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), + K.epsilon())) def eucl_dist_output_shape(shapes): @@ -37,20 +41,35 @@ def contrastive_loss(y_true, y_pred): http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf ''' margin = 1 + # print(y_true, y_pred) return K.mean(y_true * K.square(y_pred) + (1 - y_true) * K.square(K.maximum(margin - y_pred, 0))) -def create_base_network(input_dim): +def create_base_rnn_network(input_dim): '''Base network to be shared (eq. to feature extraction). ''' inp = Input(shape=input_dim) - sr1 = SimpleRNN(128)(inp) - # sr2 = LSTM(128)(sr1) - # sr2 = SimpleRNN(128)(sr) - x = Dense(128, activation='relu')(sr1) - return Model(inp, x) + # d1 = Dense(1024, activation='sigmoid')(inp) + # # d2 = Dense(2, activation='sigmoid')(d1) + ls1 = LSTM(1024, return_sequences=True)(inp) + ls2 = LSTM(512, return_sequences=True)(ls1) + ls3 = LSTM(32)(ls2) # , return_sequences=True + # sr2 = SimpleRNN(128, return_sequences=True)(sr1) + # sr3 = SimpleRNN(32)(sr2) + # x = Dense(128, activation='relu')(sr1) + return Model(inp, ls3) +def create_base_network(input_dim): + '''Base network to be shared (eq. to feature extraction). + ''' + input = Input(shape=input_dim) + x = Dense(128, activation='relu')(input) + x = Dropout(0.1)(x) + x = Dense(128, activation='relu')(x) + x = Dropout(0.1)(x) + x = Dense(128, activation='relu')(x) + return Model(input, x) def compute_accuracy(y_true, y_pred): '''Compute classification accuracy with a fixed threshold on distances. @@ -66,8 +85,8 @@ def accuracy(y_true, y_pred): # the data, shuffled and split between train and test sets -tr_pairs,te_pairs,tr_y,te_y = speech_model_data() - # y_train.shape,y_test.shape +tr_pairs, te_pairs, tr_y, te_y = speech_model_data() +# y_train.shape,y_test.shape # x_train.shape,x_test.shape # x_train = x_train.reshape(60000, 784) # x_test = x_test.reshape(10000, 784) @@ -75,11 +94,11 @@ tr_pairs,te_pairs,tr_y,te_y = speech_model_data() # x_test = x_test.astype('float32') # x_train /= 255 # x_test /= 255 -input_dim = tr_pairs.shape[2:] +input_dim = (tr_pairs.shape[2], tr_pairs.shape[3]) epochs = 20 # network definition -base_network = create_base_network(input_dim) +base_network = create_base_rnn_network(input_dim) input_a = Input(shape=input_dim) input_b = Input(shape=input_dim) @@ -90,17 +109,25 @@ processed_a = base_network(input_a) processed_b = base_network(input_b) distance = Lambda(euclidean_distance, - output_shape=eucl_dist_output_shape)([processed_a, processed_b]) + output_shape=eucl_dist_output_shape)( + [processed_a, processed_b] +) model = Model([input_a, input_b], distance) +tb_cb = TensorBoard(log_dir='./siamese_logs', histogram_freq=1, batch_size=32, + write_graph=True, write_grads=True, write_images=True, + embeddings_freq=0, embeddings_layer_names=None, + embeddings_metadata=None) # train -rms = RMSprop() +rms = RMSprop(lr=0.00001) # lr=0.001) +sgd = SGD(lr=0.001) model.compile(loss=contrastive_loss, optimizer=rms, metrics=[accuracy]) model.fit([tr_pairs[:, 0], tr_pairs[:, 1]], tr_y, batch_size=128, epochs=epochs, - validation_data=([te_pairs[:, 0], te_pairs[:, 1]], te_y)) + validation_data=([te_pairs[:, 0], te_pairs[:, 1]], te_y), + callbacks=[tb_cb]) # compute final accuracy on training and test sets y_pred = model.predict([tr_pairs[:, 0], tr_pairs[:, 1]])