FacesGAN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.externals import joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_olivetti_faces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "faces = fetch_olivetti_faces()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(400, 64, 64)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "faces.images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "#X, y = load_digits(n_class=1,return_X_y=True)\n",
    "X,y = faces.images , faces.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def showdigit(index,multiplier=70,size=200):\n",
    "    tstimage1= Image.fromarray(X[index].reshape((8,8)) * multiplier)\n",
    "    tstimage1 = tstimage1.convert('L')\n",
    "    tstimage1 = tstimage1.resize((size,size))\n",
    "    tstimage1.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def showpic(digit,multiplier=70,size=200):\n",
    "    tstimage1= Image.fromarray(digit.reshape((64,64)) * multiplier)\n",
    "    tstimage1 = tstimage1.convert('L')\n",
    "    tstimage1 = tstimage1.resize((size,size))\n",
    "    tstimage1.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.models import Model, Sequential\n",
    "from keras.layers import Flatten,Dropout, Dense, Input,merge\n",
    "from keras.layers.convolutional import Convolution2D, Deconvolution2D, UpSampling2D, MaxPooling2D\n",
    "from keras.layers.core import Reshape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#model=joblib.load(\"Sigmoid20epoch5gen.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from math import sqrt\n",
    "PIC_SHAPE = 64*64\n",
    "\n",
    "gen_input = Input(shape=(PIC_SHAPE,))\n",
    "orig_input = Input(shape=(PIC_SHAPE,))\n",
    "\n",
    "generator = Reshape((1, int(sqrt(PIC_SHAPE)), int(sqrt(PIC_SHAPE))),name = \"generator11\")(gen_input)\n",
    "generator = Convolution2D(1, 12, 12, border_mode='same',name = \"generator10\")(generator)#Dense(64*64,activation = 'sigmoid',name = \"encoder_1\")\n",
    "generator = Flatten(name = \"generator2\")(generator)\n",
    "generator = Dense(64*64,activation = 'sigmoid',name = \"encoder_1\")(generator)\n",
    "generator = Dense(64*32,activation = 'sigmoid',name = \"encoder_2\")(generator)\n",
    "generator = Dense(64*40,activation = 'sigmoid',name = \"encoder_3\")(generator)\n",
    "gen_out = Dense(PIC_SHAPE,activation = 'sigmoid',name = \"generator1\")(generator)\n",
    "#generator4= Deconvolution2D(3, 3, 3, output_shape=(None, 3, 14, 14), border_mode='valid')(generator3)\n",
    "#gen_out1 = generator(gen_input)\n",
    "#gen_out2 = generator2(gen_out1)\n",
    "#gen_out = generator3(gen_out2)\n",
    "\n",
    "merged_vector = merge([orig_input,gen_out], mode='sum',output_shape=(PIC_SHAPE,))\n",
    "\n",
    "predictor = Reshape((1, int(sqrt(PIC_SHAPE)), int(sqrt(PIC_SHAPE))),name = \"predictor11\")(merged_vector)\n",
    "predictor = Convolution2D(1, 12, 12, border_mode='same',name = \"predictor10\")(predictor)\n",
    "predictor = Flatten(name = \"predictor2\")(predictor)\n",
    "predictor = Dense(64*32, activation='sigmoid',name=\"predictor_1\")(predictor)\n",
    "predictor = Dense(64, activation='sigmoid',name=\"predictor_2\")(predictor)\n",
    "predictions = Dense(1, activation='sigmoid',name=\"predictor_3\")(predictor)\n",
    "\n",
    "model = Model(input=[gen_input,orig_input], output=predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "Exception",
     "evalue": "Layer weight shape (4096, 4096) not compatible with provided weight shape (4096, 2048)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mException\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-23-d20479fd2a79>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mstartwith\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mjoblib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"modelFACES_LAST_epoch.pkl\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset_weights\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstartwith\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mjoblib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstartwith\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m\"start\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/aanax/anaconda3/lib/python3.5/site-packages/keras/engine/topology.py\u001b[0m in \u001b[0;36mset_weights\u001b[1;34m(self, weights)\u001b[0m\n\u001b[0;32m    884\u001b[0m                                 \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    885\u001b[0m                                 \u001b[1;34m' not compatible with '\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 886\u001b[1;33m                                 'provided weight shape ' + str(w.shape))\n\u001b[0m\u001b[0;32m    887\u001b[0m             \u001b[0mweight_value_tuples\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    888\u001b[0m         \u001b[0mK\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbatch_set_value\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mweight_value_tuples\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mException\u001b[0m: Layer weight shape (4096, 4096) not compatible with provided weight shape (4096, 2048)"
     ]
    }
   ],
   "source": [
    "startwith = joblib.load(\"modelFACES_LAST_epoch.pkl\")\n",
    "model.set_weights(startwith)\n",
    "joblib.dump(startwith,\"start\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<keras.engine.topology.InputLayer at 0x7f6cb143f940>,\n",
       " <keras.layers.core.Reshape at 0x7f6cb14b4cc0>,\n",
       " <keras.layers.convolutional.Convolution2D at 0x7f6cb14b4dd8>,\n",
       " <keras.layers.core.Flatten at 0x7f6cb14b4240>,\n",
       " <keras.layers.core.Dense at 0x7f6cb1413c18>,\n",
       " <keras.layers.core.Dense at 0x7f6cb14196d8>,\n",
       " <keras.layers.core.Dense at 0x7f6cb1419748>,\n",
       " <keras.layers.core.Dense at 0x7f6cb1425048>,\n",
       " <keras.engine.topology.InputLayer at 0x7f6cb143fc50>,\n",
       " <keras.engine.topology.Merge at 0x7f6cb141ee10>,\n",
       " <keras.layers.core.Reshape at 0x7f6cb1419cf8>,\n",
       " <keras.layers.convolutional.Convolution2D at 0x7f6cb141eda0>,\n",
       " <keras.layers.core.Flatten at 0x7f6cb1425780>,\n",
       " <keras.layers.core.Dense at 0x7f6cb13c23c8>,\n",
       " <keras.layers.core.Dense at 0x7f6cb13c2438>,\n",
       " <keras.layers.core.Dense at 0x7f6cb13d27b8>]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers#[4].trainable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def generator_error(y_true,y_pred):\n",
    "    #y_pred = np.array(y_pred)\n",
    "    return np.log(np.ones(y_pred.shape)-y_pred).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from theano.tensor import basic as tensor, subtensor\n",
    "def generator_error_theano(y_true,y_pred):\n",
    "    '''\n",
    "    TODO write normalisation\n",
    "    '''\n",
    "    return -tensor.log(1.0 - y_pred)  #-(target * tensor.log(output) + (1.0 - target) *\n",
    "    #return -tensor.log(1.0 + (y_pred-1.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def tester_error(y_true,y_pred):\n",
    "    y_pred = np.array(y_pred)\n",
    "    #log(pred) if true = 0\n",
    "    #log(1-pred) if true=1\n",
    "    return np.log(y_true - y_pred*((-1)**(y_true+1))).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def tester_error_theano(target,output):\n",
    "    '''\n",
    "    y_true = 0 -> it was real\n",
    "    '''\n",
    "    \n",
    "    return -((target) * tensor.log(output) + (1.0 - target) * tensor.log(1.0 - output))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generate_input(length=64,bottom_border =0 ,top_border=1):\n",
    "    '''\n",
    "    generates list of length length with trash from custom distribution.\n",
    "    '''\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "npx = np.array(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "showpic(X[70])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bottom = np.array(X).min()\n",
    "top = np.array(X).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = X/top"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "generator_layers = 3\n",
    "tester_train_steps = 1\n",
    "n_epoch = 100\n",
    "xnum=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.optimizers import SGD\n",
    "\n",
    "opt = \"adam\"#SGD(lr = 0.0001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#showdigit(0,multiplier=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 400/400 [35:59<00:00,  5.24s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 400/400 [36:43<00:00,  5.62s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 400/400 [37:22<00:00,  5.83s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 400/400 [37:08<00:00,  5.38s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 400/400 [37:13<00:00,  5.44s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▏   | 246/400 [22:06<13:35,  5.29s/it]"
     ]
    }
   ],
   "source": [
    "import theano.sandbox.cuda\n",
    "theano.sandbox.cuda.use(\"gpu\")\n",
    "\n",
    "from tqdm import tqdm\n",
    "for epoch in range(0,n_epoch):\n",
    "    \n",
    "    print(\"Epoch \",epoch)#,\"Iter \",xnum)\n",
    "    joblib.dump(model.get_weights(),\"modelFACES_LAST_epoch.pkl\",compress=9)\n",
    "    for xnum in tqdm(range(0,len(X))):\n",
    "        \n",
    "\n",
    "        #unfreezing tester\n",
    "        #model.layers[-1].trainable = True\n",
    "        for i in model.layers:\n",
    "            if \"predictor\" in i.name:\n",
    "                i.trainable = True\n",
    "        #freezing generator\n",
    "        for i in model.layers:\n",
    "            if (\"encoder\" in i.name or \"decoder\" in i.name or \"generator\" in i.name):\n",
    "\n",
    "                i.trainable = False\n",
    "        model.compile(opt , loss = tester_error_theano)\n",
    "        #doing tester_train_steps of tester training\n",
    "        for k in range(0,tester_train_steps):\n",
    "            #for xnum in range(0,len(X)):\n",
    "            model.train_on_batch([np.zeros((PIC_SHAPE,)).reshape((-1,PIC_SHAPE)),np.array(X[xnum]).reshape((-1,PIC_SHAPE))],np.array([0.0])) #inputs original image\n",
    "            model.train_on_batch([np.random.uniform(bottom,top,PIC_SHAPE).reshape((-1,PIC_SHAPE)),np.zeros((PIC_SHAPE,)).reshape((-1,PIC_SHAPE))],np.array([1.0])) #inputs noise to generator\n",
    "            \n",
    "\n",
    "        #Unfreezing generator        \n",
    "        for i in model.layers:\n",
    "            if (\"encoder\" in i.name or \"decoder\" in i.name or \"generator\" in i.name):\n",
    "                i.trainable = True\n",
    "        #freezing tester\n",
    "        #model.layers[-1].trainable = False\n",
    "        for i in model.layers:\n",
    "            if \"predictor\" in i.name:\n",
    "                i.trainable = False\n",
    "                \n",
    "        model.compile(opt, loss = generator_error_theano)\n",
    "        #doing ? steps of generator training\n",
    "        #print(\"Ill train discriminator\")\n",
    "        for nn in range(0,2):\n",
    "            model.train_on_batch([np.random.uniform(bottom,top,PIC_SHAPE).reshape((-1,PIC_SHAPE)),np.zeros((PIC_SHAPE,)).reshape((-1,PIC_SHAPE))],np.array([1.0])) #inputs noise to generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pred = model.predict([np.random.uniform(bottom,top,PIC_SHAPE).reshape((-1,PIC_SHAPE)),np.zeros((PIC_SHAPE,)).reshape((-1,PIC_SHAPE))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#import subprocess subprocess.open (args 2 pipes -> -<)\n",
    "pred2 = model.predict([np.zeros((PIC_SHAPE,)).reshape((-1,PIC_SHAPE)),np.array(X[44]).reshape((-1,PIC_SHAPE))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  5.15669262e-06]], dtype=float32)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in range(len(model.layers)):\n",
    "            if (\"encoder\" in model.layers[i].name or \"decoder\" in model.layers[i].name or \"generator\" in model.layers[i].name):\n",
    "                last_gen = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_gen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import theano\n",
    "get_activations = theano.function([model.layers[0].input], model.layers[last_gen].output, allow_input_downcast=True)\n",
    "#activations = get_activations(X_batch) # same result as above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tst = np.random.uniform(bottom,top,PIC_SHAPE).reshape((-1,PIC_SHAPE))\n",
    "showpic(tst,size=500,multiplier = 100)\n",
    "Generated = get_activations(tst)\n",
    "showpic(Generated,size=500,multiplier = 100)\n",
    "showpic(X[31],size=500,multiplier = 100)\n",
    "#showdigit(np.random.randint(low=0,high=120),size=500,multiplier = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "joblib.dump(model,\"Sigmoid500epoch3gen.pkl\",compress=9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 0: GeForce GTX 860M (CNMeM is disabled, cuDNN 4007)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[GpuElemwise{exp,no_inplace}(<CudaNdarrayType(float32, vector)>), HostFromGpu(GpuElemwise{exp,no_inplace}.0)]\n",
      "Looping 1000 times took 0.359734 seconds\n",
      "Result is [ 1.23178029  1.61879349  1.52278066 ...,  2.20771813  2.29967761\n",
      "  1.62323296]\n",
      "Used the gpu\n"
     ]
    }
   ],
   "source": [
    "from theano import function, config, shared, sandbox\n",
    "import theano.tensor as T\n",
    "import numpy\n",
    "import time\n",
    "\n",
    "#import os\n",
    "#os.environ[\"THEANO_FLAGS\"] = \"mode=FAST_RUN,device=gpu0,floatX=float64\"\n",
    "\n",
    "import theano.sandbox.cuda\n",
    "theano.sandbox.cuda.use(\"gpu\")\n",
    "\n",
    "vlen = 10 * 30 * 768  # 10 x #cores x # threads per core\n",
    "iters = 1000\n",
    "\n",
    "rng = numpy.random.RandomState(22)\n",
    "x = shared(numpy.asarray(rng.rand(vlen), config.floatX))\n",
    "f = function([], T.exp(x))\n",
    "print(f.maker.fgraph.toposort())\n",
    "t0 = time.time()\n",
    "for i in range(iters):\n",
    "    r = f()\n",
    "t1 = time.time()\n",
    "print(\"Looping %d times took %f seconds\" % (iters, t1 - t0))\n",
    "print(\"Result is %s\" % (r,))\n",
    "if numpy.any([isinstance(x.op, T.Elemwise) for x in f.maker.fgraph.toposort()]):\n",
    "    print('Used the cpu')\n",
    "else:\n",
    "    print('Used the gpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from math import sqrt\n",
    "PIC_SHAPE = 64*64\n",
    "\n",
    "gen_input = Input(shape=(PIC_SHAPE,))\n",
    "orig_input = Input(shape=(PIC_SHAPE,))\n",
    "\n",
    "generator = Reshape((1, int(sqrt(PIC_SHAPE)), int(sqrt(PIC_SHAPE))),name = \"generator11\")(gen_input)\n",
    "generator = Convolution2D(1, 12, 12, border_mode='same',name = \"generator10\")(generator)#Dense(64*64,activation = 'sigmoid',name = \"encoder_1\")\n",
    "#generator = Convolution2D(128, 3, 3, border_mode='same',name = \"generator9\")(generator)\n",
    "#generator = MaxPooling2D((2,2),name = \"generator8\")(generator)\n",
    "#generator = Convolution2D(32, 3, 3,name = \"generator7\")(generator)\n",
    "#generator = Deconvolution2D(1, 3, 3, output_shape=(None, 1, 14, 14),subsample=(2,2), border_mode='valid',name = \"generator6\")(generator)\n",
    "generator = UpSampling2D(size=(2, 2), dim_ordering='default',name = \"generator5\")(generator)\n",
    "#generator = Deconvolution2D(1, 3, 3, output_shape=(None, 1, 32, 32), subsample=(2,2), border_mode='valid',name = \"generator4\")(generator)\n",
    "generator = Deconvolution2D(1, 12, 12, output_shape=(None, 1, 33, 33),subsample=(2,2), border_mode='same',name = \"generator3\")(generator)\n",
    "generator = Flatten(name = \"generator2\")(generator)\n",
    "#generator1 = Dense(64*64,activation = 'sigmoid',name = \"encoder_1\")(gen_input)\n",
    "#generator2 = Dense(64*32,activation = 'sigmoid',name = \"encoder_2\")(generator1)\n",
    "#generator3 = Dense(64*32,activation = 'sigmoid',name = \"encoder_3\")(generator2)\n",
    "gen_out = Dense(PIC_SHAPE,activation = 'sigmoid',name = \"generator1\")(generator)\n",
    "#generator4= Deconvolution2D(3, 3, 3, output_shape=(None, 3, 14, 14), border_mode='valid')(generator3)\n",
    "#gen_out1 = generator(gen_input)\n",
    "#gen_out2 = generator2(gen_out1)\n",
    "#gen_out = generator3(gen_out2)\n",
    "\n",
    "merged_vector = merge([orig_input,gen_out], mode='sum',output_shape=(PIC_SHAPE,))\n",
    "\n",
    "predictor1 = Dense(64*32, activation='sigmoid',name=\"predictor_1\")(merged_vector)\n",
    "predictor2 = Dense(64, activation='sigmoid',name=\"predictor_2\")(predictor1)\n",
    "predictions = Dense(1, activation='sigmoid',name=\"predictor_3\")(predictor2)\n",
    "\n",
    "model = Model(input=[gen_input,orig_input], output=predictions)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}