punchline_multilayer_baseline_text-speech.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#imports\n",
    "import torch\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run dataloader.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size=2600\n",
    "train_dataloader = DataLoader(train_set, collate_fn=collate_fn, batch_size=batch_size, shuffle=True)\n",
    "dev_dataloader = DataLoader(dev_set, collate_fn=collate_fn, batch_size=1000, shuffle=True)\n",
    "test_dataloader = DataLoader(test_set, collate_fn=collate_fn, batch_size=1000, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpu_id = 'cuda:3'\n",
    "device = torch.device(gpu_id if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# multi layer linear\n",
    "class MultiLayer(torch.nn.Module):\n",
    "\n",
    "    def __init__(self, input_dim, hidden_layer_size) -> None:\n",
    "        super().__init__()\n",
    "        self.linear1 = torch.nn.Linear(20*input_dim, hidden_layer_size)\n",
    "        self.dropout = torch.nn.Dropout(0.2)\n",
    "        self.linear2 = torch.nn.Linear(hidden_layer_size, 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = torch.flatten(x, start_dim=1)\n",
    "        h1 = torch.nn.functional.leaky_relu(self.linear1(x))\n",
    "        h2 = self.dropout(h1)\n",
    "        return self.linear2(h2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step error\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3364198/4016921915.py:102: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  context_w=np.array(self.language_sdk[hid]['context_embedding_indexes'])\n",
      "/tmp/ipykernel_3364198/4016921915.py:103: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  context_of=np.array(self.word_aligned_openface_sdk[hid]['context_features'])\n",
      "/tmp/ipykernel_3364198/4016921915.py:104: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  context_cvp=np.array(self.word_aligned_covarep_sdk[hid]['context_features'])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 0.71552973985672\n",
      "dev error:  0.7847256064414978\n",
      "2 0.6866025328636169\n",
      "dev error:  0.6840800046920776\n",
      "3 0.682052493095398\n",
      "dev error:  0.6902745962142944\n",
      "4 0.6594868898391724\n",
      "dev error:  0.687656581401825\n",
      "5 0.6501629948616028\n",
      "dev error:  0.6764499545097351\n",
      "6 0.6466951370239258\n",
      "dev error:  0.6761032342910767\n",
      "7 0.6375963687896729\n",
      "dev error:  0.6820253729820251\n",
      "8 0.6237716674804688\n",
      "dev error:  0.6749874949455261\n",
      "9 0.620297372341156\n",
      "dev error:  0.6747585535049438\n",
      "10 0.6061139106750488\n",
      "dev error:  0.6720737814903259\n",
      "11 0.5994778871536255\n",
      "dev error:  0.6721698641777039\n",
      "12 0.590377688407898\n",
      "dev error:  0.6718828678131104\n",
      "13 0.5873397588729858\n",
      "dev error:  0.6694179773330688\n",
      "14 0.5678422451019287\n",
      "dev error:  0.6704818606376648\n",
      "15 0.569064736366272\n",
      "dev error:  0.6734811067581177\n",
      "16 0.5558812618255615\n",
      "dev error:  0.6753247380256653\n",
      "17 0.54107666015625\n",
      "dev error:  0.6786374449729919\n",
      "18 0.5396151542663574\n",
      "dev error:  0.6824461817741394\n",
      "19 0.5248420834541321\n",
      "dev error:  0.6824727654457092\n",
      "20 0.5064702033996582\n",
      "dev error:  0.6872618198394775\n",
      "21 0.509445309638977\n",
      "dev error:  0.6895993947982788\n",
      "22 0.5034338235855103\n",
      "dev error:  0.7043846845626831\n",
      "23 0.4872022867202759\n",
      "dev error:  0.7096325755119324\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAxdElEQVR4nO3deXxU1f3/8dcnYQnIDsEiu5qyKSIkuAB1Z3EBrRXB5UttFa1a0V/dW1tK1dq6oVZr3bUuiNS9uICKVRFLoIACKosCwQUEUVD2fH5/nBszhEkIkMlMZt7Px+M+MnedM8Mw77n33HOOuTsiIiJlZSW7ACIikpoUECIiEpcCQkRE4lJAiIhIXAoIERGJSwEhIiJxKSAkbZjZ3WZ2TQqU4+dm9vZObP+pmR2dyDKJ7IpayS6ACIQvSeBsd5+8q8dw9/OqrkQiojMIqRHMTD9mEsSCrDLLdur91r9PelJASNKZ2T+BdsALZrbOzC43sw5m5mb2SzNbCrwebfuUmX1hZt+Y2X/MrFvMcR4ys2ujx4ebWZGZ/cbMVpjZ52Z2VgVlOMvM5pvZWjNbbGbnxqyr8Fhm1tzMnjezb83sv8A+O3i9Z5rZEjNbZWa/LbMuy8yuNLNF0frxZtYsWveSmV1YZvvZZvbTcp7nYDObamZrou0Oj1k3xcyuM7N3gO+BvaP3+wIzWwAsiLY7x8wWmtnq6DXuFXOM7baX9KKAkKRz9zOBpcAJ7t7A3f8as/owoAswIJp/CcgDWgIzgccqOPSPgMZAa+CXwJ1m1rScbVcAxwONgLOAW82sZyWPdSewAWgF/CKa4jKzrsDfgTOBvYDmQJuYTX4NnBi97r2Ar6PjAzwBDC9zrPbAv+M8T+to+bVAM+BS4F9mlhuz2ZnASKAhsCRadiJwENDVzI4E/gwMjV7bEmBcmaf6YfvyXrPUYO6uSVPSJ+BT4OiY+Q6AA3tXsE+TaJvG0fxDwLXR48OB9UCtmO1XAAdXsjzPAqN2dCwgG9gMdI5Zdz3wdjnH/T0wLmZ+D2BTyWsH5gNHxaxvFR2/FuGL/DugfbTuOuCBcp7nCuCfZZa9AoyIHk8BxpRZ78CRMfP3A3+NmW8QlaVDvO01pd+kMwhJdctKHphZtpndEF1++ZYQKgAtytl3lbtviZn/nvAltx0zG2Rm06JLKWuAY8sct7xj5RK+vJfFrFtC+faK3dbdvwNWxaxvDzwTXRZaQwiMrcCe7r6WcFYwLNp2OOWfQbUHTik5TnSsvoTAKbEszn6xy/aKfS3uvi4qa+sdHEPShAJCUkV53QrHLj8NGAIcTbjc0yFabrvzxGZWF/gXcBPhi7gJMLGSx10JbAHaxixrV8H2n8dua2b1CZeZSiwDBrl7k5gpx92XR+ufAIab2SFADvBGOc+zjHAGEXucPdz9hpht4r3nscs+IwRNSVn3iMq6vJztJc0oICRVfAnsvYNtGgIbCb9i6xMu5VSFOkBdoi97MxsE9K/Mju6+FXgaGG1m9aN6gREV7DIBON7M+ppZHWAM2/4/vBu4zszaA5hZrpkNiVk/kfClPQZ40t2Ly3meR4ETzGxAdOaVE1W2tyln+3ieAM4ysx5RiF4PvOfun+7EMaQGU0BIqvgz8Lvocsil5WzzCOGSx3JgHjCtKp44unRzETCeUCl8GvD8ThziQsLlpi8I9SAPVvBcc4ELgMcJZxNfA0Uxm9wWPferZraW8BoPitl/IyGQjo6OUd7zLCOcbV1NCL5lwGXsxP95D21SriGcXX1OuDtrWIU7SVoxd50hiojI9nQGISIicSkgREQkLgWEiIjEpYAQEZG40qaDrRYtWniHDh2SXQwRkRplxowZX7l7brx1aRMQHTp0oLCwMNnFEBGpUcys3Jb/usQkIiJxKSBERCQuBYSIiMSVNnUQIpI+Nm/eTFFRERs2bEh2UdJGTk4Obdq0oXbt2pXeRwEhIimnqKiIhg0b0qFDB8x2q7NeIYz7s2rVKoqKiujYsWOl99MlJhFJORs2bKB58+YKhypiZjRv3nynz8gUECKSkhQOVWtX3k8FxOrVMGYMzJiR7JKIiKQU1UHUqgV/+AOYQa9eyS6NiKSg0aNH06BBAy69tLyhStKTziAaNYJOnWD69GSXRESELVu2VDhf2f2qgs4gAAoKYPJkcA9nEiKS8a677joefvhhWrZsSdu2bekVXWFYtGgRF1xwAStXrqR+/frce++9tGrViu7du/PJJ5+QlZXFd999R+fOnVm8ePE2t5WuXLmS8847j6VLlwIwduxY+vTpw+jRo1m0aBGLFy+mXbt2dOrUaZv5P//5z/ziF7/gq6++Ijc3lwcffJB27drx85//nJycHP73v//Rp08fbrnllip9DxQQEALi0Ufhs8+gdetkl0ZEYl18McyaVbXH7NEDxo4td/WMGTMYN24cs2bNYsuWLfTs2fOHgBg5ciR33303eXl5vPfee5x//vm8/vrr9OjRgzfffJMjjjiCF198kQEDBmzX5mDUqFFccskl9O3bl6VLlzJgwADmz58PwLx583j77bepV68eo0eP3mb+hBNOYMSIEYwYMYIHHniAiy66iGeffRYItwRPnTqV7Ozsqn2PUEAEBQXh7/TpCggR4a233uKkk06ifv36AAwePBiAdevWMXXqVE455ZQftt24cSMAp556Kk8++SRHHHEE48aN4/zzz9/uuJMnT2bevHk/zH/77besW7fuh+eoV6/eD+ti5999912efvppAM4880wuv/zyH7Y75ZRTEhIOoIAIevQIldXTp8OJJya7NCISq4Jf+tWtuLiYJk2aMCvOGc3gwYO5+uqrWb16NTNmzODII4+Mu/+0adPIycnZbt0ee+xR4Xx5KrvdrlAlNUC9erDffqqoFhEAfvKTn/Dss8+yfv161q5dywsvvABAo0aN6NixI0899RQQWijPnj0bgAYNGlBQUMCoUaM4/vjj4/6q79+/P3fccccP8/GCJp5DDz2UcePGAfDYY4/Rr1+/3Xl5laaAKFFQAIWFoaJaRDJaz549OfXUUznggAMYNGgQBSWXoQlf0Pfffz8HHHAA3bp147nnnvth3amnnsqjjz7KqaeeGve4t99+O4WFhXTv3p2uXbty9913V6o8d9xxBw8++CDdu3fnn//8J7fddtvuvcBKMk+TL8T8/HzfrQGD7r0XRo6EBQtg332rrmAistPmz59Ply5dkl2MtBPvfTWzGe6eH297nUGUiK2oFhERBcQPunWDnBwFhIhIRAFRonZtOPBABYSISEQBEaugAGbOhAQ0WRcRqWkUELEKCuD77yFq2SgikskUELFUUS0i8gMFRKy8vNC7qwJCJKOtWbOGu+66a5f2PfbYY1mzZk3VFihJFBCxsrIgP18BIZLhKgqIHXWrPXHiRJo0aVKl5UlWF+Dqi6msggK45RbYuBHq1k12aUQkCa688koWLVpEjx49OOaYYzjuuOO45ppraNq0KR9++CEff/wxJ554IsuWLWPDhg2MGjWKkSNHAtChQwcKCwtZt24dgwYNom/fvkydOpXWrVvz3HPPbdMhH6R2F+AKiLIKCmDzZpg9G3r3TnZpRDJeEnr75oYbbuCDDz74oa+kKVOmMHPmTD744AM6duwIwAMPPECzZs1Yv349BQUFnHzyyTRv3nyb4yxYsIAnnniCe++9l6FDh/Kvf/2LM844Y5ttUrkLcAVEWbEV1QoIEYn07t37h3CA0K/SM888A8CyZctYsGDBdgHRsWNHevToAUCvXr349NNPtztuKncBntCAMLOBwG1ANnCfu99QZv2twBHRbH2gpbs3idZtBd6P1i1198GJLOsP2raFli1VDyGSIlKlt+/YbrWnTJnC5MmTeffdd6lfvz6HH344GzZs2G6fujGXqbOzs1m/fv1226RyF+AJq6Q2s2zgTmAQ0BUYbmZdY7dx90vcvYe79wDuAJ6OWb2+ZF21hQOEIUcLChQQIhmsYcOGrF27ttz133zzDU2bNqV+/fp8+OGHTJs2bZefK5W7AE/kXUy9gYXuvtjdNwHjgCEVbD8ceCKB5am8goLQWK6CD4iIpK/mzZvTp08f9ttvPy677LLt1g8cOJAtW7bQpUsXrrzySg4++OBdfq5U7gI8Yd19m9nPgIHufnY0fyZwkLtfGGfb9sA0oI27b42WbQFmAVuAG9z92Tj7jQRGArRr167XkiVLqqbwEyfCccfBlClw2GFVc0wRqTR1950YNbW772HAhJJwiLSPCn0aMNbM9im7k7vf4+757p6fm5tbdaVRi2oRkYQGxHKgbcx8m2hZPMMoc3nJ3ZdHfxcDU4ADq76I5cjNhfbtFRAiktESGRDTgTwz62hmdQgh8HzZjcysM9AUeDdmWVMzqxs9bgH0AeaV3TehVFEtklTpMtplqtiV9zNhAeHuW4ALgVeA+cB4d59rZmPMLPaupGHAON+29F2AQjObDbxBqIOo/oD45BP46qtqfVoRgZycHFatWqWQqCLuzqpVq+LeSluRhLaDcPeJwMQyy35fZn50nP2mAvsnsmw7VFIPMWMGDBiQ1KKIZJo2bdpQVFTEypUrk12UtJGTk0ObNm12ah+1pC5Pr16hTcT06QoIkWpWu3btbVotS3Kkyl1MqadRI+jUSfUQIpKxFBAVUUW1iGQwBURFCgrg889heXl354qIpC8FREXyo8aFOosQkQykgKhIjx5Qq5YCQkQykgKiIvXqwX77KSBEJCMpIHakoAAKC0ENdkQkwyggdqSgAL7+GhYtSnZJRESqlQJiR9Szq4hkKAXEjnTrBjk5CggRyTgKiB2pXRsOPFABISIZRwFRGQUFMHMmbNmS7JKIiFQbBURlFBTA99+HcapFRDKEAqIyVFEtIhlIAVEZeXmhd1cFhIhkkIwPiO+/h4svhsWLK9goKyv0y6SAEJEMkvEB8dVX8NBDcNppsHlzBRsWFMCcObBxY3UVTUQkqTI+INq1g3vugffegz/+sYINCwpCgsyeXW1lExFJpowPCIChQ+Gss+D66+HNN8vZSBXVIpJhFBCR22+HffeFM86A1avjbNC2LbRsqYAQkdSwfj28/jr8/vdw7bUJeYpaCTlqDdSgATzxBBxyCJxzDkyYAGYxG5hpCFIRSZ4NG2DaNJgyBd54IzzetCncRHPccQl5SgVEjF694Lrr4PLL4b77QlBso6AAJk6EtWuhYcOklFFEMsTGjaFytCQQ3n03LDODnj3h17+GI46Avn2hceOEFEEBUcZvfgOvvgqjRkG/ftC5c8zKgoIwLsTMmXDYYUkro4ikoa1bw1nBG2+EUJg6NVxGMgujW55/fgiEfv2gSZNqKVJCA8LMBgK3AdnAfe5+Q5n1twJHRLP1gZbu3iRaNwL4XbTuWnd/OJFlLZGVBY88At27w/Dh4d+rbt1oZUlFdWGhAkJEqs6CBXD66aWXsLt3h5EjSwOhWbOkFCthAWFm2cCdwDFAETDdzJ5393kl27j7JTHb/xo4MHrcDPgDkA84MCPa9+tElTdWq1bwwAMweDBcdRXccku0IjcX2rdXPYSIVA338GUzahTUqQP33w9DhkDz5skuGZDYu5h6AwvdfbG7bwLGAUMq2H448ET0eAAwyd1XR6EwCRiYwLJu54QT4IIL4NZb4eWXY1aoolpEqsKqVfCzn8HZZ8NBB4WGuL/4RcqEAyQ2IFoDy2Lmi6Jl2zGz9kBH4PWd3TeRbrwR9tsPRoyAL7+MFhYUhH45Vq2q7uKISLp47bVwGemFF8IXzaRJ0KZNsku1nVRpBzEMmODuW3dmJzMbaWaFZla4cuXKKi9UvXrh1tdvvgkN6YqL2bYeQkRkZ2zcCJdeCkcfHToAfe+9MJ+VKl/F20pkqZYDbWPm20TL4hlG6eWlSu/r7ve4e7675+fm5u5mcePbbz+4+WZ46SW44w7CvbBmuswkIjtn3rxwKenmm8MdSTNmhNEqU1giA2I6kGdmHc2sDiEEni+7kZl1BpoC78YsfgXob2ZNzawp0D9alhTnnx/qJC6/HGZ/0gg6dVJAiEjluMNdd4Ufl599Fi4r3Xkn1K+f7JLtUMICwt23ABcSvtjnA+Pdfa6ZjTGzwTGbDgPGubvH7Lsa+BMhZKYDY6JlSWEWbjRo3hyGDYPvD+yjgBCRHVuxovSOl8MPDxXRxx+f7FJVmsV8L9do+fn5XpjgeoHJk+GYY+DcPu9z9zvdoagIWld73bmI1AQTJ4bKy2++gZtuCiGxTf89qcHMZrh7frx1akm9E44+Gi67DG68cX8GcCInTZ9e+YDYsiW0jHzxxfAronVr6NChdOrYMTTAyM5O4CsQkYTZujXc7rhsGfzzn+EyUvfuoUO9bt2SXbpdooDYSddeC6+/VszZM++j4LX7aHNiBRuvXh0aUbz4YqjlXrMGatcOH5Y5c+Dzz7fdvnbtMEBFbHCUhEeHDiFAUvRuB5G0VlwcLhcVFYUAKJli5z/7LPwQLHHJJWEMgZyc5JV7N+kS0y74+GPo2eV7Chp/zCOze9CmTXTm6A4ffhgC4cUX4Z13wq+K3NzQ2+Lxx4drVI0ahQNt2ABLl8Knn4bpk09KH3/6KXzxxbZP3KoV/OEPoTFN7drV8lpFMs7KleH/7tSpoa5xyRJYvjz0nBqrbt3QdqFt29KpZL5TpzCWfQ1Q0SUmBcQueujwBznrzbMA2Kv5Bg5q8hEHf/0yB6/+N72YwR4H5IVAOP740HZiVy4drV9fGiCffAKPPQZvvx0+eNdfDyefnJLXNEVqjOJi+OijEAgl04IFYV2dOuE21H32iR8CLVqkxf8/BUQi3Hsvc0bewVu1j2La5p5Ms0NY6PsCkJ3tdO9uHHwwP0x5eVXwWXIPZyZXXQVz50Lv3vCXv4S7I0Rkx9avD2cFU6eWniWUjBDWvDn06ROmQw+F/PwafXmoshQQiVBUFPpROeCAcJZw1FGs/K4+//1v6AF22rTQSHLt2rB5s2ahjczBB8NRR4XP4C7bujVUgl1zTSjHoEHw5z+HsiTTpk0wa1ZIw6ZNk1sWSR/FxeEMevZseP/9cFcQhB9MJeI9jl22eTP873+hq/7Nm8OyTp1KA6FPH/jxj9PijGBnKSCSZOvWUCVREhjTpoUf/u5w991w7rm7+QTr14c7Ja6/PlSAn346/OlPoUK7uixZEirgX3459C+zbl2oH+nfH049NXSJm6DBTCQNrVsXQmD27DDNmROmdevCerMw/GOJ2C/0eI9j/3btWhoGhxwSLhGJAiKVfPttGGfipZfgySfhlFOq4KBr1oRLTWPHhl9bv/oV/Pa3oXK8qm3cCG+9FV7ASy/B/Plhefv24UzmsMNCP1Xjx4c7O+rWhYEDQ1iccMK2/7klc7mHHxclQVASBosWlf7yb9QonBWXTN27h75vakAL5JpEAZFivv8+/MD+73/h3/8ONzZVieXLYfTo0Ox7jz1C3yCXXBIe745PPikNhNdfDy+gTp0QBoMGhQDo3HnbX3DFxeEa25NPwlNPhVsAc3LC3VynngrHHrv75ZLUt3lz+NKfPz9MH35Y+jf2rGCffbYNggMOCD86MvCST3VTQKSgNWvC9+uiReHKzEEHVeHB588PZxDPPAM/+lGoK6lbF2rVClPt2vEfx85nZYUzgZdeCnd5QGiPMWhQmI44ovJf8MXFoUJw/PgQFl9+GX4FnnACDB0ajlevXhW+AVLt1q3b9su/JBAWLty2bUCbNuHHRJcu4ZLPAQfA/vvrzDKJFBAp6vPPw3jja9bAf/6TgMaW774bgmLmzPCfdMuW8IuuuLhy+9etG+6QKgmFqrgVa+vWcInqySfhX/8K95w3aBCO36JFCKbs7NJpR/P16oXGhR07hklfNFXHPVwTXbEihPqXX277uGR+6dJwObFErVqw776lQdClS3jcuTM0bJi81yNxKSBS2OLFoc4sKyv8yK6W+uXi4vBFXRIYJeFRdr5t28Re792yJQzOPn48vPJKqHTfurW0fGWnynxWW7QoDYuyU/v24dKYBGvXhnv+S6ZPPtk+CDZu3H4/s3BL6J57QsuWoduYkhDo0iVcLtL7XGMoIFLcnDnhclNubmgH17JlskuUoty3D4/vvy9tSFh2WrKk9JZGCF9srVuHsNhzz7CsuDgct2Ta0bxZ+IWcnb1zf1u0CAHVrl2YqqvfrXXrwmWe2CBYsCAs+2GYxEirVuGS5J57ln75x3vcokV4TZIWFBA1wDvvhMrqzp3Dj+qS3jhkN2zdGirH44XHV1+FL/vYKSur4nmzEBIlZ187+lvyePPm0K1KrFq1wvX4ksCIDY+SqeRymTt89134xb+j6dtvwzXLRYtCEMTrriUvb9tp333DpLuDMpICooaYOBGGDAmXnF56SfW2aWXt2nCdfunScGazdOm2j4uKQpjEatIknMGsW1f5eqMGDUK7kw4d4geB6mikDHX3XUMceyw8/HBo7zZsWKjD1Zl8mmjYMNy107Vr/PUlZzslwVFS8VurVti3ZGrUaNv52GmPPdTbr1Qpff2kmNNOC13D/PrXcPbZoUmD/s9ngOzs0s7gdqsfFpGqo4BIQRdeCKtWhTZvzZuHwajUXkhEqpsCIkX9/vehHvWWW8JNI1ddlewSiUimUUCkKDO47bZwJnH11eFMYuTIZJdKRDKJAiKFZWXBQw+FuxbPOy/cgDJ8uC43iUj1UPVniqtTByZMCL0Tn356+PvUU9t2byMikggKiBqgfn2YPBnuuCPUSwwdGm5rHzu2dEAiEZGqpoCoIerVC3c3ffQRPP106DHikktCY9zLLtu2rzQRkaqggKhhsrPhpJNCn03TpoWhGG65BfbeO1yCmjkz2SUUkXSR0IAws4Fm9pGZLTSzK8vZZqiZzTOzuWb2eMzyrWY2K5qeT2Q5a6qDDgq9Zi9aFBrWvfAC9OoVeuh+4YXK984gIhJPwgLCzLKBO4FBQFdguJl1LbNNHnAV0MfduwEXx6xe7+49omlwosqZDjp0CGcRy5bBjTeGwBg8OPS8fPfdqqcQkV2zw4CwoO0uHLs3sNDdF7v7JmAcMKTMNucAd7r71wDuvmIXnkcijRvDpZeGMSYefzx0z/OrX4Xue/beG44/Hq64Ah55JAwW9913yS6xiKSyHbaDcHc3s4nA/jt57NZAbNVpEVB2YM0fA5jZO0A2MNrdX47W5ZhZIbAFuMHdny37BGY2EhgJ0K5du50sXvqqXTu0lxg2LHQj/sYbMHdumF59ddshEjp2DP3HdesWpq5dw5mHhosWkco2lJtpZgXuPj0Bz58HHA60Af5jZvu7+xqgvbsvN7O9gdfN7H13XxS7s7vfA9wDobvvKi5bjWcWhjTt27d02ZYtYayYefNKQ2PePJg0CTZtKt2uQ4fQu+xFF0GnTtVedBFJAZUNiIOA081sCfAdYISTi+4V7LMciL001SZaFqsIeM/dNwOfmNnHhMCY7u7LCU+y2MymAAcCi5DdUqtW6fDAP/1p6fKywTFrFtx3H9x1VwiKUaPCgEZqxS2SOSo1YJCZtY+33N2XVLBPLeBj4ChCMEwHTnP3uTHbDASGu/sIM2sB/A/oARQD37v7xmj5u8AQd59X3vOlw4BBqebLL+Ef/wgh8eWX4fLTqFFwxhkafEwkXVQ0YFCl7mKKgqAJcEI0NakoHKJ9tgAXAq8A84Hx7j7XzMaYWcldSa8Aq8xsHvAGcJm7rwK6AIVmNjtafkNF4SCJseeeoVfZJUvCQEZ168K554YhC66+GpaXPR8UkbRS2TOIUYQ7jp6OFp0E3OPudySwbDtFZxCJ5w5vvRV6mX322dCZ4CmnwMUXQ+/eyS6diOyK3R6T2szmAIe4+3fR/B7Auzuog6hWCojq9ckn8Le/hXqKb78NnQiOGhXqNWrXTnbpRKSydvsSE6FSOnZE9a3RMslQHTvCzTdDURHcfjusWBFuq917b3j00XC2ISI1W2UD4kHgPTMbbWajgWnA/QkrldQYDRuGbj4+/jh079G6NZx5JgwZAp99luzSicjuqExL6ixCIJwFrI6ms9x9bGKLJjVJVlZoqf3OO6Hbj0mTQsO7Rx7R2YRITbXDgHD3YkJ3GDPd/fZo+l81lE1qoOzs0A35nDkhIEaMgBNO0B1PIjVRZS8xvWZmJ5upmZRUTl4evPlmGNTo9ddDWDz0kM4mRGqSygbEucBTwEYz+9bM1prZtwksl6SB7OxwZ9OcOdC9O5x1Fhx3XKjYFpHUV9k6iIHunuXuddy9kbs3dPdG1VA+SQP77gtTpoS7nd58M5xNPPCAziZEUl1l6yD+Vg1lkTSWlRXudpozBw48EH75Sxg0SEOliqQy1UFItdpnn1AnceedYdjUbt1CYzudTYiknp2pgxiP6iCkCmRlwfnnw/vvQ34+nHNO6Cn2zjvDpaiVK5NdQhGBynf33Rg4Hejo7mPMrB3QKnHFkkzQsSNMngz33AO/+x289lrputzc7Qcy6tYtLBeR6lHZvpj+TuiC+0h372JmTYFX3b0g0QWsLPXFVLO5h7YSZQcymjs39PVUIjd328AoKAiTiOyaivpiqvSAQe7e08z+B+DuX5tZnSoroWQ8M2jTJkz9+5cujxccc+eG/p5KguOMM8IdUk2bJqfsIumqsgGx2cyyAQcws1zCGYVIQu0oOO65B66/Poy7ff/9MGBA8soqkm4qW0l9O/AM0NLMrgPeBq5PWKlEdqAkOMaMgWnToFEjGDgQzjsP1q1LdulE0kNlR5R7DLgc+DPwOXCiuz+VyIKJVFZ+PsyYAb/5TTij6N49DGwkIrunsmcQuPuH7n6nu//N3ecnslAiO6tePbjpptBS2wwOOwwuvRQ2bEh2yURqrkoHhEhN0K8fzJ4dxs6++Wbo2RN0c5vIrlFASNpp0AD+/nd4+eVwp9PBB8Mf/gCbNye7ZCI1iwJC0taAAfDBB3DaaaEy+6CDwryIVI4CQtJakyZhVLunnw7djPfqBX/9K2zdusNdRTKeAkIywkknhQZ2xx8PV1wRguLGG2HhwmSXTCR1KSAkY+TmwoQJ8NhjUKsWXH55GPmue3cYPTpUbqtXWZFSCQ0IMxtoZh+Z2UIzu7KcbYaa2Twzm2tmj8csH2FmC6JpRCLLKZnDLNRJFBbCp5/CrbeGLjrGjIEePcLgRpdeClOnQrH6CpAMV6nO+nbpwKFrjo+BY4AiYDow3N3nxWyTR+hG/Miof6eW7r7CzJoBhUA+oXuPGUAvd/+6vOdTZ32yO1asgOefD3UVkyeHO55+9CM48UT46U/h8MOhdu1kl1Kk6lXUWV8izyB6AwvdfbG7bwLGAUPKbHMOcGfJF7+7r4iWDwAmufvqaN0kYGACyyoZrmVLOPtsmDgxjEfx+OPQt2+o4O7fP6z/v/+Dl17SmYVkjkQGRGsgdkDJomhZrB8DPzazd8xsmpkN3Il9MbORZlZoZoUrNcqMVJHGjWH4cHjqKfjqK3juORgyBF58EY49Frp0gbvuUp9Pkv6SXUldC8gDDgeGA/eaWZPK7uzu97h7vrvn52okGUmAevVg8GB46CH44otwZtG4MVxwAbRtGyq6ly5NdilFEiORAbEcaBsz3yZaFqsIeN7dN7v7J4Q6i7xK7itSrerUCWcW770XKrH794dbboG994ahQ8My3QUl6SSRATEdyDOzjtHgQsOA58ts8yzh7AEza0G45LQYeAXob2ZNo9Hr+kfLRJLODA45BJ58EhYvDr3ITpoEffqE1tqPPQabNiW7lCK7L2EB4e5bgAsJX+zzgfHuPtfMxpjZ4GizV4BVZjYPeAO4zN1Xuftq4E+EkJkOjImWiaSUdu3gL38JrbTvuiv0/XTGGdChA1x3XajDEKmpEnaba3XTba6SCoqL4ZVX4Lbbwt+cnBAYQ4aERnkdO4ZLVSKpoqLbXBUQIgkyb14YK/uRR2D9+rAsOzucXeTlhenHPy593L59WC9SnRQQIkn07bchLD7+GBYsCFPJ49hbZevUCRXeJaHRqVOo/G7cOHlll/SngBBJQe7w5Zfxg2PhwjAaXvfu8OqrsOeeyS6tpKuKAqJWdRdGRAKz0J3Hj34EP/nJtuuKi8OAR6ecElp0T54cLkGJVKdkN5QTkTiyskKr7UmTwp1QffvChx8mu1SSaRQQIins0ENhypTQrqJfP5g5M9klkkyigBBJcQccAG+/DfXrwxFHwFtvJbtEkikUECI1QF5eCIlWrcJY2y+9lOwSSSZQQIjUEG3bhrOHzp1DB4Ljxye7RJLuFBAiNUhuLrzxBhx8MAwbBvfem+wSSTpTQIjUMI0bh248Bg6EkSPhppuSXSJJVwoIkRqofn149tnQ0vqyy+C3v1VX41L11FBOpIaqU6d0AKPrr4c1a+COO0IbCpGqoIAQqcGys+Ef/4AmTeDGG0O/Tw88ALVr73jf4uKw/TffhCknJ/QDJVJCASFSw5mFMSmaNoWrrw4tr/v1K/3iL29au3b7Y515Jtx8c6gMF1FAiKQBM7jqqnC56aKLQj9OtWuH+caNwxlG48ah07+SZWWnWbPgr3+FiRPDUKpnnhmOK5lLvbmKpJl168Klp5ycnf+Cnzs33Bk1dSocdRTcfTfsu29iyimpoaLeXFWdJZJmGjSAevV27dd/t26hMd7f/w7Tp8P++8MNN8DmzVVfTkl9CggR2UZWFpx3HsyfD8cdFy5d9eoF772X7JJJdVNAiEhce+0FEyaE9harV8Mhh4T6jXiV25KeFBAiUqEhQ8KQqRdcAH/7G3TtCs8/n+xSSXVQQIjIDjVqFBrhTZ0a7ogaMgR+9jP4/PNkl0wSSQEhIpV28MFh0KLrr4cXX4QuXeCee9TNR7pSQIjITqldO1Rcv/9+qLw+99zQ/fiKFckumVS1hAaEmQ00s4/MbKGZXRln/c/NbKWZzYqms2PWbY1ZriueIikmLy+MmX3bbeFv9+4ayCjdJCwgzCwbuBMYBHQFhptZ1zibPunuPaLpvpjl62OWD05UOUVk12VlhTubpk8P3XMce2yYX78+2SWTqpDIM4jewEJ3X+zum4BxwJAEPp+IJMn++4eQGDUqVGb37h0uQUnNlsiAaA0si5kvipaVdbKZzTGzCWbWNmZ5jpkVmtk0Mzsx3hOY2chom8KVK1dWXclFZKfl5MDYseEy08qVUFAQLj8VFye7ZLKrkl1J/QLQwd27A5OAh2PWtY/6BzkNGGtm+5Td2d3vcfd8d8/PVfeTIilh4MBw9tC/P1x8cbjspNtha6ZEBsRyIPaMoE207AfuvsrdN0az9wG9YtYtj/4uBqYAByawrCJShXJz4bnnQmd///lPqMBW47qaJ5EBMR3IM7OOZlYHGAZs8xExs1Yxs4OB+dHypmZWN3rcAugDzEtgWUWkipmFW2BnzoS2bUPjul/9Cr7/Ptklk8pKWEC4+xbgQuAVwhf/eHefa2ZjzKzkrqSLzGyumc0GLgJ+Hi3vAhRGy98AbnB3BYRIDdS5M0ybBpdfHka/69kzhIakPo0HISLV5vXX4f/+LzSqO+00aNUKmjUrf6pXL9klTn8VjQehEeVEpNoceSTMmVM66t3q1RWPNVG37vah8bOfwemna7S76qCAEJFq1awZPPpoeOwO330XgqIy05w5ofL7+edDBXizZsl9LelOASEiSWMWRsBr0ADatdvx9lu3wo03wjXXwLvvwiOPwBFHJL6cmSrZ7SBERCotOxuuvDKEQ/36YdzsK66ATZuSXbL0pIAQkRonPz/cCXXOOfDXv4bR7j78MNmlSj8KCBGpkfbYI9w2++yzsGRJuH327rs1NkVVUkCISI02ZEjo2qNfv9AQb8iQ0BeU7D4FhIjUeK1ahU4Cx46FV18Nvcu+/HKyS1XzKSBEJC1kZYXuxkvGphg0KMxrbIpdp4AQkbQSOzbF7bdrbIrdoYAQkbRTMjbFyy/DV1+Fu54efDDZpap5FBAikrYGDAitrw87DH7xC/jjH3WX085QQIhIWsvNhX//G846C0aPhrPPrrj/JymlrjZEJO3Vrg333x+68/jjH+Gzz2D8eGjYMNklS206gxCRjGAWziDuuw8mTQqXnTQUasUUECKSUX75S3jhBfj449BFx/z5yS5R6lJAiEjGGTQI3nwTNmyAPn3g7beTXaLUpIAQkYzUq1foFbZlSzj6aJgwIdklSj0KCBHJWB07wjvvhHYSQ4fCrbcmu0SpRQEhIhmtefNQaf3Tn8L/+39wySVQXJzsUqUG3eYqIhmvXj148kn4zW9CC+yiIvjnP0OL7MpYvRoWLICFC8PfNWvCXVJHH12zb6VVQIiIEEarGzsW2rcPZxJffBHGvy4Z97psCJT8XbAAvv669DhmIVhuuy20v+jXD449NkydO4f1NYV5mrQ7z8/P98LCwmQXQ0TSwPjxcOaZsNdeoRJ74cIQECXMQqO7vDzYd9/wt+Tx3nuHnmWnToWJE8P0wQdhvw4dQlAMGhTG0t5jj6S8vG2Y2Qx3z4+7TgEhIrK9t94Kl5waN44fAnXrVv5Yy5aF8SomToTJk+G778L+hx9eenax774JeykVSlpAmNlA4DYgG7jP3W8os/7nwI3A8mjR39z9vmjdCOB30fJr3f3hip5LASEiNcHGjSF8Ss4uPvooLM/Lg2HD4Le/3bnw2V1JCQgzywY+Bo4BioDpwHB3nxezzc+BfHe/sMy+zYBCIB9wYAbQy92/phwKCBGpiRYtCmcXL74Ir7wCBx0U2mS0aVM9z19RQCTyNtfewEJ3X+zum4BxwJBK7jsAmOTuq6NQmAQMTFA5RUSSZp994MILw9gVEybAvHnQsye8/nqyS5bYgGgNLIuZL4qWlXWymc0xswlm1nZn9jWzkWZWaGaFKzVKuYjUcCefXDpk6jHHwF/+ktzxK5LdUO4FoIO7dyecJVRYz1CWu9/j7vnunp+bm5uQAoqIVKdOneC99+CUU+DKK0MDvm++SU5ZEhkQy4G2MfNtKK2MBsDdV7n7xmj2PqBXZfcVEUlXDRrAE0+Erj9efBEKCkpvla1OiQyI6UCemXU0szrAMOD52A3MrFXM7GCgpOPdV4D+ZtbUzJoC/aNlIiIZwQwuvjjURaxdGyqvH3+8esuQsIBw9y3AhYQv9vnAeHefa2ZjzGxwtNlFZjbXzGYDFwE/j/ZdDfyJEDLTgTHRMhGRjNKvH8ycGXqfPf10uOgi2LSpep5bDeVERGqAzZvhiivCZadDDw2tvVvHu+1nJyXrNlcREakitWvDLbeETgVnzw63wk6ZktjnVECIiNQgQ4fCf/8LTZuG3mJvuilxt8IqIEREapiuXUNInHgiXHZZCI1EjGGh7r5FRGqgRo3gqafg5pvD+BNZCfi5r4AQEamhzODSSxN3fF1iEhGRuBQQIiISlwJCRETiUkCIiEhcCggREYlLASEiInEpIEREJC4FhIiIxJU2vbma2UpgCdAC+CrJxUk1ek+2p/dke3pP4kv396W9u8cdkjNtAqKEmRWW13VtptJ7sj29J9vTexJfJr8vusQkIiJxKSBERCSudAyIe5JdgBSk92R7ek+2p/ckvox9X9KuDkJERKpGOp5BiIhIFVBAiIhIXGkTEGY20Mw+MrOFZnZlssuTKszsUzN738xmmVlhssuTDGb2gJmtMLMPYpY1M7NJZrYg+ts0mWWsbuW8J6PNbHn0WZllZscms4zVzczamtkbZjbPzOaa2ahoecZ+VtIiIMwsG7gTGAR0BYabWdfkliqlHOHuPTL1Xm7gIWBgmWVXAq+5ex7wWjSfSR5i+/cE4Nbos9LD3SdWc5mSbQvwG3fvChwMXBB9j2TsZyUtAgLoDSx098XuvgkYBwxJcpkkRbj7f4DVZRYPAR6OHj8MnFidZUq2ct6TjObun7v7zOjxWmA+0JoM/qykS0C0BpbFzBdFywQceNXMZpjZyGQXJoXs6e6fR4+/APZMZmFSyIVmNie6BJUxl1LKMrMOwIHAe2TwZyVdAkLK19fdexIuv11gZj9JdoFSjYd7vXW/N/wd2AfoAXwO3JzU0iSJmTUA/gVc7O7fxq7LtM9KugTEcqBtzHybaFnGc/fl0d8VwDOEy3ECX5pZK4Do74oklyfp3P1Ld9/q7sXAvWTgZ8XMahPC4TF3fzpanLGflXQJiOlAnpl1NLM6wDDg+SSXKenMbA8za1jyGOgPfFDxXhnjeWBE9HgE8FwSy5ISSr4EIyeRYZ8VMzPgfmC+u98SsypjPytp05I6uiVvLJANPODu1yW3RMlnZnsTzhoAagGPZ+L7YmZPAIcTum3+EvgD8CwwHmhH6CZ+qLtnTKVtOe/J4YTLSw58Cpwbc+097ZlZX+At4H2gOFp8NaEeIiM/K2kTECIiUrXS5RKTiIhUMQWEiIjEpYAQEZG4FBAiIhKXAkJEROJSQIgkkZkdbmYvJrscIvEoIEREJC4FhEglmNkZZvbfaJyEf5hZtpmtM7Nbo7EDXjOz3GjbHmY2Ler07pmSTu/MbF8zm2xms81sppntEx2+gZlNMLMPzeyxqEUvZnZDNDbBHDO7KUkvXTKYAkJkB8ysC3Aq0MfdewBbgdOBPYBCd+8GvElojQzwCHCFu3cntMotWf4YcKe7HwAcSugQD0KvoRcTxjLZG+hjZs0J3V10i45zbSJfo0g8CgiRHTsK6AVMN7NZ0fzehO4Ynoy2eRToa2aNgSbu/ma0/GHgJ1GfWK3d/RkAd9/g7t9H2/zX3YuiTvJmAR2Ab4ANwP1m9lOgZFuRaqOAENkxAx6OGWmtk7uPjrPdrvZbszHm8VaglrtvIfSmOgE4Hnh5F48tsssUECI79hrwMzNrCT+MUdye8P/nZ9E2pwFvu/s3wNdm1i9afibwZjRCWZGZnRgdo66Z1S/vCaMxCRpHw35eAhyQgNclUqFayS6ASKpz93lm9jvCyHxZwGbgAuA7oHe0bgWhngJCl9B3RwGwGDgrWn4m8A8zGxMd45QKnrYh8JyZ5RDOYP5fFb8skR1Sb64iu8jM1rl7g2SXQyRRdIlJRETi0hmEiIjEpTMIERGJSwEhIiJxKSBERCQuBYSIiMSlgBARkbj+P3o1TIbDBYQUAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best model after 13 epochs with dev set error 0.6694179773330688\n"
     ]
    }
   ],
   "source": [
    "idxs_text = torch.arange(300) # text, 300\n",
    "idxs_speech = torch.arange(300, 381) # speech, 81\n",
    "idxs_visual = torch.arange(381, 752) # visual, 371\n",
    "idxs_text_speech = torch.arange(381) # text + speech, 381\n",
    "idxs_text_visual = torch.cat((torch.arange(300), torch.arange(381, 752))) # text + visual, 671\n",
    "idxs_speech_visual = torch.arange(300, 752) # speech + visual, 452\n",
    "idxs_all = torch.arange(752) # text + speech + visual, 752\n",
    "\n",
    "model = MultiLayer(input_dim=381, hidden_layer_size=256)\n",
    "model.to(device)\n",
    "\n",
    "optimiser = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=0.002)\n",
    "epochs = 500\n",
    "patience = 10\n",
    "\n",
    "print('step', 'error')\n",
    "dev_errors = []\n",
    "best_dev_error = np.inf\n",
    "errors = []\n",
    "num_bad_epochs = 0\n",
    "num_epochs = 0\n",
    "best_epoch = 0\n",
    "for epoch in range(1, epochs+1):\n",
    "    model.train()\n",
    "    for batch_idx, batch in enumerate(train_dataloader, 0):\n",
    "        x_p,x_c,y,hid,x_p_len=map(lambda x: x.to(device), batch)\n",
    "        optimiser.zero_grad()\n",
    "        output = model(x_p[:, :, idxs_text_speech])\n",
    "        error = torch.nn.functional.binary_cross_entropy_with_logits(output, y)\n",
    "        error.backward()\n",
    "        optimiser.step()\n",
    "    \n",
    "    print(epoch, error.detach().tolist())\n",
    "    errors.append(error.detach().tolist())\n",
    "\n",
    "    with torch.no_grad():\n",
    "        model.eval()\n",
    "        matches = 0\n",
    "        total = 0\n",
    "        for batch_idx, batch in enumerate(dev_dataloader, 0):\n",
    "            x_p,x_c,y,hid,x_p_len=map(lambda x: x.to(device), batch)\n",
    "            dev_output = model(x_p[:, :, idxs_text_speech])\n",
    "            dev_error = torch.nn.functional.binary_cross_entropy_with_logits(dev_output, y)\n",
    "        dev_errors.append(float(dev_error))\n",
    "        print('dev error: ', float(dev_error))\n",
    "\n",
    "    if dev_error < best_dev_error:\n",
    "        num_bad_epochs = 0\n",
    "        best_dev_error = dev_error\n",
    "        best_epoch = epoch\n",
    "        torch.save(model, 'punchline_multilayer_baseline_text-speech.pth')\n",
    "    else:\n",
    "        num_bad_epochs += 1\n",
    "        if num_bad_epochs == patience:\n",
    "            num_epochs = epoch\n",
    "            break\n",
    "\n",
    "plt.plot(range(1, num_epochs+1), dev_errors, 'r', label='dev error')\n",
    "plt.plot(range(1, num_epochs+1), errors, 'b', label='train error')\n",
    "plt.title('train and dev error')\n",
    "plt.xlabel('epochs')\n",
    "plt.ylabel('error')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "print('best model after {} epochs with dev set error {}'.format(best_epoch, best_dev_error))\n",
    "rnn = torch.load('punchline_multilayer_baseline_text-speech.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3364198/4016921915.py:102: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  context_w=np.array(self.language_sdk[hid]['context_embedding_indexes'])\n",
      "/tmp/ipykernel_3364198/4016921915.py:103: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  context_of=np.array(self.word_aligned_openface_sdk[hid]['context_features'])\n",
      "/tmp/ipykernel_3364198/4016921915.py:104: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  context_cvp=np.array(self.word_aligned_covarep_sdk[hid]['context_features'])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test error:  0.6613044142723083\n",
      "test accuracy: 62.24%\n"
     ]
    }
   ],
   "source": [
    "hid_humor_t = {}\n",
    "hid_humor_f = {}\n",
    "hid_nonhumor_t = {}\n",
    "hid_nonhumor_f = {}\n",
    "\n",
    "with torch.no_grad():\n",
    "    matches = 0\n",
    "    total = 0\n",
    "    for batch_idx, batch in enumerate(test_dataloader, 0): \n",
    "        x_p,x_c,y,hid, x_p_len=map(lambda x: x.to(device), batch)\n",
    "        output = model(x_p[:, :, idxs_text_speech])\n",
    "        error = torch.nn.functional.binary_cross_entropy_with_logits(output, y)\n",
    "        scaled_outputs = torch.sigmoid(output)\n",
    "        predictions = (scaled_outputs > 0.5)*1\n",
    "        matches += (y == predictions).sum()\n",
    "        total += len(y)\n",
    "        for j, id in enumerate(hid.detach().tolist()):\n",
    "            if predictions[j] == 1 and y[j] == 1:\n",
    "                hid_humor_t[id] = scaled_outputs[j].detach().tolist()[0]\n",
    "            elif predictions[j] == 1 and y[j] == 0:\n",
    "                hid_humor_f[id] = scaled_outputs[j].detach().tolist()[0]\n",
    "            elif predictions[j] == 0 and y[j] == 0:\n",
    "                hid_nonhumor_t[id] = scaled_outputs[j].detach().tolist()[0]\n",
    "            elif predictions[j] == 0 and y[j] == 1:\n",
    "                hid_nonhumor_f[id] = scaled_outputs[j].detach().tolist()[0]\n",
    "            else:\n",
    "                print('something is wrong')\n",
    "    acc = matches / total\n",
    "    print('test error: ', float(error))\n",
    "    print('test accuracy: {:.2%}'.format(float(acc)))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "36cf16204b8548560b1c020c4e8fb5b57f0e4c58016f52f2d4be01e192833930"
  },
  "kernelspec": {
   "display_name": "Python 3.9.9 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}