anomalyDarkJet.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Importing all the required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "require(['notebook'],\n",
       "  function() {\n",
       "    IPython.CodeCell.config_defaults.highlight_modes['magic_text/x-c++src'] = {'reg':[/^%%cpp/]};\n",
       "    console.log(\"JupyROOT - %%cpp magic configured\");\n",
       "  }\n",
       ");\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Welcome to JupyROOT 6.07/07\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "import math\n",
    "import numpy as np\n",
    "import h5py\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy\n",
    "from scipy import ndimage\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "from tensorflow.python.framework import ops\n",
    "#from cnn_utils import *\n",
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES']='-1'\n",
    "import deepdish.io as io\n",
    "from keras.models import Sequential, Model\n",
    "#from keras.layers import Input, Add, Dense, Dropout, LeakyReLU\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "#from keras.utils import multi_gpu_model\n",
    "from sklearn.model_selection import cross_val_score, cross_val_predict, train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn import metrics\n",
    "from itertools import izip\n",
    "import random\n",
    "import pandautils as pup\n",
    "import pandas as pd\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Looking inside the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "final_jj_1MEvents.h5\r\n"
     ]
    }
   ],
   "source": [
    "ls /data/debo/jetAnomaly/UdeM_LPS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SAMPLE_PATH = os.path.join('/data/debo/jetAnomaly/UdeM_LPS')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['final_jj_1MEvents.h5']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir(SAMPLE_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/debo/env2/local/lib/python2.7/site-packages/tables/group.py:329: UserWarning: leaf ``/objects/e`` is of an unsupported type; it will become an ``UnImplemented`` node\n",
      "  % self._g_join(childname))\n",
      "/home/debo/env2/local/lib/python2.7/site-packages/tables/group.py:329: UserWarning: leaf ``/objects/gamma`` is of an unsupported type; it will become an ``UnImplemented`` node\n",
      "  % self._g_join(childname))\n",
      "/home/debo/env2/local/lib/python2.7/site-packages/tables/group.py:329: UserWarning: leaf ``/objects/jets/constituents`` is of an unsupported type; it will become an ``UnImplemented`` node\n",
      "  % self._g_join(childname))\n",
      "/home/debo/env2/local/lib/python2.7/site-packages/tables/group.py:329: UserWarning: leaf ``/objects/mu`` is of an unsupported type; it will become an ``UnImplemented`` node\n",
      "  % self._g_join(childname))\n"
     ]
    }
   ],
   "source": [
    "EventInfo = io.load(os.path.join(SAMPLE_PATH,'final_jj_1MEvents.h5'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['high_level', 'objects', 'partons']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "EventInfo.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['jets', 'mu', 'e', 'gamma']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "EventInfo['objects'].keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We will only take jets from each event"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "objects = EventInfo['objects']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['constituents', 'obs']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "objects['jets'].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'constituents': None,\n",
       " 'obs': array([[[ 5.39563283e+02, -9.23743352e-01, -1.68751582e+00, ...,\n",
       "           4.43915892e+00,  3.24084687e+00,  2.17888212e+00],\n",
       "         [ 4.60895568e+02, -1.33517978e+00,  1.39145153e+00, ...,\n",
       "           1.15711622e+01,  9.73054123e+00,  7.97088432e+00]],\n",
       " \n",
       "        [[ 7.27789084e+02, -1.65506998e+00,  1.00065047e+00, ...,\n",
       "           1.41101923e+01,  1.08286705e+01,  9.35908318e+00],\n",
       "         [ 6.24550581e+02, -1.94968494e-01, -2.15542965e+00, ...,\n",
       "           2.28335209e+01,  2.01812935e+01,  1.57388792e+01]],\n",
       " \n",
       "        [[ 4.72478786e+02, -1.46958756e+00, -2.33395113e+00, ...,\n",
       "           9.42582321e+00,  8.68241215e+00,  7.48327303e+00],\n",
       "         [ 4.48907604e+02,  1.63198306e+00,  6.18280904e-01, ...,\n",
       "           4.18570518e+01,  3.71974640e+01,  2.80914612e+01]],\n",
       " \n",
       "        ...,\n",
       " \n",
       "        [[ 4.79685628e+02,  1.58962283e+00,  2.86841754e-01, ...,\n",
       "           9.75549412e+00,  7.92058420e+00,  6.64717293e+00],\n",
       "         [ 4.06458914e+02,  7.81083118e-01, -1.99905830e+00, ...,\n",
       "           5.24107056e+01,  4.45202713e+01,  3.93323593e+01]],\n",
       " \n",
       "        [[ 5.94529968e+02, -1.37682577e+00, -1.95065062e+00, ...,\n",
       "           1.93707466e+01,  1.71963005e+01,  1.40770445e+01],\n",
       "         [ 5.61226179e+02, -5.00789036e-01,  1.14212191e+00, ...,\n",
       "           1.22992420e+01,  1.10096836e+01,  1.00030203e+01]],\n",
       " \n",
       "        [[ 1.12842868e+03,  1.21833202e+00, -1.84087801e+00, ...,\n",
       "           8.59270554e+01,  6.27936020e+01,  4.98363686e+01],\n",
       "         [ 1.12200983e+03,  4.01905617e-01,  1.32837664e+00, ...,\n",
       "           3.76097336e+01,  2.82051907e+01,  2.41330872e+01]]])}"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "objects['jets']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The 'constituents' are always 'None' and 'obs' seems jets 4momentum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "jets = objects['jets']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['constituents', 'obs']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jets.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "jets4Mom = jets['obs']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(705643, 2, 9)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jets4Mom.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[539.56328263,  -0.92374335,  -1.68751582,  35.72464581,\n",
       "         14.73322678,   5.4043684 ,   4.43915892,   3.24084687,\n",
       "          2.17888212],\n",
       "       [460.89556826,  -1.33517978,   1.39145153,  42.94837991,\n",
       "         14.90651512,  12.10266972,  11.57116222,   9.73054123,\n",
       "          7.97088432]])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jets4Mom[0,:,:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Does it seems each event has 2 jets and 9 variables correspond to pt, eta, phi, m, tau1, tau2, tau3, tau4, tau5 as mentioned in https://gitlab.com/atlas-anomaly-detection/atlas-anomaly-detection-colab/blob/master/data/UdeM-LPS_dataset.md\n",
    " So for the dataset I will preparing should I keep only 1st 4 collumns pt, eta, phi, m (converted to E,px,py,pz)and only the leading jet so each entry will then have single jet??"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env2",
   "language": "python",
   "name": "env2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}