notebooks/07 - Data Mapping, Exploring Metadata Tables.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# More mappers!\n",
    "\n",
    "# Mapping here simply means converting an existing value that's already in the data into another value using a dictionary as a lookup.\n",
    "\n",
    "# Examples of this could be mapping similar AIS Codes to standard group names, or adding a dynamic string descriptor based on the speed of the agent (e.g. slow, medium, fast). This is where you can get creative.\n",
    "___"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import trackio as tio"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "## Let's read the dataset from the last notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Type:\n",
       "    <class 'trackio.Dataset.Dataset'>\n",
       "Status:\n",
       "    0 Unprocessed CSV Files\n",
       "    0 Processed CSV Files\n",
       "    0 Unsplit Agent Files\n",
       "    172 Split Agent Files\n",
       "Static Data Fields:\n",
       "    ['AISCode', 'Agent ID', 'Length', 'MMSI', 'Name', 'Width']\n",
       "Dynamic Data Fields:\n",
       "    ['Coursing', 'Draft', 'Speed', 'Status', 'Time', 'X', 'Y']\n",
       "Metadata:\n",
       "    X: degrees\n",
       "    Y: degrees\n",
       "    CRS: EPSG:4326\n",
       "Data Path:\n",
       "    c:\\code\\trackio\\notebooks\\example0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#define the data_path\n",
    "data_path = 'example0'\n",
    "\n",
    "#read the existing Dataset\n",
    "ds = tio.read(data_path=data_path)\n",
    "\n",
    "#check the metadata available\n",
    "ds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "____\n",
    "## First, you can map metadata (static data) for the agents in your `Dataset`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[92mMaking meta mappers\u001b[0m: 100%|\u001b[32m██████████\u001b[0m| 172/172 [00:02<00:00, 64.06it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{0.0: None,\n",
       " 11.0: None,\n",
       " 12.0: None,\n",
       " 13.0: None,\n",
       " 15.0: None,\n",
       " 16.0: None,\n",
       " 17.0: None,\n",
       " 18.0: None,\n",
       " 19.0: None,\n",
       " 20.0: None,\n",
       " 21.0: None,\n",
       " 22.0: None,\n",
       " 23.0: None,\n",
       " 24.0: None,\n",
       " 25.0: None,\n",
       " 26.0: None,\n",
       " 27.0: None,\n",
       " 28.0: None,\n",
       " 29.0: None,\n",
       " 30.0: None,\n",
       " 31.0: None,\n",
       " 32.0: None,\n",
       " 33.0: None,\n",
       " 34.0: None,\n",
       " 35.0: None,\n",
       " 36.0: None,\n",
       " 37.0: None,\n",
       " 38.0: None,\n",
       " 39.0: None,\n",
       " 42.0: None,\n",
       " 44.0: None,\n",
       " 48.0: None,\n",
       " 52.0: None,\n",
       " 60.0: None,\n",
       " 84.0: None,\n",
       " 88.0: None,\n",
       " 94.0: None,\n",
       " 159.0: None,\n",
       " 169.0: None,\n",
       " 170.0: None,\n",
       " 227.0: None,\n",
       " 228.0: None,\n",
       " 240.0: None,\n",
       " 248.0: None,\n",
       " 333.0: None,\n",
       " nan: None}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#make metadata mappers\n",
    "meta_mapper = ds.make_meta_mapper(['Length'], \n",
    "                                  ncores=4)\n",
    "\n",
    "meta_mapper"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "## Or make multiple at once."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[92mMaking meta mappers\u001b[0m: 100%|\u001b[32m██████████\u001b[0m| 172/172 [00:02<00:00, 76.91it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'Length': {0.0: None,\n",
       "  11.0: None,\n",
       "  12.0: None,\n",
       "  13.0: None,\n",
       "  15.0: None,\n",
       "  16.0: None,\n",
       "  17.0: None,\n",
       "  18.0: None,\n",
       "  19.0: None,\n",
       "  20.0: None,\n",
       "  21.0: None,\n",
       "  22.0: None,\n",
       "  23.0: None,\n",
       "  24.0: None,\n",
       "  25.0: None,\n",
       "  26.0: None,\n",
       "  27.0: None,\n",
       "  28.0: None,\n",
       "  29.0: None,\n",
       "  30.0: None,\n",
       "  31.0: None,\n",
       "  32.0: None,\n",
       "  33.0: None,\n",
       "  34.0: None,\n",
       "  35.0: None,\n",
       "  36.0: None,\n",
       "  37.0: None,\n",
       "  38.0: None,\n",
       "  39.0: None,\n",
       "  42.0: None,\n",
       "  44.0: None,\n",
       "  48.0: None,\n",
       "  52.0: None,\n",
       "  60.0: None,\n",
       "  84.0: None,\n",
       "  88.0: None,\n",
       "  94.0: None,\n",
       "  159.0: None,\n",
       "  169.0: None,\n",
       "  170.0: None,\n",
       "  227.0: None,\n",
       "  228.0: None,\n",
       "  240.0: None,\n",
       "  248.0: None,\n",
       "  333.0: None,\n",
       "  nan: None},\n",
       " 'Width': {0.0: None,\n",
       "  4.0: None,\n",
       "  5.0: None,\n",
       "  6.0: None,\n",
       "  7.0: None,\n",
       "  8.0: None,\n",
       "  9.0: None,\n",
       "  10.0: None,\n",
       "  11.0: None,\n",
       "  12.0: None,\n",
       "  14.0: None,\n",
       "  18.0: None,\n",
       "  21.0: None,\n",
       "  26.0: None,\n",
       "  27.0: None,\n",
       "  32.0: None,\n",
       "  33.0: None,\n",
       "  37.0: None,\n",
       "  42.0: None,\n",
       "  nan: None}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#make metadata mappers\n",
    "meta_mappers = ds.make_meta_mapper(['Length', 'Width'], \n",
    "                                   ncores=4)\n",
    "\n",
    "meta_mappers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "## Now make up some mapped data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#descriptor by width\n",
    "for k in meta_mappers['Width'].keys():\n",
    "      if k <= 20:\n",
    "            meta_mappers['Width'][k] = 'narrow'\n",
    "      else:\n",
    "            meta_mappers['Width'][k] = 'wide'      \n",
    "            \n",
    "#descript by length\n",
    "for k in  meta_mappers['Length'].keys():\n",
    "      if k <= 20:\n",
    "            meta_mappers['Length'][k] = 'short'\n",
    "      else:\n",
    "            meta_mappers['Length'][k] = 'long' "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "## Now you can map these to the agents metadata."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[92mMapping metadata to agent\u001b[0m: 100%|\u001b[32m██████████\u001b[0m| 172/172 [00:02<00:00, 81.29it/s]\n"
     ]
    }
   ],
   "source": [
    "#map metadata\n",
    "ds = ds.map_meta(['Width'], \n",
    "                 ['How Wide?'], \n",
    "                 meta_mappers,\n",
    "                 ncores=4,\n",
    "                 fill='Unknown')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[92mMapping metadata to agent\u001b[0m: 100%|\u001b[32m██████████\u001b[0m| 172/172 [00:02<00:00, 57.79it/s]\n"
     ]
    }
   ],
   "source": [
    "#map multiple metadata at once\n",
    "ds = ds.map_meta(['Width', 'Length'], \n",
    "                 ['How Wide?', 'How Long?'], \n",
    "                 meta_mappers,\n",
    "                 ncores=4,\n",
    "                 fill='Unknown')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "## Now if you refresh the metadata tables, you'll see this data in the `agent.db` file by accessing `Dataset.agents`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[92mRefreshing metadata\u001b[0m: 100%|\u001b[32m██████████\u001b[0m| 172/172 [00:03<00:00, 52.88it/s]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New meta/databases saved to c:\\code\\trackio\\notebooks\\example0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MMSI</th>\n",
       "      <th>Name</th>\n",
       "      <th>AISCode</th>\n",
       "      <th>Length</th>\n",
       "      <th>Width</th>\n",
       "      <th>How Wide?</th>\n",
       "      <th>How Long?</th>\n",
       "      <th>npoints</th>\n",
       "      <th>ntracks</th>\n",
       "      <th>Xmin</th>\n",
       "      <th>Xmax</th>\n",
       "      <th>Ymin</th>\n",
       "      <th>Ymax</th>\n",
       "      <th>Start Time</th>\n",
       "      <th>End Time</th>\n",
       "      <th>File</th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Agent ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Vessel_367796050</th>\n",
       "      <td>367796050</td>\n",
       "      <td>SEA LION</td>\n",
       "      <td>31.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>narrow</td>\n",
       "      <td>short</td>\n",
       "      <td>1242</td>\n",
       "      <td>1</td>\n",
       "      <td>-74.07765</td>\n",
       "      <td>-74.07759</td>\n",
       "      <td>40.68621</td>\n",
       "      <td>40.68627</td>\n",
       "      <td>2021-01-01 00:17:17</td>\n",
       "      <td>2021-01-01 23:59:07</td>\n",
       "      <td>c:\\code\\trackio\\notebooks\\example0\\Vessel_3677...</td>\n",
       "      <td>POLYGON ((-74.07759 40.68621, -74.07759 40.686...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_368111610</th>\n",
       "      <td>368111610</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "      <td>-74.07277</td>\n",
       "      <td>-74.01703</td>\n",
       "      <td>40.62842</td>\n",
       "      <td>40.71297</td>\n",
       "      <td>2021-01-01 02:21:15</td>\n",
       "      <td>2021-01-01 03:02:17</td>\n",
       "      <td>c:\\code\\trackio\\notebooks\\example0\\Vessel_3681...</td>\n",
       "      <td>POLYGON ((-74.01703 40.62842, -74.01703 40.712...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_367069240</th>\n",
       "      <td>367069240</td>\n",
       "      <td>C F CAMPBELL</td>\n",
       "      <td>31.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>narrow</td>\n",
       "      <td>long</td>\n",
       "      <td>324</td>\n",
       "      <td>1</td>\n",
       "      <td>-74.1227</td>\n",
       "      <td>-73.95377</td>\n",
       "      <td>40.63918</td>\n",
       "      <td>40.76081</td>\n",
       "      <td>2021-01-01 00:19:43</td>\n",
       "      <td>2021-01-01 09:36:17</td>\n",
       "      <td>c:\\code\\trackio\\notebooks\\example0\\Vessel_3670...</td>\n",
       "      <td>POLYGON ((-73.95377 40.63918, -73.95377 40.760...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_367758160</th>\n",
       "      <td>367758160</td>\n",
       "      <td>S/V NO DOUBT</td>\n",
       "      <td>36.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>narrow</td>\n",
       "      <td>short</td>\n",
       "      <td>347</td>\n",
       "      <td>1</td>\n",
       "      <td>-74.04491</td>\n",
       "      <td>-74.0448</td>\n",
       "      <td>40.71038</td>\n",
       "      <td>40.71047</td>\n",
       "      <td>2021-01-01 00:19:53</td>\n",
       "      <td>2021-01-01 23:55:56</td>\n",
       "      <td>c:\\code\\trackio\\notebooks\\example0\\Vessel_3677...</td>\n",
       "      <td>POLYGON ((-74.04480 40.71038, -74.04480 40.710...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_338862000</th>\n",
       "      <td>338862000</td>\n",
       "      <td>FORT SCHUYLER</td>\n",
       "      <td>31.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>long</td>\n",
       "      <td>175</td>\n",
       "      <td>1</td>\n",
       "      <td>-74.12821</td>\n",
       "      <td>-74.02601</td>\n",
       "      <td>40.64329</td>\n",
       "      <td>40.70685</td>\n",
       "      <td>2021-01-01 00:17:13</td>\n",
       "      <td>2021-01-01 21:39:36</td>\n",
       "      <td>c:\\code\\trackio\\notebooks\\example0\\Vessel_3388...</td>\n",
       "      <td>POLYGON ((-74.02601 40.64329, -74.02601 40.706...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       MMSI           Name AISCode Length Width How Wide?  \\\n",
       "Agent ID                                                                    \n",
       "Vessel_367796050  367796050       SEA LION    31.0   19.0   6.0    narrow   \n",
       "Vessel_368111610  368111610            NaN     NaN    NaN   NaN   Unknown   \n",
       "Vessel_367069240  367069240   C F CAMPBELL    31.0   32.0   9.0    narrow   \n",
       "Vessel_367758160  367758160   S/V NO DOUBT    36.0   16.0   4.0    narrow   \n",
       "Vessel_338862000  338862000  FORT SCHUYLER    31.0   29.0   NaN   Unknown   \n",
       "\n",
       "                 How Long? npoints ntracks      Xmin      Xmax      Ymin  \\\n",
       "Agent ID                                                                   \n",
       "Vessel_367796050     short    1242       1 -74.07765 -74.07759  40.68621   \n",
       "Vessel_368111610   Unknown      35       1 -74.07277 -74.01703  40.62842   \n",
       "Vessel_367069240      long     324       1  -74.1227 -73.95377  40.63918   \n",
       "Vessel_367758160     short     347       1 -74.04491  -74.0448  40.71038   \n",
       "Vessel_338862000      long     175       1 -74.12821 -74.02601  40.64329   \n",
       "\n",
       "                      Ymax          Start Time            End Time  \\\n",
       "Agent ID                                                             \n",
       "Vessel_367796050  40.68627 2021-01-01 00:17:17 2021-01-01 23:59:07   \n",
       "Vessel_368111610  40.71297 2021-01-01 02:21:15 2021-01-01 03:02:17   \n",
       "Vessel_367069240  40.76081 2021-01-01 00:19:43 2021-01-01 09:36:17   \n",
       "Vessel_367758160  40.71047 2021-01-01 00:19:53 2021-01-01 23:55:56   \n",
       "Vessel_338862000  40.70685 2021-01-01 00:17:13 2021-01-01 21:39:36   \n",
       "\n",
       "                                                               File  \\\n",
       "Agent ID                                                              \n",
       "Vessel_367796050  c:\\code\\trackio\\notebooks\\example0\\Vessel_3677...   \n",
       "Vessel_368111610  c:\\code\\trackio\\notebooks\\example0\\Vessel_3681...   \n",
       "Vessel_367069240  c:\\code\\trackio\\notebooks\\example0\\Vessel_3670...   \n",
       "Vessel_367758160  c:\\code\\trackio\\notebooks\\example0\\Vessel_3677...   \n",
       "Vessel_338862000  c:\\code\\trackio\\notebooks\\example0\\Vessel_3388...   \n",
       "\n",
       "                                                           geometry  \n",
       "Agent ID                                                             \n",
       "Vessel_367796050  POLYGON ((-74.07759 40.68621, -74.07759 40.686...  \n",
       "Vessel_368111610  POLYGON ((-74.01703 40.62842, -74.01703 40.712...  \n",
       "Vessel_367069240  POLYGON ((-73.95377 40.63918, -73.95377 40.760...  \n",
       "Vessel_367758160  POLYGON ((-74.04480 40.71038, -74.04480 40.710...  \n",
       "Vessel_338862000  POLYGON ((-74.02601 40.64329, -74.02601 40.706...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#refresh metadata\n",
    "ds = ds.refresh_meta(ncores=4)\n",
    "\n",
    "ds.agents.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "## You can also do the same thing with any dynamic data field."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[92mMaking data mappers\u001b[0m: 100%|\u001b[32m██████████\u001b[0m| 172/172 [00:03<00:00, 52.66it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'Draft': {2.3: 'shallow',\n",
       "  2.5: 'shallow',\n",
       "  2.7: 'shallow',\n",
       "  2.9: 'shallow',\n",
       "  3.0: 'shallow',\n",
       "  3.3: 'shallow',\n",
       "  3.4: 'shallow',\n",
       "  3.5: 'shallow',\n",
       "  3.6: 'shallow',\n",
       "  3.7: 'shallow',\n",
       "  3.8: 'shallow',\n",
       "  3.9: 'shallow',\n",
       "  4.0: 'shallow',\n",
       "  4.1: 'shallow',\n",
       "  4.2: 'shallow',\n",
       "  4.3: 'shallow',\n",
       "  4.4: 'shallow',\n",
       "  4.5: 'shallow',\n",
       "  4.6: 'shallow',\n",
       "  4.9: 'shallow',\n",
       "  5.0: 'shallow',\n",
       "  5.2: 'deep',\n",
       "  5.5: 'deep',\n",
       "  5.8: 'deep',\n",
       "  6.1: 'deep',\n",
       "  9.4: 'deep',\n",
       "  9.8: 'deep',\n",
       "  9.9: 'deep',\n",
       "  10.1: 'deep',\n",
       "  12.0: 'deep',\n",
       "  14.0: 'deep',\n",
       "  14.5: 'deep',\n",
       "  14.9: 'deep',\n",
       "  nan: 'deep'},\n",
       " 'Speed': {0.0: 'slow',\n",
       "  0.1: 'slow',\n",
       "  0.2: 'slow',\n",
       "  0.3: 'slow',\n",
       "  0.4: 'slow',\n",
       "  0.5: 'slow',\n",
       "  0.6: 'slow',\n",
       "  0.7: 'slow',\n",
       "  0.8: 'slow',\n",
       "  0.9: 'slow',\n",
       "  1.0: 'slow',\n",
       "  1.1: 'slow',\n",
       "  1.2: 'slow',\n",
       "  1.3: 'slow',\n",
       "  1.4: 'slow',\n",
       "  1.5: 'slow',\n",
       "  1.6: 'slow',\n",
       "  1.7: 'slow',\n",
       "  1.8: 'slow',\n",
       "  1.9: 'slow',\n",
       "  2.0: 'slow',\n",
       "  2.1: 'slow',\n",
       "  2.2: 'slow',\n",
       "  2.3: 'slow',\n",
       "  2.4: 'slow',\n",
       "  2.5: 'slow',\n",
       "  2.6: 'slow',\n",
       "  2.7: 'slow',\n",
       "  2.8: 'slow',\n",
       "  2.9: 'slow',\n",
       "  3.0: 'slow',\n",
       "  3.1: 'slow',\n",
       "  3.2: 'slow',\n",
       "  3.3: 'slow',\n",
       "  3.4: 'slow',\n",
       "  3.5: 'slow',\n",
       "  3.6: 'slow',\n",
       "  3.7: 'slow',\n",
       "  3.8: 'slow',\n",
       "  3.9: 'slow',\n",
       "  4.0: 'slow',\n",
       "  4.1: 'slow',\n",
       "  4.2: 'slow',\n",
       "  4.3: 'slow',\n",
       "  4.4: 'slow',\n",
       "  4.5: 'slow',\n",
       "  4.6: 'slow',\n",
       "  4.7: 'slow',\n",
       "  4.8: 'slow',\n",
       "  4.9: 'slow',\n",
       "  5.0: 'slow',\n",
       "  5.1: 'fast',\n",
       "  5.2: 'fast',\n",
       "  5.3: 'fast',\n",
       "  5.4: 'fast',\n",
       "  5.5: 'fast',\n",
       "  5.6: 'fast',\n",
       "  5.7: 'fast',\n",
       "  5.8: 'fast',\n",
       "  5.9: 'fast',\n",
       "  6.0: 'fast',\n",
       "  6.1: 'fast',\n",
       "  6.2: 'fast',\n",
       "  6.3: 'fast',\n",
       "  6.4: 'fast',\n",
       "  6.5: 'fast',\n",
       "  6.6: 'fast',\n",
       "  6.7: 'fast',\n",
       "  6.8: 'fast',\n",
       "  6.9: 'fast',\n",
       "  7.0: 'fast',\n",
       "  7.1: 'fast',\n",
       "  7.2: 'fast',\n",
       "  7.3: 'fast',\n",
       "  7.4: 'fast',\n",
       "  7.5: 'fast',\n",
       "  7.6: 'fast',\n",
       "  7.7: 'fast',\n",
       "  7.8: 'fast',\n",
       "  7.9: 'fast',\n",
       "  8.0: 'fast',\n",
       "  8.1: 'fast',\n",
       "  8.2: 'fast',\n",
       "  8.3: 'fast',\n",
       "  8.4: 'fast',\n",
       "  8.5: 'fast',\n",
       "  8.6: 'fast',\n",
       "  8.7: 'fast',\n",
       "  8.8: 'fast',\n",
       "  8.9: 'fast',\n",
       "  9.0: 'fast',\n",
       "  9.1: 'fast',\n",
       "  9.2: 'fast',\n",
       "  9.3: 'fast',\n",
       "  9.4: 'fast',\n",
       "  9.5: 'fast',\n",
       "  9.6: 'fast',\n",
       "  9.7: 'fast',\n",
       "  9.8: 'fast',\n",
       "  9.9: 'fast',\n",
       "  10.0: 'fast',\n",
       "  10.1: 'fast',\n",
       "  10.2: 'fast',\n",
       "  10.3: 'fast',\n",
       "  10.4: 'fast',\n",
       "  10.5: 'fast',\n",
       "  10.6: 'fast',\n",
       "  10.7: 'fast',\n",
       "  10.8: 'fast',\n",
       "  10.9: 'fast',\n",
       "  11.0: 'fast',\n",
       "  11.1: 'fast',\n",
       "  11.2: 'fast',\n",
       "  11.3: 'fast',\n",
       "  11.4: 'fast',\n",
       "  11.5: 'fast',\n",
       "  11.6: 'fast',\n",
       "  11.7: 'fast',\n",
       "  11.8: 'fast',\n",
       "  11.9: 'fast',\n",
       "  12.0: 'fast',\n",
       "  12.1: 'fast',\n",
       "  12.2: 'fast',\n",
       "  12.3: 'fast',\n",
       "  12.4: 'fast',\n",
       "  12.5: 'fast',\n",
       "  12.6: 'fast',\n",
       "  12.7: 'fast',\n",
       "  12.8: 'fast',\n",
       "  12.9: 'fast',\n",
       "  13.0: 'fast',\n",
       "  13.1: 'fast',\n",
       "  13.2: 'fast',\n",
       "  13.3: 'fast',\n",
       "  13.4: 'fast',\n",
       "  13.5: 'fast',\n",
       "  13.6: 'fast',\n",
       "  13.7: 'fast',\n",
       "  13.8: 'fast',\n",
       "  13.9: 'fast',\n",
       "  14.0: 'fast',\n",
       "  14.1: 'fast',\n",
       "  14.2: 'fast',\n",
       "  14.3: 'fast',\n",
       "  14.4: 'fast',\n",
       "  14.5: 'fast',\n",
       "  14.6: 'fast',\n",
       "  14.7: 'fast',\n",
       "  14.8: 'fast',\n",
       "  14.9: 'fast',\n",
       "  15.0: 'fast',\n",
       "  15.1: 'fast',\n",
       "  15.2: 'fast',\n",
       "  15.3: 'fast',\n",
       "  15.4: 'fast',\n",
       "  15.5: 'fast',\n",
       "  15.6: 'fast',\n",
       "  15.7: 'fast',\n",
       "  15.8: 'fast',\n",
       "  15.9: 'fast',\n",
       "  16.0: 'fast',\n",
       "  16.1: 'fast',\n",
       "  16.2: 'fast',\n",
       "  16.3: 'fast',\n",
       "  16.4: 'fast',\n",
       "  16.5: 'fast',\n",
       "  16.6: 'fast',\n",
       "  16.7: 'fast',\n",
       "  16.8: 'fast',\n",
       "  16.9: 'fast',\n",
       "  17.0: 'fast',\n",
       "  17.1: 'fast',\n",
       "  17.2: 'fast',\n",
       "  17.3: 'fast',\n",
       "  17.4: 'fast',\n",
       "  17.5: 'fast',\n",
       "  17.6: 'fast',\n",
       "  17.7: 'fast',\n",
       "  17.8: 'fast',\n",
       "  17.9: 'fast',\n",
       "  18.0: 'fast',\n",
       "  18.1: 'fast',\n",
       "  18.2: 'fast',\n",
       "  18.3: 'fast',\n",
       "  18.4: 'fast',\n",
       "  18.5: 'fast',\n",
       "  18.6: 'fast',\n",
       "  18.7: 'fast',\n",
       "  18.8: 'fast',\n",
       "  18.9: 'fast',\n",
       "  19.0: 'fast',\n",
       "  19.1: 'fast',\n",
       "  19.2: 'fast',\n",
       "  19.3: 'fast',\n",
       "  19.4: 'fast',\n",
       "  19.5: 'fast',\n",
       "  19.6: 'fast',\n",
       "  19.7: 'fast',\n",
       "  19.8: 'fast',\n",
       "  19.9: 'fast',\n",
       "  20.0: 'fast',\n",
       "  20.1: 'fast',\n",
       "  20.2: 'fast',\n",
       "  20.3: 'fast',\n",
       "  20.4: 'fast',\n",
       "  20.5: 'fast',\n",
       "  20.6: 'fast',\n",
       "  20.7: 'fast',\n",
       "  20.8: 'fast',\n",
       "  20.9: 'fast',\n",
       "  21.0: 'fast',\n",
       "  21.1: 'fast',\n",
       "  21.2: 'fast',\n",
       "  21.3: 'fast',\n",
       "  21.4: 'fast',\n",
       "  21.5: 'fast',\n",
       "  21.6: 'fast',\n",
       "  21.7: 'fast',\n",
       "  21.8: 'fast',\n",
       "  21.9: 'fast',\n",
       "  22.0: 'fast',\n",
       "  22.1: 'fast',\n",
       "  22.2: 'fast',\n",
       "  22.3: 'fast',\n",
       "  22.4: 'fast',\n",
       "  22.5: 'fast',\n",
       "  22.6: 'fast',\n",
       "  22.7: 'fast',\n",
       "  22.8: 'fast',\n",
       "  22.9: 'fast',\n",
       "  23.0: 'fast',\n",
       "  23.1: 'fast',\n",
       "  23.2: 'fast',\n",
       "  23.3: 'fast',\n",
       "  23.4: 'fast',\n",
       "  23.5: 'fast',\n",
       "  23.6: 'fast',\n",
       "  23.7: 'fast',\n",
       "  23.8: 'fast',\n",
       "  23.9: 'fast',\n",
       "  24.0: 'fast',\n",
       "  24.1: 'fast',\n",
       "  24.2: 'fast',\n",
       "  24.3: 'fast',\n",
       "  24.4: 'fast',\n",
       "  24.5: 'fast',\n",
       "  24.6: 'fast',\n",
       "  24.7: 'fast',\n",
       "  24.8: 'fast',\n",
       "  24.9: 'fast',\n",
       "  25.0: 'fast',\n",
       "  25.1: 'fast',\n",
       "  25.2: 'fast',\n",
       "  25.3: 'fast',\n",
       "  25.4: 'fast',\n",
       "  25.5: 'fast',\n",
       "  25.6: 'fast',\n",
       "  25.7: 'fast',\n",
       "  25.8: 'fast',\n",
       "  25.9: 'fast',\n",
       "  26.0: 'fast',\n",
       "  26.1: 'fast',\n",
       "  26.2: 'fast',\n",
       "  26.3: 'fast',\n",
       "  26.4: 'fast',\n",
       "  26.5: 'fast',\n",
       "  26.6: 'fast',\n",
       "  26.7: 'fast',\n",
       "  26.8: 'fast',\n",
       "  26.9: 'fast',\n",
       "  27.0: 'fast',\n",
       "  27.1: 'fast',\n",
       "  27.2: 'fast',\n",
       "  27.3: 'fast',\n",
       "  27.4: 'fast',\n",
       "  27.5: 'fast',\n",
       "  27.6: 'fast',\n",
       "  27.7: 'fast',\n",
       "  27.8: 'fast',\n",
       "  27.9: 'fast',\n",
       "  28.0: 'fast',\n",
       "  28.1: 'fast',\n",
       "  28.2: 'fast',\n",
       "  28.3: 'fast',\n",
       "  28.4: 'fast',\n",
       "  28.5: 'fast',\n",
       "  28.6: 'fast',\n",
       "  28.7: 'fast',\n",
       "  28.8: 'fast',\n",
       "  29.0: 'fast',\n",
       "  29.2: 'fast',\n",
       "  29.4: 'fast',\n",
       "  29.5: 'fast',\n",
       "  29.6: 'fast',\n",
       "  29.8: 'fast',\n",
       "  29.9: 'fast',\n",
       "  30.2: 'fast',\n",
       "  30.3: 'fast',\n",
       "  30.5: 'fast',\n",
       "  30.7: 'fast',\n",
       "  30.8: 'fast',\n",
       "  30.9: 'fast',\n",
       "  31.0: 'fast',\n",
       "  31.1: 'fast',\n",
       "  31.2: 'fast',\n",
       "  31.3: 'fast',\n",
       "  31.5: 'fast',\n",
       "  31.7: 'fast',\n",
       "  32.0: 'fast',\n",
       "  32.1: 'fast',\n",
       "  32.7: 'fast',\n",
       "  33.1: 'fast',\n",
       "  33.6: 'fast',\n",
       "  34.4: 'fast',\n",
       "  34.5: 'fast',\n",
       "  34.7: 'fast',\n",
       "  34.9: 'fast',\n",
       "  35.4: 'fast',\n",
       "  35.6: 'fast',\n",
       "  36.3: 'fast',\n",
       "  36.6: 'fast',\n",
       "  37.2: 'fast',\n",
       "  40.1: 'fast',\n",
       "  42.7: 'fast',\n",
       "  43.0: 'fast',\n",
       "  102.3: 'fast'}}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#make data mapper\n",
    "data_mappers = ds.make_data_mapper(['Draft','Speed'],\n",
    "                                   ncores=4)\n",
    "\n",
    "#map the draft data to something else - doesnt have to be a string, could use a function etc.\n",
    "for k in data_mappers['Draft'].keys():\n",
    "      if k<=5:\n",
    "            data_mappers['Draft'][k] = 'shallow'\n",
    "      else:\n",
    "            data_mappers['Draft'][k] = 'deep'\n",
    "            \n",
    "#map the speed data to something else\n",
    "for k in data_mappers['Speed'].keys():\n",
    "      if k<=5:\n",
    "            data_mappers['Speed'][k] = 'slow'\n",
    "      else:\n",
    "            data_mappers['Speed'][k] = 'fast'\n",
    "\n",
    "data_mappers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[92mMapping agent dynamic data\u001b[0m: 100%|\u001b[32m██████████\u001b[0m| 172/172 [00:02<00:00, 59.64it/s]\n",
      "\u001b[92mRefreshing metadata\u001b[0m: 100%|\u001b[32m██████████\u001b[0m| 172/172 [00:02<00:00, 58.51it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New meta/databases saved to c:\\code\\trackio\\notebooks\\example0\n"
     ]
    }
   ],
   "source": [
    "#map the data\n",
    "ds = ds.map_data(['Draft','Speed'],\n",
    "                 ['Depth Requirement', 'How Fast?'],\n",
    "                 data_mappers,\n",
    "                 ncores=4)\n",
    "\n",
    "#refresh metadata\n",
    "ds = ds.refresh_meta(ncores=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "## Now read a random track to view the mapped dynamic data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time</th>\n",
       "      <th>X</th>\n",
       "      <th>Y</th>\n",
       "      <th>Draft</th>\n",
       "      <th>Status</th>\n",
       "      <th>Coursing</th>\n",
       "      <th>Speed</th>\n",
       "      <th>Depth Requirement</th>\n",
       "      <th>How Fast?</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-01-01 00:18:06</td>\n",
       "      <td>-73.97277</td>\n",
       "      <td>40.70478</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>None</td>\n",
       "      <td>slow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-01-01 00:21:10</td>\n",
       "      <td>-73.97275</td>\n",
       "      <td>40.70480</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>None</td>\n",
       "      <td>slow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-01-01 00:24:06</td>\n",
       "      <td>-73.97275</td>\n",
       "      <td>40.70480</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>None</td>\n",
       "      <td>slow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-01-01 00:27:10</td>\n",
       "      <td>-73.97275</td>\n",
       "      <td>40.70479</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>None</td>\n",
       "      <td>slow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-01-01 00:30:10</td>\n",
       "      <td>-73.97274</td>\n",
       "      <td>40.70479</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>None</td>\n",
       "      <td>slow</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Time         X         Y  Draft  Status  Coursing  Speed  \\\n",
       "0 2021-01-01 00:18:06 -73.97277  40.70478    NaN     5.0     360.0    0.1   \n",
       "1 2021-01-01 00:21:10 -73.97275  40.70480    NaN     5.0     360.0    0.0   \n",
       "2 2021-01-01 00:24:06 -73.97275  40.70480    NaN     5.0     360.0    0.0   \n",
       "3 2021-01-01 00:27:10 -73.97275  40.70479    NaN     5.0     360.0    0.0   \n",
       "4 2021-01-01 00:30:10 -73.97274  40.70479    NaN     5.0     360.0    0.1   \n",
       "\n",
       "  Depth Requirement How Fast?  \n",
       "0              None      slow  \n",
       "1              None      slow  \n",
       "2              None      slow  \n",
       "3              None      slow  \n",
       "4              None      slow  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#read single track\n",
    "ds.get_track(ds.tracks.index[24]).head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "## You can also map a dynamic data field to boolean Code columns. See [ReadMe](https://github.com/derekeden/trackio/blob/main/README.md) for more information about what a Code is in this context.\n",
    "\n",
    "## Lets map the `Coursing` data to quadrants 1, 2, 3 or 4 based on the direction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[92mMaking data mappers\u001b[0m: 100%|\u001b[32m██████████\u001b[0m| 172/172 [00:03<00:00, 56.02it/s]\n"
     ]
    }
   ],
   "source": [
    "#make coursing mapper\n",
    "coursing_mapper = ds.make_data_mapper(['Coursing'], \n",
    "                                      ncores=4)\n",
    "\n",
    "#calculate the quadrant\n",
    "for key in coursing_mapper.keys():\n",
    "    coursing_mapper[key] = int((key%360)/90) % 4 + 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "## Now, map these integer values to boolean Code columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[92mMapping agent dynamic data to coded boolean arrays\u001b[0m: 100%|\u001b[32m██████████\u001b[0m| 172/172 [00:02<00:00, 61.35it/s]\n",
      "\u001b[92mRefreshing metadata\u001b[0m: 100%|\u001b[32m██████████\u001b[0m| 172/172 [00:02<00:00, 57.44it/s]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New meta/databases saved to c:\\code\\trackio\\notebooks\\example0\n"
     ]
    }
   ],
   "source": [
    "#map quadrants to boolean codes\n",
    "ds = ds.map_data_to_codes(['Coursing'],\n",
    "                          {'Coursing':coursing_mapper},\n",
    "                          ncores=4)\n",
    "\n",
    "#refresh metadata\n",
    "ds = ds.refresh_meta(ncores=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time</th>\n",
       "      <th>X</th>\n",
       "      <th>Y</th>\n",
       "      <th>Draft</th>\n",
       "      <th>Status</th>\n",
       "      <th>Coursing</th>\n",
       "      <th>Speed</th>\n",
       "      <th>Depth Requirement</th>\n",
       "      <th>How Fast?</th>\n",
       "      <th>Code1</th>\n",
       "      <th>Code2</th>\n",
       "      <th>Code3</th>\n",
       "      <th>Code4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-01-01 00:17:19</td>\n",
       "      <td>-74.04922</td>\n",
       "      <td>40.71192</td>\n",
       "      <td>2.3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>10.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>shallow</td>\n",
       "      <td>slow</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-01-01 00:18:30</td>\n",
       "      <td>-74.04923</td>\n",
       "      <td>40.71187</td>\n",
       "      <td>2.3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>152.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>shallow</td>\n",
       "      <td>slow</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-01-01 00:19:39</td>\n",
       "      <td>-74.04921</td>\n",
       "      <td>40.71194</td>\n",
       "      <td>2.3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>346.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>shallow</td>\n",
       "      <td>slow</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-01-01 00:20:49</td>\n",
       "      <td>-74.04919</td>\n",
       "      <td>40.71193</td>\n",
       "      <td>2.3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>10.8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>shallow</td>\n",
       "      <td>slow</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-01-01 00:21:59</td>\n",
       "      <td>-74.04922</td>\n",
       "      <td>40.71193</td>\n",
       "      <td>2.3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>shallow</td>\n",
       "      <td>slow</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2021-01-01 00:23:09</td>\n",
       "      <td>-74.04921</td>\n",
       "      <td>40.71187</td>\n",
       "      <td>2.3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>357.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>shallow</td>\n",
       "      <td>slow</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2021-01-01 00:24:19</td>\n",
       "      <td>-74.04920</td>\n",
       "      <td>40.71193</td>\n",
       "      <td>2.3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>shallow</td>\n",
       "      <td>slow</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2021-01-01 00:25:29</td>\n",
       "      <td>-74.04924</td>\n",
       "      <td>40.71190</td>\n",
       "      <td>2.3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>356.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>shallow</td>\n",
       "      <td>slow</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2021-01-01 00:26:39</td>\n",
       "      <td>-74.04928</td>\n",
       "      <td>40.71182</td>\n",
       "      <td>2.3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>212.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>shallow</td>\n",
       "      <td>slow</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2021-01-01 00:27:48</td>\n",
       "      <td>-74.04926</td>\n",
       "      <td>40.71190</td>\n",
       "      <td>2.3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>shallow</td>\n",
       "      <td>slow</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Time         X         Y  Draft  Status  Coursing  Speed  \\\n",
       "0 2021-01-01 00:17:19 -74.04922  40.71192    2.3     3.0      10.2    0.0   \n",
       "1 2021-01-01 00:18:30 -74.04923  40.71187    2.3     3.0     152.7    0.0   \n",
       "2 2021-01-01 00:19:39 -74.04921  40.71194    2.3     3.0     346.6    0.0   \n",
       "3 2021-01-01 00:20:49 -74.04919  40.71193    2.3     3.0      10.8    0.0   \n",
       "4 2021-01-01 00:21:59 -74.04922  40.71193    2.3     3.0       8.1    0.0   \n",
       "5 2021-01-01 00:23:09 -74.04921  40.71187    2.3     3.0     357.3    0.0   \n",
       "6 2021-01-01 00:24:19 -74.04920  40.71193    2.3     3.0       8.4    0.0   \n",
       "7 2021-01-01 00:25:29 -74.04924  40.71190    2.3     3.0     356.9    0.0   \n",
       "8 2021-01-01 00:26:39 -74.04928  40.71182    2.3     3.0     212.0    0.0   \n",
       "9 2021-01-01 00:27:48 -74.04926  40.71190    2.3     3.0      15.3    0.0   \n",
       "\n",
       "  Depth Requirement How Fast?  Code1  Code2  Code3  Code4  \n",
       "0           shallow      slow   True  False  False  False  \n",
       "1           shallow      slow  False   True  False  False  \n",
       "2           shallow      slow  False  False  False   True  \n",
       "3           shallow      slow   True  False  False  False  \n",
       "4           shallow      slow   True  False  False  False  \n",
       "5           shallow      slow  False  False  False   True  \n",
       "6           shallow      slow   True  False  False  False  \n",
       "7           shallow      slow  False  False  False   True  \n",
       "8           shallow      slow  False  False   True  False  \n",
       "9           shallow      slow   True  False  False  False  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#read single track\n",
    "ds.get_track(ds.tracks.index[22]).head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "## Now you can see these codes stored in the `agent.db` and `track.db` metadata tables, making it possible to query by these properties without having to process them again."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AISCode</th>\n",
       "      <th>Code1</th>\n",
       "      <th>Code2</th>\n",
       "      <th>Code3</th>\n",
       "      <th>Code4</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Agent ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Vessel_367796050</th>\n",
       "      <td>31.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_368111610</th>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_367069240</th>\n",
       "      <td>31.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_367758160</th>\n",
       "      <td>36.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_338862000</th>\n",
       "      <td>31.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_367304010</th>\n",
       "      <td>31.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_366998820</th>\n",
       "      <td>31.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_367638140</th>\n",
       "      <td>90.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_566634000</th>\n",
       "      <td>70.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_338073000</th>\n",
       "      <td>31.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>172 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 AISCode Code1 Code2  Code3  Code4\n",
       "Agent ID                                          \n",
       "Vessel_367796050    31.0  True  True  False   True\n",
       "Vessel_368111610     NaN  True  True   True   True\n",
       "Vessel_367069240    31.0  True  True   True   True\n",
       "Vessel_367758160    36.0  True  True   True   True\n",
       "Vessel_338862000    31.0  True  True   True   True\n",
       "...                  ...   ...   ...    ...    ...\n",
       "Vessel_367304010    31.0  True  True   True   True\n",
       "Vessel_366998820    31.0  True  True   True   True\n",
       "Vessel_367638140    90.0  True  True   True   True\n",
       "Vessel_566634000    70.0  True  True  False  False\n",
       "Vessel_338073000    31.0  True  True   True   True\n",
       "\n",
       "[172 rows x 5 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#code is true if ANY point along ANY track for agent was True\n",
    "ds.agents.filter(like='Code')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AISCode</th>\n",
       "      <th>Code1</th>\n",
       "      <th>Code2</th>\n",
       "      <th>Code3</th>\n",
       "      <th>Code4</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Track ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Vessel_367796050_T0</th>\n",
       "      <td>31.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_368111610_T0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_367069240_T0</th>\n",
       "      <td>31.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_367758160_T0</th>\n",
       "      <td>36.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vessel_338862000_T0</th>\n",
       "      <td>31.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     AISCode  Code1  Code2  Code3  Code4\n",
       "Track ID                                                \n",
       "Vessel_367796050_T0     31.0   True   True  False   True\n",
       "Vessel_368111610_T0      NaN   True   True   True   True\n",
       "Vessel_367069240_T0     31.0   True   True   True   True\n",
       "Vessel_367758160_T0     36.0   True   True   True   True\n",
       "Vessel_338862000_T0     31.0   True   True   True   True"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#code is True if ANY point along track was True\n",
    "ds.tracks.head(5).filter(like='Code')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "## Finally, you can also drop metadata and dynamic data columns from the agents/tracks in your `Dataset`. This is useful when you want to reduce the data size and delete data you are not using anymore."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[92mDropping meta from agents\u001b[0m: 100%|\u001b[32m██████████\u001b[0m| 172/172 [00:03<00:00, 48.16it/s]\n",
      "\u001b[92mDropping dynamic data from agents\u001b[0m: 100%|\u001b[32m██████████\u001b[0m| 172/172 [00:03<00:00, 47.41it/s]\n",
      "\u001b[92mRefreshing metadata\u001b[0m: 100%|\u001b[32m██████████\u001b[0m| 172/172 [00:03<00:00, 54.82it/s]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New meta/databases saved to c:\\code\\trackio\\notebooks\\example0\n"
     ]
    }
   ],
   "source": [
    "#drop metadata from agents\n",
    "ds = ds.drop_meta(['How Wide?', 'How Long?'],\n",
    "                  ncores=4)\n",
    "\n",
    "#drop dynamic data from tracks\n",
    "ds = ds.drop_data(['Depth Requirement', 'How Fast?'],\n",
    "                  ncores=4)\n",
    "\n",
    "#refresh metadata\n",
    "ds = ds.refresh_meta(ncores=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "## Notice now that this data is not present in the metadata tables, or the actual agent/track."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['MMSI', 'Name', 'AISCode', 'Length', 'Width', 'npoints', 'ntracks',\n",
       "       'Xmin', 'Xmax', 'Ymin', 'Ymax', 'Start Time', 'End Time', 'Code1',\n",
       "       'Code2', 'Code3', 'Code4', 'File', 'geometry'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds.agents.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['MMSI', 'Name', 'AISCode', 'Length', 'Width', 'Agent ID', 'npoints',\n",
       "       'Xmin', 'Xmax', 'Ymin', 'Ymax', 'Start Time', 'End Time', 'Code1',\n",
       "       'Code2', 'Code3', 'Code4', 'File', 'Track Length', 'Duration', 'Year',\n",
       "       'Month', 'Xstart', 'Ystart', 'Xend', 'Yend', 'Effective Distance',\n",
       "       'Min Temporal Resolution', 'Mean Temporal Resolution',\n",
       "       'Max Temporal Resolution', 'Min Spatial Resolution',\n",
       "       'Mean Spatial Resolution', 'Max Spatial Resolution', 'Sinuosity',\n",
       "       'geometry'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds.tracks.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'MMSI': 367796050,\n",
       " 'Name': 'SEA LION',\n",
       " 'AISCode': 31.0,\n",
       " 'Length': 19.0,\n",
       " 'Width': 6.0,\n",
       " 'Agent ID': 'Vessel_367796050'}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#get random agent\n",
    "agent = ds.get_agent(ds.agents.index[0])\n",
    "\n",
    "#view agent meta\n",
    "agent.meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time</th>\n",
       "      <th>X</th>\n",
       "      <th>Y</th>\n",
       "      <th>Draft</th>\n",
       "      <th>Status</th>\n",
       "      <th>Coursing</th>\n",
       "      <th>Speed</th>\n",
       "      <th>Code1</th>\n",
       "      <th>Code2</th>\n",
       "      <th>Code3</th>\n",
       "      <th>Code4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-01-01 00:17:17</td>\n",
       "      <td>-74.07761</td>\n",
       "      <td>40.68623</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>157.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-01-01 00:18:26</td>\n",
       "      <td>-74.07761</td>\n",
       "      <td>40.68623</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>157.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-01-01 00:19:38</td>\n",
       "      <td>-74.07761</td>\n",
       "      <td>40.68624</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>157.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-01-01 00:20:46</td>\n",
       "      <td>-74.07761</td>\n",
       "      <td>40.68624</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>157.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-01-01 00:21:57</td>\n",
       "      <td>-74.07764</td>\n",
       "      <td>40.68623</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>157.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Time         X         Y  Draft  Status  Coursing  Speed  \\\n",
       "0 2021-01-01 00:17:17 -74.07761  40.68623    NaN     0.0     157.6    0.0   \n",
       "1 2021-01-01 00:18:26 -74.07761  40.68623    NaN     0.0     157.6    0.0   \n",
       "2 2021-01-01 00:19:38 -74.07761  40.68624    NaN     0.0     157.6    0.0   \n",
       "3 2021-01-01 00:20:46 -74.07761  40.68624    NaN     0.0     157.6    0.0   \n",
       "4 2021-01-01 00:21:57 -74.07764  40.68623    NaN     0.0     157.6    0.0   \n",
       "\n",
       "   Code1  Code2  Code3  Code4  \n",
       "0  False   True  False  False  \n",
       "1  False   True  False  False  \n",
       "2  False   True  False  False  \n",
       "3  False   True  False  False  \n",
       "4  False   True  False  False  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#view first track for agent\n",
    "agent.tracks['T0'].head(5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "trackio",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}