notebook/FAQIR/01.Parsing_FAQIRv1.0.xml_file.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JoWilTNGKcrO"
   },
   "source": [
    "**Parsing & Extracting data for FAQIR**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "uE2oNEYgcMW0"
   },
   "outputs": [],
   "source": [
    "# install required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "S0HJM9k_b7nH",
    "outputId": "85fa7bf0-7718-4cb8-ca2b-107ee4cbfb07",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: xmltodict in /Users/daciantamasan/anaconda3/lib/python3.8/site-packages (0.12.0)\r\n"
     ]
    }
   ],
   "source": [
    "!pip3 install xmltodict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "jop-QMVBD5a1"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, '../../../BERT-FAQ/')\n",
    "\n",
    "# import parsers\n",
    "from parser.faqir import FAQIR_XML_Parser\n",
    "\n",
    "# import utility functions\n",
    "from shared.utils import load_from_json\n",
    "from shared.utils import dump_to_json\n",
    "from shared.utils import make_dirs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "geF_0cu4oN7E"
   },
   "source": [
    "**Parsing & generating data from FAQIRv1.0.xml file**\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "V3my3-bmcPF6"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "../../../BERT-FAQ/parser/faqir.py:251: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  query_answer_pairs_df = pd.concat([qa_pairs_df, user_query_pair_df])\n"
     ]
    }
   ],
   "source": [
    "# read and parse FAQIRv1.0.xml\n",
    "filepath = '../../../BERT-FAQ/data/FAQIR/FAQIRv1.0.xml'\n",
    "\n",
    "faqir_xml_parser = FAQIR_XML_Parser()\n",
    "faqir_xml_parser.extract_data(filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "CxcUeRq7cPCS",
    "outputId": "2f0d32e4-0185-49ec-afe4-c84989a606df"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'queryGroupID': '0',\n",
       "  'expansions': None,\n",
       "  'author': 'mladen',\n",
       "  'question': 'How do I get a sticker off of a car window  (plastic part)?  Nail polish remover seems risky?',\n",
       "  'infneed': 'Inf. potreba -- kako skinuti naljepnicu sa staklene povrsine',\n",
       "  'exwords': ['sticker', 'glass', 'window', 'mirror'],\n",
       "  'query_string': \"My kid puts these stickers on my kitchen windows. I can't get them off properly. Any advice on what to use?\",\n",
       "  'rel_docs': None,\n",
       "  'fold': '0'},\n",
       " {'queryGroupID': '0',\n",
       "  'expansions': None,\n",
       "  'author': 'mladen',\n",
       "  'question': 'How do I get a sticker off of a car window  (plastic part)?  Nail polish remover seems risky?',\n",
       "  'infneed': 'Inf. potreba -- kako skinuti naljepnicu sa staklene povrsine',\n",
       "  'exwords': ['sticker', 'glass', 'window', 'mirror'],\n",
       "  'query_string': 'I put a sticker on my car window. How can I remove it without damaging the glass.',\n",
       "  'rel_docs': None,\n",
       "  'fold': '0'},\n",
       " {'queryGroupID': '0',\n",
       "  'expansions': None,\n",
       "  'author': 'mladen',\n",
       "  'question': 'How do I get a sticker off of a car window  (plastic part)?  Nail polish remover seems risky?',\n",
       "  'infneed': 'Inf. potreba -- kako skinuti naljepnicu sa staklene povrsine',\n",
       "  'exwords': ['sticker', 'glass', 'window', 'mirror'],\n",
       "  'query_string': 'How can I remove a sticker from my window?',\n",
       "  'rel_docs': None,\n",
       "  'fold': '0'},\n",
       " {'queryGroupID': '0',\n",
       "  'expansions': None,\n",
       "  'author': 'mladen',\n",
       "  'question': 'How do I get a sticker off of a car window  (plastic part)?  Nail polish remover seems risky?',\n",
       "  'infneed': 'Inf. potreba -- kako skinuti naljepnicu sa staklene povrsine',\n",
       "  'exwords': ['sticker', 'glass', 'window', 'mirror'],\n",
       "  'query_string': 'I glued some photos on my bathroom mirror. What would be the best way to get them off?',\n",
       "  'rel_docs': None,\n",
       "  'fold': '0'},\n",
       " {'queryGroupID': '0',\n",
       "  'expansions': None,\n",
       "  'author': 'mladen',\n",
       "  'question': 'How do I get a sticker off of a car window  (plastic part)?  Nail polish remover seems risky?',\n",
       "  'infneed': 'Inf. potreba -- kako skinuti naljepnicu sa staklene povrsine',\n",
       "  'exwords': ['sticker', 'glass', 'window', 'mirror'],\n",
       "  'query_string': 'I found some world cup album stickers stuck on my glass table. I peeled them off but I still cant get it completely clean. Any advice?',\n",
       "  'rel_docs': None,\n",
       "  'fold': '0'}]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get first 5 queries\n",
    "faqir_xml_parser.queries[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "V4e3aLFBcO-O",
    "outputId": "3f705d28-e016-457a-bf22-76e0a3272b4c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1233"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get total number of queries\n",
    "faqir_xml_parser.num_queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "WR9dr_D9pInD",
    "outputId": "cf366b46-39e2-4ec4-c2d3-90f622c0bcba"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': '8656',\n",
       "  'question': 'How to remove tree sap from car? Hi,I have some tree sap on my car, both older (pretty dry) and some newer from a redwood tree.Any tips on how to remove it?',\n",
       "  'answer': 'I have had good luck with commercial products available at auto supply stores for just a few dollars. Below are some additional suggestions if you want to try DIY:',\n",
       "  'type': 'original'},\n",
       " {'id': '40807',\n",
       "  'question': 'How do I get a sticker off of a car window  (plastic part)?  Nail polish remover seems risky?',\n",
       "  'answer': \"Goo Gone is a fantastic product (available at Target and most any hardware store or drug store) and works amazingly well on even pesky sticker residue.  Soak the area, and wipe it away.  It might take repeated applications, but you can remove the sticker without scratching, and without leaving a gooey residue.Note that Goo Gone can affect some surfaces, including some plastics, so you should be careful, especially if it is clear plastic.  I've used Goo Gone many times on plastic glassic and CD cases, so it will certainly work on many plastics, but you should be careful. In the event that Goo Gone seems too risky, you can frequently remove just the adhesive residue using plain old Scotch tape.  Remove as much of the sticker as possible (the paper/plastic backing) and then apply tape to the sticker residue firmly.  Pull the tape off sharply, and some of the residue will come with it.  This is a slow process so only do it if the area is relatively small, however, I frequently use this tactic for price tags and the like and it is surprisingly effective.\",\n",
       "  'type': 'original'},\n",
       " {'id': '57727',\n",
       "  'question': 'How to get rid of garbage disposal odor?',\n",
       "  'answer': '1. Inspect the black rubber cover that fits in the disposal. Sometimes bits of garbage collect under the flaps and create a smell.    2. Clean the rubber cover, if necessary. If yours lifts out, remove it and clean away debris with a scrub brush and warm, soapy water. If your rubber cover is installed permanently, lift up each flap and clean it with soapy water and an old toothbrush.    3. Deodorize the disposal. Cut a lemon in half and drop the fruit and a handful of baking soda into the disposal. Turn on the cold water faucet and then the disposal. The unit will clean itself as it grinds up the mixture.',\n",
       "  'type': 'original'},\n",
       " {'id': '120493',\n",
       "  'question': 'How to maintain a lawn mower?',\n",
       "  'answer': \"To properly maintain your lawnmover, you'll need to do the things below. If you use use your lawnmover year round, you might need to do maintaince twice a year.1) Changing the oil (for non-electric)2) Removing dirt, grass, and grime from the underside3) Cleaning the air filter4) Cleaning the spark plugsYou'll need the following tools to do the job:1) Putty knife2) Socket wrench3) File or sharpening tool4) Wire brush5) 30-weight oilIt actually didn't sound so hard to maintain. The most important safety concern is to remember to remove the spark plug before working on the blade.You'll find the instructions on the home depot site.\",\n",
       "  'type': 'original'},\n",
       " {'id': '62732',\n",
       "  'question': 'How to fix vertical blinds? I can no longer re-hang them because they are broken.',\n",
       "  'answer': 'Click on the link and follow the steps.',\n",
       "  'type': 'original'}]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get first 5 qa-pairs\n",
    "faqir_xml_parser.qa_pairs[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-n01v-E5pIhy",
    "outputId": "e6cb4202-f622-4c9e-ff82-375fe88b29b9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4313"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get total number of qa pairs\n",
    "faqir_xml_parser.num_qa_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': '123855',\n",
       "  'queryGroupID': '0',\n",
       "  'annotation': [{'Val': '3',\n",
       "    'Annotator': 'djurozlikovski',\n",
       "    'AnnotationTime': '2015-10-10T17:23:16.104613+02:00'},\n",
       "   {'Val': '2',\n",
       "    'Annotator': 'ferdo',\n",
       "    'AnnotationTime': '2015-10-12T11:32:02.2507956+02:00'}]},\n",
       " {'id': '82644',\n",
       "  'queryGroupID': '0',\n",
       "  'annotation': [{'Val': '4',\n",
       "    'Annotator': 'djurozlikovski',\n",
       "    'AnnotationTime': '2015-10-10T17:22:30.5355862+02:00'},\n",
       "   {'Val': '2',\n",
       "    'Annotator': 'ferdo',\n",
       "    'AnnotationTime': '2015-10-12T11:24:46.6424264+02:00'}]},\n",
       " {'id': '13285',\n",
       "  'queryGroupID': '0',\n",
       "  'annotation': [{'Val': '4',\n",
       "    'Annotator': 'djurozlikovski',\n",
       "    'AnnotationTime': '2015-10-10T17:22:33.8346817+02:00'},\n",
       "   {'Val': '1',\n",
       "    'Annotator': 'ferdo',\n",
       "    'AnnotationTime': '2015-10-12T11:24:47.8522497+02:00'}]},\n",
       " {'id': '2637',\n",
       "  'queryGroupID': '0',\n",
       "  'annotation': {'Val': '4',\n",
       "   'Annotator': 'djurozlikovski',\n",
       "   'AnnotationTime': '2015-10-10T17:22:34.4377084+02:00'}},\n",
       " {'id': '42908',\n",
       "  'queryGroupID': '0',\n",
       "  'annotation': {'Val': '4',\n",
       "   'Annotator': 'djurozlikovski',\n",
       "   'AnnotationTime': '2015-10-10T17:22:35.1737365+02:00'}}]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get first 5 ircandidates\n",
    "faqir_xml_parser.ircandidates[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10142"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get total number of ircandidates\n",
    "faqir_xml_parser.num_ircandidates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'answer': 'I have had good luck with commercial products available at auto supply stores for just a few dollars. Below are some additional suggestions if you want to try DIY:',\n",
       "  'id': 1,\n",
       "  'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'How to remove tree sap from car? Hi,I have some tree sap on my car, both older (pretty dry) and some newer from a redwood tree.Any tips on how to remove it?'},\n",
       " {'answer': \"Goo Gone is a fantastic product (available at Target and most any hardware store or drug store) and works amazingly well on even pesky sticker residue.  Soak the area, and wipe it away.  It might take repeated applications, but you can remove the sticker without scratching, and without leaving a gooey residue.Note that Goo Gone can affect some surfaces, including some plastics, so you should be careful, especially if it is clear plastic.  I've used Goo Gone many times on plastic glassic and CD cases, so it will certainly work on many plastics, but you should be careful. In the event that Goo Gone seems too risky, you can frequently remove just the adhesive residue using plain old Scotch tape.  Remove as much of the sticker as possible (the paper/plastic backing) and then apply tape to the sticker residue firmly.  Pull the tape off sharply, and some of the residue will come with it.  This is a slow process so only do it if the area is relatively small, however, I frequently use this tactic for price tags and the like and it is surprisingly effective.\",\n",
       "  'id': 2,\n",
       "  'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'How do I get a sticker off of a car window  (plastic part)?  Nail polish remover seems risky?'},\n",
       " {'answer': '1. Inspect the black rubber cover that fits in the disposal. Sometimes bits of garbage collect under the flaps and create a smell.    2. Clean the rubber cover, if necessary. If yours lifts out, remove it and clean away debris with a scrub brush and warm, soapy water. If your rubber cover is installed permanently, lift up each flap and clean it with soapy water and an old toothbrush.    3. Deodorize the disposal. Cut a lemon in half and drop the fruit and a handful of baking soda into the disposal. Turn on the cold water faucet and then the disposal. The unit will clean itself as it grinds up the mixture.',\n",
       "  'id': 3,\n",
       "  'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'How to get rid of garbage disposal odor?'},\n",
       " {'answer': \"To properly maintain your lawnmover, you'll need to do the things below. If you use use your lawnmover year round, you might need to do maintaince twice a year.1) Changing the oil (for non-electric)2) Removing dirt, grass, and grime from the underside3) Cleaning the air filter4) Cleaning the spark plugsYou'll need the following tools to do the job:1) Putty knife2) Socket wrench3) File or sharpening tool4) Wire brush5) 30-weight oilIt actually didn't sound so hard to maintain. The most important safety concern is to remember to remove the spark plug before working on the blade.You'll find the instructions on the home depot site.\",\n",
       "  'id': 4,\n",
       "  'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'How to maintain a lawn mower?'},\n",
       " {'answer': 'Click on the link and follow the steps.',\n",
       "  'id': 5,\n",
       "  'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'How to fix vertical blinds? I can no longer re-hang them because they are broken.'}]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get first 5 query_answer_pairs\n",
    "faqir_xml_parser.query_answer_pairs[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>query_type</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8656</td>\n",
       "      <td>How to remove tree sap from car? Hi,I have som...</td>\n",
       "      <td>I have had good luck with commercial products ...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>40807</td>\n",
       "      <td>How do I get a sticker off of a car window  (p...</td>\n",
       "      <td>Goo Gone is a fantastic product (available at ...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>57727</td>\n",
       "      <td>How to get rid of garbage disposal odor?</td>\n",
       "      <td>1. Inspect the black rubber cover that fits in...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>120493</td>\n",
       "      <td>How to maintain a lawn mower?</td>\n",
       "      <td>To properly maintain your lawnmover, you'll ne...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>62732</td>\n",
       "      <td>How to fix vertical blinds? I can no longer re...</td>\n",
       "      <td>Click on the link and follow the steps.</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4308</th>\n",
       "      <td>138807</td>\n",
       "      <td>How do you remove other car paint rubs from yo...</td>\n",
       "      <td>Rubbing compound then put some wax on it...bug...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4309</th>\n",
       "      <td>110019</td>\n",
       "      <td>how to replace bathroom fan?</td>\n",
       "      <td>If you didn't buy the fan yet you have to cons...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4310</th>\n",
       "      <td>127097</td>\n",
       "      <td>How do you change the small down lights used i...</td>\n",
       "      <td>The halogen is made such that the actual lamp ...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4311</th>\n",
       "      <td>65707</td>\n",
       "      <td>How do I know how much air to put in car tires...</td>\n",
       "      <td>Wow, so many answers, so few correct.  It amaz...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4312</th>\n",
       "      <td>99118</td>\n",
       "      <td>how can i change  my birthday?</td>\n",
       "      <td>Time travel back and induce labor on your moth...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4313 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                           question  \\\n",
       "0       8656  How to remove tree sap from car? Hi,I have som...   \n",
       "1      40807  How do I get a sticker off of a car window  (p...   \n",
       "2      57727           How to get rid of garbage disposal odor?   \n",
       "3     120493                      How to maintain a lawn mower?   \n",
       "4      62732  How to fix vertical blinds? I can no longer re...   \n",
       "...      ...                                                ...   \n",
       "4308  138807  How do you remove other car paint rubs from yo...   \n",
       "4309  110019                       how to replace bathroom fan?   \n",
       "4310  127097  How do you change the small down lights used i...   \n",
       "4311   65707  How do I know how much air to put in car tires...   \n",
       "4312   99118                     how can i change  my birthday?   \n",
       "\n",
       "                                                 answer query_type  label  \n",
       "0     I have had good luck with commercial products ...        faq      1  \n",
       "1     Goo Gone is a fantastic product (available at ...        faq      1  \n",
       "2     1. Inspect the black rubber cover that fits in...        faq      1  \n",
       "3     To properly maintain your lawnmover, you'll ne...        faq      1  \n",
       "4               Click on the link and follow the steps.        faq      1  \n",
       "...                                                 ...        ...    ...  \n",
       "4308  Rubbing compound then put some wax on it...bug...        faq      1  \n",
       "4309  If you didn't buy the fan yet you have to cons...        faq      1  \n",
       "4310  The halogen is made such that the actual lamp ...        faq      1  \n",
       "4311  Wow, so many answers, so few correct.  It amaz...        faq      1  \n",
       "4312  Time travel back and induce labor on your moth...        faq      1  \n",
       "\n",
       "[4313 rows x 5 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check qa_pair_df\n",
    "faqir_xml_parser.qa_pairs_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>queryGroupID</th>\n",
       "      <th>expansions</th>\n",
       "      <th>author</th>\n",
       "      <th>infneed</th>\n",
       "      <th>exwords</th>\n",
       "      <th>query_string</th>\n",
       "      <th>rel_docs</th>\n",
       "      <th>fold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>mladen</td>\n",
       "      <td>Inf. potreba -- kako skinuti naljepnicu sa sta...</td>\n",
       "      <td>[sticker, glass, window, mirror]</td>\n",
       "      <td>My kid puts these stickers on my kitchen windo...</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>mladen</td>\n",
       "      <td>Inf. potreba -- kako skinuti naljepnicu sa sta...</td>\n",
       "      <td>[sticker, glass, window, mirror]</td>\n",
       "      <td>I put a sticker on my car window. How can I re...</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>mladen</td>\n",
       "      <td>Inf. potreba -- kako skinuti naljepnicu sa sta...</td>\n",
       "      <td>[sticker, glass, window, mirror]</td>\n",
       "      <td>How can I remove a sticker from my window?</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>mladen</td>\n",
       "      <td>Inf. potreba -- kako skinuti naljepnicu sa sta...</td>\n",
       "      <td>[sticker, glass, window, mirror]</td>\n",
       "      <td>I glued some photos on my bathroom mirror. Wha...</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>mladen</td>\n",
       "      <td>Inf. potreba -- kako skinuti naljepnicu sa sta...</td>\n",
       "      <td>[sticker, glass, window, mirror]</td>\n",
       "      <td>I found some world cup album stickers stuck on...</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1178</th>\n",
       "      <td>49</td>\n",
       "      <td>None</td>\n",
       "      <td>domagoj</td>\n",
       "      <td>Inf. potreba -- prozor je pofarban i sad se ne...</td>\n",
       "      <td>[paint, painted, window]</td>\n",
       "      <td>Can someone give me some simple tips on gettin...</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1179</th>\n",
       "      <td>49</td>\n",
       "      <td>None</td>\n",
       "      <td>domagoj</td>\n",
       "      <td>Inf. potreba -- prozor je pofarban i sad se ne...</td>\n",
       "      <td>[paint, painted, window]</td>\n",
       "      <td>Is there a chemical that dissolves dried color...</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1180</th>\n",
       "      <td>49</td>\n",
       "      <td>None</td>\n",
       "      <td>domagoj</td>\n",
       "      <td>Inf. potreba -- prozor je pofarban i sad se ne...</td>\n",
       "      <td>[paint, painted, window]</td>\n",
       "      <td>Can you just scrub the dried paint from a window?</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1181</th>\n",
       "      <td>49</td>\n",
       "      <td>None</td>\n",
       "      <td>domagoj</td>\n",
       "      <td>Inf. potreba -- prozor je pofarban i sad se ne...</td>\n",
       "      <td>[paint, painted, window]</td>\n",
       "      <td>I was a real handyman and I've painted my wind...</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1182</th>\n",
       "      <td>49</td>\n",
       "      <td>None</td>\n",
       "      <td>domagoj</td>\n",
       "      <td>Inf. potreba -- prozor je pofarban i sad se ne...</td>\n",
       "      <td>[paint, painted, window]</td>\n",
       "      <td>How to make sure that newly painted window can...</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1183 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     queryGroupID expansions   author  \\\n",
       "0               0       None   mladen   \n",
       "1               0       None   mladen   \n",
       "2               0       None   mladen   \n",
       "3               0       None   mladen   \n",
       "4               0       None   mladen   \n",
       "...           ...        ...      ...   \n",
       "1178           49       None  domagoj   \n",
       "1179           49       None  domagoj   \n",
       "1180           49       None  domagoj   \n",
       "1181           49       None  domagoj   \n",
       "1182           49       None  domagoj   \n",
       "\n",
       "                                                infneed  \\\n",
       "0     Inf. potreba -- kako skinuti naljepnicu sa sta...   \n",
       "1     Inf. potreba -- kako skinuti naljepnicu sa sta...   \n",
       "2     Inf. potreba -- kako skinuti naljepnicu sa sta...   \n",
       "3     Inf. potreba -- kako skinuti naljepnicu sa sta...   \n",
       "4     Inf. potreba -- kako skinuti naljepnicu sa sta...   \n",
       "...                                                 ...   \n",
       "1178  Inf. potreba -- prozor je pofarban i sad se ne...   \n",
       "1179  Inf. potreba -- prozor je pofarban i sad se ne...   \n",
       "1180  Inf. potreba -- prozor je pofarban i sad se ne...   \n",
       "1181  Inf. potreba -- prozor je pofarban i sad se ne...   \n",
       "1182  Inf. potreba -- prozor je pofarban i sad se ne...   \n",
       "\n",
       "                               exwords  \\\n",
       "0     [sticker, glass, window, mirror]   \n",
       "1     [sticker, glass, window, mirror]   \n",
       "2     [sticker, glass, window, mirror]   \n",
       "3     [sticker, glass, window, mirror]   \n",
       "4     [sticker, glass, window, mirror]   \n",
       "...                                ...   \n",
       "1178          [paint, painted, window]   \n",
       "1179          [paint, painted, window]   \n",
       "1180          [paint, painted, window]   \n",
       "1181          [paint, painted, window]   \n",
       "1182          [paint, painted, window]   \n",
       "\n",
       "                                           query_string rel_docs fold  \n",
       "0     My kid puts these stickers on my kitchen windo...     None    0  \n",
       "1     I put a sticker on my car window. How can I re...     None    0  \n",
       "2            How can I remove a sticker from my window?     None    0  \n",
       "3     I glued some photos on my bathroom mirror. Wha...     None    0  \n",
       "4     I found some world cup album stickers stuck on...     None    0  \n",
       "...                                                 ...      ...  ...  \n",
       "1178  Can someone give me some simple tips on gettin...     None    0  \n",
       "1179  Is there a chemical that dissolves dried color...     None    0  \n",
       "1180  Can you just scrub the dried paint from a window?     None    0  \n",
       "1181  I was a real handyman and I've painted my wind...     None    0  \n",
       "1182  How to make sure that newly painted window can...     None    0  \n",
       "\n",
       "[1183 rows x 8 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check queries_df\n",
    "faqir_xml_parser.queries_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>queryGroupID</th>\n",
       "      <th>annotation</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>123855</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'Val': '3', 'Annotator': 'djurozlikovski', '...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>82644</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'Val': '4', 'Annotator': 'djurozlikovski', '...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13285</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'Val': '4', 'Annotator': 'djurozlikovski', '...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2637</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Val': '4', 'Annotator': 'djurozlikovski', 'A...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>42908</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Val': '4', 'Annotator': 'djurozlikovski', 'A...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10137</th>\n",
       "      <td>142145</td>\n",
       "      <td>49</td>\n",
       "      <td>{'Val': '4', 'Annotator': 'djurozlikovski', 'A...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10138</th>\n",
       "      <td>19742</td>\n",
       "      <td>49</td>\n",
       "      <td>{'Val': '4', 'Annotator': 'djurozlikovski', 'A...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10139</th>\n",
       "      <td>130462</td>\n",
       "      <td>49</td>\n",
       "      <td>{'Val': '4', 'Annotator': 'djurozlikovski', 'A...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10140</th>\n",
       "      <td>83859</td>\n",
       "      <td>49</td>\n",
       "      <td>{'Val': '4', 'Annotator': 'djurozlikovski', 'A...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10141</th>\n",
       "      <td>37759</td>\n",
       "      <td>49</td>\n",
       "      <td>{'Val': '4', 'Annotator': 'djurozlikovski', 'A...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10142 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id queryGroupID                                         annotation  \\\n",
       "0      123855            0  [{'Val': '3', 'Annotator': 'djurozlikovski', '...   \n",
       "1       82644            0  [{'Val': '4', 'Annotator': 'djurozlikovski', '...   \n",
       "2       13285            0  [{'Val': '4', 'Annotator': 'djurozlikovski', '...   \n",
       "3        2637            0  {'Val': '4', 'Annotator': 'djurozlikovski', 'A...   \n",
       "4       42908            0  {'Val': '4', 'Annotator': 'djurozlikovski', 'A...   \n",
       "...       ...          ...                                                ...   \n",
       "10137  142145           49  {'Val': '4', 'Annotator': 'djurozlikovski', 'A...   \n",
       "10138   19742           49  {'Val': '4', 'Annotator': 'djurozlikovski', 'A...   \n",
       "10139  130462           49  {'Val': '4', 'Annotator': 'djurozlikovski', 'A...   \n",
       "10140   83859           49  {'Val': '4', 'Annotator': 'djurozlikovski', 'A...   \n",
       "10141   37759           49  {'Val': '4', 'Annotator': 'djurozlikovski', 'A...   \n",
       "\n",
       "       label  \n",
       "0          0  \n",
       "1          0  \n",
       "2          1  \n",
       "3          0  \n",
       "4          0  \n",
       "...      ...  \n",
       "10137      0  \n",
       "10138      0  \n",
       "10139      0  \n",
       "10140      0  \n",
       "10141      0  \n",
       "\n",
       "[10142 rows x 4 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check ircandidates_df\n",
    "faqir_xml_parser.ircandidates_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>answer</th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>query_type</th>\n",
       "      <th>question</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I have had good luck with commercial products ...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>faq</td>\n",
       "      <td>How to remove tree sap from car? Hi,I have som...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Goo Gone is a fantastic product (available at ...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>faq</td>\n",
       "      <td>How do I get a sticker off of a car window  (p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1. Inspect the black rubber cover that fits in...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>faq</td>\n",
       "      <td>How to get rid of garbage disposal odor?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>To properly maintain your lawnmover, you'll ne...</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>faq</td>\n",
       "      <td>How to maintain a lawn mower?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Click on the link and follow the steps.</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>faq</td>\n",
       "      <td>How to fix vertical blinds? I can no longer re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16320</th>\n",
       "      <td>Get a bucket, fold some cotton towels into it,...</td>\n",
       "      <td>16320</td>\n",
       "      <td>1</td>\n",
       "      <td>user_query</td>\n",
       "      <td>Cigarette smoke odor is so annoying. Some frie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16321</th>\n",
       "      <td>Get a bucket, fold some cotton towels into it,...</td>\n",
       "      <td>16321</td>\n",
       "      <td>1</td>\n",
       "      <td>user_query</td>\n",
       "      <td>I love smoking a pipe but the smell is quite d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16322</th>\n",
       "      <td>Get a bucket, fold some cotton towels into it,...</td>\n",
       "      <td>16322</td>\n",
       "      <td>1</td>\n",
       "      <td>user_query</td>\n",
       "      <td>What are the best tips for getting rid of the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16323</th>\n",
       "      <td>Get a bucket, fold some cotton towels into it,...</td>\n",
       "      <td>16323</td>\n",
       "      <td>1</td>\n",
       "      <td>user_query</td>\n",
       "      <td>The weed smell is really hard to get rid of. D...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16324</th>\n",
       "      <td>Get a bucket, fold some cotton towels into it,...</td>\n",
       "      <td>16324</td>\n",
       "      <td>1</td>\n",
       "      <td>user_query</td>\n",
       "      <td>My roommate started smoking marijuana every th...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16324 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  answer     id  label  \\\n",
       "1      I have had good luck with commercial products ...      1      1   \n",
       "2      Goo Gone is a fantastic product (available at ...      2      1   \n",
       "3      1. Inspect the black rubber cover that fits in...      3      1   \n",
       "4      To properly maintain your lawnmover, you'll ne...      4      1   \n",
       "5                Click on the link and follow the steps.      5      1   \n",
       "...                                                  ...    ...    ...   \n",
       "16320  Get a bucket, fold some cotton towels into it,...  16320      1   \n",
       "16321  Get a bucket, fold some cotton towels into it,...  16321      1   \n",
       "16322  Get a bucket, fold some cotton towels into it,...  16322      1   \n",
       "16323  Get a bucket, fold some cotton towels into it,...  16323      1   \n",
       "16324  Get a bucket, fold some cotton towels into it,...  16324      1   \n",
       "\n",
       "       query_type                                           question  \n",
       "1             faq  How to remove tree sap from car? Hi,I have som...  \n",
       "2             faq  How do I get a sticker off of a car window  (p...  \n",
       "3             faq           How to get rid of garbage disposal odor?  \n",
       "4             faq                      How to maintain a lawn mower?  \n",
       "5             faq  How to fix vertical blinds? I can no longer re...  \n",
       "...           ...                                                ...  \n",
       "16320  user_query  Cigarette smoke odor is so annoying. Some frie...  \n",
       "16321  user_query  I love smoking a pipe but the smell is quite d...  \n",
       "16322  user_query  What are the best tips for getting rid of the ...  \n",
       "16323  user_query  The weed smell is really hard to get rid of. D...  \n",
       "16324  user_query  My roommate started smoking marijuana every th...  \n",
       "\n",
       "[16324 rows x 5 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check query_answer_pairs_df\n",
    "faqir_xml_parser.query_answer_pairs_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "Xrt0bmmopISL"
   },
   "outputs": [],
   "source": [
    "# Dump data to json files\n",
    "\n",
    "dump_to_json(faqir_xml_parser.data, '../../../BERT-FAQ/data/FAQIR/FAQIRv1.0.json', sort_keys=False)\n",
    "dump_to_json(faqir_xml_parser.query_answer_pairs, '../../../BERT-FAQ/data/FAQIR/query_answer_pairs.json', sort_keys=False)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "01_Generating_Ground_Truth_Dataset_for_FAQIR.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}