"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
   "name": "assignment8_template.ipynb",
   "provenance": [],
   "collapsed_sections": [],
   "toc_visible": true
},
"kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
}
},
"cells": [
{
   "cell_type": "code",
   "metadata": {
   "id": "fKNaurvjTShp",
   "colab_type": "code",
   "colab": {}
   },
   "source": [
   "grading = False"
   ],
   "execution_count": 0,
   "outputs": []
},
{
   "cell_type": "code",
   "metadata": {
   "id": "9y7pkknmTW8m",
   "colab_type": "code",
   "colab": {}
   },
   "source": [
   "from copy import deepcopy\n",
   "\n",
   "import pandas as pd\n",
   "import spacy\n",
   "from sklearn.feature_extraction.text import TfidfVectorizer\n",
   "from sklearn.linear_model import PassiveAggressiveClassifier\n",
   "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, classification_report\n",
   "from sklearn.naive_bayes import MultinomialNB\n",
   "from sklearn.svm import SVC, LinearSVC\n",
   "from sklearn.model_selection import GridSearchCV"
   ],
   "execution_count": 0,
   "outputs": []
},
{
   "cell_type": "code",
   "metadata": {
   "id": "-MC3gFWWTYNq",
   "colab_type": "code",
   "colab": {}
   },
   "source": [
   "def load_questions():\n",
   " \"\"\"\n",
   " Task 1a)\n",
   "\n",
   " Load the questions and their classes.\n",
   "\n",
   " Complete the following steps:\n",
   "    - load the file \"train_5500.label.txt\" with \"ISO-8859-1\" encoding\n",
   "    - read the lines of the file\n",
   "    - consider only the lines that start with \"LOC:city\" or \"HUM:ind\"\n",
   "    - create a list in the following format:\n",
   "    [ (text, class), ... ]\n",
   "    - \"text\" is the content of the line AFTER the label\n",
   "       LOC:city What is the name of the city that Maurizio Pellegrin lives in ?\n",
   "    For this line the content would be: \"What is the name of the city that Maurizio Pellegrin lives in ?\"\n",
   "    - class is based on the label: \"CITY\" for \"LOC:city\" and \"HUMAN\" for \"HUM:ind\"\n",
   "    For the example above this would be \"CITY\"\n",
   "    - keep only the first 128 lines with class \"HUMAN\" (labelled as \"HUM:ind\")\n",
   "    - keep all samples for \"CITY\"\n",
   "    - return the list in the format described above\n",
   " \n",
   " INPUT:\n",
   "    - no input\n",
   " OUTPUT:\n",
   "    - texts: a list of tuples\n",
   "    Each tuple should represent ONE (1) line from the text file \"train_5500.label.txt\"\n",
   "    The first element in the tuple is the the question part of the line.\n",
   "    That means everything after the label.\n",
   "    Example:\n",
   "       Consider this line from the text file:\n",
   "          HUM:ind Name the scar-faced bounty hunter of The Old West .\n",
   "       To your output list you have to add this tuple:\n",
   "          (\"Name the scar-faced bounty hunter of The Old West .\", \"HUM:ind\")\n",
   "                      ^ the text of the question                ^ class\n",
   " \"\"\"\n",
   " texts = []\n",
   " ### YOUR CODE HERE ###\n",
   " f = open(\"train_5500.label.txt\", encoding=\"ISO-8859-1\")\n",
   " lines = f.readlines()\n",
   " hum_cout = 0\n",
   " for line in lines:\n",
   "    line = line.strip()\n",
   "    if line.startswith(\"LOC:city\"):\n",
   "    content = line[len(\"LOC:city\")+1:]\n",
   "    texts.append((content, \"CITY\"))\n",
   "    elif line.startswith(\"HUM:ind\") and hum_cout < 128:\n",
   "    content = line[len(\"HUM:ind\") + 1:]\n",
   "    texts.append((content, \"HUMAN\"))\n",
   "    hum_cout += 1\n",
   " return texts"
   ],
   "execution_count": 0,
   "outputs": []
},
{
   "cell_type": "code",
   "metadata": {
   "id": "8YKQ4hcGO0_r",
   "colab_type": "code",
   "colab": {
      "base_uri": "https://localhost:8080/",
      "height": 34
   },
   "outputId": "3d87375c-aa7f-4bfb-c353-62289f96c516"
   },
   "source": [
   "if not grading:\n",
   " # you need to import this before you can use it\n",
   " from collections import Counter\n",
   " # this is the function you must implement in task 1a)\n",
   " texts = load_questions()\n",
   " # this counts how often the second element (accessed with x[1]) appears\n",
   " # this means it counts how often each class appears in your dataset\n",
   " count = Counter(x[1] for x in texts)\n",
   " # the \"CITY\" class should appear 129 times, the \"HUMAN\" class should appear 128 times\n",
   " print(count)\n",
   " # Counter({'CITY': 129, 'HUMAN': 128})"
   ],
   "execution_count": 5,
   "outputs": [
   {
      "output_type": "stream",
      "text": [
         "Counter({'CITY': 129, 'HUMAN': 128})\n"
      ],
      "name": "stdout"
   }
   ]
},
{
   "cell_type": "code",
   "metadata": {
   "id": "j2Ohb-E6Tgm5",
   "colab_type": "code",
   "colab": {}
   },
   "source": [
   "def build_dataframe_q(texts):\n",
   " \"\"\"\n",
   " Task 1b)\n",
   "\n",
   " Process the texts and create a DataFrame you can use for training a classifier.\n",
   "\n",
   " Complete the following steps:\n",
   "    - load the spacy model \"en_core_web_sm\"\n",
   "    - for each text, create a spacy doc object\n",
   "    - for each token in this doc object, do:\n",
   "    - if the token begins an entity, keep the entity type\n",
   "    - elif the token is outside an entity, and not a stopword, keep token.text\n",
   "    - if the token is a stopword, continue\n",
   "    - keep token.text\n",
   "       (if the token is outside an entity then this means you keep token.text twice)\n",
   "    - you can find everything you need here: https://spacy.io/api/token\n",
   "    - create a dictionary for the current document in this format:\n",
   "       {\n",
   "          \"text\": list of the tokens joined with whitespace (use \" \".join(tokens)),\n",
   "          \"class\": the class of the text,\n",
   "       }\n",
   "    - store this dictionary, for each document, in a list\n",
   "    - create a new dataframe where each row is represented by one of the\n",
   "       document dictionaries\n",
   "    - return this dataframe\n",
   "\n",
   " vvvvvvvvvvvvvvvvvvvv\n",
   " To check whether or not a token is inside an entity, you can use this code.\n",
   " You need to put in a for-loop. This for-loop must iterate over each\n",
   " token in a text.\n",
   "\n",
   "    t = []\n",
   "    if token.ent_iob == 3:\n",
   "       t.append(token.ent_type_)\n",
   "    elif token.ent_iob == 2 and not token.is_stop:\n",
   "       t.append(token.text)\n",
   "    # if token is not a stopword, then use \"continue\" here\n",
   "    # if the token is NOT a stopword, add it to your list \"t\"\n",
   "\n",
   " \"token\" comes from a spaCy Document object. Refer to the spaCy documentation\n",
   " or the lecture slides to see how to process the tokens in a text with\n",
   " spaCy.\n",
   " You also need to check if a token is a stopword. If it is a stopword, use\n",
   " the \"continue\" keyword to go to the next token in your for-loop.\n",
   " After you have processed all tokens for the current text, combine them \n",
   " together from a list of strings (your \"t\" list) to a string.\n",
   " vvvvvvvvvvvvvvvvvvvv\n",
   "\n",
   " INPUT:\n",
   "    - texts: The list of texts that you get from your task 1a) function.\n",
   "    You need to pre-process the texts. In this case, that means that you have to\n",
   "    remove stopwords.\n",
   " OUTPUT:\n",
   "    - df: A DataFrame, where each row represents one text / question from the \"texts\"\n",
   "    list that you get as a function argument.\n",
   " PSEUDOCODE:\n",
   "    This is not actual code but it describes what your function will have to do.\n",
   "    If you translate this \"fake\" code into Python then your function is correct.\n",
   " ```\n",
   " rows = []\n",
   " nlp = load spacy model\n",
   " for text, class in texts:\n",
   "    doc = make spaCy document object out of \"text\"\n",
   "    t = []\n",
   "    for token in doc:\n",
   "          if token.ent_iob == 3:\n",
   "             t.append(token.ent_type_)\n",
   "          elif token.ent_iob == 2 and not token.is_stop:\n",
   "             t.append(token.text)\n",
   "          if token is a stopword:\n",
   "             continue\n",
   "          append \"token.text\" to the list \"t\"\n",
   "    t = \"Â \".join(t)\n",
   "    text_information = dictionary that has the keys \"text\" and \"class\" that maps to the joined together string (t) and the class of the text (from the for loop)\n",
   "    append \"text_information\" to the list \"rows\"\n",
   " df = a DataFrame that contains the content of the \"rows\" list as actual DataFrame rows\n",
   " return df\n",
   " ```\n",
   " \"\"\"\n",
   " rows = []\n",
   " ### YOUR CODE HERE ###\n",
   " nlp = spacy.load(\"en_core_web_sm\")\n",
   " for text, c in texts:\n",
   "    doc = nlp(text)\n",
   "    t= []\n",
   "    for token in doc:\n",
   "    if token.ent_iob == 3:\n",
   "       t.append(token.ent_type_)\n",
   "    elif token.ent_iob == 2 and not token.is_stop:\n",
   "       t.append(token.text)\n",
   "    if token.is_stop:\n",
   "          continue\n",
   "    t.append(token.text)\n",
   "    t = \"Â \".join(t)\n",
   "    text_information = {'text': t, 'class': c}\n",
   "    rows.append(text_information)\n",
   " df = pd.DataFrame(rows)\n",
   " return df"
   ],
   "execution_count": 0,
   "outputs": []
},
{
   "cell_type": "code",
   "metadata": {
   "id": "XENeY7BDSj9M",
   "colab_type": "code",
   "colab": {
      "base_uri": "https://localhost:8080/",
      "height": 204
   },
   "outputId": "22b932ab-1743-431d-a744-5ca99ebda750

import matplotlib.pyplot as plt %matplotlib inline %precision 16...