11_kod_dane_tekstowe.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d30b30c4",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>my arm still hurts from when i pulled it yeste...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>I have so much to do outside! Been looking at ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>@AbsolutSara Yes, I knew about the clusterfark...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>Just woke up and i feel relieved Haha now i ha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>LOVING the hot weather forecast for the rest o...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment                                               text\n",
       "0          0  my arm still hurts from when i pulled it yeste...\n",
       "1          1  I have so much to do outside! Been looking at ...\n",
       "2          0  @AbsolutSara Yes, I knew about the clusterfark...\n",
       "3          0  Just woke up and i feel relieved Haha now i ha...\n",
       "4          1  LOVING the hot weather forecast for the rest o..."
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "data = pd.read_csv(\"data/tweets.csv\", header=None, usecols=[1,6])\n",
    "data.columns = [\"sentiment\", \"text\"]\n",
    "data[\"sentiment\"] = data.sentiment.replace({4:1})\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "45fc5615",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30000,)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = data.text\n",
    "y = data.sentiment\n",
    "\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "64fac738",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f6517084",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, \n",
    "                                                test_size=0.1, \n",
    "                                                random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "264370db",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b9bb42e9",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer()"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = CountVectorizer()\n",
    "vectorizer.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c0bb1af3",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['00', '000', '000thb', ..., 'ø¹ø', 'ø¹øª', 'ø¹ù'], dtype=object)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.get_feature_names_out()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2809b4bf",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "37682"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vectorizer.get_feature_names_out())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "605d9a3d",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "X_train_vec = vectorizer.transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cea016c5",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<27000x37682 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 320153 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "85e0232c",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0003146732795106024"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "320153/(27000*37682)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4d2710b2",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_vec[:10,:10].toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2d91fec3",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "?CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "611c34a0",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7011"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = CountVectorizer(min_df=3)\n",
    "vectorizer.fit(X_train)\n",
    "len(vectorizer.get_feature_names_out())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "fcff6929",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7000"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = CountVectorizer(min_df=3, max_df=0.1)\n",
    "vectorizer.fit(X_train)\n",
    "len(vectorizer.get_feature_names_out())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7404901b",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "?CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1a8ea869",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5000"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = CountVectorizer(max_features=5000)\n",
    "vectorizer.fit(X_train)\n",
    "len(vectorizer.get_feature_names_out())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9c18c018",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import make_pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5308858e",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nor/anaconda3/envs/Elearn/lib/python3.9/site-packages/sklearn/svm/_base.py:1199: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.735"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline = make_pipeline(CountVectorizer(min_df=3), LinearSVC())\n",
    "pipeline.fit(X_train, y_train)\n",
    "accuracy_score(y_test, pipeline.predict(X_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52bcbc1a",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Jeśli podczas wywoływania transform pojawią się słowa, które nie pojawiły się podczas fit - zostaną pominięte."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c32844d5",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['maszynowe', 'uczenie', 'uczenie maszynowe'], dtype=object)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts = [\"Lubię uczenie maszynowe\", \n",
    "         \"Uczenie maszynowe jest najlepsze\"]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(1,2), min_df=2)\n",
    "vectorizer.fit(texts)\n",
    "vectorizer.get_feature_names_out()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5cf00235",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['jest', 'lubię', 'maszynowe', 'super', 'uczenie'], dtype=object)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts = [\"Lubię uczenie maszynowe <3\", \n",
    "         \"I uczenie maszynowe jest super!!\"]\n",
    "\n",
    "vectorizer = CountVectorizer()\n",
    "vectorizer.fit(texts)\n",
    "vectorizer.get_feature_names_out()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e4ba088",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Nie ma słowa \"I\" i liczby \"3\" - bo to \"słowa\" jednoznakowe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "b09118dd",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "?CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "da982cec",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['3', 'i', 'jest', 'lubię', 'maszynowe', 'super', 'uczenie'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = CountVectorizer(token_pattern='(?u)\\\\b\\\\w+\\\\b')\n",
    "vectorizer.fit(texts)\n",
    "vectorizer.get_feature_names_out()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47ae83e8",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Przygotowanie danych tekstowych do modelowania"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "6468eaa1",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "346                                 is home for the night \n",
       "13028                                   @ThisismyiQ Im up \n",
       "8821     @vickycornell and by the way, your damn right....\n",
       "25676    3 boys asleep and oldest not home from school ...\n",
       "7534     @MelFresh27 I'm going to try the servo across ...\n",
       "                               ...                        \n",
       "29802             portobello road and wimbledon for a bbq \n",
       "5390     @Shannenp it is absolutely genius!  i should s...\n",
       "860      @piginthepoke later gator, will keep you updat...\n",
       "15795    @aaronh Interesting! No, I didn't know. It's a...\n",
       "23654    @Wossy I'm listening too, it's not the same pr...\n",
       "Name: text, Length: 27000, dtype: object"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "23562e61",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7ea5a335",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "x = \"@JessMcFlyxxx haha you ok?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "4e7939e5",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'  haha you ok?'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.sub(\"@\\w+\", \" \", x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "806b4d14",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "X_train = X_train.apply(lambda x: re.sub(\"@\\w+\", \" \", x))\n",
    "X_test = X_test.apply(lambda x: re.sub(\"@\\w+\", \" \", x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c4cd1a39",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"Sleeping alone tonight cause Crystal went to Josie's house. \",\n",
       " '  There probably are a few down in the depths, I sometimes wonder what I would do if I hauled one up ',\n",
       " \"  she woke up this morning when I was moving the wheel down and she was making these squeeky noises  she's mad at me still\",\n",
       " \"Going to bedd soon. I am sooooooo tired  gonna fall asleep to the sound of &quot;You're the one that I want the one that I want ooh ooh ooh ya&quot;\",\n",
       " \"migraines suck   I hope truffles don't make them worse ;)\",\n",
       " \"Today wasn't so bad  I started reading just listen and today I decided that I'm going to iron my bangs since I havent done so in like days\",\n",
       " 'Today was just one long hang over ',\n",
       " '  receptionist! hehe ',\n",
       " \"  kriiiiista, please say ur alive, not seen u in almost a week! I'm going to be shit at coping when u go to Uni  xxx\",\n",
       " 'going to bed tomorrow long day first school then working... ']"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(X_train[20:30])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f9bde89c",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ooh ya '"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = \"ooh ya&quot;\"\n",
    "x.replace(\"&quot;\", \" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "1aaae69a",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "X_train = X_train.apply(lambda x: x.replace(\"&quot;\", \" \"))\n",
    "X_test = X_test.apply(lambda x: x.replace(\"&quot;\", \" \"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0cae6217",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Czyszczenie tektów polega na usunięciu zawartości, której nie chcemy uwzględniać w modelowaniu."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "858e0171",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Konkrety zależą od celu działania algorytmu."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04328813",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "W życiu zazwyczaj trzeba czyścić i często jest to duża część pracy. "
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Elearn",
   "language": "python",
   "name": "elearn"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}