![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\"){kind=icon}
"
1401 | ]
1402 | },
1403 | {
1404 | "cell_type": "code",
1405 | "source": [
1406 | "!pip install transformers"
1407 | ],
1408 | "metadata": {
1409 | "colab": {
1410 | "base_uri": "https://localhost:8080/"
1411 | },
1412 | "id": "Swz2kIIIdA7V",
1413 | "outputId": "493226cb-73b6-4274-d8a5-33538410b5b2"
1414 | },
1415 | "execution_count": null,
1416 | "outputs": [
1417 | {
1418 | "output_type": "stream",
1419 | "name": "stdout",
1420 | "text": [
1421 | "Collecting transformers\n",
1422 | " Downloading transformers-4.31.0-py3-none-any.whl (7.4 MB)\n",
1423 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.4/7.4 MB\u001b[0m \u001b[31m15.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
1424 | "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.2)\n",
1425 | "Collecting huggingface-hub<1.0,>=0.14.1 (from transformers)\n",
1426 | " Downloading huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\n",
1427 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m26.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
1428 | "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.23.5)\n",
1429 | "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
1430 | "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
1431 | "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n",
1432 | "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
1433 | "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n",
1434 | " Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
1435 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m36.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
1436 | "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n",
1437 | " Downloading safetensors-0.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
1438 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m48.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
1439 | "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n",
1440 | "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.6.0)\n",
1441 | "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.7.1)\n",
1442 | "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.2.0)\n",
1443 | "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
1444 | "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.16)\n",
1445 | "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.7.22)\n",
1446 | "Installing collected packages: tokenizers, safetensors, huggingface-hub, transformers\n",
1447 | "Successfully installed huggingface-hub-0.16.4 safetensors-0.3.2 tokenizers-0.13.3 transformers-4.31.0\n"
1448 | ]
1449 | }
1450 | ]
1451 | },
1452 | {
1453 | "cell_type": "code",
1454 | "execution_count": null,
1455 | "metadata": {
1456 | "colab": {
1457 | "base_uri": "https://localhost:8080/"
1458 | },
1459 | "id": "UpEfXvKoZfDc",
1460 | "outputId": "e78fd96d-5b76-43df-eb49-b97279a35697"
1461 | },
1462 | "outputs": [
1463 | {
1464 | "output_type": "stream",
1465 | "name": "stdout",
1466 | "text": [
1467 | "Mounted at /content/gdrive\n"
1468 | ]
1469 | }
1470 | ],
1471 | "source": [
1472 | "from google.colab import drive\n",
1473 | "drive.mount('/content/gdrive', force_remount=True)\n",
1474 | "root_dir = \"/content/gdrive/My Drive/training/\""
1475 | ]
1476 | },
1477 | {
1478 | "cell_type": "code",
1479 | "source": [
1480 | "import os\n",
1481 | "os.listdir(root_dir)"
1482 | ],
1483 | "metadata": {
1484 | "colab": {
1485 | "base_uri": "https://localhost:8080/"
1486 | },
1487 | "id": "buoZFap_aGav",
1488 | "outputId": "56b029e5-cf1e-4e18-8248-3d5b7a0fce49"
1489 | },
1490 | "execution_count": null,
1491 | "outputs": [
1492 | {
1493 | "output_type": "execute_result",
1494 | "data": {
1495 | "text/plain": [
1496 | "['train.csv', '.ipynb_checkpoints']"
1497 | ]
1498 | },
1499 | "metadata": {},
1500 | "execution_count": 4
1501 | }
1502 | ]
1503 | },
1504 | {
1505 | "cell_type": "code",
1506 | "source": [
1507 | "import pandas as pd\n",
1508 | "\n",
1509 | "root_path = f\"{root_dir}training/train.csv\"\n",
1510 | "df = pd.read_csv(f\"{root_dir}train.csv\")\n",
1511 | "df.head()"
1512 | ],
1513 | "metadata": {
1514 | "colab": {
1515 | "base_uri": "https://localhost:8080/",
1516 | "height": 206
1517 | },
1518 | "id": "DJBFucr5ahZb",
1519 | "outputId": "90cfbc8e-fc29-4f5a-fdaa-9470e697cec2"
1520 | },
1521 | "execution_count": null,
1522 | "outputs": [
1523 | {
1524 | "output_type": "execute_result",
1525 | "data": {
1526 | "text/plain": [
1527 | " category text\n",
1528 | "0 SearchDynamicNews Which city hosted a major international confer...\n",
1529 | "1 SearchDynamicNews What is the current status of the cryptocurren...\n",
1530 | "2 SearchDynamicNews What is the current status of the global clima...\n",
1531 | "3 NormalChat I'm currently reading a mystery novel.\n",
1532 | "4 NormalChat Have you seen the latest episode of that TV show?"
1533 | ],
1534 | "text/html": [
1535 | "\n",
1536 | "\n",
1537 | " | \n", 1557 | " | category | \n", 1558 | "text | \n", 1559 | "
|---|---|---|
| 0 | \n", 1564 | "SearchDynamicNews | \n", 1565 | "Which city hosted a major international confer... | \n", 1566 | "
| 1 | \n", 1569 | "SearchDynamicNews | \n", 1570 | "What is the current status of the cryptocurren... | \n", 1571 | "
| 2 | \n", 1574 | "SearchDynamicNews | \n", 1575 | "What is the current status of the global clima... | \n", 1576 | "
| 3 | \n", 1579 | "NormalChat | \n", 1580 | "I'm currently reading a mystery novel. | \n", 1581 | "
| 4 | \n", 1584 | "NormalChat | \n", 1585 | "Have you seen the latest episode of that TV show? | \n", 1586 | "
| \n", 1807 | " | category | \n", 1808 | "text | \n", 1809 | "encoded_text | \n", 1810 | "
|---|---|---|---|
| 0 | \n", 1815 | "SearchDynamicNews | \n", 1816 | "Which city hosted a major international confer... | \n", 1817 | "1 | \n", 1818 | "
| 1 | \n", 1821 | "SearchDynamicNews | \n", 1822 | "What is the current status of the cryptocurren... | \n", 1823 | "1 | \n", 1824 | "
| 2 | \n", 1827 | "SearchDynamicNews | \n", 1828 | "What is the current status of the global clima... | \n", 1829 | "1 | \n", 1830 | "
| 3 | \n", 1833 | "NormalChat | \n", 1834 | "I'm currently reading a mystery novel. | \n", 1835 | "0 | \n", 1836 | "
| 4 | \n", 1839 | "NormalChat | \n", 1840 | "Have you seen the latest episode of that TV show? | \n", 1841 | "0 | \n", 1842 | "
| 5 | \n", 1845 | "SearchDynamicNews | \n", 1846 | "What are the updates on international trade ag... | \n", 1847 | "1 | \n", 1848 | "
| 6 | \n", 1851 | "SearchDynamicNews | \n", 1852 | "Who is the CEO of Twitter? | \n", 1853 | "1 | \n", 1854 | "
| 7 | \n", 1857 | "SearchDynamicNews | \n", 1858 | "What is the current situation of the pandemic ... | \n", 1859 | "1 | \n", 1860 | "
| 8 | \n", 1863 | "NormalChat | \n", 1864 | "Board game nights are so much fun! | \n", 1865 | "0 | \n", 1866 | "
| 9 | \n", 1869 | "SearchDynamicNews | \n", 1870 | "What is the breakthrough in cancer research? | \n", 1871 | "1 | \n", 1872 | "
| \n", 2093 | " | category | \n", 2094 | "Phrase | \n", 2095 | "Sentiment | \n", 2096 | "
|---|---|---|---|
| 0 | \n", 2101 | "SearchDynamicNews | \n", 2102 | "Which city hosted a major international confer... | \n", 2103 | "1 | \n", 2104 | "
| 1 | \n", 2107 | "SearchDynamicNews | \n", 2108 | "What is the current status of the cryptocurren... | \n", 2109 | "1 | \n", 2110 | "
| 2 | \n", 2113 | "SearchDynamicNews | \n", 2114 | "What is the current status of the global clima... | \n", 2115 | "1 | \n", 2116 | "
| 3 | \n", 2119 | "NormalChat | \n", 2120 | "I'm currently reading a mystery novel. | \n", 2121 | "0 | \n", 2122 | "
| 4 | \n", 2125 | "NormalChat | \n", 2126 | "Have you seen the latest episode of that TV show? | \n", 2127 | "0 | \n", 2128 | "
| 5 | \n", 2131 | "SearchDynamicNews | \n", 2132 | "What are the updates on international trade ag... | \n", 2133 | "1 | \n", 2134 | "
| 6 | \n", 2137 | "SearchDynamicNews | \n", 2138 | "Who is the CEO of Twitter? | \n", 2139 | "1 | \n", 2140 | "
| 7 | \n", 2143 | "SearchDynamicNews | \n", 2144 | "What is the current situation of the pandemic ... | \n", 2145 | "1 | \n", 2146 | "
| 8 | \n", 2149 | "NormalChat | \n", 2150 | "Board game nights are so much fun! | \n", 2151 | "0 | \n", 2152 | "
| 9 | \n", 2155 | "SearchDynamicNews | \n", 2156 | "What is the breakthrough in cancer research? | \n", 2157 | "1 | \n", 2158 | "