├── .env.example ├── .gitignore ├── Fetch Magang Report.ipynb └── Fetch Studi Independen Report.ipynb /.env.example: -------------------------------------------------------------------------------- 1 | EMAIL= 2 | PASSWORD= -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | Daily Report.csv 3 | Weekly Report.csv 4 | *.csv 5 | *.xlsx -------------------------------------------------------------------------------- /Fetch Magang Report.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 104, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "data": { 10 | "text/plain": [ 11 | "'id'" 12 | ] 13 | }, 14 | "execution_count": 104, 15 | "metadata": {}, 16 | "output_type": "execute_result" 17 | } 18 | ], 19 | "source": [ 20 | "from dotenv import dotenv_values\n", 21 | "from datetime import timedelta, datetime\n", 22 | "\n", 23 | "import pandas as pd\n", 24 | "import requests\n", 25 | "import locale\n", 26 | "\n", 27 | "config = dotenv_values(\".env\")\n", 28 | "locale.setlocale(locale.LC_ALL, \"id\")" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 105, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "url = 'https://api.kampusmerdeka.kemdikbud.go.id/user/auth/login/mbkm'\n", 38 | "headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}\n", 39 | "obj = {\n", 40 | " 'email': config.get('EMAIL'),\n", 41 | " 'password': config.get('PASSWORD')\n", 42 | "}\n", 43 | "\n", 44 | "response = requests.post(url, json=obj, headers=headers)\n", 45 | "token = response.json()['data'].get('access_token')" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 106, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "headers = dict()\n", 55 | "headers[\"Accept\"] = \"application/json\"\n", 56 | "headers[\"Authorization\"] = f'Bearer {token}'\n", 57 | "headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 107, 63 | "metadata": {}, 64 | "outputs": [], 65 | "source": [ 66 | "url = \"https://api.kampusmerdeka.kemdikbud.go.id/mbkm/mahasiswa/active-activities/my\"\n", 67 | "response = requests.get(url, headers=headers)\n", 68 | "activity_id = response.json().get('data')[0].get('id')\n" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 108, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "url = f'https://api.kampusmerdeka.kemdikbud.go.id/magang/report/allweeks/{activity_id}'\n", 78 | "response = requests.get(url, headers=headers)\n", 79 | "reports = response.json().get('data')" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 110, 85 | "metadata": {}, 86 | "outputs": [], 87 | "source": [ 88 | "weekly_reports = []\n", 89 | "daily_reports = []\n", 90 | "\n", 91 | "for report in reversed(list(reports)):\n", 92 | " date = report.get('start_date')[0:10]\n", 93 | " date = datetime.strptime(date, '%Y-%m-%d')\n", 94 | "\n", 95 | " date_format = date.strftime(\"%d %B %Y\")\n", 96 | " date_format = date_format.split()\n", 97 | "\n", 98 | " weekly_reports.append([report.get(\"learned_weekly\"), f\"{date_format[1]} {date_format[2]}\"])\n", 99 | "\n", 100 | " for j, daily_report in enumerate(report.get('daily_report')):\n", 101 | " if j == 0:\n", 102 | " dt = date \n", 103 | " else:\n", 104 | " dt = dt + timedelta(days=1)\n", 105 | "\n", 106 | " d = dt.strftime(\"%A, %d %B %Y\")\n", 107 | " daily_reports.append([daily_report.get('report'), d])" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": 111, 113 | "metadata": {}, 114 | "outputs": [ 115 | { 116 | "data": { 117 | "text/html": [ 118 | "
\n", 119 | "\n", 132 | "\n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | "
DeskripsiBulan
12Pada minggu ini masih berfokus untuk menangani...November 2021
13Pada minggu ini team backend berfokus untuk me...November 2021
14Pada minggu ini team backend berfokus kepada p...November 2021
\n", 158 | "
" 159 | ], 160 | "text/plain": [ 161 | " Deskripsi Bulan\n", 162 | "12 Pada minggu ini masih berfokus untuk menangani... November 2021\n", 163 | "13 Pada minggu ini team backend berfokus untuk me... November 2021\n", 164 | "14 Pada minggu ini team backend berfokus kepada p... November 2021" 165 | ] 166 | }, 167 | "execution_count": 111, 168 | "metadata": {}, 169 | "output_type": "execute_result" 170 | } 171 | ], 172 | "source": [ 173 | "df_weekly = pd.DataFrame(weekly_reports, columns=['Deskripsi', 'Bulan'])\n", 174 | "df_weekly[12:15]" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": 112, 180 | "metadata": {}, 181 | "outputs": [ 182 | { 183 | "data": { 184 | "text/html": [ 185 | "
\n", 186 | "\n", 199 | "\n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | "
DeskripsiHari
35Membuat Endpoint (Terdiri Atas Model, Controll...Senin, 11 Oktober 2021
36Mengerjakan CRUD (Create Read Update Delete) y...Selasa, 12 Oktober 2021
37Melakukan testing dan evaluasi setiap endpoint...Rabu, 13 Oktober 2021
\n", 225 | "
" 226 | ], 227 | "text/plain": [ 228 | " Deskripsi Hari\n", 229 | "35 Membuat Endpoint (Terdiri Atas Model, Controll... Senin, 11 Oktober 2021\n", 230 | "36 Mengerjakan CRUD (Create Read Update Delete) y... Selasa, 12 Oktober 2021\n", 231 | "37 Melakukan testing dan evaluasi setiap endpoint... Rabu, 13 Oktober 2021" 232 | ] 233 | }, 234 | "execution_count": 112, 235 | "metadata": {}, 236 | "output_type": "execute_result" 237 | } 238 | ], 239 | "source": [ 240 | "df_daily = pd.DataFrame(daily_reports, columns=['Deskripsi', 'Hari'])\n", 241 | "df_daily[35:38]" 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": 113, 247 | "metadata": {}, 248 | "outputs": [], 249 | "source": [ 250 | "df_weekly.to_csv('Weekly Report.csv', index=False)\n", 251 | "df_daily.to_csv('Daily Report.csv', index=False)" 252 | ] 253 | } 254 | ], 255 | "metadata": { 256 | "interpreter": { 257 | "hash": "2bf746931fd58f1246b6df4740a30aa94cbb7406c646ac7ef817c7ae9c273626" 258 | }, 259 | "kernelspec": { 260 | "display_name": "Python 3.9.0 64-bit", 261 | "language": "python", 262 | "name": "python3" 263 | }, 264 | "language_info": { 265 | "codemirror_mode": { 266 | "name": "ipython", 267 | "version": 3 268 | }, 269 | "file_extension": ".py", 270 | "mimetype": "text/x-python", 271 | "name": "python", 272 | "nbconvert_exporter": "python", 273 | "pygments_lexer": "ipython3", 274 | "version": "3.10.2" 275 | }, 276 | "orig_nbformat": 4 277 | }, 278 | "nbformat": 4, 279 | "nbformat_minor": 2 280 | } 281 | -------------------------------------------------------------------------------- /Fetch Studi Independen Report.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Studi Independen Crawler" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Step 1: Credential Prerequisite\n", 15 | "\n", 16 | "Before you begin using this script. Please create `.env` file from the template `.env.example`. After creating that file, please fill your Kampus Merdeka email and password to the `.env` file." 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "## Step 2: Installing library" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "!pip install python-dotenv pandas requests openpyxl" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": null, 38 | "metadata": {}, 39 | "outputs": [], 40 | "source": [ 41 | "from dotenv import dotenv_values\n", 42 | "from datetime import timedelta, datetime\n", 43 | "\n", 44 | "import pandas as pd\n", 45 | "import requests\n", 46 | "import locale" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": {}, 52 | "source": [ 53 | "## Step 3: Login to Kampus Merdeka" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": null, 59 | "metadata": {}, 60 | "outputs": [], 61 | "source": [ 62 | "config = dotenv_values(\".env\")\n", 63 | "locale.setlocale(locale.LC_ALL, \"id\")\n", 64 | "\n", 65 | "def login() -> str:\n", 66 | " try:\n", 67 | " url = 'https://api.kampusmerdeka.kemdikbud.go.id/user/auth/login/mbkm'\n", 68 | " headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}\n", 69 | " obj = {\n", 70 | " 'email': config.get('EMAIL'),\n", 71 | " 'password': config.get('PASSWORD')\n", 72 | " }\n", 73 | " response = requests.post(url, json=obj, headers=headers)\n", 74 | " token = response.json()['data'].get('access_token')\n", 75 | " except Exception:\n", 76 | " raise ValueError(response.json()[\"error\"][\"message\"])\n", 77 | " return token\n", 78 | "\n", 79 | "token = login()\n", 80 | "headers = dict()\n", 81 | "headers[\"Accept\"] = \"application/json\"\n", 82 | "headers[\"Authorization\"] = f'Bearer {token}'\n", 83 | "headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'\n", 84 | "print(\"Login success\")" 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "metadata": {}, 90 | "source": [ 91 | "## Step 4: Choose your Kampus Merdeka Studi Independen Activity" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": null, 97 | "metadata": {}, 98 | "outputs": [], 99 | "source": [ 100 | "def get_activity_id():\n", 101 | " url = \"https://api.kampusmerdeka.kemdikbud.go.id/mbkm/mahasiswa/activities\"\n", 102 | " response = requests.get(url, headers=headers)\n", 103 | " \n", 104 | " activities = response.json().get('data')\n", 105 | " \n", 106 | " print(\"Your Studi Independen activities\")\n", 107 | " for i in range(len(activities)):\n", 108 | " activity = activities[i][\"nama_kegiatan\"]\n", 109 | " print(f\"{i+1}. {activity}\")\n", 110 | " input_num = int(input(\"Choose your activity from number above: \"))\n", 111 | " return activities[input_num - 1][\"id\"]\n", 112 | " \n", 113 | "activity_id = get_activity_id()\n", 114 | "print(f\"Got Activity Id : {activity_id}\")" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": {}, 120 | "source": [ 121 | "## Step 5: Get Reports" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": null, 127 | "metadata": {}, 128 | "outputs": [], 129 | "source": [ 130 | "url = f'https://api.kampusmerdeka.kemdikbud.go.id/studi/report/allweeks/{activity_id}'\n", 131 | "response = requests.get(url, headers=headers)\n", 132 | "reports = response.json().get('data')\n", 133 | "\n", 134 | "counter_minggu_ke = 1\n", 135 | "\n", 136 | "minggu_tanggal_cols = []\n", 137 | "kegiatan_cols = []\n", 138 | "hasil_cols = []\n", 139 | "\n", 140 | "for weekly_report in reports[::-1]:\n", 141 | " \n", 142 | " # Handle Minggu Tanggal Cols\n", 143 | " start_date = datetime.strptime(weekly_report[\"start_date\"], \"%Y-%m-%dT%H:%M:%SZ\")\n", 144 | " end_date = datetime.strptime(weekly_report[\"end_date\"], \"%Y-%m-%dT%H:%M:%SZ\")\n", 145 | " start_date_str = start_date.strftime(\"%d %B %Y\")\n", 146 | " end_date_str = end_date.strftime(\"%d %B %Y\")\n", 147 | " minggu_tanggal = f\"Minggu ke-{counter_minggu_ke} \\n({start_date_str} - {end_date_str})\"\n", 148 | " minggu_tanggal_cols.append(minggu_tanggal)\n", 149 | " counter_minggu_ke += 1\n", 150 | " \n", 151 | " # Handle Kegiatan Cols\n", 152 | " daily_reports = weekly_report[\"daily_report\"]\n", 153 | " counter = 1\n", 154 | " final_kegiatan_str = \"\"\n", 155 | " for daily_report in daily_reports:\n", 156 | " report = daily_report[\"report\"]\n", 157 | " final_kegiatan_str += f\"{counter}. {report} \\n\"\n", 158 | " counter += 1\n", 159 | " kegiatan_cols.append(final_kegiatan_str)\n", 160 | " \n", 161 | " # Handle Hasil Cols\n", 162 | " hasil_cols.append(weekly_report[\"learned_weekly\"])\n", 163 | "\n", 164 | "reports = pd.DataFrame({\n", 165 | " \"Minggu/Tgl\" : minggu_tanggal_cols,\n", 166 | " \"Kegiatan Harian\": kegiatan_cols,\n", 167 | " \"Laporan Mingguan\": hasil_cols\n", 168 | "})" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": null, 174 | "metadata": {}, 175 | "outputs": [], 176 | "source": [ 177 | "reports" 178 | ] 179 | }, 180 | { 181 | "cell_type": "markdown", 182 | "metadata": {}, 183 | "source": [ 184 | "## Step 6: Output to Files" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": null, 190 | "metadata": {}, 191 | "outputs": [], 192 | "source": [ 193 | "def output_files(df: pd.DataFrame):\n", 194 | " output_types = [\"csv\", \"excel\"]\n", 195 | " print(\"Output Types\")\n", 196 | " for i in range(len(output_types)):\n", 197 | " output_type = output_types[i]\n", 198 | " print(f\"{i+1}. {output_type}\")\n", 199 | " input_num = int(input(\"Choose your output type from number above: \")) - 1\n", 200 | " \n", 201 | " if input_num == 0:\n", 202 | " df.to_csv(\"./reports.csv\",index=False)\n", 203 | " elif input_num == 1:\n", 204 | " df.to_excel(\"./reports.xlsx\", index=False)\n", 205 | " else:\n", 206 | " raise ValueError(\"Not a valid input\")\n", 207 | "\n", 208 | " print(f\"Creating {output_types[input_num]} with a file name `reports`\")\n", 209 | " \n", 210 | "output_files(reports)" 211 | ] 212 | } 213 | ], 214 | "metadata": { 215 | "kernelspec": { 216 | "display_name": "Python 3.8.0 64-bit", 217 | "language": "python", 218 | "name": "python3" 219 | }, 220 | "language_info": { 221 | "codemirror_mode": { 222 | "name": "ipython", 223 | "version": 3 224 | }, 225 | "file_extension": ".py", 226 | "mimetype": "text/x-python", 227 | "name": "python", 228 | "nbconvert_exporter": "python", 229 | "pygments_lexer": "ipython3", 230 | "version": "3.8.0" 231 | }, 232 | "orig_nbformat": 4, 233 | "vscode": { 234 | "interpreter": { 235 | "hash": "570feb405e2e27c949193ac68f46852414290d515b0ba6e5d90d076ed2284471" 236 | } 237 | } 238 | }, 239 | "nbformat": 4, 240 | "nbformat_minor": 2 241 | } 242 | --------------------------------------------------------------------------------