├── test ├── test_helper.exs ├── search_api_test.exs └── class_parser_test.exs ├── .gitignore ├── README.md ├── lib ├── .cookie_agent.ex.swp ├── .search_api.ex.swo ├── .search_api.ex.swp ├── cookie_agent.ex ├── class_parser.ex └── search_api.ex ├── mix.lock ├── mix.exs └── config └── config.exs /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /_build 2 | /deps 3 | erl_crash.dump 4 | *.ez 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ClassParser 2 | =========== 3 | 4 | ** TODO: Add description ** 5 | -------------------------------------------------------------------------------- /lib/.cookie_agent.ex.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iheanyi/class_parser_elixir/master/lib/.cookie_agent.ex.swp -------------------------------------------------------------------------------- /lib/.search_api.ex.swo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iheanyi/class_parser_elixir/master/lib/.search_api.ex.swo -------------------------------------------------------------------------------- /lib/.search_api.ex.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iheanyi/class_parser_elixir/master/lib/.search_api.ex.swp -------------------------------------------------------------------------------- /test/search_api_test.exs: -------------------------------------------------------------------------------- 1 | defmodule SearchApiTest do 2 | use ExUnit.Case 3 | 4 | test "the truth" do 5 | assert 1 + 1 == 2 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /test/class_parser_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ClassParserTest do 2 | use ExUnit.Case 3 | 4 | test "the truth" do 5 | assert 1 + 1 == 2 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{"floki": {:hex, :floki, "0.3.3"}, 2 | "hackney": {:hex, :hackney, "1.3.1"}, 3 | "httpoison": {:hex, :httpoison, "0.7.2"}, 4 | "idna": {:hex, :idna, "1.0.2"}, 5 | "mochiweb": {:hex, :mochiweb, "2.12.2"}, 6 | "poison": {:hex, :poison, "1.5.0"}, 7 | "ssl_verify_hostname": {:hex, :ssl_verify_hostname, "1.0.5"}} 8 | -------------------------------------------------------------------------------- /lib/cookie_agent.ex: -------------------------------------------------------------------------------- 1 | defmodule CookieAgent do 2 | 3 | @doc """ 4 | Starts a blank cookie. 5 | """ 6 | def start_link(cookie_string) do 7 | Agent.start_link(fn -> cookie_string end, name: :cookie) 8 | end 9 | 10 | @doc """ 11 | Gets the cookie's value 12 | """ 13 | def get(agent) do 14 | Agent.get(agent, fn cookie -> cookie end) 15 | end 16 | 17 | @doc """ 18 | Updates the cookie's value 19 | """ 20 | def set(agent, value) do 21 | IO.puts 'Updating agent with value: #{value}' 22 | Agent.get_and_update(agent, fn cookie -> {:ok, value} end) 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/class_parser.ex: -------------------------------------------------------------------------------- 1 | defmodule ClassParser do 2 | use Application 3 | 4 | # See http://elixir-lang.org/docs/stable/elixir/Application.html 5 | # for more information on OTP Applications 6 | def start(_type, _args) do 7 | import Supervisor.Spec, warn: false 8 | 9 | children = [ 10 | # Define workers and child supervisors to be supervised 11 | # worker(ClassParser.Worker, [arg1, arg2, arg3]) 12 | ] 13 | 14 | # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html 15 | # for other strategies and supported options 16 | opts = [strategy: :one_for_one, name: ClassParser.Supervisor] 17 | Supervisor.start_link(children, opts) 18 | end 19 | 20 | end 21 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule ClassParser.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [app: :class_parser, 6 | version: "0.0.1", 7 | elixir: "~> 1.0", 8 | build_embedded: Mix.env == :prod, 9 | start_permanent: Mix.env == :prod, 10 | deps: deps] 11 | end 12 | 13 | # Configuration for the OTP application 14 | # 15 | # Type `mix help compile.app` for more information 16 | def application do 17 | [applications: [:logger, :httpoison], 18 | mod: {ClassParser, []}] 19 | end 20 | 21 | # Dependencies can be Hex packages: 22 | # 23 | # {:mydep, "~> 0.3.0"} 24 | # 25 | # Or git/path repositories: 26 | # 27 | # {:mydep, git: "https://github.com/elixir-lang/mydep.git", tag: "0.1.0"} 28 | # 29 | # Type `mix help deps` for more examples and options 30 | defp deps do 31 | [ 32 | {:floki, "~> 0.3"}, 33 | {:httpoison, "~> 0.7.2"}, 34 | {:poison, "~> 1.5"} 35 | ] 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Mix.Config module. 3 | use Mix.Config 4 | 5 | # This configuration is loaded before any dependency and is restricted 6 | # to this project. If another project depends on this project, this 7 | # file won't be loaded nor affect the parent project. For this reason, 8 | # if you want to provide default values for your application for third- 9 | # party users, it should be done in your mix.exs file. 10 | 11 | # Sample configuration: 12 | # 13 | # config :logger, :console, 14 | # level: :info, 15 | # format: "$date $time [$level] $metadata$message\n", 16 | # metadata: [:user_id] 17 | 18 | # It is also possible to import configuration files, relative to this 19 | # directory. For example, you can emulate configuration per environment 20 | # by uncommenting the line below and defining dev.exs, test.exs and such. 21 | # Configuration from the imported file will override the ones defined 22 | # here (which is why it is important to import them last). 23 | # 24 | # import_config "#{Mix.env}.exs" 25 | -------------------------------------------------------------------------------- /lib/search_api.ex: -------------------------------------------------------------------------------- 1 | defmodule SearchApi do 2 | @base_url "https://ssb.cc.nd.edu/StudentRegistrationSsb/ssb" 3 | @term_url "#{@base_url}/classSearch" 4 | @search_result_url "#{@base_url}/searchResults" 5 | # @cookie_agent = elem(CookieAgent.start_link(''), 1) 6 | 7 | def fetch_everything do 8 | cookie = authenticate_request 9 | fetch_terms 10 | |> Enum.map(fn(term) -> fetch_term_departments(term["code"], cookie) end) 11 | |> Enum.map(fn({k, v}) -> Enum.map(v, fn(item) -> 12 | IO.puts(item["code"]) 13 | fetch_term_department_courses(cookie, k, item["code"], 0) end) 14 | end) 15 | end 16 | 17 | def fetch_terms do 18 | # Let's only fetch for the five most recent terms, for simplicity. 19 | HTTPoison.get!("#{@term_url}/getTerms?searchTerm=&offset=1&max=1").body 20 | |> Poison.decode! 21 | 22 | # Returns it as a JSON object. 23 | end 24 | 25 | def parse_for_terms(term) do 26 | end 27 | 28 | def fetch_term_departments(term, cookie) do 29 | # Term Object! 30 | IO.puts "https://ssb.cc.nd.edu/StudentRegistrationSsb/ssb/classSearch/get_subject?searchTerm=&term=#{term}&offset=1&max=200" 31 | final_value = HTTPoison.get!("https://ssb.cc.nd.edu/StudentRegistrationSsb/ssb/classSearch/get_subject?searchTerm=&term=#{term}&offset=1&max=200").body 32 | |> Poison.decode! 33 | 34 | {term, final_value} 35 | end 36 | 37 | def authenticate_request(header_cookie \\ "") do 38 | map_header = %{'Set-Cookie' => header_cookie, 'Content-type' => 39 | 'mulitpart/form_data'} 40 | 41 | if(header_cookie == "") do 42 | headers = ['Content-Type': 'application/x-www-form-urlencoded'] 43 | end 44 | 45 | resp = HTTPoison.post!("#{@base_url}/term/search?mode=search&term=201510", 46 | "{\"term\": 201510}", ["Set-Cookie": header_cookie, "Content-Type": 47 | "application/x-www-form-urlencoded;charset=UTF-8"]) 48 | 49 | IO.puts(header_cookie) 50 | resp_body = Poison.decode! resp.body 51 | IO.puts(resp.body) 52 | 53 | resp_headers = Enum.at(resp.headers, 3) 54 | resp_body 55 | 56 | cookie = (Tuple.to_list(resp_headers)) |> List.last; 57 | 58 | actual_cookie = String.split(cookie, ";", trim: true) |> List.first; 59 | IO.puts(actual_cookie) 60 | # Check length of the dictionary keys 61 | #IO.puts length Dict.keys(resp) 62 | #IO.puts(resp.headers) 63 | #resp_headers 64 | 65 | if(header_cookie == "") do 66 | IO.puts("Blank header cookie!") 67 | #authenticate_request(actual_cookie) 68 | end 69 | 70 | 71 | case CookieAgent.start_link(actual_cookie) do 72 | {:ok, cookie_agent} -> IO.puts 'Successfully started Cookie Agent' 73 | {:error, reason} -> IO.puts 'Error starting the cookie agent.' 74 | CookieAgent.set(:cookie, actual_cookie) 75 | end 76 | 77 | 78 | actual_cookie 79 | end 80 | 81 | def fetch_term_department_courses(cookie, term, department_code, offset) do 82 | # Gonna have to recursively call this one, probably, in order to get every 83 | # single department that we need. 84 | cookie_header = CookieAgent.get(:cookie) 85 | 86 | IO.puts CookieAgent.get(:cookie) 87 | json_response = 88 | HTTPoison.get!("#{@search_result_url}/searchResults?txt_subject=#{department_code}&txt_term=#{term}&startDatepicker=&endDatepicker=&pageOffset=#{offset}&pageMaxSize=299&sortColumn=subjectDescription&sortDirection=asc", 89 | ['Cookie': cookie_header]).body 90 | |> Poison.decode! 91 | 92 | #Enum.map(json_response, fn {k,v} -> IO.puts "%{k}: #{v}" end) 93 | 94 | if(!json_response["success"]) do 95 | # The call failed for some reason, probably unauthenticated. 96 | IO.puts("The API call failed. Reauthenticate.") 97 | authenticate_request 98 | fetch_term_department_courses(cookie, term, department_code, offset) 99 | # Reauthenticate and retry the same request. 100 | end 101 | 102 | json_response 103 | end 104 | 105 | def fetch_departments do 106 | HTTPoison.get!("https://ssb.cc.nd.edu/StudentRegistrationSsb/ssb/classSearch/get_subject?searchTerm=&term=201510&offset=1&max=200").body 107 | |> Poison.decode! 108 | end 109 | end 110 | 111 | --------------------------------------------------------------------------------