├── lib ├── athel_web │ ├── templates │ │ ├── page │ │ │ └── index.html.eex │ │ ├── article │ │ │ └── show.html.eex │ │ ├── admin │ │ │ └── new_group.html.eex │ │ ├── group │ │ │ ├── show.html.eex │ │ │ └── index.html.eex │ │ └── layout │ │ │ └── app.html.eex │ ├── views │ │ ├── layout_view.ex │ │ ├── page_view.ex │ │ ├── admin_view.ex │ │ ├── article_view.ex │ │ ├── group_view.ex │ │ ├── error_view.ex │ │ ├── view_common.ex │ │ └── error_helpers.ex │ ├── controllers │ │ ├── page_controller.ex │ │ ├── article_controller.ex │ │ ├── admin_controller.ex │ │ └── group_controller.ex │ ├── gettext.ex │ ├── router.ex │ ├── channels │ │ └── user_socket.ex │ └── endpoint.ex ├── athel │ ├── vault.ex │ ├── models │ │ ├── encrypted_binary_field.ex │ │ ├── role.ex │ │ ├── article_search_index.ex │ │ ├── foreigner.ex │ │ ├── group.ex │ │ ├── user.ex │ │ ├── attachment.ex │ │ └── article.ex │ ├── repo.ex │ ├── event │ │ ├── supervisor.ex │ │ ├── nntp_broadcaster.ex │ │ └── moderation_handler.ex │ ├── scraper_supervisor.ex │ ├── application.ex │ ├── nntp │ │ ├── supervisor.ex │ │ ├── defs.ex │ │ ├── formattable.ex │ │ ├── client.ex │ │ ├── protocol.ex │ │ └── parser.ex │ ├── user_cache.ex │ ├── services │ │ ├── auth_service.ex │ │ ├── multipart.ex │ │ └── nntp_service.ex │ └── scraper.ex ├── athel.ex └── athel_web.ex ├── test ├── nntp │ ├── scandavian_org.txt │ ├── formattable_test.exs │ └── parser_test.exs ├── services │ ├── danish_body.txt │ ├── auth_service_test.exs │ ├── nntp_service_test.exs │ └── multipart_test.exs ├── views │ ├── page_view_test.exs │ ├── layout_view_test.exs │ └── error_view_test.exs ├── test_helper.exs ├── controllers │ ├── page_controller_test.exs │ └── group_controller_test.exs ├── user_cache_test.exs ├── models │ ├── user_test.exs │ ├── group_test.exs │ ├── attachment_test.exs │ └── article_test.exs └── support │ ├── channel_case.ex │ ├── conn_case.ex │ ├── test_data.ex │ └── model_case.ex ├── README.md ├── todo.txt ├── priv ├── repo │ ├── migrations │ │ ├── 20180905024527_create_foreigners.exs │ │ ├── 20190329023750_create_role.exs │ │ ├── 20160819035916_create_attachment.exs │ │ ├── 20160810163507_create_user.exs │ │ ├── 20160718151737_create_group.exs │ │ ├── 20190226213046_article_full_text_search.exs │ │ └── 20160718181754_create_article.exs │ └── seeds.exs ├── keys │ ├── testing.cert │ ├── example.com.crt │ ├── example.com.key │ └── testing.key └── gettext │ ├── en │ └── LC_MESSAGES │ │ └── errors.po │ └── errors.pot ├── assets ├── package.json ├── brunch-config.js └── styles │ ├── app.less │ └── _normalize.less ├── LICENSE ├── .gitignore ├── config ├── prod.exs ├── test.exs ├── config.exs └── dev.exs ├── mix.exs ├── INSTALL.md ├── .credo.exs └── mix.lock /lib/athel_web/templates/page/index.html.eex: -------------------------------------------------------------------------------- 1 |

2 | Wording and words 3 |

4 | -------------------------------------------------------------------------------- /lib/athel/vault.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Vault do 2 | use Cloak.Vault, otp_app: :athel 3 | end -------------------------------------------------------------------------------- /test/nntp/scandavian_org.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruhlio/athel/HEAD/test/nntp/scandavian_org.txt -------------------------------------------------------------------------------- /test/services/danish_body.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruhlio/athel/HEAD/test/services/danish_body.txt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Athel 2 | 3 | An NNTP backed messageboard 4 | 5 | # Running 6 | 7 | See [Running](./running.md) 8 | -------------------------------------------------------------------------------- /test/views/page_view_test.exs: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.PageViewTest do 2 | use AthelWeb.ConnCase, async: true 3 | end 4 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start(timeout: 7_500) 2 | 3 | Ecto.Adapters.SQL.Sandbox.mode(Athel.Repo, :manual) 4 | 5 | -------------------------------------------------------------------------------- /test/views/layout_view_test.exs: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.LayoutViewTest do 2 | use AthelWeb.ConnCase, async: true 3 | end 4 | -------------------------------------------------------------------------------- /test/controllers/page_controller_test.exs: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.PageControllerTest do 2 | use AthelWeb.ConnCase, async: true 3 | end 4 | -------------------------------------------------------------------------------- /lib/athel/models/encrypted_binary_field.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.EncryptedBinaryField do 2 | use Cloak.Fields.Binary, vault: Athel.Vault 3 | end -------------------------------------------------------------------------------- /lib/athel/repo.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Repo do 2 | use Ecto.Repo, 3 | otp_app: :athel, 4 | adapter: Ecto.Adapters.Postgres 5 | end 6 | -------------------------------------------------------------------------------- /lib/athel_web/views/layout_view.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.LayoutView do 2 | use AthelWeb, :view 3 | 4 | def title() do 5 | "Athel" 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /lib/athel_web/views/page_view.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.PageView do 2 | use AthelWeb, :view 3 | 4 | def title(_, _) do 5 | "Home" 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /lib/athel_web/views/admin_view.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.AdminView do 2 | use AthelWeb, :view 3 | 4 | def title(_, _) do 5 | "Admin" 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /todo.txt: -------------------------------------------------------------------------------- 1 | ban/delete by post/poster 2 | increment group watermarks 3 | configure logging to file 4 | implement moderated posting status 5 | add remote info to server/client logging 6 | -------------------------------------------------------------------------------- /lib/athel_web/views/article_view.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.ArticleView do 2 | use AthelWeb, :view 3 | 4 | def title(_, assigns) do 5 | List.first(assigns.articles).subject 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /lib/athel_web/controllers/page_controller.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.PageController do 2 | use AthelWeb, :controller 3 | 4 | def index(conn, _params) do 5 | render conn, "index.html" 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /lib/athel_web/views/group_view.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.GroupView do 2 | use AthelWeb, :view 3 | 4 | def title("show.html", assigns) do 5 | assigns.group.name 6 | end 7 | def title(_t, _) do 8 | "Groups" 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /lib/athel.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel do 2 | def model do 3 | quote do 4 | use Ecto.Schema 5 | use Timex.Ecto.Timestamps 6 | 7 | import Ecto 8 | import Ecto.Changeset 9 | import Ecto.Query 10 | end 11 | end 12 | 13 | end 14 | -------------------------------------------------------------------------------- /lib/athel/models/role.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Role do 2 | use Ecto.Schema 3 | import Ecto.Changeset 4 | 5 | alias Athel.User 6 | 7 | @type t :: %__MODULE__{} 8 | 9 | @primary_key false 10 | schema "roles" do 11 | field :name, :string 12 | field :group_name, :string 13 | 14 | belongs_to :user, User, 15 | foreign_key: :user_email, 16 | references: :email, 17 | type: :string 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/athel_web/templates/article/show.html.eex: -------------------------------------------------------------------------------- 1 |
2 |

<%= List.first(@articles).subject %>

3 | <%= for article <- @articles do %> 4 |
5 |
6 | <%= format_date(article.date) %> | <%= article.from %> 7 |
8 |
9 | <%= format_article_body(article.body) %> 10 |
11 |
12 | <% end %> 13 |
14 | -------------------------------------------------------------------------------- /priv/repo/migrations/20180905024527_create_foreigners.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.Repo.Migrations.CreateForeigners do 2 | use Ecto.Migration 3 | 4 | def change do 5 | create table(:foreigners) do 6 | add :hostname, :string, null: false 7 | add :port, :integer, null: false 8 | add :username, :string 9 | add :password, :binary 10 | add :interval, :integer, null: false 11 | 12 | timestamps() 13 | end 14 | 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /lib/athel_web/views/error_view.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.ErrorView do 2 | use AthelWeb, :view 3 | 4 | def render("404.html", _assigns) do 5 | "Page not found" 6 | end 7 | 8 | def render("500.html", _assigns) do 9 | "Internal server error" 10 | end 11 | 12 | # In case no render clause matches or no 13 | # template is found, let's render it as 500 14 | def template_not_found(_template, assigns) do 15 | render "500.html", assigns 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/athel/event/supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Event.Supervisor do 2 | use Supervisor 3 | 4 | import Supervisor.Spec 5 | 6 | def start_link() do 7 | Supervisor.start_link(__MODULE__, :ok, name: __MODULE__) 8 | end 9 | 10 | def init(:ok) do 11 | children = [ 12 | worker(Athel.Event.NntpBroadcaster, []), 13 | worker(Athel.Event.ModerationHandler, []) 14 | ] 15 | 16 | Supervisor.init(children, strategy: :one_for_one) 17 | end 18 | 19 | end 20 | -------------------------------------------------------------------------------- /assets/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "repository": {}, 3 | "license": "ISC", 4 | "scripts": { 5 | "deploy": "brunch build --production", 6 | "watch": "brunch watch --stdin" 7 | }, 8 | "dependencies": { 9 | "phoenix": "file:../deps/phoenix", 10 | "phoenix_html": "file:../deps/phoenix_html" 11 | }, 12 | "devDependencies": { 13 | "autoprefixer": "^7.1.2", 14 | "brunch": "^2.10.10", 15 | "less-brunch": "^2.10.0", 16 | "postcss-brunch": "^2.0.5" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /priv/repo/migrations/20190329023750_create_role.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.Repo.Migrations.CreateRole do 2 | use Ecto.Migration 3 | 4 | def change do 5 | create table(:roles, primary_key: false) do 6 | add :name, :string, primary_key: true 7 | add :user_email, references(:users, column: :email, type: :string), primary_key: true 8 | add :group_name, references(:groups, column: :name, type: :string), primary_key: true 9 | 10 | timestamps() 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/athel/scraper_supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.ScraperSupervisor do 2 | use Supervisor 3 | 4 | import Supervisor.Spec 5 | 6 | def start_link() do 7 | Supervisor.start_link(__MODULE__, :ok, name: __MODULE__) 8 | end 9 | 10 | def init(:ok) do 11 | foreigners = Athel.Repo.all(Athel.Foreigner) 12 | children = Enum.map foreigners, fn foreigner -> 13 | worker(Athel.Scraper, [foreigner], id: foreigner.hostname) 14 | end 15 | 16 | Supervisor.init(children, strategy: :one_for_one) 17 | end 18 | 19 | 20 | end 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018 Jeff Ruhl 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /priv/repo/migrations/20160819035916_create_attachment.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.Repo.Migrations.CreateAttachment do 2 | use Ecto.Migration 3 | 4 | def change do 5 | create table(:attachments) do 6 | add :filename, :string 7 | add :type, :string 8 | add :hash, :binary 9 | add :content, :binary 10 | 11 | timestamps() 12 | end 13 | 14 | create table(:attachments_to_articles, primary_key: false) do 15 | add :attachment_id, references(:attachments) 16 | add :message_id, references(:articles, column: :message_id, type: :string) 17 | end 18 | 19 | end 20 | 21 | end 22 | -------------------------------------------------------------------------------- /priv/repo/seeds.exs: -------------------------------------------------------------------------------- 1 | # Script for populating the database. You can run it as: 2 | # 3 | # mix run priv/repo/seeds.exs 4 | # 5 | # Inside the script, you can read and write to any of your 6 | # repositories directly: 7 | # 8 | # Athel.Repo.insert!(%Athel.SomeModel{}) 9 | # 10 | # We recommend using the bang functions (`insert!`, `update!` 11 | # and so on) as they will fail if something goes wrong. 12 | 13 | Athel.Repo.insert!(%Athel.Foreigner{ 14 | hostname: 'localhost', 15 | port: 9119, 16 | interval: 5, 17 | }) 18 | 19 | Athel.Repo.insert!(%Athel.Group{ 20 | name: 'test', 21 | description: 'test', 22 | }) 23 | -------------------------------------------------------------------------------- /test/user_cache_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.UserCacheTest do 2 | use ExUnit.Case 3 | alias Athel.{UserCache, User, TestData} 4 | 5 | setup do 6 | :ok = Ecto.Adapters.SQL.Sandbox.checkout(Athel.Repo) 7 | Ecto.Adapters.SQL.Sandbox.mode(Athel.Repo, {:shared, self()}) 8 | 9 | :ok 10 | end 11 | 12 | test "put" do 13 | user = %User{email: "santa@north.pole", status: "active", public_key: TestData.public_key} 14 | UserCache.put(user) 15 | stored_user = UserCache.get("santa@north.pole") 16 | assert stored_user.email == "santa@north.pole" 17 | {key_type, _, _} = stored_user.decoded_public_key 18 | assert key_type == :RSAPublicKey 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /test/views/error_view_test.exs: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.ErrorViewTest do 2 | use AthelWeb.ConnCase, async: true 3 | 4 | # Bring render/3 and render_to_string/3 for testing custom views 5 | import Phoenix.View 6 | 7 | test "renders 404.html" do 8 | assert render_to_string(AthelWeb.ErrorView, "404.html", []) == 9 | "Page not found" 10 | end 11 | 12 | test "render 500.html" do 13 | assert render_to_string(AthelWeb.ErrorView, "500.html", []) == 14 | "Internal server error" 15 | end 16 | 17 | test "render any other" do 18 | assert render_to_string(AthelWeb.ErrorView, "505.html", []) == 19 | "Internal server error" 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /priv/repo/migrations/20160810163507_create_user.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.Repo.Migrations.CreateUser do 2 | use Ecto.Migration 3 | 4 | def up do 5 | execute "CREATE TYPE user_status AS ENUM ('active', 'pending', 'locked')" 6 | 7 | create table(:users, primary_key: false) do 8 | add :email, :string, size: 255, primary_key: true 9 | add :hashed_password, :binary, null: true 10 | add :salt, :binary, null: true 11 | add :public_key, :text, null: true 12 | add :status, :user_status, null: false 13 | 14 | timestamps() 15 | end 16 | end 17 | 18 | def down do 19 | drop table(:users) 20 | execute "DROP TYPE user_status" 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /priv/repo/migrations/20160718151737_create_group.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.Repo.Migrations.CreateGroup do 2 | use Ecto.Migration 3 | 4 | def up do 5 | execute "CREATE TYPE group_status AS ENUM ('y', 'n', 'm')" 6 | 7 | create table(:groups, primary_key: false) do 8 | add :name, :string, size: 128, primary_key: true 9 | add :description, :string, size: 256, null: false 10 | add :low_watermark, :integer, null: false 11 | add :high_watermark, :integer, null: false 12 | add :status, :group_status, null: false 13 | 14 | timestamps() 15 | end 16 | end 17 | 18 | def down do 19 | drop table(:groups) 20 | execute "DROP TYPE group_status" 21 | end 22 | 23 | end 24 | -------------------------------------------------------------------------------- /lib/athel/models/article_search_index.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.ArticleSearchIndex do 2 | use Ecto.Schema 3 | 4 | @type t :: %__MODULE__{} 5 | 6 | @primary_key {:message_id, :string, autogenerate: false} 7 | schema "article_search_index" do 8 | field :from, :string 9 | field :subject, :string 10 | field :date, :utc_datetime 11 | field :status, :string 12 | field :document, {:array, :map} 13 | 14 | many_to_many :groups, Athel.Group, 15 | join_through: "articles_to_groups", 16 | join_keys: [message_id: :message_id, group_id: :id] 17 | end 18 | 19 | def update_view() do 20 | Ecto.Adapters.SQL.query!(Athel.Repo, "REFRESH MATERIALIZED VIEW article_search_index") 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/athel/event/nntp_broadcaster.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Event.NntpBroadcaster do 2 | use GenStage 3 | 4 | # API 5 | 6 | def start_link() do 7 | GenStage.start_link(__MODULE__, :ok, name: __MODULE__) 8 | end 9 | 10 | def new_article(article) do 11 | GenStage.cast(__MODULE__, {:new_article, article}) 12 | end 13 | 14 | # Callbacks 15 | 16 | @impl true 17 | def init(:ok) do 18 | {:producer, :ok, dispatcher: GenStage.BroadcastDispatcher} 19 | end 20 | 21 | @impl true 22 | def handle_cast({:new_article, article}, state) do 23 | {:noreply, [article], state} 24 | end 25 | 26 | @impl true 27 | def handle_demand(_demand, state) do 28 | {:noreply, [], state} 29 | end 30 | 31 | end 32 | -------------------------------------------------------------------------------- /lib/athel_web/controllers/article_controller.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.ArticleController do 2 | use AthelWeb, :controller 3 | use Timex 4 | 5 | alias Athel.Article 6 | 7 | def show(conn, %{"message_id" => id}) do 8 | articles = Repo.all( 9 | from a in Article, 10 | join: t in fragment( 11 | """ 12 | (WITH RECURSIVE thread AS ( 13 | SELECT * FROM articles WHERE message_id = ? 14 | UNION ALL 15 | SELECT a.* FROM articles a 16 | JOIN thread t on a.parent_message_id = t.message_id) 17 | SELECT * FROM thread) 18 | """, ^id), on: a.message_id == t.message_id) 19 | 20 | render(conn, "show.html", articles: articles) 21 | end 22 | 23 | end 24 | -------------------------------------------------------------------------------- /lib/athel/event/moderation_handler.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Event.ModerationHandler do 2 | use GenStage 3 | 4 | alias Athel.{Article, Group} 5 | alias Athel.Event.NntpBroadcaster 6 | 7 | def start_link() do 8 | GenStage.start_link(__MODULE__, :ok) 9 | end 10 | 11 | @impl true 12 | def init(:ok) do 13 | {:consumer, :ok, subscribe_to: [{NntpBroadcaster, selector: &article_selector/1}]} 14 | end 15 | 16 | defp article_selector(%Article{groups: groups}) do 17 | case groups do 18 | [%Group{name: "ctl"}] -> true 19 | _ -> false 20 | end 21 | end 22 | 23 | @impl true 24 | def handle_events(events, _from, state) do 25 | for event <- events do 26 | IO.inspect(event) 27 | end 28 | 29 | {:noreply, [], state} 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/athel/models/foreigner.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Foreigner do 2 | use Ecto.Schema 3 | import Ecto.Changeset 4 | 5 | @type t :: %__MODULE__{} 6 | 7 | @derive {Inspect, except: [:password]} 8 | schema "foreigners" do 9 | field :hostname, :string 10 | field :port, :integer 11 | field :username, :string 12 | field :password, Athel.EncryptedBinaryField 13 | field :interval, :integer 14 | 15 | timestamps() 16 | end 17 | 18 | @doc false 19 | def changeset(foreigner, attrs) do 20 | foreigner 21 | |> cast(attrs, [:hostname, :port, :username, :password, :interval]) 22 | |> validate_required([:hostname, :port, :interval]) 23 | end 24 | end 25 | 26 | defimpl String.Chars, for: Athel.Foreigner do 27 | def to_string(f) do 28 | "#{f.hostname}:#{f.port}" 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/athel_web/gettext.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.Gettext do 2 | @moduledoc """ 3 | A module providing Internationalization with a gettext-based API. 4 | 5 | By using [Gettext](https://hexdocs.pm/gettext), 6 | your module gains a set of macros for translations, for example: 7 | 8 | import Athel.Gettext 9 | 10 | # Simple translation 11 | gettext "Here is the string to translate" 12 | 13 | # Plural translation 14 | ngettext "Here is the string to translate", 15 | "Here are the strings to translate", 16 | 3 17 | 18 | # Domain-based translation 19 | dgettext "errors", "Here is the error message to translate" 20 | 21 | See the [Gettext Docs](https://hexdocs.pm/gettext) for detailed usage. 22 | """ 23 | use Gettext, otp_app: :athel 24 | end 25 | -------------------------------------------------------------------------------- /lib/athel/application.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Application do 2 | use Application 3 | 4 | def start(_type, _args) do 5 | import Supervisor.Spec 6 | 7 | children = [ 8 | supervisor(Athel.Repo, []), 9 | supervisor(AthelWeb.Endpoint, []), 10 | supervisor(Athel.Nntp.Supervisor, []), 11 | supervisor(Athel.Event.Supervisor, []), 12 | supervisor(Athel.ScraperSupervisor, []), 13 | worker(Athel.UserCache, []) 14 | ] 15 | 16 | opts = [strategy: :one_for_one, name: Athel.Supervisor] 17 | Supervisor.start_link(children, opts) 18 | end 19 | 20 | # Tell Phoenix to update the endpoint configuration 21 | # whenever the application is updated. 22 | def config_change(changed, _new, removed) do 23 | AthelWeb.Endpoint.config_change(changed, removed) 24 | :ok 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # App artifacts 2 | /_build 3 | /db 4 | /deps 5 | /*.ez 6 | .elixir_ls 7 | 8 | # Generated on crash by the VM 9 | erl_crash.dump 10 | 11 | # Static artifacts 12 | /assets/node_modules 13 | 14 | # Since we are building assets from web/static, 15 | # we ignore priv/static. You may want to comment 16 | # this depending on your deployment strategy. 17 | /priv/static/ 18 | 19 | # The config/prod.secret.exs file by default contains sensitive 20 | # data and you should not commit it into version control. 21 | # 22 | # Alternatively, you may comment the line below and commit the 23 | # secrets file as long as you replace its contents by environment 24 | # variables. 25 | /config/prod.secret.exs 26 | 27 | # emacs temp files 28 | .\#* 29 | 30 | # distillery 31 | rel/* 32 | !rel/config.exs 33 | 34 | # Intellij 35 | .idea/ 36 | *.iml 37 | -------------------------------------------------------------------------------- /lib/athel/nntp/supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Nntp.Supervisor do 2 | use Supervisor 3 | 4 | @type opts :: [port: non_neg_integer, 5 | pool_size: non_neg_integer, 6 | timeout: non_neg_integer, 7 | keyfile: String.t, 8 | certfile: String.t 9 | ] 10 | 11 | def start_link do 12 | Supervisor.start_link(__MODULE__, :ok, name: __MODULE__) 13 | end 14 | 15 | def init(:ok) do 16 | config = Application.fetch_env!(:athel, Athel.Nntp) 17 | 18 | children = [ 19 | # :ranch_sup already started by phoenix/cowboy 20 | :ranch.child_spec( 21 | :nntp, 22 | config[:pool_size], 23 | :ranch_tcp, [port: config[:port]], 24 | Athel.Nntp.Protocol, config 25 | ) 26 | ] 27 | 28 | supervise(children, strategy: :one_for_one) 29 | end 30 | 31 | end 32 | -------------------------------------------------------------------------------- /lib/athel_web/router.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.Router do 2 | use AthelWeb, :router 3 | 4 | pipeline :browser do 5 | plug :accepts, ["html"] 6 | plug :fetch_session 7 | plug :fetch_flash 8 | plug :protect_from_forgery 9 | plug :put_secure_browser_headers 10 | end 11 | 12 | pipeline :api do 13 | plug :accepts, ["json"] 14 | end 15 | 16 | scope "/", AthelWeb do 17 | pipe_through :browser 18 | 19 | get "/", PageController, :index 20 | 21 | scope "/groups" do 22 | get "/", GroupController, :index 23 | get "/:name", GroupController, :show 24 | post "/:name", GroupController, :create_topic 25 | 26 | get "/:group_name/articles/:message_id", ArticleController, :show 27 | end 28 | 29 | scope "/admin" do 30 | get "/new_group", AdminController, :new_group 31 | post "/new_group", AdminController, :create_group 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /test/models/user_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.UserTest do 2 | use Athel.ModelCase 3 | 4 | alias Athel.User 5 | 6 | test "valid" do 7 | {:ok, hash} = Multihash.encode(:sha2_512, :crypto.hash(:sha512, "knarly")) 8 | attrs = %{email: "me@him.who", 9 | hashed_password: hash, 10 | salt: "WHENWILLIEVER", 11 | status: "active"} 12 | assert User.changeset(%User{}, attrs).valid? 13 | end 14 | 15 | test "email" do 16 | assert_invalid_format(%User{}, :email, 17 | ["asd@sdf.p", 18 | "I'M GONNA@do.nothing", 19 | "cannot.feel@m@.legs"]) 20 | 21 | superlong = fn -> "HA" end 22 | |> Stream.repeatedly() 23 | |> Enum.take(200) 24 | |> Enum.join 25 | assert_too_long(%User{}, :email, "#{superlong}@bob.com") 26 | end 27 | 28 | test "hashed password" do 29 | assert_invalid(%User{}, :hashed_password, 30 | [<<0xf>>, <<0x63, 0x22, 0x44>>], "invalid multihash") 31 | end 32 | 33 | end 34 | -------------------------------------------------------------------------------- /assets/brunch-config.js: -------------------------------------------------------------------------------- 1 | exports.config = { 2 | files: { 3 | stylesheets: { joinTo: "css/app.css" } 4 | }, 5 | 6 | conventions: { 7 | // This option sets where we should place non-css and non-js assets in. 8 | // By default, we set this to "assets/static". Files in this directory 9 | // will be copied to `paths.public`, which is "priv/static" by default. 10 | assets: /^(static)/ 11 | }, 12 | 13 | // Phoenix paths configuration 14 | paths: { 15 | // Dependencies and current project directories to watch 16 | watched: ["static", "styles"], 17 | 18 | // Where to compile files to 19 | public: "../priv/static" 20 | }, 21 | 22 | // Configure your plugins 23 | plugins: { 24 | postcss: { 25 | processors: [ 26 | require('autoprefixer')(['last 4 versions']) 27 | ] 28 | }, 29 | less: {} 30 | }, 31 | 32 | modules: { 33 | }, 34 | 35 | npm: { 36 | enabled: true 37 | } 38 | }; 39 | -------------------------------------------------------------------------------- /lib/athel_web/templates/admin/new_group.html.eex: -------------------------------------------------------------------------------- 1 | <%= form_for @changeset, admin_path(@conn, :create_group), [class: "pure-form pure-form-aligned"], fn form -> %> 2 |
3 | Create a group 4 | 5 |
6 | <%= label(form, :name, "Name") %> 7 | <%= text_input(form, :name, class: error_class(@changeset, :name)) %> 8 |
9 | 10 |
11 | <%= label(form, :description, "Description") %> 12 | <%= textarea(form, :description, class: error_class(@changeset, :description)) %> 13 |
14 | 15 |
16 | <%= label(form, :status, "Status") %> 17 | <%= select(form, :status, ["Posting allowed": "y", "Posting disallowed": "n", "Posting moderated": "m"], class: error_class(@changeset, :status)) %> 18 |
19 | 20 |
21 | 22 |
23 |
24 | <% end %> 25 | -------------------------------------------------------------------------------- /priv/repo/migrations/20190226213046_article_full_text_search.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.Repo.Migrations.ArticleFullTextSearch do 2 | use Ecto.Migration 3 | 4 | def up do 5 | alter table(:articles) do 6 | add :language, :string, null: false, default: "english" 7 | end 8 | 9 | execute """ 10 | CREATE MATERIALIZED VIEW article_search_index AS 11 | SELECT message_id, 12 | parent_message_id, 13 | "from", 14 | subject, 15 | date, 16 | status, 17 | language, 18 | setweight(to_tsvector(language::regconfig, subject), 'A') || setweight(to_tsvector(language::regconfig, body), 'B') AS document 19 | FROM articles 20 | """ 21 | execute "CREATE INDEX index_search_articles ON article_search_index USING GIN(document)" 22 | end 23 | 24 | def down do 25 | execute "DROP INDEX index_search_articles" 26 | execute "DROP MATERIALIZED VIEW article_search_index" 27 | 28 | alter table(:articles) do 29 | remove :language 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /config/prod.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | # For production, we configure the host to read the PORT 4 | # from the system environment. Therefore, you will need 5 | # to set PORT=80 before running your server. 6 | # 7 | # You should also configure the url host to something 8 | # meaningful, we use this information when generating URLs. 9 | # 10 | # Finally, we also include the path to a manifest 11 | # containing the digested version of static files. This 12 | # manifest is generated by the mix phoenix.digest task 13 | # which you typically run after static files are built. 14 | 15 | config :athel, AthelWeb.Endpoint, 16 | http: [port: {:system, "PORT"}], 17 | url: [host: "localhost", port: "POST"], 18 | cache_static_manifest: "priv/static/manifest.json", 19 | server: true, 20 | root: ".", 21 | version: Mix.Project.config[:version] 22 | 23 | # Do not print debug messages in production 24 | config :logger, level: :info 25 | 26 | # Finally import the config/prod.secret.exs 27 | # which should be versioned separately. 28 | import_config "prod.secret.exs" 29 | -------------------------------------------------------------------------------- /lib/athel_web/templates/group/show.html.eex: -------------------------------------------------------------------------------- 1 |

<%= @group.name %>

2 | 3 | <%= if Enum.empty? @group.articles do %> 4 |

No articles

5 | <% end %> 6 | 7 | 17 | 18 | <%= if Enum.count(@pages) > 1 do %> 19 | 34 | <% end %> 35 | -------------------------------------------------------------------------------- /config/test.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | # We don't run a server during test. If one is required, 4 | # you can enable the server option below. 5 | config :athel, AthelWeb.Endpoint, 6 | http: [port: 4001], 7 | server: false 8 | 9 | # Print only warnings and errors during test 10 | config :logger, level: :warn 11 | 12 | # Configure your database 13 | config :athel, Athel.Repo, 14 | username: "athel", 15 | password: "athel", 16 | database: "athel_test", 17 | hostname: "localhost", 18 | pool: Ecto.Adapters.SQL.Sandbox 19 | 20 | config :athel, Athel.Nntp, 21 | port: 8119, 22 | hostname: "example.com", 23 | pool_size: 10, 24 | timeout: 5_000, 25 | keyfile: Path.expand("../priv/keys/testing.key", __DIR__), 26 | certfile: Path.expand("../priv/keys/testing.cert", __DIR__), 27 | cacertfile: Path.expand("../priv/keys/example.com.crt", __DIR__), 28 | max_request_size: 250_000, 29 | max_attachment_size: 100, 30 | max_attachment_count: 3 31 | 32 | config :athel, Athel.Vault, 33 | ciphers: [ 34 | default: {Cloak.Ciphers.AES.GCM, tag: "AES.GCM.V1", key: Base.decode64!("7/TfD+toi48MB2bpPZRnsfc8pgvpY1QEQWvfYyfGsVw=")} 35 | ] 36 | -------------------------------------------------------------------------------- /lib/athel_web/controllers/admin_controller.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.AdminController do 2 | use AthelWeb, :controller 3 | 4 | alias Athel.Group 5 | 6 | def new_group(conn, _params) do 7 | render(conn, "new_group.html", changeset: Group.changeset(%Group{})) 8 | end 9 | 10 | def create_group(conn, %{"group" => 11 | %{ 12 | "name" => name, 13 | "description" => description, 14 | "status" => status} 15 | }) do 16 | changeset = Group.changeset(%Group{}, 17 | %{ 18 | name: name, 19 | description: description, 20 | status: status, 21 | low_watermark: 0, 22 | high_watermark: 0 23 | }) 24 | 25 | if changeset.valid? do 26 | Repo.insert!(changeset) 27 | conn 28 | |> put_flash(:success, "Group created") 29 | |> redirect(to: "/") 30 | else 31 | conn 32 | |> put_flash(:error, "Please correct the errors and resubmit") 33 | |> render("new_group.html", changeset: changeset) 34 | end 35 | end 36 | 37 | end 38 | -------------------------------------------------------------------------------- /test/support/channel_case.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.ChannelCase do 2 | @moduledoc """ 3 | This module defines the test case to be used by 4 | channel tests. 5 | 6 | Such tests rely on `Phoenix.ChannelTest` and also 7 | import other functionality to make it easier 8 | to build and query models. 9 | 10 | Finally, if the test case interacts with the database, 11 | it cannot be async. For this reason, every test runs 12 | inside a transaction which is reset at the beginning 13 | of the test unless the test case is marked as async. 14 | """ 15 | 16 | use ExUnit.CaseTemplate 17 | 18 | using do 19 | quote do 20 | # Import conveniences for testing with channels 21 | use Phoenix.ChannelTest 22 | 23 | alias Athel.Repo 24 | import Ecto 25 | import Ecto.Changeset 26 | import Ecto.Query 27 | 28 | 29 | # The default endpoint for testing 30 | @endpoint AthelWeb.Endpoint 31 | end 32 | end 33 | 34 | setup tags do 35 | :ok = Ecto.Adapters.SQL.Sandbox.checkout(Athel.Repo) 36 | 37 | unless tags[:async] do 38 | Ecto.Adapters.SQL.Sandbox.mode(Athel.Repo, {:shared, self()}) 39 | end 40 | 41 | :ok 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Mix.Config module. 3 | # 4 | # This configuration file is loaded before any dependency and 5 | # is restricted to this project. 6 | use Mix.Config 7 | 8 | config :phoenix, :json_library, Jason 9 | 10 | # General application configuration 11 | config :athel, 12 | ecto_repos: [Athel.Repo] 13 | 14 | # Configures the endpoint 15 | config :athel, AthelWeb.Endpoint, 16 | url: [host: "localhost"], 17 | secret_key_base: "Kq4pSxruZ4nUA7dTY8Ydsl9u8zvzqqnDCi18oMMWQfArkNW4jWfUFX0Mu6UmW4DX", 18 | render_errors: [view: AthelWeb.ErrorView, accepts: ~w(html json)], 19 | pubsub: [name: Athel.PubSub, 20 | adapter: Phoenix.PubSub.PG2] 21 | 22 | # Configures Elixir's Logger 23 | config :logger, :console, 24 | format: "$time $metadata[$level] $message\n", 25 | metadata: [:request_id, :module] 26 | 27 | config :codepagex, :encodings, [ 28 | :ascii, 29 | ~r[iso8859]i, 30 | ~r[VENDORS/MICSFT/WINDOWS/CP12] 31 | ] 32 | 33 | # Import environment specific config. This must remain at the bottom 34 | # of this file so it overrides the configuration defined above. 35 | import_config "#{Mix.env}.exs" 36 | -------------------------------------------------------------------------------- /lib/athel_web/views/view_common.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.ViewCommon do 2 | use Timex 3 | import Phoenix.HTML.Tag 4 | 5 | def format_date(date) do 6 | Timex.format!(date, "%a, %d %b %Y %T", :strftime) 7 | end 8 | 9 | def error_class(changeset, input) do 10 | if changeset.errors[input] do 11 | "input-error" 12 | else 13 | "" 14 | end 15 | end 16 | 17 | def format_article_body(body) do 18 | paragraphs = body 19 | |> String.split("\n") 20 | |> Enum.chunk_by(&(&1 == "")) 21 | |> Enum.filter(&(&1 != [""])) 22 | 23 | Enum.map(paragraphs, fn paragraph -> 24 | lines = Enum.map(paragraph, &([process_line(&1), tag(:br)])) 25 | content_tag(:p, lines) 26 | end) 27 | end 28 | 29 | @quote_class_count 3 30 | 31 | defp process_line(line) do 32 | case count_quotes(line) do 33 | 0 -> line 34 | level -> 35 | class_level = rem(level, @quote_class_count) 36 | content_tag(:span, line, class: "quote-#{class_level}") 37 | end 38 | end 39 | 40 | defp count_quotes(line), do: count_quotes(line, 0) 41 | defp count_quotes(<<">", rest :: binary>>, count), do: count_quotes(rest, count + 1) 42 | defp count_quotes(_, count), do: count 43 | end 44 | -------------------------------------------------------------------------------- /lib/athel_web/channels/user_socket.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.UserSocket do 2 | use Phoenix.Socket 3 | 4 | ## Channels 5 | # channel "room:*", PhoenixDistilleryWeb.RoomChannel 6 | 7 | # Socket params are passed from the client and can 8 | # be used to verify and authenticate a user. After 9 | # verification, you can put default assigns into 10 | # the socket that will be set for all channels, ie 11 | # 12 | # {:ok, assign(socket, :user_id, verified_user_id)} 13 | # 14 | # To deny connection, return `:error`. 15 | # 16 | # See `Phoenix.Token` documentation for examples in 17 | # performing token verification on connect. 18 | def connect(_params, socket, _connect_info) do 19 | {:ok, socket} 20 | end 21 | 22 | # Socket id's are topics that allow you to identify all sockets for a given user: 23 | # 24 | # def id(socket), do: "user_socket:#{socket.assigns.user_id}" 25 | # 26 | # Would allow you to broadcast a "disconnect" event and terminate 27 | # all active sockets and channels for a given user: 28 | # 29 | # PhoenixDistilleryWeb.Endpoint.broadcast("user_socket:#{user.id}", "disconnect", %{}) 30 | # 31 | # Returning `nil` makes this socket anonymous. 32 | def id(_socket), do: nil 33 | end 34 | -------------------------------------------------------------------------------- /lib/athel/models/group.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Group do 2 | use Ecto.Schema 3 | import Ecto.Changeset 4 | 5 | @type t :: %__MODULE__{} 6 | 7 | @primary_key {:name, :string, autogenerate: false} 8 | schema "groups" do 9 | field :description, :string 10 | field :low_watermark, :integer 11 | field :high_watermark, :integer 12 | field :status, :string 13 | 14 | many_to_many :articles, Athel.Article, 15 | join_through: "articles_to_groups", 16 | join_keys: [group_name: :name, message_id: :message_id] 17 | many_to_many :article_search_indexes, Athel.ArticleSearchIndex, 18 | join_through: "articles_to_groups", 19 | join_keys: [group_name: :name, message_id: :message_id] 20 | 21 | timestamps() 22 | end 23 | 24 | @doc """ 25 | Builds a changeset based on the `struct` and `params`. 26 | """ 27 | def changeset(struct, params \\ %{}) do 28 | struct 29 | |> cast(params, [:name, :description, :low_watermark, :high_watermark, :status]) 30 | |> validate_required([:name, :low_watermark, :high_watermark, :status]) 31 | |> validate_format(:name, ~r/^[a-zA-Z0-9_.-]{1,128}$/) 32 | |> unique_constraint(:name, name: "groups_pkey") 33 | |> validate_inclusion(:status, ["y", "n", "m"]) 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /priv/repo/migrations/20160718181754_create_article.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.Repo.Migrations.CreateArticle do 2 | use Ecto.Migration 3 | 4 | def up do 5 | execute "CREATE TYPE article_status AS ENUM ('active', 'banned')" 6 | 7 | create table(:articles, primary_key: false) do 8 | add :message_id, :string, primary_key: true, size: 192 9 | add :parent_message_id, :string, size: 192 10 | add :from, :string, null: true 11 | add :subject, :string, null: false 12 | add :date, :utc_datetime, null: false 13 | add :content_type, :string, null: false 14 | add :status, :article_status, null: false 15 | add :headers, :map, null: false 16 | add :body, :text, null: false 17 | 18 | timestamps() 19 | end 20 | 21 | create index(:articles, :parent_message_id) 22 | 23 | create table(:articles_to_groups, primary_key: false) do 24 | add :message_id, references(:articles, column: :message_id, type: :string) 25 | add :group_name, references(:groups, column: :name, type: :string) 26 | end 27 | 28 | end 29 | 30 | def down do 31 | drop table(:articles_to_groups) 32 | drop index(:articles, :parent_message_id) 33 | drop table(:articles) 34 | execute "DROP TYPE article_status" 35 | end 36 | 37 | end 38 | -------------------------------------------------------------------------------- /test/support/conn_case.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.ConnCase do 2 | @moduledoc """ 3 | This module defines the test case to be used by 4 | tests that require setting up a connection. 5 | 6 | Such tests rely on `Phoenix.ConnTest` and also 7 | import other functionality to make it easier 8 | to build and query models. 9 | 10 | Finally, if the test case interacts with the database, 11 | it cannot be async. For this reason, every test runs 12 | inside a transaction which is reset at the beginning 13 | of the test unless the test case is marked as async. 14 | """ 15 | 16 | use ExUnit.CaseTemplate 17 | 18 | using do 19 | quote do 20 | # Import conveniences for testing with connections 21 | use Phoenix.ConnTest 22 | 23 | alias Athel.Repo 24 | import Ecto 25 | import Ecto.Changeset 26 | import Ecto.Query 27 | 28 | import AthelWeb.Router.Helpers 29 | 30 | # The default endpoint for testing 31 | @endpoint AthelWeb.Endpoint 32 | end 33 | end 34 | 35 | setup tags do 36 | :ok = Ecto.Adapters.SQL.Sandbox.checkout(Athel.Repo) 37 | 38 | unless tags[:async] do 39 | Ecto.Adapters.SQL.Sandbox.mode(Athel.Repo, {:shared, self()}) 40 | end 41 | 42 | {:ok, conn: Phoenix.ConnTest.build_conn()} 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/athel_web/templates/group/index.html.eex: -------------------------------------------------------------------------------- 1 |

Groups

2 | 3 | <%= if Enum.empty? @groups do %> 4 |

No groups

5 | <% else %> 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | <%= for group <- @groups do %> 14 | 15 | 20 | 23 | 26 | 36 | 37 | <% end %> 38 |
NameLow WatermarkHigh WatermarkStatus
16 | 17 | <%= group.name %> 18 | 19 | 21 | <%= group.low_watermark %> 22 | 24 | <%= group.high_watermark %> 25 | 27 | <%= case group.status do %> 28 | <% "y" -> %> 29 | Posting permitted 30 | <% "n" -> %> 31 | Posting not permitted 32 | <% "m" -> %> 33 | Posting moderated 34 | <% end %> 35 |
39 | <% end %> 40 | -------------------------------------------------------------------------------- /test/models/group_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.GroupTest do 2 | use Athel.ModelCase 3 | 4 | alias Athel.Group 5 | 6 | @valid_attrs %{high_watermark: 42, low_watermark: 42, name: "fun.times", status: "m", description: "OOSE OOSE"} 7 | @invalid_attrs %{high_watermark: -1, low_watermark: -1, name: "yo la tengo", status: "BANANAPANIC"} 8 | 9 | test "changeset with valid attributes" do 10 | changeset = Group.changeset(%Group{}, @valid_attrs) 11 | assert changeset.valid? 12 | end 13 | 14 | test "changeset with invalid attributes" do 15 | changeset = Group.changeset(%Group{}, @invalid_attrs) 16 | refute changeset.valid? 17 | end 18 | 19 | test "name format" do 20 | changeset = Group.changeset(%Group{}, @valid_attrs) 21 | assert changeset.valid? 22 | 23 | changeset = Group.changeset(%Group{}, %{@valid_attrs | name: "MR. BRIGGS"}) 24 | assert changeset.errors[:name] == {"has invalid format", [validation: :format]} 25 | end 26 | 27 | test "name uniqueness" do 28 | changeset = Group.changeset(%Group{}, @valid_attrs) 29 | Repo.insert!(changeset) 30 | 31 | changeset = Group.changeset(%Group{}, @valid_attrs) 32 | {:error, changeset} = Repo.insert(changeset) 33 | {message, _} = changeset.errors[:name] 34 | assert message == "has already been taken" 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /priv/keys/testing.cert: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDjDCCAnSgAwIBAgIJAPConSVqTLskMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV 3 | BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX 4 | aWRnaXRzIFB0eSBMdGQxFDASBgNVBAMMC2V4YW1wbGUuY29tMB4XDTE3MDgyNTE2 5 | MDg0MVoXDTI4MTExMTE2MDg0MVowWzELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNv 6 | bWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEUMBIG 7 | A1UEAwwLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB 8 | AQCcvweLntg34Gqzt2CKuDU0uH8LSlvvOhga1TdLqm7lqdA/I4cRihFHPh5m4kP5 9 | zDu+BVJoFh+Gbw6GPYd19duEn5zM5rkHZg7UZWqp4i2pjntifwXdUQZIsFZ6KZZp 10 | 6S1HcHd1FeIpb8/VaIwZg807ldtHDTl0JrrDoJTzm3PTFuEUBGorD0Nj+Pwl1uKZ 11 | pTnOqOJek0mkMtXv0+PaX14BBkouSivXF0xcULV2YdNczC5EUFFZKJuKk0cInocM 12 | q00Vuvklpq1zGouj740hKmbwyKNoJi8sAYUtAnybsFchHpJFW4AKhlQzjTAYvnY+ 13 | Yja6i0Ui/xcJm3jRl9XfMYkzAgMBAAGjUzBRMB0GA1UdDgQWBBQZQTTHIY6P3wr1 14 | 6MIxEoNCJoLBFjAfBgNVHSMEGDAWgBQZQTTHIY6P3wr16MIxEoNCJoLBFjAPBgNV 15 | HRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAjfG8aNOq9gAlI8Ugli3a3 16 | y1rgEZITCUp5nVFRaXsLjY7h5M8X6h2nAcpEBEOZAHY0yakzrb+7oLB1OZuGxMKy 17 | 3SbkLcsNYjGLLCc9qLl59ldzmmQKz/FTEPrGhhLmbYq3tRZANNHomqvn4I2LJ56p 18 | TiOcJWC8DX5e7TjUWroN+ECd89wt6eYG7BUQ2wVAwic/gQj0VbBbGyaWE/xZ80kN 19 | s2ncWoNF9F1GZB5kZxv4hTHRDM3MWToGca7TRUc2idq4RZqSfl+mlKP6hd+ldPYO 20 | EoFj54/NF587ybaFLrk0EnjHqFTTNa5MODY4+b6PU4qQctGGxvLSRgg5T9dEFhUX 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /lib/athel_web/templates/layout/app.html.eex: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | <%= @view_module.title(@view_template, assigns) %> 11 | "> 12 | 13 | 14 | 15 |
16 | 21 |
22 | 23 |
24 | <%= for flash_type <- [:info, :success, :warning, :error] do %> 25 | <% flash = get_flash(@conn, flash_type) %> 26 | <%= if flash do %> 27 |
28 | <%= flash %> 29 |
30 | <% end %> 31 | <% end %> 32 | 33 | <%= render @view_module, @view_template, assigns %> 34 |
35 | 36 | 37 | -------------------------------------------------------------------------------- /priv/keys/example.com.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDlTCCAn2gAwIBAgIJAIi4OVDTktKcMA0GCSqGSIb3DQEBCwUAMGExCzAJBgNV 3 | BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMRUwEwYDVQQHDAxFeGFtcGxlIFRv 4 | d24xEDAOBgNVBAoMB0V4YW1wbGUxFDASBgNVBAMMC2V4YW1wbGUuY29tMB4XDTE2 5 | MDkyNjAwNDYzNFoXDTMwMDYwNTAwNDYzNFowYTELMAkGA1UEBhMCQVUxEzARBgNV 6 | BAgMClNvbWUtU3RhdGUxFTATBgNVBAcMDEV4YW1wbGUgVG93bjEQMA4GA1UECgwH 7 | RXhhbXBsZTEUMBIGA1UEAwwLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA 8 | A4IBDwAwggEKAoIBAQCb1lKsPEemSESt8t3/RedcB6/3mQ0QbZx5xpFwX2DB2oYM 9 | PEvh49icE4nbDAadQnPC25H/+J0N98BCMlRDxCcYquXwF1ZJIf+Nrw/J7O5+Zb3K 10 | JmP2OFfcjpzOECtNRDFykwhXpztjvi53W1W34gK540RZ5miAP59gEI0zk4qDOeAl 11 | Idv0WDNdcU1I/8KlI4rHVA9oEq9+8yqt+XbUyvRngmMfHgimElByOGCbdKjZ7J9C 12 | 7BVuV5g85hP5MH5YR/UzoAZ4CRdWzaJs5n6K/Mvs4e62BM+gVdv3J7FEDvKTNcFZ 13 | 8Wn6+xqKXHdwZ5yza2ibvjALclxk683MHeiiniGPAgMBAAGjUDBOMB0GA1UdDgQW 14 | BBR1rsgRDY8+7aPFA3tlih7uf8GpGTAfBgNVHSMEGDAWgBR1rsgRDY8+7aPFA3tl 15 | ih7uf8GpGTAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQB1ALWQRtsD 16 | t1bAgtskWKhuAOxsRrbJCW8U3n5TMYXPyWd+HLXIuotqC6zHxKxyzYugh1TUz+JM 17 | dhd5/DQf0qNJLb5Q+QkdQTsT5jIXLvpFy8u0dE/8jhNCDoIbkFugGEDJphSogipG 18 | A/AcS5FZ61vl96mNAAwmgDfFydAUqszSCihBLxBBxym9asOMlUAG2hlx/EHBn+Cv 19 | /QWorg3ujs6IXtYbDSu71zAixiwxUyBC00rmgprcq693LlQGJiZ41G5cKu/aRwzZ 20 | mVqu/bzfyTNntGTNgEaOrljgLhbJRHmQIt27p4yIHFU/Qs3CARfhK0/XdEUuzkUz 21 | PuqvYSJ18kQm 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /lib/athel/user_cache.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.UserCache do 2 | use GenServer 3 | 4 | # API 5 | 6 | def start_link(), do: start_link([]) 7 | def start_link([]) do 8 | GenServer.start_link(__MODULE__, :ok, name: UserCache) 9 | end 10 | 11 | def get(email) do 12 | GenServer.call(UserCache, {:get, email}) 13 | end 14 | 15 | def put(user) do 16 | GenServer.cast(UserCache, {:put, user}) 17 | end 18 | 19 | # Impl 20 | 21 | @impl true 22 | def init(:ok) do 23 | {:ok, %{}, {:continue, :load_users}} 24 | end 25 | 26 | @impl true 27 | def handle_continue(:load_users, _state) do 28 | users = %{} 29 | # Athel.User 30 | # |> Athel.Repo.all() 31 | # |> Enum.reduce(%{}, fn user, acc -> 32 | # loaded_user = %{user | decoded_public_key: load_key(user.public_key)} 33 | # Map.put(acc, user.email, loaded_user) 34 | # end) 35 | 36 | {:noreply, users} 37 | end 38 | 39 | @impl true 40 | def handle_call({:get, email}, _from, users) do 41 | {:reply, Map.get(users, email), users} 42 | end 43 | 44 | @impl true 45 | def handle_cast({:put, user}, users) do 46 | loaded_user = %{user | decoded_public_key: load_key(user.public_key)} 47 | {:noreply, Map.put(users, user.email, loaded_user)} 48 | end 49 | 50 | defp load_key(key) do 51 | [pem] = :public_key.pem_decode(key) 52 | :public_key.pem_entry_decode(pem) 53 | end 54 | 55 | end 56 | -------------------------------------------------------------------------------- /lib/athel_web/views/error_helpers.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.ErrorHelpers do 2 | @moduledoc """ 3 | Conveniences for translating and building error messages. 4 | """ 5 | 6 | use Phoenix.HTML 7 | 8 | @doc """ 9 | Generates tag for inlined form input errors. 10 | """ 11 | def error_tag(form, field) do 12 | if error = form.errors[field] do 13 | content_tag :span, translate_error(error), class: "help-block" 14 | end 15 | end 16 | 17 | @doc """ 18 | Translates an error message using gettext. 19 | """ 20 | def translate_error({msg, opts}) do 21 | # Because error messages were defined within Ecto, we must 22 | # call the Gettext module passing our Gettext backend. We 23 | # also use the "errors" domain as translations are placed 24 | # in the errors.po file. 25 | # Ecto will pass the :count keyword if the error message is 26 | # meant to be pluralized. 27 | # On your own code and templates, depending on whether you 28 | # need the message to be pluralized or not, this could be 29 | # written simply as: 30 | # 31 | # dngettext "errors", "1 file", "%{count} files", count 32 | # dgettext "errors", "is invalid" 33 | # 34 | if count = opts[:count] do 35 | Gettext.dngettext(Athel.Gettext, "errors", msg, msg, count, opts) 36 | else 37 | Gettext.dgettext(Athel.Gettext, "errors", msg, opts) 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /lib/athel/models/user.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.User do 2 | use Ecto.Schema 3 | import Ecto.Changeset 4 | 5 | alias Athel.Role 6 | 7 | @type t :: %__MODULE__{} 8 | 9 | @derive {Inspect, except: [:hashed_password, :salt]} 10 | @primary_key {:email, :string, autogenerate: false} 11 | schema "users" do 12 | field :hashed_password, :binary 13 | field :salt, :binary 14 | field :status, :string 15 | 16 | field :public_key, :string 17 | field :decoded_public_key, :binary, virtual: true 18 | 19 | timestamps() 20 | 21 | has_many :roles, Role 22 | end 23 | 24 | def changeset(struct, params \\ %{}) do 25 | struct 26 | |> cast(params, [:email, :hashed_password, :salt, :status, :public_key]) 27 | |> validate_required([:email, :status]) 28 | |> validate_length(:email, max: 255) 29 | |> validate_format(:email, ~r/^[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}$/i) 30 | |> validate_length(:salt, min: 8) 31 | |> validate_hash(:hashed_password) 32 | |> validate_inclusion(:status, ["active", "pending", "locked"]) 33 | end 34 | 35 | defp validate_hash(changeset, field) do 36 | case get_field(changeset, field) do 37 | nil -> changeset 38 | hash -> 39 | case Multihash.decode(hash) do 40 | {:ok, _} -> changeset 41 | {:error, _} -> add_error(changeset, field, "invalid multihash") 42 | end 43 | end 44 | end 45 | 46 | end 47 | -------------------------------------------------------------------------------- /lib/athel_web/endpoint.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.Endpoint do 2 | use Phoenix.Endpoint, otp_app: :athel 3 | 4 | socket "/socket", AthelWeb.UserSocket, 5 | websocket: true, 6 | longpoll: false 7 | 8 | # Serve at "/" the static files from "priv/static" directory. 9 | # 10 | # You should set gzip to true if you are running phoenix.digest 11 | # when deploying your static files in production. 12 | plug Plug.Static, 13 | at: "/", 14 | from: :athel, 15 | gzip: false, 16 | only: ~w(css fonts images js favicon.ico robots.txt) 17 | 18 | # Code reloading can be explicitly enabled under the 19 | # :code_reloader configuration of your endpoint. 20 | if code_reloading? do 21 | socket "/phoenix/live_reload/socket", Phoenix.LiveReloader.Socket 22 | plug Phoenix.LiveReloader 23 | plug Phoenix.CodeReloader 24 | end 25 | 26 | plug Plug.RequestId 27 | plug Plug.Logger 28 | 29 | plug Plug.Parsers, 30 | parsers: [:urlencoded, :multipart, :json], 31 | pass: ["*/*"], 32 | json_decoder: Phoenix.json_library() 33 | 34 | plug Plug.MethodOverride 35 | plug Plug.Head 36 | 37 | # The session will be stored in the cookie and signed, 38 | # this means its contents can be read but not tampered with. 39 | # Set :encryption_salt if you would also like to encrypt it. 40 | plug Plug.Session, 41 | store: :cookie, 42 | key: "_athel_key", 43 | signing_salt: "6J7dmZrf" 44 | 45 | plug AthelWeb.Router 46 | end 47 | -------------------------------------------------------------------------------- /test/models/attachment_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.AttachmentTest do 2 | use Athel.ModelCase 3 | 4 | alias Athel.Attachment 5 | 6 | @valid_attrs %{content: "some content", type: "some/type"} 7 | @invalid_attrs %{} 8 | 9 | test "changeset with valid attributes" do 10 | changeset = Attachment.changeset(%Attachment{}, @valid_attrs) 11 | assert changeset.valid? 12 | end 13 | 14 | test "changeset with invalid attributes" do 15 | changeset = Attachment.changeset(%Attachment{}, @invalid_attrs) 16 | refute changeset.valid? 17 | end 18 | 19 | test "hashes content" do 20 | {:ok, hash} = Multihash.encode(:sha1, 21 | <<183, 10, 51, 217, 73, 25, 65, 174, 198, 46, 152, 98, 8, 119, 198, 138, 45, 97, 22 | 114, 81>>) 23 | changeset = Attachment.changeset(%Attachment{}, 24 | %{@valid_attrs | content: "hello, sailor"}) 25 | assert changeset.changes[:hash] == hash 26 | end 27 | 28 | test "takes content type at face value" do 29 | changeset = Attachment.changeset(%Attachment{}, 30 | %{@valid_attrs | content: "

BIG TEXT

"}) 31 | assert changeset.changes[:type] == "some/type" 32 | end 33 | 34 | test "limits content length" do 35 | assert Attachment.changeset(%Attachment{}, %{@valid_attrs | content: "under 100"}).valid? 36 | long_content = Stream.repeatedly(fn -> "f" end) |> Enum.take(101) |> Enum.join("") 37 | assert_invalid(%Attachment{}, :content, long_content, "should be at most") 38 | end 39 | 40 | end 41 | -------------------------------------------------------------------------------- /test/services/auth_service_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.AuthServiceTest do 2 | use Athel.ModelCase 3 | alias Athel.{AuthService, UserCache, TestData} 4 | 5 | @signature "6E0FCFD3028899FE8BE11F44FC931FCE3435B8A6D0565AACDB6DFE28B23BB3C31030E1FE481D2070EFB837697287488ED0E4CEF061D748DE7F733A05C9589AE08A60C7784CE383106979BB0E9F2872FC0B5B9E834DA7E9943CF2687AF5E55FCC6FF77C7160FBC0B06ECFAE70DD98EA22AAE8177832F78C6F146A8F1E2EFF43C469CCEF9D9D68E125729517C4DB04170A979C164B34BD8A70103D6903EC5C2D396C255C23000C3538005F5B21D8A7A09A354A07BD775325D1F21A83E7A042BA9796B17B25DE4D9BC2AF8D9058863C88B7749B9FC52F187AEB9C05F37E8512AB7B7CC081D48B517A7A91DEE9527B3E65BC50B8AEE41374F91283CF1488FDDECEAA" 6 | 7 | setup do 8 | {:ok, user} = AuthService.create_user("jimbo@thrilling.chilling", "cherrypie") 9 | {:ok, user: user} 10 | end 11 | 12 | test "create invalid user" do 13 | {:error, _} = AuthService.create_user("who", "what") 14 | end 15 | 16 | test "login", %{user: user} do 17 | {:ok, logged_in_user} = AuthService.login("jimbo@thrilling.chilling", "cherrypie") 18 | assert user.email == logged_in_user.email 19 | refute user.salt == logged_in_user.salt 20 | 21 | assert AuthService.login("jimbo@thrilling.chilling", "HARDBODIES") == :invalid_credentials 22 | assert AuthService.login("peeweeherman", "jeepers") == :invalid_credentials 23 | end 24 | 25 | test "verify", %{user: user} do 26 | user = %{user | public_key: TestData.public_key} 27 | UserCache.put(user) 28 | 29 | assert AuthService.verify("nobody@example.com", "416", "asd") == false 30 | assert AuthService.verify(user.email, "416", "asd") == false 31 | assert AuthService.verify(user.email, @signature, "And now we have to say goodbye\n") == true 32 | end 33 | 34 | end 35 | -------------------------------------------------------------------------------- /priv/keys/example.com.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCb1lKsPEemSESt 3 | 8t3/RedcB6/3mQ0QbZx5xpFwX2DB2oYMPEvh49icE4nbDAadQnPC25H/+J0N98BC 4 | MlRDxCcYquXwF1ZJIf+Nrw/J7O5+Zb3KJmP2OFfcjpzOECtNRDFykwhXpztjvi53 5 | W1W34gK540RZ5miAP59gEI0zk4qDOeAlIdv0WDNdcU1I/8KlI4rHVA9oEq9+8yqt 6 | +XbUyvRngmMfHgimElByOGCbdKjZ7J9C7BVuV5g85hP5MH5YR/UzoAZ4CRdWzaJs 7 | 5n6K/Mvs4e62BM+gVdv3J7FEDvKTNcFZ8Wn6+xqKXHdwZ5yza2ibvjALclxk683M 8 | HeiiniGPAgMBAAECggEAP1uazyXO558YNTS55zBviO4jL+JM+nHmHWiK9woAF7CV 9 | sWHOZC+zgHk9Ig64nbVHxWBp8o0MpYIl64P02HxmfNP2mm+SiDdHZD5Zh/pJWKBa 10 | 0lZba96qciSVQf427Lod9Hws9x4pujq3P5WluxYrj5ID1x0jPYkgbfksv1xsAz+h 11 | Es7jR+A2w74TU03CdHr8ZmpdoAs42M0QrRgohvtl/EkLxxBkPxxnCTT6IQGAy1sw 12 | I4QYU7Vel7gbWby6elok2nzf2w0S7af8BDIOsAOx1NB/Zy7RHTzGWtyy7r8tULFz 13 | q1BCYOrJnb1yT+RIAYj/AtLMKskBKchTWFsjs/9/oQKBgQDNvhcvIwJtyRvMy4Jx 14 | BXjvgfjqJrUCKZlfulQ3ii6RtlZSYHfVxREUpVT2m6jIYEoJf53dgpoBWIh2NyZV 15 | /r+wKwEt6c1dHFMdJtaSvKbwC5gTYFokzTFcvmpW6qBZ2Bbz6DnaBcuY17xg/vPS 16 | 1/+MddajeKPS+BAk+pQKeMKk/wKBgQDB53NNoDcWgaOEQy3OlJz72rYt2bL8YGkB 17 | yC5uXkxJGZRxqs62Pdun8xa+zraxjU1tH+6hwyj+AI7RMKdR7nnk+ABo7s1TCJiA 18 | Bq3E1gcOohhEz+yELoEXX9VTjWZLfESuv+CSR/l1JReWzNry0SQVmr8ySIb1iz4q 19 | xGPx+6uzcQKBgQDI+VEIWHh86aBgUsNex+u0eg++GoViUWRi4E532mFXMPftjBJD 20 | HTdsJXxzUOZ0paps0N5SjMsHWYYjhAfMpQZ2feuu/939gDeoGFIuEF45yfmJo+sq 21 | W85GPDMAKDzuxmjVZRlt4Y9aBBMd5K4kXZ5hhJJgKO5OnMaYeLW37PKl3QKBgGeT 22 | +0O8EbE0DuTX/eAcAr+GVUqov7OQzIbnJ+ZM+PMTdvhBBarT4EIW2E+UnIK7uGBS 23 | bmZ6masVITUdiEN74CEvWQi0h3mTXeMFrk03Bw4KCGy5pN32+X5C8vFu1vX7q7St 24 | SojZaafp6G/lfg+3KE9iGkAB/hWsC8lMnxbkGRQBAoGAfs3gUMkNZjz9RfPATqbt 25 | SDyNcCKpgHCelXc7nIH5LEPgsiH5meXn2dXo5pYKKiM1ZS+ub9eZm/8DvWnj8A2e 26 | /U8EtrEyME38hpWko7i52cSJCl2w+HymvR3Lt/Y96Kl6vZ2aDCX/+pBR+SC4HDhM 27 | GCaK2mVQigMvbpc3RSk/QUo= 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /priv/keys/testing.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCcvweLntg34Gqz 3 | t2CKuDU0uH8LSlvvOhga1TdLqm7lqdA/I4cRihFHPh5m4kP5zDu+BVJoFh+Gbw6G 4 | PYd19duEn5zM5rkHZg7UZWqp4i2pjntifwXdUQZIsFZ6KZZp6S1HcHd1FeIpb8/V 5 | aIwZg807ldtHDTl0JrrDoJTzm3PTFuEUBGorD0Nj+Pwl1uKZpTnOqOJek0mkMtXv 6 | 0+PaX14BBkouSivXF0xcULV2YdNczC5EUFFZKJuKk0cInocMq00Vuvklpq1zGouj 7 | 740hKmbwyKNoJi8sAYUtAnybsFchHpJFW4AKhlQzjTAYvnY+Yja6i0Ui/xcJm3jR 8 | l9XfMYkzAgMBAAECggEAKGwnQP1s2zQXsFMZJY0Nw5PUx4+cl9wOfVUBFpVUVgvt 9 | 9WpvGbnWbN37LyMozpG50m5C6y7RYHThdQMHHQeTXedfo4PYsazDJEknMbpvdiuV 10 | bDg/xexwR2yaUJTLAnMsxyCc3egP1AnOukVk4+uWkMg7rV4es/KM9YhDAXPUcdos 11 | Sp/OG0GfqhyjFAK1dsX/IK8wIofPwa1M5V1L86NmTRtD3VZj+6qnhHGrmezn5Bg/ 12 | UuKOclFrOnJGNVg5vXG95Nn9uMPgxu2MLTyXw4tykHbi7LSzcH9eAWUTohl7Jxkf 13 | d7S8LBGLM8fXpwbQ4wfA09w/G9pZ8+gDIHGsbJrp8QKBgQDLYbOm9xhtTVAZeXQQ 14 | unM+pXDtt518HUAtUjSE9K3zUR96diCo+DJH/q7F5BkSL84u5rGaQIG5Ra+3agH+ 15 | lDm0/oopc6dkJnKRRBedZGNj9iKKksWSgvvtwNcELMFh8HO+yQNyn5FEEgUX2kXJ 16 | nJCl3VSCBqAivwqr0CgmmVzHSwKBgQDFTJYnt79amUh8ohbZqk9aSgmUBLS6EinU 17 | o1hfK74kI8SQfI4BxYtAwWc5qWQRRfPP9/hVzw3zfkneiph8z/DatqFMtRvBKS1J 18 | FOZD+64IU8wj9SV5yeyTfOfxzchMeUNJnrmCoOZxKHAgIzrcUToJgD2EqHlX5O34 19 | PlSSzqoMuQKBgQCQn6JLyYwyNXcPFmGlf6Bx3N2H/TjcyEQZtkoofYGw82/p+lRR 20 | M2U18vI/QGtflmUMzvleUh6tK9O/Hn/ak3bRsOt4fIh83CY+DGiqgHd43s9DMQmT 21 | nNcfAzEjA9xkE8OK2JA+EyAOgq3if1F/A3mMqO3uJF39N1KUSMo0YHwsLwKBgQCv 22 | k00YgUMvS7Me/luZTh8ZuUM2zs1JvLou+UG+R74IiS/2aHEzEGmwsau7u4tKd9bV 23 | ntUG/6BprFvuR6YVhDLRX67BBXZyecNMAuY7X3BrBq9m3FSCQfhe88uw+jCiJVOE 24 | 41Qw9CC+WH8XimJqB3q/U7jrIcYCOr6uqEE49+KKOQKBgQDKRHNyOPbrpE+GMjlK 25 | bVB92Rt1f4SDksLfbHSbVdOee2Vy8UHrW4BtLxuwSOeNrF6MKtvsCOcMbhp3hjfT 26 | 0x3QwkCPa0IPHsyQaka5wG73YK5iSHJdJvVTtHgpJ7b4IX+3Lj5H6RuxhZEh/6hE 27 | omrPRZ7IZvPkic+aZiNW9PARvw== 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [app: :athel, 6 | version: "0.1.0", 7 | elixir: "~> 1.8", 8 | elixirc_paths: elixirc_paths(Mix.env), 9 | compilers: [:phoenix, :gettext] ++ Mix.compilers, 10 | build_embedded: Mix.env == :prod, 11 | start_permanent: Mix.env == :prod, 12 | aliases: aliases(), 13 | deps: deps()] 14 | end 15 | 16 | # Configuration for the OTP application. 17 | # 18 | # Type `mix help compile.app` for more information. 19 | def application do 20 | [mod: {Athel.Application, []}, 21 | extra_applications: [:logger, :runtime_tools]] 22 | end 23 | 24 | # Specifies which paths to compile per environment. 25 | defp elixirc_paths(:test), do: ["lib", "test/support"] 26 | defp elixirc_paths(_), do: ["lib"] 27 | 28 | defp deps do 29 | [ 30 | {:gen_stage, "~> 0.14"}, 31 | {:phoenix, "~> 1.4.0"}, 32 | {:phoenix_ecto, "~> 4.0.0"}, 33 | {:postgrex, ">= 0.0.0"}, 34 | {:ecto_sql, "~> 3.0"}, 35 | {:phoenix_html, "~> 2.12.0"}, 36 | {:phoenix_live_reload, "~> 1.2", only: :dev}, 37 | {:gettext, "~> 0.16.0"}, 38 | {:plug_cowboy, "~> 2.0"}, 39 | {:jason, "~> 1.0.0"}, 40 | {:timex, "~> 3.4"}, 41 | {:cloak, "~> 0.9.0"}, 42 | {:ex_multihash, github: "ruhlio/ex_multihash"}, 43 | {:codepagex, "~> 0.1.4"}, 44 | {:distillery, "~> 2.0.0"}, 45 | 46 | # dev 47 | {:credo, "~> 1.0.0", only: [:dev, :test], runtime: false}, 48 | {:dialyxir, "~> 1.0.0-rc.4", only: [:dev], runtime: false} 49 | ] 50 | end 51 | 52 | defp aliases do 53 | ["ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"], 54 | "ecto.reset": ["ecto.drop", "ecto.setup"], 55 | test: ["ecto.migrate", "test"]] 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /lib/athel_web.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb do 2 | @moduledoc """ 3 | A module that keeps using definitions for controllers, 4 | views and so on. 5 | 6 | This can be used in your application as: 7 | 8 | use AthelWeb, :controller 9 | use AthelWeb, :view 10 | 11 | The definitions below will be executed for every view, 12 | controller, etc, so keep them short and clean, focused 13 | on imports, uses and aliases. 14 | 15 | Do NOT define functions inside the quoted expressions 16 | below. 17 | """ 18 | 19 | def controller do 20 | quote do 21 | use Phoenix.Controller, namespace: AthelWeb 22 | 23 | alias Athel.Repo 24 | import Ecto 25 | import Ecto.Query 26 | 27 | import AthelWeb.Router.Helpers 28 | import AthelWeb.Gettext 29 | end 30 | end 31 | 32 | def view do 33 | quote do 34 | use Phoenix.View, root: "lib/athel_web/templates", namespace: AthelWeb 35 | 36 | # Import convenience functions from controllers 37 | import Phoenix.Controller, only: [get_csrf_token: 0, get_flash: 2, view_module: 1] 38 | 39 | # Use all HTML functionality (forms, tags, etc) 40 | use Phoenix.HTML 41 | 42 | import AthelWeb.Router.Helpers 43 | import AthelWeb.ErrorHelpers 44 | import AthelWeb.Gettext 45 | 46 | import AthelWeb.ViewCommon 47 | end 48 | end 49 | 50 | def router do 51 | quote do 52 | use Phoenix.Router 53 | end 54 | end 55 | 56 | def channel do 57 | quote do 58 | use Phoenix.Channel 59 | 60 | alias Athel.Repo 61 | import Ecto 62 | import Ecto.Query 63 | import AthelWeb.Gettext 64 | end 65 | end 66 | 67 | @doc """ 68 | When used, dispatch to the appropriate controller/view/etc. 69 | """ 70 | defmacro __using__(which) when is_atom(which) do 71 | apply(__MODULE__, which, []) 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/athel/services/auth_service.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.AuthService do 2 | alias Ecto.Changeset 3 | alias Athel.Repo 4 | alias Athel.User 5 | 6 | @spec create_user(String.t, String.t) :: {:ok, User.t} | {:error, Changeset.t} 7 | def create_user(email, password) do 8 | salt = create_salt() 9 | changeset = User.changeset(%User{}, 10 | %{email: email, 11 | salt: salt, 12 | hashed_password: hash_password(password, salt), 13 | status: "active"}) 14 | Repo.insert(changeset) 15 | end 16 | 17 | @spec login(String.t, String.t) :: {:ok, User.t} | :invalid_credentials 18 | def login(email, password) do 19 | case Repo.get_by(User, email: email) do 20 | nil -> :invalid_credentials 21 | user -> 22 | if hash_password(password, user.salt) == user.hashed_password do 23 | {:ok, update_login(user, password)} 24 | else 25 | :invalid_credentials 26 | end 27 | end 28 | 29 | end 30 | 31 | @spec verify(String.t, String.t, String.t) :: boolean 32 | def verify(email, signature, message) do 33 | case Athel.UserCache.get(email) do 34 | nil -> false 35 | user -> 36 | case Base.decode16(signature) do 37 | {:ok, decoded_signature} -> 38 | :public_key.verify(message, :sha512, decoded_signature, user.decoded_public_key) 39 | _ -> false 40 | end 41 | end 42 | end 43 | 44 | defp update_login(user, password) do 45 | new_salt = create_salt() 46 | user = Changeset.change user, 47 | salt: new_salt, 48 | hashed_password: hash_password(password, new_salt) 49 | Repo.update!(user) 50 | end 51 | 52 | defp create_salt do 53 | :crypto.strong_rand_bytes(32) 54 | end 55 | 56 | defp hash_password(password, salt) do 57 | hash = :crypto.hash(:sha512, password <> salt) 58 | {:ok, multihash} = Multihash.encode(:sha2_512, hash) 59 | multihash 60 | end 61 | 62 | end 63 | -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | # External Dependencies 2 | 3 | ## Debian 4 | `apt install elixir npm postgres libmagic-dev` 5 | 6 | Make sure to run `ln -s /usr/bin/nodejs /usr/bin/node` afterwards 7 | 8 | # Dependency setup 9 | 10 | ## PostgreSQL 11 | 12 | To setup for development: 13 | 1. `sudo -u postgres psql` 14 | 2. `create user athel password 'athel';` 15 | 3. `create database athel_dev owner athel` 16 | 4. `create database athel_test owner athel` 17 | 18 | ## Elixir 19 | 20 | 1. `mix deps.get` - pull in dependencies 21 | 2. Setup emagic 22 | 3. `mix ecto.migrate` - initialize database 23 | 4. `mix test` - run automated tests 24 | 25 | ## Node.js 26 | 27 | Node is only being used as a build system for the (currently non-existent) frontend. 28 | This is optional if you only care about NNTP 29 | - `npm -g install npm` - update npm to the latest version 30 | - `npm install` - pull local dependencies 31 | - `npm install -g brunch` - will put `brunch` into the path, can be skipped by using =node_modules/brunch/bin/brunch= directly 32 | 33 | # Running 34 | 35 | `mix phoenix.server` should get you going, the NNTP server will be running on port 8119 36 | 37 | `mix test` to run tests. The error output is expected for a few tests for the time being. 38 | 39 | # Deployment 40 | 41 | ## Configuration 42 | 43 | Create `config/prod.secret.exs` and fill out/copy over the Athel.Nntp 44 | and Athel.Repo configs. Ideally this will be pulled in at runtime 45 | instead of compiletime in the future. 46 | 47 | ## Assets 48 | 49 | `./node_modules/brunch/bin/brunch b -p` builds static assets, and 50 | `MIX_ENV=prod mix phoenix.digest` sets them up for caching 51 | 52 | ## Packaging 53 | 54 | [Distillery](https://hexdocs.pm/distillery/) is used to create the release package. 55 | `MIX_ENV=prod mix release --env=prod` will generate the release, with `--upgrade` 56 | added for generating an upgrade release. Make sure that the system 57 | you deploy on is binary compatible with the system that you build on. 58 | 59 | ## TODO Deployment 60 | 61 | 62 | -------------------------------------------------------------------------------- /config/dev.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | # For development, we disable any cache and enable 4 | # debugging and code reloading. 5 | # 6 | # The watchers configuration can be used to run external 7 | # watchers to your application. For example, we use it 8 | # with brunch.io to recompile .js and .css sources. 9 | config :athel, AthelWeb.Endpoint, 10 | http: [port: 4000], 11 | debug_errors: true, 12 | code_reloader: true, 13 | check_origin: false, 14 | watchers: [node: ["node_modules/brunch/bin/brunch", "watch", 15 | cd: Path.expand("../assets", __DIR__)]] 16 | 17 | 18 | # Watch static and templates for browser reloading. 19 | config :athel, AthelWeb.Endpoint, 20 | live_reload: [ 21 | patterns: [ 22 | ~r{priv/static/.*(js|css|png|jpeg|jpg|gif|svg)$}, 23 | ~r{priv/gettext/.*(po)$}, 24 | ~r{lib/athel_web/views/.*(ex)$}, 25 | ~r{lib/athel_web/templates/.*(eex)$} 26 | ] 27 | ] 28 | 29 | # Do not include metadata nor timestamps in development logs 30 | config :logger, :console, format: "[$level] $message\n" 31 | 32 | # Set a higher stacktrace during development. Avoid configuring such 33 | # in production as building large stacktraces may be expensive. 34 | config :phoenix, :stacktrace_depth, 20 35 | 36 | # Configure your database 37 | config :athel, Athel.Repo, 38 | username: "athel", 39 | password: "athel", 40 | database: "athel_dev", 41 | hostname: "localhost", 42 | pool_size: 10 43 | 44 | config :athel, Athel.Nntp, 45 | port: 8119, 46 | hostname: "localhost", 47 | pool_size: 10, 48 | timeout: 61_000, 49 | keyfile: Path.expand("../priv/keys/testing.key", __DIR__), 50 | certfile: Path.expand("../priv/keys/testing.cert", __DIR__), 51 | cacertfile: Path.expand("../priv/keys/example.com.crt", __DIR__), 52 | max_request_size: 75_000_000, 53 | max_attachment_size: 20_000_000, 54 | max_attachment_count: 3 55 | 56 | config :athel, Athel.Vault, 57 | ciphers: [ 58 | default: {Cloak.Ciphers.AES.GCM, tag: "AES.GCM.V1", key: Base.decode64!("7/TfD+toi48MB2bpPZRnsfc8pgvpY1QEQWvfYyfGsVw=")} 59 | ] 60 | -------------------------------------------------------------------------------- /lib/athel/models/attachment.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Attachment do 2 | use Ecto.Schema 3 | import Ecto.Changeset 4 | alias Ecto.Changeset 5 | alias Athel.Article 6 | 7 | @type t :: %__MODULE__{} 8 | 9 | schema "attachments" do 10 | field :filename, :string 11 | field :type, :string 12 | field :hash, :binary 13 | field :content, :binary 14 | 15 | many_to_many :article, Article, 16 | join_through: "attachments_to_articles", 17 | join_keys: [attachment_id: :id, message_id: :message_id] 18 | 19 | timestamps() 20 | end 21 | 22 | def changeset(struct, params \\ %{}) do 23 | max_attachment_size = Application.fetch_env!(:athel, Athel.Nntp)[:max_attachment_size] 24 | 25 | struct 26 | |> cast(params, [:content, :filename, :type]) 27 | |> hash_changeset 28 | |> validate_required([:type, :hash, :content]) 29 | |> validate_length(:content, max: max_attachment_size) 30 | end 31 | 32 | @spec hash_content(binary) :: {:ok, binary} | {:error, String.t} 33 | def hash_content(content) do 34 | Multihash.encode(:sha1, :crypto.hash(:sha, content)) 35 | end 36 | 37 | defp hash_changeset(changeset = %Changeset{changes: changes}) do 38 | case changes[:content] do 39 | nil -> changeset 40 | content -> 41 | {:ok, hash} = hash_content(content) 42 | put_change(changeset, :hash, hash) 43 | end 44 | end 45 | 46 | end 47 | 48 | defimpl Athel.Nntp.Formattable, for: Athel.Attachment do 49 | alias Athel.Nntp.Formattable 50 | 51 | def format(attachment) do 52 | headers = 53 | %{"Content-Type" => attachment.type, 54 | "Content-Transfer-Encoding" => "base64"} 55 | |> add_disposition_header(attachment.filename) 56 | 57 | [Formattable.format(headers), Base.encode64(attachment.content), "\r\n"] 58 | end 59 | 60 | defp add_disposition_header(headers, nil) do 61 | headers 62 | end 63 | 64 | defp add_disposition_header(headers, filename) do 65 | Map.put(headers, 66 | "Content-Disposition", "attachment, filename=\"#{filename}\"") 67 | end 68 | 69 | end 70 | -------------------------------------------------------------------------------- /lib/athel/nntp/defs.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Nntp.Defs do 2 | 3 | defmacro command(name, function, opts) do 4 | max_args = Keyword.get(opts, :max_args, 0) 5 | auth = Keyword.get(opts, :auth, [required: false]) 6 | unauthorized_response = Keyword.get(auth, :response, {483, "Unauthorized"}) 7 | 8 | max_args_clause = quote do 9 | length(args) > unquote(max_args) -> 10 | {:reply, {:continue, {501, "Too many arguments"}}, state} 11 | end 12 | 13 | auth_clause = quote do 14 | !is_authenticated(state) -> 15 | Logger.info( 16 | "Access denied for #{get_username(state)}@#{unquote(name)}") 17 | {:reply, {:continue, unquote(unauthorized_response)}, state} 18 | end 19 | 20 | action_clause = quote do 21 | true -> 22 | try do 23 | case __MODULE__.unquote(function)(args, state) do 24 | {action, response} -> 25 | {:reply, {action, response}, state} 26 | {action, response, state_updates} -> 27 | new_state = Map.merge(state, state_updates) 28 | {:reply, {action, response}, new_state} 29 | end 30 | rescue 31 | _ in FunctionClauseError -> 32 | Logger.info( 33 | "Invalid arguments passed to '#{unquote(name)}': #{inspect args}") 34 | {:reply, {:error, {501, "Invalid #{unquote(name)} arguments"}}, state} 35 | end 36 | end 37 | 38 | clauses = List.flatten(if auth[:required] do 39 | [max_args_clause, auth_clause, action_clause] 40 | else 41 | [max_args_clause, action_clause] 42 | end) 43 | 44 | quote do 45 | def handle_call({unquote(name), args}, _sender, state) do 46 | # credo:disable-for-next-line Credo.Check.Refactor.CondStatements 47 | cond do 48 | unquote(clauses) 49 | end 50 | end 51 | end 52 | 53 | end 54 | 55 | def is_authenticated(%{authentication: %Athel.User{}}), do: true 56 | def is_authenticated(_), do: false 57 | 58 | def get_username(%{authentication: %Athel.User{email: email}}) do 59 | "user #{email}" 60 | end 61 | def get_username(_), do: "unauthorized client" 62 | 63 | end 64 | -------------------------------------------------------------------------------- /lib/athel/nntp/formattable.ex: -------------------------------------------------------------------------------- 1 | defprotocol Athel.Nntp.Formattable do 2 | @spec format(t) :: iodata 3 | def format(formattable) 4 | end 5 | 6 | defimpl Athel.Nntp.Formattable, for: List do 7 | def format(list) do 8 | [Enum.reduce(list, [], &escape_line/2), ".\r\n"] 9 | end 10 | 11 | defp escape_line(<<".", rest :: binary>>, acc) do 12 | [acc, "..", rest, "\r\n"] 13 | end 14 | 15 | defp escape_line(line, acc) when is_number(line) or is_boolean(line) do 16 | escape_line(to_string(line), acc) 17 | end 18 | 19 | defp escape_line(line, acc) do 20 | [acc, line, "\r\n"] 21 | end 22 | 23 | end 24 | 25 | defimpl Athel.Nntp.Formattable, for: Range do 26 | alias Athel.Nntp.Formattable 27 | 28 | def format(range) do 29 | range |> Enum.to_list |> Formattable.format 30 | end 31 | end 32 | 33 | defimpl Athel.Nntp.Formattable, for: Map do 34 | def format(headers) do 35 | formatted = Enum.reduce(headers, [], fn ({key, value}, acc) -> 36 | format_header({format_key(key), value}, acc) 37 | end) 38 | [formatted, "\r\n"] 39 | end 40 | 41 | defp format_header({_, value}, acc) when is_nil(value) do 42 | acc 43 | end 44 | # multiple header values 45 | defp format_header({key, values}, acc) when is_list(values) do 46 | Enum.reduce(values, acc, &format_header({key, &1}, &2)) 47 | end 48 | # paramterized header 49 | defp format_header({key, %{value: value, params: params}}, acc) do 50 | formatted_params = Enum.reduce(params, [], &format_param/2) 51 | [acc, key, ": ", value, formatted_params, "\r\n"] 52 | end 53 | # default 54 | defp format_header({key, value}, acc) do 55 | [acc, key, ": ", value, "\r\n"] 56 | end 57 | 58 | defp format_param({key, value}, acc) do 59 | formatted_key = String.downcase(key) 60 | if value =~ ~r/\s/ do 61 | [acc, "; ", formatted_key, "=\"", value, "\""] 62 | else 63 | [acc, "; ", formatted_key, "=", value] 64 | end 65 | end 66 | 67 | defp format_key("MESSAGE-ID"), do: "Message-ID" 68 | defp format_key("MIME-VERSION"), do: "MIME-Version" 69 | defp format_key(key) do 70 | key 71 | |> String.split("-") 72 | |> Enum.map(&String.capitalize/1) 73 | |> Enum.join("-") 74 | end 75 | end 76 | 77 | -------------------------------------------------------------------------------- /test/support/test_data.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.TestData do 2 | def public_key, do: """ 3 | -----BEGIN PUBLIC KEY----- 4 | MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArqZrJaCElI7RnTP3qrSs 5 | MoM9LEgRQhmJK5I5p9tBaGPRKXSBZ072NREHM+z19KCCItnLbPXp4DKPJt8BGHuM 6 | lFq3jz+RSwc8I6E10NpLcHsdPftC/X/5qVCNSol6rjQxPWNzXdF5xmDURQA5svop 7 | VWbclv3uJaDg8sORDwcedUkEtkEZCRkHR8a3ZO2vjhIRPqPuh1keRG0EK4wypXnM 8 | dlyQC3Ci3cnXQQgj3HsJCuqdt2mN7TNQLlGNqLm/DufrkBS22Kr6SV1M0lw9n/CI 9 | /lBG9i0WOiKmJZS2ka5sXEP3rtTJprl5i4yNCjjAmXdmAL8PJ3cs4tHWt2BsFOXQ 10 | AwIDAQAB 11 | -----END PUBLIC KEY----- 12 | """ 13 | 14 | def private_key, do: """ 15 | -----BEGIN RSA PRIVATE KEY----- 16 | MIIEowIBAAKCAQEArqZrJaCElI7RnTP3qrSsMoM9LEgRQhmJK5I5p9tBaGPRKXSB 17 | Z072NREHM+z19KCCItnLbPXp4DKPJt8BGHuMlFq3jz+RSwc8I6E10NpLcHsdPftC 18 | /X/5qVCNSol6rjQxPWNzXdF5xmDURQA5svopVWbclv3uJaDg8sORDwcedUkEtkEZ 19 | CRkHR8a3ZO2vjhIRPqPuh1keRG0EK4wypXnMdlyQC3Ci3cnXQQgj3HsJCuqdt2mN 20 | 7TNQLlGNqLm/DufrkBS22Kr6SV1M0lw9n/CI/lBG9i0WOiKmJZS2ka5sXEP3rtTJ 21 | prl5i4yNCjjAmXdmAL8PJ3cs4tHWt2BsFOXQAwIDAQABAoIBAHbNNW1u90Cmteed 22 | hgdUxx3FMEOC8lpoTGqbGSUZfDCqVYlBexTvHYOThbbIpbY1yNA0HrCLxv9+5Omo 23 | IHKq+EGiQ+Lpdsf2r+38p0LeexqUZJvY4wTVnNqTtMjTI+SEFEqR79QNviw3ia02 24 | LgmVKbCyO7NqICjwepQoe/AhA5L7bCxxKaTPm7ZD4cVhaf5IlMub5J1TWcxljxLZ 25 | ZTL3OftZbDdIv5nXNUydXsnfngeLxtEH3tANt+vn+WFjRYP+Jsd9XMN0yOF9cQuZ 26 | gqK8cuo73gMyG5FL/6hylyvQwAsXk54ehrrH9WucvmcupmBnbf2queBUjXNiEwef 27 | FUUr4IECgYEA4heacRgq2f4wyg7BDsk3hMspC128mrpqpZ0ln0UADs0LKnBtDCq+ 28 | 10oOUWA52DHnLQrEmWjUW9pkNesy8Dsn31N7lKo6kueHwYLAS1WZVvOEaUKNY523 29 | oi3zjaH6aOnvseTJ8cuqO2ZQFljIQ/Zl+KyoRwKK5d2QrgRa5CFpgRECgYEAxcDG 30 | 0y6kt02Vsnr6EyNWVsTtb+pUUwctGLPLBeFD/FrDoQBxmR6MlAeuwoaN6TWSahmw 31 | sZVGE/56wuMlygfZEbGyuHgSYsxPOXCr1Czl/Xi26oBUzMEUnrx4EaGJc3LtuB+1 32 | OrX4eneMS6u8n46Fx27p7JR5WTxuWZeyvgVZf9MCgYBSWsam62awgSbEcxtfh2vx 33 | sw8AVOSed8jhCpzppviea5Hlo44VIHzjbtZITgTD+2l5vrJeLxErZCGcgk/LscCU 34 | WJRrUpaDbFLG6hmhV0zDn3Bb5yIZZxm8uYA91wKftJba9buZl9YqTNpfSXepSdda 35 | /YlOVF7D3DEXMf7pmkIUAQKBgQC3uyZ/q4SKcmE1VKDoCxr6vzjDlHoIMlCp9NIa 36 | gnNCEapU+i6RTxrZplGulolfNdD1Fy1dsQ1NIlE4pQbFMIlzsSAV2Cls9dpdyds7 37 | 5QNCf1ejhNxE6NeZrA36g5VLWGqZeYxOIifc0RnebI9xx19wLhLVJhWg3U7BmvoN 38 | JrdC1QKBgFxvTgCYqvt/amtcdKiO+sC6J8KWdTwM6wq0NB+v0WuBxcAXLXrj50m/ 39 | pah+4nAk7RnOzlDgm3LePf/p5ak/IocY5PODKCqKPowLQGHmwhfZBAiOXX7Iuib3 40 | GwaR4mDTsrXo2M4wYDHhSpPI8e2TXVApgLyBBsuAf3csmLzlCJEI 41 | -----END RSA PRIVATE KEY----- 42 | """ 43 | 44 | end 45 | -------------------------------------------------------------------------------- /priv/gettext/en/LC_MESSAGES/errors.po: -------------------------------------------------------------------------------- 1 | ## `msgid`s in this file come from POT (.pot) files. 2 | ## 3 | ## Do not add, change, or remove `msgid`s manually here as 4 | ## they're tied to the ones in the corresponding POT file 5 | ## (with the same domain). 6 | ## 7 | ## Use `mix gettext.extract --merge` or `mix gettext.merge` 8 | ## to merge POT files into PO files. 9 | msgid "" 10 | msgstr "" 11 | "Language: en\n" 12 | 13 | ## From Ecto.Changeset.cast/4 14 | msgid "can't be blank" 15 | msgstr "" 16 | 17 | ## From Ecto.Changeset.unique_constraint/3 18 | msgid "has already been taken" 19 | msgstr "" 20 | 21 | ## From Ecto.Changeset.put_change/3 22 | msgid "is invalid" 23 | msgstr "" 24 | 25 | ## From Ecto.Changeset.validate_format/3 26 | msgid "has invalid format" 27 | msgstr "" 28 | 29 | ## From Ecto.Changeset.validate_subset/3 30 | msgid "has an invalid entry" 31 | msgstr "" 32 | 33 | ## From Ecto.Changeset.validate_exclusion/3 34 | msgid "is reserved" 35 | msgstr "" 36 | 37 | ## From Ecto.Changeset.validate_confirmation/3 38 | msgid "does not match confirmation" 39 | msgstr "" 40 | 41 | ## From Ecto.Changeset.no_assoc_constraint/3 42 | msgid "is still associated to this entry" 43 | msgstr "" 44 | 45 | msgid "are still associated to this entry" 46 | msgstr "" 47 | 48 | ## From Ecto.Changeset.validate_length/3 49 | msgid "should be %{count} character(s)" 50 | msgid_plural "should be %{count} character(s)" 51 | msgstr[0] "" 52 | msgstr[1] "" 53 | 54 | msgid "should have %{count} item(s)" 55 | msgid_plural "should have %{count} item(s)" 56 | msgstr[0] "" 57 | msgstr[1] "" 58 | 59 | msgid "should be at least %{count} character(s)" 60 | msgid_plural "should be at least %{count} character(s)" 61 | msgstr[0] "" 62 | msgstr[1] "" 63 | 64 | msgid "should have at least %{count} item(s)" 65 | msgid_plural "should have at least %{count} item(s)" 66 | msgstr[0] "" 67 | msgstr[1] "" 68 | 69 | msgid "should be at most %{count} character(s)" 70 | msgid_plural "should be at most %{count} character(s)" 71 | msgstr[0] "" 72 | msgstr[1] "" 73 | 74 | msgid "should have at most %{count} item(s)" 75 | msgid_plural "should have at most %{count} item(s)" 76 | msgstr[0] "" 77 | msgstr[1] "" 78 | 79 | ## From Ecto.Changeset.validate_number/3 80 | msgid "must be less than %{number}" 81 | msgstr "" 82 | 83 | msgid "must be greater than %{number}" 84 | msgstr "" 85 | 86 | msgid "must be less than or equal to %{number}" 87 | msgstr "" 88 | 89 | msgid "must be greater than or equal to %{number}" 90 | msgstr "" 91 | 92 | msgid "must be equal to %{number}" 93 | msgstr "" 94 | -------------------------------------------------------------------------------- /priv/gettext/errors.pot: -------------------------------------------------------------------------------- 1 | ## This file is a PO Template file. 2 | ## 3 | ## `msgid`s here are often extracted from source code. 4 | ## Add new translations manually only if they're dynamic 5 | ## translations that can't be statically extracted. 6 | ## 7 | ## Run `mix gettext.extract` to bring this file up to 8 | ## date. Leave `msgstr`s empty as changing them here as no 9 | ## effect: edit them in PO (`.po`) files instead. 10 | 11 | ## From Ecto.Changeset.cast/4 12 | msgid "can't be blank" 13 | msgstr "" 14 | 15 | ## From Ecto.Changeset.unique_constraint/3 16 | msgid "has already been taken" 17 | msgstr "" 18 | 19 | ## From Ecto.Changeset.put_change/3 20 | msgid "is invalid" 21 | msgstr "" 22 | 23 | ## From Ecto.Changeset.validate_format/3 24 | msgid "has invalid format" 25 | msgstr "" 26 | 27 | ## From Ecto.Changeset.validate_subset/3 28 | msgid "has an invalid entry" 29 | msgstr "" 30 | 31 | ## From Ecto.Changeset.validate_exclusion/3 32 | msgid "is reserved" 33 | msgstr "" 34 | 35 | ## From Ecto.Changeset.validate_confirmation/3 36 | msgid "does not match confirmation" 37 | msgstr "" 38 | 39 | ## From Ecto.Changeset.no_assoc_constraint/3 40 | msgid "is still associated to this entry" 41 | msgstr "" 42 | 43 | msgid "are still associated to this entry" 44 | msgstr "" 45 | 46 | ## From Ecto.Changeset.validate_length/3 47 | msgid "should be %{count} character(s)" 48 | msgid_plural "should be %{count} character(s)" 49 | msgstr[0] "" 50 | msgstr[1] "" 51 | 52 | msgid "should have %{count} item(s)" 53 | msgid_plural "should have %{count} item(s)" 54 | msgstr[0] "" 55 | msgstr[1] "" 56 | 57 | msgid "should be at least %{count} character(s)" 58 | msgid_plural "should be at least %{count} character(s)" 59 | msgstr[0] "" 60 | msgstr[1] "" 61 | 62 | msgid "should have at least %{count} item(s)" 63 | msgid_plural "should have at least %{count} item(s)" 64 | msgstr[0] "" 65 | msgstr[1] "" 66 | 67 | msgid "should be at most %{count} character(s)" 68 | msgid_plural "should be at most %{count} character(s)" 69 | msgstr[0] "" 70 | msgstr[1] "" 71 | 72 | msgid "should have at most %{count} item(s)" 73 | msgid_plural "should have at most %{count} item(s)" 74 | msgstr[0] "" 75 | msgstr[1] "" 76 | 77 | ## From Ecto.Changeset.validate_number/3 78 | msgid "must be less than %{number}" 79 | msgstr "" 80 | 81 | msgid "must be greater than %{number}" 82 | msgstr "" 83 | 84 | msgid "must be less than or equal to %{number}" 85 | msgstr "" 86 | 87 | msgid "must be greater than or equal to %{number}" 88 | msgstr "" 89 | 90 | msgid "must be equal to %{number}" 91 | msgstr "" 92 | -------------------------------------------------------------------------------- /lib/athel_web/controllers/group_controller.ex: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.GroupController do 2 | use AthelWeb, :controller 3 | 4 | @articles_per_page 15 5 | @page_link_count 5 6 | 7 | def index(conn, _params) do 8 | groups = Repo.all(Athel.Group) 9 | render(conn, "index.html", groups: groups) 10 | end 11 | 12 | def show(conn, params = %{"name" => name}) do 13 | {page, _} = params |> Map.get("page", "0") |> Integer.parse 14 | query = Map.get(params, "query", "") 15 | {group, article_count} = load_group(name, page, query) 16 | page_params = calculate_page_params(page, article_count) 17 | 18 | render(conn, "show.html", 19 | page_params ++ [group: group, article_count: article_count]) 20 | end 21 | 22 | defp calculate_page_params(page, article_count) do 23 | page_count = ceil(article_count / @articles_per_page) - 1 24 | # are these div/rem results inlined? 25 | pages_per_direction = div(@page_link_count, 2) 26 | page_offset = rem(@page_link_count, 2) 27 | page_range = if (pages_per_direction + page) >= page_count do 28 | max(0, page_count - @page_link_count)..page_count 29 | else 30 | start_page = max(0, page - pages_per_direction) 31 | end_page = min(page_count, start_page + @page_link_count - page_offset) 32 | start_page..end_page 33 | end 34 | 35 | [page: page, 36 | per_page: @articles_per_page, 37 | page_count: page_count, 38 | pages: page_range] 39 | end 40 | 41 | defp load_group(name, page, query) do 42 | group = Repo.one!(from g in Athel.Group, 43 | where: g.name == ^name) 44 | article_source = if "" != query do 45 | Athel.ArticleSearchIndex 46 | else 47 | Athel.Article 48 | end 49 | 50 | article_query = from a in article_source, 51 | join: a2g in ^(from "articles_to_groups"), on: a2g.message_id == a.message_id, 52 | where: is_nil(a.parent_message_id), 53 | where: a2g.group_name == ^name, 54 | order_by: [desc: a.date], 55 | limit: @articles_per_page, 56 | offset: ^(page * @articles_per_page), 57 | select: {a, over(count())} 58 | 59 | articles = if "" != query do 60 | article_query 61 | |> where([a], fragment("? @@ to_tsquery(?::regconfig, ?)", a.document, a.language, ^query)) 62 | |> Repo.all 63 | else 64 | Repo.all(article_query) 65 | end 66 | 67 | group = %{group | articles: Enum.map(articles, fn {article, _} -> article end)} 68 | article_count = 69 | case articles do 70 | [] -> 0 71 | [{_, count} | _] -> count 72 | end 73 | {group, article_count} 74 | end 75 | 76 | end 77 | -------------------------------------------------------------------------------- /test/models/article_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.ArticleTest do 2 | use Athel.ModelCase 3 | 4 | alias Athel.Article 5 | 6 | @valid_attrs %{message_id: "123@banana", 7 | body: "some content", 8 | content_type: "text/xml", 9 | date: Timex.now(), 10 | from: "some content", 11 | parent: nil, 12 | subject: "some content", 13 | status: "active", 14 | headers: %{"CONTENT-TYPE" => %{value: "text/xml", params: %{"CHARSET" => "UTF-8"}}}} 15 | @invalid_attrs %{} 16 | 17 | test "changeset with valid attributes" do 18 | group = setup_models() 19 | changeset = %Article{} 20 | |> Article.changeset(@valid_attrs) 21 | |> put_assoc(:groups, [group]) 22 | assert changeset.valid? 23 | end 24 | 25 | test "changeset with invalid attributes" do 26 | changeset = Article.changeset(%Article{}, @invalid_attrs) 27 | refute changeset.valid? 28 | end 29 | 30 | test "message id format" do 31 | assert_invalid(%Article{}, :message_id, "", "remove surrounding brackets") 32 | end 33 | 34 | test "string date" do 35 | changeset = Article.changeset(%Article{}, %{@valid_attrs | date: "Tue, 04 Jul 2012 04:51:23 -0500"}) 36 | assert changeset.changes[:date] == Timex.to_datetime({{2012, 7, 4}, {9, 51, 23}}, "Etc/UTC") 37 | end 38 | 39 | test "message id uniqueness" do 40 | group = setup_models() 41 | 42 | changeset = %Article{} 43 | |> Article.changeset(@valid_attrs) 44 | |> put_assoc(:groups, [group]) 45 | Repo.insert!(changeset) 46 | 47 | changeset = Article.changeset(%Article{}, @valid_attrs) 48 | assert_raise Ecto.ConstraintError, fn -> Repo.insert(changeset) end 49 | end 50 | 51 | test "parent reference" do 52 | changeset = Article.changeset(%Article{}, @valid_attrs) 53 | Repo.insert! changeset 54 | 55 | parent = Repo.one!(from a in Article, limit: 1) 56 | changeset = %Article{} 57 | |> Article.changeset(%{@valid_attrs | message_id: "fuggg@fin"}) 58 | |> put_assoc(:parent, parent) 59 | Repo.insert! changeset 60 | end 61 | 62 | test "status" do 63 | assert_invalid(%Article{}, :status, "decimated", "is invalid") 64 | end 65 | 66 | test "joins list bodies" do 67 | changeset = Article.changeset(%Article{}, %{@valid_attrs | body: ["multiline", "fantasy"]}) 68 | assert changeset.valid? 69 | assert changeset.changes[:body] == "multiline\nfantasy" 70 | end 71 | 72 | test "dedicated header field values overwrite values from %Article{:headers}" do 73 | changeset = Article.changeset(%Article{}, @valid_attrs) 74 | article = changeset |> Repo.insert!() |> Repo.preload(:groups) |> Repo.preload(:attachments) 75 | {headers, _} = Article.get_headers(article) 76 | assert headers["CONTENT-TYPE"] == "text/xml" 77 | end 78 | 79 | test "trims down a null body" do 80 | changeset = Article.changeset(%Article{}, %{@valid_attrs | body: <<0, 0, 0, 0>>}) 81 | assert changeset.changes[:body] == "" 82 | end 83 | 84 | end 85 | -------------------------------------------------------------------------------- /test/support/model_case.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.ModelCase do 2 | @moduledoc """ 3 | This module defines the test case to be used by 4 | model tests. 5 | 6 | You may define functions here to be used as helpers in 7 | your model tests. See `errors_on/2`'s definition as reference. 8 | 9 | Finally, if the test case interacts with the database, 10 | it cannot be async. For this reason, every test runs 11 | inside a transaction which is reset at the beginning 12 | of the test unless the test case is marked as async. 13 | """ 14 | 15 | use ExUnit.CaseTemplate 16 | alias Ecto.Changeset 17 | alias Athel.Repo 18 | 19 | using do 20 | quote do 21 | alias Athel.Repo 22 | 23 | import Ecto 24 | import Ecto.Changeset 25 | import Ecto.Query 26 | import Athel.ModelCase 27 | end 28 | end 29 | 30 | setup tags do 31 | :ok = Ecto.Adapters.SQL.Sandbox.checkout(Athel.Repo) 32 | 33 | unless tags[:async] do 34 | Ecto.Adapters.SQL.Sandbox.mode(Athel.Repo, {:shared, self()}) 35 | end 36 | 37 | :ok 38 | end 39 | 40 | @spec error(Changeset.t, atom) :: String.t 41 | def error(changeset, key) do 42 | {actual_message, _} = changeset.errors[key] 43 | actual_message 44 | end 45 | 46 | @spec assert_invalid(struct, atom, list | any, String.t) :: nil 47 | def assert_invalid(struct, attr, invalid_values, error_message) when is_list(invalid_values) do 48 | for value <- invalid_values do 49 | assert_invalid(struct, attr, value, error_message) 50 | end 51 | end 52 | 53 | def assert_invalid(struct, attr, invalid_value, error_message) do 54 | changeset = struct.__struct__.changeset(struct, %{attr => invalid_value}) 55 | assert error(changeset, attr) =~ error_message 56 | end 57 | 58 | def assert_invalid_format(module, attr, invalid_values) do 59 | assert_invalid(module, attr, invalid_values, "has invalid format") 60 | end 61 | 62 | def assert_too_long(module, attr, invalid_values) do 63 | assert_invalid(module, attr, invalid_values, "should be at most") 64 | end 65 | 66 | @spec setup_models(non_neg_integer) :: Athel.Group.t 67 | def setup_models(article_count \\ 0) do 68 | group = Athel.Repo.insert!( 69 | %Athel.Group{name: "fun.times", 70 | description: "Funners of the world unite", 71 | status: "y", 72 | low_watermark: 0, 73 | high_watermark: 0}) 74 | 75 | if article_count > 0 do 76 | for index <- 0..(article_count - 1) do 77 | changeset = 78 | %Athel.Article{} 79 | |> Athel.Article.changeset(%{ 80 | message_id: "0#{index}@test.com", 81 | from: "Me", 82 | subject: "Talking to myself", 83 | # second precision causes times to all line up, 84 | # ensure dates differ for ordering and that any articles created 85 | # afterwards have later dates 86 | date: Timex.now() |> DateTime.add(-(article_count * 2) + index), 87 | parent_message_id: nil, 88 | content_type: "text/plain", 89 | headers: %{}, 90 | body: "LET'S ROCK OUT FOR JESUS & AMERICA", 91 | status: "active"}) 92 | |> Changeset.put_assoc(:groups, [group]) 93 | changeset |> Athel.Repo.insert!() |> Repo.preload(:attachments) 94 | end 95 | end 96 | 97 | group 98 | end 99 | 100 | end 101 | -------------------------------------------------------------------------------- /assets/styles/app.less: -------------------------------------------------------------------------------- 1 | @import "_normalize"; 2 | 3 | @primary-color: #708090; 4 | @error-color: rgb(202, 60, 60); 5 | 6 | .unstyle-list() { 7 | padding: 0; 8 | list-style-type: none; 9 | } 10 | 11 | .flash { 12 | color: #fff; 13 | padding: 0.25em; 14 | } 15 | 16 | .flash-info { 17 | background-color: rgb(66, 184, 221); 18 | } 19 | 20 | .flash-success { 21 | background-color: rgb(28, 184, 65); 22 | } 23 | 24 | .flash-warning { 25 | background-color: rgb(223, 117, 20); 26 | } 27 | 28 | .flash-error { 29 | background-color: @error-color; 30 | } 31 | 32 | input { 33 | display: inline-block; 34 | box-shadow: inset 0 1px 3px #ddd; 35 | } 36 | 37 | .input-error { 38 | border: 2px solid @error-color; 39 | } 40 | 41 | .input-field label { 42 | display: inline-block; 43 | text-align: right; 44 | vertical-align: middle; 45 | margin-right: 1em; 46 | } 47 | 48 | .input-field input { 49 | vertical-align: middle; 50 | } 51 | 52 | // base styles 53 | 54 | a:link, a:visited, a:active { 55 | color: #777; 56 | } 57 | a:hover { 58 | color: #444; 59 | } 60 | 61 | 62 | // layout 63 | 64 | body { 65 | padding: 0 1em; 66 | display: flex; 67 | flex-flow: row; 68 | } 69 | 70 | header { 71 | flex: 1 auto; 72 | 73 | .navigation { 74 | .unstyle-list(); 75 | display: flex; 76 | align-items: stretch; 77 | flex-direction: column; 78 | } 79 | 80 | .navigation li { 81 | display: inline; 82 | font-size: 1.1em; 83 | border-right: 1px dashed @primary-color; 84 | border-bottom: 1px dashed @primary-color; 85 | padding-bottom: 0.1em; 86 | margin-bottom: 0.5em; 87 | 88 | &:hover { 89 | border-right: 1px dashed lighten(@primary-color, 30%); 90 | } 91 | 92 | a { 93 | color: @primary-color; 94 | text-decoration: none; 95 | } 96 | 97 | a:hover { 98 | color: #000; 99 | } 100 | } 101 | } 102 | 103 | main { 104 | flex: 2 75%; 105 | padding-left: 2em; 106 | } 107 | 108 | // pagination 109 | 110 | .pagination { 111 | .unstyle-list(); 112 | display: flex; 113 | 114 | li a { 115 | display: block; 116 | padding: 0.25em 0.25em; 117 | } 118 | } 119 | 120 | // groups 121 | 122 | .groups-table { 123 | td, th { 124 | padding: 0.25em; 125 | } 126 | 127 | tr:nth-child(even) { 128 | background-color: #eee; 129 | } 130 | tr:nth-child(odd) { 131 | background-color: #ddd; 132 | } 133 | // don't shade the th row 134 | tr:nth-child(1) { 135 | background-color: #fff; 136 | } 137 | } 138 | 139 | .articles { 140 | .unstyle-list(); 141 | margin-bottom: 0.5em; 142 | 143 | li { 144 | margin-bottom: 0.5em; 145 | padding-bottom: 0.25em; 146 | } 147 | 148 | li a { 149 | border-left: 1px dashed @primary-color; 150 | border-bottom: 1px dashed @primary-color; 151 | padding: 0.1em 0.1em; 152 | } 153 | } 154 | 155 | .article { 156 | border-bottom: 2px dashed @primary-color; 157 | margin-bottom: 0.5em; 158 | } 159 | 160 | .article:last-child { 161 | border-bottom: 0; 162 | margin-bottom: 0; 163 | } 164 | 165 | .article .metadata { 166 | font-weight: 600; 167 | } 168 | 169 | // taken from twilight-bright 170 | 171 | .quote-1 { 172 | color: #6b82a7; 173 | background-color: #f1f4f8; 174 | } 175 | 176 | .quote-2 { 177 | color: #cf7900; 178 | background-color: #fdf9f2; 179 | } 180 | 181 | .quote-3 { 182 | color: #5f9411; 183 | background-color: #eff8e9; 184 | } 185 | -------------------------------------------------------------------------------- /lib/athel/nntp/client.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Nntp.Client do 2 | alias Athel.Nntp.Parser 3 | require Logger 4 | 5 | @spec connect(String.t | :inet.ip_address, :inet.port_number) :: {:ok, :inet.socket} | {:error, String.t} 6 | def connect(address, port) do 7 | res = 8 | with {:ok, socket} <- :gen_tcp.connect(String.to_charlist(address), port, active: false), 9 | {:ok, _greeting} <- :gen_tcp.recv(socket, 0), 10 | do: {:ok, socket} 11 | 12 | format_error(res) 13 | end 14 | 15 | @spec list(:inet.socket, list(String.t)) :: {:ok, list(String.t)} | {:error, String.t} 16 | def list(socket, arguments \\ []) do 17 | command = 18 | if Enum.empty?(arguments) do 19 | "LIST\r\n" 20 | else 21 | "LIST #{Enum.join(arguments, " ")}\r\n" 22 | end 23 | 24 | res = 25 | with :ok <- :gen_tcp.send(socket, command), 26 | {:ok, {215, _}, body} <- read_and_parse(socket, [], &Parser.parse_code_line/2), 27 | {:ok, lines, _} <- read_and_parse(socket, body, &Parser.parse_multiline/2), 28 | do: {:ok, lines} 29 | 30 | format_error(res) 31 | end 32 | 33 | @spec set_group(:inet.socket, String.t) :: :ok | {:error, String.t} 34 | def set_group(socket, group) do 35 | res = 36 | with :ok <- :gen_tcp.send(socket, "GROUP #{group}\r\n"), 37 | {:ok, {211, _}, _} <- read_and_parse(socket, [], &Parser.parse_code_line/2), 38 | do: :ok 39 | 40 | format_error(res) 41 | end 42 | 43 | @spec xover(:inet.socket, integer) :: {:ok, []} | {:error, list(String.t)} 44 | def xover(socket, start) do 45 | res = 46 | with :ok <- :gen_tcp.send(socket, "XOVER #{start}-\r\n"), 47 | {:ok, {224, _}, body} <- read_and_parse(socket, [], &Parser.parse_code_line/2), 48 | {:ok, lines, _} <- read_and_parse(socket, body, &Parser.parse_multiline/2), 49 | do: {:ok, lines} 50 | 51 | format_error(res) 52 | end 53 | 54 | @spec get_article(:inet.socket, String.t) :: {:ok, {map(), list(String.t)}} | {:error, String.t} 55 | def get_article(socket, id) do 56 | res = 57 | with :ok <- :gen_tcp.send(socket, "ARTICLE <#{id}>\r\n"), 58 | {:ok, {220, _}, header_buffer} <- read_and_parse(socket, [], &Parser.parse_code_line/2), 59 | {:ok, headers, body_buffer} <- read_and_parse(socket, header_buffer, &Parser.parse_headers/2), 60 | {:ok, body, _} <- read_and_parse(socket, body_buffer, &Parser.parse_multiline/2), 61 | do: {:ok, {headers, body}} 62 | 63 | format_error(res) 64 | end 65 | 66 | @spec quit(:inet.socket) :: :ok | {:error, String.t} 67 | def quit(socket) do 68 | res = 69 | with :ok <- :gen_tcp.send(socket, "QUIT\r\n"), 70 | do: :gen_tcp.close(socket) 71 | 72 | format_error(res) 73 | end 74 | 75 | defp read_and_parse(socket, buffer, parser, parser_state \\ nil) do 76 | next_buffer = 77 | if "" == buffer or [] == buffer do 78 | read(socket, buffer) 79 | else 80 | buffer 81 | end 82 | 83 | if is_error(next_buffer) do 84 | next_buffer 85 | else 86 | case parser.(next_buffer, parser_state) do 87 | {:need_more, parser_state} -> read_and_parse(socket, [], parser, parser_state) 88 | other -> other 89 | end 90 | end 91 | end 92 | 93 | defp read(socket, buffer) do 94 | case :gen_tcp.recv(socket, 0, 10_000) do 95 | {:ok, received} -> [buffer, received] 96 | error -> error 97 | end 98 | end 99 | 100 | defp is_error({:error, _}), do: true 101 | defp is_error(_), do: false 102 | 103 | defp format_error(:ok), do: :ok 104 | defp format_error({:ok, {code, message}, _}) when is_number(code) do 105 | {:error, {code, message}} 106 | end 107 | defp format_error(tup) when is_tuple(tup), do: tup 108 | defp format_error(other), do: {:error, other} 109 | 110 | end 111 | -------------------------------------------------------------------------------- /test/controllers/group_controller_test.exs: -------------------------------------------------------------------------------- 1 | defmodule AthelWeb.GroupControllerTest do 2 | use AthelWeb.ConnCase 3 | alias Athel.Group 4 | alias Athel.Repo 5 | 6 | test "index no groups", %{conn: conn} do 7 | conn = get conn, "/groups" 8 | resp = html_response(conn, 200) 9 | assert resp =~ "Groups" 10 | assert resp =~ "No groups" 11 | end 12 | 13 | test "index groups", %{conn: conn} do 14 | create_group!() 15 | conn = get conn, "/groups" 16 | assert html_response(conn, 200) =~ "cool.runnings" 17 | end 18 | 19 | test "show group with no articles" do 20 | create_group!() 21 | conn = get build_conn(), "/groups/cool.runnings" 22 | resp = html_response(conn, 200) 23 | assert resp =~ "cool.runnings" 24 | assert resp =~ "No articles" 25 | end 26 | 27 | test "pagination", %{conn: conn} do 28 | Athel.ModelCase.setup_models(200) 29 | 30 | request = get conn, "/groups/fun.times" 31 | response = html_response(request, 200) 32 | for page <- 0..4 do 33 | assert response =~ "fun.times?page=#{page}" 34 | end 35 | assert response =~ "fun.times?page=13" 36 | assert count_instances(response, "Talking to myself") == 15 37 | 38 | request = get conn, "/groups/fun.times?page=13" 39 | response = html_response(request, 200) 40 | for page <- 9..13 do 41 | assert response =~ "fun.times?page=#{page}" 42 | end 43 | assert count_instances(response, "Talking to myself") == 5 44 | end 45 | 46 | test "pagination without enough articles to show all pages", %{conn: conn} do 47 | Athel.ModelCase.setup_models(32) 48 | 49 | request = get conn, "/groups/fun.times" 50 | response = html_response(request, 200) 51 | 52 | for page <- 0..2 do 53 | assert response =~ "fun.times?page=#{page}" 54 | end 55 | refute response =~ "fun.times?pages=3" 56 | 57 | request = get conn, "/groups/fun.times?page=1" 58 | response = html_response(request, 200) 59 | 60 | for page <- 0..2 do 61 | assert response =~ "fun.times?page=#{page}" 62 | end 63 | refute response =~ "fun.times?pages=3" 64 | end 65 | 66 | test "no pagination when too few articles", %{conn: conn} do 67 | Athel.ModelCase.setup_models(5) 68 | 69 | request = get conn, "/groups/fun.times" 70 | response = html_response(request, 200) 71 | 72 | refute response =~ "fun.times?page=" 73 | end 74 | 75 | test "search", %{conn: conn} do 76 | Athel.ModelCase.setup_models(5) 77 | Repo.update_all from(a in Athel.Article, where: a.message_id == "01@test.com"), 78 | set: [subject: "Asphalt"] 79 | Athel.ArticleSearchIndex.update_view() 80 | 81 | request = get conn, "/groups/fun.times?query=asphalt" 82 | response = html_response(request, 200) 83 | assert response =~ "Asphalt" 84 | assert count_instances(response, "/articles") == 1 85 | 86 | request = get conn, "/groups/fun.times?query=OREODUNK" 87 | response = html_response(request, 200) 88 | assert count_instances(response, "/articles") == 0 89 | end 90 | 91 | test "doesn't show child articles", %{conn: conn} do 92 | Athel.ModelCase.setup_models(5) 93 | Repo.update_all from(a in Athel.Article, where: a.message_id == "01@test.com" or a.message_id == "02@test.com"), 94 | set: [subject: "Asphalt"] 95 | Repo.update_all from(a in Athel.Article, where: a.message_id == "02@test.com"), 96 | set: [parent_message_id: "01@test.com"] 97 | Athel.ArticleSearchIndex.update_view() 98 | 99 | request = get conn, "/groups/fun.times" 100 | response = html_response(request, 200) 101 | assert count_instances(response, "/articles") == 4 102 | 103 | request = get conn, "/groups/fun.times?query=asphalt" 104 | response = html_response(request, 200) 105 | assert count_instances(response, "/articles") == 1 106 | end 107 | 108 | defp create_group!() do 109 | Repo.insert!(%Group{ 110 | name: "cool.runnings", 111 | description: "Get on up, it's bobsled time!", 112 | status: "y", 113 | low_watermark: 0, 114 | high_watermark: 0}) 115 | end 116 | 117 | defp count_instances(string, match) do 118 | length(String.split(string, match)) - 1 119 | end 120 | end 121 | -------------------------------------------------------------------------------- /test/nntp/formattable_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.Nntp.FormatTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Athel.Nntp.Formattable 5 | alias Athel.{Group, Article, Attachment} 6 | 7 | test "multiline multiline" do 8 | assert format(~w(cat in the hat)) == "cat\r\nin\r\nthe\r\nhat\r\n.\r\n" 9 | end 10 | 11 | test "singleline multiline" do 12 | assert format(~w(HORSE)) == "HORSE\r\n.\r\n" 13 | end 14 | 15 | test "empty multiline" do 16 | assert format([]) == ".\r\n" 17 | end 18 | 19 | test "multiline with non-binary lines" do 20 | assert format(1..5) == "1\r\n2\r\n3\r\n4\r\n5\r\n.\r\n" 21 | end 22 | 23 | test "article" do 24 | article = create_article() 25 | assert format(article) == "Content-Type: text/plain\r\nDate: Wed, 04 May 2016 03:02:01 -0500\r\nFrom: Me\r\nMessage-ID: <123@test.com>\r\nNewsgroups: fun.times,blow.away\r\nReferences: <547@heav.en>\r\nSubject: Talking to myself\r\n\r\nhow was your day?\r\nyou're too kind to ask\r\n.\r\n" 26 | end 27 | 28 | test "article without optional fields" do 29 | article = %{create_article() | parent_message_id: nil, from: nil, date: nil} 30 | assert format(article) == "Content-Type: text/plain\r\nMessage-ID: <123@test.com>\r\nNewsgroups: fun.times,blow.away\r\nSubject: Talking to myself\r\n\r\nhow was your day?\r\nyou're too kind to ask\r\n.\r\n" 31 | end 32 | 33 | test "plain attachment" do 34 | attachment = 35 | %Attachment{type: "text/plain", 36 | content: "That's a much more interesting story"} 37 | assert format(attachment) == "Content-Transfer-Encoding: base64\r\nContent-Type: text/plain\r\n\r\nVGhhdCdzIGEgbXVjaCBtb3JlIGludGVyZXN0aW5nIHN0b3J5\r\n" 38 | end 39 | 40 | test "file attachment" do 41 | attachment = 42 | %Attachment{type: "text/plain", 43 | filename: "cool.txt", 44 | content: "That's a much more interesting story"} 45 | assert format(attachment) == "Content-Disposition: attachment, filename=\"cool.txt\"\r\nContent-Transfer-Encoding: base64\r\nContent-Type: text/plain\r\n\r\nVGhhdCdzIGEgbXVjaCBtb3JlIGludGVyZXN0aW5nIHN0b3J5\r\n" 46 | end 47 | 48 | test "article with attachments" do 49 | attachments = [%Attachment{type: "text/plain", content: "Motorcycles"}, 50 | %Attachment{type: "text/plain", content: "Big rigs"}] 51 | article = %{create_article() | attachments: attachments} 52 | result = format(article) 53 | [_, boundary] = Regex.run ~r/boundary="(.*)"\r\n/m, result 54 | [_, body] = Regex.run ~r/\r\n\r\n(.*)\z/ms, result 55 | assert body == "how was your day?\r\nyou're too kind to ask\r\n\r\n--#{boundary}Content-Transfer-Encoding: base64\r\nContent-Type: text/plain\r\n\r\nTW90b3JjeWNsZXM=\r\n\r\n--#{boundary}Content-Transfer-Encoding: base64\r\nContent-Type: text/plain\r\n\r\nQmlnIHJpZ3M=\r\n\r\n--#{boundary}--\r\n.\r\n" 56 | end 57 | 58 | test "header names" do 59 | # small erlang maps have sorted entries 60 | headers = %{"MESSAGE-ID" => "123@example.com", 61 | "PARAMETERS" => %{value: "text/plain", 62 | params: %{"CHARSET" => "UTF-8", 63 | "SPACES" => "WHERE ARE\tTHEY"}}, 64 | "ARRAY" => ["alchemist", "test", "report"], 65 | "KEBAB-CASE" => "shawarma"} 66 | assert format(headers) == "Array: alchemist\r\nArray: test\r\nArray: report\r\nKebab-Case: shawarma\r\nMessage-ID: 123@example.com\r\nParameters: text/plain; charset=UTF-8; spaces=\"WHERE ARE\tTHEY\"\r\n\r\n" 67 | end 68 | 69 | defp format(formattable) do 70 | formattable |> Formattable.format |> IO.iodata_to_binary 71 | end 72 | 73 | defp create_article do 74 | groups = 75 | [%Group{name: "fun.times", 76 | status: "y", 77 | low_watermark: 0, 78 | high_watermark: 0}, 79 | %Group{name: "blow.away", 80 | status: "m", 81 | low_watermark: 0, 82 | high_watermark: 0}] 83 | 84 | %Article{message_id: "123@test.com", 85 | from: "Me", 86 | subject: "Talking to myself", 87 | date: Timex.to_datetime({{2016, 5, 4}, {3, 2, 1}}, "America/Chicago"), 88 | headers: %{}, 89 | parent_message_id: "547@heav.en", 90 | content_type: "text/plain", 91 | groups: groups, 92 | attachments: [], 93 | body: "how was your day?\nyou're too kind to ask"} 94 | end 95 | end 96 | -------------------------------------------------------------------------------- /lib/athel/scraper.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Scraper do 2 | use GenServer 3 | require Logger 4 | import Ecto.Query 5 | 6 | alias Athel.{Nntp, NntpService, Repo, Article} 7 | 8 | def start_link(foreigner = %Athel.Foreigner{}) do 9 | GenServer.start_link(__MODULE__, foreigner, []) 10 | end 11 | 12 | @impl true 13 | def init(foreigner) do 14 | {:ok, %{foreigner: foreigner, groups: [], message_id_index: nil}, {:continue, :initialize_session}} 15 | end 16 | 17 | @impl true 18 | def handle_continue(:initialize_session, state = %{foreigner: foreigner}) do 19 | {:ok, session} = connect(foreigner) 20 | groups = find_groups(session) 21 | if Enum.empty?(groups) do 22 | Nntp.Client.quit(session) 23 | Logger.warn "No common groups found at #{foreigner}" 24 | {:noreply, state} 25 | else 26 | message_id_index = find_message_id_index(session) 27 | Nntp.Client.quit(session) 28 | Logger.info "Found common groups #{inspect groups} at #{foreigner}" 29 | Process.send_after(self(), :run, 0) 30 | 31 | {:noreply, %{state | groups: groups, message_id_index: message_id_index}} 32 | end 33 | end 34 | 35 | defp find_groups(session) do 36 | local_groups = Athel.Group |> Athel.Repo.all |> Enum.map(&(&1.name)) |> MapSet.new 37 | {:ok, foreign_group_resp} = Nntp.Client.list(session) 38 | foreign_groups = foreign_group_resp 39 | |> Enum.map(fn line -> line |> String.split |> List.first end) 40 | |> MapSet.new() 41 | 42 | local_groups 43 | |> MapSet.intersection(foreign_groups) 44 | |> MapSet.to_list() 45 | end 46 | 47 | defp find_message_id_index(session) do 48 | case Nntp.Client.list(session, ["OVERVIEW.FMT"]) do 49 | {:ok, format} -> 50 | # add one to skip leading listing number in XOVER 51 | Enum.find_index(format, &("Message-ID:" == &1)) + 1 52 | _error -> 4 53 | end 54 | end 55 | 56 | @impl true 57 | def handle_info(:run, state = %{foreigner: foreigner, 58 | groups: groups, 59 | message_id_index: message_id_index}) do 60 | stream = Task.async_stream(groups, Athel.Scraper, :scrape_group, [foreigner, message_id_index], 61 | timeout: 600_000, on_timeout: :kill_task, ordered: false) 62 | results = Enum.to_list(stream) 63 | Logger.info "Finished #{foreigner}: #{inspect results}" 64 | 65 | Process.send_after(self(), :run, state.foreigner.interval) 66 | {:noreply, state} 67 | end 68 | 69 | def scrape_group(group, foreigner, message_id_index) do 70 | {:ok, session} = connect(foreigner) 71 | :ok = Nntp.Client.set_group(session, group) 72 | {:ok, overviews} = Nntp.Client.xover(session, 1) 73 | 74 | ids = extract_ids(overviews, message_id_index) 75 | id_set = MapSet.new(ids) 76 | existing_ids = MapSet.new Repo.all( 77 | from article in Article, 78 | select: article.message_id) 79 | new_ids = id_set |> MapSet.difference(existing_ids) |> MapSet.to_list() 80 | Logger.info "Found #{length new_ids} new messages for group #{group}" 81 | fetch_ids(new_ids, session, group, foreigner) 82 | end 83 | 84 | defp fetch_ids([id | rest], session, group, foreigner) do 85 | case fetch_id(session, id) do 86 | true -> fetch_ids(rest, session, group, foreigner) 87 | false -> 88 | Nntp.Client.quit(session) 89 | {:ok, new_session} = connect(foreigner) 90 | :ok = Nntp.Client.set_group(new_session, group) 91 | fetch_ids(rest, new_session, group, foreigner) 92 | end 93 | end 94 | defp fetch_ids([], session, _, _) do 95 | Nntp.Client.quit(session) 96 | end 97 | 98 | defp extract_ids(overviews, message_id_index) do 99 | Enum.map(overviews, fn line -> 100 | raw_id = line 101 | |> String.split("\t") 102 | |> Enum.at(message_id_index) 103 | if "" == raw_id do 104 | Logger.warn("Unexpected XOVER line: #{line}") 105 | else 106 | NntpService.extract_message_id(raw_id) 107 | end 108 | end) 109 | end 110 | 111 | defp fetch_id(session, id) do 112 | Logger.debug fn -> "Taking #{id}" end 113 | case Nntp.Client.get_article(session, id) do 114 | {:ok, {headers, body}} -> 115 | case NntpService.take_article(headers, body, true) do 116 | {:error, %Ecto.Changeset{:errors => errors}} -> 117 | Logger.warn "Failed to take #{id}: #{inspect errors}" 118 | {:error, reason} -> 119 | Logger.warn "Skipping #{id} due to #{inspect reason}" 120 | _ -> 121 | Logger.info "Took #{id}" 122 | end 123 | true 124 | {:error, {code, message}} when is_number(code) -> 125 | Logger.warn "Server rejected request for #{id}: #{message}" 126 | true 127 | {:error, reason} -> 128 | # Failed parse, can't recover 129 | Logger.error "Failed to parse #{id}: #{inspect reason}" 130 | false 131 | end 132 | end 133 | 134 | defp connect(foreigner) do 135 | Nntp.Client.connect(foreigner.hostname, foreigner.port) 136 | end 137 | end 138 | -------------------------------------------------------------------------------- /.credo.exs: -------------------------------------------------------------------------------- 1 | # This file contains the configuration for Credo and you are probably reading 2 | # this after creating it with `mix credo.gen.config`. 3 | # 4 | # If you find anything wrong or unclear in this file, please report an 5 | # issue on GitHub: https://github.com/rrrene/credo/issues 6 | # 7 | %{ 8 | # 9 | # You can have as many configs as you like in the `configs:` field. 10 | configs: [ 11 | %{ 12 | # 13 | # Run any exec using `mix credo -C `. If no exec name is given 14 | # "default" is used. 15 | # 16 | name: "default", 17 | # 18 | # These are the files included in the analysis: 19 | files: %{ 20 | # 21 | # You can give explicit globs or simply directories. 22 | # In the latter case `**/*.{ex,exs}` will be used. 23 | # 24 | included: ["lib/", "src/", "test/", "web/", "apps/"], 25 | excluded: [~r"/_build/", ~r"/deps/", ~r"/node_modules/"] 26 | }, 27 | # 28 | # If you create your own checks, you must specify the source files for 29 | # them here, so they can be loaded by Credo before running the analysis. 30 | # 31 | requires: [], 32 | # 33 | # If you want to enforce a style guide and need a more traditional linting 34 | # experience, you can change `strict` to `true` below: 35 | # 36 | strict: false, 37 | # 38 | # If you want to use uncolored output by default, you can change `color` 39 | # to `false` below: 40 | # 41 | color: true, 42 | # 43 | # You can customize the parameters of any check by adding a second element 44 | # to the tuple. 45 | # 46 | # To disable a check put `false` as second element: 47 | # 48 | # {Credo.Check.Design.DuplicatedCode, false} 49 | # 50 | checks: [ 51 | # 52 | ## Consistency Checks 53 | # 54 | {Credo.Check.Consistency.ExceptionNames, []}, 55 | {Credo.Check.Consistency.LineEndings, []}, 56 | {Credo.Check.Consistency.ParameterPatternMatching, []}, 57 | {Credo.Check.Consistency.SpaceAroundOperators, []}, 58 | {Credo.Check.Consistency.SpaceInParentheses, []}, 59 | {Credo.Check.Consistency.TabsOrSpaces, []}, 60 | 61 | # 62 | ## Design Checks 63 | # 64 | # You can customize the priority of any check 65 | # Priority values are: `low, normal, high, higher` 66 | # 67 | {Credo.Check.Design.AliasUsage, 68 | [priority: :low, if_nested_deeper_than: 2, if_called_more_often_than: 0]}, 69 | # You can also customize the exit_status of each check. 70 | # If you don't want TODO comments to cause `mix credo` to fail, just 71 | # set this value to 0 (zero). 72 | # 73 | {Credo.Check.Design.TagTODO, [exit_status: 2]}, 74 | {Credo.Check.Design.TagFIXME, []}, 75 | 76 | # 77 | ## Readability Checks 78 | # 79 | {Credo.Check.Readability.AliasOrder, []}, 80 | {Credo.Check.Readability.FunctionNames, []}, 81 | {Credo.Check.Readability.LargeNumbers, []}, 82 | {Credo.Check.Readability.MaxLineLength, [priority: :low, max_length: 120]}, 83 | {Credo.Check.Readability.ModuleAttributeNames, []}, 84 | {Credo.Check.Readability.ModuleDoc, []}, 85 | {Credo.Check.Readability.ModuleNames, []}, 86 | {Credo.Check.Readability.ParenthesesInCondition, []}, 87 | {Credo.Check.Readability.ParenthesesOnZeroArityDefs, []}, 88 | {Credo.Check.Readability.PredicateFunctionNames, []}, 89 | {Credo.Check.Readability.PreferImplicitTry, []}, 90 | {Credo.Check.Readability.RedundantBlankLines, []}, 91 | {Credo.Check.Readability.Semicolons, []}, 92 | {Credo.Check.Readability.SpaceAfterCommas, []}, 93 | {Credo.Check.Readability.StringSigils, []}, 94 | {Credo.Check.Readability.TrailingBlankLine, []}, 95 | {Credo.Check.Readability.TrailingWhiteSpace, []}, 96 | {Credo.Check.Readability.VariableNames, []}, 97 | 98 | # 99 | ## Refactoring Opportunities 100 | # 101 | {Credo.Check.Refactor.CondStatements, []}, 102 | {Credo.Check.Refactor.CyclomaticComplexity, []}, 103 | {Credo.Check.Refactor.FunctionArity, []}, 104 | {Credo.Check.Refactor.LongQuoteBlocks, []}, 105 | {Credo.Check.Refactor.MapInto, []}, 106 | {Credo.Check.Refactor.MatchInCondition, []}, 107 | {Credo.Check.Refactor.NegatedConditionsInUnless, []}, 108 | {Credo.Check.Refactor.NegatedConditionsWithElse, []}, 109 | {Credo.Check.Refactor.Nesting, []}, 110 | # {Credo.Check.Refactor.PipeChainStart, 111 | # [excluded_argument_types: [:atom, :binary, :fn, :keyword], excluded_functions: []]}, 112 | {Credo.Check.Refactor.UnlessWithElse, []}, 113 | 114 | # 115 | ## Warnings 116 | # 117 | {Credo.Check.Warning.BoolOperationOnSameValues, []}, 118 | {Credo.Check.Warning.ExpensiveEmptyEnumCheck, []}, 119 | {Credo.Check.Warning.IExPry, []}, 120 | {Credo.Check.Warning.IoInspect, []}, 121 | {Credo.Check.Warning.LazyLogging, []}, 122 | {Credo.Check.Warning.OperationOnSameValues, []}, 123 | {Credo.Check.Warning.OperationWithConstantResult, []}, 124 | {Credo.Check.Warning.RaiseInsideRescue, []}, 125 | {Credo.Check.Warning.UnusedEnumOperation, []}, 126 | {Credo.Check.Warning.UnusedFileOperation, []}, 127 | {Credo.Check.Warning.UnusedKeywordOperation, []}, 128 | {Credo.Check.Warning.UnusedListOperation, []}, 129 | {Credo.Check.Warning.UnusedPathOperation, []}, 130 | {Credo.Check.Warning.UnusedRegexOperation, []}, 131 | {Credo.Check.Warning.UnusedStringOperation, []}, 132 | {Credo.Check.Warning.UnusedTupleOperation, []}, 133 | 134 | # 135 | # Controversial and experimental checks (opt-in, just remove `, false`) 136 | # 137 | {Credo.Check.Consistency.MultiAliasImportRequireUse, false}, 138 | {Credo.Check.Design.DuplicatedCode, false}, 139 | {Credo.Check.Readability.Specs, false}, 140 | {Credo.Check.Refactor.ABCSize, false}, 141 | {Credo.Check.Refactor.AppendSingleItem, false}, 142 | {Credo.Check.Refactor.DoubleBooleanNegation, false}, 143 | {Credo.Check.Refactor.VariableRebinding, false}, 144 | {Credo.Check.Warning.MapGetUnsafePass, false}, 145 | {Credo.Check.Warning.UnsafeToAtom, false} 146 | 147 | # 148 | # Custom checks can be created using `mix credo.gen.check`. 149 | # 150 | ] 151 | } 152 | ] 153 | } 154 | -------------------------------------------------------------------------------- /lib/athel/models/article.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Article do 2 | use Ecto.Schema 3 | import Ecto.Changeset 4 | alias Athel.{Group, Attachment} 5 | 6 | @type t :: %__MODULE__{} 7 | 8 | @primary_key {:message_id, :string, autogenerate: false} 9 | schema "articles" do 10 | field :from, :string 11 | field :subject, :string 12 | field :date, :utc_datetime 13 | field :content_type, :string 14 | field :status, :string 15 | field :headers, :map 16 | field :body, :string 17 | 18 | many_to_many :groups, Group, 19 | join_through: "articles_to_groups", 20 | join_keys: [message_id: :message_id, group_name: :name] 21 | belongs_to :parent, __MODULE__, 22 | foreign_key: :parent_message_id, 23 | references: :message_id, 24 | type: :string 25 | many_to_many :attachments, Attachment, 26 | join_through: "attachments_to_articles", 27 | join_keys: [message_id: :message_id, attachment_id: :id] 28 | 29 | timestamps() 30 | end 31 | 32 | @date_format "%a, %d %b %Y %H:%M:%S %z" 33 | def date_format, do: @date_format 34 | 35 | def changeset(article, params \\ %{}) do 36 | article 37 | |> cast(params, [:message_id, :from, :subject, :status, :headers]) 38 | |> cast_assoc(:groups) 39 | |> cast_date(params) 40 | |> cast_content_type(params) 41 | |> cast_body(params) 42 | |> cast_assoc(:parent, required: false) 43 | |> validate_headers() 44 | |> validate_required([:subject, :date, :content_type]) 45 | |> validate_inclusion(:status, ["active", "banned"]) 46 | |> validate_length(:from, max: 255) 47 | |> validate_length(:subject, max: 255) 48 | |> validate_length(:message_id, max: 192) 49 | |> validate_format(:message_id, ~r/^[^<].*[^>]$/, message: "remove surrounding brackets") 50 | end 51 | 52 | def get_headers(article) do 53 | headers = Map.merge(article.headers, 54 | %{"NEWSGROUPS" => format_article_groups(article.groups), 55 | "MESSAGE-ID" => format_message_id(article.message_id), 56 | "REFERENCES" => format_message_id(article.parent_message_id), 57 | "FROM" => article.from, 58 | "SUBJECT" => article.subject, 59 | "DATE" => format_date(article.date)}) 60 | 61 | if Enum.empty?(article.attachments) do 62 | {headers |> Map.put("CONTENT-TYPE", article.content_type), nil} 63 | else 64 | boundary = Ecto.UUID.generate() 65 | headers = headers 66 | |> Map.put("MIME-VERSION", "1.0") 67 | |> Map.put("CONTENT-TYPE", "multipart/mixed; boundary=\"#{boundary}\"") 68 | {headers, boundary} 69 | end 70 | end 71 | 72 | 73 | defp cast_date(changeset, params) do 74 | date = params[:date] 75 | parse_value(changeset, :date, fn _ -> 76 | cond do 77 | is_binary(date) -> 78 | with {:ok, parsed_date} <- Timex.parse(date, @date_format, Timex.Parse.DateTime.Tokenizers.Strftime), do: {:ok, parsed_date |> sanitize_date()} 79 | nil == date -> 80 | {:ok, nil} 81 | true -> 82 | {:ok, date |> Timex.to_datetime() |> sanitize_date()} 83 | end 84 | end) 85 | end 86 | 87 | defp sanitize_date(date) do 88 | date 89 | |> DateTime.truncate(:second) 90 | |> Timex.Timezone.convert("Etc/UTC") 91 | end 92 | 93 | defp validate_headers(changeset) do 94 | if get_field(changeset, :headers) == nil do 95 | add_error(changeset, :headers, "is required") 96 | else 97 | changeset 98 | end 99 | end 100 | 101 | defp cast_content_type(changeset, params) do 102 | content_type = params[:content_type] 103 | parse_value(changeset, :content_type, fn _ -> 104 | case content_type do 105 | nil -> {:ok, "text/plain"} 106 | %{value: type} -> {:ok, type} 107 | type -> {:ok, type} 108 | end 109 | end) 110 | end 111 | 112 | defp cast_body(changeset, params = %{:body => body}) when is_list(body) do 113 | joined_body = Enum.join(body, "\n") 114 | cast_body(changeset, %{params | body: joined_body}) 115 | end 116 | defp cast_body(changeset, params) do 117 | body = 118 | case params[:body] do 119 | nil -> nil 120 | body -> String.trim(body, "\0") 121 | end 122 | new_body = 123 | case params[:content_type] do 124 | %{params: %{"CHARSET" => charset}} -> 125 | case map_encoding(charset) do 126 | nil -> body 127 | encoding -> 128 | Codepagex.to_string!(body, encoding) 129 | end 130 | _ -> body 131 | end 132 | 133 | new_changes = Map.put(changeset.changes, :body, new_body) 134 | %{changeset | changes: new_changes} 135 | catch 136 | e -> Ecto.Changeset.add_error(changeset, :body, Exception.message(e)) 137 | end 138 | 139 | defp map_encoding(encoding) do 140 | encoding = String.upcase(encoding) 141 | case Regex.run(~r/^ISO-8859-(\d+)$/, encoding) do 142 | [_, subtype] -> "ISO8859/8859-#{subtype}" 143 | nil -> 144 | case Regex.run(~r/^WINDOWS-12(\d{2})$/, encoding) do 145 | [_, digits] -> "VENDORS/MICSFT/WINDOWS/CP12#{digits}" 146 | nil -> nil 147 | end 148 | end 149 | end 150 | 151 | defp parse_value(changeset = %{changes: changes, errors: errors}, key, parser) do 152 | value = Map.get(changeset.changes, key, "") 153 | case parser.(value) do 154 | {:error, message} -> 155 | error = {key, {message, []}} 156 | %{changeset | errors: [error | errors], valid?: false} 157 | {:ok, new_value} -> 158 | new_changes = Map.put(changes, key, new_value) 159 | %{changeset | changes: new_changes} 160 | end 161 | end 162 | 163 | defp format_message_id(message_id) when is_nil(message_id), do: nil 164 | defp format_message_id(message_id), do: "<#{message_id}>" 165 | 166 | defp format_date(date) when is_nil(date), do: nil 167 | defp format_date(date) do 168 | Timex.format!(date, Athel.Article.date_format, :strftime) 169 | end 170 | 171 | defp format_article_groups([]), do: "" 172 | defp format_article_groups(groups) do 173 | groups 174 | |> Enum.reduce(:first, fn 175 | group, :first -> [group.name] 176 | group, acc -> [acc, ?,, group.name] 177 | end) 178 | |> IO.iodata_to_binary 179 | end 180 | 181 | end 182 | 183 | defimpl Athel.Nntp.Formattable, for: Athel.Article do 184 | alias Athel.Nntp.Formattable 185 | 186 | def format(article) do 187 | {headers, boundary} = Athel.Article.get_headers(article) 188 | split_body = String.split(article.body, "\n") 189 | body = if is_nil(boundary) do 190 | Formattable.format(split_body) 191 | else 192 | attachments = article.attachments |> Enum.map(fn attachment -> 193 | ["\r\n--", boundary, Formattable.format(attachment)] 194 | end) 195 | #NOTE: article lines won't be escaped (nested list won't pattern match), 196 | # which is ok since attachment content is always base64'd 197 | Formattable.format(split_body ++ [attachments, "--#{boundary}--"]) 198 | end 199 | [Formattable.format(headers), body] 200 | end 201 | 202 | end 203 | 204 | -------------------------------------------------------------------------------- /lib/athel/nntp/protocol.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Nntp.Protocol do 2 | @behaviour :ranch_protocol 3 | 4 | require Logger 5 | 6 | alias Athel.Nntp.{SessionHandler, Parser, Formattable} 7 | 8 | defmodule CommunicationError do 9 | defexception message: "Error while communicating with client" 10 | end 11 | 12 | defmodule State do 13 | defstruct [:transport, 14 | :socket, 15 | :buffer, 16 | :session_handler, 17 | :opts] 18 | @type t :: %State{transport: :ranch_transport, 19 | socket: :inet.socket, 20 | buffer: iodata, 21 | session_handler: pid, 22 | opts: Athel.Nntp.opts} 23 | end 24 | 25 | @spec start_link(:ranch.ref, :inet.socket, module, Athel.Nntp.opts) :: {:ok, pid} 26 | def start_link(_ref, socket, transport, opts) do 27 | # no accept_ack(ref) because connection always starts as plain tcp 28 | {:ok, session_handler} = SessionHandler.start_link() 29 | state = %State{socket: socket, 30 | transport: transport, 31 | session_handler: session_handler, 32 | buffer: [], 33 | opts: opts 34 | } 35 | pid = spawn_link(__MODULE__, :init, [state]) 36 | {:ok, pid} 37 | end 38 | 39 | @spec init(State.t) :: nil 40 | def init(state) do 41 | Logger.info "Welcoming client" 42 | send_status(state, {200, "WELCOME FRIEND"}) 43 | recv_command(state) 44 | end 45 | 46 | defp recv_command(state) do 47 | {action, buffer} = 48 | case read_and_parse(state, &Parser.parse_command/1) do 49 | {:ok, command, buffer} -> 50 | {GenServer.call(state.session_handler, command), buffer} 51 | {:error, type} -> 52 | {{:continue, {501, "Syntax error in #{type}"}}, []} 53 | end 54 | state = %{state | buffer: buffer} 55 | 56 | Logger.debug fn -> "Next NNTP protocol action is #{inspect action}" end 57 | 58 | case action do 59 | {:continue, response} -> 60 | send_status(state, response) 61 | recv_command(state) 62 | #TODO: count errors and kill connection if too many occur 63 | {:error, response} -> 64 | # we're in a bad state, clear any remaining input 65 | state = %{state | buffer: []} 66 | send_status(state, response) 67 | recv_command(state) 68 | {{:recv_article, type}, response} -> 69 | unless is_nil(response), do: send_status(state, response) 70 | recv_article(type, state) 71 | # clear remaining article out of buffer without clearing any 72 | # of the following commands 73 | {:kill_article, response} -> 74 | send_status(state, response) 75 | kill_article(state) 76 | {:start_tls, {good_response, bad_response}} -> 77 | start_tls(state, good_response, bad_response) 78 | {:quit, response} -> 79 | send_status(state, response) 80 | close(state) 81 | end 82 | end 83 | 84 | defp recv_article(type, state) do 85 | message = 86 | with {:ok, headers, buffer} <- read_and_parse(state, &Parser.parse_headers/1), 87 | {:ok, body, buffer} <- read_and_parse(%{state | buffer: buffer}, 88 | &Parser.parse_multiline/1), 89 | message_name <- "#{type}_article" |> String.to_atom, 90 | do: {{message_name, headers, body}, buffer} 91 | 92 | {response, buffer} = 93 | case message do 94 | {:error, type} -> 95 | message = "Syntax error in #{type}" 96 | Logger.debug fn -> message end 97 | {{501, message}, []} 98 | {message, buffer} -> 99 | {GenServer.call(state.session_handler, message), buffer} 100 | end 101 | 102 | send_status(state, response) 103 | recv_command(%{state | buffer: buffer}) 104 | end 105 | 106 | defp kill_article(state) do 107 | buffer = 108 | with {:ok, _, buffer} <- read_and_parse(state, &Parser.parse_headers/1), 109 | {:ok, _, buffer} <- read_and_parse(%{state | buffer: buffer}, 110 | &Parser.parse_multiline/1), 111 | do: buffer 112 | 113 | buffer = 114 | case buffer do 115 | {:error, _} -> [] 116 | buffer -> buffer 117 | end 118 | 119 | recv_command(%{state | buffer: buffer}) 120 | end 121 | 122 | defp start_tls(state = %State{transport: :ranch_tcp}, good_response, _) do 123 | send_status(state, good_response) 124 | 125 | opts = [keyfile: state.opts[:keyfile], 126 | certfile: state.opts[:certfile], 127 | cacertfile: state.opts[:cacertfile], 128 | verify: :verify_none] 129 | result = :ssl.ssl_accept(state.socket, opts, state.opts[:timeout]) 130 | case result do 131 | {:ok, socket} -> 132 | %{state | transport: :ranch_ssl, socket: socket} 133 | |> recv_command 134 | {:error, reason} -> 135 | Logger.error "Failed to accept SSL: #{inspect reason}" 136 | close(state) 137 | end 138 | end 139 | 140 | defp start_tls(state = %State{transport: :ranch_ssl}, _, bad_response) do 141 | send_status(state, bad_response) 142 | recv_command(state) 143 | end 144 | 145 | defp read_and_parse(state = %State{buffer: buffer}, parser) do 146 | buffer = 147 | case buffer do 148 | [] -> read(state) 149 | "" -> read(%{state | buffer: []}) 150 | _ -> buffer 151 | end 152 | 153 | case parser.(buffer) do 154 | :need_more -> 155 | next_state = %{state | buffer: read(state)} 156 | read_and_parse(next_state, parser) 157 | other -> other 158 | end 159 | end 160 | 161 | defp read(%State 162 | {transport: transport, 163 | socket: socket, 164 | buffer: buffer, 165 | opts: opts}) do 166 | case transport.recv(socket, 0, opts[:timeout]) do 167 | {:ok, received} -> 168 | buffer = [buffer, received] 169 | bytes_read = IO.iodata_length(buffer) 170 | if bytes_read > opts[:max_request_size] do 171 | raise CommunicationError, message: "Max request buffer length exceeded" 172 | end 173 | buffer 174 | {:error, reason} -> 175 | raise CommunicationError, message: "Failed to read from client: #{reason}" 176 | end 177 | end 178 | 179 | defp send_status(%State{transport: transport, socket: socket}, {code, message}) do 180 | case transport.send(socket, "#{code} #{message}\r\n") do 181 | :ok -> nil 182 | {:error, reason} -> 183 | raise CommunicationError, message: "Failed to send status to client: #{reason}" 184 | end 185 | end 186 | 187 | defp send_status(%State{transport: transport, socket: socket}, {code, message, body}) do 188 | body = body |> Formattable.format 189 | case transport.send(socket, "#{code} #{message}\r\n#{body}") do 190 | :ok -> nil 191 | {:error, reason} -> 192 | raise CommunicationError, message: "Failed to send status with body to client: #{reason}" 193 | end 194 | end 195 | 196 | defp close(state) do 197 | case state.transport.close(state.socket) do 198 | :ok -> nil 199 | {:error, reason} -> Logger.error "Failed to close connection: #{inspect reason}" 200 | end 201 | end 202 | 203 | end 204 | -------------------------------------------------------------------------------- /lib/athel/services/multipart.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Multipart do 2 | alias Athel.Nntp.Parser 3 | 4 | @type attachment :: %{ 5 | type: String.t, 6 | filename: String.t, 7 | params: %{optional(atom) => any}, 8 | content: binary, 9 | attachments: list(attachment)} 10 | 11 | @spec read_attachments(%{optional(String.t) => String.t}, list(String.t)) :: {:ok, list(attachment)} | {:error, atom} 12 | def read_attachments(headers, body) do 13 | mime_version = headers["MIME-VERSION"] 14 | 15 | case mime_version do 16 | "1.0" -> 17 | check_multipart_type(headers, "multipart/mixed", fn boundary -> 18 | {raw_attachments, _} = parse(body, boundary) 19 | attachments = Enum.map(raw_attachments, &cast_attachment/1) 20 | {:ok, attachments} 21 | end) 22 | nil -> 23 | {:ok, nil} 24 | _ -> 25 | {:error, :invalid_mime_version} 26 | end 27 | catch 28 | reason -> {:error, reason} 29 | end 30 | 31 | defp check_multipart_type(headers, valid_type, process) do 32 | content_type = headers["CONTENT-TYPE"] 33 | 34 | case content_type do 35 | %{value: ^valid_type, params: %{"BOUNDARY" => boundary}} -> 36 | process.(boundary) 37 | %{value: type} -> 38 | if is_multipart(type) do 39 | fail(:unhandled_multipart_type) 40 | else 41 | {:ok, nil} 42 | end 43 | type -> 44 | if is_multipart(type) do 45 | fail(:invalid_multipart_type) 46 | else 47 | {:ok, nil} 48 | end 49 | end 50 | end 51 | 52 | defp is_multipart(content_type) when is_nil(content_type), do: false 53 | defp is_multipart(content_type) do 54 | content_type =~ ~r/^multipart\// 55 | end 56 | 57 | defp cast_attachment({ 58 | %{"CONTENT-TYPE" => %{value: "multipart/signed", 59 | params: %{"MICALG" => micalg, "PROTOCOL" => protocol}}}, 60 | _, 61 | [{headers, body, []}, {_, signature, []}]}) 62 | do 63 | stripped_signature = strip_signature(signature) 64 | 65 | %{type: "multipart/signed", 66 | params: %{micalg: micalg, protocol: protocol, signature: stripped_signature}, 67 | filename: nil, 68 | content: decode_body(headers, body), 69 | attachments: []} 70 | end 71 | defp cast_attachment({%{"CONTENT-TYPE" => %{value: "multipart/signed"}}, _, _}) do 72 | fail(:invalid_multipart_signed_type) 73 | end 74 | defp cast_attachment({headers, body, attachments}) do 75 | %{type: get_type(headers), 76 | params: %{}, 77 | filename: get_filename(headers), 78 | content: decode_body(headers, body), 79 | attachments: attachments} 80 | end 81 | 82 | defp get_type(headers) do 83 | case headers["CONTENT-TYPE"] do 84 | nil -> "text/plain" 85 | %{value: type} -> type 86 | type -> type 87 | end 88 | end 89 | 90 | defp get_filename(headers) do 91 | case headers["CONTENT-DISPOSITION"] do 92 | nil -> 93 | nil 94 | %{value: type, params: %{"FILENAME" => filename}} when type in ["attachment", "form-data"] -> 95 | filename 96 | content_disposition when is_map(content_disposition) -> 97 | fail(:unhandled_content_disposition) 98 | _ -> 99 | nil 100 | end 101 | end 102 | 103 | defp decode_body(headers, body) do 104 | encoding = 105 | case Map.get(headers, "CONTENT-TRANSFER-ENCODING", "") do 106 | [encoding | _rest] -> String.upcase(encoding) 107 | encoding -> String.upcase(encoding) 108 | end 109 | 110 | case encoding do 111 | "" -> 112 | body 113 | ignore when ignore in ["7BIT", "8BIT", "BINARY", "QUOTED-PRINTABLE"] -> 114 | body 115 | "BASE64" -> 116 | deserialized = body 117 | |> Enum.join("") 118 | |> Base.decode64(body) 119 | case deserialized do 120 | {:ok, body} -> body 121 | :error -> fail(:invalid_transfer_encoding) 122 | end 123 | _ -> 124 | fail(:unhandled_transfer_encoding) 125 | end 126 | end 127 | 128 | defp strip_signature(["-----BEGIN PGP SIGNATURE-----" | rest]) do 129 | strip_signature(rest, []) 130 | end 131 | defp strip_signature(_), do: fail(:invalid_signature) 132 | defp strip_signature(["-----END PGP SIGNATURE-----" | _], acc) do 133 | Enum.reverse(acc) 134 | end 135 | defp strip_signature([next | rest], acc) do 136 | strip_signature(rest, [next | acc]) 137 | end 138 | defp strip_signature(_, _), do: throw fail(:invalid_signature) 139 | 140 | ### Parsing 141 | 142 | defp parse(lines, boundary) do 143 | next_sigil = "--#{boundary}" 144 | terminator = "--#{boundary}--" 145 | 146 | # ignore comments (the top level body) 147 | case parse_body(lines, terminator, next_sigil, []) do 148 | {:continue, _, rest} -> 149 | parse_attachments(rest, terminator, next_sigil, []) 150 | {:terminate, _, _} -> 151 | {[], nil} 152 | end 153 | end 154 | 155 | defp parse_nested(headers, body) do 156 | check_multipart_type(headers, "multipart/signed", fn boundary -> 157 | parse(body, boundary) 158 | end) 159 | end 160 | 161 | defp parse_body([], _, _, _) do 162 | fail(:unterminated_body) 163 | end 164 | defp parse_body([line | rest], terminator, _, acc) when line == terminator do 165 | {:terminate, Enum.reverse(acc), rest} 166 | end 167 | defp parse_body([line | rest], _, next_sigil, acc) when line == next_sigil do 168 | {:continue, Enum.reverse(acc), rest} 169 | end 170 | defp parse_body([line | rest], terminator, next_sigil, acc) do 171 | parse_body(rest, terminator, next_sigil, [line | acc]) 172 | end 173 | 174 | defp parse_attachments([], _, _, _) do 175 | fail(:unterminated_body) 176 | end 177 | defp parse_attachments(lines, terminator, next_sigil, attachments) do 178 | case parse_attachment(lines, terminator, next_sigil) do 179 | {:continue, attachment, rest} -> 180 | parse_attachments(rest, terminator, next_sigil, [attachment | attachments]) 181 | {:terminate, attachment, rest} -> 182 | {Enum.reverse([attachment | attachments]), rest} 183 | end 184 | end 185 | 186 | defp parse_attachment(lines, terminator, next_sigil) do 187 | {headers, rest} = read_headers(lines) 188 | 189 | case parse_nested(headers, rest) do 190 | {:ok, nil} -> 191 | case parse_body(rest, terminator, next_sigil, []) do 192 | {:continue, body, rest} -> 193 | {:continue, {headers, body, []}, rest} 194 | {:terminate, body, rest} -> 195 | {:terminate, {headers, body, []}, rest} 196 | end 197 | {attachments, rest} -> 198 | # eat deadspace until beginning of next attachment 199 | case Enum.split_while(rest, &(&1 != next_sigil)) do 200 | {_deadspace, []} -> 201 | {:terminate, {headers, [], attachments}, []} 202 | {_deadspace, [_sigil]} -> 203 | throw :unterminated_body 204 | {_deadspace, [_sigil | rest]} -> 205 | {:continue, {headers, [], attachments}, rest} 206 | end 207 | end 208 | end 209 | 210 | defp read_headers(lines, state \\ nil) 211 | defp read_headers([line | lines], state) do 212 | case Parser.parse_headers("#{line}\r\n", state) do 213 | {:ok, headers, _} -> {headers, lines} 214 | {:error, reason} -> fail(reason) 215 | {:need_more, state} -> read_headers(lines, state) 216 | end 217 | end 218 | defp read_headers([], _), do: fail(:unterminated_headers) 219 | 220 | defp fail(reason) do 221 | throw reason 222 | end 223 | 224 | end 225 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "artificery": {:hex, :artificery, "0.2.6", "f602909757263f7897130cbd006b0e40514a541b148d366ad65b89236b93497a", [:mix], [], "hexpm"}, 3 | "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"}, 4 | "certifi": {:hex, :certifi, "2.4.2", "75424ff0f3baaccfd34b1214184b6ef616d89e420b258bb0a5ea7d7bc628f7f0", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"}, 5 | "cloak": {:hex, :cloak, "0.9.2", "29503a561eb0ee47dfc0a3b1a79ee0d47283489301376dbb166dacb1211b7d5d", [:mix], [{:ecto, ">= 1.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:flow, "~> 0.14", [hex: :flow, repo: "hexpm", optional: false]}, {:pbkdf2, "~> 2.0", [hex: :pbkdf2, repo: "hexpm", optional: true]}, {:poison, ">= 1.5.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, 6 | "codepagex": {:hex, :codepagex, "0.1.4", "dae3bc57e9334c324914b32ed61c0a30929fac3e73dc71fc611ed7eeb2dcb867", [:mix], [], "hexpm"}, 7 | "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm"}, 8 | "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"}, 9 | "cowboy": {:hex, :cowboy, "2.6.1", "f2e06f757c337b3b311f9437e6e072b678fcd71545a7b2865bdaa154d078593f", [:rebar3], [{:cowlib, "~> 2.7.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, 10 | "cowlib": {:hex, :cowlib, "2.7.0", "3ef16e77562f9855a2605900cedb15c1462d76fb1be6a32fc3ae91973ee543d2", [:rebar3], [], "hexpm"}, 11 | "credo": {:hex, :credo, "1.0.0", "aaa40fdd0543a0cf8080e8c5949d8c25f0a24e4fc8c1d83d06c388f5e5e0ea42", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, 12 | "db_connection": {:hex, :db_connection, "2.0.3", "b4e8aa43c100e16f122ccd6798cd51c48c79fd391c39d411f42b3cd765daccb0", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, 13 | "decimal": {:hex, :decimal, "1.6.0", "bfd84d90ff966e1f5d4370bdd3943432d8f65f07d3bab48001aebd7030590dcc", [:mix], [], "hexpm"}, 14 | "dialyxir": {:hex, :dialyxir, "1.0.0-rc.4", "71b42f5ee1b7628f3e3a6565f4617dfb02d127a0499ab3e72750455e986df001", [:mix], [{:erlex, "~> 0.1", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm"}, 15 | "distillery": {:hex, :distillery, "2.0.12", "6e78fe042df82610ac3fa50bd7d2d8190ad287d120d3cd1682d83a44e8b34dfb", [:mix], [{:artificery, "~> 0.2", [hex: :artificery, repo: "hexpm", optional: false]}], "hexpm"}, 16 | "ecto": {:hex, :ecto, "3.0.6", "d33ab5b3f7553a41507d4b0ad5bf192d533119c4ad08f3a5d63d85aa12117dc9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, 17 | "ecto_sql": {:hex, :ecto_sql, "3.0.4", "e7a0feb0b2484b90981c56d5cd03c52122c1c31ded0b95ed213b7c5c07ae6737", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, 18 | "emagic": {:git, "https://github.com/JasonZhu/erlang_magic.git", "5f8d98ea2d7f3e62ded8c6acb160420592fe3300", []}, 19 | "erlex": {:hex, :erlex, "0.2.1", "cee02918660807cbba9a7229cae9b42d1c6143b768c781fa6cee1eaf03ad860b", [:mix], [], "hexpm"}, 20 | "ex_multihash": {:git, "https://github.com/ruhlio/ex_multihash.git", "a0763e822d554f627602ee8729a5d235f77dfb66", []}, 21 | "file_system": {:hex, :file_system, "0.2.6", "fd4dc3af89b9ab1dc8ccbcc214a0e60c41f34be251d9307920748a14bf41f1d3", [:mix], [], "hexpm"}, 22 | "flow": {:hex, :flow, "0.14.3", "0d92991fe53035894d24aa8dec10dcfccf0ae00c4ed436ace3efa9813a646902", [:mix], [{:gen_stage, "~> 0.14.0", [hex: :gen_stage, repo: "hexpm", optional: false]}], "hexpm"}, 23 | "fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], []}, 24 | "gen_stage": {:hex, :gen_stage, "0.14.1", "9d46723fda072d4f4bb31a102560013f7960f5d80ea44dcb96fd6304ed61e7a4", [:mix], [], "hexpm"}, 25 | "gettext": {:hex, :gettext, "0.16.1", "e2130b25eebcbe02bb343b119a07ae2c7e28bd4b146c4a154da2ffb2b3507af2", [:mix], [], "hexpm"}, 26 | "hackney": {:hex, :hackney, "1.15.0", "287a5d2304d516f63e56c469511c42b016423bcb167e61b611f6bad47e3ca60e", [:rebar3], [{:certifi, "2.4.2", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, 27 | "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"}, 28 | "jason": {:hex, :jason, "1.0.1", "ef108e64c6e086364b9f15b0073cf794061670af8f331d545d7308c0ba2e67f9", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, 29 | "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"}, 30 | "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"}, 31 | "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], [], "hexpm"}, 32 | "monad": {:hex, :monad, "1.0.5", "bd02263a8dad0894433ca3283ebb6f71a55799e1cd17bda1e8b2ea9e14eeb9c5", [], [], "hexpm"}, 33 | "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, 34 | "phoenix": {:hex, :phoenix, "1.4.0", "56fe9a809e0e735f3e3b9b31c1b749d4b436e466d8da627b8d82f90eaae714d2", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, 35 | "phoenix_ecto": {:hex, :phoenix_ecto, "4.0.0", "c43117a136e7399ea04ecaac73f8f23ee0ffe3e07acfcb8062fe5f4c9f0f6531", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, 36 | "phoenix_html": {:hex, :phoenix_html, "2.12.0", "1fb3c2e48b4b66d75564d8d63df6d53655469216d6b553e7e14ced2b46f97622", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, 37 | "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.2.0", "3bb31a9fbd40ffe8652e60c8660dffd72dd231efcdf49b744fb75b9ef7db5dd2", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm"}, 38 | "phoenix_pubsub": {:hex, :phoenix_pubsub, "1.1.1", "6668d787e602981f24f17a5fbb69cc98f8ab085114ebfac6cc36e10a90c8e93c", [:mix], [], "hexpm"}, 39 | "plug": {:hex, :plug, "1.7.1", "8516d565fb84a6a8b2ca722e74e2cd25ca0fc9d64f364ec9dbec09d33eb78ccd", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}], "hexpm"}, 40 | "plug_cowboy": {:hex, :plug_cowboy, "2.0.1", "d798f8ee5acc86b7d42dbe4450b8b0dadf665ce588236eb0a751a132417a980e", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, 41 | "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, 42 | "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, 43 | "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], [], "hexpm"}, 44 | "postgrex": {:hex, :postgrex, "0.14.1", "63247d4a5ad6b9de57a0bac5d807e1c32d41e39c04b8a4156a26c63bcd8a2e49", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, 45 | "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, 46 | "socket": {:git, "https://github.com/meh/elixir-socket.git", "8832c5e89b76172a4ca4d8dfbc8cb73ae8ed087b", []}, 47 | "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, 48 | "telemetry": {:hex, :telemetry, "0.3.0", "099a7f3ce31e4780f971b4630a3c22ec66d22208bc090fe33a2a3a6a67754a73", [:rebar3], [], "hexpm"}, 49 | "timex": {:hex, :timex, "3.4.2", "d74649c93ad0e12ce5b17cf5e11fbd1fb1b24a3d114643e86dba194b64439547", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, 50 | "timex_ecto": {:hex, :timex_ecto, "3.3.0", "d5bdef09928e7a60f10a0baa47ce653f29b43d6fee87b30b236b216d0e36b98d", [:mix], [{:ecto, "~> 2.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: false]}], "hexpm"}, 51 | "tzdata": {:hex, :tzdata, "0.5.19", "7962a3997bf06303b7d1772988ede22260f3dae1bf897408ebdac2b4435f4e6a", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, 52 | "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"}, 53 | } 54 | -------------------------------------------------------------------------------- /lib/athel/services/nntp_service.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.NntpService do 2 | require Logger 3 | 4 | import Ecto.Query 5 | alias Ecto.{Changeset, UUID} 6 | alias Athel.{Repo, Group, Article, Attachment, Multipart} 7 | alias Athel.Event.NntpBroadcaster 8 | 9 | @type changeset_params :: %{optional(binary) => term} | %{optional(atom) => term} 10 | @type headers :: %{optional(String.t) => String.t} 11 | @type new_article_result :: {:ok, Article.t} | {:error, Changeset.t} 12 | @type indexed_article :: {non_neg_integer, Article.t} 13 | 14 | @spec get_groups() :: list(Group.t) 15 | def get_groups do 16 | Repo.all(from g in Group, order_by: :name) 17 | end 18 | 19 | @spec get_group(String.t) :: Group.t | nil 20 | def get_group(group_name) do 21 | Repo.get_by(Group, name: group_name) 22 | end 23 | 24 | @spec get_groups_created_after(Timex.DateTime) :: list(Group.t) 25 | def get_groups_created_after(date) do 26 | Repo.all(from g in Group, 27 | where: g.inserted_at > ^date, 28 | order_by: g.inserted_at) 29 | end 30 | 31 | @spec get_article(String.t) :: Article.t | nil 32 | def get_article(message_id) do 33 | Repo.one(from a in Article, 34 | where: a.message_id == ^message_id and a.status == "active", 35 | left_join: at in assoc(a, :attachments), 36 | preload: [attachments: at], 37 | preload: [:groups]) 38 | end 39 | 40 | @spec get_articles_created_after(String.t, Timex.DateTime) :: list(Article.t) 41 | def get_articles_created_after(group_name, date) do 42 | Repo.all(from a in Article, 43 | join: g in assoc(a, :groups), 44 | left_join: at in assoc(a, :attachments), 45 | preload: [attachments: at], 46 | preload: [:groups], 47 | where: a.date > ^date and g.name == ^group_name, 48 | order_by: a.date) 49 | end 50 | 51 | @spec get_article_by_index(String.t, integer) :: indexed_article | nil 52 | def get_article_by_index(group_name, index) do 53 | group_name 54 | |> base_by_index(index) 55 | |> limit(1) 56 | |> Repo.one 57 | end 58 | 59 | @spec get_article_by_index(String.t, integer, :infinity) :: list(indexed_article) 60 | def get_article_by_index(group_name, lower_bound, :infinity) do 61 | group_name 62 | |> base_by_index(lower_bound) 63 | |> Repo.all 64 | end 65 | 66 | @spec get_article_by_index(String.t, integer, integer) :: list(indexed_article) 67 | def get_article_by_index(group_name, lower_bound, upper_bound) do 68 | count = max(upper_bound - lower_bound, 0) 69 | group_name 70 | |> base_by_index(lower_bound) 71 | |> limit(^count) 72 | |> Repo.all 73 | end 74 | 75 | defp base_by_index(group_name, lower_bound) do 76 | Article 77 | |> where(status: "active") 78 | |> join(:left, [a], at in assoc(a, :attachments)) 79 | |> preload([_a, at], attachments: at) 80 | |> preload(:groups) 81 | |> offset(^lower_bound) 82 | |> order_by(:date) 83 | # subqueries with fragments in the `select` not supported, whole query must 84 | # be a fragment to be joined on 85 | # see: https://github.com/elixir-ecto/ecto/issues/1416 86 | # make `row_number()` zero-indexed to match up with `offset` 87 | |> join(:inner, [a], i in fragment(""" 88 | SELECT (row_number() OVER (ORDER BY a.date) - 1) as index, 89 | a.message_id as message_id 90 | FROM articles a 91 | JOIN articles_to_groups a2g ON a2g.message_id = a.message_id AND a2g.group_name = ? 92 | """, ^group_name), on: i.message_id == a.message_id) 93 | |> select([a, _at, i], {i.index, a}) 94 | end 95 | 96 | @spec post_article(headers, list(String.t)) :: new_article_result 97 | def post_article(headers, body) do 98 | config = Application.fetch_env!(:athel, Athel.Nntp) 99 | hostname = config[:hostname] 100 | 101 | #TODO: user logged in user's name/email for FROM 102 | save_article(headers, body, 103 | %{message_id: generate_message_id(hostname), 104 | from: headers["FROM"], 105 | subject: headers["SUBJECT"], 106 | date: Timex.now(), 107 | content_type: headers["CONTENT-TYPE"], 108 | status: "active"}, 109 | false) 110 | end 111 | 112 | @spec take_article(headers, list(String.t)) :: new_article_result 113 | def take_article(headers, body, allow_orphan \\ false) do 114 | save_article(headers, body, 115 | %{message_id: extract_message_id(headers["MESSAGE-ID"]), 116 | date: headers["DATE"], 117 | from: headers["FROM"], 118 | subject: headers["SUBJECT"], 119 | content_type: headers["CONTENT-TYPE"], 120 | status: "active"}, 121 | allow_orphan) 122 | end 123 | 124 | @spec extract_message_id(String.t) :: String.t 125 | def extract_message_id(nil), do: nil 126 | def extract_message_id(raw_id) do 127 | String.slice(raw_id, 1, String.length(raw_id) - 2) 128 | end 129 | 130 | defp save_article(headers, body, params, allow_orphan) do 131 | with {:ok, {body, attachments}} <- read_body(headers, body) do 132 | changeset = %Article{} 133 | |> Article.changeset(params |> Map.put(:body, body) |> Map.put(:headers, headers)) 134 | |> Changeset.prepare_changes(set_groups(headers)) 135 | |> Changeset.prepare_changes(set_parent(headers, allow_orphan)) 136 | |> Changeset.prepare_changes(set_attachments(attachments)) 137 | 138 | result = Repo.insert(changeset) 139 | with {:ok, article} <- result, do: NntpBroadcaster.new_article(article) 140 | result 141 | end 142 | end 143 | 144 | defp set_groups(headers) do 145 | group_names = headers 146 | |> Map.get("NEWSGROUPS", "") 147 | |> String.split(",") 148 | |> Enum.map(&String.trim/1) 149 | 150 | fn changeset -> 151 | group_query = from g in Group, where: g.name in ^group_names 152 | groups = changeset.repo.all(group_query) 153 | 154 | cond do 155 | Enum.empty?(groups) || length(group_names) != length(groups) -> 156 | Changeset.add_error(changeset, :groups, "is invalid") 157 | Enum.any?(groups, &(&1.status == "n")) -> 158 | #TODO: move this to the base changeset? or the group changeset? 159 | Changeset.add_error(changeset, :groups, "doesn't allow posting") 160 | true -> 161 | changeset.repo.update_all(group_query, inc: [high_watermark: 1]) 162 | Changeset.put_assoc(changeset, :groups, groups) 163 | end 164 | end 165 | end 166 | 167 | defp set_parent(headers, allow_orphan) do 168 | parent_message_id = 169 | case headers["REFERENCES"] do 170 | nil -> nil 171 | references -> references 172 | |> String.split(" ") 173 | |> List.last 174 | |> extract_message_id 175 | end 176 | 177 | fn changeset -> 178 | if allow_orphan do 179 | Changeset.put_change(changeset, :parent_message_id, parent_message_id) 180 | else 181 | parent = if !is_nil(parent_message_id) do 182 | changeset.repo.get(Article, parent_message_id) 183 | end 184 | 185 | if is_nil(parent) && !is_nil(parent_message_id) do 186 | Changeset.add_error(changeset, :parent, "is invalid") 187 | else 188 | Changeset.put_assoc(changeset, :parent, parent) 189 | end 190 | end 191 | end 192 | end 193 | 194 | defp set_attachments(attachments) do 195 | config = Application.fetch_env!(:athel, Athel.Nntp) 196 | max_attachment_count = config[:max_attachment_count] 197 | 198 | fn changeset -> 199 | if length(attachments) > max_attachment_count do 200 | Changeset.add_error(changeset, :attachments, 201 | "limited to #{max_attachment_count}") 202 | else 203 | # mapping attachments by hash eliminates duplicate uploads 204 | hashed_attachments = 205 | Enum.reduce(attachments, %{}, fn attachment, acc -> 206 | {:ok, hash} = Attachment.hash_content(attachment.content) 207 | Map.put(acc, hash, attachment) 208 | end) 209 | 210 | attachments = hashed_attachments |> Enum.map(fn {hash, attachment} -> 211 | existing_attachment = changeset.repo.one(from a in Attachment, 212 | where: a.hash == ^hash) 213 | if is_nil(existing_attachment) do 214 | Attachment.changeset(%Attachment{}, attachment) 215 | else 216 | existing_attachment 217 | end 218 | end) 219 | Changeset.put_assoc(changeset, :attachments, attachments) 220 | end 221 | end 222 | end 223 | 224 | defp read_body(headers, body) do 225 | case Multipart.read_attachments(headers, body) do 226 | {:ok, nil} -> {:ok, {body, []}} 227 | {:ok, attachments} -> {:ok, split_attachments(attachments)} 228 | error -> error 229 | end 230 | end 231 | 232 | defp split_attachments([]), do: {[], []} 233 | defp split_attachments(attachments) do 234 | [first | rest] = attachments 235 | if is_nil(first.filename) do 236 | body = split_body(first.content) 237 | {body, rest |> join_attachment_contents} 238 | else 239 | {[], attachments |> join_attachment_contents} 240 | end 241 | end 242 | 243 | defp split_body(body) when is_list(body), do: body 244 | defp split_body(body) do 245 | body |> String.split(~r/(\r\n)|\n|\r/) 246 | end 247 | 248 | defp join_attachment_contents(attachments) do 249 | attachments |> Enum.map(fn attachment -> 250 | %{attachment | content: attachment.content |> join_body} 251 | end) 252 | end 253 | 254 | defp join_body(body) when is_list(body), do: Enum.join(body, "\n") 255 | defp join_body(body), do: body 256 | 257 | @spec new_topic(list(Group.t), changeset_params) :: new_article_result 258 | def new_topic(groups, params) do 259 | config = Application.fetch_env!(:athel, Athel.Nntp) 260 | hostname = config[:hostname] 261 | 262 | params = Map.merge(params, 263 | %{message_id: generate_message_id(hostname), 264 | parent: nil, 265 | date: Timex.now()}) 266 | 267 | %Article{} 268 | |> Article.changeset(params) 269 | |> Changeset.put_assoc(:groups, groups) 270 | |> Repo.insert 271 | end 272 | 273 | defp generate_message_id(hostname) do 274 | id = UUID.generate() |> String.replace("-", ".") 275 | "#{id}@#{hostname}" 276 | end 277 | 278 | end 279 | -------------------------------------------------------------------------------- /assets/styles/_normalize.less: -------------------------------------------------------------------------------- 1 | /*! normalize.css v4.2.0 | MIT License | github.com/necolas/normalize.css */ 2 | 3 | /** 4 | * 1. Change the default font family in all browsers (opinionated). 5 | * 2. Correct the line height in all browsers. 6 | * 3. Prevent adjustments of font size after orientation changes in IE and iOS. 7 | */ 8 | 9 | /* Document 10 | ========================================================================== */ 11 | 12 | html { 13 | font-family: sans-serif; /* 1 */ 14 | line-height: 1.15; /* 2 */ 15 | -ms-text-size-adjust: 100%; /* 3 */ 16 | -webkit-text-size-adjust: 100%; /* 3 */ 17 | } 18 | 19 | /* Sections 20 | ========================================================================== */ 21 | 22 | /** 23 | * Remove the margin in all browsers (opinionated). 24 | */ 25 | 26 | body { 27 | margin: 0; 28 | } 29 | 30 | /** 31 | * Add the correct display in IE 9-. 32 | */ 33 | 34 | article, 35 | aside, 36 | footer, 37 | header, 38 | nav, 39 | section { 40 | display: block; 41 | } 42 | 43 | /** 44 | * Correct the font size and margin on `h1` elements within `section` and 45 | * `article` contexts in Chrome, Firefox, and Safari. 46 | */ 47 | 48 | h1 { 49 | font-size: 2em; 50 | margin: 0.67em 0; 51 | } 52 | 53 | /* Grouping content 54 | ========================================================================== */ 55 | 56 | /** 57 | * Add the correct display in IE 9-. 58 | * 1. Add the correct display in IE. 59 | */ 60 | 61 | figcaption, 62 | figure, 63 | main { /* 1 */ 64 | display: block; 65 | } 66 | 67 | /** 68 | * Add the correct margin in IE 8. 69 | */ 70 | 71 | figure { 72 | margin: 1em 40px; 73 | } 74 | 75 | /** 76 | * 1. Add the correct box sizing in Firefox. 77 | * 2. Show the overflow in Edge and IE. 78 | */ 79 | 80 | hr { 81 | box-sizing: content-box; /* 1 */ 82 | height: 0; /* 1 */ 83 | overflow: visible; /* 2 */ 84 | } 85 | 86 | /** 87 | * 1. Correct the inheritance and scaling of font size in all browsers. 88 | * 2. Correct the odd `em` font sizing in all browsers. 89 | */ 90 | 91 | pre { 92 | font-family: monospace, monospace; /* 1 */ 93 | font-size: 1em; /* 2 */ 94 | } 95 | 96 | /* Text-level semantics 97 | ========================================================================== */ 98 | 99 | /** 100 | * 1. Remove the gray background on active links in IE 10. 101 | * 2. Remove gaps in links underline in iOS 8+ and Safari 8+. 102 | */ 103 | 104 | a { 105 | background-color: transparent; /* 1 */ 106 | -webkit-text-decoration-skip: objects; /* 2 */ 107 | } 108 | 109 | /** 110 | * Remove the outline on focused links when they are also active or hovered 111 | * in all browsers (opinionated). 112 | */ 113 | 114 | a:active, 115 | a:hover { 116 | outline-width: 0; 117 | } 118 | 119 | /** 120 | * 1. Remove the bottom border in Firefox 39-. 121 | * 2. Add the correct text decoration in Chrome, Edge, IE, Opera, and Safari. 122 | */ 123 | 124 | abbr[title] { 125 | border-bottom: none; /* 1 */ 126 | text-decoration: underline; /* 2 */ 127 | text-decoration: underline dotted; /* 2 */ 128 | } 129 | 130 | /** 131 | * Prevent the duplicate application of `bolder` by the next rule in Safari 6. 132 | */ 133 | 134 | b, 135 | strong { 136 | font-weight: inherit; 137 | } 138 | 139 | /** 140 | * Add the correct font weight in Chrome, Edge, and Safari. 141 | */ 142 | 143 | b, 144 | strong { 145 | font-weight: bolder; 146 | } 147 | 148 | /** 149 | * 1. Correct the inheritance and scaling of font size in all browsers. 150 | * 2. Correct the odd `em` font sizing in all browsers. 151 | */ 152 | 153 | code, 154 | kbd, 155 | samp { 156 | font-family: monospace, monospace; /* 1 */ 157 | font-size: 1em; /* 2 */ 158 | } 159 | 160 | /** 161 | * Add the correct font style in Android 4.3-. 162 | */ 163 | 164 | dfn { 165 | font-style: italic; 166 | } 167 | 168 | /** 169 | * Add the correct background and color in IE 9-. 170 | */ 171 | 172 | mark { 173 | background-color: #ff0; 174 | color: #000; 175 | } 176 | 177 | /** 178 | * Add the correct font size in all browsers. 179 | */ 180 | 181 | small { 182 | font-size: 80%; 183 | } 184 | 185 | /** 186 | * Prevent `sub` and `sup` elements from affecting the line height in 187 | * all browsers. 188 | */ 189 | 190 | sub, 191 | sup { 192 | font-size: 75%; 193 | line-height: 0; 194 | position: relative; 195 | vertical-align: baseline; 196 | } 197 | 198 | sub { 199 | bottom: -0.25em; 200 | } 201 | 202 | sup { 203 | top: -0.5em; 204 | } 205 | 206 | /* Embedded content 207 | ========================================================================== */ 208 | 209 | /** 210 | * Add the correct display in IE 9-. 211 | */ 212 | 213 | audio, 214 | video { 215 | display: inline-block; 216 | } 217 | 218 | /** 219 | * Add the correct display in iOS 4-7. 220 | */ 221 | 222 | audio:not([controls]) { 223 | display: none; 224 | height: 0; 225 | } 226 | 227 | /** 228 | * Remove the border on images inside links in IE 10-. 229 | */ 230 | 231 | img { 232 | border-style: none; 233 | } 234 | 235 | /** 236 | * Hide the overflow in IE. 237 | */ 238 | 239 | svg:not(:root) { 240 | overflow: hidden; 241 | } 242 | 243 | /* Forms 244 | ========================================================================== */ 245 | 246 | /** 247 | * 1. Change font properties to `inherit` in all browsers (opinionated). 248 | * 2. Remove the margin in Firefox and Safari. 249 | */ 250 | 251 | button, 252 | input, 253 | optgroup, 254 | select, 255 | textarea { 256 | font: inherit; /* 1 */ 257 | margin: 0; /* 2 */ 258 | } 259 | 260 | /** 261 | * Restore the font weight unset by the previous rule. 262 | */ 263 | 264 | optgroup { 265 | font-weight: bold; 266 | } 267 | 268 | /** 269 | * Show the overflow in IE. 270 | * 1. Show the overflow in Edge. 271 | */ 272 | 273 | button, 274 | input { /* 1 */ 275 | overflow: visible; 276 | } 277 | 278 | /** 279 | * Remove the inheritance of text transform in Edge, Firefox, and IE. 280 | * 1. Remove the inheritance of text transform in Firefox. 281 | */ 282 | 283 | button, 284 | select { /* 1 */ 285 | text-transform: none; 286 | } 287 | 288 | /** 289 | * 1. Prevent a WebKit bug where (2) destroys native `audio` and `video` 290 | * controls in Android 4. 291 | * 2. Correct the inability to style clickable types in iOS and Safari. 292 | */ 293 | 294 | button, 295 | html [type="button"], /* 1 */ 296 | [type="reset"], 297 | [type="submit"] { 298 | -webkit-appearance: button; /* 2 */ 299 | } 300 | 301 | /** 302 | * Remove the inner border and padding in Firefox. 303 | */ 304 | 305 | button::-moz-focus-inner, 306 | [type="button"]::-moz-focus-inner, 307 | [type="reset"]::-moz-focus-inner, 308 | [type="submit"]::-moz-focus-inner { 309 | border-style: none; 310 | padding: 0; 311 | } 312 | 313 | /** 314 | * Restore the focus styles unset by the previous rule. 315 | */ 316 | 317 | button:-moz-focusring, 318 | [type="button"]:-moz-focusring, 319 | [type="reset"]:-moz-focusring, 320 | [type="submit"]:-moz-focusring { 321 | outline: 1px dotted ButtonText; 322 | } 323 | 324 | /** 325 | * Change the border, margin, and padding in all browsers (opinionated). 326 | */ 327 | 328 | fieldset { 329 | border: 1px solid #c0c0c0; 330 | margin: 0 2px; 331 | padding: 0.35em 0.625em 0.75em; 332 | } 333 | 334 | /** 335 | * 1. Correct the text wrapping in Edge and IE. 336 | * 2. Correct the color inheritance from `fieldset` elements in IE. 337 | * 3. Remove the padding so developers are not caught out when they zero out 338 | * `fieldset` elements in all browsers. 339 | */ 340 | 341 | legend { 342 | box-sizing: border-box; /* 1 */ 343 | color: inherit; /* 2 */ 344 | display: table; /* 1 */ 345 | max-width: 100%; /* 1 */ 346 | padding: 0; /* 3 */ 347 | white-space: normal; /* 1 */ 348 | } 349 | 350 | /** 351 | * 1. Add the correct display in IE 9-. 352 | * 2. Add the correct vertical alignment in Chrome, Firefox, and Opera. 353 | */ 354 | 355 | progress { 356 | display: inline-block; /* 1 */ 357 | vertical-align: baseline; /* 2 */ 358 | } 359 | 360 | /** 361 | * Remove the default vertical scrollbar in IE. 362 | */ 363 | 364 | textarea { 365 | overflow: auto; 366 | } 367 | 368 | /** 369 | * 1. Add the correct box sizing in IE 10-. 370 | * 2. Remove the padding in IE 10-. 371 | */ 372 | 373 | [type="checkbox"], 374 | [type="radio"] { 375 | box-sizing: border-box; /* 1 */ 376 | padding: 0; /* 2 */ 377 | } 378 | 379 | /** 380 | * Correct the cursor style of increment and decrement buttons in Chrome. 381 | */ 382 | 383 | [type="number"]::-webkit-inner-spin-button, 384 | [type="number"]::-webkit-outer-spin-button { 385 | height: auto; 386 | } 387 | 388 | /** 389 | * 1. Correct the odd appearance in Chrome and Safari. 390 | * 2. Correct the outline style in Safari. 391 | */ 392 | 393 | [type="search"] { 394 | -webkit-appearance: textfield; /* 1 */ 395 | outline-offset: -2px; /* 2 */ 396 | } 397 | 398 | /** 399 | * Remove the inner padding and cancel buttons in Chrome and Safari on OS X. 400 | */ 401 | 402 | [type="search"]::-webkit-search-cancel-button, 403 | [type="search"]::-webkit-search-decoration { 404 | -webkit-appearance: none; 405 | } 406 | 407 | /** 408 | * 1. Correct the inability to style clickable types in iOS and Safari. 409 | * 2. Change font properties to `inherit` in Safari. 410 | */ 411 | 412 | ::-webkit-file-upload-button { 413 | -webkit-appearance: button; /* 1 */ 414 | font: inherit; /* 2 */ 415 | } 416 | 417 | /* Interactive 418 | ========================================================================== */ 419 | 420 | /* 421 | * Add the correct display in IE 9-. 422 | * 1. Add the correct display in Edge, IE, and Firefox. 423 | */ 424 | 425 | details, /* 1 */ 426 | menu { 427 | display: block; 428 | } 429 | 430 | /* 431 | * Add the correct display in all browsers. 432 | */ 433 | 434 | summary { 435 | display: list-item; 436 | } 437 | 438 | /* Scripting 439 | ========================================================================== */ 440 | 441 | /** 442 | * Add the correct display in IE 9-. 443 | */ 444 | 445 | canvas { 446 | display: inline-block; 447 | } 448 | 449 | /** 450 | * Add the correct display in IE. 451 | */ 452 | 453 | template { 454 | display: none; 455 | } 456 | 457 | /* Hidden 458 | ========================================================================== */ 459 | 460 | /** 461 | * Add the correct display in IE 10-. 462 | */ 463 | 464 | [hidden] { 465 | display: none; 466 | } 467 | -------------------------------------------------------------------------------- /test/nntp/parser_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.Nntp.ParserTest do 2 | # credo:disable-for-this-file Credo.Check.Consistency.TabsOrSpaces 3 | use ExUnit.Case, async: true 4 | 5 | import Athel.Nntp.Parser 6 | 7 | test "code and multiline" do 8 | input = "215 Order of fields in overview database\r\nSubject:\r\nFrom:\r\nDate:\r\nMessage-ID:\r\nReferences:\r\nBytes:\r\nLines:\r\nXref:full\r\n.\r\n" 9 | {:ok, {215, "Order of fields in overview database"}, rest} = parse_code_line(input) 10 | assert {:ok, ["Subject:", "From:", "Date:", "Message-ID:", "References:", "Bytes:", "Lines:", "Xref:full"], ""} == parse_multiline(rest) 11 | end 12 | 13 | test "valid code line" do 14 | assert parse_code_line(["203 all clear", "\r\n"]) == {:ok, {203, "all clear"}, ""} 15 | end 16 | 17 | test "incomplete newline" do 18 | {:need_more, {:line, acc, 391}} = parse_code_line("391 HEYO\r") 19 | assert IO.iodata_to_binary(acc) == "HEYO\r" 20 | end 21 | 22 | test "invalid newline" do 23 | assert parse_code_line("404 SMELL YA LATER\r\a") == {:error, :line} 24 | assert parse_code_line("404 SMELL YA LATER\n") == {:error, :line} 25 | end 26 | 27 | test "newline split across inputs" do 28 | {:need_more, good_state} = parse_code_line("404 WHADDUP\r") 29 | {:need_more, bad_state} = parse_code_line("404 WHADDUP") 30 | assert parse_code_line("\nmore", good_state) == {:ok, {404, "WHADDUP"}, "more"} 31 | assert parse_code_line("\nmore", bad_state) == {:error, :line} 32 | end 33 | 34 | test "code too short" do 35 | assert parse_code_line("31 GET ME") == {:error, :code} 36 | end 37 | 38 | test "code too long" do 39 | assert parse_code_line("3124 GIRAFFE NECK") == {:error, :code} 40 | end 41 | 42 | test "truncated code" do 43 | {:need_more, {:code, {acc, count}}} = parse_code_line("12") 44 | assert IO.iodata_to_binary(acc) == "12" 45 | assert count == 2 46 | end 47 | 48 | test "truncated line" do 49 | {:need_more, {:line, acc, code}} = parse_code_line("123 i could just") 50 | assert code == 123 51 | assert IO.iodata_to_binary(acc) == "i could just" 52 | end 53 | 54 | test "non-numerical code" do 55 | assert parse_code_line("!@# ok") == {:error, :code} 56 | end 57 | 58 | test "valid multiline" do 59 | multiline = parse_multiline(["hey\r\nthere\r\nparty\r\npeople\r\n", ".\r\n"]) 60 | assert multiline == {:ok, ~w(hey there party people), ""} 61 | end 62 | 63 | test "valid escaped multiline" do 64 | multiline = parse_multiline("i put periods\r\n.. at the beginning\r\n.. of my lines\r\n.\r\n") 65 | assert multiline == {:ok, ["i put periods", ". at the beginning", ". of my lines"], ""} 66 | end 67 | 68 | test "unescaped multiline" do 69 | assert parse_multiline("I DO\r\n.WHAT I WANT\r\n.\r\n") == {:error, :multiline} 70 | end 71 | 72 | test "unterminated multiline" do 73 | {:need_more, {line_acc, acc}} = parse_multiline("I SMELL LONDON\r\nI SMELL FRANCE\r\nI SMELL AN UNTERMINATED MULTILINE\r\nwut") 74 | assert IO.iodata_to_binary(line_acc) == "wut" 75 | assert acc == ["I SMELL AN UNTERMINATED MULTILINE", "I SMELL FRANCE", "I SMELL LONDON"] 76 | end 77 | 78 | test "valid headers" do 79 | headers = parse_headers(["Content-Type: funky/nasty\r\nBoogie-Nights: You missed that boat\r\n", "\r\n"]) 80 | assert headers == {:ok, 81 | %{"CONTENT-TYPE" => "funky/nasty", 82 | "BOOGIE-NIGHTS" => "You missed that boat"}, 83 | ""} 84 | end 85 | 86 | test "multiline duplicate headers" do 87 | input = "References: \r 88 | \r 89 | \r 90 | \r 91 | Original-Received: from quimby.gnus.org ([80.91.224.244])\r 92 | by main.gmane.org with esmtp (Exim 3.35 #1 (Debian))\r 93 | id 1827JK-0006HD-00\r 94 | for ; Thu, 17 Oct 2002 11:51:22 +0200\r 95 | Original-Received: from hawk.netfonds.no ([80.91.224.246])\r 96 | by quimby.gnus.org with esmtp (Exim 3.12 #1 (Debian))\r 97 | id 1828BC-0005rd-00\r 98 | for ; Thu, 17 Oct 2002 12:47:02 +0200\r\n\r\n" 99 | 100 | assert parse_headers(input) == {:ok, 101 | %{"REFERENCES" => " ", 102 | "ORIGINAL-RECEIVED" => [ 103 | "from hawk.netfonds.no ([80.91.224.246]) by quimby.gnus.org with esmtp (Exim 3.12 #1 (Debian)) id 1828BC-0005rd-00 for ; Thu, 17 Oct 2002 12:47:02 +0200", 104 | "from quimby.gnus.org ([80.91.224.244]) by main.gmane.org with esmtp (Exim 3.35 #1 (Debian)) id 1827JK-0006HD-00 for ; Thu, 17 Oct 2002 11:51:22 +0200" 105 | ] 106 | }, ""} 107 | end 108 | 109 | test "header value with parameters" do 110 | result = parse_headers( 111 | ["Content-Type: attachment; boundary=hearsay; pants=off\r\n", 112 | "\r\n"]) 113 | assert result == {:ok, 114 | %{"CONTENT-TYPE" => %{value: "attachment", params: %{"BOUNDARY" => "hearsay", 115 | "PANTS" => "off"}}}, 116 | ""} 117 | end 118 | 119 | test "header with params followed other headers" do 120 | result = parse_headers( 121 | ["Content-Type: text/plain; charset=utf-8\r\n", 122 | "Subject: None\r\n", 123 | "Kind: of\r\n", 124 | "\r\n"]) 125 | assert result == {:ok, 126 | %{"CONTENT-TYPE" => %{value: "text/plain", params: %{"CHARSET" => "utf-8"}}, 127 | "SUBJECT" => "None", 128 | "KIND" => "of"}, 129 | ""} 130 | end 131 | 132 | test "header value with delimited parameter" do 133 | result = parse_headers( 134 | ["Content-Type: attachment; boundary= \"hearsay\" ; unnecessary=\t\"unfortunately\" \r\n", 135 | "\r\n"]) 136 | assert result == {:ok, 137 | %{"CONTENT-TYPE" => %{value: "attachment", 138 | params: %{"BOUNDARY" => "hearsay", 139 | "UNNECESSARY" => "unfortunately"}}}, 140 | ""} 141 | 142 | result = parse_headers( 143 | ["Content-Type: form-data; boundary=keep_this\"\r\n", 144 | "\r\n"]) 145 | assert result == {:ok, 146 | %{"CONTENT-TYPE" => %{value: "form-data", 147 | params: %{"BOUNDARY" => "keep_this\""}}}, 148 | ""} 149 | end 150 | 151 | test "header with params split across lines" do 152 | result = parse_headers("Content-Type: multipart/signed; boundary=\"=-=-=\";\r\n\tmicalg=pgp-sha1; protocol=\"application/pgp-signature\"\r\n\r\n") 153 | assert result == {:ok, 154 | %{"CONTENT-TYPE" => %{value: "multipart/signed", 155 | params: %{"BOUNDARY" => "=-=-=", 156 | "MICALG" => "pgp-sha1", 157 | "PROTOCOL" => "application/pgp-signature"}}}, 158 | ""} 159 | end 160 | 161 | test "header with trailing spaces before params, and params starting on next line on next read" do 162 | {:need_more, state} = parse_headers("Content-Type: multipart/alternative; \r\n") 163 | {:ok, _, _} = parse_headers( 164 | "\tboundary=\"----=_Part_5262_18691742.1203610743730\"\r\n\r\n", state) 165 | end 166 | 167 | test "header with param names split across lines & reads" do 168 | {:need_more, state} = parse_headers("Content-Type: multipart/alternative; chars\r\n") 169 | {:need_more, next_state} = parse_headers("\tet=utf8; LESS=chess; \r\n", state) 170 | {:ok, headers, _} = parse_headers("\tmore=params\r\n\r\n", next_state) 171 | assert headers == %{"CONTENT-TYPE" => %{value: "multipart/alternative", 172 | params: %{"CHARSET" => "utf8", 173 | "LESS" => "chess", 174 | "MORE" => "params"}}} 175 | end 176 | 177 | #TODO 178 | # test "header with param values split across lines" do 179 | # {:ok, headers, _} = parse_headers("Content-Type: multipart/alternative; charset=\"utf\r\n\t8\"; LES\r\n\tS=\"ches\r\n\ts\"\r\n\r\n") 180 | # assert headers == %{"CONTENT-TYPE" => {"multipart/alternative", 181 | # %{"CHARSET" => "utf8", 182 | # "LESS" => "chess"}}} 183 | # end 184 | 185 | test "header values with invalid characters" do 186 | {:ok, headers, _} = "test/nntp/scandavian_org.txt" 187 | |> File.stream!() 188 | |> Enum.to_list() 189 | |> parse_headers() 190 | assert headers["ORGANIZATION"] == "Erzs�bet the Vampire" 191 | end 192 | 193 | test "unterminated header entry parameters" do 194 | {:need_more, _} = parse_headers("Content-Type: attachment; boundary") 195 | {:need_more, _} = parse_headers("Content-Type: attachment; boundary=nope;") 196 | end 197 | 198 | test "prematurely terminated header param" do 199 | # could complete on next line 200 | {:need_more, _} = parse_headers("Content-Type: attachment; boundary\r\n") 201 | end 202 | 203 | test "header name with whitespace" do 204 | assert parse_headers("OH YEAH: BROTHER\r\n\r\n") == {:error, :header_name} 205 | end 206 | 207 | test "unterminated headers" do 208 | {:need_more, state} = parse_headers("this-train: is off the tracks\r\n") 209 | assert state.headers == %{"THIS-TRAIN" => "is off the tracks"} 210 | end 211 | 212 | test "newline terminated header name" do 213 | assert parse_headers("i-just-cant-seem-to-ever-shut-my-piehole\r\n") == {:error, :header_name} 214 | end 215 | 216 | test "unterminated header name" do 217 | {:need_more, _} = parse_headers("just-must-fuss") 218 | end 219 | 220 | test "unterminated header value" do 221 | {:need_more, _} = parse_headers("welcome: to the danger zone") 222 | end 223 | 224 | test "valid command with arguments" do 225 | command = parse_command("ADULT supervision is required\r\nsomething") 226 | assert command == {:ok, {"ADULT", ~w(supervision is required)}, "something"} 227 | end 228 | 229 | test "valid command without arguments" do 230 | command = parse_command("WUT\r\n") 231 | assert command == {:ok, {"WUT", []}, ""} 232 | end 233 | 234 | test "command name is upcased" do 235 | assert parse_command("AsDf\r\n") == {:ok, {"ASDF", []}, ""} 236 | end 237 | 238 | test "no command" do 239 | command = parse_command("\r\n") 240 | assert command == {:error, :command} 241 | end 242 | 243 | test "incomplete newline in command" do 244 | result = parse_command("HAL\nP MY NEW\rLINES\r\n") 245 | assert result == {:error, :command} 246 | end 247 | 248 | test "unterminated command" do 249 | {:need_more, good_state} = parse_command("MARKET FRESH HORSEMEAT\r") 250 | {:need_more, bad_state} = parse_command("MARKET FRESH HORSEMEAT") 251 | {command, [arg0], _} = good_state 252 | assert IO.iodata_to_binary(command) == "MARKET" 253 | assert IO.iodata_to_binary(arg0) == "FRESH" 254 | 255 | assert {:error, :command} == parse_command("\n", bad_state) 256 | assert {:ok, {"MARKET", ["FRESH", "HORSEMEAT"]}, "uh"} == parse_command("\nuh", good_state) 257 | end 258 | 259 | test "unterminated command without arguments" do 260 | {:need_more, {[], [], acc}} = parse_command("ants") 261 | assert IO.iodata_to_binary(acc) == "ants" 262 | end 263 | 264 | end 265 | -------------------------------------------------------------------------------- /test/services/nntp_service_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.NntpServiceTest do 2 | # 3 | use Athel.ModelCase 4 | 5 | import Athel.NntpService 6 | alias Athel.{Group, Article, Attachment} 7 | 8 | test "get groups" do 9 | setup_models() 10 | Repo.insert! %Group{ 11 | name: "dude", 12 | description: "what", 13 | low_watermark: 0, 14 | high_watermark: 0, 15 | status: "y" 16 | } 17 | 18 | assert get_groups() |> Enum.map(&(&1.name)) == ["dude", "fun.times"] 19 | end 20 | 21 | test "get group" do 22 | setup_models() 23 | 24 | assert get_group("COSTANZA") == nil 25 | assert %Group{name: "fun.times"} = get_group("fun.times") 26 | end 27 | 28 | test "get article" do 29 | setup_models(2) 30 | 31 | article = Repo.get!(Article, "00@test.com") 32 | changeset = change(article, status: "banned") 33 | Repo.update!(changeset) 34 | assert get_article("asd") == nil 35 | assert get_article("01@test.com").message_id == "01@test.com" 36 | end 37 | 38 | test "get article by index" do 39 | group = setup_models(5) 40 | 41 | {index, article} = get_article_by_index(group.name, 2) 42 | assert {index, article.message_id} == {2, "02@test.com"} 43 | 44 | articles = get_article_by_index(group.name, 2, :infinity) 45 | assert message_ids(articles) == [ 46 | {2, "02@test.com"}, 47 | {3, "03@test.com"}, 48 | {4, "04@test.com"} 49 | ] 50 | 51 | articles = get_article_by_index(group.name, 2, 4) 52 | assert message_ids(articles) == [ 53 | {2, "02@test.com"}, 54 | {3, "03@test.com"} 55 | ] 56 | 57 | articles = get_article_by_index(group.name, 7, 5) 58 | assert articles == [] 59 | 60 | group = Repo.update! Group.changeset(group, %{low_watermark: 0}) 61 | article = Repo.get!(Article, "02@test.com") 62 | changeset = change(article, status: "banned") 63 | Repo.update!(changeset) 64 | assert group.name |> get_article_by_index(0, :infinity) |> message_ids == [ 65 | {0, "00@test.com"}, 66 | {1, "01@test.com"}, 67 | {3, "03@test.com"}, 68 | {4, "04@test.com"} 69 | ] 70 | end 71 | 72 | test "post" do 73 | group = setup_models() 74 | 75 | {:error, changeset} = post_article(%{"NEWSGROUPS" => "heyo", "SUBJECT" => "nothing"}, []) 76 | assert error(changeset, :groups) == "is invalid" 77 | 78 | {:error, changeset} = post_article(%{"SUBJECT" => "nothing"}, []) 79 | assert error(changeset, :groups) == "is invalid" 80 | 81 | {:error, changeset} = post_article(%{"REFERENCES" => "nothing", "SUBJECT" => "nothing"}, []) 82 | assert error(changeset, :parent) == "is invalid" 83 | 84 | headers = %{ 85 | "FROM" => "Triple One", 86 | "SUBJECT" => "Colors", 87 | "CONTENT-TYPE" => "text/plain", 88 | "NEWSGROUPS" => "fun.times" 89 | } 90 | body = ["All I see are these colors", "we walk with distant lovers", "but really what is it all to me"] 91 | {:ok, posted_article} = post_article(headers, body) 92 | 93 | article = Article |> Repo.get(posted_article.message_id) |> Repo.preload(:groups) 94 | assert article.message_id =~ ~r/example.com$/ 95 | assert article.from == headers["FROM"] 96 | assert article.subject == headers["SUBJECT"] 97 | assert article.content_type == headers["CONTENT-TYPE"] 98 | assert article.body == Enum.join(body, "\n") 99 | assert (article.groups |> Enum.map(&(&1.name))) == [group.name] 100 | end 101 | 102 | test "post with references list" do 103 | group = setup_models(5) 104 | 105 | headers = %{ 106 | "FROM" => "geriatrics", 107 | "SUBJECT" => "Water Polo", 108 | "CONTENT-TYPE" => "text/plain", 109 | "NEWSGROUPS" => group.name, 110 | "REFERENCES" => "<01@test.com> <02@test.com> <03@test.com>" 111 | } 112 | body = ["Too cheap for a horse"] 113 | {:ok, posted_article} = post_article(headers, body) 114 | assert posted_article.parent_message_id == "03@test.com" 115 | end 116 | 117 | test "post with non-UTF-8 body encoding" do 118 | setup_models() 119 | 120 | headers = %{ 121 | "SUBJECT" => "Re: idea for GMane: treat RSS feeds as mailing lists 122 | ", 123 | "NEWSGROUPS" => "fun.times", 124 | #TODO: decode characterset here also 125 | "FROM" => "asjo@koldfront.dk (Adam =?iso-8859-1?Q?Sj=F8gren?=)", 126 | "CONTENT-TYPE" => %{value: "text/plain", params: %{"CHARSET" => "iso-8859-1"}} 127 | } 128 | body = File.stream!("test/services/danish_body.txt") |> Enum.to_list 129 | 130 | {:ok, article} = post_article(headers, body) 131 | assert article.body == 132 | " \"You have to photosynthesize\" Adam Sjøgren\n\n asjo@koldfront.dk\n" 133 | end 134 | 135 | test "post with attachments" do 136 | setup_models() 137 | 138 | headers = 139 | %{"SUBJECT" => "gnarly", 140 | "FROM" => "Chef Mandude ", 141 | "NEWSGROUPS" => "fun.times", 142 | "MIME-VERSION" => "1.0", 143 | "CONTENT-TYPE" => %{value: "multipart/mixed", params: %{"BOUNDARY" => "surfsup"}}} 144 | body = 145 | ["", 146 | "--surfsup", 147 | "Content-Transfer-Encoding: base64", 148 | "Content-Disposition: attachment; filename=\"phatwave.jpg\"", 149 | "", 150 | "TkFVR0hU", 151 | "--surfsup--"] 152 | {:ok, article} = post_article(headers, body) 153 | 154 | assert article.body == "" 155 | [attachment] = article.attachments 156 | assert attachment.filename == "phatwave.jpg" 157 | assert attachment.content == "NAUGHT" 158 | end 159 | 160 | test "post takes first attachment content as body if it is text without filename" do 161 | setup_models() 162 | 163 | headers = 164 | %{"SUBJECT" => "gnarly", 165 | "FROM" => "Chef Mandude ", 166 | "NEWSGROUPS" => "fun.times", 167 | "MIME-VERSION" => "1.0", 168 | "CONTENT-TYPE" => %{value: "multipart/mixed", params: %{"BOUNDARY" => "surfsup"}}} 169 | single_attachment_body = 170 | ["--surfsup", 171 | "Content-Transfer-Encoding: base64", 172 | "", 173 | "Q2FuJ3QgZ2V0IG15DQpsaW5lIGVuZGluZ3MKY29uc2lzdGVudA1pIHF1aXQ=", 174 | "--surfsup--"] 175 | multi_attachment_body = 176 | ["--surfsup", 177 | "Content-Transfer-Encoding: base64", 178 | "", 179 | "Q2FuJ3QgZ2V0IG15DQpsaW5lIGVuZGluZ3MKY29uc2lzdGVudA1pIHF1aXQ=", 180 | "--surfsup", 181 | "Content-Transfer-Encoding: base64", 182 | "Content-Disposition: attachment; filename=\"turbo_killer.gif\"", 183 | "", 184 | "c2htb2tpbic=", 185 | "--surfsup--"] 186 | {:ok, single_attachment_article} = post_article(headers, single_attachment_body) 187 | {:ok, multi_attachment_article} = post_article(headers, multi_attachment_body) 188 | 189 | assert single_attachment_article.body == "Can't get my\nline endings\nconsistent\ni quit" 190 | assert multi_attachment_article.body == single_attachment_article.body 191 | 192 | assert single_attachment_article.attachments == [] 193 | [attachment] = multi_attachment_article.attachments 194 | assert attachment.filename == "turbo_killer.gif" 195 | assert attachment.content == "shmokin'" 196 | end 197 | 198 | test "post with too many attachments/plaintext attachments handling" do 199 | setup_models() 200 | 201 | headers = 202 | %{"SUBJECT" => "gnarly", 203 | "FROM" => "Chef Mandude ", 204 | "NEWSGROUPS" => "fun.times", 205 | "MIME-VERSION" => "1.0", 206 | "CONTENT-TYPE" => %{value: "multipart/mixed", params: %{"BOUNDARY" => "surfsup"}}} 207 | body = 208 | ["--surfsup", 209 | "Content-Type: text/plain", 210 | "", 211 | "body", 212 | "--surfsup", 213 | "Content-Type: text/plain", 214 | "", 215 | "one", 216 | "--surfsup", 217 | "Content-Type: text/plain", 218 | "", 219 | "two", 220 | "--surfsup", 221 | "Content-Type: text/plain", 222 | "", 223 | "three", 224 | "--surfsup", 225 | "Content-Type: text/plain", 226 | "", 227 | "four", 228 | "--surfsup--"] 229 | 230 | {:error, changeset} = post_article(headers, body) 231 | assert changeset.errors[:attachments] == {"limited to 3", []} 232 | end 233 | 234 | test "merges identical attachments" do 235 | setup_models() 236 | 237 | headers = 238 | %{"SUBJECT" => "gnarly", 239 | "FROM" => "Chef Mandude ", 240 | "NEWSGROUPS" => "fun.times", 241 | "MIME-VERSION" => "1.0", 242 | "CONTENT-TYPE" => %{value: "multipart/mixed", params: %{"BOUNDARY" => "surfsup"}}} 243 | 244 | body = 245 | ["--surfsup", 246 | "Content-Transfer-Encoding: base64", 247 | "Content-Disposition: attachment; filename=\"turbo_killer.gif\"", 248 | "", 249 | "Q2FuJ3QgZ2V0IG15DQpsaW5lIGVuZGluZ3MKY29uc2lzdGVudA1pIHF1aXQ=", 250 | "--surfsup", 251 | "Content-Transfer-Encoding: base64", 252 | "Content-Disposition: attachment; filename=\"turbo_killer.gif\"", 253 | "", 254 | "Q2FuJ3QgZ2V0IG15DQpsaW5lIGVuZGluZ3MKY29uc2lzdGVudA1pIHF1aXQ=", 255 | "--surfsup--"] 256 | body_redux = 257 | ["--surfsup", 258 | "Content-Transfer-Encoding: base64", 259 | "Content-Disposition: attachment; filename=\"turbo_killer.gif\"", 260 | "", 261 | "Q2FuJ3QgZ2V0IG15DQpsaW5lIGVuZGluZ3MKY29uc2lzdGVudA1pIHF1aXQ=", 262 | "--surfsup--"] 263 | 264 | {:ok, _} = post_article(headers, body) 265 | {:ok, _} = post_article(headers, body_redux) 266 | 267 | assert 1 == Repo.one(from a in Attachment, select: count(a.id)) 268 | end 269 | 270 | test "take" do 271 | setup_models() 272 | 273 | date = Timex.to_datetime({{2012, 7, 4}, {10, 51, 23}}, "Etc/UTC") 274 | headers = %{ 275 | "MESSAGE-ID" => "", 276 | "DATE" => "Tue, 04 Jul 2012 04:51:23 -0600", 277 | "FROM" => "ur mum", 278 | "SUBJECT" => "heehee", 279 | "CONTENT-TYPE" => "text/plain", 280 | "NEWSGROUPS" => "fun.times" 281 | } 282 | body = ["brass monkey"] 283 | {:ok, taken_article} = take_article(headers, body) 284 | 285 | assert taken_article.date == date 286 | assert taken_article.message_id == "not@really.here" 287 | end 288 | 289 | test "get new groups" do 290 | Repo.insert!(%Group { 291 | name: "old.timer", 292 | description: "I can smell the graveworms", 293 | low_watermark: 0, 294 | high_watermark: 0, 295 | status: "y", 296 | inserted_at: ~N[1969-12-31 23:59:59]}) 297 | Repo.insert!(%Group { 298 | name: "young.whippersnapper", 299 | description: "I wanna be you fetish queen", 300 | low_watermark: 0, 301 | high_watermark: 0, 302 | status: "y", 303 | inserted_at: ~N[2012-03-04 05:55:55]}) 304 | assert ~N[2010-04-05 22:22:22] 305 | |> get_groups_created_after() 306 | |> Enum.map(&(&1.name)) == ["young.whippersnapper"] 307 | end 308 | 309 | test "get new articles" do 310 | group = setup_models() 311 | 312 | articles = [ 313 | %{message_id: "good@old.boys", date: ~N[1969-12-31 23:59:59]}, 314 | %{message_id: "cherry@burger.pies", date: ~N[2012-03-04 05:55:55]} 315 | ] 316 | for article <- articles do 317 | changeset = %Article{} 318 | |> Article.changeset(Map.merge(article, 319 | %{from: "whoever", 320 | subject: "whatever", 321 | body: ["however"], 322 | content_type: "text/plain", 323 | headers: %{}, 324 | status: "active"})) 325 | |> put_assoc(:groups, [group]) 326 | Repo.insert!(changeset) 327 | end 328 | 329 | assert get_articles_created_after("fun.times", ~N[2010-04-05 22:22:22]) 330 | |> Enum.map(&(&1.message_id)) == ["cherry@burger.pies"] 331 | assert get_articles_created_after("bad.times", ~N[2010-04-05 22:22:22]) == [] 332 | end 333 | 334 | test "extracting message id" do 335 | assert extract_message_id("") == "asd" 336 | assert extract_message_id(nil) == nil 337 | end 338 | 339 | defp message_ids(articles) do 340 | Enum.map(articles, fn {row, article} -> {row, article.message_id} end) 341 | end 342 | end 343 | -------------------------------------------------------------------------------- /test/services/multipart_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Athel.MultipartTest do 2 | # credo:disable-for-this-file Credo.Check.Consistency.TabsOrSpaces 3 | use ExUnit.Case, async: true 4 | 5 | import Athel.Multipart 6 | 7 | @headers %{"MIME-VERSION" => "1.0", 8 | "CONTENT-TYPE" => %{value: "multipart/mixed", params: %{"BOUNDARY" => "lapalabra"}}} 9 | 10 | test "unsupported MIME version" do 11 | assert read_attachments(%{"MIME-VERSION" => "0.3"}, []) == {:error, :invalid_mime_version} 12 | assert read_attachments(%{}, []) == {:ok, nil} 13 | end 14 | 15 | test "content type header" do 16 | headers = %{"MIME-VERSION" => "1.0", "CONTENT-TYPE" => nil} 17 | 18 | assert read_attachments(%{headers | "CONTENT-TYPE" => 19 | %{value: "multipart/parallel", params: %{"BOUNDARY" => "word"}}}, []) == {:error, :unhandled_multipart_type} 20 | assert read_attachments(%{headers | "CONTENT-TYPE" => "multipart/mixed"}, []) == {:error, :invalid_multipart_type} 21 | 22 | assert read_attachments(headers, []) == {:ok, nil} 23 | assert read_attachments(%{headers | "CONTENT-TYPE" => 24 | %{value: "text/plain", params: %{"CHARSET" => "UTF8"}}}, []) == {:ok, nil} 25 | assert read_attachments(%{headers | "CONTENT-TYPE" => "text/plain"}, []) == {:ok, nil} 26 | end 27 | 28 | test "no attachments" do 29 | assert read_attachments(@headers, 30 | ["something", 31 | "or", 32 | "the", 33 | "other", 34 | "--lapalabra--"]) == {:ok, []} 35 | end 36 | 37 | test "one attachment" do 38 | attachment = 39 | %{type: "text/notplain", 40 | params: %{}, 41 | filename: "cristo.txt", 42 | content: ["yo", "te", "llamo", "cristo", ""], 43 | attachments: []} 44 | 45 | assert read_attachments(@headers, 46 | ["IGNORE", 47 | "ME", 48 | "--lapalabra", 49 | "Content-Type: text/notplain", 50 | "Content-Disposition: attachment ; filename=\"cristo.txt\"", 51 | "", 52 | "yo", 53 | "te", 54 | "llamo", 55 | "cristo", 56 | "", 57 | "--lapalabra--"]) == {:ok, [attachment]} 58 | end 59 | 60 | test "two attachments" do 61 | attachments = 62 | [%{type: "text/notplain", 63 | params: %{}, 64 | filename: "cristo.txt", 65 | content: ["yo", "te", "llamo", "cristo"], 66 | attachments: []}, 67 | %{type: "text/html", 68 | params: %{}, 69 | filename: "my_homepage.html", 70 | content: ["

Cool things that I say to my friends

", "Fire, walk with me"], 71 | attachments: []}] 72 | 73 | assert read_attachments(@headers, 74 | ["IGNORE", 75 | "ME", 76 | "--lapalabra", 77 | "Content-Type: text/notplain", 78 | "Content-Disposition: attachment ; filename=\"cristo.txt\"", 79 | "", 80 | "yo", 81 | "te", 82 | "llamo", 83 | "cristo", 84 | "--lapalabra", 85 | "Content-Type: text/html", 86 | "Content-Disposition: attachment; filename=\"my_homepage.html\"", 87 | "", 88 | "

Cool things that I say to my friends

", 89 | "Fire, walk with me", 90 | "--lapalabra--"]) == {:ok, attachments} 91 | end 92 | 93 | test "signed attachment" do 94 | {:ok, [post, signature]} = read_attachments(@headers, String.split(signed_attachment(), "\n")) 95 | 96 | # remove terminating \n 97 | {_, post_body} = signed_attachment_body() |> String.split("\n") |> List.pop_at(-1) 98 | {_, sig_body} = signed_attachment_signature() |> String.split("\n") |> List.pop_at(-1) 99 | 100 | assert post == %{type: "multipart/signed", 101 | filename: nil, 102 | params: %{ 103 | micalg: "pgp-sha1", 104 | protocol: "application/pgp-signature", 105 | signature: sig_body 106 | }, 107 | content: post_body, 108 | attachments: []} 109 | assert signature == %{type: "text/plain", 110 | filename: nil, 111 | params: %{}, 112 | content: ["_______________________________________________", 113 | "Gmane-discuss mailing list", 114 | "Gmane-discuss@hawk.netfonds.no", 115 | "http://hawk.netfonds.no/cgi-bin/mailman/listinfo/gmane-discuss", 116 | ""], 117 | attachments: []} 118 | end 119 | 120 | test "signed attachment with no following attachments" do 121 | body = signed_attachment() 122 | |> String.split("--lapalabra\nContent-Type: text/plain; charset=\"us-ascii\"") 123 | |> List.first 124 | |> String.split("\n") 125 | 126 | {:ok, [post]} = read_attachments(@headers, body) 127 | assert post.type == "multipart/signed" 128 | end 129 | 130 | test "signed attachment with invalid signature" do 131 | body = signed_attachment() 132 | |> String.replace("-----BEGIN PGP SIGNATURE-----", "COMMENCE EMBEZZLING") 133 | |> String.split("\n") 134 | 135 | assert {:error, :invalid_signature} = read_attachments(@headers, body) 136 | end 137 | 138 | test "calls for help after the terminator are ignored" do 139 | {:ok, [attachment]} = read_attachments(@headers, 140 | ["--lapalabra", 141 | "Content-Type: text/plain", 142 | "", 143 | "siempre", 144 | "estas", 145 | "aquí", 146 | "--lapalabra--", 147 | "ayúdame"]) 148 | assert attachment == %{ 149 | type: "text/plain", 150 | params: %{}, 151 | filename: nil, 152 | content: ["siempre", "estas", "aquí"], 153 | attachments: [] 154 | } 155 | end 156 | 157 | test "base64 content" do 158 | {:ok, [attachment]} = read_attachments(@headers, 159 | ["--lapalabra", 160 | "Content-Transfer-Encoding: base64", 161 | "", 162 | "Q2FuJ3QgZ2V0IG15DQpsaW5lIGVuZGluZ3MKY29uc2lzdGVudA1pIHF1aXQ=", 163 | "--lapalabra--"]) 164 | assert attachment.content == "Can't get my\r\nline endings\nconsistent\ri quit" 165 | end 166 | 167 | # this is real 168 | test "duplicate transfer encoding headers" do 169 | {:ok, [attachment]} = read_attachments(@headers, 170 | ["--lapalabra", 171 | "Content-Transfer-Encoding: base64", 172 | "Content-Transfer-Encoding: base64", 173 | "", 174 | "Q2FuJ3QgZ2V0IG15DQpsaW5lIGVuZGluZ3MKY29uc2lzdGVudA1pIHF1aXQ=", 175 | "--lapalabra--"]) 176 | assert attachment.content == "Can't get my\r\nline endings\nconsistent\ri quit" 177 | end 178 | 179 | test "attachment without headers" do 180 | {:ok, [attachment]} = read_attachments(@headers, 181 | ["--lapalabra", 182 | "", 183 | "just a body", 184 | "kinda shoddy", 185 | "--lapalabra--"]) 186 | assert attachment == %{ 187 | type: "text/plain", 188 | params: %{}, 189 | filename: nil, 190 | content: ["just a body", "kinda shoddy"], 191 | attachments: [] 192 | } 193 | end 194 | 195 | test "missing terminator with no attachments" do 196 | assert read_attachments(@headers, 197 | ["just", 198 | "can't", 199 | "stop", 200 | "myself"]) == {:error, :unterminated_body} 201 | end 202 | 203 | test "missing terminator with an attachment" do 204 | assert read_attachments(@headers, 205 | ["la gloria", 206 | "de Dios", 207 | "--lapalabra", 208 | "Content-Type: text/plain", 209 | "", 210 | "get at me", 211 | "--lapalabra"]) == {:error, :unterminated_body} 212 | 213 | assert read_attachments(@headers, 214 | ["--lapalabra", 215 | "Content-Type: text/plain", 216 | "", 217 | "sacrebleu!"]) == {:error, :unterminated_body} 218 | end 219 | 220 | test "missing newline after headers" do 221 | assert read_attachments(@headers, 222 | ["--lapalabra", 223 | "Content-Type: bread/wine", 224 | "this is my body", 225 | "--lapalabra--"]) == {:error, :header_name} 226 | 227 | assert read_attachments(@headers, 228 | ["--lapalabra", 229 | "Content-Type: bread/wine"]) == {:error, :unterminated_headers} 230 | end 231 | 232 | test "unhandled encoding type" do 233 | assert read_attachments(@headers, 234 | ["--lapalabra", 235 | "Content-Transfer-Encoding: base58", 236 | "", 237 | "--lapalabra--"]) == {:error, :unhandled_transfer_encoding} 238 | end 239 | 240 | test "invalid encoding" do 241 | assert read_attachments(@headers, 242 | ["--lapalabra", 243 | "Content-Transfer-Encoding: base64", 244 | "", 245 | "can't b64", 246 | "two lines", 247 | "--lapalabra--"]) == {:error, :invalid_transfer_encoding} 248 | 249 | assert read_attachments(@headers, 250 | ["--lapalabra", 251 | "Content-Transfer-Encoding: base64", 252 | "", 253 | "whoopsy", 254 | "--lapalabra--"]) == {:error, :invalid_transfer_encoding} 255 | end 256 | 257 | test "unhandled content disposition" do 258 | assert read_attachments(@headers, 259 | ["--lapalabra", 260 | "Content-Disposition: inline; filename=\"nice_heels.jpg\"", 261 | "", 262 | "sometimes frosted", 263 | "sometimes sprinkled", 264 | "--lapalabra--"]) == {:error, :unhandled_content_disposition} 265 | end 266 | 267 | defp signed_attachment, do: """ 268 | --lapalabra 269 | Content-Type: multipart/signed; boundary="=-=-="; 270 | micalg=pgp-sha1; protocol="application/pgp-signature" 271 | 272 | --=-=-= 273 | Content-Type: text/plain 274 | Content-Transfer-Encoding: quoted-printable 275 | 276 | Olly Betts writes: 277 | 278 | > Rainer M Krug writes: 279 | >> 2) http://search.gmane.org/nov.php needs to support search in gwene 280 | >> groups. 281 | > 282 | > Only gmane.* groups are indexed currently - if that changed, the search 283 | > forms would probably just work. 284 | 285 | That would be gret. 286 | 287 | > 288 | >> My question: could this be implemented in gmane? 289 | > 290 | > The main blocker for doing this would be that the search machine doesn't 291 | > have a lot of spare disk space. 292 | 293 | OK. 294 | 295 | > 296 | > A new machine is planned, but I'm not sure exactly when it'll actually 297 | > happen. 298 | 299 | If the indexing of gmane could be added after the new machine is 300 | available, that would be great. 301 | 302 | Could you please keep us posted? 303 | 304 | Thanks, 305 | 306 | Rainer 307 | 308 | > 309 | > Cheers, 310 | > Olly 311 | 312 | =2D-=20 313 | Rainer M. Krug 314 | email: Rainerkrugsde 315 | PGP: 0x0F52F982 316 | 317 | --=-=-= 318 | Content-Type: application/pgp-signature 319 | 320 | -----BEGIN PGP SIGNATURE----- 321 | Version: GnuPG/MacGPG2 v2.0.22 (Darwin) 322 | 323 | iQEcBAEBAgAGBQJUQM65AAoJENvXNx4PUvmC3/8H/iV8GbKm6D8exL2Czxc+ADEF 324 | XaPVO7pYKK2cBWLIZ+AmhEyiVBKa01/Ch6tkNjmR9snCtI0TH3R0srdjzuRu1yhx 325 | CjMngcN2SSL1QXY4OYdYWfIY2/5RueIjm37/u3Y/qeJHoMJsE/nLb4jmvWLXdWAb 326 | Ns6WUDuL5WFunDvm6qH6IBLPLTU8mLsKG2yhbdXUx+ObBnHQec0laNLIqwIt1eRa 327 | Q0blwRK4TyqHf0XpdV8iB04b2EHYZUyuQsJc42In9fYesGHxmKwWkGEb3GA5C8X6 328 | lKD62Xa8IEGAQu8dkgrXJrwcslyAIFHtX7ICtwqqvgQ2LFvHLEwUaPKDcsvqXqw= 329 | =C/U1 330 | -----END PGP SIGNATURE----- 331 | --=-=-=-- 332 | 333 | 334 | --lapalabra 335 | Content-Type: text/plain; charset="us-ascii" 336 | MIME-Version: 1.0 337 | Content-Transfer-Encoding: 7bit 338 | Content-Disposition: inline 339 | 340 | _______________________________________________ 341 | Gmane-discuss mailing list 342 | Gmane-discuss@hawk.netfonds.no 343 | http://hawk.netfonds.no/cgi-bin/mailman/listinfo/gmane-discuss 344 | 345 | --lapalabra-- 346 | """ 347 | 348 | defp signed_attachment_body, do: """ 349 | Olly Betts writes: 350 | 351 | > Rainer M Krug writes: 352 | >> 2) http://search.gmane.org/nov.php needs to support search in gwene 353 | >> groups. 354 | > 355 | > Only gmane.* groups are indexed currently - if that changed, the search 356 | > forms would probably just work. 357 | 358 | That would be gret. 359 | 360 | > 361 | >> My question: could this be implemented in gmane? 362 | > 363 | > The main blocker for doing this would be that the search machine doesn't 364 | > have a lot of spare disk space. 365 | 366 | OK. 367 | 368 | > 369 | > A new machine is planned, but I'm not sure exactly when it'll actually 370 | > happen. 371 | 372 | If the indexing of gmane could be added after the new machine is 373 | available, that would be great. 374 | 375 | Could you please keep us posted? 376 | 377 | Thanks, 378 | 379 | Rainer 380 | 381 | > 382 | > Cheers, 383 | > Olly 384 | 385 | =2D-=20 386 | Rainer M. Krug 387 | email: Rainerkrugsde 388 | PGP: 0x0F52F982 389 | 390 | """ 391 | 392 | defp signed_attachment_signature, do: """ 393 | Version: GnuPG/MacGPG2 v2.0.22 (Darwin) 394 | 395 | iQEcBAEBAgAGBQJUQM65AAoJENvXNx4PUvmC3/8H/iV8GbKm6D8exL2Czxc+ADEF 396 | XaPVO7pYKK2cBWLIZ+AmhEyiVBKa01/Ch6tkNjmR9snCtI0TH3R0srdjzuRu1yhx 397 | CjMngcN2SSL1QXY4OYdYWfIY2/5RueIjm37/u3Y/qeJHoMJsE/nLb4jmvWLXdWAb 398 | Ns6WUDuL5WFunDvm6qH6IBLPLTU8mLsKG2yhbdXUx+ObBnHQec0laNLIqwIt1eRa 399 | Q0blwRK4TyqHf0XpdV8iB04b2EHYZUyuQsJc42In9fYesGHxmKwWkGEb3GA5C8X6 400 | lKD62Xa8IEGAQu8dkgrXJrwcslyAIFHtX7ICtwqqvgQ2LFvHLEwUaPKDcsvqXqw= 401 | =C/U1 402 | """ 403 | 404 | end 405 | -------------------------------------------------------------------------------- /lib/athel/nntp/parser.ex: -------------------------------------------------------------------------------- 1 | defmodule Athel.Nntp.Parser do 2 | 3 | @type parse_result(parsed) :: {:ok, parsed, iodata} | {:error, atom} | {:need_more, any()} 4 | 5 | @spec parse_code_line(iodata) :: parse_result({integer, String.t}) 6 | def parse_code_line(input, state \\ nil) do 7 | {result, rest} = input |> IO.iodata_to_binary |> code_line(state) 8 | {:ok, result, rest} 9 | catch 10 | e -> e 11 | end 12 | 13 | @spec parse_multiline(iodata, list(String.t)) :: parse_result(list(String.t)) 14 | def parse_multiline(input, state \\ nil) do 15 | {lines, rest} = input |> IO.iodata_to_binary |> multiline(state) 16 | {:ok, lines, rest} 17 | catch 18 | e -> e 19 | end 20 | 21 | @spec parse_headers(iodata) :: parse_result(%{optional(String.t) => String.t | list(String.t)}) 22 | def parse_headers(input, state \\ nil) do 23 | {headers, rest} = input |> IO.iodata_to_binary |> headers(state) 24 | {:ok, headers, rest} 25 | catch 26 | e -> e 27 | end 28 | 29 | @spec parse_command(iodata) :: parse_result({String.t, list(String.t)}) 30 | def parse_command(input, state \\ nil) do 31 | {name, arguments, rest} = input |> IO.iodata_to_binary |> command(state) 32 | {:ok, {name, arguments}, rest} 33 | catch 34 | e -> e 35 | end 36 | 37 | @digits '0123456789' 38 | 39 | defp code(input, nil) do 40 | code(input, {[], 0}) 41 | end 42 | defp code(<>, {acc, count}) when digit in @digits do 43 | code(rest, {[acc, digit], count + 1}) 44 | end 45 | defp code("", {acc, count}) when count < 3 do 46 | need_more({acc, count}) 47 | end 48 | defp code(next, state) do 49 | end_code(state, next) 50 | end 51 | 52 | defp end_code({acc, 3}, rest) do 53 | {acc |> IO.iodata_to_binary |> String.to_integer, rest} 54 | end 55 | defp end_code(_in, _acc) do 56 | syntax_error(:code) 57 | end 58 | 59 | @whitespace ' \t' 60 | 61 | defp skip_whitespace(<>) when char in @whitespace do 62 | rest 63 | end 64 | defp skip_whitespace(string) do 65 | string 66 | end 67 | 68 | defp line("", acc) do 69 | need_more(acc) 70 | end 71 | defp line(<<"\r\n", rest :: binary>>, acc) do 72 | {IO.iodata_to_binary(acc), rest} 73 | end 74 | # newline split across reads 75 | defp line(<>, acc) do 76 | need_more([acc, ?\r]) 77 | end 78 | # if there is more after \r, enforce \n 79 | defp line(<>, _) when next != ?\n do 80 | syntax_error(:line) 81 | end 82 | # end of newline split across reads. Read backwards to confirm \r preceded 83 | defp line(<>, acc) do 84 | [head | tail] = acc 85 | case tail do 86 | '\r' -> {IO.iodata_to_binary(head), rest} 87 | _ -> syntax_error(:line) 88 | end 89 | end 90 | defp line(<>, acc) do 91 | line(rest, [acc, char]) 92 | end 93 | defp line(_, _) do 94 | syntax_error(:line) 95 | end 96 | 97 | # code_line state is {:code, current_code} | {:line, current_line, code} 98 | # resume 99 | defp code_line(input, {:code, acc}) do 100 | code_line(input, acc) 101 | end 102 | defp code_line(input, {:line, acc, code}) do 103 | end_code_line(input, acc, code) 104 | end 105 | # parse 106 | defp code_line(input, acc) do 107 | {code, rest} = try do 108 | code(input, acc) 109 | catch 110 | {:need_more, acc} -> need_more({:code, acc}) 111 | end 112 | 113 | end_code_line(rest, [], code) 114 | end 115 | 116 | defp end_code_line(input, acc, code) do 117 | {line, rest} = 118 | try do 119 | input |> skip_whitespace |> line(acc) 120 | catch 121 | {:need_more, acc} -> need_more({:line, acc, code}) 122 | end 123 | {{code, line}, rest} 124 | end 125 | 126 | # multiline state is {current_line, lines} 127 | # first iteration 128 | defp multiline(input, nil) do 129 | multiline(input, {[], []}) 130 | end 131 | # no input 132 | defp multiline("", state) do 133 | need_more(state) 134 | end 135 | # parsing 136 | defp multiline(input, {line_acc, acc}) do 137 | {line, rest} = 138 | try do 139 | line(input, line_acc) 140 | catch 141 | {:need_more, next_line_acc} -> need_more({next_line_acc, acc}) 142 | end 143 | 144 | #IO.puts "at #{inspect(line)} with #{inspect rest} after #{inspect input}" 145 | case line do 146 | # escaped leading . 147 | <<"..", line_rest :: binary>> -> 148 | escaped_line = "." <> line_rest 149 | multiline(rest, {[], [escaped_line | acc]}) 150 | # termination 151 | <<".">> -> {Enum.reverse(acc), rest} 152 | # leading . that neither terminated nor was escaped is invalid 153 | <<".", _ :: binary>> -> 154 | syntax_error(:multiline) 155 | # else keep reading 156 | line -> multiline(rest, {[], [line | acc]}) 157 | end 158 | end 159 | 160 | # previous is for handling multiline headers 161 | # :need_more is only handled at the top level (via reading in a whole line at a time) for simplicity 162 | # initialize 163 | defp headers(input, nil) do 164 | headers(input, %{ 165 | line_acc: [], 166 | headers: %{}, 167 | header_name: nil, 168 | param_name_acc: []}) 169 | end 170 | defp headers("", state) do 171 | need_more(state) 172 | end 173 | # parse 174 | defp headers(input, state) do 175 | {line, rest} = 176 | try do 177 | line(input, state.line_acc) 178 | catch 179 | {:need_more, next_line_acc} -> need_more(%{state | line_acc: next_line_acc}) 180 | end 181 | 182 | cond do 183 | # empty line signifies end of headers 184 | "" == line -> {state.headers, rest} 185 | # multiline header 186 | line =~ ~r/^\s+/ -> 187 | case state.header_name do 188 | nil -> syntax_error(:multiline_header) 189 | header_name -> 190 | # resuming header value parse 191 | new_headers = Map.update(state.headers, header_name, "", 192 | &merge_header_lines(&1, line, state)) 193 | headers(rest, %{state | line_acc: [], headers: new_headers}) 194 | end 195 | true -> 196 | {name, rest_value} = header_name(line, []) 197 | named_state = %{state | header_name: name, param_name_acc: []} 198 | value = header_value(skip_whitespace(rest_value), [], named_state) 199 | new_headers = Map.update(state.headers, name, value, &merge_headers(&1, value)) 200 | 201 | headers(rest, %{named_state | line_acc: [], headers: new_headers}) 202 | end 203 | end 204 | 205 | # values are inserted backwards while merging 206 | 207 | defp merge_header_lines(prev, line, state) when is_list(prev) do 208 | [last | rest] = prev 209 | [merge_header_lines(last, line, state) | rest] 210 | end 211 | defp merge_header_lines(%{value: value, params: params}, line, state) do 212 | next_params = header_params(line, params, state) 213 | %{value: value, params: next_params} 214 | catch 215 | {:need_more, {new_params, param_name_acc}} -> 216 | handle_incomplete_params_parse(state, value, Map.merge(params, new_params), param_name_acc) 217 | end 218 | defp merge_header_lines(prev, line, state) do 219 | next = header_value(line, [], state) 220 | "#{prev} #{next}" 221 | end 222 | 223 | @spec handle_incomplete_params_parse(map(), String.t, map(), iodata) :: no_return 224 | defp handle_incomplete_params_parse(state, header_value, params, param_name_acc) do 225 | new_headers = Map.put(state.headers, state.header_name, %{value: header_value, params: params}) 226 | need_more(%{state | headers: new_headers, param_name_acc: param_name_acc}) 227 | end 228 | 229 | defp merge_headers(prev, new) when is_list(prev) do 230 | [new | prev] 231 | end 232 | defp merge_headers(prev, new) do 233 | [new, prev] 234 | end 235 | 236 | # hit EOL without ":" separator 237 | defp header_name("", _), do: syntax_error(:header_name) 238 | # termination 239 | defp header_name(<<":", rest :: binary>>, acc) do 240 | {acc |> IO.iodata_to_binary |> String.upcase, rest} 241 | end 242 | # no whitespace in header names 243 | defp header_name(<>, _) when next in @whitespace, do: syntax_error(:header_name) 244 | # parse 245 | defp header_name(<>, acc) do 246 | header_name(rest, [acc, next]) 247 | end 248 | 249 | @param_headers ["CONTENT-TYPE", "CONTENT-DISPOSITION"] 250 | 251 | # termination 252 | defp header_value("", acc, _), do: terminate_header_value(acc) 253 | # start of params 254 | defp header_value(<<";", rest :: binary>>, acc, state = %{header_name: header_name}) 255 | when header_name in @param_headers do 256 | params = header_params(rest, %{}, state) 257 | %{value: terminate_header_value(acc), params: params} 258 | catch 259 | {:need_more, {params, param_name_acc}} -> 260 | handle_incomplete_params_parse(state, terminate_header_value(acc), params, param_name_acc) 261 | end 262 | # parse 263 | defp header_value(<>, acc, state) do 264 | header_value(rest, [acc, char], state) 265 | end 266 | 267 | defp terminate_header_value(acc) do 268 | acc 269 | |> IO.iodata_to_binary 270 | |> Codepagex.to_string!(:ascii, Codepagex.use_utf_replacement()) 271 | |> String.trim 272 | end 273 | 274 | # termination 275 | defp header_params("", params, _), do: params 276 | # parse 277 | defp header_params(input, params, state) do 278 | {param_name, value_input} = try do 279 | header_param_name(input, state.param_name_acc) 280 | catch 281 | {:need_more, param_name_acc} -> need_more({params, param_name_acc}) 282 | end 283 | {param_value, next_param_input} = header_param_value(value_input, {[], false}) 284 | cleared_state = %{state | param_name_acc: []} 285 | header_params(next_param_input, params |> Map.put(param_name, param_value), cleared_state) 286 | end 287 | 288 | # haven't found "=" separator 289 | defp header_param_name("", acc) do 290 | need_more(acc) 291 | end 292 | # eat leading whitespace 293 | defp header_param_name(<>, acc) when char in @whitespace do 294 | header_param_name(rest, acc) 295 | end 296 | # termination 297 | defp header_param_name(<<"=", rest :: binary>>, acc) do 298 | {acc |> IO.iodata_to_binary |> String.upcase, rest} 299 | end 300 | # parse 301 | defp header_param_name(<>, acc) do 302 | header_param_name(rest, [acc, char]) 303 | end 304 | 305 | # termination 306 | defp header_param_value("", {acc, _}) do 307 | {acc |> IO.iodata_to_binary, ""} 308 | end 309 | defp header_param_value(<<";", rest :: binary>>, {acc, _}) do 310 | {acc |> IO.iodata_to_binary, rest} 311 | end 312 | # handle boundary="commadelimited" 313 | defp header_param_value(<<"\"", rest :: binary>>, {acc, delimited}) do 314 | cond do 315 | delimited -> 316 | case Regex.run ~r/^( |\t)*(;?(.*))/s, rest do 317 | nil -> syntax_error(:header_param_value) 318 | [_, _, _, rest2] -> 319 | {acc |> IO.iodata_to_binary, rest2} 320 | end 321 | IO.iodata_to_binary(acc) =~ ~r/^\s*$/ -> 322 | header_param_value(rest, {[], true}) 323 | not delimited -> 324 | header_param_value(rest, {[acc, "\""], false}) 325 | end 326 | end 327 | # parse 328 | defp header_param_value(<>, {acc, delimited}) do 329 | header_param_value(rest, {[acc, char], delimited}) 330 | end 331 | 332 | # command state is {command_name, arguments, current_identifier} 333 | # initialize 334 | defp command(input, nil) do 335 | command(input, {[], [], []}) 336 | end 337 | # end of command 338 | defp command(<<"\r\n", rest :: binary>>, state) do 339 | end_command(state, rest) 340 | end 341 | # newline split across reads 342 | defp command(<>, {name, arguments, acc}) do 343 | need_more({name, arguments, [acc, ?\r]}) 344 | end 345 | # if there is more after \r, enforce \n 346 | defp command(<>, _) when next != ?\n do 347 | syntax_error(:command) 348 | end 349 | # end of newline split across reads. Read backwards to confirm \r preceded 350 | defp command(<>, {command, arguments, acc}) do 351 | [head | tail] = acc 352 | case tail do 353 | '\r' -> end_command({command, arguments, head}, rest) 354 | _ -> syntax_error(:command) 355 | end 356 | end 357 | # need more 358 | defp command("", acc) do 359 | need_more(acc) 360 | end 361 | # end of command name 362 | defp command(<>, {[], [], acc}) when next in @whitespace do 363 | command(rest, {acc, [], []}) 364 | end 365 | # reading command name 366 | defp command(<>, {[], [], acc}) do 367 | command(rest, {[], [], [acc, next]}) 368 | end 369 | # end of argument 370 | defp command(<>, {name, arguments, acc}) when next in @whitespace do 371 | command(rest, {name, [acc | arguments], []}) 372 | end 373 | # reading argument 374 | defp command(<>, {name, arguments, acc}) do 375 | command(rest, {name, arguments, [acc, next]}) 376 | end 377 | 378 | defp end_command({name, arguments, acc}, rest) do 379 | {name, arguments} = 380 | if Enum.empty? name do 381 | if Enum.empty? acc do 382 | syntax_error(:command) 383 | else 384 | {acc, []} 385 | end 386 | else 387 | if Enum.empty? acc do 388 | {name, arguments} 389 | else 390 | {name, [acc | arguments]} 391 | end 392 | end 393 | 394 | name = name |> IO.iodata_to_binary |> String.upcase 395 | arguments = arguments |> Enum.map(&IO.iodata_to_binary/1) |> Enum.reverse 396 | {name, arguments, rest} 397 | end 398 | 399 | defp need_more(state) do 400 | throw {:need_more, state} 401 | end 402 | 403 | defp syntax_error(type) do 404 | throw {:error, type} 405 | end 406 | 407 | end 408 | --------------------------------------------------------------------------------