├── test ├── test_helper.exs ├── hex2txt_web │ └── controllers │ │ ├── page_controller_test.exs │ │ ├── error_json_test.exs │ │ └── error_html_test.exs └── support │ └── conn_case.ex ├── .env ├── rel ├── overlays │ └── bin │ │ ├── server.bat │ │ └── server └── env.sh.eex ├── priv └── static │ ├── favicon.ico │ ├── robots.txt │ └── images │ └── logo.svg ├── .formatter.exs ├── assets ├── css │ └── app.css ├── js │ └── app.js ├── tailwind.config.js └── vendor │ └── topbar.js ├── lib ├── hex2txt.ex ├── hex2txt_web │ ├── controllers │ │ ├── page_html.ex │ │ ├── page_controller.ex │ │ ├── error_json.ex │ │ ├── error_html.ex │ │ └── page_html │ │ │ └── home.html.heex │ ├── components │ │ ├── layouts.ex │ │ ├── layouts │ │ │ ├── root.html.heex │ │ │ └── app.html.heex │ │ └── core_components.ex │ ├── router.ex │ ├── endpoint.ex │ └── telemetry.ex ├── hex2txt │ ├── application.ex │ ├── generator.ex │ └── hex_scraper.ex └── hex2txt_web.ex ├── .github └── workflows │ └── fly-deploy.yml ├── config ├── prod.exs ├── test.exs ├── config.exs ├── dev.exs └── runtime.exs ├── fly.toml ├── .dockerignore ├── justfile ├── README.md ├── .gitignore ├── flake.lock ├── mix.exs ├── flake.nix ├── Dockerfile ├── mix.lock ├── notebooks └── hex2context.livemd └── LICENSE /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | NODE_NAME=hex2txt 2 | NODE_COOKIE=hex2txtcookie 3 | -------------------------------------------------------------------------------- /rel/overlays/bin/server.bat: -------------------------------------------------------------------------------- 1 | set PHX_SERVER=true 2 | call "%~dp0\hex2txt" start 3 | -------------------------------------------------------------------------------- /priv/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mjrusso/hex2txt/HEAD/priv/static/favicon.ico -------------------------------------------------------------------------------- /rel/overlays/bin/server: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -eu 3 | 4 | cd -P -- "$(dirname -- "$0")" 5 | PHX_SERVER=true exec ./hex2txt start 6 | -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | [ 2 | import_deps: [:phoenix], 3 | plugins: [Phoenix.LiveView.HTMLFormatter], 4 | inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}"] 5 | ] 6 | -------------------------------------------------------------------------------- /assets/css/app.css: -------------------------------------------------------------------------------- 1 | @import "tailwindcss/base"; 2 | @import "tailwindcss/components"; 3 | @import "tailwindcss/utilities"; 4 | 5 | /* This file is for your main application CSS */ 6 | -------------------------------------------------------------------------------- /priv/static/robots.txt: -------------------------------------------------------------------------------- 1 | # See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file 2 | # 3 | # To ban all spiders from the entire site uncomment the next two lines: 4 | # User-agent: * 5 | # Disallow: / 6 | -------------------------------------------------------------------------------- /lib/hex2txt.ex: -------------------------------------------------------------------------------- 1 | defmodule Hex2txt do 2 | @moduledoc """ 3 | Hex2txt keeps the contexts that define your domain 4 | and business logic. 5 | 6 | Contexts are also responsible for managing your data, regardless 7 | if it comes from the database, an external API or others. 8 | """ 9 | end 10 | -------------------------------------------------------------------------------- /test/hex2txt_web/controllers/page_controller_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Hex2txtWeb.PageControllerTest do 2 | use Hex2txtWeb.ConnCase 3 | 4 | test "GET /", %{conn: conn} do 5 | conn = get(conn, ~p"/") 6 | assert html_response(conn, 200) =~ "Peace of mind from prototype to production" 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /lib/hex2txt_web/controllers/page_html.ex: -------------------------------------------------------------------------------- 1 | defmodule Hex2txtWeb.PageHTML do 2 | @moduledoc """ 3 | This module contains pages rendered by PageController. 4 | 5 | See the `page_html` directory for all templates available. 6 | """ 7 | use Hex2txtWeb, :html 8 | 9 | embed_templates "page_html/*" 10 | end 11 | -------------------------------------------------------------------------------- /test/hex2txt_web/controllers/error_json_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Hex2txtWeb.ErrorJSONTest do 2 | use Hex2txtWeb.ConnCase, async: true 3 | 4 | test "renders 404" do 5 | assert Hex2txtWeb.ErrorJSON.render("404.json", %{}) == %{errors: %{detail: "Not Found"}} 6 | end 7 | 8 | test "renders 500" do 9 | assert Hex2txtWeb.ErrorJSON.render("500.json", %{}) == 10 | %{errors: %{detail: "Internal Server Error"}} 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/hex2txt_web/controllers/page_controller.ex: -------------------------------------------------------------------------------- 1 | defmodule Hex2txtWeb.PageController do 2 | use Hex2txtWeb, :controller 3 | 4 | def home(conn, _params) do 5 | # The home page is often custom made, 6 | # so skip the default app layout. 7 | render(conn, :home, layout: false) 8 | end 9 | 10 | def llms_txt(conn, %{"package" => package} = params) do 11 | text(conn, Hex2txt.Generator.generate(package, Map.get(params, "version"))) 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /test/hex2txt_web/controllers/error_html_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Hex2txtWeb.ErrorHTMLTest do 2 | use Hex2txtWeb.ConnCase, async: true 3 | 4 | # Bring render_to_string/4 for testing custom views 5 | import Phoenix.Template 6 | 7 | test "renders 404.html" do 8 | assert render_to_string(Hex2txtWeb.ErrorHTML, "404", "html", []) == "Not Found" 9 | end 10 | 11 | test "renders 500.html" do 12 | assert render_to_string(Hex2txtWeb.ErrorHTML, "500", "html", []) == "Internal Server Error" 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/hex2txt_web/components/layouts.ex: -------------------------------------------------------------------------------- 1 | defmodule Hex2txtWeb.Layouts do 2 | @moduledoc """ 3 | This module holds different layouts used by your application. 4 | 5 | See the `layouts` directory for all templates available. 6 | The "root" layout is a skeleton rendered as part of the 7 | application router. The "app" layout is set as the default 8 | layout on both `use Hex2txtWeb, :controller` and 9 | `use Hex2txtWeb, :live_view`. 10 | """ 11 | use Hex2txtWeb, :html 12 | 13 | embed_templates "layouts/*" 14 | end 15 | -------------------------------------------------------------------------------- /rel/env.sh.eex: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # configure node for distributed erlang with IPV6 support 4 | export ERL_AFLAGS="-proto_dist inet6_tcp" 5 | export ECTO_IPV6="true" 6 | export DNS_CLUSTER_QUERY="${FLY_APP_NAME}.internal" 7 | export RELEASE_DISTRIBUTION="name" 8 | export RELEASE_NODE="${FLY_APP_NAME}-${FLY_IMAGE_REF##*-}@${FLY_PRIVATE_IP}" 9 | 10 | # Uncomment to send crash dumps to stderr 11 | # This can be useful for debugging, but may log sensitive information 12 | # export ERL_CRASH_DUMP=/dev/stderr 13 | # export ERL_CRASH_DUMP_BYTES=4096 14 | -------------------------------------------------------------------------------- /.github/workflows/fly-deploy.yml: -------------------------------------------------------------------------------- 1 | # See https://fly.io/docs/app-guides/continuous-deployment-with-github-actions/ 2 | 3 | name: Fly Deploy 4 | on: 5 | push: 6 | branches: 7 | - main 8 | jobs: 9 | deploy: 10 | name: Deploy app 11 | runs-on: ubuntu-latest 12 | concurrency: deploy-group # optional: ensure only one action runs at a time 13 | steps: 14 | - uses: actions/checkout@v4 15 | - uses: superfly/flyctl-actions/setup-flyctl@master 16 | - run: flyctl deploy --remote-only 17 | env: 18 | FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} 19 | -------------------------------------------------------------------------------- /config/prod.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | # Note we also include the path to a cache manifest 4 | # containing the digested version of static files. This 5 | # manifest is generated by the `mix assets.deploy` task, 6 | # which you should run after static files are built and 7 | # before starting your production server. 8 | config :hex2txt, Hex2txtWeb.Endpoint, cache_static_manifest: "priv/static/cache_manifest.json" 9 | 10 | # Do not print debug messages in production 11 | config :logger, level: :info 12 | 13 | # Runtime production configuration, including reading 14 | # of environment variables, is done on config/runtime.exs. 15 | -------------------------------------------------------------------------------- /config/test.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | # We don't run a server during test. If one is required, 4 | # you can enable the server option below. 5 | config :hex2txt, Hex2txtWeb.Endpoint, 6 | http: [ip: {127, 0, 0, 1}, port: 4002], 7 | secret_key_base: "ktwv3NT74quUx7lEpSwFmtWFiK3YDZxEEt4B4cF81NGnCp9MFzFBw7QkJ6GLqYeB", 8 | server: false 9 | 10 | # Print only warnings and errors during test 11 | config :logger, level: :warning 12 | 13 | # Initialize plugs at runtime for faster test compilation 14 | config :phoenix, :plug_init_mode, :runtime 15 | 16 | # Enable helpful, but potentially expensive runtime checks 17 | config :phoenix_live_view, 18 | enable_expensive_runtime_checks: true 19 | -------------------------------------------------------------------------------- /lib/hex2txt_web/components/layouts/root.html.heex: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | <.live_title default="hex2txt" suffix=" · hex.pm package → llms.txt"> 8 | {assigns[:page_title]} 9 | 10 | 11 | 13 | 14 | 15 | {@inner_content} 16 | 17 | 18 | -------------------------------------------------------------------------------- /lib/hex2txt_web/controllers/error_json.ex: -------------------------------------------------------------------------------- 1 | defmodule Hex2txtWeb.ErrorJSON do 2 | @moduledoc """ 3 | This module is invoked by your endpoint in case of errors on JSON requests. 4 | 5 | See config/config.exs. 6 | """ 7 | 8 | # If you want to customize a particular status code, 9 | # you may add your own clauses, such as: 10 | # 11 | # def render("500.json", _assigns) do 12 | # %{errors: %{detail: "Internal Server Error"}} 13 | # end 14 | 15 | # By default, Phoenix returns the status message from 16 | # the template name. For example, "404.json" becomes 17 | # "Not Found". 18 | def render(template, _assigns) do 19 | %{errors: %{detail: Phoenix.Controller.status_message_from_template(template)}} 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /fly.toml: -------------------------------------------------------------------------------- 1 | # fly.toml app configuration file generated for hex2txt on 2024-12-15T09:00:00-05:00 2 | # 3 | # See https://fly.io/docs/reference/configuration/ for information about how to use this file. 4 | # 5 | 6 | app = 'hex2txt' 7 | primary_region = 'yyz' 8 | kill_signal = 'SIGTERM' 9 | 10 | [build] 11 | 12 | [env] 13 | PHX_HOST = 'hex2txt.fly.dev' 14 | PORT = '8080' 15 | 16 | [http_service] 17 | internal_port = 8080 18 | force_https = true 19 | auto_stop_machines = 'stop' 20 | auto_start_machines = true 21 | min_machines_running = 0 22 | processes = ['app'] 23 | 24 | [http_service.concurrency] 25 | type = 'connections' 26 | hard_limit = 1000 27 | soft_limit = 1000 28 | 29 | [[vm]] 30 | memory = '1gb' 31 | cpu_kind = 'shared' 32 | cpus = 1 33 | -------------------------------------------------------------------------------- /lib/hex2txt_web/controllers/error_html.ex: -------------------------------------------------------------------------------- 1 | defmodule Hex2txtWeb.ErrorHTML do 2 | @moduledoc """ 3 | This module is invoked by your endpoint in case of errors on HTML requests. 4 | 5 | See config/config.exs. 6 | """ 7 | use Hex2txtWeb, :html 8 | 9 | # If you want to customize your error pages, 10 | # uncomment the embed_templates/1 call below 11 | # and add pages to the error directory: 12 | # 13 | # * lib/hex2txt_web/controllers/error_html/404.html.heex 14 | # * lib/hex2txt_web/controllers/error_html/500.html.heex 15 | # 16 | # embed_templates "error_html/*" 17 | 18 | # The default is to render a plain text page based on 19 | # the template name. For example, "404.html" becomes 20 | # "Not Found". 21 | def render(template, _assigns) do 22 | Phoenix.Controller.status_message_from_template(template) 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/hex2txt/application.ex: -------------------------------------------------------------------------------- 1 | defmodule Hex2txt.Application do 2 | # See https://hexdocs.pm/elixir/Application.html 3 | # for more information on OTP Applications 4 | @moduledoc false 5 | 6 | use Application 7 | 8 | @impl true 9 | def start(_type, _args) do 10 | children = [ 11 | Hex2txtWeb.Telemetry, 12 | {DNSCluster, query: Application.get_env(:hex2txt, :dns_cluster_query) || :ignore}, 13 | {Phoenix.PubSub, name: Hex2txt.PubSub}, 14 | # Start a worker by calling: Hex2txt.Worker.start_link(arg) 15 | # {Hex2txt.Worker, arg}, 16 | # Start to serve requests, typically the last entry 17 | Hex2txtWeb.Endpoint 18 | ] 19 | 20 | # See https://hexdocs.pm/elixir/Supervisor.html 21 | # for other strategies and supported options 22 | opts = [strategy: :one_for_one, name: Hex2txt.Supervisor] 23 | Supervisor.start_link(children, opts) 24 | end 25 | 26 | # Tell Phoenix to update the endpoint configuration 27 | # whenever the application is updated. 28 | @impl true 29 | def config_change(changed, _new, removed) do 30 | Hex2txtWeb.Endpoint.config_change(changed, removed) 31 | :ok 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /test/support/conn_case.ex: -------------------------------------------------------------------------------- 1 | defmodule Hex2txtWeb.ConnCase do 2 | @moduledoc """ 3 | This module defines the test case to be used by 4 | tests that require setting up a connection. 5 | 6 | Such tests rely on `Phoenix.ConnTest` and also 7 | import other functionality to make it easier 8 | to build common data structures and query the data layer. 9 | 10 | Finally, if the test case interacts with the database, 11 | we enable the SQL sandbox, so changes done to the database 12 | are reverted at the end of every test. If you are using 13 | PostgreSQL, you can even run database tests asynchronously 14 | by setting `use Hex2txtWeb.ConnCase, async: true`, although 15 | this option is not recommended for other databases. 16 | """ 17 | 18 | use ExUnit.CaseTemplate 19 | 20 | using do 21 | quote do 22 | # The default endpoint for testing 23 | @endpoint Hex2txtWeb.Endpoint 24 | 25 | use Hex2txtWeb, :verified_routes 26 | 27 | # Import conveniences for testing with connections 28 | import Plug.Conn 29 | import Phoenix.ConnTest 30 | import Hex2txtWeb.ConnCase 31 | end 32 | end 33 | 34 | setup _tags do 35 | {:ok, conn: Phoenix.ConnTest.build_conn()} 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/hex2txt_web/components/layouts/app.html.heex: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 | 5 | 6 | 7 |

8 | v{Application.spec(:phoenix, :vsn)} 9 |

10 |
11 |
12 | 13 | @elixirphoenix 14 | 15 | 16 | GitHub 17 | 18 | 22 | Get Started 23 | 24 |
25 |
26 |
27 |
28 |
29 | <.flash_group flash={@flash} /> 30 | {@inner_content} 31 |
32 |
33 | -------------------------------------------------------------------------------- /lib/hex2txt/generator.ex: -------------------------------------------------------------------------------- 1 | defmodule Hex2txt.Generator do 2 | alias Hex2txt.HexScraper 3 | 4 | def generate(package_name, package_version \\ nil) do 5 | title = 6 | "# " <> 7 | if package_version do 8 | "#{package_name} v#{package_version}" 9 | else 10 | package_name 11 | end 12 | 13 | description = 14 | HexScraper.get_package_description!(package_name) 15 | |> String.trim() 16 | |> String.split("\n", trim: false) 17 | |> Enum.map(&"> #{&1}") 18 | |> Enum.join("\n") 19 | 20 | search_data = HexScraper.get_search_data!(package_name, package_version) 21 | 22 | docs = 23 | search_data["items"] 24 | |> Enum.filter( 25 | &(&1["type"] in [ 26 | "module", 27 | "type", 28 | "function", 29 | "protocol", 30 | "behaviour", 31 | "callback", 32 | "macro" 33 | ]) 34 | ) 35 | |> Enum.map(fn item -> 36 | "### #{item["title"]} (#{item["type"]})\n\n#{item["doc"]}" 37 | end) 38 | |> Enum.join("\n\n") 39 | 40 | [ 41 | title, 42 | description, 43 | "## Docs", 44 | docs, 45 | "## Links", 46 | "- [Online documentation](https://hexdocs.pm/#{package_name})" 47 | ] 48 | |> Enum.join("\n\n") 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # This file excludes paths from the Docker build context. 2 | # 3 | # By default, Docker's build context includes all files (and folders) in the 4 | # current directory. Even if a file isn't copied into the container it is still sent to 5 | # the Docker daemon. 6 | # 7 | # There are multiple reasons to exclude files from the build context: 8 | # 9 | # 1. Prevent nested folders from being copied into the container (ex: exclude 10 | # /assets/node_modules when copying /assets) 11 | # 2. Reduce the size of the build context and improve build time (ex. /build, /deps, /doc) 12 | # 3. Avoid sending files containing sensitive information 13 | # 14 | # More information on using .dockerignore is available here: 15 | # https://docs.docker.com/engine/reference/builder/#dockerignore-file 16 | 17 | .dockerignore 18 | 19 | # Ignore git, but keep git HEAD and refs to access current commit hash if needed: 20 | # 21 | # $ cat .git/HEAD | awk '{print ".git/"$2}' | xargs cat 22 | # d0b8727759e1e0e7aa3d41707d12376e373d5ecc 23 | .git 24 | !.git/HEAD 25 | !.git/refs 26 | 27 | # Common development/test artifacts 28 | /cover/ 29 | /doc/ 30 | /test/ 31 | /tmp/ 32 | .elixir_ls 33 | 34 | # Mix artifacts 35 | /_build/ 36 | /deps/ 37 | *.ez 38 | 39 | # Generated on crash by the VM 40 | erl_crash.dump 41 | 42 | # Static artifacts - These should be fetched and built inside the Docker image 43 | /assets/node_modules/ 44 | /priv/static/assets/ 45 | /priv/static/cache_manifest.json 46 | -------------------------------------------------------------------------------- /lib/hex2txt_web/router.ex: -------------------------------------------------------------------------------- 1 | defmodule Hex2txtWeb.Router do 2 | use Hex2txtWeb, :router 3 | 4 | pipeline :browser do 5 | plug :accepts, ["html"] 6 | plug :fetch_session 7 | plug :fetch_live_flash 8 | plug :put_root_layout, html: {Hex2txtWeb.Layouts, :root} 9 | plug :protect_from_forgery 10 | plug :put_secure_browser_headers 11 | end 12 | 13 | pipeline :api do 14 | plug :accepts, ["json"] 15 | end 16 | 17 | scope "/", Hex2txtWeb do 18 | pipe_through :browser 19 | 20 | get "/", PageController, :home 21 | 22 | get "/:package/llms.txt", PageController, :llms_txt 23 | get "/:package/:version/llms.txt", PageController, :llms_txt 24 | end 25 | 26 | # Other scopes may use custom stacks. 27 | # scope "/api", Hex2txtWeb do 28 | # pipe_through :api 29 | # end 30 | 31 | # Enable LiveDashboard in development 32 | if Application.compile_env(:hex2txt, :dev_routes) do 33 | # If you want to use the LiveDashboard in production, you should put 34 | # it behind authentication and allow only admins to access it. 35 | # If your application does not have an admins-only section yet, 36 | # you can use Plug.BasicAuth to set up some basic authentication 37 | # as long as you are also using SSL (which you should anyway). 38 | import Phoenix.LiveDashboard.Router 39 | 40 | scope "/dev" do 41 | pipe_through :browser 42 | 43 | live_dashboard "/dashboard", metrics: Hex2txtWeb.Telemetry 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | dev: replapp 2 | 3 | test: 4 | mix test 5 | 6 | lint: 7 | mix format --check-formatted 8 | 9 | build: 10 | mix compile --warnings-as-errors --force 11 | 12 | compile: build 13 | 14 | ci: 15 | mix ci 16 | 17 | setup: 18 | mix setup 19 | 20 | clean: 21 | mix clean 22 | mix deps.clean --unused mix 23 | mix phx.digest.clean 24 | 25 | clean-all: 26 | mix deps.clean --all --force 27 | mix clean --all --force 28 | phx.digest.clean 29 | mix local.hex --force 30 | mix local.rebar --force 31 | mix deps.get 32 | 33 | # Start the Phoenix webserver, enabling the node to accept connections from 34 | # other nodes (including those nodes run by Livebook). 35 | runapp: 36 | elixir --name $NODE_NAME@127.0.0.1 --cookie $NODE_COOKIE -S mix phx.server 37 | 38 | # Like `runapp`, but also starts an IEx instance. 39 | replapp: 40 | iex --name $NODE_NAME@127.0.0.1 --cookie $NODE_COOKIE -S mix phx.server 41 | 42 | # Start an IEx instance that connects to a node started with `runapp` or 43 | # `replapp`. (`iexc` is short for "IEx connect".) 44 | iexc: 45 | iex --cookie "$NODE_COOKIE" --remsh "$NODE_NAME@127.0.0.1" 46 | 47 | print-versions: print-erlang-version print-elixir-version print-node-version 48 | 49 | print-erlang-version: 50 | elixir -e 'IO.puts(System.otp_release)' 51 | 52 | print-elixir-version: 53 | elixir -e 'IO.puts(System.version)' 54 | 55 | print-node-version: 56 | node -e 'console.log(process.version.replace(/^v/,""))' 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hex2txt 2 | 3 | Convert documentation for [hex.pm packages](https://hex.pm) to [`/llms.txt` 4 | files](https://llmstxt.org). 5 | 6 | ## Why? 7 | 8 | The LLM you're using to help with development might not know about the 9 | libraries you're using. Use `hex2txt` to generate documentation for 10 | inference-time consumption. 11 | 12 | If the documentation is too large for your LLM's context window, use the 13 | [`hex2context` Livebook](./notebooks/hex2context.livemd). `hex2context` 14 | performs Retrieval-Augmented Generation, including only the most relevant 15 | subset of documentation for the task-at-hand. 16 | 17 | ## Try It 18 | 19 | ### `hex2txt` 20 | 21 | See [https://hex2txt.fly.dev](https://hex2txt.fly.dev). 22 | 23 | Some examples: 24 | 25 | - https://hex2txt.fly.dev/geo/llms.txt 26 | - https://hex2txt.fly.dev/flop_phoenix/llms.txt 27 | - https://hex2txt.fly.dev/phoenix_live_view/llms.txt 28 | - https://hex2txt.fly.dev/elixir/1.18.0-rc.0/llms.txt 29 | - https://hex2txt.fly.dev/ecto_ulid_next/1.0.2/llms.txt 30 | - https://hex2txt.fly.dev/phoenix/1.7.17/llms.txt 31 | 32 | ### `hex2context` 33 | 34 | Run the notebook directly by clicking on this badge: 35 | 36 | [![Run in Livebook](https://livebook.dev/badge/v1/blue.svg)](https://livebook.dev/run?url=https%3A%2F%2Fgithub.com%2Fmjrusso%2Fhex2txt%2Fblob%2Fmain%2Fnotebooks%2Fhex2context.livemd) 37 | 38 | ## License 39 | 40 | hex2txt is released under the terms of the [Apache License 2.0](LICENSE). 41 | 42 | Copyright (c) 2024, [Michael Russo](https://mjrusso.com). 43 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where 3rd-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Temporary files, for example, from tests. 23 | /tmp/ 24 | 25 | # Ignore package tarball (built via "mix hex.build"). 26 | hex2txt-*.tar 27 | 28 | # Ignore assets that are produced by build tools. 29 | /priv/static/assets/ 30 | 31 | # Ignore digested assets cache. 32 | /priv/static/cache_manifest.json 33 | 34 | # In case you use Node.js/npm, you want to ignore these. 35 | npm-debug.log 36 | /assets/node_modules/ 37 | 38 | # Ignore artifacts produced when running in Nix shell. These artifacts are not 39 | # Nix-specific, but are configured as part of the project's Flake's 40 | # `shellHook`. These include configuration files and scripts related to Mix and 41 | # Hex, as well as files related to storing shell history for Erlang and IEx. 42 | .nix-shell/ 43 | 44 | # Developer-specific extensions for IEx. 45 | .iex.local.exs 46 | 47 | # Livebook notebooks that are not committed to source control. 48 | *.local.livemd 49 | 50 | # Local sqlite database files. 51 | hex2context.db* 52 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "flake-utils": { 4 | "inputs": { 5 | "systems": "systems" 6 | }, 7 | "locked": { 8 | "lastModified": 1731533236, 9 | "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", 10 | "owner": "numtide", 11 | "repo": "flake-utils", 12 | "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", 13 | "type": "github" 14 | }, 15 | "original": { 16 | "owner": "numtide", 17 | "repo": "flake-utils", 18 | "type": "github" 19 | } 20 | }, 21 | "nixpkgs": { 22 | "locked": { 23 | "lastModified": 1735291276, 24 | "narHash": "sha256-NYVcA06+blsLG6wpAbSPTCyLvxD/92Hy4vlY9WxFI1M=", 25 | "owner": "NixOS", 26 | "repo": "nixpkgs", 27 | "rev": "634fd46801442d760e09493a794c4f15db2d0cbb", 28 | "type": "github" 29 | }, 30 | "original": { 31 | "owner": "NixOS", 32 | "ref": "nixos-unstable", 33 | "repo": "nixpkgs", 34 | "type": "github" 35 | } 36 | }, 37 | "root": { 38 | "inputs": { 39 | "flake-utils": "flake-utils", 40 | "nixpkgs": "nixpkgs" 41 | } 42 | }, 43 | "systems": { 44 | "locked": { 45 | "lastModified": 1681028828, 46 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 47 | "owner": "nix-systems", 48 | "repo": "default", 49 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 50 | "type": "github" 51 | }, 52 | "original": { 53 | "owner": "nix-systems", 54 | "repo": "default", 55 | "type": "github" 56 | } 57 | } 58 | }, 59 | "root": "root", 60 | "version": 7 61 | } 62 | -------------------------------------------------------------------------------- /lib/hex2txt_web/endpoint.ex: -------------------------------------------------------------------------------- 1 | defmodule Hex2txtWeb.Endpoint do 2 | use Phoenix.Endpoint, otp_app: :hex2txt 3 | 4 | # The session will be stored in the cookie and signed, 5 | # this means its contents can be read but not tampered with. 6 | # Set :encryption_salt if you would also like to encrypt it. 7 | @session_options [ 8 | store: :cookie, 9 | key: "_hex2txt_key", 10 | signing_salt: "YVqHb7DK", 11 | same_site: "Lax" 12 | ] 13 | 14 | socket "/live", Phoenix.LiveView.Socket, 15 | websocket: [connect_info: [session: @session_options]], 16 | longpoll: [connect_info: [session: @session_options]] 17 | 18 | # Serve at "/" the static files from "priv/static" directory. 19 | # 20 | # You should set gzip to true if you are running phx.digest 21 | # when deploying your static files in production. 22 | plug Plug.Static, 23 | at: "/", 24 | from: :hex2txt, 25 | gzip: false, 26 | only: Hex2txtWeb.static_paths() 27 | 28 | # Code reloading can be explicitly enabled under the 29 | # :code_reloader configuration of your endpoint. 30 | if code_reloading? do 31 | socket "/phoenix/live_reload/socket", Phoenix.LiveReloader.Socket 32 | plug Phoenix.LiveReloader 33 | plug Phoenix.CodeReloader 34 | end 35 | 36 | plug Phoenix.LiveDashboard.RequestLogger, 37 | param_key: "request_logger", 38 | cookie_key: "request_logger" 39 | 40 | plug Plug.RequestId 41 | plug Plug.Telemetry, event_prefix: [:phoenix, :endpoint] 42 | 43 | plug Plug.Parsers, 44 | parsers: [:urlencoded, :multipart, :json], 45 | pass: ["*/*"], 46 | json_decoder: Phoenix.json_library() 47 | 48 | plug Plug.MethodOverride 49 | plug Plug.Head 50 | plug Plug.Session, @session_options 51 | plug Hex2txtWeb.Router 52 | end 53 | -------------------------------------------------------------------------------- /assets/js/app.js: -------------------------------------------------------------------------------- 1 | // If you want to use Phoenix channels, run `mix help phx.gen.channel` 2 | // to get started and then uncomment the line below. 3 | // import "./user_socket.js" 4 | 5 | // You can include dependencies in two ways. 6 | // 7 | // The simplest option is to put them in assets/vendor and 8 | // import them using relative paths: 9 | // 10 | // import "../vendor/some-package.js" 11 | // 12 | // Alternatively, you can `npm install some-package --prefix assets` and import 13 | // them using a path starting with the package name: 14 | // 15 | // import "some-package" 16 | // 17 | 18 | // Include phoenix_html to handle method=PUT/DELETE in forms and buttons. 19 | import "phoenix_html" 20 | // Establish Phoenix Socket and LiveView configuration. 21 | import {Socket} from "phoenix" 22 | import {LiveSocket} from "phoenix_live_view" 23 | import topbar from "../vendor/topbar" 24 | 25 | let csrfToken = document.querySelector("meta[name='csrf-token']").getAttribute("content") 26 | let liveSocket = new LiveSocket("/live", Socket, { 27 | longPollFallbackMs: 2500, 28 | params: {_csrf_token: csrfToken} 29 | }) 30 | 31 | // Show progress bar on live navigation and form submits 32 | topbar.config({barColors: {0: "#29d"}, shadowColor: "rgba(0, 0, 0, .3)"}) 33 | window.addEventListener("phx:page-loading-start", _info => topbar.show(300)) 34 | window.addEventListener("phx:page-loading-stop", _info => topbar.hide()) 35 | 36 | // connect if there are any LiveViews on the page 37 | liveSocket.connect() 38 | 39 | // expose liveSocket on window for web console debug logs and latency simulation: 40 | // >> liveSocket.enableDebug() 41 | // >> liveSocket.enableLatencySim(1000) // enabled for duration of browser session 42 | // >> liveSocket.disableLatencySim() 43 | window.liveSocket = liveSocket 44 | 45 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Config module. 3 | # 4 | # This configuration file is loaded before any dependency and 5 | # is restricted to this project. 6 | 7 | # General application configuration 8 | import Config 9 | 10 | config :hex2txt, 11 | generators: [timestamp_type: :utc_datetime] 12 | 13 | # Configures the endpoint 14 | config :hex2txt, Hex2txtWeb.Endpoint, 15 | url: [host: "localhost"], 16 | adapter: Bandit.PhoenixAdapter, 17 | render_errors: [ 18 | formats: [html: Hex2txtWeb.ErrorHTML, json: Hex2txtWeb.ErrorJSON], 19 | layout: false 20 | ], 21 | pubsub_server: Hex2txt.PubSub, 22 | live_view: [signing_salt: "xgE1Q34P"] 23 | 24 | # Configure esbuild (the version is required) 25 | config :esbuild, 26 | version: "0.17.11", 27 | hex2txt: [ 28 | args: 29 | ~w(js/app.js --bundle --target=es2017 --outdir=../priv/static/assets --external:/fonts/* --external:/images/*), 30 | cd: Path.expand("../assets", __DIR__), 31 | env: %{"NODE_PATH" => Path.expand("../deps", __DIR__)} 32 | ] 33 | 34 | # Configure tailwind (the version is required) 35 | config :tailwind, 36 | version: "3.4.3", 37 | hex2txt: [ 38 | args: ~w( 39 | --config=tailwind.config.js 40 | --input=css/app.css 41 | --output=../priv/static/assets/app.css 42 | ), 43 | cd: Path.expand("../assets", __DIR__) 44 | ] 45 | 46 | # Configures Elixir's Logger 47 | config :logger, :console, 48 | format: "$time $metadata[$level] $message\n", 49 | metadata: [:request_id] 50 | 51 | # Use Jason for JSON parsing in Phoenix 52 | config :phoenix, :json_library, Jason 53 | 54 | # Import environment specific config. This must remain at the bottom 55 | # of this file so it overrides the configuration defined above. 56 | import_config "#{config_env()}.exs" 57 | -------------------------------------------------------------------------------- /lib/hex2txt/hex_scraper.ex: -------------------------------------------------------------------------------- 1 | defmodule Hex2txt.HexScraper do 2 | @hexpm "https://hex.pm" 3 | @hexdocs "https://hexdocs.pm" 4 | 5 | def get_package_description!("elixir") do 6 | "Elixir is a dynamic, functional language for building scalable and maintainable applications." 7 | end 8 | 9 | def get_package_description!(package_name) do 10 | req = Req.new() |> ReqEasyHTML.attach() 11 | 12 | Req.get!(req, url: hexpm_package_url(package_name)).body[ 13 | "meta[name=\"description\"]" 14 | ] 15 | |> Map.get(:nodes) 16 | |> Floki.attribute("content") 17 | |> List.first() 18 | end 19 | 20 | def get_search_data!(package_name, package_version \\ nil) do 21 | get!(package_name, package_version, "script[src^=\"dist/search_data-\"]", "searchData") 22 | end 23 | 24 | def get_sidebar_items!(package_name, package_version \\ nil) do 25 | get!(package_name, package_version, "script[src^=\"dist/sidebar_items-\"]", "sidebarNodes") 26 | end 27 | 28 | defp get!(package_name, package_version, selector, js_var) do 29 | find_json_url!(package_name, package_version, selector) 30 | |> decode_json!(js_var) 31 | end 32 | 33 | defp find_json_url!(package_name, package_version, selector) do 34 | req = Req.new() |> ReqEasyHTML.attach() 35 | 36 | src = 37 | Req.get!(req, url: hexdocs_asset_url("search.html", package_name, package_version)).body[ 38 | selector 39 | ] 40 | |> Map.get(:nodes) 41 | |> Floki.attribute("src") 42 | |> List.first() 43 | 44 | hexdocs_asset_url(src, package_name, package_version) 45 | end 46 | 47 | defp decode_json!(url, js_var) do 48 | req = Req.new() 49 | 50 | Req.get!(req, url: url).body 51 | |> String.trim_leading("#{js_var}=") 52 | |> Jason.decode!() 53 | end 54 | 55 | defp hexpm_package_url(package_name) do 56 | "#{@hexpm}/packages/#{package_name}" 57 | end 58 | 59 | defp hexdocs_asset_url(src, package_name, nil) do 60 | "#{@hexdocs}/#{package_name}/#{src}" 61 | end 62 | 63 | defp hexdocs_asset_url(src, package_name, package_version) do 64 | "#{@hexdocs}/#{package_name}/#{package_version}/#{src}" 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /lib/hex2txt_web/telemetry.ex: -------------------------------------------------------------------------------- 1 | defmodule Hex2txtWeb.Telemetry do 2 | use Supervisor 3 | import Telemetry.Metrics 4 | 5 | def start_link(arg) do 6 | Supervisor.start_link(__MODULE__, arg, name: __MODULE__) 7 | end 8 | 9 | @impl true 10 | def init(_arg) do 11 | children = [ 12 | # Telemetry poller will execute the given period measurements 13 | # every 10_000ms. Learn more here: https://hexdocs.pm/telemetry_metrics 14 | {:telemetry_poller, measurements: periodic_measurements(), period: 10_000} 15 | # Add reporters as children of your supervision tree. 16 | # {Telemetry.Metrics.ConsoleReporter, metrics: metrics()} 17 | ] 18 | 19 | Supervisor.init(children, strategy: :one_for_one) 20 | end 21 | 22 | def metrics do 23 | [ 24 | # Phoenix Metrics 25 | summary("phoenix.endpoint.start.system_time", 26 | unit: {:native, :millisecond} 27 | ), 28 | summary("phoenix.endpoint.stop.duration", 29 | unit: {:native, :millisecond} 30 | ), 31 | summary("phoenix.router_dispatch.start.system_time", 32 | tags: [:route], 33 | unit: {:native, :millisecond} 34 | ), 35 | summary("phoenix.router_dispatch.exception.duration", 36 | tags: [:route], 37 | unit: {:native, :millisecond} 38 | ), 39 | summary("phoenix.router_dispatch.stop.duration", 40 | tags: [:route], 41 | unit: {:native, :millisecond} 42 | ), 43 | summary("phoenix.socket_connected.duration", 44 | unit: {:native, :millisecond} 45 | ), 46 | summary("phoenix.channel_joined.duration", 47 | unit: {:native, :millisecond} 48 | ), 49 | summary("phoenix.channel_handled_in.duration", 50 | tags: [:event], 51 | unit: {:native, :millisecond} 52 | ), 53 | 54 | # VM Metrics 55 | summary("vm.memory.total", unit: {:byte, :kilobyte}), 56 | summary("vm.total_run_queue_lengths.total"), 57 | summary("vm.total_run_queue_lengths.cpu"), 58 | summary("vm.total_run_queue_lengths.io") 59 | ] 60 | end 61 | 62 | defp periodic_measurements do 63 | [ 64 | # A module, function and arguments to be invoked periodically. 65 | # This function must call :telemetry.execute/3 and a metric must be added above. 66 | # {Hex2txtWeb, :count_users, []} 67 | ] 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Hex2txt.MixProject do 2 | use Mix.Project 3 | 4 | def project do 5 | [ 6 | app: :hex2txt, 7 | version: "0.1.0", 8 | elixir: "~> 1.14", 9 | elixirc_paths: elixirc_paths(Mix.env()), 10 | start_permanent: Mix.env() == :prod, 11 | aliases: aliases(), 12 | deps: deps() 13 | ] 14 | end 15 | 16 | # Configuration for the OTP application. 17 | # 18 | # Type `mix help compile.app` for more information. 19 | def application do 20 | [ 21 | mod: {Hex2txt.Application, []}, 22 | extra_applications: [:logger, :runtime_tools] 23 | ] 24 | end 25 | 26 | # Specifies which paths to compile per environment. 27 | defp elixirc_paths(:test), do: ["lib", "test/support"] 28 | defp elixirc_paths(_), do: ["lib"] 29 | 30 | # Specifies your project dependencies. 31 | # 32 | # Type `mix help deps` for examples and options. 33 | defp deps do 34 | [ 35 | {:phoenix, "~> 1.7.18"}, 36 | {:phoenix_html, "~> 4.1"}, 37 | {:phoenix_live_reload, "~> 1.2", only: :dev}, 38 | {:phoenix_live_view, "~> 1.0.0"}, 39 | {:floki, ">= 0.30.0"}, 40 | {:phoenix_live_dashboard, "~> 0.8.3"}, 41 | {:esbuild, "~> 0.8", runtime: Mix.env() == :dev}, 42 | {:tailwind, "~> 0.2", runtime: Mix.env() == :dev}, 43 | {:heroicons, 44 | github: "tailwindlabs/heroicons", 45 | tag: "v2.1.1", 46 | sparse: "optimized", 47 | app: false, 48 | compile: false, 49 | depth: 1}, 50 | {:telemetry_metrics, "~> 1.0"}, 51 | {:telemetry_poller, "~> 1.0"}, 52 | {:jason, "~> 1.2"}, 53 | {:dns_cluster, "~> 0.1.1"}, 54 | {:bandit, "~> 1.5"}, 55 | {:req, "~> 0.5.8"}, 56 | {:req_easyhtml, "~> 0.1.3"} 57 | ] 58 | end 59 | 60 | # Aliases are shortcuts or tasks specific to the current project. 61 | # For example, to install project dependencies and perform other setup tasks, run: 62 | # 63 | # $ mix setup 64 | # 65 | # See the documentation for `Mix` for more info on aliases. 66 | defp aliases do 67 | [ 68 | setup: ["deps.get", "assets.setup", "assets.build"], 69 | "assets.setup": ["tailwind.install --if-missing", "esbuild.install --if-missing"], 70 | "assets.build": ["tailwind hex2txt", "esbuild hex2txt"], 71 | "assets.deploy": [ 72 | "tailwind hex2txt --minify", 73 | "esbuild hex2txt --minify", 74 | "phx.digest" 75 | ] 76 | ] 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /config/dev.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | # For development, we disable any cache and enable 4 | # debugging and code reloading. 5 | # 6 | # The watchers configuration can be used to run external 7 | # watchers to your application. For example, we can use it 8 | # to bundle .js and .css sources. 9 | config :hex2txt, Hex2txtWeb.Endpoint, 10 | # Binding to loopback ipv4 address prevents access from other machines. 11 | # Change to `ip: {0, 0, 0, 0}` to allow access from other machines. 12 | http: [ip: {127, 0, 0, 1}, port: 4000], 13 | check_origin: false, 14 | code_reloader: true, 15 | debug_errors: true, 16 | secret_key_base: "cF7Ri6AMsN/i4S/09pzUfZkjpWZV3Od7clcUU6oJ3njWAvM5uNBaioTWZJrolqNm", 17 | watchers: [ 18 | esbuild: {Esbuild, :install_and_run, [:hex2txt, ~w(--sourcemap=inline --watch)]}, 19 | tailwind: {Tailwind, :install_and_run, [:hex2txt, ~w(--watch)]} 20 | ] 21 | 22 | # ## SSL Support 23 | # 24 | # In order to use HTTPS in development, a self-signed 25 | # certificate can be generated by running the following 26 | # Mix task: 27 | # 28 | # mix phx.gen.cert 29 | # 30 | # Run `mix help phx.gen.cert` for more information. 31 | # 32 | # The `http:` config above can be replaced with: 33 | # 34 | # https: [ 35 | # port: 4001, 36 | # cipher_suite: :strong, 37 | # keyfile: "priv/cert/selfsigned_key.pem", 38 | # certfile: "priv/cert/selfsigned.pem" 39 | # ], 40 | # 41 | # If desired, both `http:` and `https:` keys can be 42 | # configured to run both http and https servers on 43 | # different ports. 44 | 45 | # Watch static and templates for browser reloading. 46 | config :hex2txt, Hex2txtWeb.Endpoint, 47 | live_reload: [ 48 | patterns: [ 49 | ~r"priv/static/(?!uploads/).*(js|css|png|jpeg|jpg|gif|svg)$", 50 | ~r"lib/hex2txt_web/(controllers|live|components)/.*(ex|heex)$" 51 | ] 52 | ] 53 | 54 | # Enable dev routes for dashboard and mailbox 55 | config :hex2txt, dev_routes: true 56 | 57 | # Do not include metadata nor timestamps in development logs 58 | config :logger, :console, format: "[$level] $message\n" 59 | 60 | # Set a higher stacktrace during development. Avoid configuring such 61 | # in production as building large stacktraces may be expensive. 62 | config :phoenix, :stacktrace_depth, 20 63 | 64 | # Initialize plugs at runtime for faster development compilation 65 | config :phoenix, :plug_init_mode, :runtime 66 | 67 | config :phoenix_live_view, 68 | # Include HEEx debug annotations as HTML comments in rendered markup 69 | debug_heex_annotations: true, 70 | # Enable helpful, but potentially expensive runtime checks 71 | enable_expensive_runtime_checks: true 72 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "Nix-Flake-based development environment for hex2txt"; 3 | inputs = { 4 | nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; 5 | flake-utils.url = "github:numtide/flake-utils"; 6 | }; 7 | 8 | outputs = { self, nixpkgs, flake-utils }: 9 | flake-utils.lib.eachDefaultSystem (system: 10 | let 11 | pkgs = nixpkgs.legacyPackages.${system}; 12 | elixir = pkgs.elixir; 13 | elixir-ls = pkgs.elixir-ls; 14 | next-ls = pkgs.next-ls; 15 | lexical = pkgs.lexical; 16 | node = pkgs.nodejs_20; 17 | 18 | base-packages = [ elixir pkgs.git node pkgs.just ]; 19 | 20 | elixir-language-servers = [ elixir-ls next-ls lexical ]; 21 | 22 | base-scripts = [ ]; 23 | 24 | in { 25 | 26 | # A stripped-down dev shell, for use in CI environments. 27 | # 28 | # Example usage: 29 | # 30 | # nix develop .#ci -c COMMAND 31 | # 32 | # This shell is faster to build than the default, because it doesn't 33 | # include the Elixir language servers and other dependencies that are 34 | # only useful for development. 35 | devShells.ci = 36 | pkgs.mkShell { packages = base-packages ++ base-scripts; }; 37 | 38 | devShells.default = pkgs.mkShell { 39 | packages = base-packages ++ elixir-language-servers ++ base-scripts 40 | 41 | ++ pkgs.lib.optionals pkgs.stdenv.isLinux 42 | (with pkgs; [ inotify-tools libnotify ]) 43 | 44 | ++ pkgs.lib.optionals pkgs.stdenv.isDarwin 45 | (with pkgs.darwin.apple_sdk.frameworks; [ 46 | pkgs.terminal-notifier 47 | CoreFoundation 48 | CoreServices 49 | ]); 50 | 51 | shellHook = '' 52 | export PATH=$PWD/assets/node_modules/.bin:$PATH 53 | 54 | # Store configuration files and scripts used by Mix in this local directory. 55 | mkdir -p .nix-shell/.mix 56 | export MIX_HOME=$PWD/.nix-shell/.mix 57 | export PATH=$MIX_HOME/bin:$PATH 58 | export PATH=$MIX_HOME/escripts:$PATH 59 | 60 | # Store cache and configuration files used by Hex in this local directory. 61 | mkdir -p .nix-shell/.hex 62 | export HEX_HOME=$PWD/.nix-shell/.hex 63 | export PATH=$HEX_HOME/bin:$PATH 64 | 65 | # Enable shell history for IEx. 66 | export ERL_AFLAGS="-kernel shell_history enabled -kernel shell_history_path '\"$PWD/.nix-shell/.erlang-history\"'" 67 | ''; 68 | 69 | }; 70 | }); 71 | 72 | } 73 | -------------------------------------------------------------------------------- /assets/tailwind.config.js: -------------------------------------------------------------------------------- 1 | // See the Tailwind configuration guide for advanced usage 2 | // https://tailwindcss.com/docs/configuration 3 | 4 | const plugin = require("tailwindcss/plugin") 5 | const fs = require("fs") 6 | const path = require("path") 7 | 8 | module.exports = { 9 | content: [ 10 | "./js/**/*.js", 11 | "../lib/hex2txt_web.ex", 12 | "../lib/hex2txt_web/**/*.*ex" 13 | ], 14 | theme: { 15 | extend: { 16 | colors: { 17 | brand: "#1E90FF", 18 | } 19 | }, 20 | }, 21 | plugins: [ 22 | require("@tailwindcss/forms"), 23 | // Allows prefixing tailwind classes with LiveView classes to add rules 24 | // only when LiveView classes are applied, for example: 25 | // 26 | //
27 | // 28 | plugin(({addVariant}) => addVariant("phx-click-loading", [".phx-click-loading&", ".phx-click-loading &"])), 29 | plugin(({addVariant}) => addVariant("phx-submit-loading", [".phx-submit-loading&", ".phx-submit-loading &"])), 30 | plugin(({addVariant}) => addVariant("phx-change-loading", [".phx-change-loading&", ".phx-change-loading &"])), 31 | 32 | // Embeds Heroicons (https://heroicons.com) into your app.css bundle 33 | // See your `CoreComponents.icon/1` for more information. 34 | // 35 | plugin(function({matchComponents, theme}) { 36 | let iconsDir = path.join(__dirname, "../deps/heroicons/optimized") 37 | let values = {} 38 | let icons = [ 39 | ["", "/24/outline"], 40 | ["-solid", "/24/solid"], 41 | ["-mini", "/20/solid"], 42 | ["-micro", "/16/solid"] 43 | ] 44 | icons.forEach(([suffix, dir]) => { 45 | fs.readdirSync(path.join(iconsDir, dir)).forEach(file => { 46 | let name = path.basename(file, ".svg") + suffix 47 | values[name] = {name, fullPath: path.join(iconsDir, dir, file)} 48 | }) 49 | }) 50 | matchComponents({ 51 | "hero": ({name, fullPath}) => { 52 | let content = fs.readFileSync(fullPath).toString().replace(/\r?\n|\r/g, "") 53 | let size = theme("spacing.6") 54 | if (name.endsWith("-mini")) { 55 | size = theme("spacing.5") 56 | } else if (name.endsWith("-micro")) { 57 | size = theme("spacing.4") 58 | } 59 | return { 60 | [`--hero-${name}`]: `url('data:image/svg+xml;utf8,${content}')`, 61 | "-webkit-mask": `var(--hero-${name})`, 62 | "mask": `var(--hero-${name})`, 63 | "mask-repeat": "no-repeat", 64 | "background-color": "currentColor", 65 | "vertical-align": "middle", 66 | "display": "inline-block", 67 | "width": size, 68 | "height": size 69 | } 70 | } 71 | }, {values}) 72 | }) 73 | ] 74 | } 75 | -------------------------------------------------------------------------------- /priv/static/images/logo.svg: -------------------------------------------------------------------------------- 1 | 7 | -------------------------------------------------------------------------------- /lib/hex2txt_web.ex: -------------------------------------------------------------------------------- 1 | defmodule Hex2txtWeb do 2 | @moduledoc """ 3 | The entrypoint for defining your web interface, such 4 | as controllers, components, channels, and so on. 5 | 6 | This can be used in your application as: 7 | 8 | use Hex2txtWeb, :controller 9 | use Hex2txtWeb, :html 10 | 11 | The definitions below will be executed for every controller, 12 | component, etc, so keep them short and clean, focused 13 | on imports, uses and aliases. 14 | 15 | Do NOT define functions inside the quoted expressions 16 | below. Instead, define additional modules and import 17 | those modules here. 18 | """ 19 | 20 | def static_paths, do: ~w(assets fonts images favicon.ico robots.txt) 21 | 22 | def router do 23 | quote do 24 | use Phoenix.Router, helpers: false 25 | 26 | # Import common connection and controller functions to use in pipelines 27 | import Plug.Conn 28 | import Phoenix.Controller 29 | import Phoenix.LiveView.Router 30 | end 31 | end 32 | 33 | def channel do 34 | quote do 35 | use Phoenix.Channel 36 | end 37 | end 38 | 39 | def controller do 40 | quote do 41 | use Phoenix.Controller, 42 | formats: [:html, :json], 43 | layouts: [html: Hex2txtWeb.Layouts] 44 | 45 | import Plug.Conn 46 | 47 | unquote(verified_routes()) 48 | end 49 | end 50 | 51 | def live_view do 52 | quote do 53 | use Phoenix.LiveView, 54 | layout: {Hex2txtWeb.Layouts, :app} 55 | 56 | unquote(html_helpers()) 57 | end 58 | end 59 | 60 | def live_component do 61 | quote do 62 | use Phoenix.LiveComponent 63 | 64 | unquote(html_helpers()) 65 | end 66 | end 67 | 68 | def html do 69 | quote do 70 | use Phoenix.Component 71 | 72 | # Import convenience functions from controllers 73 | import Phoenix.Controller, 74 | only: [get_csrf_token: 0, view_module: 1, view_template: 1] 75 | 76 | # Include general helpers for rendering HTML 77 | unquote(html_helpers()) 78 | end 79 | end 80 | 81 | defp html_helpers do 82 | quote do 83 | # HTML escaping functionality 84 | import Phoenix.HTML 85 | # Core UI components 86 | import Hex2txtWeb.CoreComponents 87 | 88 | # Shortcut for generating JS commands 89 | alias Phoenix.LiveView.JS 90 | 91 | # Routes generation with the ~p sigil 92 | unquote(verified_routes()) 93 | end 94 | end 95 | 96 | def verified_routes do 97 | quote do 98 | use Phoenix.VerifiedRoutes, 99 | endpoint: Hex2txtWeb.Endpoint, 100 | router: Hex2txtWeb.Router, 101 | statics: Hex2txtWeb.static_paths() 102 | end 103 | end 104 | 105 | @doc """ 106 | When used, dispatch to the appropriate controller/live_view/etc. 107 | """ 108 | defmacro __using__(which) when is_atom(which) do 109 | apply(__MODULE__, which, []) 110 | end 111 | end 112 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Find eligible builder and runner images on Docker Hub. We use Ubuntu/Debian 2 | # instead of Alpine to avoid DNS resolution issues in production. 3 | # 4 | # https://hub.docker.com/r/hexpm/elixir/tags?page=1&name=ubuntu 5 | # https://hub.docker.com/_/ubuntu?tab=tags 6 | # 7 | # This file is based on these images: 8 | # 9 | # - https://hub.docker.com/r/hexpm/elixir/tags - for the build image 10 | # - https://hub.docker.com/_/debian?tab=tags&page=1&name=bullseye-20241202-slim - for the release image 11 | # - https://pkgs.org/ - resource for finding needed packages 12 | # - Ex: hexpm/elixir:1.17.3-erlang-27.1.2-debian-bullseye-20241202-slim 13 | # 14 | ARG ELIXIR_VERSION=1.18.1 15 | ARG OTP_VERSION=27.2 16 | ARG DEBIAN_VERSION=bullseye-20241223-slim 17 | 18 | ARG BUILDER_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}" 19 | ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}" 20 | 21 | FROM ${BUILDER_IMAGE} as builder 22 | 23 | # install build dependencies 24 | RUN apt-get update -y && apt-get install -y build-essential git \ 25 | && apt-get clean && rm -f /var/lib/apt/lists/*_* 26 | 27 | # prepare build dir 28 | WORKDIR /app 29 | 30 | # install hex + rebar 31 | RUN mix local.hex --force && \ 32 | mix local.rebar --force 33 | 34 | # set build ENV 35 | ENV MIX_ENV="prod" 36 | 37 | # install mix dependencies 38 | COPY mix.exs mix.lock ./ 39 | RUN mix deps.get --only $MIX_ENV 40 | RUN mkdir config 41 | 42 | # copy compile-time config files before we compile dependencies 43 | # to ensure any relevant config change will trigger the dependencies 44 | # to be re-compiled. 45 | COPY config/config.exs config/${MIX_ENV}.exs config/ 46 | RUN mix deps.compile 47 | 48 | COPY priv priv 49 | 50 | COPY lib lib 51 | 52 | COPY assets assets 53 | 54 | # compile assets 55 | RUN mix assets.deploy 56 | 57 | # Compile the release 58 | RUN mix compile 59 | 60 | # Changes to config/runtime.exs don't require recompiling the code 61 | COPY config/runtime.exs config/ 62 | 63 | COPY rel rel 64 | RUN mix release 65 | 66 | # start a new build stage so that the final image will only contain 67 | # the compiled release and other runtime necessities 68 | FROM ${RUNNER_IMAGE} 69 | 70 | RUN apt-get update -y && \ 71 | apt-get install -y libstdc++6 openssl libncurses5 locales ca-certificates \ 72 | && apt-get clean && rm -f /var/lib/apt/lists/*_* 73 | 74 | # Set the locale 75 | RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen 76 | 77 | ENV LANG en_US.UTF-8 78 | ENV LANGUAGE en_US:en 79 | ENV LC_ALL en_US.UTF-8 80 | 81 | WORKDIR "/app" 82 | RUN chown nobody /app 83 | 84 | # set runner ENV 85 | ENV MIX_ENV="prod" 86 | 87 | # Only copy the final release from the build stage 88 | COPY --from=builder --chown=nobody:root /app/_build/${MIX_ENV}/rel/hex2txt ./ 89 | 90 | USER nobody 91 | 92 | # If using an environment that doesn't automatically reap zombie processes, it is 93 | # advised to add an init process such as tini via `apt-get install` 94 | # above and adding an entrypoint. See https://github.com/krallin/tini for details 95 | # ENTRYPOINT ["/tini", "--"] 96 | 97 | CMD ["/app/bin/server"] 98 | -------------------------------------------------------------------------------- /config/runtime.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | # config/runtime.exs is executed for all environments, including 4 | # during releases. It is executed after compilation and before the 5 | # system starts, so it is typically used to load production configuration 6 | # and secrets from environment variables or elsewhere. Do not define 7 | # any compile-time configuration in here, as it won't be applied. 8 | # The block below contains prod specific runtime configuration. 9 | 10 | # ## Using releases 11 | # 12 | # If you use `mix release`, you need to explicitly enable the server 13 | # by passing the PHX_SERVER=true when you start it: 14 | # 15 | # PHX_SERVER=true bin/hex2txt start 16 | # 17 | # Alternatively, you can use `mix phx.gen.release` to generate a `bin/server` 18 | # script that automatically sets the env var above. 19 | if System.get_env("PHX_SERVER") do 20 | config :hex2txt, Hex2txtWeb.Endpoint, server: true 21 | end 22 | 23 | if config_env() == :prod do 24 | # The secret key base is used to sign/encrypt cookies and other secrets. 25 | # A default value is used in config/dev.exs and config/test.exs but you 26 | # want to use a different value for prod and you most likely don't want 27 | # to check this value into version control, so we use an environment 28 | # variable instead. 29 | secret_key_base = 30 | System.get_env("SECRET_KEY_BASE") || 31 | raise """ 32 | environment variable SECRET_KEY_BASE is missing. 33 | You can generate one by calling: mix phx.gen.secret 34 | """ 35 | 36 | host = System.get_env("PHX_HOST") || "example.com" 37 | port = String.to_integer(System.get_env("PORT") || "4000") 38 | 39 | config :hex2txt, :dns_cluster_query, System.get_env("DNS_CLUSTER_QUERY") 40 | 41 | config :hex2txt, Hex2txtWeb.Endpoint, 42 | url: [host: host, port: 443, scheme: "https"], 43 | http: [ 44 | # Enable IPv6 and bind on all interfaces. 45 | # Set it to {0, 0, 0, 0, 0, 0, 0, 1} for local network only access. 46 | # See the documentation on https://hexdocs.pm/bandit/Bandit.html#t:options/0 47 | # for details about using IPv6 vs IPv4 and loopback vs public addresses. 48 | ip: {0, 0, 0, 0, 0, 0, 0, 0}, 49 | port: port 50 | ], 51 | secret_key_base: secret_key_base 52 | 53 | # ## SSL Support 54 | # 55 | # To get SSL working, you will need to add the `https` key 56 | # to your endpoint configuration: 57 | # 58 | # config :hex2txt, Hex2txtWeb.Endpoint, 59 | # https: [ 60 | # ..., 61 | # port: 443, 62 | # cipher_suite: :strong, 63 | # keyfile: System.get_env("SOME_APP_SSL_KEY_PATH"), 64 | # certfile: System.get_env("SOME_APP_SSL_CERT_PATH") 65 | # ] 66 | # 67 | # The `cipher_suite` is set to `:strong` to support only the 68 | # latest and more secure SSL ciphers. This means old browsers 69 | # and clients may not be supported. You can set it to 70 | # `:compatible` for wider support. 71 | # 72 | # `:keyfile` and `:certfile` expect an absolute path to the key 73 | # and cert in disk or a relative path inside priv, for example 74 | # "priv/ssl/server.key". For all supported SSL configuration 75 | # options, see https://hexdocs.pm/plug/Plug.SSL.html#configure/1 76 | # 77 | # We also recommend setting `force_ssl` in your config/prod.exs, 78 | # ensuring no data is ever sent via http, always redirecting to https: 79 | # 80 | # config :hex2txt, Hex2txtWeb.Endpoint, 81 | # force_ssl: [hsts: true] 82 | # 83 | # Check `Plug.SSL` for all available options in `force_ssl`. 84 | end 85 | -------------------------------------------------------------------------------- /assets/vendor/topbar.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @license MIT 3 | * topbar 2.0.0, 2023-02-04 4 | * https://buunguyen.github.io/topbar 5 | * Copyright (c) 2021 Buu Nguyen 6 | */ 7 | (function (window, document) { 8 | "use strict"; 9 | 10 | // https://gist.github.com/paulirish/1579671 11 | (function () { 12 | var lastTime = 0; 13 | var vendors = ["ms", "moz", "webkit", "o"]; 14 | for (var x = 0; x < vendors.length && !window.requestAnimationFrame; ++x) { 15 | window.requestAnimationFrame = 16 | window[vendors[x] + "RequestAnimationFrame"]; 17 | window.cancelAnimationFrame = 18 | window[vendors[x] + "CancelAnimationFrame"] || 19 | window[vendors[x] + "CancelRequestAnimationFrame"]; 20 | } 21 | if (!window.requestAnimationFrame) 22 | window.requestAnimationFrame = function (callback, element) { 23 | var currTime = new Date().getTime(); 24 | var timeToCall = Math.max(0, 16 - (currTime - lastTime)); 25 | var id = window.setTimeout(function () { 26 | callback(currTime + timeToCall); 27 | }, timeToCall); 28 | lastTime = currTime + timeToCall; 29 | return id; 30 | }; 31 | if (!window.cancelAnimationFrame) 32 | window.cancelAnimationFrame = function (id) { 33 | clearTimeout(id); 34 | }; 35 | })(); 36 | 37 | var canvas, 38 | currentProgress, 39 | showing, 40 | progressTimerId = null, 41 | fadeTimerId = null, 42 | delayTimerId = null, 43 | addEvent = function (elem, type, handler) { 44 | if (elem.addEventListener) elem.addEventListener(type, handler, false); 45 | else if (elem.attachEvent) elem.attachEvent("on" + type, handler); 46 | else elem["on" + type] = handler; 47 | }, 48 | options = { 49 | autoRun: true, 50 | barThickness: 3, 51 | barColors: { 52 | 0: "rgba(26, 188, 156, .9)", 53 | ".25": "rgba(52, 152, 219, .9)", 54 | ".50": "rgba(241, 196, 15, .9)", 55 | ".75": "rgba(230, 126, 34, .9)", 56 | "1.0": "rgba(211, 84, 0, .9)", 57 | }, 58 | shadowBlur: 10, 59 | shadowColor: "rgba(0, 0, 0, .6)", 60 | className: null, 61 | }, 62 | repaint = function () { 63 | canvas.width = window.innerWidth; 64 | canvas.height = options.barThickness * 5; // need space for shadow 65 | 66 | var ctx = canvas.getContext("2d"); 67 | ctx.shadowBlur = options.shadowBlur; 68 | ctx.shadowColor = options.shadowColor; 69 | 70 | var lineGradient = ctx.createLinearGradient(0, 0, canvas.width, 0); 71 | for (var stop in options.barColors) 72 | lineGradient.addColorStop(stop, options.barColors[stop]); 73 | ctx.lineWidth = options.barThickness; 74 | ctx.beginPath(); 75 | ctx.moveTo(0, options.barThickness / 2); 76 | ctx.lineTo( 77 | Math.ceil(currentProgress * canvas.width), 78 | options.barThickness / 2 79 | ); 80 | ctx.strokeStyle = lineGradient; 81 | ctx.stroke(); 82 | }, 83 | createCanvas = function () { 84 | canvas = document.createElement("canvas"); 85 | var style = canvas.style; 86 | style.position = "fixed"; 87 | style.top = style.left = style.right = style.margin = style.padding = 0; 88 | style.zIndex = 100001; 89 | style.display = "none"; 90 | if (options.className) canvas.classList.add(options.className); 91 | document.body.appendChild(canvas); 92 | addEvent(window, "resize", repaint); 93 | }, 94 | topbar = { 95 | config: function (opts) { 96 | for (var key in opts) 97 | if (options.hasOwnProperty(key)) options[key] = opts[key]; 98 | }, 99 | show: function (delay) { 100 | if (showing) return; 101 | if (delay) { 102 | if (delayTimerId) return; 103 | delayTimerId = setTimeout(() => topbar.show(), delay); 104 | } else { 105 | showing = true; 106 | if (fadeTimerId !== null) window.cancelAnimationFrame(fadeTimerId); 107 | if (!canvas) createCanvas(); 108 | canvas.style.opacity = 1; 109 | canvas.style.display = "block"; 110 | topbar.progress(0); 111 | if (options.autoRun) { 112 | (function loop() { 113 | progressTimerId = window.requestAnimationFrame(loop); 114 | topbar.progress( 115 | "+" + 0.05 * Math.pow(1 - Math.sqrt(currentProgress), 2) 116 | ); 117 | })(); 118 | } 119 | } 120 | }, 121 | progress: function (to) { 122 | if (typeof to === "undefined") return currentProgress; 123 | if (typeof to === "string") { 124 | to = 125 | (to.indexOf("+") >= 0 || to.indexOf("-") >= 0 126 | ? currentProgress 127 | : 0) + parseFloat(to); 128 | } 129 | currentProgress = to > 1 ? 1 : to; 130 | repaint(); 131 | return currentProgress; 132 | }, 133 | hide: function () { 134 | clearTimeout(delayTimerId); 135 | delayTimerId = null; 136 | if (!showing) return; 137 | showing = false; 138 | if (progressTimerId != null) { 139 | window.cancelAnimationFrame(progressTimerId); 140 | progressTimerId = null; 141 | } 142 | (function loop() { 143 | if (topbar.progress("+.1") >= 1) { 144 | canvas.style.opacity -= 0.05; 145 | if (canvas.style.opacity <= 0.05) { 146 | canvas.style.display = "none"; 147 | fadeTimerId = null; 148 | return; 149 | } 150 | } 151 | fadeTimerId = window.requestAnimationFrame(loop); 152 | })(); 153 | }, 154 | }; 155 | 156 | if (typeof module === "object" && typeof module.exports === "object") { 157 | module.exports = topbar; 158 | } else if (typeof define === "function" && define.amd) { 159 | define(function () { 160 | return topbar; 161 | }); 162 | } else { 163 | this.topbar = topbar; 164 | } 165 | }.call(this, window, document)); 166 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "bandit": {:hex, :bandit, "1.6.1", "9e01b93d72ddc21d8c576a704949e86ee6cde7d11270a1d3073787876527a48f", [:mix], [{:hpax, "~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "5a904bf010ea24b67979835e0507688e31ac873d4ffc8ed0e5413e8d77455031"}, 3 | "castore": {:hex, :castore, "1.0.10", "43bbeeac820f16c89f79721af1b3e092399b3a1ecc8df1a472738fd853574911", [:mix], [], "hexpm", "1b0b7ea14d889d9ea21202c43a4fa015eb913021cb535e8ed91946f4b77a8848"}, 4 | "dns_cluster": {:hex, :dns_cluster, "0.1.3", "0bc20a2c88ed6cc494f2964075c359f8c2d00e1bf25518a6a6c7fd277c9b0c66", [:mix], [], "hexpm", "46cb7c4a1b3e52c7ad4cbe33ca5079fbde4840dedeafca2baf77996c2da1bc33"}, 5 | "easyhtml": {:hex, :easyhtml, "0.3.2", "050adfc8074f53b261f7dfe83303d864f1fbf5988245b369f8fdff1bf4c4b3e6", [:mix], [{:floki, "~> 0.35", [hex: :floki, repo: "hexpm", optional: false]}], "hexpm", "b6a936f91612a4870aa3e828cd8da5a08d9e3b6221b4d3012b6ec70b87845d06"}, 6 | "esbuild": {:hex, :esbuild, "0.8.2", "5f379dfa383ef482b738e7771daf238b2d1cfb0222bef9d3b20d4c8f06c7a7ac", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "558a8a08ed78eb820efbfda1de196569d8bfa9b51e8371a1934fbb31345feda7"}, 7 | "file_system": {:hex, :file_system, "1.0.1", "79e8ceaddb0416f8b8cd02a0127bdbababe7bf4a23d2a395b983c1f8b3f73edd", [:mix], [], "hexpm", "4414d1f38863ddf9120720cd976fce5bdde8e91d8283353f0e31850fa89feb9e"}, 8 | "finch": {:hex, :finch, "0.19.0", "c644641491ea854fc5c1bbaef36bfc764e3f08e7185e1f084e35e0672241b76d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc5324ce209125d1e2fa0fcd2634601c52a787aff1cd33ee833664a5af4ea2b6"}, 9 | "floki": {:hex, :floki, "0.37.0", "b83e0280bbc6372f2a403b2848013650b16640cd2470aea6701f0632223d719e", [:mix], [], "hexpm", "516a0c15a69f78c47dc8e0b9b3724b29608aa6619379f91b1ffa47109b5d0dd3"}, 10 | "heroicons": {:git, "https://github.com/tailwindlabs/heroicons.git", "88ab3a0d790e6a47404cba02800a6b25d2afae50", [tag: "v2.1.1", sparse: "optimized", depth: 1]}, 11 | "hpax": {:hex, :hpax, "1.0.1", "c857057f89e8bd71d97d9042e009df2a42705d6d690d54eca84c8b29af0787b0", [:mix], [], "hexpm", "4e2d5a4f76ae1e3048f35ae7adb1641c36265510a2d4638157fbcb53dda38445"}, 12 | "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, 13 | "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"}, 14 | "mint": {:hex, :mint, "1.6.2", "af6d97a4051eee4f05b5500671d47c3a67dac7386045d87a904126fd4bbcea2e", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "5ee441dffc1892f1ae59127f74afe8fd82fda6587794278d924e4d90ea3d63f9"}, 15 | "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, 16 | "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, 17 | "phoenix": {:hex, :phoenix, "1.7.18", "5310c21443514be44ed93c422e15870aef254cf1b3619e4f91538e7529d2b2e4", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "1797fcc82108442a66f2c77a643a62980f342bfeb63d6c9a515ab8294870004e"}, 18 | "phoenix_html": {:hex, :phoenix_html, "4.1.1", "4c064fd3873d12ebb1388425a8f2a19348cef56e7289e1998e2d2fa758aa982e", [:mix], [], "hexpm", "f2f2df5a72bc9a2f510b21497fd7d2b86d932ec0598f0210fed4114adc546c6f"}, 19 | "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.5", "d5f44d7dbd7cfacaa617b70c5a14b2b598d6f93b9caa8e350c51d56cd4350a9b", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7 or ~> 1.2.0", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "1d73920515554d7d6c548aee0bf10a4780568b029d042eccb336db29ea0dad70"}, 20 | "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.5.3", "f2161c207fda0e4fb55165f650f7f8db23f02b29e3bff00ff7ef161d6ac1f09d", [:mix], [{:file_system, "~> 0.3 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "b4ec9cd73cb01ff1bd1cac92e045d13e7030330b74164297d1aee3907b54803c"}, 21 | "phoenix_live_view": {:hex, :phoenix_live_view, "1.0.1", "5389a30658176c0de816636ce276567478bffd063c082515a6e8368b8fc9a0db", [:mix], [{:floki, "~> 0.36", [hex: :floki, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c0f517e6f290f10dbb94343ac22e0109437fb1fa6f0696e7c73967b789c1c285"}, 22 | "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"}, 23 | "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, 24 | "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"}, 25 | "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, 26 | "req": {:hex, :req, "0.5.8", "50d8d65279d6e343a5e46980ac2a70e97136182950833a1968b371e753f6a662", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "d7fc5898a566477e174f26887821a3c5082b243885520ee4b45555f5d53f40ef"}, 27 | "req_easyhtml": {:hex, :req_easyhtml, "0.1.3", "628d808b08ad9a71121249dfaacdb23961bfc46eafd91ee8fe204d43a04b19f5", [:mix], [{:easyhtml, "~> 0.3.0", [hex: :easyhtml, repo: "hexpm", optional: false]}, {:req, "~> 0.4.0 or ~> 0.5.0", [hex: :req, repo: "hexpm", optional: false]}], "hexpm", "bf86cc4961d7c00e088c56293b8fd602334b3faee75b5bcbad41de286be15fb7"}, 28 | "tailwind": {:hex, :tailwind, "0.2.4", "5706ec47182d4e7045901302bf3a333e80f3d1af65c442ba9a9eed152fb26c2e", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "c6e4a82b8727bab593700c998a4d98cf3d8025678bfde059aed71d0000c3e463"}, 29 | "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, 30 | "telemetry_metrics": {:hex, :telemetry_metrics, "1.0.0", "29f5f84991ca98b8eb02fc208b2e6de7c95f8bb2294ef244a176675adc7775df", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f23713b3847286a534e005126d4c959ebcca68ae9582118ce436b521d1d47d5d"}, 31 | "telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"}, 32 | "thousand_island": {:hex, :thousand_island, "1.3.7", "1da7598c0f4f5f50562c097a3f8af308ded48cd35139f0e6f17d9443e4d0c9c5", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0139335079953de41d381a6134d8b618d53d084f558c734f2662d1a72818dd12"}, 33 | "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, 34 | "websock_adapter": {:hex, :websock_adapter, "0.5.8", "3b97dc94e407e2d1fc666b2fb9acf6be81a1798a2602294aac000260a7c4a47d", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "315b9a1865552212b5f35140ad194e67ce31af45bcee443d4ecb96b5fd3f3782"}, 35 | } 36 | -------------------------------------------------------------------------------- /notebooks/hex2context.livemd: -------------------------------------------------------------------------------- 1 | # hex2context 2 | 3 | ```elixir 4 | Mix.install([ 5 | {:sqlite_vec, github: "joelpaulkoch/sqlite_vec"}, 6 | {:ecto, "~> 3.12"}, 7 | {:ecto_sql, "~> 3.12"}, 8 | {:ecto_sqlite3, "~> 0.17.2"}, 9 | {:kino, "~> 0.14.1"}, 10 | {:nx, "~> 0.9.1"}, 11 | {:bumblebee, "~> 0.6.0"}, 12 | {:exla, "~> 0.9.0"}, 13 | {:axon, "~> 0.7.0"}, 14 | {:text_chunker, "~> 0.3.1"}, 15 | {:req, "~> 0.5.8"}, 16 | {:plug, "~> 1.16"} 17 | ]) 18 | 19 | Nx.global_default_backend(EXLA.Backend) 20 | ``` 21 | 22 | ## Documentation 23 | 24 | _`hex2context` is a [Livebook](https://livebook.dev) companion to `hex2txt`._ 25 | 26 | ### Introduction 27 | 28 | `hex2context` uses Retrieval-Augmented Generation (RAG) to include only the most relevant snippets of documentation from the `llms.txt` files generated by [`hex2txt`](https://hex2txt.fly.dev). 29 | 30 | (Excluding documentation that is not as relevant to the task-at-hand make more efficient use of limited LLM context window sizes, and reduces inference costs.) 31 | 32 | ### Usage 33 | 34 | #### Step 1: Generate Embeddings 35 | 36 | First, generate embeddings for every combination of package/version that you will be interested in querying later. For example: 37 | 38 | 39 | 40 | ```elixir 41 | Hex2context.ingest_docs([ 42 | {"phoenix", "1.7.18"}, 43 | {"phoenix_html", "4.1.1"}, 44 | {"phoenix_live_view", "1.0.1"}, 45 | {"flop", "0.26.1"}, 46 | {"flop_phoenix", "0.23.1"}, 47 | {"ecto", "3.12.5"} 48 | ]) 49 | ``` 50 | 51 | Embeddings are generated locally (i.e., on the computer running this Livebook) and durably written to a Sqlite database in the same folder on disk that this notebook is running from. For larger packages (like many in the example above), generating these embeddings may take some time. However, once persisted, they do not need to be re-computed. 52 | 53 | #### Step 2: Query for Relevant Documentation Snippets 54 | 55 | This notebook uses [`Kino.Proxy`](https://hexdocs.pm/kino/Kino.Proxy.html) to expose the following HTTP API: 56 | 57 | * `GET /proxy/sessions/:id/:package_name/:package_version?query=:query` 58 | 59 | Here: 60 | 61 | * `:id` is the identifier of the current Livebook session (check your browser's address bar for the identifier of your current session; [more info](https://news.livebook.dev/livebook-0.13-expose-an-http-api-from-your-notebook-2wE6GY)) 62 | * `:package_name` is the name of the package to query documentation for 63 | * `:package_version` is the version number of the package to query documentation for 64 | * `:query` is the (URL-encoded) query used for similarity search 65 | 66 | Documentation relevant to the provided `:query` (for the indicated package) will be returned from this endpoint, and can be fed in directly to your AI coding assistnat of choice. 67 | 68 | Note that this endpoint will return an error if embeddings for the indicated package (name _and_ version) have not been ingested yet. 69 | 70 | For example: 71 | 72 | ``` 73 | curl "http://localhost:52039/proxy/sessions/6lq7fwziy23shg77c7vjzjtqnoy4hlpfkzhbsqsiuel7vaqr/phoenix_live_view/1.0.0/?query=phx-click" 74 | ``` 75 | 76 | ### Thanks 77 | 78 | With thanks to the following resources: 79 | 80 | * https://github.com/dwyl/rag-elixir-doc/ 81 | * https://github.com/dwyl/rag-elixir-doc/blob/main/rag-elixir.livemd 82 | * https://bitcrowd.dev/a-rag-for-elixir-in-elixir/ 83 | * https://gist.github.com/joelpaulkoch/9192abd23bd2e6ff76be314c24173974 84 | 85 | ## Infrastructure (DB) 86 | 87 | ```elixir 88 | defmodule Hex2context.Repo do 89 | use Ecto.Repo, 90 | otp_app: :hex2context, 91 | adapter: Ecto.Adapters.SQLite3 92 | end 93 | ``` 94 | 95 | ```elixir 96 | Kino.start_child( 97 | {Hex2context.Repo, 98 | database: Path.join(__DIR__, "hex2context.db"), load_extensions: [SqliteVec.path()]} 99 | ) 100 | ``` 101 | 102 | ```elixir 103 | defmodule Hex2context.Repo.Migrations.CreateEmbeddingsTable do 104 | use Ecto.Migration 105 | 106 | def up do 107 | execute(~s" 108 | CREATE TABLE embeddings( 109 | id INTEGER PRIMARY KEY, 110 | package_name TEXT NOT NULL, 111 | package_version TEXT NOT NULL, 112 | doc_chunk TEXT NOT NULL, 113 | embedding FLOAT[384] NOT NULL, 114 | UNIQUE(package_name, package_version, doc_chunk) 115 | ); 116 | 117 | CREATE INDEX idx_package ON embeddings(package_name, package_version); 118 | ") 119 | end 120 | 121 | def down do 122 | execute("DROP TABLE embeddings") 123 | end 124 | end 125 | ``` 126 | 127 | ```elixir 128 | defmodule Hex2context.Repo.Migrations do 129 | alias Hex2context.Repo.Migrations.CreateEmbeddingsTable 130 | def migrate, do: Ecto.Migrator.up(Hex2context.Repo, 1, CreateEmbeddingsTable) 131 | def rollback, do: Ecto.Migrator.down(Hex2context.Repo, 1, CreateEmbeddingsTable) 132 | end 133 | ``` 134 | 135 | ```elixir 136 | Hex2context.Repo.Migrations.migrate() 137 | ``` 138 | 139 | ## Infrastructure (ML) 140 | 141 | ```elixir 142 | defmodule Hex2context.Serving do 143 | def build_serving_for_embeddings() do 144 | repo = {:hf, "sentence-transformers/all-MiniLM-L6-v2"} 145 | 146 | {:ok, model_info} = Bumblebee.load_model(repo) 147 | {:ok, tokenizer} = Bumblebee.load_tokenizer(repo) 148 | 149 | Bumblebee.Text.text_embedding( 150 | model_info, 151 | tokenizer, 152 | output_pool: :mean_pooling, 153 | output_attribute: :hidden_state, 154 | embedding_processor: :l2_norm, 155 | compile: [batch_size: 1, sequence_length: [2000]], 156 | defn_options: [compiler: EXLA] 157 | ) 158 | end 159 | end 160 | ``` 161 | 162 | ```elixir 163 | Kino.start_child( 164 | {Nx.Serving, 165 | serving: Hex2context.Serving.build_serving_for_embeddings(), 166 | name: Hex2context.EmbeddingServing, 167 | batch_timeout: 100} 168 | ) 169 | ``` 170 | 171 | ## Schema 172 | 173 | ```elixir 174 | defmodule Hex2context.Embedding do 175 | use Ecto.Schema 176 | 177 | schema "embeddings" do 178 | field(:embedding, SqliteVec.Ecto.Float32) 179 | field(:package_name, :string) 180 | field(:package_version, :string) 181 | field(:doc_chunk, :string) 182 | end 183 | end 184 | ``` 185 | 186 | ## Application (RAG) 187 | 188 | ```elixir 189 | defmodule Hex2context do 190 | require Logger 191 | import Ecto.Query 192 | import SqliteVec.Ecto.Query 193 | alias Hex2context.Embedding 194 | 195 | @hex2txt "https://hex2txt.fly.dev" 196 | @req Req.new() 197 | @chunk_retrieval_limit 20 198 | 199 | def ingest_docs(packages_list) when is_list(packages_list) do 200 | for {package_name, package_version} <- packages_list do 201 | ingest_docs(package_name, package_version) 202 | end 203 | 204 | :ok 205 | end 206 | 207 | def ingest_docs(package_name, package_version, opts \\ []) do 208 | force_refresh? = Keyword.get(opts, :force, false) 209 | already_loaded? = has_docs?(package_name, package_version) 210 | 211 | case {already_loaded?, force_refresh?} do 212 | {true, false} -> 213 | Logger.info("Docs already ingested for #{package_name} v#{package_version}; specify `force: true` to refresh") 214 | :ok 215 | 216 | _ -> 217 | fetch_docs_and_generate_embeddings(package_name, package_version) |> 218 | persist_embeddings(package_name, package_version) 219 | 220 | :ok 221 | end 222 | end 223 | 224 | def retrieve_docs(package_name, package_version, query) do 225 | with true <- has_docs?(package_name, package_version) do 226 | %{embedding: tensor} = Nx.Serving.batched_run(Hex2context.EmbeddingServing, query) 227 | 228 | query_embedding = SqliteVec.Float32.new(tensor) 229 | 230 | doc_chunks = Hex2context.Repo.all( 231 | from(i in Embedding, 232 | where: i.package_name == ^package_name, 233 | where: i.package_version == ^package_version, 234 | order_by: vec_distance_L2(i.embedding, vec_f32(query_embedding)), 235 | limit: ^@chunk_retrieval_limit, 236 | select: i.doc_chunk 237 | ) 238 | ) 239 | 240 | {:ok, doc_chunks} 241 | else 242 | false -> 243 | {:err, 244 | "No docs for #{package_name} v#{package_version}; must pre-generate with `Hex2context.ingest_docs/2`"} 245 | end 246 | end 247 | 248 | defp has_docs?(package_name, package_version) do 249 | Hex2context.Repo.exists?( 250 | from(i in Embedding, 251 | where: i.package_name == ^package_name, 252 | where: i.package_version == ^package_version 253 | ) 254 | ) 255 | end 256 | 257 | defp fetch_docs_and_generate_embeddings(package_name, package_version) do 258 | url = @hex2txt <> "/" <> package_name <> "/" <> package_version <> "/llms.txt" 259 | docs = Req.get!(@req, url: url, http_errors: :raise).body 260 | 261 | chunks = 262 | docs 263 | |> TextChunker.split(format: :markdown) 264 | |> Enum.map(fn chunk -> 265 | %TextChunker.Chunk{chunk | text: String.trim(chunk.text)} 266 | end) 267 | 268 | chunk_count = Enum.count(chunks) 269 | 270 | Logger.info( 271 | "Fetched #{chunk_count} documentation chunks for #{package_name} v#{package_version}" 272 | ) 273 | 274 | embeddings = 275 | chunks 276 | |> Enum.map(& &1.text) 277 | |> Enum.with_index() 278 | |> Enum.map(fn {chunk, index} -> 279 | Logger.info("Computing embedding for chunk #{index}/#{chunk_count}...") 280 | Nx.Serving.batched_run(Hex2context.EmbeddingServing, chunk) 281 | end) 282 | 283 | Logger.info("Finished computing embeddings for #{chunk_count} chunks") 284 | 285 | {chunks, embeddings} 286 | end 287 | 288 | defp persist_embeddings({chunks, embeddings}, package_name, package_version) do 289 | for {%TextChunker.Chunk{text: text}, %{embedding: tensor}} <- Enum.zip(chunks, embeddings) do 290 | Hex2context.Repo.insert( 291 | %Embedding{ 292 | embedding: SqliteVec.Float32.new(tensor), 293 | package_name: package_name, 294 | package_version: package_version, 295 | doc_chunk: text 296 | }, 297 | on_conflict: :replace_all 298 | ) 299 | end 300 | end 301 | end 302 | ``` 303 | 304 | ## HTTP API Server 305 | 306 | ```elixir 307 | defmodule Hex2context.API do 308 | use Plug.Router 309 | 310 | plug :match 311 | plug Plug.Parsers, parsers: [:urlencoded] 312 | plug :dispatch 313 | 314 | get "/:package_name/:package_version" do 315 | query = conn.params["query"] || "" 316 | 317 | case Hex2context.retrieve_docs(package_name, package_version, query) do 318 | {:err, message} -> 319 | send_resp(conn, 404, message) 320 | 321 | {:ok, doc_chunks} -> 322 | send_resp(conn, 200, doc_chunks |> Enum.join("\n\n")) 323 | 324 | _ -> 325 | send_resp(conn, 500, "Unable to retrieve documentation") 326 | end 327 | end 328 | 329 | match _ do 330 | send_resp(conn, 404, "Not found") 331 | end 332 | end 333 | ``` 334 | 335 | ```elixir 336 | Kino.Proxy.listen(Hex2context.API) 337 | ``` 338 | 339 | ## Playground 340 | 341 | ```elixir 342 | Hex2context.ingest_docs([ 343 | {"flop", "0.26.1"}, 344 | {"flop_phoenix", "0.23.1"}, 345 | ]) 346 | ``` 347 | 348 | ```elixir 349 | Hex2context.retrieve_docs("flop", "0.26.1", "filter date") 350 | ``` 351 | 352 | ```elixir 353 | Hex2context.ingest_docs("geo", "4.0.1", force: true) 354 | ``` 355 | 356 | ```elixir 357 | Hex2context.retrieve_docs("geo", "4.0.1", "WKB") 358 | ``` 359 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /lib/hex2txt_web/components/core_components.ex: -------------------------------------------------------------------------------- 1 | defmodule Hex2txtWeb.CoreComponents do 2 | @moduledoc """ 3 | Provides core UI components. 4 | 5 | At first glance, this module may seem daunting, but its goal is to provide 6 | core building blocks for your application, such as modals, tables, and 7 | forms. The components consist mostly of markup and are well-documented 8 | with doc strings and declarative assigns. You may customize and style 9 | them in any way you want, based on your application growth and needs. 10 | 11 | The default components use Tailwind CSS, a utility-first CSS framework. 12 | See the [Tailwind CSS documentation](https://tailwindcss.com) to learn 13 | how to customize them or feel free to swap in another framework altogether. 14 | 15 | Icons are provided by [heroicons](https://heroicons.com). See `icon/1` for usage. 16 | """ 17 | use Phoenix.Component 18 | 19 | alias Phoenix.LiveView.JS 20 | 21 | @doc """ 22 | Renders a modal. 23 | 24 | ## Examples 25 | 26 | <.modal id="confirm-modal"> 27 | This is a modal. 28 | 29 | 30 | JS commands may be passed to the `:on_cancel` to configure 31 | the closing/cancel event, for example: 32 | 33 | <.modal id="confirm" on_cancel={JS.navigate(~p"/posts")}> 34 | This is another modal. 35 | 36 | 37 | """ 38 | attr :id, :string, required: true 39 | attr :show, :boolean, default: false 40 | attr :on_cancel, JS, default: %JS{} 41 | slot :inner_block, required: true 42 | 43 | def modal(assigns) do 44 | ~H""" 45 | 327 | """ 328 | end 329 | 330 | def input(%{type: "select"} = assigns) do 331 | ~H""" 332 |
333 | <.label for={@id}>{@label} 334 | 344 | <.error :for={msg <- @errors}>{msg} 345 |
346 | """ 347 | end 348 | 349 | def input(%{type: "textarea"} = assigns) do 350 | ~H""" 351 |
352 | <.label for={@id}>{@label} 353 | 363 | <.error :for={msg <- @errors}>{msg} 364 |
365 | """ 366 | end 367 | 368 | # All other inputs text, datetime-local, url, password, etc. are handled here... 369 | def input(assigns) do 370 | ~H""" 371 |
372 | <.label for={@id}>{@label} 373 | 385 | <.error :for={msg <- @errors}>{msg} 386 |
387 | """ 388 | end 389 | 390 | @doc """ 391 | Renders a label. 392 | """ 393 | attr :for, :string, default: nil 394 | slot :inner_block, required: true 395 | 396 | def label(assigns) do 397 | ~H""" 398 | 401 | """ 402 | end 403 | 404 | @doc """ 405 | Generates a generic error message. 406 | """ 407 | slot :inner_block, required: true 408 | 409 | def error(assigns) do 410 | ~H""" 411 |

412 | <.icon name="hero-exclamation-circle-mini" class="mt-0.5 h-5 w-5 flex-none" /> 413 | {render_slot(@inner_block)} 414 |

415 | """ 416 | end 417 | 418 | @doc """ 419 | Renders a header with title. 420 | """ 421 | attr :class, :string, default: nil 422 | 423 | slot :inner_block, required: true 424 | slot :subtitle 425 | slot :actions 426 | 427 | def header(assigns) do 428 | ~H""" 429 |
430 |
431 |

432 | {render_slot(@inner_block)} 433 |

434 |

435 | {render_slot(@subtitle)} 436 |

437 |
438 |
{render_slot(@actions)}
439 |
440 | """ 441 | end 442 | 443 | @doc ~S""" 444 | Renders a table with generic styling. 445 | 446 | ## Examples 447 | 448 | <.table id="users" rows={@users}> 449 | <:col :let={user} label="id">{user.id} 450 | <:col :let={user} label="username">{user.username} 451 | 452 | """ 453 | attr :id, :string, required: true 454 | attr :rows, :list, required: true 455 | attr :row_id, :any, default: nil, doc: "the function for generating the row id" 456 | attr :row_click, :any, default: nil, doc: "the function for handling phx-click on each row" 457 | 458 | attr :row_item, :any, 459 | default: &Function.identity/1, 460 | doc: "the function for mapping each row before calling the :col and :action slots" 461 | 462 | slot :col, required: true do 463 | attr :label, :string 464 | end 465 | 466 | slot :action, doc: "the slot for showing user actions in the last table column" 467 | 468 | def table(assigns) do 469 | assigns = 470 | with %{rows: %Phoenix.LiveView.LiveStream{}} <- assigns do 471 | assign(assigns, row_id: assigns.row_id || fn {id, _item} -> id end) 472 | end 473 | 474 | ~H""" 475 |
476 | 477 | 478 | 479 | 480 | 483 | 484 | 485 | 490 | 491 | 503 | 514 | 515 | 516 |
{col[:label]} 481 | Actions 482 |
496 |
497 | 498 | 499 | {render_slot(col, @row_item.(row))} 500 | 501 |
502 | 		504 |
505 | 506 | 510 | {render_slot(action, @row_item.(row))} 511 | 512 |
513 |
517 |
518 | """ 519 | end 520 | 521 | @doc """ 522 | Renders a data list. 523 | 524 | ## Examples 525 | 526 | <.list> 527 | <:item title="Title">{@post.title} 528 | <:item title="Views">{@post.views} 529 | 530 | """ 531 | slot :item, required: true do 532 | attr :title, :string, required: true 533 | end 534 | 535 | def list(assigns) do 536 | ~H""" 537 |
538 |
539 |
540 |
{item.title}
541 |
{render_slot(item)}
542 |
543 |
544 |
545 | """ 546 | end 547 | 548 | @doc """ 549 | Renders a back navigation link. 550 | 551 | ## Examples 552 | 553 | <.back navigate={~p"/posts"}>Back to posts 554 | """ 555 | attr :navigate, :any, required: true 556 | slot :inner_block, required: true 557 | 558 | def back(assigns) do 559 | ~H""" 560 |
561 | <.link 562 | navigate={@navigate} 563 | class="text-sm font-semibold leading-6 text-zinc-900 hover:text-zinc-700" 564 | > 565 | <.icon name="hero-arrow-left-solid" class="h-3 w-3" /> 566 | {render_slot(@inner_block)} 567 | 568 |
569 | """ 570 | end 571 | 572 | @doc """ 573 | Renders a [Heroicon](https://heroicons.com). 574 | 575 | Heroicons come in three styles – outline, solid, and mini. 576 | By default, the outline style is used, but solid and mini may 577 | be applied by using the `-solid` and `-mini` suffix. 578 | 579 | You can customize the size and colors of the icons by setting 580 | width, height, and background color classes. 581 | 582 | Icons are extracted from the `deps/heroicons` directory and bundled within 583 | your compiled app.css by the plugin in your `assets/tailwind.config.js`. 584 | 585 | ## Examples 586 | 587 | <.icon name="hero-x-mark-solid" /> 588 | <.icon name="hero-arrow-path" class="ml-1 w-3 h-3 animate-spin" /> 589 | """ 590 | attr :name, :string, required: true 591 | attr :class, :string, default: nil 592 | 593 | def icon(%{name: "hero-" <> _} = assigns) do 594 | ~H""" 595 | 596 | """ 597 | end 598 | 599 | ## JS Commands 600 | 601 | def show(js \\ %JS{}, selector) do 602 | JS.show(js, 603 | to: selector, 604 | time: 300, 605 | transition: 606 | {"transition-all transform ease-out duration-300", 607 | "opacity-0 translate-y-4 sm:translate-y-0 sm:scale-95", 608 | "opacity-100 translate-y-0 sm:scale-100"} 609 | ) 610 | end 611 | 612 | def hide(js \\ %JS{}, selector) do 613 | JS.hide(js, 614 | to: selector, 615 | time: 200, 616 | transition: 617 | {"transition-all transform ease-in duration-200", 618 | "opacity-100 translate-y-0 sm:scale-100", 619 | "opacity-0 translate-y-4 sm:translate-y-0 sm:scale-95"} 620 | ) 621 | end 622 | 623 | def show_modal(js \\ %JS{}, id) when is_binary(id) do 624 | js 625 | |> JS.show(to: "##{id}") 626 | |> JS.show( 627 | to: "##{id}-bg", 628 | time: 300, 629 | transition: {"transition-all transform ease-out duration-300", "opacity-0", "opacity-100"} 630 | ) 631 | |> show("##{id}-container") 632 | |> JS.add_class("overflow-hidden", to: "body") 633 | |> JS.focus_first(to: "##{id}-content") 634 | end 635 | 636 | def hide_modal(js \\ %JS{}, id) do 637 | js 638 | |> JS.hide( 639 | to: "##{id}-bg", 640 | transition: {"transition-all transform ease-in duration-200", "opacity-100", "opacity-0"} 641 | ) 642 | |> hide("##{id}-container") 643 | |> JS.hide(to: "##{id}", transition: {"block", "block", "hidden"}) 644 | |> JS.remove_class("overflow-hidden", to: "body") 645 | |> JS.pop_focus() 646 | end 647 | 648 | @doc """ 649 | Translates an error message using gettext. 650 | """ 651 | def translate_error({msg, opts}) do 652 | # You can make use of gettext to translate error messages by 653 | # uncommenting and adjusting the following code: 654 | 655 | # if count = opts[:count] do 656 | # Gettext.dngettext(Hex2txtWeb.Gettext, "errors", msg, msg, count, opts) 657 | # else 658 | # Gettext.dgettext(Hex2txtWeb.Gettext, "errors", msg, opts) 659 | # end 660 | 661 | Enum.reduce(opts, msg, fn {key, value}, acc -> 662 | String.replace(acc, "%{#{key}}", fn _ -> to_string(value) end) 663 | end) 664 | end 665 | 666 | @doc """ 667 | Translates the errors for a field from a keyword list of errors. 668 | """ 669 | def translate_errors(errors, field) when is_list(errors) do 670 | for {^field, {msg, opts}} <- errors, do: translate_error({msg, opts}) 671 | end 672 | end 673 | -------------------------------------------------------------------------------- /lib/hex2txt_web/controllers/page_html/home.html.heex: -------------------------------------------------------------------------------- 1 | <.flash_group flash={@flash} /> 2 | 3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |

11 | hex2txt 12 |

13 | 14 |

15 | quickly convert 16 | 17 | hex.pm 18 | 19 | package docs to an 20 | 21 | /llms.txt 22 | 23 | file 24 |

25 | 26 |

27 | …this helps AI coding assistants write better Elixir code by teaching them 28 | about the packages you're using 29 |

30 |
31 |
32 | 33 |

34 | Usage Details 35 |

36 | 37 |
38 |
39 |

Latest Version

40 |

Generate an /llms.txt file 41 | for the most recent version of a package:

42 | 43 | https://hex2txt.fly.dev/<package>/llms.txt 44 | 45 |

46 | Examples: 47 |

48 | 86 |
87 | 88 |
89 |

Specific Version

90 |

Generate an /llms.txt file 91 | for a specific version of a package:

92 | 93 | https://hex2txt.fly.dev/<package>/<version>/llms.txt 94 | 95 |

96 | Examples: 97 |

98 | 136 |
137 |
138 | Replace <package> 139 | with the desired package name and 140 | <version> 141 | with the specific version number. 142 |
143 |
144 | 145 |
146 |
150 |
151 |
152 |
153 |
154 |
155 |
156 |
157 |
158 |
159 |
160 |

161 | Small context windows getting in the way? 162 |

163 |

164 | Try hex2context, 165 | a self-contained Livebook notebook that uses 166 | Retrieval-Augmented Generation (RAG) to only 167 | include the most relevant snippets of 168 | documentation. 169 |

170 | 171 |

172 | Excluding less-relevant documentation makes 173 | more efficient use of limited context window 174 | sizes (and reduces inference costs, too). 175 |

176 | 177 |
178 | 182 | Run in Livebook 187 | 188 |
189 |
190 |
191 |
192 |
193 |
194 |
195 |
196 |
197 |
198 |
199 |
200 | 201 |
202 |

203 | FAQ 204 |

205 |
206 |
207 |
208 |

209 | Isn't this just generating a Markdown file? 210 |

211 |

212 | Yes. Although the implementation is currently 213 | straightforward, the end goal is to produce files that are 214 | optimized specifically for inference-time consumption by an 215 | LLM (or LLM-adjacent tooling). The /llms.txt 216 | component is a signal that these files are intended for use 217 | by machines, not humans. 218 |

219 |
220 |
221 | 222 |
223 |
224 |

225 | Why provide "inference-time" documentation? 226 |

227 |

228 | Here's an example of two real sessions using 229 | 230 | Aider: 231 | 232 |

    233 |
  • 234 | Session 1: 235 | Aider hallucinates and generates broken code 236 | (it is not familiar with the 237 | 241 | EasyHTML 242 | 243 | package) 244 |
    • 245 |
  • 246 | Session 2: 247 | Aider generates perfect code, because 248 | the EasyHTML docs are loaded into context (via the 249 | 250 | /web 251 | 252 | https://hex2txt.fly.dev/easyhtml/llms.txt 253 | 254 | 255 | command) 256 |
    • 257 |
258 |
Session 1 ⁉️
259 | 260 |
261 |
262 |
263 |
264 |
265 |
266 |
267 | $ 268 |

269 | aider --sonnet
270 |

271 |
272 |
273 |
Main model: claude-3-5-sonnet-20241022 with diff edit format, 274 | infinite output
Weak model: claude-3-5-haiku-20241022

275 | 276 | > /ask Using EasyHTML, how to find all OpenGraph tags in an HTML document? Use Elixir. 277 | 278 | 279 |

Using EasyHTML in Elixir, you can find all OpenGraph tags like 280 | this:
281 | 282 |
283 | ``` 284 | def find_og_tags(html) do 285 | {:ok, document} = EasyHTML.parse(html) 286 | 287 | document 288 | |> EasyHTML.find("meta[property^='og:']") 289 | |> Enum.map(fn tag -> 290 | %{ 291 | property: EasyHTML.attr(tag, "property"), 292 | content: EasyHTML.attr(tag, "content") 293 | } 294 | end) 295 | end 296 | ``` 297 |
You can use it like this: 298 |
299 | 300 |
301 | ``` 302 | html = """ 303 | <html> 304 | <head> 305 | <meta property="og:title" content="My Page Title"> 306 | <meta property="og:description" content="Page Description"> 307 | <meta property="og:image" content="image.jpg"> 308 | </head> 309 | </html> 310 | """ 311 | 312 | og_tags = find_og_tags(html) 313 | # Returns a list of maps with property and content: 314 | # [ 315 | # %{property: "og:title", content: "My Page Title"}, 316 | # %{property: "og:description", content: "Page Description"}, 317 | # %{property: "og:image", content: "image.jpg"} 318 | # ] 319 | ``` 320 |
The CSS selector 321 | meta[property^='og:'] 322 | will match all meta tags where the property attribute starts with "og:". The function returns a list of maps 323 | containing the property and content values for each OpenGraph tag found. 324 |
325 |
326 | 327 | Tokens: 309 sent, 286 received. Cost: $0.0052 message, $0.0052 session. 328 | 329 |
330 |
331 |
332 | 333 |
Session 2 ✅
334 | 335 |
336 |
337 |
338 |
339 |
340 |
341 |
342 | $ 343 |

344 | aider --sonnet
345 |

346 |
347 |
348 |
Main model: claude-3-5-sonnet-20241022 with diff edit format, 349 | infinite output
Weak model: claude-3-5-haiku-20241022

350 | 351 | 352 | > /web https://hex2txt.fly.dev/easyhtml/llms.txt 353 | 354 | 355 |
Scraping https://hex2txt.fly.dev/easyhtml/llms.txt...
356 | ... added to chat.

357 | 358 | 359 | > /ask Using EasyHTML, how to find all OpenGraph tags in an HTML document? 360 | 361 | 362 |

363 | Based on the documentation shown, you can use EasyHTML's Access implementation (the 364 | [] 365 | operator) to find meta tags with OpenGraph properties. Here's 366 | how you would likely do it:
367 | 368 |
369 | ``` 370 | html = EasyHTML.parse!(document) 371 | html["meta[property^='og:']"] 372 | ``` 373 |
This would find all 374 | <meta> 375 | tags where the property 376 | attribute starts with "og:" which is the standard prefix for OpenGraph tags. 377 |
For example, if you had HTML like:
378 | 379 |
380 | ``` 381 | html = """ 382 | <html> 383 | <head> 384 | <meta property="og:title" content="My Page Title"> 385 | <meta property="og:description" content="Page description"> 386 | </head> 387 | <body>...</body> 388 | </html> 389 | """ 390 | ``` 391 |
The above code would find those meta tags.

392 | Note that EasyHTML is a wrapper around Floki and uses its CSS selector syntax, 393 | so this should work as expected. You can then enumerate over the results since EasyHTML implements Enumerable.

394 | However, if you need to get the specific content of these tags, you might need 395 | to use additional Floki functions as this isn't explicitly shown in the documentation provided.
396 | 397 |
398 | 399 | Tokens: 548 sent, 234 received. Cost: $0.0052 message, $0.0052 session. 400 | 401 |
402 |
403 |
404 |

405 |
406 |
407 | 408 |
409 |
410 |

411 | Why doesn't this work for <package-name>? 412 |

413 |

414 | This works by scraping output files produced 415 | by ExDoc, 419 | making numerous assumptions (for example, relying on ExDoc 420 | to generate JavaScript files with embedded JSON assigned to 421 | specific JS variable names). This is obviously fragile. For 422 | this reason, documentation published with older versions of 423 | ExDoc might not work. 424 |

425 |
426 |
427 | 428 |
429 |
430 |

431 | Shouldn't this be part of ExDoc? 432 |

433 |

434 | Yes, I think so (with, perhaps, additional web-facing tooling 435 | facilitated by other Hex-related projects). Iterating on this 436 | prototype and collecting community feedback is the best way 437 | to determine how useful this feature is and to inform 438 | requirements. 439 |

440 |
441 |
442 | 443 |
444 |
445 |

446 | What are the limitations? 447 |

448 |

449 | This is currently prototype-quality code, without proper 450 | error handling (among other deficiencies). 451 |

452 |

453 | But the biggest practical issue is the size of generated 454 | documentation files, specifically for packages with a large 455 | API footprint (such as Phoenix, Ecto, Elixir, etc.). These 456 | docs can consume several hundreds of thousands of tokens and 457 | easily exhaust all available LLM context space. 458 |

459 |

460 | We need to find ways to reduce the file size (e.g. by only 461 | including information for a subset of modules, or by dropping 462 | examples, etc.) Additionally, there may be clever ways to use 463 | embeddings to dynamically include only the relevant 464 | components of documentation for the task-at-hand (see, for 465 | example, the 466 | hex2context Livebook, which offers 467 | one potential solution to this problem). 468 |

469 | 470 |

471 | Please 472 | 476 | submit a PR on GitHub 477 | 478 | if you'd like to help 479 | contribute. Some work (and lots of experimentation) will 480 | be required to discover how to most effectively assemble 481 | LLM-specific documentation. 482 |

483 |
484 |
485 |
486 |
487 |
488 |
489 | 490 |
491 |
492 |
493 | 508 | 509 | 528 |
529 |
530 |
531 |
532 |
533 | --------------------------------------------------------------------------------