├── .envrc.dist
├── .github
    └── dependabot.yml
├── .gitignore
├── Brewfile
├── Brewfile.lock.json
├── README.md
├── app
    └── Memex.app
    │   └── Contents
    │       ├── Info.plist
    │       ├── MacOS
    │           └── WebkitWrapper
    │       └── Resources
    │           ├── app.asar.unpacked
    │               ├── build
    │               │   ├── app.json
    │               │   └── icon.png
    │               └── package.json
    │           └── icon.icns
├── bin-dev
    └── start.sh
├── docker-compose.yaml
└── memex
    ├── .formatter.exs
    ├── .gitignore
    ├── README.md
    ├── assets
        ├── css
        │   ├── app.css
        │   ├── codemirror.css
        │   ├── mapbox.css
        │   └── rings.css
        ├── jest.config.js
        ├── js
        │   ├── app.js
        │   ├── editor.js
        │   ├── force_input_value.js
        │   ├── infinite_scroll.js
        │   ├── map.js
        │   ├── search_language
        │   │   ├── index.js
        │   │   ├── index.test.js
        │   │   ├── syntax.grammer
        │   │   ├── syntax.js
        │   │   └── syntax.terms.js
        │   └── sidebar.js
        ├── package.json
        ├── postcss.config.js
        ├── tailwind.config.js
        └── yarn.lock
    ├── config
        ├── config.exs
        ├── dev.exs
        ├── prod.exs
        ├── prod.secret.exs
        └── test.exs
    ├── lib
        ├── memex.ex
        ├── memex
        │   ├── application.ex
        │   ├── connector.ex
        │   ├── importer.ex
        │   ├── importers
        │   │   ├── apple_message.ex
        │   │   ├── apple_photos.ex
        │   │   ├── apple_podcasts.ex
        │   │   ├── arc.ex
        │   │   ├── fish_shell.ex
        │   │   ├── github.ex
        │   │   ├── money_money.ex
        │   │   ├── notes.ex
        │   │   └── safari.ex
        │   ├── repo.ex
        │   ├── scheduler.ex
        │   ├── schema
        │   │   ├── catalog.ex
        │   │   ├── document.ex
        │   │   ├── encrypted
        │   │   │   └── map.ex
        │   │   ├── importer_config.ex
        │   │   ├── importer_log.ex
        │   │   └── relation.ex
        │   ├── search
        │   │   ├── legacy_query.ex
        │   │   ├── postgres.ex
        │   │   ├── query.ex
        │   │   └── sidebars.ex
        │   └── vault.ex
        ├── memex_web.ex
        └── memex_web
        │   ├── channels
        │       └── user_socket.ex
        │   ├── components
        │       ├── badge.ex
        │       ├── close_circles.ex
        │       ├── core_components.ex
        │       ├── dates_facet.ex
        │       ├── forms
        │       │   └── toggle_switch.ex
        │       ├── icon.ex
        │       ├── icons
        │       │   ├── plus_icon.ex
        │       │   └── settings_icon.ex
        │       ├── layouts.ex
        │       ├── layouts
        │       │   ├── app.html.heex
        │       │   ├── live.html.leex
        │       │   └── root.html.heex
        │       ├── map.ex
        │       ├── search_bar.ex
        │       ├── search_result_stats.ex
        │       ├── sidebars_component.ex
        │       ├── text.ex
        │       ├── time_duration.ex
        │       ├── timeline.ex
        │       └── timeline
        │       │   ├── card.ex
        │       │   ├── date_border.ex
        │       │   ├── provider_icon.ex
        │       │   ├── time.ex
        │       │   └── vertical_line.ex
        │   ├── controllers
        │       ├── alfred_controller.ex
        │       ├── arc_controller.ex
        │       ├── error_html.ex
        │       ├── error_json.ex
        │       ├── index_controller.ex
        │       └── photo_controller.ex
        │   ├── endpoint.ex
        │   ├── gettext.ex
        │   ├── live
        │       ├── page_live.ex
        │       └── sidebars
        │       │   ├── activity_live.ex
        │       │   ├── generic_live.ex
        │       │   ├── person_live.ex
        │       │   └── setttings_live.ex
        │   ├── router.ex
        │   ├── telemetry.ex
        │   └── views
        │       ├── error_helpers.ex
        │       └── timeline_view.ex
    ├── mix.exs
    ├── mix.lock
    ├── priv
        ├── gettext
        │   ├── en
        │   │   └── LC_MESSAGES
        │   │   │   └── errors.po
        │   └── errors.pot
        ├── repo
        │   ├── migrations
        │   │   ├── 20210619105629_add_documents.exs
        │   │   ├── 20210619110040_add_relations.exs
        │   │   ├── 20211009134648_create_importer_config.exs
        │   │   └── 20221230125036_create_importer_log.exs
        │   └── seeds.exs
        └── static
        │   └── images
        │       ├── Arc-small.png
        │       ├── Arc.png
        │       ├── GitHub-small.png
        │       ├── GitHub.icns
        │       ├── MoneyMoney-small.png
        │       ├── MoneyMoney.icns
        │       ├── Photos-small.png
        │       ├── Photos.icns
        │       ├── Podcasts-small.png
        │       ├── Podcasts.icns
        │       ├── Terminal.icns
        │       ├── Twitter-small.png
        │       ├── Twitter.icns
        │       ├── iMessage-small.png
        │       ├── iMessage.icns
        │       ├── memex.icns
        │       ├── memex.png
        │       ├── memex
        │           └── memex.afdesign
        │       ├── phoenix.png
        │       ├── safari-small.png
        │       ├── safari.icns
        │       └── terminal-small.png
    └── test
        ├── memex
            ├── importers
            │   └── github_importer_test.exs
            └── search
            │   └── query_test.exs
        ├── memex_web
            ├── live
            │   └── page_live_test.exs
            └── views
            │   ├── error_view_test.exs
            │   ├── layout_view_test.exs
            │   └── timeline_view_test.exs
        ├── support
            ├── channel_case.ex
            └── conn_case.ex
        └── test_helper.exs


/.envrc.dist:
--------------------------------------------------------------------------------
 1 | export MEILISEARCH_HOST=http://127.0.0.1:7700
 2 | export INDEX_NAME=memex
 3 | 
 4 | export GITHUB_USER_NAME=adri
 5 | export GITHUB_PERSONAL_TOKEN=
 6 | 
 7 | export POSTGRES_DSN=postgresql://postgres:memex@localhost:65432/memex
 8 | 
 9 | # Set a random string, used to encrypt secrets stored in the database
10 | export SECRETS_ENCRYPTION_KEY= 
11 | 
12 | # Optional, used to resolve Foursquare categories
13 | # and places collected by the Arc app.
14 | export FOURSQUARE_CLIENT_ID=
15 | export FOURSQUARE_CLIENT_SECRET=
16 | 
17 | # Optional, used to display a map preview of locations
18 | export MAPBOX_API_KEY=
19 | 


--------------------------------------------------------------------------------
/.github/dependabot.yml:
--------------------------------------------------------------------------------
 1 | # To get started with Dependabot version updates, you'll need to specify which
 2 | # package ecosystems to update and where the package manifests are located.
 3 | # Please see the documentation for all configuration options:
 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
 5 | 
 6 | version: 2
 7 | updates:
 8 |   - package-ecosystem: "npm" # See documentation for possible values
 9 |     directory: "/memex/assets" # Location of package manifests
10 |     schedule:
11 |       interval: "weekly"
12 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | data.ms/
 2 | dumps/
 3 | data/
 4 | data*/
 5 | backup/
 6 | node_modules/
 7 | auth.json
 8 | .envrc
 9 | twitter.db
10 | github.db
11 | 


--------------------------------------------------------------------------------
/Brewfile:
--------------------------------------------------------------------------------
 1 | # Install https://github.com/Homebrew/homebrew-bundle
 2 | # Run: brew bundle
 3 | 
 4 | brew "direnv"
 5 | brew "dockerize"
 6 | brew "elixir"
 7 | brew "fswatch"
 8 | brew "jq"
 9 | brew "sqlcipher"
10 | 


--------------------------------------------------------------------------------
/Brewfile.lock.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "entries": {
 3 |     "brew": {
 4 |       "jq": {
 5 |         "version": "1.6",
 6 |         "bottle": {
 7 |           "cellar": ":any",
 8 |           "prefix": "/usr/local",
 9 |           "files": {
10 |             "big_sur": {
11 |               "url": "https://homebrew.bintray.com/bottles/jq-1.6.big_sur.bottle.1.tar.gz",
12 |               "sha256": "bf0f8577632af7b878b6425476f5b1ab9c3bf66d65affb0c455048a173a0b6bf"
13 |             },
14 |             "catalina": {
15 |               "url": "https://homebrew.bintray.com/bottles/jq-1.6.catalina.bottle.1.tar.gz",
16 |               "sha256": "820a3c85fcbb63088b160c7edf125d7e55fc2c5c1d51569304499c9cc4b89ce8"
17 |             },
18 |             "mojave": {
19 |               "url": "https://homebrew.bintray.com/bottles/jq-1.6.mojave.bottle.1.tar.gz",
20 |               "sha256": "71f0e76c5b22e5088426c971d5e795fe67abee7af6c2c4ae0cf4c0eb98ed21ff"
21 |             },
22 |             "high_sierra": {
23 |               "url": "https://homebrew.bintray.com/bottles/jq-1.6.high_sierra.bottle.1.tar.gz",
24 |               "sha256": "dffcffa4ea13e8f0f2b45c5121e529077e135ae9a47254c32182231662ee9b72"
25 |             },
26 |             "sierra": {
27 |               "url": "https://homebrew.bintray.com/bottles/jq-1.6.sierra.bottle.1.tar.gz",
28 |               "sha256": "bb4d19dc026c2d72c53eed78eaa0ab982e9fcad2cd2acc6d13e7a12ff658e877"
29 |             }
30 |           }
31 |         }
32 |       },
33 |       "sqlcipher": {
34 |         "version": "4.4.2",
35 |         "bottle": {
36 |           "cellar": ":any",
37 |           "prefix": "/usr/local",
38 |           "files": {
39 |             "big_sur": {
40 |               "url": "https://homebrew.bintray.com/bottles/sqlcipher-4.4.2.big_sur.bottle.tar.gz",
41 |               "sha256": "cac60b27489ae08e4be4fffcb80c66208cc628889ce89295802983581a39febc"
42 |             },
43 |             "catalina": {
44 |               "url": "https://homebrew.bintray.com/bottles/sqlcipher-4.4.2.catalina.bottle.tar.gz",
45 |               "sha256": "b6e926a4630f8b4547e5d71f5195bdde461ef08e54b9ae90a45644b11543cd9d"
46 |             },
47 |             "mojave": {
48 |               "url": "https://homebrew.bintray.com/bottles/sqlcipher-4.4.2.mojave.bottle.tar.gz",
49 |               "sha256": "8d216a324ade956c2ec9b4dc94a676b27342e590c948b1c80ba49d602c885ccb"
50 |             }
51 |           }
52 |         }
53 |       },
54 |       "direnv": {
55 |         "version": "2.25.2",
56 |         "bottle": {
57 |           "cellar": ":any_skip_relocation",
58 |           "prefix": "/usr/local",
59 |           "files": {
60 |             "big_sur": {
61 |               "url": "https://homebrew.bintray.com/bottles/direnv-2.25.2.big_sur.bottle.tar.gz",
62 |               "sha256": "c79af7963300877b62c3debbe068b9323579586f7a1b934a4e86f91cdaa65b30"
63 |             },
64 |             "catalina": {
65 |               "url": "https://homebrew.bintray.com/bottles/direnv-2.25.2.catalina.bottle.tar.gz",
66 |               "sha256": "cc182495b66533285fc592dae75c76bcb034ae214b2eed34220ecbd60bbfe8a8"
67 |             },
68 |             "mojave": {
69 |               "url": "https://homebrew.bintray.com/bottles/direnv-2.25.2.mojave.bottle.tar.gz",
70 |               "sha256": "b9b3f2b300c8928de283e76cf3685c6efddf4435032c86e374827567cbffda95"
71 |             }
72 |           }
73 |         }
74 |       }
75 |     }
76 |   },
77 |   "system": {
78 |     "macos": {
79 |       "big_sur": {
80 |         "HOMEBREW_VERSION": "2.6.1-86-gced0da1",
81 |         "HOMEBREW_PREFIX": "/usr/local",
82 |         "Homebrew/homebrew-core": "d48643e337e7ba2b56de39a916f3e889a8fd2870",
83 |         "CLT": "12.2.0.0.1.1604076827",
84 |         "Xcode": "12.2",
85 |         "macOS": "11.0.1"
86 |       }
87 |     }
88 |   }
89 | }
90 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Building a Memex
 2 | 
 3 | Search of my personal data (including but not limited to notes, messages, financial transactions,
 4 | photos, videos, visited places, traveled routes, browser history, CLI commands,
 5 | version control commits, ...).
 6 | 
 7 | Similar to the quantified self movement where as much data as possible is collected.
 8 | However, the idea is to build a tool to remind myself of things I did and learned from
 9 | the past instead of focusing on the data visualization part. The notes database is a
10 | subset of this idea.
11 | 
12 | Inspired by the [talk Building a Memex](https://www.youtube.com/watch?v=DFWxvQn4cf8&t=1616s) by Andrew Louis.
13 | Andrew has written many interesting [blog posts](https://hyfen.net/memex/) while building a Memex.
14 | 
15 | ### What is a Memex?
16 | 
17 | > Memex is [...] a device in which individuals would compress and store
18 | > all of their books, records, and communications, "mechanized so that
19 | > it may be consulted with exceeding speed and flexibility".
20 | 
21 | Source: [Wikipedia](https://en.wikipedia.org/wiki/Memex)
22 | 
23 | ### How does it look like?
24 | <img width="876" alt="memex" src="https://github.com/adri/memex/assets/133832/953dd65f-1948-4e88-9684-2c5a52b9a547">
25 | 
26 | What it can do:
27 | 
28 | * Search with auto-suggest and search result highlights 🔍
29 | * Timeline with clickable filters ⏲️
30 | * Super fast ⚡
31 | 
32 | ### Installation
33 | 
34 | ```
35 | # Setup environment (once)
36 | brew bundle
37 | cp .envrc.dist .envrc
38 | direnv allow
39 | 
40 | # Start services
41 | ./bin-dev/start.sh
42 | 
43 | open http://localhost:4000
44 | 
45 | # Click the 'cog' wheel icon on the right to configure
46 | ```
47 | 
48 | ### Links
49 | 
50 | - Icons https://macosicons.com, https://heroicons.com
51 | 


--------------------------------------------------------------------------------
/app/Memex.app/Contents/Info.plist:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 3 | <plist version="1.0">
 4 |   <dict>
 5 |     <key>CFBundleName</key>
 6 |     <string>Memex</string>
 7 |     <key>CFBundleIdentifier</key>
 8 |     <string>com.chromeless.webkit.custom-1625040392025</string>
 9 |     <key>CFBundleExecutable</key>
10 |     <string>WebkitWrapper</string>
11 |     <key>CFBundleIconFile</key>
12 |     <string>icon.icns</string>
13 |     <key>LSMinimumSystemVersion</key>
14 |     <string>10.10</string>
15 |     <key>CFBundlePackageType</key>
16 |     <string>APPL</string>
17 |     <key>CFBundleInfoDictionaryVersion</key>
18 |     <string>6.0</string>
19 |     <key>CFBundleVersion</key>
20 |     <string>2.0.2</string>
21 |     <key>CFBundleGetInfoString</key>
22 |     <string>2.0.2</string>
23 |     <key>CFBundleShortVersionString</key>
24 |     <string>2.0.2</string>
25 |     <key>NSPrincipalClass</key>
26 |     <string>NSApplication</string>
27 |     <key>NSMainNibFile</key>
28 |     <string>MainMenu</string>
29 |     <key>CFBundleURLTypes</key>
30 |     <array>
31 |       <dict>
32 |         <key>CFBundleURLName</key>
33 |         <string>HTTPS Protocol</string>
34 |         <key>CFBundleURLSchemes</key>
35 |         <array>
36 |           <string>https</string>
37 |         </array>
38 |       </dict>
39 |       <dict>
40 |         <key>CFBundleURLName</key>
41 |         <string>HTTP Protocol</string>
42 |         <key>CFBundleURLSchemes</key>
43 |         <array>
44 |           <string>http</string>
45 |         </array>
46 |       </dict>
47 |     </array>
48 |   </dict>
49 | </plist>
50 | 


--------------------------------------------------------------------------------
/app/Memex.app/Contents/MacOS/WebkitWrapper:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/app/Memex.app/Contents/MacOS/WebkitWrapper


--------------------------------------------------------------------------------
/app/Memex.app/Contents/Resources/app.asar.unpacked/build/app.json:
--------------------------------------------------------------------------------
1 | {"id":"custom-1625040392025","name":"Memex","url":"http://127.0.0.1:4000","engine":"webkit","opts":{"slug":"memex"}}
2 | 


--------------------------------------------------------------------------------
/app/Memex.app/Contents/Resources/app.asar.unpacked/build/icon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/app/Memex.app/Contents/Resources/app.asar.unpacked/build/icon.png


--------------------------------------------------------------------------------
/app/Memex.app/Contents/Resources/app.asar.unpacked/package.json:
--------------------------------------------------------------------------------
1 | {"version":"2.0.2"}


--------------------------------------------------------------------------------
/app/Memex.app/Contents/Resources/icon.icns:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/app/Memex.app/Contents/Resources/icon.icns


--------------------------------------------------------------------------------
/bin-dev/start.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | set -e
 3 | 
 4 | # Start Postgres
 5 | docker compose up -d
 6 | dockerize -wait-retry-interval 5s -wait tcp://127.0.0.1:65432 -timeout 5m &> /dev/null
 7 | 
 8 | # Start Elixir frontend
 9 | cd memex
10 | mix deps.get
11 | mix ecto.migrate
12 | cd assets && yarn && cd ..
13 | iex -S mix phx.server
14 | 


--------------------------------------------------------------------------------
/docker-compose.yaml:
--------------------------------------------------------------------------------
 1 | version: "2.2"
 2 | services:
 3 |   postgres:
 4 |     image: postgres:13
 5 |     ports:
 6 |       - "65432:5432"
 7 |     environment:
 8 |       - POSTGRES_USER=postgres
 9 |       - POSTGRES_PASSWORD=memex
10 |       - POSTGRES_DB=memex
11 |     volumes:
12 |       - /dev/urandom:/dev/random # Required to get non-blocking entropy source
13 |       - postgres-db:/var/lib/postgresql/data
14 |       - ./data/postgres-share:/postgres-share:ro
15 |     healthcheck:
16 |       test:
17 |         [
18 |           "CMD",
19 |           "psql",
20 |           "-h",
21 |           "localhost",
22 |           "-U",
23 |           "postgres",
24 |           "-c",
25 |           "select 1",
26 |           "memex",
27 |         ]
28 |       interval: 10s
29 |       timeout: 10s
30 |       retries: 5
31 |     restart: always
32 | 
33 | volumes:
34 |   postgres-db:
35 | 


--------------------------------------------------------------------------------
/memex/.formatter.exs:
--------------------------------------------------------------------------------
 1 | [
 2 |   surface_inputs: ["{lib,test}/**/*.{ex,exs,sface}"],
 3 |   import_deps: [:ecto, :ecto_sql, :phoenix, :surface],
 4 |   inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"],
 5 |   subdirectories: ["priv/*/migrations"],
 6 |   plugins: [
 7 |     Surface.Formatter.Plugin,
 8 |     Phoenix.LiveView.HTMLFormatter,
 9 |     Styler
10 |   ]
11 | ]
12 | 


--------------------------------------------------------------------------------
/memex/.gitignore:
--------------------------------------------------------------------------------
 1 | # The directory Mix will write compiled artifacts to.
 2 | /_build/
 3 | 
 4 | # If you run "mix test --cover", coverage assets end up here.
 5 | /cover/
 6 | 
 7 | # The directory Mix downloads your dependencies sources to.
 8 | /deps/
 9 | 
10 | # Where 3rd-party dependencies like ExDoc output generated docs.
11 | /doc/
12 | 
13 | # Ignore .fetch files in case you like to edit your project deps locally.
14 | /.fetch
15 | 
16 | # If the VM crashes, it generates a dump, let's ignore it too.
17 | erl_crash.dump
18 | 
19 | # Also ignore archive artifacts (built via "mix archive.build").
20 | *.ez
21 | 
22 | # Ignore package tarball (built via "mix hex.build").
23 | memex-*.tar
24 | 
25 | # If NPM crashes, it generates a log, let's ignore it too.
26 | npm-debug.log
27 | 
28 | # The directory NPM downloads your dependencies sources to.
29 | /assets/node_modules/
30 | 
31 | # Ignore assets that are produced by build tools.
32 | /priv/static/assets/
33 | 
34 | # Ignore digested assets cache.
35 | /priv/static/cache_manifest.json
36 | 
37 | # The directory used by ExUnit :tmp_dir
38 | /tmp/
39 | 


--------------------------------------------------------------------------------
/memex/README.md:
--------------------------------------------------------------------------------
 1 | # Memex
 2 | 
 3 | To start your Phoenix server:
 4 | 
 5 |   * Install dependencies with `mix deps.get`
 6 |   * Install Node.js dependencies with `npm install` inside the `assets` directory
 7 |   * Start Phoenix endpoint with `mix phx.server`
 8 | 
 9 | Now you can visit [`localhost:4000`](http://localhost:4000) from your browser.
10 | 
11 | Ready to run in production? Please [check our deployment guides](https://hexdocs.pm/phoenix/deployment.html).
12 | 
13 | ## Learn more
14 | 
15 |   * Official website: https://www.phoenixframework.org/
16 |   * Guides: https://hexdocs.pm/phoenix/overview.html
17 |   * Docs: https://hexdocs.pm/phoenix
18 |   * Forum: https://elixirforum.com/c/phoenix-forum
19 |   * Source: https://github.com/phoenixframework/phoenix
20 | 


--------------------------------------------------------------------------------
/memex/assets/css/app.css:
--------------------------------------------------------------------------------
 1 | @import "tailwindcss/base";
 2 | @import "tailwindcss/components";
 3 | @import "tailwindcss/utilities";
 4 | @import "./rings.css";
 5 | @import "./mapbox.css";
 6 | @import "./codemirror.css";
 7 | 
 8 | em {
 9 |   @apply dark:bg-yellow-300/70 bg-yellow-300/70 rounded-sm not-italic;
10 | }
11 | 


--------------------------------------------------------------------------------
/memex/assets/css/codemirror.css:
--------------------------------------------------------------------------------
 1 | .ͼ1.cm-focused {
 2 |   /* Codemirror shows a focus ring by default, disable it. */
 3 |   @apply outline-none;
 4 | }
 5 | 
 6 | .ͼ1.cm-content {
 7 |   @apply p-0 pt-2;
 8 | }
 9 | 
10 | .ͼ2 .cm-content {
11 |   @apply caret-black dark:caret-white p-0;
12 | }
13 | 
14 | .ͼ1 .cm-line {
15 |   @apply font-sans text-black text-base dark:text-white;
16 | }
17 | 
18 | /* Style codemirror autocomplete with darkmode */
19 | .ͼ2 .cm-tooltip {
20 |   @apply bg-white dark:bg-black text-black dark:text-white
21 |   border-0 rounded-md overflow-hidden shadow-lg font-sans text-base;
22 | }
23 | 
24 | .ͼ2 .cm-tooltip-autocomplete ul li {
25 |   @apply font-sans;
26 | }
27 | 
28 | .ͼ2 .cm-tooltip-autocomplete ul li[aria-selected] {
29 |   @apply bg-gray-300 text-black dark:bg-gray-700 dark:text-white;
30 | }
31 | 


--------------------------------------------------------------------------------
/memex/assets/jest.config.js:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * For a detailed explanation regarding each configuration property, visit:
  3 |  * https://jestjs.io/docs/configuration
  4 |  */
  5 | 
  6 | /** @type {import('jest').Config} */
  7 | const config = {
  8 |   // All imported modules in your tests should be mocked automatically
  9 |   // automock: false,
 10 | 
 11 |   // Stop running tests after `n` failures
 12 |   // bail: 0,
 13 | 
 14 |   // The directory where Jest should store its cached dependency information
 15 |   // cacheDirectory: "/private/var/folders/0j/gnst16td735c3mx8r36fp4480000gp/T/jest_dy",
 16 | 
 17 |   // Automatically clear mock calls, instances, contexts and results before every test
 18 |   // clearMocks: false,
 19 | 
 20 |   // Indicates whether the coverage information should be collected while executing the test
 21 |   // collectCoverage: false,
 22 | 
 23 |   // An array of glob patterns indicating a set of files for which coverage information should be collected
 24 |   // collectCoverageFrom: undefined,
 25 | 
 26 |   // The directory where Jest should output its coverage files
 27 |   // coverageDirectory: undefined,
 28 | 
 29 |   // An array of regexp pattern strings used to skip coverage collection
 30 |   // coveragePathIgnorePatterns: [
 31 |   //   "/node_modules/"
 32 |   // ],
 33 | 
 34 |   // Indicates which provider should be used to instrument code for coverage
 35 |   coverageProvider: "v8",
 36 | 
 37 |   // A list of reporter names that Jest uses when writing coverage reports
 38 |   // coverageReporters: [
 39 |   //   "json",
 40 |   //   "text",
 41 |   //   "lcov",
 42 |   //   "clover"
 43 |   // ],
 44 | 
 45 |   // An object that configures minimum threshold enforcement for coverage results
 46 |   // coverageThreshold: undefined,
 47 | 
 48 |   // A path to a custom dependency extractor
 49 |   // dependencyExtractor: undefined,
 50 | 
 51 |   // Make calling deprecated APIs throw helpful error messages
 52 |   // errorOnDeprecated: false,
 53 | 
 54 |   // The default configuration for fake timers
 55 |   // fakeTimers: {
 56 |   //   "enableGlobally": false
 57 |   // },
 58 | 
 59 |   // Force coverage collection from ignored files using an array of glob patterns
 60 |   // forceCoverageMatch: [],
 61 | 
 62 |   // A path to a module which exports an async function that is triggered once before all test suites
 63 |   // globalSetup: undefined,
 64 | 
 65 |   // A path to a module which exports an async function that is triggered once after all test suites
 66 |   // globalTeardown: undefined,
 67 | 
 68 |   // A set of global variables that need to be available in all test environments
 69 |   // globals: {},
 70 | 
 71 |   // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
 72 |   // maxWorkers: "50%",
 73 | 
 74 |   // An array of directory names to be searched recursively up from the requiring module's location
 75 |   // moduleDirectories: [
 76 |   //   "node_modules"
 77 |   // ],
 78 | 
 79 |   // An array of file extensions your modules use
 80 |   // moduleFileExtensions: [
 81 |   //   "js",
 82 |   //   "mjs",
 83 |   //   "cjs",
 84 |   //   "jsx",
 85 |   //   "ts",
 86 |   //   "tsx",
 87 |   //   "json",
 88 |   //   "node"
 89 |   // ],
 90 | 
 91 |   // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
 92 |   // moduleNameMapper: {},
 93 | 
 94 |   // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
 95 |   // modulePathIgnorePatterns: [],
 96 | 
 97 |   // Activates notifications for test results
 98 |   // notify: false,
 99 | 
100 |   // An enum that specifies notification mode. Requires { notify: true }
101 |   // notifyMode: "failure-change",
102 | 
103 |   // A preset that is used as a base for Jest's configuration
104 |   // preset: undefined,
105 | 
106 |   // Run tests from one or more projects
107 |   // projects: undefined,
108 | 
109 |   // Use this configuration option to add custom reporters to Jest
110 |   // reporters: undefined,
111 | 
112 |   // Automatically reset mock state before every test
113 |   // resetMocks: false,
114 | 
115 |   // Reset the module registry before running each individual test
116 |   // resetModules: false,
117 | 
118 |   // A path to a custom resolver
119 |   // resolver: undefined,
120 | 
121 |   // Automatically restore mock state and implementation before every test
122 |   // restoreMocks: false,
123 | 
124 |   // The root directory that Jest should scan for tests and modules within
125 |   // rootDir: undefined,
126 | 
127 |   // A list of paths to directories that Jest should use to search for files in
128 |   // roots: [
129 |   //   "<rootDir>"
130 |   // ],
131 | 
132 |   // Allows you to use a custom runner instead of Jest's default test runner
133 |   // runner: "jest-runner",
134 | 
135 |   // The paths to modules that run some code to configure or set up the testing environment before each test
136 |   // setupFiles: [],
137 | 
138 |   // A list of paths to modules that run some code to configure or set up the testing framework before each test
139 |   // setupFilesAfterEnv: [],
140 | 
141 |   // The number of seconds after which a test is considered as slow and reported as such in the results.
142 |   // slowTestThreshold: 5,
143 | 
144 |   // A list of paths to snapshot serializer modules Jest should use for snapshot testing
145 |   // snapshotSerializers: [],
146 | 
147 |   // The test environment that will be used for testing
148 |   // testEnvironment: "jest-environment-node",
149 | 
150 |   // Options that will be passed to the testEnvironment
151 |   // testEnvironmentOptions: {},
152 | 
153 |   // Adds a location field to test results
154 |   // testLocationInResults: false,
155 | 
156 |   // The glob patterns Jest uses to detect test files
157 |   // testMatch: [
158 |   //   "**/__tests__/**/*.[jt]s?(x)",
159 |   //   "**/?(*.)+(spec|test).[tj]s?(x)"
160 |   // ],
161 | 
162 |   // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
163 |   // testPathIgnorePatterns: [
164 |   //   "/node_modules/"
165 |   // ],
166 | 
167 |   // The regexp pattern or array of patterns that Jest uses to detect test files
168 |   // testRegex: [],
169 | 
170 |   // This option allows the use of a custom results processor
171 |   // testResultsProcessor: undefined,
172 | 
173 |   // This option allows use of a custom test runner
174 |   // testRunner: "jest-circus/runner",
175 | 
176 |   // A map from regular expressions to paths to transformers
177 |   // transform: undefined,
178 |   transform: {
179 |     "^.+\\.js?$": "esbuild-jest",
180 |   },
181 | 
182 |   // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
183 |   // transformIgnorePatterns: [
184 |   //   "/node_modules/",
185 |   //   "\\.pnp\\.[^\\/]+$"
186 |   // ],
187 | 
188 |   // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
189 |   // unmockedModulePathPatterns: undefined,
190 | 
191 |   // Indicates whether each individual test should be reported during the run
192 |   // verbose: undefined,
193 | 
194 |   // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
195 |   // watchPathIgnorePatterns: [],
196 | 
197 |   // Whether to use watchman for file crawling
198 |   // watchman: true,
199 | };
200 | 
201 | module.exports = config;
202 | 


--------------------------------------------------------------------------------
/memex/assets/js/app.js:
--------------------------------------------------------------------------------
 1 | import "phoenix_html";
 2 | import { Socket } from "phoenix";
 3 | import { LiveSocket } from "phoenix_live_view";
 4 | import { InfiniteScroll } from "./infinite_scroll";
 5 | import { ForceInputValue } from "./force_input_value";
 6 | import { Sidebar } from "./sidebar";
 7 | import { Map } from "./map";
 8 | import { Editor } from "./editor";
 9 | 
10 | let csrfToken = document
11 |   .querySelector("meta[name='csrf-token']")
12 |   .getAttribute("content");
13 | let liveSocket = new LiveSocket("/live", Socket, {
14 |   hooks: { InfiniteScroll, ForceInputValue, Sidebar, Map, Editor },
15 |   params: { _csrf_token: csrfToken },
16 | });
17 | 
18 | liveSocket.connect();
19 | 
20 | // expose liveSocket on window for web console debug logs and latency simulation:
21 | // >> liveSocket.enableDebug()
22 | // >> liveSocket.enableLatencySim(1000)  // enabled for duration of browser session
23 | // >> liveSocket.disableLatencySim()
24 | window.liveSocket = liveSocket;
25 | 
26 | window.addEventListener("memex:clipcopy", (event) => {
27 |   if ("clipboard" in navigator) {
28 |     const text = event.target.textContent;
29 |     navigator.clipboard.writeText(text);
30 |   } else {
31 |     alert("Sorry, your browser does not support clipboard copy.");
32 |   }
33 | });
34 | 


--------------------------------------------------------------------------------
/memex/assets/js/editor.js:
--------------------------------------------------------------------------------
  1 | import {
  2 |   keymap,
  3 |   EditorView,
  4 |   placeholder,
  5 |   highlightSpecialChars,
  6 | } from "@codemirror/view";
  7 | import { CompletionContext } from "@codemirror/autocomplete";
  8 | import {
  9 |   defaultHighlightStyle,
 10 |   syntaxHighlighting,
 11 |   foldKeymap,
 12 |   HighlightStyle,
 13 |   syntaxTree,
 14 | } from "@codemirror/language";
 15 | import { EditorState } from "@codemirror/state";
 16 | import {
 17 |   autocompletion,
 18 |   completionKeymap,
 19 |   closeBrackets,
 20 |   closeBracketsKeymap,
 21 |   completionStatus,
 22 |   acceptCompletion,
 23 | } from "@codemirror/autocomplete";
 24 | import { Search, parseStateToFilters } from "./search_language";
 25 | import * as terms from "./search_language/syntax.terms.js";
 26 | import { tags } from "@lezer/highlight";
 27 | import { inlineSuggestion } from "codemirror-extension-inline-suggestion";
 28 | 
 29 | // Enforces that the input won't split over multiple lines (basically prevents
 30 | // Enter from inserting a new line)
 31 | const singleLine = EditorState.transactionFilter.of((transaction) =>
 32 |   transaction.newDoc.lines > 1 ? [] : transaction
 33 | );
 34 | 
 35 | // Nice query language:
 36 | // - https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering
 37 | // - Qdrant: https://qdrant.tech/documentation/concepts/filtering/#geo
 38 | //   { filter: { must: [{ key: "", match: { value: "" } }] }, must_not: ..., should: ...}
 39 | 
 40 | // const query = [
 41 | //   { type: "provider", operator: "=", value: "Messages" }, // Auto convert "provider:Messages", "provider IN (Messages, Emails)"
 42 | //   { type: "provider", operator: "exists" }, // auto convert "provider exists"
 43 | //   { type: "provider", operator: "is_empty" }, // auto convert "provider is empty"
 44 | //   { type: "date", operator: ">=", value: "2022-03-01T00:00:00Z" }, // auto convert "date > 1 month ago"
 45 | //   { type: "_allSearchableFields", operator: "prefix", value: "Els" }, // auto convert "Els"
 46 | //   // auto convert '"Els Philipp" or "Els Remijnse"'
 47 | //   {
 48 | //     type: "or", operator: "", value: [ // ⚠️ no operator?
 49 | //       { type: "_allSearchableFields", operator: "=", value: "Els Philipp" },
 50 | //       { type: "_allSearchableFields", operator: "=", value: "Els Remijnse" },
 51 | //     ]
 52 | //   },
 53 | //   // auto convert "Els near Amsterdam" or "Els near Amsterdam within 20km". "near + [city] + within [distance]"
 54 | //   {
 55 | //     type: "_geoRadius",
 56 | //     operator: "radius",
 57 | //     value: {
 58 | //       type: "geopoint",
 59 | //       latitude: 32.1212312,
 60 | //       longitude: 4.0123123,
 61 | //       distance: "20km",
 62 | //     },
 63 | //   },
 64 | //   {
 65 | //     type: "_geoRadius",
 66 | //     operator: "radius",
 67 | //     value: {
 68 | //       type: "geopoint",
 69 | //       latitude: 32.1212312,
 70 | //       longitude: 4.0123123,
 71 | //       distance: "20km",
 72 | //     },
 73 | //   },
 74 | //   {
 75 | //     type: "",
 76 | //     operator: "",
 77 | //     value:,
 78 | //   },
 79 | // ];
 80 | 
 81 | const myHighlightStyle = HighlightStyle.define([
 82 |   {
 83 |     tag: tags.string,
 84 |     class: "bg-yellow-300/50 rounded-sm",
 85 |   },
 86 |   {
 87 |     tag: tags.propertyName,
 88 |     class: "dark:bg-gray-900 bg-gray-100 rounded-sm",
 89 |   },
 90 |   { tag: tags.invalid, class: "dark:bg-red-800 bg-red-200 rounded-sm" },
 91 |   { tag: tags.number, class: "dark:text-indigo-200 text-indigo-500" },
 92 |   { tag: tags.className, color: "red" },
 93 |   { tag: tags.arithmeticOperator, color: "blue" },
 94 |   { tag: tags.logicOperator, color: "#f5d", fontStyle: "italic" },
 95 | ]);
 96 | 
 97 | export const Editor = {
 98 |   mounted() {
 99 |     const view = new EditorView({
100 |       state: EditorState.create({
101 |         doc: this.el.value ?? "",
102 |         extensions: [
103 |           closeBrackets(),
104 |           syntaxHighlighting(defaultHighlightStyle, { fallback: true }),
105 |           syntaxHighlighting(myHighlightStyle),
106 |           highlightSpecialChars(),
107 |           singleLine,
108 |           placeholder("Search..."),
109 |           keymap.of([
110 |             ...closeBracketsKeymap,
111 |             ...foldKeymap,
112 |             ...completionKeymap,
113 |           ]),
114 |           Search(),
115 |           autocompletion({
116 |             icons: false,
117 |             selectOnOpen: true,
118 |             activateOnTyping: true,
119 |             closeOnBlur: false,
120 |             maxRenderedOptions: 3,
121 |           }),
122 |           keymap.of([{ key: "Tab", run: acceptCompletion }]), // needed because otherwise tab doesn't autocomplete
123 |           Search().language.data.of({
124 |             autocomplete: (context) => this.autocomplete(context),
125 |           }),
126 |           inlineSuggestion({
127 |             fetchFn: async (state) => {
128 |               console.log({ state });
129 |               return "";
130 |             },
131 |             delay: 0,
132 |           }),
133 |         ],
134 |       }),
135 |       parent: document.getElementById("editor"),
136 |     });
137 |     this.view = view;
138 | 
139 |     // Synchronise the form's textarea with the editor on change
140 |     this.el.form.addEventListener("keyup", (event) => {
141 |       event.preventDefault();
142 | 
143 |       if (
144 |         ["ArrowUp", "ArrowDown", "Enter"].includes(event.code) &&
145 |         completionStatus(view.state) == null
146 |       ) {
147 |         console.log("key-pressed", { key: event.code });
148 |         this.pushEvent("key-pressed", { key: event.code });
149 |         return;
150 |       }
151 | 
152 |       this.search();
153 |     });
154 | 
155 |     this.el.form.addEventListener("addFilter", (event) =>
156 |       this.addFilter(event)
157 |     );
158 |     this.el.form.addEventListener("removeFilter", (event) =>
159 |       this.removeFilter(event)
160 |     );
161 |     this.el.form.addEventListener("resetFilters", (event) =>
162 |       this.resetFilters(event)
163 |     );
164 |     this.el.form.addEventListener("search", (event) => this.search(event));
165 | 
166 |     view.focus();
167 |   },
168 |   addFilter(event) {
169 |     const text = this.view.state.doc.toString();
170 |     const filter = `${event.detail.key}:"${event.detail.value}"`;
171 | 
172 |     this.view.dispatch({
173 |       changes: { from: 0, to: text.length, insert: `${text} ${filter}` },
174 |     });
175 |   },
176 |   removeFilter(event) {
177 |     const text = this.view.state.doc.toString();
178 |     const pattern = new RegExp(`\\s*${event.detail.key}:".*"`);
179 |     const match = pattern.exec(text);
180 |     if (match == null) {
181 |       return;
182 |     }
183 | 
184 |     this.view.dispatch({
185 |       changes: {
186 |         from: match.index,
187 |         to: match.index + match[0].length,
188 |         insert: "",
189 |       },
190 |     });
191 |   },
192 |   resetFilters() {
193 |     const text = this.view.state.doc.toString();
194 |     this.view.dispatch({
195 |       changes: { from: 0, to: text.length, insert: "" },
196 |     });
197 |   },
198 |   search() {
199 |     const query = this.view.state.doc.toString();
200 |     const filters = parseStateToFilters(query, syntaxTree(this.view.state));
201 |     console.log({ query, filters });
202 | 
203 |     this.pushEvent("search", { query, filters });
204 |   },
205 |   autocomplete(context) {
206 |     const token = context.tokenBefore(["FilterExpression", "Identifier"]);
207 |     if (!token || token.from == token.to) return null;
208 | 
209 |     const completion = {
210 |       sources: ["keyword", "search", "field_keys", "field_values"],
211 |       key: null,
212 |       value: token?.text,
213 |     };
214 | 
215 |     // Future to determine if this has been interrupted
216 |     // const interruptedFuture = new Promise<'failed'>((resolve) => {
217 |     //   context.addEventListener('abort', () => {
218 |     //       resolve('failed')
219 |     //   })
220 |     // })
221 | 
222 |     // if (token.type.id == terms.FilterExpression) {
223 |     //   completion.sources = ["field_values"];
224 |     //   completion.key = token.firstChild;
225 |     //   completion.value = token.firstChild?.nextSibling().nextSibling.value;
226 |     // }
227 | 
228 |     console.log({ completion });
229 | 
230 |     return new Promise((resolve) => {
231 |       this.pushEvent("completion", completion, (reply, ref) => {
232 |         resolve({
233 |           from: token.from,
234 |           options: reply.options,
235 |         });
236 |       });
237 |     });
238 |   },
239 |   unmounted() {
240 |     this.view.destroy();
241 |   },
242 | };
243 | 


--------------------------------------------------------------------------------
/memex/assets/js/force_input_value.js:
--------------------------------------------------------------------------------
 1 | export const ForceInputValue = {
 2 |   mounted() {
 3 |     this.handleEvent(
 4 |       "force-input-value",
 5 |       ({ value }) => (this.el.value = value)
 6 |     );
 7 | 
 8 |     // CMD + K to focus on the search input
 9 |     this.listener = document.addEventListener("keydown", (e) => {
10 |       if (e.code !== "KeyK" || e.metaKey === false) return;
11 | 
12 |       this.el.focus();
13 |       this.el.select();
14 |     });
15 |   },
16 | };
17 | 


--------------------------------------------------------------------------------
/memex/assets/js/infinite_scroll.js:
--------------------------------------------------------------------------------
 1 | export const InfiniteScroll = {
 2 |   page() {
 3 |     return this.el.dataset.page;
 4 |   },
 5 |   loadMore(entries) {
 6 |     const target = entries[0];
 7 | 
 8 |     if (target.isIntersecting && this.pending == this.page()) {
 9 |       this.pending = this.page() + 1;
10 |       this.pushEvent("load-more", {});
11 |     }
12 |   },
13 |   mounted() {
14 |     this.pending = this.page();
15 |     this.observer = new IntersectionObserver(
16 |       (entries) => this.loadMore(entries),
17 |       {
18 |         root: null, // window by default
19 |         rootMargin: "0px",
20 |         threshold: 1.0,
21 |       }
22 |     );
23 |     this.observer.observe(this.el);
24 |   },
25 |   beforeDestroy() {
26 |     this.observer.unobserve(this.el);
27 |   },
28 |   updated() {
29 |     if (this.el.dataset.query && this.el.dataset.query != this.query) {
30 |       document.body.scrollTo(0, 0);
31 |     }
32 | 
33 |     this.query = this.el.dataset.query;
34 |     this.pending = this.page();
35 |   },
36 | };
37 | 


--------------------------------------------------------------------------------
/memex/assets/js/map.js:
--------------------------------------------------------------------------------
 1 | export const Map = {
 2 |   async mounted() {
 3 |     const mapboxgl = await import("mapbox-gl");
 4 |     this.mapboxgl = mapboxgl.default;
 5 |     const response = await fetch(this.el.dataset.url, { method: "GET" });
 6 |     const geojson = await response.json();
 7 |     this.map = new this.mapboxgl.Map({
 8 |       container: this.el.id,
 9 |       accessToken:
10 |         "pk.eyJ1IjoiYWRyaXAxMjMiLCJhIjoiY2tvN2Vpa3BlMGE4MTJ2cDd3Nmhrc25uMCJ9.SgEyt_86NvLxnETEfGjrcQ",
11 |       style: "mapbox://styles/mapbox/dark-v10",
12 |     });
13 |     const map = this.map;
14 | 
15 |     map.fitBounds(this.getBounds(geojson), {
16 |       padding: 30,
17 |       maxZoom: 14.15,
18 |       duration: 0,
19 |     });
20 | 
21 |     map.on("load", () => {
22 |       map.addSource("route", {
23 |         type: "geojson",
24 |         data: geojson,
25 |       });
26 | 
27 |       map.addLayer({
28 |         id: "route",
29 |         type: "line",
30 |         source: "route",
31 |         layout: {
32 |           "line-join": "round",
33 |           "line-cap": "round",
34 |         },
35 |         paint: {
36 |           "line-color": "#FCD34D",
37 |           "line-width": 5,
38 |         },
39 |       });
40 |     });
41 |   },
42 |   unmounted() {
43 |     this.map.remove();
44 |   },
45 |   updated() {
46 |     if (!this.map) return;
47 |     const map = this.map;
48 |     const items = JSON.parse(this.el.dataset.items);
49 | 
50 |     map.on("load", () => {
51 |       for (let i = 0; i < items.length; i++) {
52 |         const item = items[i];
53 |         // if geometry: { type: "Point", create marker with properties config
54 |         this.createMarker(item.data).addTo(map);
55 |       }
56 |     });
57 |   },
58 |   createMarker(feature) {
59 |     const el = document.createElement("div");
60 |     for (let prop of Object.keys(feature.properties.style)) {
61 |       el.style[prop.toString()] = feature.properties.style[prop.toString()];
62 |     }
63 | 
64 |     return new this.mapboxgl.Marker(el).setLngLat(feature.geometry.coordinates);
65 |   },
66 |   getBounds(geojson) {
67 |     const first = geojson.geometry.coordinates[0];
68 |     let bounds = new this.mapboxgl.LngLatBounds(first, first);
69 | 
70 |     geojson.geometry.coordinates.forEach((coordinate) => {
71 |       bounds.extend(coordinate);
72 |     });
73 | 
74 |     return bounds;
75 |   },
76 | };
77 | 


--------------------------------------------------------------------------------
/memex/assets/js/search_language/index.js:
--------------------------------------------------------------------------------
 1 | import { parser } from "./syntax.js";
 2 | import * as terms from "./syntax.terms.js";
 3 | import {
 4 |   LRLanguage,
 5 |   LanguageSupport,
 6 |   bracketMatching,
 7 |   syntaxTree,
 8 | } from "@codemirror/language";
 9 | import { styleTags, tags as t } from "@lezer/highlight";
10 | 
11 | export const parseStateToFilters = (query, state) => {
12 |   const filters = [];
13 |   const tree = state;
14 | 
15 |   console.log({ tree: tree.toString() });
16 |   let cursor = tree.cursor();
17 | 
18 |   do {
19 |     if (
20 |       [terms.Prefix, terms.NotPrefix, terms.Exact, terms.NotExact].includes(
21 |         cursor.node.type.id
22 |       )
23 |     ) {
24 |       filters.push({
25 |         type: cursor.node.type.name,
26 |         value: getNodeContent(query, cursor.node.firstChild),
27 |       });
28 |     }
29 | 
30 |     if (cursor.node.type.id === terms.FilterExpression) {
31 |       filters.push({
32 |         type: cursor.node.firstChild.nextSibling.type.name,
33 |         key: getNodeContent(query, cursor.node.firstChild),
34 |         value: getNodeContent(
35 |           query,
36 |           cursor.node.firstChild.nextSibling.nextSibling
37 |         ),
38 |       });
39 |     }
40 |   } while (cursor.next());
41 | 
42 |   return filters;
43 | };
44 | 
45 | function getNodeContent(query, node) {
46 |   const content = query.slice(node.from, node.to).trim().replace(/\"/g, "");
47 | 
48 |   if (node.type.id === terms.Number) {
49 |     return Number(content);
50 |   }
51 | 
52 |   return content;
53 | }
54 | 
55 | export const SearchLanguage = LRLanguage.define({
56 |   name: "search",
57 |   parser: parser.configure({
58 |     props: [
59 |       styleTags({
60 |         FilterKey: t.propertyName,
61 |         FilterValue: t.string,
62 |         Identifier: t.string,
63 |         QuotedString: t.string,
64 |         Prefix: t.arithmeticOperator,
65 |         Number: t.number,
66 |         // FilterExpression: t.definition,
67 |         "and or && ||": t.logicOperator,
68 |         ": > >= < <= =": t.compareOperator,
69 |         "⚠": t.invalid,
70 |         // "( )": t.paren,
71 |         // "[ ]": t.squareBracket,
72 |         // "{ }": t.brace,
73 |       }),
74 |     ],
75 |   }),
76 |   languageData: {
77 |     // commentTokens: { line: ";" },
78 |   },
79 | });
80 | 
81 | export function Search() {
82 |   return new LanguageSupport(
83 |     SearchLanguage,
84 |     bracketMatching({ brackets: "()" })
85 |   );
86 | }
87 | 


--------------------------------------------------------------------------------
/memex/assets/js/search_language/index.test.js:
--------------------------------------------------------------------------------
 1 | import { describe, expect } from "@jest/globals";
 2 | import { Search, parseStateToFilters } from "./index";
 3 | 
 4 | const search = (query) =>
 5 |   parseStateToFilters(query, Search().language.parser.parse(query));
 6 | 
 7 | describe("Search parser", () => {
 8 |   test("parse field equals", () => {
 9 |     expect(search("person:Peter")).toEqual([
10 |       { type: "Equals", key: "person", value: "Peter" },
11 |     ]);
12 |     expect(search("person=Peter")).toEqual([
13 |       { type: "Equals", key: "person", value: "Peter" },
14 |     ]);
15 |   });
16 | 
17 |   test("prefix search", () => {
18 |     expect(search("d")).toEqual([{ type: "Prefix", value: "d" }]);
19 |     expect(search("d e")).toEqual([
20 |       { type: "Prefix", value: "d" },
21 |       { type: "Prefix", value: "e" },
22 |     ]);
23 |   });
24 | 
25 |   test("comparison", () => {
26 |     expect(search("d>1")).toEqual([
27 |       { type: "GreaterThan", key: "d", value: 1 },
28 |     ]);
29 |     expect(search("d>=1")).toEqual([
30 |       { type: "GreaterThanEquals", key: "d", value: 1 },
31 |     ]);
32 |     expect(search("d<1")).toEqual([{ type: "LessThan", key: "d", value: 1 }]);
33 |     expect(search("d<=1")).toEqual([
34 |       { type: "LessThanEquals", key: "d", value: 1 },
35 |     ]);
36 |     expect(search("d<1.4")).toEqual([
37 |       { type: "LessThan", key: "d", value: 1.4 },
38 |     ]);
39 |   });
40 | 
41 |   test("mixed", () => {
42 |     expect(search("d:1 e")).toEqual([
43 |       { type: "Equals", key: "d", value: "1" },
44 |       { type: "Prefix", value: "e" },
45 |     ]);
46 |     expect(search("e d:1")).toEqual([
47 |       { type: "Prefix", value: "e" },
48 |       { type: "Equals", key: "d", value: "1" },
49 |     ]);
50 |   });
51 | });
52 | 


--------------------------------------------------------------------------------
/memex/assets/js/search_language/syntax.grammer:
--------------------------------------------------------------------------------
 1 | @precedence {
 2 |   compare @left,
 3 |   equality @left,
 4 |   and @left,
 5 |   or @left
 6 | }
 7 | 
 8 | @top Search { expression+ }
 9 | 
10 | expression { (Prefix | NotPrefix | Exact | NotExact | FilterExpression | BooleanExpression) }
11 | 
12 | Exact { QuotedString }
13 | NotExact { not QuotedString }
14 | Prefix { Identifier }
15 | NotPrefix { not Identifier }
16 | 
17 | FilterExpression {
18 |   FilterKey !compare op<GreaterThan, ">"> Number |
19 |   FilterKey !compare op<GreaterThanEquals, ">="> Number |
20 |   FilterKey !compare op<LessThan, "<"> Number |
21 |   FilterKey !compare op<LessThanEquals, "<="> Number |
22 |   FilterKey !equality op<Equals, ":" | "=" > FilterValue |
23 |   FilterKey !equality op<NotEquals, ":!" | "!=" > FilterValue
24 | }
25 | 
26 | BooleanExpression {
27 |   expression !and (op<Logic, "&&"> | Logic { kw<"and"> }) expression |
28 |   expression !or (op<Logic, "||"> | Logic { kw<"or"> }) expression
29 | }
30 | 
31 | kw<term> { @specialize[@name={term}]<Identifier, term> }
32 | 
33 | FilterKey { Identifier }
34 | FilterValue { Identifier | QuotedString }
35 | 
36 | @skip { space }
37 | 
38 | @tokens {
39 |   space { @whitespace+ }
40 |   Identifier { $[a-zA-Z0-9_]+ }
41 |   not { "!" | "-" }
42 |   Number { $[0-9]+ | $[0-9]+ "." $[0-9]+ }
43 |   QuotedString {  "\"" (![\\"\n] | "\\" (![\n] | "\n"))+ "\"" }
44 | 
45 |   op[@name={name}]<name, body> {body}
46 | }
47 | 


--------------------------------------------------------------------------------
/memex/assets/js/search_language/syntax.js:
--------------------------------------------------------------------------------
 1 | // This file was generated by lezer-generator. You probably shouldn't edit it.
 2 | import {LRParser} from "@lezer/lr"
 3 | const spec_Identifier = {__proto__:null,and:42, or:48}
 4 | export const parser = LRParser.deserialize({
 5 |   version: 14,
 6 |   states: "#rOVQPOOObQPO'#C^O!`QPO'#C`OOQO'#Ca'#CaO!hQPO'#CdO!|QPO'#CuOOQO'#Cy'#CyQVQPOOOOQO,58z,58zOOQO,58},58}O#hQQO,59OO#mQPO,59OOVQPO,59YOVQPO,59YOOQO'#Cp'#CpOOQO'#Cs'#CsOOQO-E6s-E6sOOQO1G.j1G.jOOQO'#Cl'#ClOOQO1G.t1G.tO#uQPO1G.t",
 7 |   stateData: "$a~OlOS~ORPOUROnQO~ORQXUQXYXX[XX]XX^XX_XXaXXcQXeQXfQXhQXjQXnQX~ORWOUXO~OYYO[YO]YO^YO_ZOaZO~Oc[Oe^Of]Oh_ORiXUiXjiXniX~OZaO~ORbOUbO~Oc[Oe^ORbiUbifbihbijbinbi~O",
 8 |   goto: "!gnPPoPooPoouPPPPPP{PoP!OPP!SP!VPPP!]XUOV[]XSOV[]RaZT[TdR]TQVOR`VSTOVQc[Rd]",
 9 |   nodeNames: "⚠ Search Prefix Identifier NotPrefix Exact QuotedString NotExact FilterExpression FilterKey GreaterThan Number GreaterThanEquals LessThan LessThanEquals Equals FilterValue NotEquals BooleanExpression Logic Logic and Logic Logic or",
10 |   maxTerm: 30,
11 |   skippedNodes: [0],
12 |   repeatNodeCount: 1,
13 |   tokenData: "'{~RgX^!jpq!jqr#_rs#lvw%`}!O%k!Q![%p![!]&w!^!_'P!_!`'^!`!a'c!c!}&f#R#S&f#T#o&f#p#q'p#y#z!j$f$g!j#BY#BZ!j$IS$I_!j$I|$JO!j$JT$JU!j$KV$KW!j&FU&FV!j~!oYl~X^!jpq!j#y#z!j$f$g!j#BY#BZ!j$IS$I_!j$I|$JO!j$JT$JU!j$KV$KW!j&FU&FV!j~#dPn~!_!`#g~#lOa~~#oVOY$UZr$Us#O$U#O#P$v#P;'S$U;'S;=`%Y<%lO$U~$XWOY$UZr$Urs$qs#O$U#O#P$v#P;'S$U;'S;=`%Y<%lO$U~$vOU~~$yTOY$UYZ$UZ;'S$U;'S;=`%Y<%lO$U~%]P;=`<%l$U~%cPvw%f~%kOc~~%pOn~R%wTRPZQ!O!P&W!Q![%p!c!}&f#R#S&f#T#o&fQ&ZP!Q![&^Q&cPZQ!Q![&^P&kSRP!Q![&f!c!}&f#R#S&f#T#o&f~&|P_~qr#g~'UP]~!_!`'X~'^O^~~'cO_~~'hPY~!_!`'k~'pO[~~'sP#p#q'v~'{Of~",
14 |   tokenizers: [0, 1],
15 |   topRules: {"Search":[0,1]},
16 |   specialized: [{term: 3, get: value => spec_Identifier[value] || -1}],
17 |   tokenPrec: 0
18 | })
19 | 


--------------------------------------------------------------------------------
/memex/assets/js/search_language/syntax.terms.js:
--------------------------------------------------------------------------------
 1 | // This file was generated by lezer-generator. You probably shouldn't edit it.
 2 | export const
 3 |   Search = 1,
 4 |   Prefix = 2,
 5 |   Identifier = 3,
 6 |   NotPrefix = 4,
 7 |   Exact = 5,
 8 |   QuotedString = 6,
 9 |   NotExact = 7,
10 |   FilterExpression = 8,
11 |   FilterKey = 9,
12 |   Number = 11,
13 |   FilterValue = 16,
14 |   BooleanExpression = 18
15 | 


--------------------------------------------------------------------------------
/memex/assets/js/sidebar.js:
--------------------------------------------------------------------------------
 1 | export const Sidebar = {
 2 |   mounted() {
 3 |     this.toggleOnEscape();
 4 |   },
 5 |   updated() {
 6 |     if (this.el.dataset.state === "open") {
 7 |       this.lockScroll();
 8 |     } else {
 9 |       this.unlockScroll();
10 |     }
11 |   },
12 |   toggleOnEscape() {
13 |     this.listener = document.addEventListener("keyup", (e) => {
14 |       if (e.code !== "Escape") return;
15 | 
16 |       this.pushEvent("close-last-sidebar");
17 |     });
18 |   },
19 |   lockScroll() {
20 |     document.body.style.overflow = `hidden`;
21 |   },
22 |   unlockScroll() {
23 |     document.body.style.overflow = `auto`;
24 |   },
25 | };
26 | 


--------------------------------------------------------------------------------
/memex/assets/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "scripts": {
 3 |     "deploy": "NODE_ENV=production tailwindcss --postcss --minify -i css/app.css -o ../priv/static/assets/app.css",
 4 |     "test": "jest"
 5 |   },
 6 |   "dependencies": {
 7 |     "@codemirror/lang-css": "^6.2.1",
 8 |     "@codemirror/language": "6.9.2",
 9 |     "@lezer/highlight": "^1.0.0",
10 |     "@lezer/lr": "^1.3.13",
11 |     "codemirror": "^6.0.1",
12 |     "codemirror-extension-inline-suggestion": "^0.0.1",
13 |     "mapbox-gl": "2.15.0"
14 |   },
15 |   "devDependencies": {
16 |     "@lezer/generator": "^1.0.0",
17 |     "autoprefixer": "^10.4.16",
18 |     "esbuild": "^0.19.2",
19 |     "esbuild-jest": "^0.5.0",
20 |     "jest": "^29.6.2",
21 |     "postcss": "^8.4.31",
22 |     "postcss-import": "^15.1.0",
23 |     "tailwindcss": "^3.3.3"
24 |   }
25 | }
26 | 


--------------------------------------------------------------------------------
/memex/assets/postcss.config.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 |   plugins: {
3 |     'postcss-import': {},
4 |     tailwindcss: {},
5 |     autoprefixer: {},
6 |   },
7 | };
8 | 


--------------------------------------------------------------------------------
/memex/assets/tailwind.config.js:
--------------------------------------------------------------------------------
 1 | const colors = require("tailwindcss/colors");
 2 | const plugin = require("tailwindcss/plugin");
 3 | 
 4 | module.exports = {
 5 |   content: [
 6 |     "../lib/**/*.ex",
 7 |     "../lib/**/*.leex",
 8 |     "../lib/**/*.heex",
 9 |     "../lib/**/*.eex",
10 |     "./js/**/*.js",
11 |   ],
12 |   theme: {
13 |     colors: {
14 |       transparent: "transparent",
15 |       current: "currentColor",
16 |       black: colors.black,
17 |       white: colors.white,
18 |       gray: { ...colors.zinc, 800: "#262728" }, // or colors.neutral
19 |       green: colors.green,
20 |       indigo: colors.indigo,
21 |       red: colors.rose,
22 |       yellow: colors.amber,
23 |     },
24 |     extend: {},
25 |   },
26 |   plugins: [
27 |     require("@tailwindcss/forms"),
28 |     plugin(({ addVariant }) =>
29 |       addVariant("phx-no-feedback", [".phx-no-feedback&", ".phx-no-feedback &"])
30 |     ),
31 |     plugin(({ addVariant }) =>
32 |       addVariant("phx-click-loading", [
33 |         ".phx-click-loading&",
34 |         ".phx-click-loading &",
35 |       ])
36 |     ),
37 |     plugin(({ addVariant }) =>
38 |       addVariant("phx-submit-loading", [
39 |         ".phx-submit-loading&",
40 |         ".phx-submit-loading &",
41 |       ])
42 |     ),
43 |     plugin(({ addVariant }) =>
44 |       addVariant("phx-change-loading", [
45 |         ".phx-change-loading&",
46 |         ".phx-change-loading &",
47 |       ])
48 |     ),
49 |   ],
50 | };
51 | 


--------------------------------------------------------------------------------
/memex/config/config.exs:
--------------------------------------------------------------------------------
 1 | # This file is responsible for configuring your application
 2 | # and its dependencies with the aid of the Mix.Config module.
 3 | #
 4 | # This configuration file is loaded before any dependency and
 5 | # is restricted to this project.
 6 | 
 7 | # General application configuration
 8 | import Config
 9 | 
10 | # Configures the endpoint
11 | config :memex, MemexWeb.Endpoint,
12 |   adapter: Bandit.PhoenixAdapter,
13 |   url: [host: "localhost"],
14 |   http: [ip: {0, 0, 0, 0}],
15 |   secret_key_base: "BP29B2R/vGyAzIXMvxN0W8Qs/Ok1UjxP7/mDqoAL872Ima1bZMKhZ09ZYQqlTn96",
16 |   render_errors: [
17 |     formats: [html: MemexWeb.ErrorHTML, json: MemexWeb.ErrorJSON],
18 |     layout: false
19 |   ],
20 |   pubsub_server: Memex.PubSub,
21 |   live_view: [signing_salt: "lj/89OzE"]
22 | 
23 | # Configures Elixir's Logger
24 | config :logger, :console,
25 |   format: "$time $metadata[$level] $message\n",
26 |   metadata: [:request_id]
27 | 
28 | config :memex, ecto_repos: [Memex.Repo]
29 | 
30 | config :memex, Memex.Repo,
31 |   url: System.get_env("POSTGRES_DSN"),
32 |   pool_size: 5,
33 |   timeout: 60_000
34 | 
35 | config :elixir, :time_zone_database, Tzdata.TimeZoneDatabase
36 | 
37 | config :esbuild,
38 |   version: "0.15.18",
39 |   default: [
40 |     args:
41 |       ~w(js/app.js --bundle --target=es2017 --splitting --format=esm --outdir=../priv/static/assets --external:/fonts/* --external:/images/*),
42 |     cd: Path.expand("../assets", __DIR__),
43 |     env: %{"NODE_PATH" => Path.expand("../deps", __DIR__)}
44 |   ]
45 | 
46 | # Configure tailwind (the version is required)
47 | config :tailwind,
48 |   version: "3.2.4",
49 |   default: [
50 |     args: ~w(
51 |       --config=tailwind.config.js
52 |       --input=css/app.css
53 |       --output=../priv/static/assets/app.css
54 |     ),
55 |     cd: Path.expand("../assets", __DIR__)
56 |   ]
57 | 
58 | # Use Jason for JSON parsing in Phoenix
59 | config :phoenix, :json_library, Jason
60 | 
61 | # config :tesla, adapter: Tesla.Adapter.Hackney
62 | config :tesla, :adapter, {Tesla.Adapter.Finch, name: Memex.Finch}
63 | 
64 | # Import environment specific config. This must remain at the bottom
65 | # of this file so it overrides the configuration defined above.
66 | import_config "#{config_env()}.exs"
67 | 


--------------------------------------------------------------------------------
/memex/config/dev.exs:
--------------------------------------------------------------------------------
 1 | import Config
 2 | 
 3 | # For development, we disable any cache and enable
 4 | # debugging and code reloading.
 5 | #
 6 | # The watchers configuration can be used to run external
 7 | # watchers to your application. For example, we use it
 8 | # with webpack to recompile .js and .css sources.
 9 | config :memex, MemexWeb.Endpoint,
10 |   http: [port: 4000],
11 |   debug_errors: true,
12 |   code_reloader: true,
13 |   check_origin: false,
14 |   watchers: [
15 |     esbuild: {Esbuild, :install_and_run, [:default, ~w(--watch --splitting)]},
16 |     tailwind: {Tailwind, :install_and_run, [:default, ~w(--watch)]}
17 |   ]
18 | 
19 | # ## SSL Support
20 | #
21 | # In order to use HTTPS in development, a self-signed
22 | # certificate can be generated by running the following
23 | # Mix task:
24 | #
25 | #     mix phx.gen.cert
26 | #
27 | # Note that this task requires Erlang/OTP 20 or later.
28 | # Run `mix help phx.gen.cert` for more information.
29 | #
30 | # The `http:` config above can be replaced with:
31 | #
32 | #     https: [
33 | #       port: 4001,
34 | #       cipher_suite: :strong,
35 | #       keyfile: "priv/cert/selfsigned_key.pem",
36 | #       certfile: "priv/cert/selfsigned.pem"
37 | #     ],
38 | #
39 | # If desired, both `http:` and `https:` keys can be
40 | # configured to run both http and https servers on
41 | # different ports.
42 | 
43 | # Watch static and templates for browser reloading.
44 | config :memex, MemexWeb.Endpoint,
45 |   live_reload: [
46 |     patterns: [
47 |       ~r"priv/static/.*(js|css|png|jpeg|jpg|gif|svg)$",
48 |       ~r"priv/gettext/.*(po)$",
49 |       ~r"lib/memex_web/(controllers|live|components)/.*(ex|heex)$"
50 |     ]
51 |   ]
52 | 
53 | # Do not include metadata nor timestamps in development logs
54 | config :logger, :console, format: "[$level] $message\n"
55 | 
56 | # Set a higher stacktrace during development. Avoid configuring such
57 | # in production as building large stacktraces may be expensive.
58 | config :phoenix, :stacktrace_depth, 20
59 | 
60 | # Initialize plugs at runtime for faster development compilation
61 | config :phoenix, :plug_init_mode, :runtime
62 | 


--------------------------------------------------------------------------------
/memex/config/prod.exs:
--------------------------------------------------------------------------------
 1 | import Config
 2 | 
 3 | # For production, don't forget to configure the url host
 4 | # to something meaningful, Phoenix uses this information
 5 | # when generating URLs.
 6 | #
 7 | # Note we also include the path to a cache manifest
 8 | # containing the digested version of static files. This
 9 | # manifest is generated by the `mix phx.digest` task,
10 | # which you should run after static files are built and
11 | # before starting your production server.
12 | config :memex, MemexWeb.Endpoint,
13 |   url: [host: "example.com", port: 80],
14 |   cache_static_manifest: "priv/static/cache_manifest.json"
15 | 
16 | # Do not print debug messages in production
17 | config :logger, level: :info
18 | 
19 | # ## SSL Support
20 | #
21 | # To get SSL working, you will need to add the `https` key
22 | # to the previous section and set your `:url` port to 443:
23 | #
24 | #     config :memex, MemexWeb.Endpoint,
25 | #       ...
26 | #       url: [host: "example.com", port: 443],
27 | #       https: [
28 | #         port: 443,
29 | #         cipher_suite: :strong,
30 | #         keyfile: System.get_env("SOME_APP_SSL_KEY_PATH"),
31 | #         certfile: System.get_env("SOME_APP_SSL_CERT_PATH"),
32 | #         transport_options: [socket_opts: [:inet6]]
33 | #       ]
34 | #
35 | # The `cipher_suite` is set to `:strong` to support only the
36 | # latest and more secure SSL ciphers. This means old browsers
37 | # and clients may not be supported. You can set it to
38 | # `:compatible` for wider support.
39 | #
40 | # `:keyfile` and `:certfile` expect an absolute path to the key
41 | # and cert in disk or a relative path inside priv, for example
42 | # "priv/ssl/server.key". For all supported SSL configuration
43 | # options, see https://hexdocs.pm/plug/Plug.SSL.html#configure/1
44 | #
45 | # We also recommend setting `force_ssl` in your endpoint, ensuring
46 | # no data is ever sent via http, always redirecting to https:
47 | #
48 | #     config :memex, MemexWeb.Endpoint,
49 | #       force_ssl: [hsts: true]
50 | #
51 | # Check `Plug.SSL` for all available options in `force_ssl`.
52 | 
53 | # Finally import the config/prod.secret.exs which loads secrets
54 | # and configuration from environment variables.
55 | import_config "prod.secret.exs"
56 | 


--------------------------------------------------------------------------------
/memex/config/prod.secret.exs:
--------------------------------------------------------------------------------
 1 | # In this file, we load production configuration and secrets
 2 | # from environment variables. You can also hardcode secrets,
 3 | # although such is generally not recommended and you have to
 4 | # remember to add this file to your .gitignore.
 5 | import Config
 6 | 
 7 | secret_key_base =
 8 |   System.get_env("SECRET_KEY_BASE") ||
 9 |     raise """
10 |     environment variable SECRET_KEY_BASE is missing.
11 |     You can generate one by calling: mix phx.gen.secret
12 |     """
13 | 
14 | config :memex, MemexWeb.Endpoint,
15 |   http: [
16 |     port: String.to_integer(System.get_env("PORT") || "4000"),
17 |     transport_options: [socket_opts: [:inet6]]
18 |   ],
19 |   secret_key_base: secret_key_base
20 | 
21 | # ## Using releases (Elixir v1.9+)
22 | #
23 | # If you are doing OTP releases, you need to instruct Phoenix
24 | # to start each relevant endpoint:
25 | #
26 | #     config :memex, MemexWeb.Endpoint, server: true
27 | #
28 | # Then you can assemble a release by calling `mix release`.
29 | # See `mix help release` for more information.
30 | 


--------------------------------------------------------------------------------
/memex/config/test.exs:
--------------------------------------------------------------------------------
 1 | import Config
 2 | 
 3 | # We don't run a server during test. If one is required,
 4 | # you can enable the server option below.
 5 | config :memex, MemexWeb.Endpoint,
 6 |   http: [port: 4002],
 7 |   server: false
 8 | 
 9 | # Print only warnings and errors during test
10 | config :logger, level: :warn
11 | 


--------------------------------------------------------------------------------
/memex/lib/memex.ex:
--------------------------------------------------------------------------------
 1 | defmodule Memex do
 2 |   @moduledoc """
 3 |   Memex keeps the contexts that define your domain
 4 |   and business logic.
 5 | 
 6 |   Contexts are also responsible for managing your data, regardless
 7 |   if it comes from the database, an external API or others.
 8 |   """
 9 | end
10 | 


--------------------------------------------------------------------------------
/memex/lib/memex/application.ex:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Application do
 2 |   # See https://hexdocs.pm/elixir/Application.html
 3 |   # for more information on OTP Applications
 4 |   @moduledoc false
 5 | 
 6 |   use Application
 7 | 
 8 |   def start(_type, _args) do
 9 |     children = [
10 |       # Start the Telemetry supervisor
11 |       MemexWeb.Telemetry,
12 |       {Finch, name: Memex.Finch},
13 |       {ConCache, [name: :search, ttl_check_interval: false]},
14 |       # Start the PubSub system
15 |       {Phoenix.PubSub, name: Memex.PubSub},
16 |       # Start the Endpoint (http/https)
17 |       MemexWeb.Endpoint,
18 |       Memex.Repo,
19 |       # Start a worker by calling: Memex.Worker.start_link(arg)
20 |       # {Memex.Worker, arg}
21 |       Memex.Vault,
22 |       Memex.Scheduler
23 |     ]
24 | 
25 |     # See https://hexdocs.pm/elixir/Supervisor.html
26 |     # for other strategies and supported options
27 |     opts = [strategy: :one_for_one, name: Memex.Supervisor]
28 |     Supervisor.start_link(children, opts)
29 |   end
30 | 
31 |   # Tell Phoenix to update the endpoint configuration
32 |   # whenever the application is updated.
33 |   def config_change(changed, _new, removed) do
34 |     MemexWeb.Endpoint.config_change(changed, removed)
35 |     :ok
36 |   end
37 | end
38 | 


--------------------------------------------------------------------------------
/memex/lib/memex/connector.ex:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Connector do
 2 |   @moduledoc false
 3 |   alias Exqlite.Basic, as: Sqlite3
 4 |   alias Exqlite.Connection
 5 | 
 6 |   def sqlite_json(path, query, args \\ [], setup \\ [], connection_options \\ []) do
 7 |     options =
 8 |       connection_options
 9 |       |> Keyword.merge(
10 |         database: path,
11 |         mode: :readonly
12 |       )
13 |       |> Keyword.filter(fn {_k, v} -> not is_nil(v) end)
14 | 
15 |     with {:ok, conn} <- Connection.connect(options),
16 |          {:ok, _} <- sqlite_queries(conn, setup),
17 |          {:ok, rows} <- sqlite_query(conn, query, args) do
18 |       {:ok, Enum.map(rows, &Jason.decode!(&1))}
19 |     end
20 |   end
21 | 
22 |   defp sqlite_queries(conn, queries) do
23 |     Enum.reduce_while(queries, {:ok, []}, fn query, {:ok, acc} ->
24 |       case sqlite_query(conn, query) do
25 |         {:ok, rows} -> {:cont, {:ok, [rows | acc]}}
26 |         {:error, _} = err -> {:halt, err}
27 |       end
28 |     end)
29 |   end
30 | 
31 |   defp sqlite_query(conn, query, args \\ []) do
32 |     with {:ok, rows, _columns} <- Sqlite3.rows(Sqlite3.exec(conn, query, args)) do
33 |       {:ok, rows}
34 |     end
35 |   end
36 | 
37 |   def cmd(command, args \\ []) do
38 |     case System.cmd(command, args) do
39 |       {result, 0} -> {:ok, result}
40 |       {result, exit_status} -> {:error, result, exit_status}
41 |     end
42 |   end
43 | 
44 |   def shell(command) do
45 |     case System.shell(command) do
46 |       {result, 0} -> {:ok, result}
47 |       {result, exit_status} -> {:error, result, exit_status}
48 |     end
49 |   end
50 | 
51 |   def json_file(path, compresed) do
52 |     path
53 |     |> File.stream!(
54 |       if compresed do
55 |         [:compressed]
56 |       else
57 |         []
58 |       end
59 |     )
60 |     |> Enum.into("")
61 |     |> Jason.decode()
62 |   end
63 | end
64 | 


--------------------------------------------------------------------------------
/memex/lib/memex/importer.ex:
--------------------------------------------------------------------------------
  1 | defmodule Memex.Importer do
  2 |   @moduledoc false
  3 |   import Memex.Connector
  4 | 
  5 |   alias Memex.Repo
  6 |   alias Memex.Schema.Document
  7 |   alias Memex.Schema.ImporterLog
  8 | 
  9 |   @max_records_per_batch 20_000
 10 |   @pubsub Memex.PubSub
 11 | 
 12 |   defmodule Sqlite do
 13 |     @moduledoc false
 14 |     defstruct [:location, :query, setup: [], connection_options: []]
 15 |   end
 16 | 
 17 |   defmodule Command do
 18 |     @moduledoc false
 19 |     defstruct [:command, :arguments]
 20 |   end
 21 | 
 22 |   defmodule Shell do
 23 |     @moduledoc false
 24 |     defstruct [:command]
 25 |   end
 26 | 
 27 |   defmodule JsonEndpoint do
 28 |     defstruct [:url, :headers]
 29 |   end
 30 | 
 31 |   defmodule JsonFile do
 32 |     @moduledoc false
 33 |     defstruct [:location, :compressed]
 34 |   end
 35 | 
 36 |   def parse_body(""), do: {:error, :no_data}
 37 |   def parse_body([]), do: {:error, :no_data}
 38 |   def parse_body(list), do: {:ok, Enum.map(list, &[body: &1])}
 39 | 
 40 |   def available_importers do
 41 |     with {:ok, list} <- :application.get_key(:memex, :modules) do
 42 |       list
 43 |       |> Enum.filter(&(&1 |> Module.split() |> Enum.take(2) == ~w|Memex Importers|))
 44 |       |> Enum.map(fn module ->
 45 |         {:module, module} = Code.ensure_loaded(module)
 46 |         module
 47 |       end)
 48 |       |> Enum.filter(&function_exported?(&1, :provider, 0))
 49 |       |> Map.new(fn module -> {module.provider(), module} end)
 50 |     end
 51 |   end
 52 | 
 53 |   def configured_importers do
 54 |     Repo.all(Memex.Schema.ImporterConfig)
 55 |   end
 56 | 
 57 |   def register_importers do
 58 |     existing_importers = Enum.map(configured_importers(), & &1.provider)
 59 | 
 60 |     available_importers()
 61 |     |> Enum.reject(fn {id, _importer} -> Enum.member?(existing_importers, id) end)
 62 |     |> Enum.reject(fn {_id, module} -> function_exported?(module, :required_config, 0) end)
 63 |     |> Enum.map(fn {id, importer} ->
 64 |       create_importer(
 65 |         id,
 66 |         importer.provider(),
 67 |         importer.provider(),
 68 |         importer.default_config(),
 69 |         %{}
 70 |       )
 71 |     end)
 72 |   end
 73 | 
 74 |   def create_importer(name, provider, display_name, encrypted_secrets, config_overwrite) do
 75 |     Repo.insert(%Memex.Schema.ImporterConfig{
 76 |       id: name,
 77 |       provider: provider,
 78 |       display_name: display_name,
 79 |       encrypted_secrets: encrypted_secrets,
 80 |       config_overwrite: config_overwrite
 81 |     })
 82 |   end
 83 | 
 84 |   def subscribe do
 85 |     Phoenix.PubSub.subscribe(@pubsub, topic())
 86 |   end
 87 | 
 88 |   defp broadcast!(msg) do
 89 |     Phoenix.PubSub.broadcast!(@pubsub, topic(), {__MODULE__, msg})
 90 |   end
 91 | 
 92 |   defp topic, do: "importer:*"
 93 | 
 94 |   def insert(list) do
 95 |     with {:ok, documents} <- parse_body(list) do
 96 |       bulk_upsert_documents(documents)
 97 |     end
 98 |   end
 99 | 
100 |   @doc """
101 |   Fetches, transforms and stores documents from a given importer config.
102 |   """
103 |   def import(config) do
104 |     {:ok, module} = get_module(config)
105 | 
106 |     if function_exported?(module, :fetch, 1) do
107 |       do_import(config)
108 |     else
109 |       {:skipped, :fetch_not_supported}
110 |     end
111 |   end
112 | 
113 |   defp do_import(config) do
114 |     {:ok, log} = Repo.insert(%ImporterLog{state: "running", log: "", config_id: config.id})
115 |     broadcast!(log)
116 | 
117 |     try do
118 |       with {:ok, module} <- get_module(config),
119 |            {:ok, merged_config} <- merge_module_config(module, config),
120 |            {:fetch, {:ok, result}} <- {:fetch, fetch(module, merged_config)},
121 |            {:transform, {:ok, documents, invalid}} <-
122 |              {:transform, transform(module, result, merged_config)},
123 |            # todo: keep fetching until items show up that are already stored
124 |            {:store, {:ok}} <- {:store, store(module, documents, log)} do
125 |         update_log(log, "success", Kernel.inspect(invalid))
126 | 
127 |         {:ok, documents, invalid}
128 |       else
129 |         {step, error} ->
130 |           update_log(log, "error", Kernel.inspect(error))
131 |           {:error, step, error}
132 |       end
133 |     catch
134 |       error, reason ->
135 |         update_log(log, "error", Kernel.inspect(reason))
136 | 
137 |         IO.puts(:stderr, Exception.format(:error, reason, __STACKTRACE__))
138 | 
139 |         {:error, error, reason}
140 |     end
141 |   end
142 | 
143 |   def get_dirs_to_watch do
144 |     configured_importers()
145 |     |> Enum.map(fn config ->
146 |       with {:ok, module} <- get_module(config),
147 |            {:ok, merged_config} <- merge_module_config(module, config),
148 |            :watcher <- merged_config["schedule"] do
149 |         merged_config["location"]
150 |       else
151 |         _ -> false
152 |       end
153 |     end)
154 |     |> Enum.filter(fn dir -> dir !== false end)
155 |     |> Enum.filter(fn dir -> not is_nil(dir) end)
156 |   end
157 | 
158 |   defp get_module(config) do
159 |     case available_importers()[config.provider] do
160 |       nil ->
161 |         {:error, :no_importer_available}
162 | 
163 |       module ->
164 |         {:ok, module}
165 |     end
166 |   end
167 | 
168 |   defp merge_module_config(module, config) do
169 |     merged =
170 |       module.default_config()
171 |       |> Map.merge(config.config_overwrite)
172 |       |> Map.merge(config.encrypted_secrets)
173 | 
174 |     {:ok, merged}
175 |   end
176 | 
177 |   defp update_log(log, state, message) do
178 |     log
179 |     |> Ecto.Changeset.change(%{state: state, log: message})
180 |     |> Repo.update!()
181 |     |> broadcast!()
182 |   end
183 | 
184 |   defp fetch(module, config) do
185 |     case module.fetch(config) do
186 |       %Sqlite{
187 |         location: location,
188 |         query: query,
189 |         setup: setup,
190 |         connection_options: connection_options
191 |       } ->
192 |         sqlite_json(location, query, [], setup, connection_options)
193 | 
194 |       %Command{command: command, arguments: args} ->
195 |         cmd(command, args)
196 | 
197 |       %Shell{command: command} ->
198 |         shell(command)
199 | 
200 |       %JsonEndpoint{url: url, headers: headers} ->
201 |         response = Tesla.get!(url, headers: headers)
202 |         {:ok, Jason.decode!(response.body)}
203 | 
204 |       %JsonFile{location: location, compressed: compressed} ->
205 |         json_file(location, compressed)
206 | 
207 |       _ ->
208 |         {:error, :unkown_fetch_type}
209 |     end
210 |   end
211 | 
212 |   defp transform(module, result, config) do
213 |     fields = module.__schema__(:fields)
214 | 
215 |     result =
216 |       if function_exported?(module, :transform, 1) do
217 |         module.transform(result)
218 |       else
219 |         result
220 |       end
221 | 
222 |     result =
223 |       if function_exported?(module, :transform, 2) do
224 |         module.transform(result, config)
225 |       else
226 |         result
227 |       end
228 | 
229 |     documents = Enum.map(result, fn item -> Ecto.Changeset.cast(struct(module), item, fields) end)
230 | 
231 |     valid =
232 |       documents
233 |       |> Enum.filter(fn document -> document.valid? end)
234 |       |> Enum.map(fn document -> document.changes end)
235 | 
236 |     invalid = Enum.filter(documents, fn document -> not document.valid? end)
237 | 
238 |     {:ok, valid, invalid}
239 |   end
240 | 
241 |   defp store(_module, documents, log) do
242 |     documents
243 |     |> Enum.map(fn document -> [body: document, importer_log_id: log.id] end)
244 |     |> bulk_upsert_documents()
245 | 
246 |     {:ok}
247 |   end
248 | 
249 |   def bulk_upsert_documents(documents) do
250 |     {current_batch, next_batch} = Enum.split(documents, @max_records_per_batch)
251 | 
252 |     Repo.insert_all(Document, current_batch,
253 |       on_conflict: {:replace, [:body]},
254 |       conflict_target: :id
255 |     )
256 | 
257 |     case next_batch do
258 |       [] -> {:ok}
259 |       _ -> bulk_upsert_documents(next_batch)
260 |     end
261 |   end
262 | 
263 |   def config(id) do
264 |     Repo.get(Memex.Schema.ImporterConfig, id).config
265 |   end
266 | end
267 | 


--------------------------------------------------------------------------------
/memex/lib/memex/importers/apple_message.ex:
--------------------------------------------------------------------------------
  1 | defmodule Memex.Importers.AppleMessages do
  2 |   @moduledoc false
  3 |   use Ecto.Schema
  4 | 
  5 |   alias Memex.Importer
  6 | 
  7 |   @primary_key false
  8 |   schema "document" do
  9 |     field :provider, :string
 10 |     field :verb, :string
 11 |     field :id, :string
 12 |     field :date_month, :string
 13 |     field :timestamp_utc, :string
 14 |     field :timestamp_unix, :integer
 15 |     field :message_direction, :string
 16 |     field :message_service, :string
 17 |     field :message_text, :string
 18 |     field :person_name, :string
 19 |     field :person_id, :string
 20 |   end
 21 | 
 22 |   def provider, do: "iMessage"
 23 | 
 24 |   def default_config do
 25 |     %{
 26 |       "location" => "#{System.user_home!()}/Library/Messages/chat.db",
 27 |       "contacts_db" =>
 28 |         "#{System.user_home!()}/Library/Application Support/AddressBook/Sources/AAE8D6A5-FEED-47BB-82CF-1A51C6789400/AddressBook-v22.abcddb",
 29 |       "schedule" => :watcher
 30 |     }
 31 |   end
 32 | 
 33 |   def fetch(config) do
 34 |     date_correction = "/ 1000000000 + 978307200"
 35 | 
 36 |     %Importer.Sqlite{
 37 |       location: config["location"],
 38 |       connection_options: [
 39 |         journal_mode: :wal
 40 |       ],
 41 |       setup: [
 42 |         """
 43 |         ATTACH DATABASE '#{config["contacts_db"]}' as contacts;
 44 |         """
 45 |       ],
 46 |       query: """
 47 |       WITH contact AS (
 48 |         SELECT
 49 |           record.ZUNIQUEID as id,
 50 |           record.ZFIRSTNAME AS first_name,
 51 |           record.ZLASTNAME AS last_name,
 52 |           REPLACE(phone.ZFULLNUMBER, ' ', '') AS phone_number,
 53 |           email.ZADDRESSNORMALIZED AS email_address
 54 |         FROM
 55 |           contacts.ZABCDRECORD AS record
 56 |         LEFT JOIN contacts.ZABCDPHONENUMBER AS phone ON phone.ZOWNER = record.Z_PK
 57 |         LEFT JOIN contacts.ZABCDEMAILADDRESS AS email ON email.ZOWNER = record.Z_PK
 58 |       )
 59 |       SELECT
 60 |         json_object(
 61 |           'provider', 'iMessage',
 62 |           'verb', 'messaged',
 63 |           'id', 'imessage-' || message.guid,
 64 |           'date_month', strftime('%Y-%m', message.date #{date_correction}, 'unixepoch'),
 65 |           'timestamp_utc', datetime(message.date #{date_correction}, 'unixepoch'),
 66 |           'timestamp_unix', CAST(strftime('%s', datetime(message.date #{date_correction}, 'unixepoch')) as INT),
 67 |           'message_direction', (CASE WHEN message.is_from_me THEN 'sent' ELSE 'received' END),
 68 |           'message_service', chat.service_name,
 69 |           'message_text', message.text,
 70 |           'person_name', CASE WHEN contact.first_name IS NULL AND contact.last_name IS NULL THEN chat.chat_identifier ELSE contact.first_name || ' ' || contact.last_name END,
 71 |           'person_id', contact.id
 72 |         ) AS json
 73 |       FROM
 74 |         main.chat chat
 75 |         JOIN chat_message_join ON chat.ROWID = chat_message_join.chat_id
 76 |         JOIN message ON chat_message_join.message_id = message.ROWID
 77 |         LEFT JOIN contact ON contact.phone_number = chat.chat_identifier OR contact.email_address = chat.chat_identifier
 78 |       WHERE message.text IS NOT NULL
 79 |       GROUP BY message.guid
 80 |       """
 81 |     }
 82 |   end
 83 | 
 84 |   defmodule TimeLineItem do
 85 |     @moduledoc false
 86 |     use Surface.Component
 87 | 
 88 |     prop item, :map, required: true
 89 | 
 90 |     def render(assigns) do
 91 |       ~F"""
 92 |       <div class="text-xs text-gray-400 dark:text-gray-500 truncate">
 93 |         {case @item["message_direction"] do
 94 |           "sent" -> "Sent to "
 95 |           "received" -> "Received from "
 96 |         end}
 97 |         {raw(@item["_formatted"]["person_name"])}
 98 |       </div>
 99 |       {raw(@item["_formatted"]["message_text"])}
100 |       """
101 |     end
102 |   end
103 | end
104 | 


--------------------------------------------------------------------------------
/memex/lib/memex/importers/apple_photos.ex:
--------------------------------------------------------------------------------
  1 | defmodule Memex.Importers.ApplePhotos do
  2 |   @moduledoc false
  3 |   use Ecto.Schema
  4 | 
  5 |   alias Memex.Importer
  6 | 
  7 |   @primary_key false
  8 |   schema "document" do
  9 |     field(:provider, :string)
 10 |     field(:verb, :string)
 11 |     field(:id, :string)
 12 |     field(:date_month, :string)
 13 |     field(:timestamp_utc, :string)
 14 |     field(:timestamp_unix, :integer)
 15 |     field(:photo_file_path, :string)
 16 |     field(:photo_file_name, :string)
 17 |     field(:photo_kind, :string)
 18 |     field(:photo_labels, {:array, :string})
 19 |     field(:place_name, {:array, :string})
 20 |     field(:person_id, {:array, :string})
 21 |     field(:person_name, {:array, :string})
 22 |     field(:location_latitude, :float)
 23 |     field(:location_longitude, :float)
 24 |     field(:device_name, :string)
 25 |   end
 26 | 
 27 |   def provider, do: "Photos"
 28 | 
 29 |   def default_config do
 30 |     %{
 31 |       "location" => "#{System.user_home!()}/Pictures/Photos\ Library.photoslibrary",
 32 |       "schedule" => :watcher
 33 |     }
 34 |   end
 35 | 
 36 |   def fetch(config) do
 37 |     %Importer.Sqlite{
 38 |       location: "#{config["location"]}/database/Photos.sqlite",
 39 |       connection_options: [
 40 |         journal_mode: :wal
 41 |       ],
 42 |       setup: [
 43 |         """
 44 |         -- Machine learning metadata information from psi.sqlite
 45 |         ATTACH DATABASE '#{config["location"]}/database/search/psi.sqlite' as psi;
 46 |         """
 47 |       ],
 48 |       query: """
 49 |       -- The UUID is split into two integers (uuid_0, uuid_1) in 'psi' and needs to be converted manually.
 50 |       WITH metadata AS (
 51 |           SELECT
 52 |               substr(printf('%p', assets.uuid_0), 15, 2)
 53 |                   || substr(printf('%p', assets.uuid_0), 13, 2)
 54 |                   || substr(printf('%p', assets.uuid_0), 11, 2)
 55 |                   || substr(printf('%p', assets.uuid_0), 9, 2)
 56 |                   || '-'
 57 |                   || substr(printf('%p', assets.uuid_0), 7, 2)
 58 |                   || substr(printf('%p', assets.uuid_0), 5, 2)
 59 |                   || '-'
 60 |                   || substr(printf('%p', assets.uuid_0), 3, 2)
 61 |                   || substr(printf('%p', assets.uuid_0), 1, 2)
 62 |                   || '-'
 63 |                   || substr(printf('%p', assets.uuid_1), 15, 2)
 64 |                   || substr(printf('%p', assets.uuid_1), 13, 2)
 65 |                   || '-'
 66 |                   || substr(printf('%p', assets.uuid_1), 11, 2)
 67 |                   || substr(printf('%p', assets.uuid_1), 9, 2)
 68 |                   || substr(printf('%p', assets.uuid_1), 7, 2)
 69 |                   || substr(printf('%p', assets.uuid_1), 5, 2)
 70 |                   || substr(printf('%p', assets.uuid_1), 3, 2)
 71 |                   || substr(printf('%p', assets.uuid_1), 1, 2)
 72 |               as uuid,
 73 |               assets.uuid_0,
 74 |               assets.uuid_1,
 75 |               json_group_array(substr(groups.content_string,1,instr(groups.content_string,char(0)))) as labels,
 76 |               json_group_array(substr(places.content_string,1,instr(places.content_string,char(0)))) as place_name
 77 |           FROM
 78 |               psi.assets
 79 |               JOIN psi.ga ON assets.rowid = ga.assetid
 80 |               JOIN psi.groups ON ga.groupid = groups.rowid
 81 |                   AND groups.category NOT IN (
 82 |                       2058, -- file name
 83 |                       2037, -- empty string
 84 |                       2047, -- empty string
 85 |                       2021 -- people, this is imported via ZPERSON
 86 |                   )
 87 |               LEFT JOIN groups people ON ga.groupid = people.rowid
 88 |             AND people.category=2021
 89 |               LEFT JOIN groups places ON ga.groupid = places.rowid
 90 |             AND places.category=5
 91 |           GROUP BY
 92 |               assets.rowid
 93 |       )
 94 |       SELECT
 95 |           json_object(
 96 |               'provider', 'Photos',
 97 |               'verb', 'photographed',
 98 |               'id', 'photos-' || ZASSET.ZUUID,
 99 |               'uuid', metadata.uuid,
100 |               'date_month', strftime('%Y-%m', ZASSET.ZDATECREATED + strftime('%s', '2001-01-01'), 'unixepoch'),
101 |               'timestamp_utc', datetime(ZASSET.ZDATECREATED + strftime('%s', '2001-01-01'), 'unixepoch'),
102 |               'timestamp_unix', CAST(ZASSET.ZDATECREATED + strftime('%s', '2001-01-01') AS INT),
103 |               'timezone_name', ZADDITIONALASSETATTRIBUTES.ZTIMEZONENAME,
104 |               'photo_file_path', ZASSET.ZDIRECTORY || '/' || ZASSET.ZFILENAME,
105 |               'photo_file_name', ZADDITIONALASSETATTRIBUTES.ZORIGINALFILENAME,
106 |               'photo_kind', CASE ZASSET.ZKIND WHEN 0 THEN 'photo' ELSE 'movie' END,
107 |               'photo_labels', json_extract(metadata.labels, '$'),
108 |               'place_name', json_extract(metadata.place_name, '$'),
109 |               'person_id', json_group_array(ZPERSON.ZPERSONURI),
110 |               'person_name', json_group_array(ZPERSON.ZFULLNAME),
111 |               'location_latitude', CASE WHEN ZASSET.ZLATITUDE == -180.0 AND ZASSET.ZLONGITUDE == -180.0 THEN NULL ELSE ZASSET.ZLATITUDE END,
112 |               'location_longitude', CASE WHEN ZASSET.ZLATITUDE == -180.0 AND ZASSET.ZLONGITUDE == -180.0 THEN NULL ELSE ZASSET.ZLONGITUDE END,
113 |               'device_name', ZEXTENDEDATTRIBUTES.ZCAMERAMAKE || ' ' || ZEXTENDEDATTRIBUTES.ZCAMERAMODEL
114 |           ) AS json
115 |       FROM ZASSET
116 |       LEFT JOIN ZADDITIONALASSETATTRIBUTES ON ZASSET.ZADDITIONALATTRIBUTES = ZADDITIONALASSETATTRIBUTES.Z_PK
117 |       LEFT JOIN ZEXTENDEDATTRIBUTES ON ZEXTENDEDATTRIBUTES.ZASSET = ZASSET.Z_PK
118 |       LEFT JOIN ZDETECTEDFACE ON ZDETECTEDFACE.ZASSETFORFACE = ZASSET.Z_PK
119 |       LEFT JOIN ZPERSON ON ZDETECTEDFACE.ZPERSONFORFACE = ZPERSON.Z_PK
120 |       LEFT JOIN metadata ON ZASSET.ZUUID=metadata.uuid
121 |       GROUP BY ZASSET.Z_PK
122 |       ORDER BY ZASSET.Z_PK DESC
123 |       LIMIT 200
124 |       """
125 |     }
126 |   end
127 | 
128 |   def transform(result, _config) do
129 |     Enum.map(result, fn item ->
130 |       %{
131 |         item
132 |         | "place_name" => Enum.reject(item["place_name"] || [], fn x -> x in ["", nil] end),
133 |           "photo_labels" => Enum.reject(item["photo_labels"] || [], fn x -> x in ["", nil] end),
134 |           "person_id" => Enum.reject(item["person_id"] || [], fn x -> x in ["", nil] end),
135 |           "person_name" => Enum.reject(item["person_name"] || [], fn x -> x in ["", nil] end)
136 |       }
137 |     end)
138 |   end
139 | 
140 |   defmodule TimeLineItem do
141 |     @moduledoc false
142 |     use Surface.Component
143 | 
144 |     alias MemexWeb.Router.Helpers, as: Routes
145 | 
146 |     prop item, :map, required: true
147 | 
148 |     def render(assigns) do
149 |       ~F"""
150 |       <img
151 |         class="object-cover float-left h-20 w-20 -m-4 rounded-l mr-4"
152 |         width="60"
153 |         height="60"
154 |         src={Routes.photo_path(MemexWeb.Endpoint, :image, @item["photo_file_path"])}
155 |       />
156 |       <p class="text-xs truncate">{raw(Enum.join(@item["_formatted"]["photo_labels"], ", "))}</p>
157 |       <p class="text-xs text-gray-400 dark:text-gray-500">
158 |         {raw(@item["_formatted"]["device_name"])}
159 |       </p>
160 |       """
161 |     end
162 |   end
163 | end
164 | 
165 | #!/usr/bin/env bash
166 | # Imports photos and videos from Apple Photos
167 | # including labels.
168 | 
169 | # PHOTOS_DB_PATH=${PHOTOS_DB_PATH:=/Users/$(whoami)/Pictures/Photos\ Library.photoslibrary/database/Photos.sqlite}
170 | # PSI_DB_PATH=${PSI_DB_PATH:=/Users/$(whoami)/Pictures/Photos\ Library.photoslibrary/database/search/psi.sqlite}
171 | 
172 | # Categories
173 | # 1 to 12: various parts of the reverse geolocation data (1 is areas of interest and 12 is country)
174 | # 1: area of interest
175 | # 2: street
176 | # 3: appears to be additional city-level/neighborhood info but not sure how this maps into other place data < city
177 | # 5: additional city-level info < city
178 | # 6: city
179 | # 7: county? > city
180 | # 9: sub-administrative area
181 | # 10: state/administrative area name
182 | # 11: state/administrative area abbreviation
183 | # 12: country
184 | # 1014: creation month
185 | # 1015: creation year
186 | # 2016: keyword
187 | # 2017: title
188 | # 2018: description
189 | # 2021: person in image
190 | # 2024: label from ML process
191 | # 2027: meal (e.g. dining, lunch)
192 | # 2029: holiday?
193 | # 2030: season
194 | # 2037: Group of people in image
195 | # 2044: videos
196 | # 2046: live photos
197 | # 2049: time-lapse
198 | # 2053: portrait
199 | # 2054: selfies
200 | # 2055: favorites
201 | 
202 | # sqlite3 -readonly "$PHOTOS_DB_PATH" "
203 | 


--------------------------------------------------------------------------------
/memex/lib/memex/importers/apple_podcasts.ex:
--------------------------------------------------------------------------------
  1 | defmodule Memex.Importers.ApplePodcasts do
  2 |   @moduledoc false
  3 |   use Ecto.Schema
  4 | 
  5 |   alias Memex.Importer
  6 | 
  7 |   @primary_key false
  8 |   schema "document" do
  9 |     field(:provider, :string)
 10 |     field(:verb, :string)
 11 |     field(:id, :string)
 12 |     field(:date_month, :string)
 13 |     field(:timestamp_utc, :string)
 14 |     field(:timestamp_unix, :integer)
 15 |     field(:episode_published_at_utc, :string)
 16 |     field(:episode_title, :string)
 17 |     field(:episode_description, :string)
 18 |     field(:episode_id, :string)
 19 |     field(:episode_author, :string)
 20 |     field(:episode_webpage_url, :string)
 21 |     field(:episode_playback_url, :string)
 22 |     field(:podcast_author, :string)
 23 |     field(:podcast_title, :string)
 24 |     field(:podcast_description_html, :string)
 25 |     field(:podcast_id, :string)
 26 |     field(:podcast_image_url, :string)
 27 |     field(:podcast_category, :string)
 28 |     field(:podcast_webpage_url, :string)
 29 |   end
 30 | 
 31 |   def provider, do: "Podcasts"
 32 | 
 33 |   def default_config do
 34 |     %{
 35 |       "location" =>
 36 |         "#{System.user_home!()}/Library/Group Containers/243LU875E5.groups.com.apple.podcasts/Documents/MTLibrary.sqlite",
 37 |       "schedule" => :watcher
 38 |     }
 39 |   end
 40 | 
 41 |   def fetch(config) do
 42 |     date_correction = "+ 978307200"
 43 | 
 44 |     %Importer.Sqlite{
 45 |       location: "#{config["location"]}",
 46 |       connection_options: [
 47 |         journal_mode: :wal
 48 |       ],
 49 |       setup: [],
 50 |       query: """
 51 |       SELECT
 52 |         json_object(
 53 |           'provider', 'Podcasts',
 54 |           'verb', 'listened',
 55 |           'id', 'podcast-' || episode.ZSTORETRACKID,
 56 |           'date_month', strftime('%Y-%m', episode.ZLASTDATEPLAYED #{date_correction}, 'unixepoch'),
 57 |           'timestamp_utc', datetime(episode.ZLASTDATEPLAYED #{date_correction}, 'unixepoch'),
 58 |           'timestamp_unix', CAST(strftime('%s', datetime(episode.ZLASTDATEPLAYED #{date_correction}, 'unixepoch')) as INT),
 59 |           'episode_title', episode.ZTITLE,
 60 |           'episode_description_html', episode.ZITEMDESCRIPTION,
 61 |           'episode_id', CAST(episode.ZSTORETRACKID as text),
 62 |           'episode_author', episode.ZAUTHOR,
 63 |           'episode_webpage_url', episode.ZWEBPAGEURL,
 64 |           'episode_playback_url', episode.ZENCLOSUREURL,
 65 |           'episode_published_at_utc', datetime(episode.ZPUBDATE #{date_correction}, 'unixepoch'),
 66 |           'podcast_author', pod.ZAUTHOR,
 67 |           'podcast_title', pod.ZTITLE,
 68 |           'podcast_description', pod.ZITEMDESCRIPTION,
 69 |           'podcast_id', CAST(pod.ZSTORECOLLECTIONID as text),
 70 |           'podcast_image_url', pod.ZIMAGEURL,
 71 |           'podcast_category', pod.ZCATEGORY,
 72 |           'podcast_webpage_url', pod.ZWEBPAGEURL
 73 |         ) AS json
 74 |       FROM ZMTEPISODE AS episode
 75 |       JOIN ZMTPODCAST AS pod ON pod.Z_PK = episode.ZPODCAST
 76 |       WHERE episode.ZLASTDATEPLAYED IS NOT NULL
 77 |       ORDER BY episode.ZLASTDATEPLAYED DESC
 78 |       LIMIT 1000
 79 |       """
 80 |     }
 81 |   end
 82 | 
 83 |   defmodule TimeLineItem do
 84 |     @moduledoc false
 85 |     use Surface.Component
 86 | 
 87 |     alias MemexWeb.Router.Helpers, as: Routes
 88 | 
 89 |     prop item, :map, required: true
 90 | 
 91 |     def render(assigns) do
 92 |       ~F"""
 93 |       <img
 94 |         class="object-cover float-left h-20 w-20 -m-4 rounded-l mr-4"
 95 |         width="60"
 96 |         height="60"
 97 |         src={Routes.photo_path(MemexWeb.Endpoint, :https_proxy, url: @item["podcast_image_url"])}
 98 |       />
 99 |       <p class="truncate">{raw(@item["_formatted"]["episode_title"])}</p>
100 |       <div class="text-xs text-gray-400 dark:text-gray-500 truncate">
101 |         <a href={@item["episode_webpage_url"]} target="_blank" class="underline">
102 |           {@item["_formatted"]["podcast_title"]} by {@item["_formatted"]["episode_author"]}</a>
103 |       </div>
104 |       """
105 |     end
106 |   end
107 | end
108 | 


--------------------------------------------------------------------------------
/memex/lib/memex/importers/arc.ex:
--------------------------------------------------------------------------------
  1 | defmodule Memex.Importers.Arc do
  2 |   @moduledoc false
  3 |   use Ecto.Schema
  4 | 
  5 |   alias Memex.Importer
  6 | 
  7 |   @primary_key false
  8 |   schema "document" do
  9 |     field(:provider, :string)
 10 |     field(:verb, :string)
 11 |     field(:id, :string)
 12 |     field(:id_next, :string)
 13 |     field(:id_previous, :string)
 14 |     field(:date_month, :string)
 15 |     field(:timestamp_utc, :string)
 16 |     field(:timestamp_unix, :integer)
 17 |     field(:timestamp_start_unix, :integer)
 18 |     field(:timestamp_start_utc, :string)
 19 |     field(:place_name, :string)
 20 |     field(:place_address, :string)
 21 |     field(:place_latitude, :float)
 22 |     field(:place_longitude, :float)
 23 |     field(:place_altitude, :float)
 24 |     field(:place_foursquare_venue_id, :string)
 25 |     field(:place_foursquare_category_id, :string)
 26 |     field(:activity_type, :string)
 27 |     field(:activity_step_count, :integer)
 28 |     field(:activity_floors_ascended, :integer)
 29 |     field(:activity_floors_descended, :integer)
 30 |     field(:activity_heart_rate_average, :float)
 31 |     field(:activity_heart_rate_max, :float)
 32 |     field(:activity_active_energy_burned, :float)
 33 |   end
 34 | 
 35 |   def provider, do: "Arc"
 36 | 
 37 |   def default_config do
 38 |     %{
 39 |       "location" => "#{System.user_home!()}/Library/Mobile\ Documents/iCloud~com~bigpaua~LearnerCoacher/",
 40 |       "schedule" => :watcher
 41 |     }
 42 |   end
 43 | 
 44 |   def fetch(config) do
 45 |     location =
 46 |       "#{config["location"]}/Documents/Export/JSON/Monthly/*.json.gz"
 47 |       |> Path.wildcard()
 48 |       |> Enum.sort(:desc)
 49 |       |> Enum.at(0)
 50 | 
 51 |     %Importer.JsonFile{
 52 |       location: location,
 53 |       compressed: true
 54 |     }
 55 |   end
 56 | 
 57 |   def transform(result, config) do
 58 |     result["timelineItems"]
 59 |     |> Enum.map(&parse_item(&1, config))
 60 |     |> Enum.filter(fn item -> item != %{} end)
 61 |   end
 62 | 
 63 |   defp parse_common(item) do
 64 |     {:ok, start_date, _} = DateTime.from_iso8601(item["endDate"])
 65 |     {:ok, end_date, _} = DateTime.from_iso8601(item["startDate"])
 66 | 
 67 |     %{
 68 |       provider: "Arc",
 69 |       id: "arc-#{item["itemId"]}",
 70 |       id_next: "arc-#{item["nextItemId"]}",
 71 |       id_previous: "arc-#{item["previousItemId"]}",
 72 |       date_month: Calendar.strftime(end_date, "%Y-%m"),
 73 |       timestamp_unix: DateTime.to_unix(end_date),
 74 |       timestamp_utc: item["endDate"],
 75 |       timestamp_start_unix: DateTime.to_unix(start_date),
 76 |       timestamp_start_utc: item["startDate"],
 77 |       activity_step_count: item["stepCount"],
 78 |       activity_floors_ascended: item["floorsAscended"],
 79 |       activity_floors_descended: item["floorsDescended"],
 80 |       activity_heart_rate_average: item["averageHeartRate"],
 81 |       activity_heart_rate_max: item["maxHeartRate"],
 82 |       activity_active_energy_burned: item["activeEnergyBurned"]
 83 |     }
 84 |   end
 85 | 
 86 |   defp parse_item(%{"isVisit" => true, "place" => place} = item, _config) do
 87 |     item
 88 |     |> parse_common()
 89 |     |> Map.merge(%{
 90 |       verb: "visited",
 91 |       place_altitude: item["altitude"]
 92 |     })
 93 |     |> Map.merge(parse_place(place))
 94 |   end
 95 | 
 96 |   defp parse_item(%{"isVisit" => true, "placeId" => placeId} = item, config) do
 97 |     path = "#{config["location"]}Documents/Backups/Place/#{String.at(placeId, 0)}/#{placeId}.json"
 98 | 
 99 |     {:ok, place} =
100 |       path
101 |       |> Path.wildcard()
102 |       |> Memex.Connector.json_file(false)
103 | 
104 |     item
105 |     |> parse_common()
106 |     |> Map.merge(%{
107 |       verb: "visited",
108 |       place_altitude: item["altitude"]
109 |     })
110 |     |> Map.merge(parse_place(place))
111 |   end
112 | 
113 |   defp parse_item(%{"isVisit" => false} = item, _config) do
114 |     item
115 |     |> parse_common()
116 |     |> Map.merge(%{
117 |       verb: "moved",
118 |       activity_type: item["activityType"]
119 |     })
120 |   end
121 | 
122 |   defp parse_item(_item, _config), do: %{}
123 | 
124 |   defp parse_place(place) do
125 |     %{
126 |       place_name: place["name"],
127 |       place_address: place["address"],
128 |       place_latitude: place["center"]["latitude"] || nil,
129 |       place_longitude: place["center"]["longitude"] || nil,
130 |       place_foursquare_venue_id: place["foursquareVenueId"],
131 |       place_foursquare_category_id: place["foursquareCategoryId"]
132 |     }
133 |   end
134 | 
135 |   defmodule TimelineItem do
136 |     @moduledoc false
137 |     use Surface.Component
138 | 
139 |     prop(item, :map)
140 | 
141 |     def render(assigns) do
142 |       ~F"""
143 |       <div :if={@item["verb"] === "visited" || @item["verb"] === "visit"} class="flex-grow truncate">
144 |         {raw(@item["_formatted"]["place_name"] || @item["_formatted"]["place_address"])}
145 |         <p class="text-xs text-gray-400 dark:text-gray-500">
146 |           Spent {MemexWeb.TimelineView.human_time_between(@item["timestamp_unix"], @item["timestamp_start_unix"])}.
147 |           <span :if={@item["place_name"]}>{raw(@item["_formatted"]["place_name"])}</span>
148 |           <span :if={@item["place_address"]}>{raw(@item["_formatted"]["place_address"])}</span>
149 |         </p>
150 |       </div>
151 |       <div :if={@item["verb"] === "moved"} class="flex-grow truncate">
152 |         Finished {raw(@item["_formatted"]["activity_type"])}
153 |         <p class="text-xs text-gray-400 dark:text-gray-500">
154 |           {MemexWeb.TimelineView.human_time_between(@item["timestamp_unix"], @item["timestamp_start_unix"])}.
155 |         </p>
156 |       </div>
157 |       """
158 |     end
159 |   end
160 | end
161 | 


--------------------------------------------------------------------------------
/memex/lib/memex/importers/fish_shell.ex:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Importers.FishShell do
 2 |   @moduledoc false
 3 |   use Ecto.Schema
 4 | 
 5 |   alias Memex.Importer
 6 | 
 7 |   @primary_key false
 8 |   schema "document" do
 9 |     field :provider, :string
10 |     field :verb, :string
11 |     field :id, :string
12 |     field :date_month, :string
13 |     field :timestamp_utc, :string
14 |     field :timestamp_unix, :integer
15 |     field :command, :string
16 |   end
17 | 
18 |   def provider, do: "terminal"
19 | 
20 |   def default_config do
21 |     %{
22 |       "schedule" => :watcher
23 |     }
24 |   end
25 | 
26 |   def fetch(_config) do
27 |     %Importer.Command{
28 |       command: "fish",
29 |       arguments: ["--command=history --show-time='<-SNIP->%s-;-%F %T-;-%Y-%m-;-'"]
30 |     }
31 |   end
32 | 
33 |   def transform(result) do
34 |     result
35 |     |> String.split("<-SNIP->")
36 |     |> Enum.map(&String.split(&1, "-;-"))
37 |     |> Enum.filter(&(&1 != [""]))
38 |     |> Enum.map(fn [timestamp_unix, timestamp_utc, date_month, command] ->
39 |       %{
40 |         id: "terminal-#{Base.encode16(:crypto.hash(:sha256, timestamp_unix <> command))}",
41 |         verb: "commanded",
42 |         provider: "terminal",
43 |         timestamp_unix: timestamp_unix,
44 |         timestamp_utc: timestamp_utc,
45 |         date_month: date_month,
46 |         command: String.trim(command)
47 |       }
48 |     end)
49 |   end
50 | 
51 |   defmodule TimeLineItem do
52 |     @moduledoc false
53 |     use Surface.Component
54 | 
55 |     alias Phoenix.LiveView.JS
56 | 
57 |     prop item, :map, required: true
58 | 
59 |     def render(assigns) do
60 |       ~F"""
61 |       <pre phx-click={open()} class="text-sm overflow-scroll"><code>{raw(String.replace(String.trim(@item["_formatted"]["command"]), "\n", "<br />"))}</code></pre>
62 |       """
63 |     end
64 | 
65 |     def open do
66 |       JS.dispatch("memex:clipcopy")
67 |     end
68 |   end
69 | end
70 | 


--------------------------------------------------------------------------------
/memex/lib/memex/importers/github.ex:
--------------------------------------------------------------------------------
  1 | defmodule Memex.Importers.Github do
  2 |   @moduledoc """
  3 |   [Github Event Documentation](https://docs.github.com/en/developers/webhooks-and-events/events/github-event-types)
  4 |   """
  5 |   use Ecto.Schema
  6 | 
  7 |   alias Memex.Importer
  8 | 
  9 |   @provider "GitHub"
 10 |   @defaults %{"provider" => @provider}
 11 |   @ignore_item %{}
 12 | 
 13 |   @primary_key false
 14 |   schema "document" do
 15 |     field :provider, :string
 16 |     field :verb, :string
 17 |     field :id, :string
 18 |     field :date_month, :string
 19 |     field :timestamp_utc, :string
 20 |     field :timestamp_unix, :integer
 21 |     field :repo_name, :string
 22 |     field :issue_title, :string
 23 |     field :issue_body, :string
 24 |     field :issue_url, :string
 25 |     field :review_body, :string
 26 |     field :review_state, :string
 27 |     field :review_url, :string
 28 |     field :comment_body, :string
 29 |     field :comment_url, :string
 30 |     field :github_user_name, :string
 31 |     field :github_user_avatar, :string
 32 |   end
 33 | 
 34 |   def provider, do: @provider
 35 | 
 36 |   def default_config do
 37 |     %{
 38 |       "user_name" => "",
 39 |       "page" => 1,
 40 |       "access_token" => "",
 41 |       "ignore_repos" => ["adri/notes"]
 42 |     }
 43 |   end
 44 | 
 45 |   def required_config do
 46 |     ["user_name", "access_token"]
 47 |   end
 48 | 
 49 |   def fetch(config) do
 50 |     %Importer.JsonEndpoint{
 51 |       url: "https://api.github.com/users/#{config["user_name"]}/events?page=#{config["page"]}",
 52 |       headers: [
 53 |         {"Accept", "application/vnd.github.v3+json"},
 54 |         {"User-Agent", "curl/7.64.1"},
 55 |         {"Authorization", "Basic #{Base.encode64("#{config["user_name"]}:#{config["access_token"]}")}"}
 56 |       ]
 57 |     }
 58 |   end
 59 | 
 60 |   def transform(result, config) do
 61 |     result
 62 |     |> Enum.filter(fn event -> not Enum.member?(config["ignore_repos"], event["repo"]["name"]) end)
 63 |     |> Enum.map(&parse_item(&1))
 64 |     |> Enum.filter(&match?(%{"verb" => _}, &1))
 65 |   end
 66 | 
 67 |   defp parse_item(item) do
 68 |     @defaults
 69 |     |> Map.merge(parse_common(item))
 70 |     |> Map.merge(parse_type(item))
 71 |     |> Map.merge(parse_comment(item))
 72 |     |> Map.merge(parse_review(item))
 73 |     |> Map.merge(parse_issue(item))
 74 |     |> Map.merge(parse_user(item))
 75 |   end
 76 | 
 77 |   defp parse_common(%{} = item) do
 78 |     {:ok, date, _} = DateTime.from_iso8601(item["created_at"])
 79 | 
 80 |     %{
 81 |       "id" => "#{@provider}_" <> item["id"],
 82 |       "date_month" => Calendar.strftime(date, "%Y-%m"),
 83 |       "timestamp_utc" => item["created_at"],
 84 |       "timestamp_unix" => DateTime.to_unix(date),
 85 |       "repo_name" => item["repo"]["name"]
 86 |       # fetch repo for other data?
 87 |       # 'repo_description',
 88 |       # 'repo_homepage',
 89 |       # 'repo_license',
 90 |       # 'repo_language',
 91 |       # 'repo_stars_count',
 92 |     }
 93 |   end
 94 | 
 95 |   defp parse_type(%{"type" => "WatchEvent"}), do: %{"verb" => "liked"}
 96 |   # ignore duplicate events, there is another PullRequestReviewCommentEvent
 97 |   defp parse_type(%{"payload" => %{"review" => %{"state" => "commented"}}}), do: @ignore_item
 98 |   defp parse_type(%{"type" => "PullRequestReviewEvent"}), do: %{"verb" => "reviewed"}
 99 |   defp parse_type(%{"type" => "PullRequestReviewCommentEvent"}), do: %{"verb" => "commented"}
100 |   defp parse_type(%{"type" => "IssueCommentEvent"}), do: %{"verb" => "commented"}
101 | 
102 |   defp parse_type(%{"type" => "PullRequestEvent"} = item) do
103 |     case item["payload"] do
104 |       %{"action" => "closed", "pull_request" => %{"merged" => true}} -> %{"verb" => "merged"}
105 |       %{"action" => "closed"} -> %{"verb" => "closed"}
106 |       %{"action" => "opened"} -> %{"verb" => "requested"}
107 |       %{"action" => "reopened"} -> %{"verb" => "requested"}
108 |       _ -> @ignore_item
109 |     end
110 |   end
111 | 
112 |   defp parse_type(_item) do
113 |     @ignore_item
114 |   end
115 | 
116 |   defp parse_issue(%{"payload" => %{"issue" => issue}}),
117 |     do: %{"issue_title" => issue["title"], "issue_body" => issue["body"], "issue_url" => issue["html_url"]}
118 | 
119 |   defp parse_issue(%{"payload" => %{"pull_request" => issue}}),
120 |     do: %{"issue_title" => issue["title"], "issue_body" => issue["body"], "issue_url" => issue["html_url"]}
121 | 
122 |   defp parse_issue(_item), do: %{}
123 | 
124 |   defp parse_review(%{"payload" => %{"review" => review}}),
125 |     do: %{"review_body" => review["body"], "review_state" => review["state"], "review_url" => review["html_url"]}
126 | 
127 |   defp parse_review(_item), do: %{}
128 | 
129 |   defp parse_comment(%{"payload" => %{"comment" => comment}}),
130 |     do: %{"comment_body" => comment["body"], "comment_url" => comment["html_url"]}
131 | 
132 |   defp parse_comment(_item), do: %{}
133 | 
134 |   defp parse_user(%{"payload" => %{"issue" => %{"user" => user}}}), do: map_user(user)
135 |   defp parse_user(%{"payload" => %{"pull_request" => %{"user" => user}}}), do: map_user(user)
136 |   defp parse_user(%{"payload" => %{"user" => user}}), do: map_user(user)
137 |   defp parse_user(_item), do: %{}
138 | 
139 |   defp map_user(user), do: %{"github_user_name" => user["login"], "github_user_avatar" => user["avatar_url"]}
140 | 
141 |   defmodule TimeLineItem do
142 |     @moduledoc false
143 |     use Surface.Component
144 | 
145 |     alias MemexWeb.Router.Helpers, as: Routes
146 | 
147 |     prop item, :map
148 | 
149 |     def render(assigns) do
150 |       ~F"""
151 |       <p class="text-xs text-gray-400 dark:text-gray-500">
152 |         {raw(@item["_formatted"]["verb"])} in {raw(@item["_formatted"]["repo_name"])}
153 |         <span
154 |           :if={@item["issue_url"] && not Enum.member?(["merged", "requested"], @item["verb"])}
155 |           class="text-xs text-gray-400 dark:text-gray-500"
156 |         >
157 |           <img
158 |             class="rounded-full float-left mr-2"
159 |             src={Routes.photo_path(MemexWeb.Endpoint, :https_proxy, url: @item["github_user_avatar"])}
160 |             width="16"
161 |             height="16"
162 |           />
163 |           <a href={@item["issue_url"]} target="_blank">{raw(@item["_formatted"]["issue_title"])}</a>
164 |         </span>
165 |       </p>
166 |       <p :if={@item["comment_body"]} class="mb-2">
167 |         {raw(@item["_formatted"]["comment_body"])}
168 |       </p>
169 |       <p :if={@item["review_body"]} class="mb-2">
170 |         <span :if={@item["review_state"] === "approved"}>✅</span>
171 | 
172 |         {raw(@item["_formatted"]["review_body"])}
173 |       </p>
174 |       <p :if={Enum.member?(["merged", "requested"], @item["verb"])} class="mb-2">
175 |         <a href={@item["issue_url"]} target="_blank">{raw(@item["_formatted"]["issue_title"])}</a>
176 |         <div class="text-sm text-gray-400 dark:text-gray-500">{raw(Earmark.as_html!(@item["_formatted"]["issue_body"], compact_output: true))}</div>
177 |       </p>
178 | 
179 |       <p :if={@item["repo_description"]} class="text-sm text-gray-400 dark:text-gray-400 truncate">
180 |         {raw(@item["_formatted"]["repo_description"])}
181 |       </p>
182 | 
183 |       <p :if={@item["repo_license"]} class="text-xs text-gray-400 dark:text-gray-500 truncate">
184 |         <span class="capitalize">{@item["repo_license"]}</span>,
185 |         {@item["repo_language"]}, {@item["repo_stars_count"]} stars
186 |       </p>
187 |       """
188 |     end
189 |   end
190 | 
191 |   defmodule HomepageItem do
192 |     @moduledoc false
193 |     use Surface.Component
194 | 
195 |     def render(assigns) do
196 |       ~F"""
197 |       test
198 |       """
199 |     end
200 |   end
201 | end
202 | 


--------------------------------------------------------------------------------
/memex/lib/memex/importers/money_money.ex:
--------------------------------------------------------------------------------
  1 | defmodule Memex.Importers.MoneyMoney do
  2 |   @moduledoc false
  3 |   use Ecto.Schema
  4 | 
  5 |   alias Memex.Importer
  6 | 
  7 |   @primary_key false
  8 |   schema "document" do
  9 |     field :provider, :string
 10 |     field :verb, :string
 11 |     field :id, :string
 12 |     field :date_month, :string
 13 |     field :timestamp_utc, :string
 14 |     field :timestamp_unix, :integer
 15 |     field :transaction_account_name, :string
 16 |     field :transaction_category, :string
 17 |     field :transaction_amount, :float
 18 |     field :transaction_currency, :string
 19 |     field :transaction_recipient, :string
 20 |     field :transaction_reference, :string
 21 |     field :transaction_purpose, :string
 22 |   end
 23 | 
 24 |   def provider, do: "MoneyMoney"
 25 | 
 26 |   def default_config do
 27 |     %{
 28 |       "location" => "#{System.user_home!()}/Library/Containers/com.moneymoney-app.retail/Data/Library/Application\ Support/MoneyMoney/Database/MoneyMoney.sqlite",
 29 |       "database_password" => "set_your_password_here",
 30 |       "bank_timezone" => "Europe/Amsterdam",
 31 |       "schedule" => :watcher
 32 |     }
 33 |   end
 34 | 
 35 |   def required_config do
 36 |     ["database_password"]
 37 |   end
 38 | 
 39 |   def fetch(config) do
 40 |     %Importer.Sqlite{
 41 |       location: config["location"],
 42 |       connection_options: [
 43 |         custom_pragmas: [
 44 |           key: config["database_password"],
 45 |           cipher_compatibility: 4
 46 |         ]
 47 |       ],
 48 |       setup: [],
 49 |       query: """
 50 |       SELECT
 51 |           json_object(
 52 |               'provider', 'MoneyMoney',
 53 |               'verb', 'transacted',
 54 |               'id', 'moneymoney-' || transactions.rowid,
 55 |               'date_month', strftime('%Y-%m', transactions.timestamp, 'unixepoch', 'utc'),
 56 |               'timestamp_utc', datetime(transactions.timestamp, 'unixepoch', 'utc'),
 57 |               'timestamp_unix', transactions.timestamp,
 58 |               'transaction_account_name', accounts.name,
 59 |               'transaction_category', categories.name,
 60 |               'transaction_amount', transactions.amount,
 61 |               'transaction_currency', transactions.currency,
 62 |               'transaction_recipient', transactions.name,
 63 |               'transaction_reference', transactions.eref,
 64 |               'transaction_purpose', transactions.unformatted_purpose
 65 |         ) AS json
 66 |       FROM transactions
 67 |       LEFT JOIN accounts ON transactions.local_account_key=accounts.rowid
 68 |       LEFT JOIN categories ON transactions.category_key=categories.rowid
 69 |       """
 70 |     }
 71 |   end
 72 | 
 73 |   def transform(results, config) do
 74 |     time_zone = config["bank_timezone"]
 75 | 
 76 |     results
 77 |     |> Enum.map(fn result -> parse_time(result, "transaction_purpose", time_zone) end)
 78 |     |> Enum.map(fn result -> parse_time(result, "transaction_reference", time_zone) end)
 79 |   end
 80 | 
 81 |   @german_datetime_regex ~r/(?<day>\d{2})-(?<month>\d{2})-(?<year>\d{4}) (?<hours>\d{2}):(?<minutes>\d{2})/
 82 |   defp parse_time(result, key, bank_timezone) do
 83 |     case Regex.named_captures(@german_datetime_regex, result[key]) do
 84 |       %{"day" => day, "month" => month, "year" => year, "hours" => hours, "minutes" => minutes} ->
 85 |         datetime =
 86 |           "#{year}-#{month}-#{day}T#{hours}:#{minutes}:00"
 87 |           |> NaiveDateTime.from_iso8601!()
 88 |           |> DateTime.from_naive!(bank_timezone)
 89 | 
 90 |         result
 91 |         |> Map.put("timestamp_unix", DateTime.to_unix(datetime))
 92 |         |> Map.put(
 93 |           "timestamp_utc",
 94 |           datetime |> DateTime.shift_zone!("Etc/UTC") |> DateTime.to_iso8601()
 95 |         )
 96 | 
 97 |       _ ->
 98 |         result
 99 |     end
100 |   end
101 | 
102 |   defmodule TimeLineItem do
103 |     @moduledoc false
104 |     use Surface.Component
105 | 
106 |     prop item, :map, required: true
107 | 
108 |     def render(assigns) do
109 |       ~F"""
110 |       <span class="font-mono">{MemexWeb.TimelineView.number_to_currency(
111 |           abs(@item["transaction_amount"]),
112 |           @item["transaction_currency"]
113 |         )}</span>
114 |       {if @item["transaction_amount"] < 0 do
115 |         "to "
116 |       else
117 |         "from "
118 |       end}
119 |       <span>{raw(@item["_formatted"]["transaction_recipient"])}</span>
120 |       <span
121 |         :if={@item["_formatted"]["transaction_category"]}
122 |         class="float-right rounded-full dark:bg-gray-700 p-2 text-xs"
123 |       >{raw(@item["_formatted"]["transaction_category"])}</span>
124 |       <div class="text-xs text-gray-400 dark:text-gray-500 truncate">
125 |         {raw(@item["_formatted"]["transaction_account_name"])} - {raw(@item["_formatted"]["transaction_purpose"])}
126 |       </div>
127 |       """
128 |     end
129 |   end
130 | end
131 | 


--------------------------------------------------------------------------------
/memex/lib/memex/importers/notes.ex:
--------------------------------------------------------------------------------
  1 | defmodule Memex.Importers.Notes do
  2 |   @moduledoc false
  3 |   use Ecto.Schema
  4 | 
  5 |   alias Memex.Importer
  6 | 
  7 |   @primary_key false
  8 |   schema "document" do
  9 |     field :provider, :string
 10 |     field :verb, :string
 11 |     field :id, :string
 12 |     field :date_month, :string
 13 |     field :timestamp_utc, :string
 14 |     field :timestamp_unix, :integer
 15 |     field :commit_sha, :string
 16 |     field :commit_diff, :string
 17 |   end
 18 | 
 19 |   def provider, do: "git-notes"
 20 | 
 21 |   def default_config do
 22 |     %{
 23 |       "schedule" => :watcher
 24 |     }
 25 |   end
 26 | 
 27 |   def required_config do
 28 |     ["path"]
 29 |   end
 30 | 
 31 |   def fetch(_config) do
 32 |     %Importer.Shell{
 33 |       command: """
 34 |       cd #{File.cwd!()}/../data/notes
 35 |       git fetch -q && git reset -q --hard origin/master && \
 36 |         git log -n6 --pretty=format:'<-SNIP->%H-;-%at-;-%B-;-' \
 37 |         --patch \
 38 |         --no-color \
 39 |         --no-ext-diff \
 40 |         --unified=0
 41 |       """
 42 |     }
 43 |   end
 44 | 
 45 |   def transform(result) do
 46 |     result
 47 |     |> String.split("<-SNIP->")
 48 |     |> Enum.map(&String.split(&1, "-;-"))
 49 |     |> Enum.filter(&(&1 != [""]))
 50 |     |> Enum.map(fn [sha1, timestamp_unix, _, diff] ->
 51 |       timestamp = DateTime.from_unix!(String.to_integer(timestamp_unix))
 52 | 
 53 |       %{
 54 |         id: "git-#{sha1}",
 55 |         verb: "committed",
 56 |         provider: provider(),
 57 |         timestamp_unix: String.to_integer(timestamp_unix),
 58 |         timestamp_utc: Calendar.strftime(timestamp, "%Y-%m-%d %H:%M:%S"),
 59 |         date_month: Calendar.strftime(timestamp, "%Y-%m"),
 60 |         commit_sha: sha1,
 61 |         commit_diff: String.replace(diff, "\\\n\\\\ No newline at end of file", "")
 62 |       }
 63 |     end)
 64 |   end
 65 | 
 66 |   defmodule TimeLineItem do
 67 |     @moduledoc false
 68 |     use Surface.Component
 69 | 
 70 |     prop item, :map, required: true
 71 | 
 72 |     def render(assigns) do
 73 |       ~F"""
 74 |       <div class="truncate" :if={Enum.empty?(parse_patch(@item["commit_diff"]))}>
 75 |         {Jason.encode!(@item, pretty: true)}
 76 |       </div>
 77 |       <div :for={patch <- parse_patch(@item["commit_diff"])}>
 78 |         <a
 79 |           href={"obsidian://open?#{URI.encode_query(%{"vault" => "Wiki_Synced", "file" => MemexWeb.TimelineView.nl2br(patch.from)})}"}
 80 |           target="_blank"
 81 |           class="text-xs text-gray-400 dark:text-gray-400"
 82 |         >
 83 |           {raw(MemexWeb.TimelineView.highlight_line_text(patch.from, @item["_formatted"]["commit_diff"]))}
 84 |         </a>
 85 |         <span :for={chunk <- patch.chunks}>
 86 |           <div
 87 |             :for={line <- chunk.lines}
 88 |             class={
 89 |               "text-sm break-normal",
 90 |               "line-through text-gray-600": MemexWeb.TimelineView.line_type(line) == "remove"
 91 |             }
 92 |           >
 93 |             <span>{raw(MemexWeb.TimelineView.highlight_line_text(line.text, @item["_formatted"]["commit_diff"]))}</span>
 94 |           </div>
 95 |         </span>
 96 |       </div>
 97 |       """
 98 |     end
 99 | 
100 |     defp parse_patch(patch) do
101 |       case GitDiff.parse_patch(String.trim(patch)) do
102 |         {:ok, parsed_diff} -> parsed_diff
103 |         {:error, error} ->
104 |           error |> IO.inspect()
105 |           []
106 |       end
107 |     end
108 |   end
109 | end
110 | 


--------------------------------------------------------------------------------
/memex/lib/memex/importers/safari.ex:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Importers.Safari do
 2 |   @moduledoc false
 3 |   use Ecto.Schema
 4 | 
 5 |   alias Memex.Importer
 6 | 
 7 |   @primary_key false
 8 |   schema "document" do
 9 |     field :provider, :string
10 |     field :verb, :string
11 |     field :id, :string
12 |     field :date_month, :string
13 |     field :timestamp_utc, :string
14 |     field :timestamp_unix, :integer
15 |     field :website_title, :string
16 |     field :website_url, :string
17 |     field :device_name, :string
18 |   end
19 | 
20 |   def provider, do: "Safari"
21 | 
22 |   def default_config do
23 |     %{
24 |       "location" => "#{System.user_home!()}/Library/Safari/History.db",
25 |       "schedule" => :watcher
26 |     }
27 |   end
28 | 
29 |   def fetch(config) do
30 |     %Importer.Sqlite{
31 |       location: config["location"],
32 |       connection_options: [
33 |         journal_mode: :wal
34 |       ],
35 |       setup: [],
36 |       query: """
37 |       SELECT
38 |           json_object(
39 |               'provider', 'Safari',
40 |               'verb', 'browsed',
41 |               'id', 'safari-' || history_visits.id,
42 |               'date_month', strftime('%Y-%m', history_visits.visit_time + 978314400, 'unixepoch', 'utc'),
43 |               'timestamp_utc', datetime(history_visits.visit_time + 978314400, 'unixepoch', 'utc'),
44 |               'timestamp_unix', CAST(strftime('%s', datetime(history_visits.visit_time + 978314400, 'unixepoch', 'utc')) AS INT),
45 |               'website_title', history_visits.title,
46 |               'website_url', history_items.URL,
47 |               'device_name', (CASE origin WHEN 1 THEN 'iPhone 11 Pro' WHEN 2 THEN 'iPad' WHEN 0 THEN 'MacBook Pro' END)
48 |           )
49 |       FROM
50 |           history_visits
51 |           INNER JOIN history_items ON history_items.id = history_visits.history_item
52 |       WHERE
53 |           history_visits.redirect_destination IS NULL
54 |       GROUP BY
55 |           history_visits.id
56 |       ORDER BY
57 |           history_visits.visit_time DESC
58 |       LIMIT 10000
59 |       """
60 |     }
61 |   end
62 | 
63 |   defmodule TimeLineItem do
64 |     @moduledoc false
65 |     use Surface.Component
66 | 
67 |     prop item, :map, required: true
68 | 
69 |     def render(assigns) do
70 |       ~F"""
71 |       <p class="truncate">{raw(@item["_formatted"]["website_title"])}</p>
72 |       <div class="text-xs text-gray-400 dark:text-gray-500 truncate">
73 |         {@item["device_name"]}: <a href={@item["website_url"]} target="_blank" class="underline">{raw(@item["_formatted"]["website_url"])}</a>
74 |       </div>
75 |       """
76 |     end
77 |   end
78 | end
79 | 


--------------------------------------------------------------------------------
/memex/lib/memex/repo.ex:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Repo do
 2 |   use Ecto.Repo,
 3 |     otp_app: :memex,
 4 |     adapter: Ecto.Adapters.Postgres
 5 | 
 6 |   @doc """
 7 |   A small wrapper around `Repo.transaction/2'.
 8 | 
 9 |   Commits the transaction if the lambda returns `{:ok, result}`, rolling it
10 |   back if the lambda returns `{:error, reason}`. In both cases, the function
11 |   returns the result of the lambda.
12 |   """
13 |   @spec transact((() -> any()), keyword()) :: {:ok, any()} | {:error, any()}
14 |   def transact(fun, opts \\ []) do
15 |     transaction(
16 |       fn ->
17 |         case fun.() do
18 |           {:ok, value} -> value
19 |           :ok -> :transaction_commited
20 |           {:error, reason} -> rollback(reason)
21 |           :error -> rollback(:transaction_rollback_error)
22 |         end
23 |       end,
24 |       opts
25 |     )
26 |   end
27 | end
28 | 


--------------------------------------------------------------------------------
/memex/lib/memex/scheduler.ex:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Scheduler do
 2 |   @moduledoc false
 3 |   use GenServer
 4 | 
 5 |   alias Memex.Importer
 6 | 
 7 |   def start_link(_opts) do
 8 |     GenServer.start_link(__MODULE__, %{})
 9 |   end
10 | 
11 |   def init(_args) do
12 |     schedule()
13 |     {:ok, watcher_pid} = register_watcher()
14 |     # Importer.register_importers()
15 | 
16 |     {:ok, %{watcher_pid: watcher_pid}}
17 |   end
18 | 
19 |   def handle_info(:run, state) do
20 |     run()
21 |     schedule()
22 | 
23 |     {:noreply, state}
24 |   end
25 | 
26 |   def handle_info({:file_event, watcher_pid, {path, events}}, %{watcher_pid: watcher_pid} = state) do
27 |     {:noreply, state}
28 |   end
29 | 
30 |   def handle_info({:file_event, watcher_pid, :stop}, %{watcher_pid: watcher_pid} = state) do
31 |     {:noreply, state}
32 |   end
33 | 
34 |   defp run do
35 |     Enum.map(Importer.configured_importers(), fn importer -> Importer.import(importer) end)
36 |     # todo:
37 |     # - loop all importers
38 |     # - loop all documents of an importer
39 |     #   - get schedule from fetch config
40 |     #   - if it's :watcher, start a watcher that runs the import
41 |     #   - if it's :interval, 10, :minutes schedule a job
42 |     #   - (future) if it's :auto, get schedule a job based on history
43 |   end
44 | 
45 |   defp register_watcher do
46 |     {:ok, pid} = FileSystem.start_link(dirs: [Importer.get_dirs_to_watch()])
47 |     FileSystem.subscribe(pid)
48 | 
49 |     {:ok, pid}
50 |   end
51 | 
52 |   defp schedule do
53 |     Process.send_after(self(), :run, 1 * 60 * 60 * 1000)
54 |   end
55 | end
56 | 


--------------------------------------------------------------------------------
/memex/lib/memex/schema/catalog.ex:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Schema.Catalog do
 2 |   @moduledoc false
 3 |   alias Memex.Importer
 4 | 
 5 |   def search(key, value, sources) do
 6 |     Enum.reduce(sources, [], fn context, acc -> acc ++ search_source(context, key, value) end)
 7 |   end
 8 | 
 9 |   defp search_source("field_keys", _key, _value) do
10 |     Importer.available_importers()
11 |     |> Map.values()
12 |     |> Enum.reduce([], fn module, acc ->
13 |       acc ++ module.__schema__(:fields)
14 |     end)
15 |     |> Enum.map(fn field -> %{"type" => "field_keys", "value" => field} end)
16 |   end
17 | 
18 |   defp search_source(_context, _key, _value), do: []
19 | end
20 | 


--------------------------------------------------------------------------------
/memex/lib/memex/schema/document.ex:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Schema.Document do
 2 |   @moduledoc false
 3 |   use Ecto.Schema
 4 | 
 5 |   alias Memex.Schema.ImporterLog
 6 |   alias Memex.Schema.Relation
 7 | 
 8 |   @primary_key {:id, :string, autogenerate: false}
 9 | 
10 |   schema "documents" do
11 |     field(:body, :map)
12 |     field(:created_at, :utc_datetime)
13 |     field(:update_at, :utc_datetime, virtual: true)
14 |     has_many(:relations, Relation, foreign_key: :id)
15 |     belongs_to(:importer_log, ImporterLog, type: :binary_id)
16 |   end
17 | end
18 | 


--------------------------------------------------------------------------------
/memex/lib/memex/schema/encrypted/map.ex:
--------------------------------------------------------------------------------
1 | defmodule Memex.Schema.Encrypted.Map do
2 |   @moduledoc false
3 |   use Cloak.Ecto.Map, vault: Memex.Vault
4 | end
5 | 


--------------------------------------------------------------------------------
/memex/lib/memex/schema/importer_config.ex:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Schema.ImporterConfig do
 2 |   @moduledoc false
 3 |   use Ecto.Schema
 4 | 
 5 |   @primary_key {:id, :string, autogenerate: false}
 6 |   @foreign_key_type :string
 7 | 
 8 |   schema "importer_config" do
 9 |     field(:provider, :string)
10 |     field(:display_name, :string)
11 |     field(:encrypted_secrets, Memex.Schema.Encrypted.Map)
12 |     field(:config_overwrite, :map)
13 |     timestamps()
14 |   end
15 | end
16 | 


--------------------------------------------------------------------------------
/memex/lib/memex/schema/importer_log.ex:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Schema.ImporterLog do
 2 |   @moduledoc false
 3 |   use Ecto.Schema
 4 | 
 5 |   import Ecto.Query
 6 | 
 7 |   alias Memex.Repo
 8 | 
 9 |   defmodule Memex.Schema.ImporterLog.Query do
10 |     @moduledoc false
11 |     defstruct select: :hits_with_highlights,
12 |               filters: %{},
13 |               aggregates: %{},
14 |               limit: nil,
15 |               order_by: []
16 |   end
17 | 
18 |   @primary_key {:id, :binary_id, autogenerate: true}
19 |   @foreign_key_type :binary_id
20 | 
21 |   schema "importer_log" do
22 |     # processing, success or error
23 |     field(:state, :string)
24 |     field(:log, :string)
25 |     belongs_to(:config, Memex.Schema.ImporterConfig, type: :string)
26 |     timestamps()
27 |   end
28 | 
29 |   def aggregate(%Memex.Schema.ImporterLog.Query{} = query) do
30 |     from(il in __MODULE__)
31 |     |> add_filters(query.filters)
32 |     |> add_aggregates(query.aggregates)
33 |     |> add_order_by(query.order_by)
34 |   end
35 | 
36 |   defp add_filters(q, filters) do
37 |     Enum.reduce(filters, q, fn
38 |       {"month", month}, q ->
39 |         from(q in q, where: fragment("to_char(?.created_at, 'yyyy-mm') = ?", q, ^month))
40 | 
41 |       _, q ->
42 |         q
43 |     end)
44 |   end
45 | 
46 |   defp add_aggregates(q, aggregates) do
47 |     Enum.reduce(aggregates, q, fn
48 |       {"count", month}, q ->
49 |         from(q in q, where: fragment("to_char(?.created_at, 'yyyy-mm') = ?", q, ^month))
50 | 
51 |       _, q ->
52 |         q
53 |     end)
54 |   end
55 | 
56 |   defp add_order_by(q, order_by) do
57 |     Enum.reduce(order_by, q, fn
58 |       "updated_at_desc", q ->
59 |         from(q in q, order_by: [desc: q.updated_at])
60 | 
61 |       _, q ->
62 |         q
63 |     end)
64 |   end
65 | 
66 |   def count_by_config do
67 |     from(
68 |       il in __MODULE__,
69 |       group_by: il.config_id,
70 |       select: {il.config_id, count(il.id)}
71 |     )
72 |     |> Repo.all()
73 |     |> Map.new()
74 |   end
75 | 
76 |   # last import date and state for each config
77 |   def last_imports do
78 |     from(
79 |       il in __MODULE__,
80 |       distinct: il.config_id,
81 |       select:
82 |         {il.config_id,
83 |          %{
84 |            "inserted_at" => il.inserted_at,
85 |            "state" => il.state,
86 |            "log" => il.log,
87 |            "duration" => il.updated_at - il.inserted_at
88 |          }},
89 |       order_by: [desc: il.inserted_at]
90 |     )
91 |     |> Repo.all()
92 |     |> Map.new()
93 |   end
94 | end
95 | 


--------------------------------------------------------------------------------
/memex/lib/memex/schema/relation.ex:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Schema.Relation do
 2 |   @moduledoc false
 3 |   use Ecto.Schema
 4 | 
 5 |   alias Memex.Schema.Document
 6 | 
 7 |   @primary_key {:id, :string, autogenerate: false}
 8 | 
 9 |   schema "relations" do
10 |     field(:type, :string)
11 |     field(:metadata, :map)
12 |     field(:created_at, :naive_datetime, virtual: true)
13 |     belongs_to(:source, Document)
14 |   end
15 | end
16 | 


--------------------------------------------------------------------------------
/memex/lib/memex/search/legacy_query.ex:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Search.LegacyQuery do
 2 |   @moduledoc """
 3 |   Options for select:
 4 |   - `:hits_with_highlights`: Return hits with highlights (formatted as HTML)
 5 |   - `:total_hits`: Return total hits count
 6 |   - `facet: "month"`: Return facet counts for each month
 7 |   """
 8 |   defstruct select: :hits_with_highlights,
 9 |             query: "",
10 |             filters: %{},
11 |             page: 1,
12 |             limit: nil,
13 |             order_by: []
14 | 
15 |   def add_filter(query, key, value), do: put_in(query.filters[key], value)
16 | 
17 |   def remove_filter(query, key), do: elem(pop_in(query.filters[key]), 1)
18 | 
19 |   def select(query, select), do: put_in(query.select, select)
20 | 
21 |   def has_filters(query), do: query.filters !== %{}
22 | 
23 |   def to_string(query) do
24 |     []
25 |     |> Enum.concat([query.query])
26 |     |> Enum.concat(Enum.map(query.filters, fn {key, value} -> "#{key}:#{maybe_quote(value)}" end))
27 |     |> Enum.join(" ")
28 |     |> String.trim()
29 |   end
30 | 
31 |   defp maybe_quote(value) do
32 |     if String.contains?(value, " ") do
33 |       "\"#{value}\""
34 |     else
35 |       value
36 |     end
37 |   end
38 | 
39 |   def from_string(string) do
40 |     filters = Regex.scan(~r/(\w+):("[^"]*"|[^\s]+)/, string)
41 | 
42 |     %__MODULE__{
43 |       query: parse_query(string, filters),
44 |       filters: parse_filters(filters)
45 |     }
46 |   end
47 | 
48 |   defp parse_query(string, []), do: String.trim(string)
49 | 
50 |   defp parse_query(string, filters) do
51 |     string
52 |     |> String.replace(Enum.map(filters, fn [filter, _, _] -> filter end), "")
53 |     |> String.trim()
54 |   end
55 | 
56 |   defp parse_filters([]), do: %{}
57 | 
58 |   defp parse_filters(filters) do
59 |     Enum.reduce(filters, %{}, fn [_, key, value], acc -> Map.put(acc, key, String.trim(value, "\"")) end)
60 |   end
61 | end
62 | 


--------------------------------------------------------------------------------
/memex/lib/memex/search/query.ex:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Search.Query do
 2 |   @moduledoc """
 3 |   Options for select:
 4 |   - `:hits_with_highlights`: Return hits with highlights (formatted as HTML)
 5 |   - `:total_hits`: Return total hits count
 6 |   - `facet: "month"`: Return facet counts for each month
 7 |   """
 8 |   defstruct select: :hits_with_highlights,
 9 |             filters: [],
10 |             page: 1,
11 |             limit: nil,
12 |             order_by: []
13 | 
14 |   def remove_filter(query, key) do
15 |     put_in(
16 |       query.filters,
17 |       Enum.filter(query.filters, fn
18 |         %{"key" => k} -> k != key
19 |         _ -> true
20 |       end)
21 |     )
22 |   end
23 | 
24 |   def add_filter(query, type, key, value) do
25 |     put_in(query.filters, query.filters ++ [%{"type" => type, "key" => key, "value" => value}])
26 |   end
27 | 
28 |   def select(query, select), do: put_in(query.select, select)
29 | 
30 |   def has_filters(query), do: query.filters !== %{}
31 | 
32 |   def to_string(query) do
33 |     []
34 |     |> Enum.concat([query.query])
35 |     |> Enum.concat(Enum.map(query.filters, fn {key, value} -> "#{key}:#{maybe_quote(value)}" end))
36 |     |> Enum.join(" ")
37 |     |> String.trim()
38 |   end
39 | 
40 |   defp maybe_quote(value) do
41 |     if String.contains?(value, " ") do
42 |       "\"#{value}\""
43 |     else
44 |       value
45 |     end
46 |   end
47 | 
48 |   def from_filters(filters) do
49 |     %__MODULE__{
50 |       filters: filters
51 |     }
52 |   end
53 | end
54 | 


--------------------------------------------------------------------------------
/memex/lib/memex/search/sidebars.ex:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Search.Sidebars do
 2 |   @moduledoc """
 3 |   Manages opening and closing of sidebars. There should always be a closed
 4 |   sidebar at the end so that a new one can animate in.
 5 |   """
 6 |   @closed %{"closed" => true}
 7 |   @topic "sidebars"
 8 | 
 9 |   def init, do: append_closed([])
10 | 
11 |   def opened(data), do: Map.merge(data, %{"closed" => false})
12 | 
13 |   def open(sidebars, data), do: append_closed(Enum.drop(sidebars, -1) ++ [opened(data)])
14 | 
15 |   def close_last([%{"closed" => false} = _data]), do: append_closed([])
16 |   def close_last(sidebars), do: append_closed(Enum.drop(sidebars, -2))
17 | 
18 |   def subscribe, do: MemexWeb.Endpoint.subscribe(@topic)
19 | 
20 |   def broadcast_open(sidebar), do: MemexWeb.Endpoint.broadcast(@topic, "open-sidebar", sidebar)
21 | 
22 |   defp append_closed(sidebars), do: sidebars ++ [@closed]
23 | end
24 | 


--------------------------------------------------------------------------------
/memex/lib/memex/vault.ex:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Vault do
 2 |   @moduledoc false
 3 |   use Cloak.Vault, otp_app: :memex
 4 | 
 5 |   @impl GenServer
 6 |   def init(config) do
 7 |     config =
 8 |       Keyword.put(config, :ciphers,
 9 |         default: {Cloak.Ciphers.AES.GCM, tag: "AES.GCM.V1", key: decode_env!("SECRETS_ENCRYPTION_KEY")}
10 |       )
11 | 
12 |     {:ok, config}
13 |   end
14 | 
15 |   defp decode_env!(var) do
16 |     var
17 |     |> System.get_env()
18 |     |> Base.decode64!()
19 |   end
20 | end
21 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web.ex:
--------------------------------------------------------------------------------
  1 | defmodule MemexWeb do
  2 |   @moduledoc """
  3 |   The entrypoint for defining your web interface, such
  4 |   as controllers, components, channels, and so on.
  5 | 
  6 |   This can be used in your application as:
  7 | 
  8 |       use MemexWeb, :controller
  9 |       use MemexWeb, :view
 10 |       use MemexWeb, :html
 11 | 
 12 |   The definitions below will be executed for every controller,
 13 |   component, etc, so keep them short and clean, focused
 14 |   on imports, uses and aliases.
 15 | 
 16 |   Do NOT define functions inside the quoted expressions
 17 |   below. Instead, define additional modules and import
 18 |   those modules here.
 19 |   """
 20 | 
 21 |   def static_paths do
 22 |     ~w(assets fonts images favicon.ico robots.txt)
 23 |   end
 24 | 
 25 |   def controller do
 26 |     quote do
 27 |       use Phoenix.Controller,
 28 |         formats: [:html, :json],
 29 |         layouts: [html: MemexWeb.Layouts]
 30 | 
 31 |       import MemexWeb.Gettext
 32 |       import Plug.Conn
 33 | 
 34 |       alias MemexWeb.Router.Helpers, as: Routes
 35 | 
 36 |       unquote(verified_routes())
 37 |     end
 38 |   end
 39 | 
 40 |   def live_view do
 41 |     quote do
 42 |       use Phoenix.LiveView,
 43 |         layout: {MemexWeb.Layouts, :app}
 44 | 
 45 |       unquote(html_helpers())
 46 |       unquote(liveview_helpers())
 47 |     end
 48 |   end
 49 | 
 50 |   def live_component do
 51 |     quote do
 52 |       use Phoenix.LiveComponent
 53 | 
 54 |       unquote(html_helpers())
 55 |     end
 56 |   end
 57 | 
 58 |   def surface_live_view do
 59 |     quote do
 60 |       use Surface.LiveView,
 61 |         layout: {MemexWeb.Layouts, :app}
 62 | 
 63 |       unquote(html_helpers())
 64 |       unquote(liveview_helpers())
 65 |     end
 66 |   end
 67 | 
 68 |   def router do
 69 |     quote do
 70 |       use Phoenix.Router
 71 | 
 72 |       import Phoenix.Controller
 73 |       import Phoenix.LiveView.Router
 74 |       import Plug.Conn
 75 |     end
 76 |   end
 77 | 
 78 |   def channel do
 79 |     quote do
 80 |       use Phoenix.Channel
 81 | 
 82 |       import MemexWeb.Gettext
 83 |     end
 84 |   end
 85 | 
 86 |   def html do
 87 |     quote do
 88 |       use Phoenix.Component
 89 |       # Import convenience functions from controllers
 90 |       import Phoenix.Controller,
 91 |         only: [get_csrf_token: 0, view_module: 1, view_template: 1]
 92 | 
 93 |       # Include general helpers for rendering HTML
 94 |       unquote(html_helpers())
 95 |     end
 96 |   end
 97 | 
 98 |   defp html_helpers do
 99 |     quote do
100 |       # HTML escaping functionality
101 |       # Core UI components and translation
102 |       import MemexWeb.CoreComponents
103 |       import MemexWeb.Gettext
104 |       import Phoenix.HTML
105 | 
106 |       # Shortcut for generating JS commands
107 |       alias MemexWeb.Router.Helpers, as: Routes
108 |       alias Phoenix.LiveView.JS
109 | 
110 |       # Routes generation with the ~p sigil
111 |       unquote(verified_routes())
112 |     end
113 |   end
114 | 
115 |   def verified_routes do
116 |     quote do
117 |       use Phoenix.VerifiedRoutes,
118 |         endpoint: MemexWeb.Endpoint,
119 |         router: MemexWeb.Router,
120 |         statics: MemexWeb.static_paths()
121 |     end
122 |   end
123 | 
124 |   defp liveview_helpers do
125 |     quote do
126 |       defp async_query(socket, key, default, query) do
127 |         socket
128 |         |> assign_default_if_not_set(key, default)
129 |         |> assign_async(key, fn -> {key, Memex.Search.Postgres.query(query)} end)
130 |       end
131 | 
132 |       defp assign_default_if_not_set(socket, key, default) do
133 |         case socket.assigns[String.to_atom("#{key}")] do
134 |           nil -> assign(socket, String.to_atom("#{key}"), default)
135 |           _ -> socket
136 |         end
137 |       end
138 | 
139 |       defp assign_async(socket, key, callback) do
140 |         # convert string to atom
141 |         cancel_current_assign(socket.assigns[async_pid_key(key)])
142 | 
143 |         pid = self()
144 |         child_pid = spawn(fn -> send(pid, {:async_assign, callback.()}) end)
145 | 
146 |         assign(socket, async_pid_key(key), child_pid)
147 |       end
148 | 
149 |       defp assign_async_loading?(socket, key) do
150 |         socket.assigns[async_pid_key(key)] != nil
151 |       end
152 | 
153 |       defp cancel_current_assign(pid), do: pid && Process.exit(pid, :kill)
154 | 
155 |       defp async_pid_key(key) do
156 |         String.to_atom("#{key}_pid")
157 |       end
158 | 
159 |       @impl true
160 |       def handle_info({:async_assign, {key, result}}, socket) do
161 |         {:noreply, assign(socket, String.to_atom("#{key}"), result)}
162 |       end
163 |     end
164 |   end
165 | 
166 |   @doc """
167 |   When used, dispatch to the appropriate controller/view/etc.
168 |   """
169 |   defmacro __using__(which) when is_atom(which) do
170 |     apply(__MODULE__, which, [])
171 |   end
172 | end
173 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/channels/user_socket.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.UserSocket do
 2 |   use Phoenix.Socket
 3 | 
 4 |   ## Channels
 5 |   # channel "room:*", MemexWeb.RoomChannel
 6 | 
 7 |   # Socket params are passed from the client and can
 8 |   # be used to verify and authenticate a user. After
 9 |   # verification, you can put default assigns into
10 |   # the socket that will be set for all channels, ie
11 |   #
12 |   #     {:ok, assign(socket, :user_id, verified_user_id)}
13 |   #
14 |   # To deny connection, return `:error`.
15 |   #
16 |   # See `Phoenix.Token` documentation for examples in
17 |   # performing token verification on connect.
18 |   @impl true
19 |   def connect(_params, socket, _connect_info) do
20 |     {:ok, socket}
21 |   end
22 | 
23 |   # Socket id's are topics that allow you to identify all sockets for a given user:
24 |   #
25 |   #     def id(socket), do: "user_socket:#{socket.assigns.user_id}"
26 |   #
27 |   # Would allow you to broadcast a "disconnect" event and terminate
28 |   # all active sockets and channels for a given user:
29 |   #
30 |   #     MemexWeb.Endpoint.broadcast("user_socket:#{user.id}", "disconnect", %{})
31 |   #
32 |   # Returning `nil` makes this socket anonymous.
33 |   @impl true
34 |   def id(_socket), do: nil
35 | end
36 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/badge.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Components.Badge do
 2 |   @moduledoc false
 3 |   use Surface.Component
 4 | 
 5 |   prop click, :event
 6 |   prop values, :keyword, default: []
 7 |   prop class, :css_class
 8 | 
 9 |   slot default
10 |   slot icon
11 | 
12 |   def render(assigns) do
13 |     ~F"""
14 |     <button
15 |       :on-click={@click}
16 |       :values={@values}
17 |       class={
18 |         "bg-blue-100 text-blue-800 text-xs font-medium inline-flex items-center px-2.5 py-0.5 rounded-md",
19 |         @class
20 |       }
21 |     >
22 |       <#slot {@icon} />
23 |       <#slot />
24 |     </button>
25 |     """
26 |   end
27 | end
28 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/close_circles.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.CloseCircles do
 2 |   @moduledoc false
 3 |   use Surface.Component
 4 | 
 5 |   def render(assigns) do
 6 |     ~F"""
 7 |     <section class="demo-container watch-container">
 8 |       <article class="watch">
 9 |         <div class="bg-image" />
10 |         <div class="screen">
11 |           <section class="dials">
12 |             <div class="dial move">
13 |               <div class="dial-background one" />
14 |               <div class="dial-container container1">
15 |                 <div class="wedge" />
16 |               </div>
17 |               <div class="dial-container container2">
18 |                 <div class="wedge" />
19 |               </div>
20 |               <div class="marker start" />
21 |               <div class="marker end" />
22 |             </div>
23 |             <div class="dial exercise">
24 |               <div class="dial-background two" />
25 |               <div class="dial-container container1">
26 |                 <div class="wedge" />
27 |               </div>
28 |               <div class="marker start" />
29 |               <div class="marker end" />
30 |             </div>
31 |             <div class="dial stand">
32 |               <div class="dial-background three" />
33 |               <div class="dial-container container1">
34 |                 <div class="wedge" />
35 |               </div>
36 |               <div class="marker start" />
37 |               <div class="marker end" />
38 |             </div>
39 |           </section>
40 |         </div>
41 |       </article>
42 |     </section>
43 |     """
44 |   end
45 | end
46 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/dates_facet.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.DatesFacet do
 2 |   @moduledoc false
 3 |   use Surface.Component
 4 | 
 5 |   alias Phoenix.LiveView.JS
 6 | 
 7 |   @target "#search-input"
 8 | 
 9 |   prop(dates, :list)
10 |   prop(max_count, :number)
11 |   prop(loading, :boolean, default: false)
12 | 
13 |   def render(assigns) do
14 |     assigns = update(assigns)
15 | 
16 |     ~F"""
17 |     <div id="date-facet" class={"cursor-pointer relative", "opacity-50": @loading}>
18 |       <div
19 |         :for={{date, count} <- @dates}
20 |         class="group text-xs hover:bg-gray-100 hover:dark:bg-gray-600"
21 |         id={"facet-#{date}"}
22 |         phx-click={filter_by_date(date)}
23 |       >
24 |         <span
25 |           class="absolute opacity-0 group-hover:opacity-100 text-gray-500 dark:text-gray-100 leading-3"
26 |           style="font-size: 8px; margin-top: -2px"
27 |         >{date} ({count})</span>
28 |         <button
29 |           id={date}
30 |           title={"#{date} (#{count})"}
31 |           class="dark:bg-gray-700 bg-gray-300 group-hover:dark:bg-gray-500 group-hover:bg-gray-200 ml-auto block mb-px h-2 transition-all"
32 |           style={"width: #{unless @max_count == 0 do
33 |             round(max(1, 100 / @max_count * count))
34 |           else
35 |             1
36 |           end}%;"}
37 |         />
38 |       </div>
39 |     </div>
40 |     """
41 |   end
42 | 
43 |   defp filter_by_date(date) do
44 |     %JS{}
45 |     |> JS.dispatch("removeFilter", to: @target, detail: %{key: "time"})
46 |     |> JS.dispatch("removeFilter", to: @target, detail: %{key: "month"})
47 |     |> JS.dispatch("addFilter", to: @target, detail: %{key: "month", value: date})
48 |     |> JS.dispatch("search", to: @target)
49 |   end
50 | 
51 |   defp update(assigns) do
52 |     assign(assigns, :max_count, max_count(assigns.dates))
53 |   end
54 | 
55 |   defp max_count(dates) do
56 |     dates
57 |     |> Enum.max_by(fn {_date, count} -> count end, fn -> {"", 0} end)
58 |     |> elem(1)
59 |   end
60 | end
61 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/forms/toggle_switch.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Components.Forms.ToggleSwitch do
 2 |   @moduledoc false
 3 |   use Surface.Component
 4 | 
 5 |   prop id, :string
 6 |   prop click, :event
 7 |   prop checked, :boolean, default: false
 8 |   prop values, :keyword, default: []
 9 | 
10 |   def render(assigns) do
11 |     ~F"""
12 |     <div class="relative inline-block w-10 mr-2 align-middle select-none transition duration-200 ease-in">
13 |       <input
14 |         type="checkbox"
15 |         :on-click={@click}
16 |         :values={@values}
17 |         checked={@checked}
18 |         name={@id}
19 |         id={@id}
20 |         class="toggle-checkbox absolute block w-6 h-6 rounded-full bg-white dark:bg-gray-600 border-4 dark:border-gray-500 appearance-none cursor-pointer checked:right-0 checked:border-green-400 checked:bg-green-100"
21 |       />
22 |       <label
23 |         for={@id}
24 |         class="toggle-label block overflow-hidden h-6 rounded-full bg-gray-300 dark:bg-gray-800 cursor-pointer"
25 |       />
26 |     </div>
27 |     """
28 |   end
29 | end
30 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/icon.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Components.Icon do
 2 |   @moduledoc false
 3 |   defmodule Plus do
 4 |     @moduledoc false
 5 |     use Surface.Component
 6 | 
 7 |     def render(assigns) do
 8 |       ~F"""
 9 |       <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" viewBox="0 0 20 20" fill="currentColor">
10 |         <path
11 |           fill-rule="evenodd"
12 |           d="M10 5a1 1 0 011 1v3h3a1 1 0 110 2h-3v3a1 1 0 11-2 0v-3H6a1 1 0 110-2h3V6a1 1 0 011-1z"
13 |           clip-rule="evenodd"
14 |         />
15 |       </svg>
16 |       """
17 |     end
18 |   end
19 | 
20 |   defmodule Settings do
21 |     @moduledoc false
22 |     use Surface.Component
23 | 
24 |     def render(assigns) do
25 |       ~F"""
26 |       <svg
27 |         xmlns="http://www.w3.org/2000/svg"
28 |         class="dark:text-gray-400 h-6 w-6"
29 |         fill="none"
30 |         viewBox="0 0 24 24"
31 |         stroke="currentColor"
32 |       >
33 |         <path
34 |           stroke-linecap="round"
35 |           stroke-linejoin="round"
36 |           stroke-width="2"
37 |           d="M10.325 4.317c.426-1.756 2.924-1.756 3.35 0a1.724 1.724 0 002.573 1.066c1.543-.94 3.31.826 2.37 2.37a1.724 1.724 0 001.065 2.572c1.756.426 1.756 2.924 0 3.35a1.724 1.724 0 00-1.066 2.573c.94 1.543-.826 3.31-2.37 2.37a1.724 1.724 0 00-2.572 1.065c-.426 1.756-2.924 1.756-3.35 0a1.724 1.724 0 00-2.573-1.066c-1.543.94-3.31-.826-2.37-2.37a1.724 1.724 0 00-1.065-2.572c-1.756-.426-1.756-2.924 0-3.35a1.724 1.724 0 001.066-2.573c-.94-1.543.826-3.31 2.37-2.37.996.608 2.296.07 2.572-1.065z"
38 |         />
39 |         <path
40 |           stroke-linecap="round"
41 |           stroke-linejoin="round"
42 |           stroke-width="2"
43 |           d="M15 12a3 3 0 11-6 0 3 3 0 016 0z"
44 |         />
45 |       </svg>
46 |       """
47 |     end
48 |   end
49 | end
50 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/icons/plus_icon.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Components.Icons.PlusIcon do
 2 |   @moduledoc false
 3 |   use Surface.Component
 4 | 
 5 |   def render(assigns) do
 6 |     ~F"""
 7 |     <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" viewBox="0 0 20 20" fill="currentColor">
 8 |       <path
 9 |         fill-rule="evenodd"
10 |         d="M10 5a1 1 0 011 1v3h3a1 1 0 110 2h-3v3a1 1 0 11-2 0v-3H6a1 1 0 110-2h3V6a1 1 0 011-1z"
11 |         clip-rule="evenodd"
12 |       />
13 |     </svg>
14 |     """
15 |   end
16 | end
17 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/icons/settings_icon.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Components.Icons.SettingsIcon do
 2 |   @moduledoc false
 3 |   use Surface.Component
 4 | 
 5 |   def render(assigns) do
 6 |     ~F"""
 7 |     <svg
 8 |       xmlns="http://www.w3.org/2000/svg"
 9 |       class="dark:text-gray-400 h-6 w-6"
10 |       fill="none"
11 |       viewBox="0 0 24 24"
12 |       stroke="currentColor"
13 |     >
14 |       <path
15 |         stroke-linecap="round"
16 |         stroke-linejoin="round"
17 |         stroke-width="2"
18 |         d="M10.325 4.317c.426-1.756 2.924-1.756 3.35 0a1.724 1.724 0 002.573 1.066c1.543-.94 3.31.826 2.37 2.37a1.724 1.724 0 001.065 2.572c1.756.426 1.756 2.924 0 3.35a1.724 1.724 0 00-1.066 2.573c.94 1.543-.826 3.31-2.37 2.37a1.724 1.724 0 00-2.572 1.065c-.426 1.756-2.924 1.756-3.35 0a1.724 1.724 0 00-2.573-1.066c-1.543.94-3.31-.826-2.37-2.37a1.724 1.724 0 00-1.065-2.572c-1.756-.426-1.756-2.924 0-3.35a1.724 1.724 0 001.066-2.573c-.94-1.543.826-3.31 2.37-2.37.996.608 2.296.07 2.572-1.065z"
19 |       />
20 |       <path
21 |         stroke-linecap="round"
22 |         stroke-linejoin="round"
23 |         stroke-width="2"
24 |         d="M15 12a3 3 0 11-6 0 3 3 0 016 0z"
25 |       />
26 |     </svg>
27 |     """
28 |   end
29 | end
30 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/layouts.ex:
--------------------------------------------------------------------------------
1 | defmodule MemexWeb.Layouts do
2 |   @moduledoc false
3 |   use MemexWeb, :html
4 | 
5 |   embed_templates "layouts/*"
6 | end
7 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/layouts/app.html.heex:
--------------------------------------------------------------------------------
1 | <main>
2 |   <.flash_group flash={@flash} />
3 |   <%= @inner_content %>
4 | </main>
5 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/layouts/live.html.leex:
--------------------------------------------------------------------------------
 1 | <main role="main">
 2 |   <p class="alert alert-info" role="alert"
 3 |     phx-click="lv:clear-flash"
 4 |     phx-value-key="info"><%= live_flash(@flash, :info) %></p>
 5 | 
 6 |   <p class="alert alert-danger" role="alert"
 7 |     phx-click="lv:clear-flash"
 8 |     phx-value-key="error"><%= live_flash(@flash, :error) %></p>
 9 | 
10 |   <%= @inner_content %>
11 | </main>
12 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/layouts/root.html.heex:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="en" style="scrollbar-gutter: stable;">
 3 |   <head>
 4 |     <meta charset="utf-8" />
 5 |     <meta name="viewport" content="width=device-width, initial-scale=1" />
 6 |     <meta name="csrf-token" content={get_csrf_token()} />
 7 |     <.live_title><%= assigns[:page_title] || "Memex" %></.live_title>
 8 |     <link phx-track-static rel="stylesheet" href={~p"/assets/app.css"} />
 9 |     <link rel="shortcut icon" type="image/png" href={~p"/images/memex.png"} />
10 |     <link
11 |       rel="shortcut icon"
12 |       type="image/png"
13 |       href={~p"/images/memex.png"}
14 |       color="darkhex"
15 |       media="(prefers-color-scheme: dark)"
16 |     />
17 |     <script defer phx-track-static type="module" src={~p"/assets/app.js"}>
18 |     </script>
19 |   </head>
20 |   <body class="dark:bg-gray-800 bg-gray-200">
21 |     <%= @inner_content %>
22 |   </body>
23 | </html>
24 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/map.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Map do
 2 |   @moduledoc false
 3 |   use Surface.Component
 4 | 
 5 |   alias MemexWeb.Router.Helpers, as: Routes
 6 | 
 7 |   prop geojson_path, :string, required: true
 8 |   prop items, :list, default: []
 9 |   prop height, :integer, default: 250
10 | 
11 |   def render(assigns) do
12 |     ~F"""
13 |     <div
14 |       class="rounded-md overflow-hidden mb-2 min-w-full dark:bg-gray-900"
15 |       style={"height: #{@height}px"}
16 |     >
17 |       <div
18 |         class="relative flex-grow rounded-md"
19 |         style={"height: #{@height}px"}
20 |         phx-hook="Map"
21 |         phx-update="ignore"
22 |         id="map"
23 |         data-items={format_items(@items)}
24 |         data-url={@geojson_path}
25 |       />
26 |     </div>
27 |     """
28 |   end
29 | 
30 |   defp format_items(items) do
31 |     items
32 |     |> Enum.map(&item_to_geojson(&1))
33 |     |> Enum.filter(fn item -> item !== false end)
34 |     |> Jason.encode!()
35 |   end
36 | 
37 |   defp item_to_geojson(%{"provider" => "Photos"} = item) do
38 |     %{
39 |       "type" => "geojson",
40 |       "data" => %{
41 |         "type" => "Feature",
42 |         "geometry" => %{
43 |           "type" => "Point",
44 |           "coordinates" => [item["location_longitude"], item["location_latitude"]]
45 |         },
46 |         "properties" => %{
47 |           "style" => %{
48 |             "width" => "40px",
49 |             "height" => "40px",
50 |             "backgroundImage" => "url(#{Routes.photo_path(MemexWeb.Endpoint, :image, item["photo_file_path"])})",
51 |             "backgroundSize" => "100%",
52 |             "borderColor" => "white",
53 |             "borderWidth" => "2px",
54 |             "borderRadius" => "0.125rem"
55 |           }
56 |         }
57 |       }
58 |     }
59 |   end
60 | 
61 |   defp item_to_geojson(_item), do: false
62 | end
63 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/search_bar.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.SearchBar do
 2 |   @moduledoc false
 3 |   use Surface.Component
 4 | 
 5 |   prop(query, :string)
 6 | 
 7 |   def render(assigns) do
 8 |     ~F"""
 9 |     <div class="p-5 pt-0 font-san items-center justify-center">
10 |       <header class="block fixed md:ml-4 text-black w-8/12 dark:text-white shadow-lg dark:bg-black bg-white rounded-xl">
11 |         <div class="flex justify-center items-center flex-grow relative">
12 |           <button class="flex items-center justify-center absolute left-0 top-5 rounded-l-xl px-4 text-black dark:text-white">
13 |             <svg
14 |               class="h-4 w-4 text-grey-dark"
15 |               fill="currentColor"
16 |               xmlns="http://www.w3.org/2000/svg"
17 |               viewBox="0 0 24 24"
18 |             >
19 |               <path d="M16.32 14.9l5.39 5.4a1 1 0 0 1-1.42 1.4l-5.38-5.38a8 8 0 1 1 1.41-1.41zM10 16a6 6 0 1 0 0-12 6 6 0 0 0 0 12z" />
20 |             </svg>
21 |           </button>
22 |           <form class="w-full relative" id="search-form" role="search" novalidate="">
23 |             <div
24 |               id="editor"
25 |               phx-update="ignore"
26 |               class="transition-colors h-14 focus:ring-blue-900 focus:ring-2 z-1 w-full text-black dark:text-white bg-transparent pl-10 pr-2 py-4 border-0"
27 |             />
28 |             <textarea
29 |               rows="1"
30 |               autocapitalize="off"
31 |               autocomplete="off"
32 |               autofocus="true"
33 |               class="hidden"
34 |               id="search-input"
35 |               maxlength="512"
36 |               name="query"
37 |               phx-debounce="10"
38 |               phx-hook="Editor"
39 |               placeholder="Search..."
40 |               spellcheck="false"
41 |               type="search"
42 |             >{@query}</textarea>
43 |           </form>
44 | 
45 |           <span class="absolute right-5 hidden sm:block text-gray-500 text-sm leading-5 py-0.5 px-1.5 border border-gray-700 rounded-md"><span class="sr-only">Press
46 |             </span><kbd class="font-sans"><abbr title="Command" class="no-underline">⌘</abbr></kbd><span class="sr-only">
47 |               and
48 |             </span><kbd class="font-sans">K</kbd><span class="sr-only">
49 |               to search</span></span>
50 |         </div>
51 |       </header>
52 |     </div>
53 |     """
54 |   end
55 | end
56 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/search_result_stats.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.SearchResultStats do
 2 |   @moduledoc false
 3 |   use Surface.Component
 4 | 
 5 |   prop(total_hits, :number)
 6 | 
 7 |   def render(assigns) do
 8 |     ~F"""
 9 |     <div class="text-xs dark:text-gray-600 text-gray-400 ml-12 md:ml-20 mt-1">
10 |       <div class="flex w-auto items-start">
11 |         <div class="flex-none self-stretch border-l border-r dark:border-gray-800 border-gray-200 bg-gray-400 dark:bg-gray-700 w-2 -ml-1" />
12 | 
13 |         <div class="pl-3">
14 |           <span class="pt-4 inline-block">
15 |             {#if @total_hits != nil}
16 |               Found {MemexWeb.TimelineView.number_short(@total_hits)} results
17 |             {#else}
18 |               &nbsp;
19 |             {/if}
20 |           </span>
21 |         </div>
22 |       </div>
23 |     </div>
24 |     """
25 |   end
26 | end
27 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/sidebars_component.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.SidebarsComponent do
 2 |   @moduledoc false
 3 |   use Surface.Component
 4 | 
 5 |   prop(sidebars, :list, required: true)
 6 |   prop(socket, :any, required: true)
 7 | 
 8 |   def render(assigns) do
 9 |     ~F"""
10 |     <div
11 |       phx-hook="Sidebar"
12 |       id="sidebars"
13 |       data-state={if length(@sidebars) > 1 do
14 |         "open"
15 |       else
16 |         "closed"
17 |       end}
18 |       aria-hidden
19 |     />
20 |     <div
21 |       :for.with_index={{hit, index} <- @sidebars}
22 |       id={"sidebar-#{index}"}
23 |       class={
24 |         "fixed inset-0 transition backdrop-filter backdrop-blur-md",
25 |         "backdrop-filter-none pointer-events-none": hit["closed"],
26 |         "bg-black/50": not hit["closed"]
27 |       }
28 |     >
29 |       <div phx-click="close-last-sidebar" class="fixed inset-0 text-white" />
30 |       <div class={"fixed border-l dark:border-gray-700 dark:bg-gray-800 bg-gray-50 p-5 inset-y-0 right-0 shadow-2xl #{width(index, length(@sidebars))} overflow-scroll transform transition-transform #{if hit["closed"] do
31 |         "translate-x-full"
32 |       end}"}>
33 |         <button phx-click="close-last-sidebar" aria-label="Close" class="dark:text-white float-right">
34 |           <svg
35 |             xmlns="http://www.w3.org/2000/svg"
36 |             class="h-6 w-6"
37 |             fill="none"
38 |             viewBox="0 0 24 24"
39 |             stroke="currentColor"
40 |           ><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" /></svg>
41 |         </button>
42 |         <div class="-mt-2" :if={hit["type"] === "generic"}>{live_render(@socket, MemexWeb.Sidebars.GenericLive, id: "generic", session: hit)}</div>
43 |         <div class="-mt-2" :if={hit["type"] === "person"}>{live_render(@socket, MemexWeb.Sidebars.PersonLive, id: "person", session: %{"hit" => hit})}</div>
44 |         <div class="-mt-2" :if={hit["type"] === "activity"}>{live_render(@socket, MemexWeb.Sidebars.ActivityLive, id: "activity", session: hit)}</div>
45 |         <div class="-mt-2" :if={hit["type"] === "settings"}>{live_render(@socket, MemexWeb.Sidebars.SettingsLive, id: "settings")}</div>
46 |         <div
47 |           :if={hit["location_latitude"] && hit["location_longitude"]}
48 |           class="rounded-md col-span-3"
49 |           style="height: 200px"
50 |           phx-hook="Map"
51 |           id={"map-#{index}"}
52 |           data-points={points(hit)}
53 |         />
54 |       </div>
55 |     </div>
56 |     """
57 |   end
58 | 
59 |   defp width(index, _length) do
60 |     case index do
61 |       0 -> "w-11/12"
62 |       1 -> "w-10/12"
63 |       2 -> "w-9/12"
64 |       3 -> "w-8/12"
65 |       _ -> "w-7/12"
66 |     end
67 |   end
68 | 
69 |   defp points(hit) do
70 |     Jason.encode!([
71 |       %{
72 |         "lat" => hit["location_latitude"],
73 |         "lon" => hit["location_longitude"]
74 |       }
75 |     ])
76 |   end
77 | end
78 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/text.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Components.Text do
 2 |   @moduledoc false
 3 |   defmodule H1 do
 4 |     @moduledoc false
 5 |     use Surface.Component
 6 | 
 7 |     slot default
 8 | 
 9 |     def render(assigns) do
10 |       ~F"""
11 |       <div class="dark:text-white text-2xl font-bold"><#slot /></div>
12 |       """
13 |     end
14 |   end
15 | 
16 |   defmodule H2 do
17 |     @moduledoc false
18 |     use Surface.Component
19 | 
20 |     slot default
21 | 
22 |     def render(assigns) do
23 |       ~F"""
24 |       <div class="text-white text-xl font-bold mt-5"><#slot /></div>
25 |       """
26 |     end
27 |   end
28 | 
29 |   defmodule SubtTitle do
30 |     @moduledoc false
31 |     use Surface.Component
32 | 
33 |     slot default
34 | 
35 |     def render(assigns) do
36 |       ~F"""
37 |       <p class="dark:text-gray-400"><#slot /></p>
38 |       """
39 |     end
40 |   end
41 | end
42 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/time_duration.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.TimeDuration do
 2 |   @moduledoc false
 3 |   use Surface.Component
 4 | 
 5 |   prop start_time, :datetime, required: true
 6 |   prop end_time, :datetime, required: true
 7 | 
 8 |   def render(assigns) do
 9 |     ~F"""
10 |     <div class="inline-flex rounded-md bg-gray-200 dark:bg-gray-900">
11 |       <div class="p-2 pr-1 text-gray-300">{to_time(@start_time)}</div>
12 |       <svg
13 |         class="self-center text-gray-300"
14 |         xmlns="http://www.w3.org/2000/svg"
15 |         width="14"
16 |         height="14"
17 |         fill="none"
18 |         viewBox="0 0 24 24"
19 |         stroke="currentColor"
20 |       ><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 5l7 7-7 7" /></svg>
21 |       <div class="p-2 pl-1 text-gray-300">{to_time(@end_time)}</div>
22 |     </div>
23 |     """
24 |   end
25 | 
26 |   defp to_time(timestamp) do
27 |     timestamp
28 |     |> MemexWeb.TimelineView.date_from_timestamp()
29 |     |> DateTime.shift_zone!("Europe/Amsterdam")
30 |     |> Calendar.strftime("%H:%M")
31 |   end
32 | end
33 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/timeline/card.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Timeline.Card do
 2 |   @moduledoc false
 3 |   use Surface.Component
 4 | 
 5 |   prop class, :css_class
 6 |   prop selected, :boolean, default: false
 7 | 
 8 |   @doc "Middle section content the card"
 9 |   slot content, required: true
10 | 
11 |   @doc "Image to the left of the card"
12 |   slot media
13 | 
14 |   @doc "Section on the right of the card"
15 |   slot right
16 | 
17 |   def render(assigns) do
18 |     ~F"""
19 |     <div class={
20 |       @class,
21 |       "flex-grow rounded-md bg-white dark:bg-gray-900 hover:shadow-lg hover:pt-3 hover:pb-5 hover:ring-gray-700 hover:ring-2 cursor-pointer transition-all p-4 my-2 shadow-md overflow-hidden dark:text-white",
22 |       "shadow-lg pt-3 pb-5 ring-gray-700 ring-2": @selected
23 |     }>
24 |       <div class="float-right"><#slot {@right} /></div>
25 |       <div class="float-left"><#slot {@media} /></div>
26 |       <#slot {@content} />
27 |     </div>
28 |     """
29 |   end
30 | end
31 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/timeline/date_border.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Timeline.DateBorder do
 2 |   @moduledoc false
 3 |   use Surface.Component
 4 | 
 5 |   prop(date, :datetime, required: true)
 6 |   prop(days_between, :number, required: true)
 7 | 
 8 |   def render(assigns) do
 9 |     ~F"""
10 |     <div class="text-base font-medium text-gray-500 p-3 dark:text-gray-500">
11 |       <span>{format(@date)}</span>
12 | 
13 |       <span class="hidden md:inline-block text-xs transition-opacity opacity-0 group-hover:opacity-100">
14 |         {days_between_to_text(@days_between)}
15 |       </span>
16 |     </div>
17 |     """
18 |   end
19 | 
20 |   defp format(date) do
21 |     Calendar.strftime(date, "%A, %d %B %Y")
22 |   end
23 | 
24 |   defp days_between_to_text(days_between) do
25 |     case days_between do
26 |       0 -> ""
27 |       1 -> "1 day before"
28 |       d when d <= 31 -> "#{d} days before"
29 |       d when d <= 31 * 12 -> "#{round(d / 31)} months before"
30 |       d -> "#{d / 365} years before"
31 |     end
32 |   end
33 | end
34 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/timeline/provider_icon.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Timeline.ProviderIcon do
 2 |   @moduledoc false
 3 |   use Surface.Component
 4 | 
 5 |   prop(provider, :string)
 6 | 
 7 |   def render(assigns) do
 8 |     ~F"""
 9 |     <figure
10 |       :if={is_nil(icon(@provider))}
11 |       class="flex-none self-center inline-block rounded-full w-2 h-2 -ml-2 dark:bg-gray-400"
12 |     />
13 |     <div :if={icon(@provider)} class="flex-none self-center inline-block w-6 h-6 -ml-4">
14 |       <img src={"#{icon(@provider)}"}>
15 |     </div>
16 |     """
17 |   end
18 | 
19 |   def icon(provider), do: MemexWeb.TimelineView.icon_by_provider(provider)
20 | end
21 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/timeline/time.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Timeline.Time do
 2 |   @moduledoc false
 3 |   use Surface.Component
 4 | 
 5 |   alias Phoenix.LiveView.JS
 6 | 
 7 |   @target "#search-input"
 8 | 
 9 |   prop(date, :datetime)
10 | 
11 |   def render(assigns) do
12 |     ~F"""
13 |     <div
14 |       phx-click={filter_by_time(@date)}
15 |       class="flex-none cursor-pointer self-center text-right text-gray-400 dark:text-gray-500 text-xs md:text-md -ml-12 md:-ml-32 w-12 md:w-32 pr-5"
16 |     >
17 |       {to_time(@date)}
18 |     </div>
19 |     """
20 |   end
21 | 
22 |   defp filter_by_time(date) do
23 |     %JS{}
24 |     |> JS.dispatch("resetFilters", to: @target)
25 |     |> JS.dispatch("addFilter", to: @target, detail: %{key: "time", value: to_iso(date)})
26 |     |> JS.dispatch("search", to: @target)
27 |   end
28 | 
29 |   defp to_time(date), do: Calendar.strftime(date, "%H:%M")
30 |   defp to_iso(date), do: DateTime.to_iso8601(date)
31 | end
32 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/components/timeline/vertical_line.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Timeline.VerticalLine do
 2 |   @moduledoc false
 3 |   use Surface.Component
 4 | 
 5 |   prop(class, :string, default: "")
 6 | 
 7 |   def render(assigns) do
 8 |     ~F"""
 9 |     <div class={"flex-none self-stretch border-l border-r dark:border-gray-800 border-gray-200 w-2 -ml-1 #{@class}"} />
10 |     """
11 |   end
12 | end
13 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/controllers/alfred_controller.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.AlfredController do
 2 |   use MemexWeb, :controller
 3 | 
 4 |   alias Memex.Search.LegacyQuery, as: Query
 5 |   alias Memex.Search.Postgres, as: Search
 6 | 
 7 |   def search(conn, params) do
 8 |     query = %{Query.from_string(params["q"] || "") | limit: 20}
 9 | 
10 |     with {:ok, results} <- Search.query(query) do
11 |       conn
12 |       |> put_resp_content_type("application/json", "utf-8")
13 |       |> send_resp(200, Jason.encode!(alfred_format_hits(results)))
14 |     else
15 |       {:error, error} ->
16 |         conn
17 |         |> put_resp_content_type("application/json", "utf-8")
18 |         |> send_resp(400, Jason.encode!(%{"error" => error}))
19 |     end
20 |   end
21 | 
22 |   defp alfred_format_hits(hits) do
23 |     %{
24 |       items: Enum.map(hits, &alfred_format_hit(&1))
25 |     }
26 |   end
27 | 
28 |   defp alfred_format_hit(hit) do
29 |     %{
30 |       # uid: hit["id"], # Exclude ID to enforce the item order
31 |       type: "default",
32 |       title: alfred_format_title(hit),
33 |       subtitle: alfred_format_subtitle(hit),
34 |       arg: alfred_format_arg(hit),
35 |       # autocomplete: "Desktop",
36 |       valid: alfred_format_valid(hit),
37 |       icon: %{
38 |         path: alfred_format_icon_path(hit)
39 |       },
40 |       text: %{
41 |         copy: alfred_format_title(hit),
42 |         largetype: alfred_format_title(hit)
43 |       }
44 |     }
45 |   end
46 | 
47 |   defp alfred_format_arg(%{"provider" => "Safari"} = hit), do: hit["website_url"]
48 |   defp alfred_format_arg(%{"provider" => "Github"} = hit), do: hit["repo_homepage"]
49 |   defp alfred_format_arg(%{"provider" => "Twitter"} = hit), do: hit["tweet_url"]
50 |   defp alfred_format_arg(_hit), do: nil
51 | 
52 |   defp alfred_format_title(%{"provider" => "Safari"} = hit), do: hit["website_title"]
53 |   defp alfred_format_title(%{"provider" => "iMessage"} = hit), do: hit["message_text"]
54 |   defp alfred_format_title(%{"provider" => "GitHub"} = hit), do: hit["repo_name"]
55 |   defp alfred_format_title(%{"provider" => "Twitter"} = hit), do: hit["tweet_full_text"]
56 |   defp alfred_format_title(%{"provider" => "terminal"} = hit), do: hit["command"]
57 | 
58 |   defp alfred_format_title(%{"provider" => "MoneyMoney"} = hit),
59 |     do:
60 |       MemexWeb.TimelineView.number_to_currency(abs(hit["transaction_amount"]), hit["transaction_currency"]) <>
61 |         ": #{hit["transaction_recipient"]} #{hit["transaction_category"]}"
62 | 
63 |   defp alfred_format_title(_hit), do: "unknown"
64 | 
65 |   defp alfred_format_subtitle(%{"provider" => "Safari"} = hit), do: hit["website_url"]
66 |   defp alfred_format_subtitle(%{"provider" => "Twitter"} = hit), do: hit["tweet_user_screen_name"]
67 | 
68 |   defp alfred_format_subtitle(%{"provider" => "MoneyMoney"} = hit),
69 |     do: "#{hit["transaction_account_name"]} #{hit["transaction_purpose"]}"
70 | 
71 |   defp alfred_format_subtitle(%{"provider" => "GitHub"} = hit),
72 |     do: "#{hit["repo_description"]} #{hit["repo_license"]} #{hit["repo_language"]} #{hit["repo_stars_count"]} stars"
73 | 
74 |   defp alfred_format_subtitle(%{"provider" => "iMessage"} = hit) do
75 |     case hit["message_direction"] do
76 |       "sent" -> "Sent to "
77 |       "received" -> "Received from "
78 |     end <>
79 |       hit["person_name"]
80 |   end
81 | 
82 |   defp alfred_format_subtitle(_hit), do: ""
83 | 
84 |   defp alfred_format_icon_path(hit),
85 |     do: Path.absname("assets/static/" <> MemexWeb.TimelineView.icon_by_provider(hit["provider"]))
86 | 
87 |   defp alfred_format_valid(hit) do
88 |     case alfred_format_arg(hit) do
89 |       nil -> false
90 |       _ -> true
91 |     end
92 |   end
93 | end
94 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/controllers/arc_controller.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.ArcController do
 2 |   use MemexWeb, :controller
 3 | 
 4 |   @arc_export_path "/Users/adrimbp/Library/Mobile\ Documents/iCloud~com~bigpaua~LearnerCoacher/Documents/Export/"
 5 | 
 6 |   def geopoint(conn, %{"date" => date, "id" => timeline_item_id} = _params) do
 7 |     date
 8 |     |> find_timeline_item(timeline_item_id)
 9 |     |> case do
10 |       {:ok, item} ->
11 |         conn
12 |         |> put_resp_content_type("application/json", "utf-8")
13 |         |> json(item)
14 | 
15 |       {:error, _error} ->
16 |         send_resp(conn, 404, "Not found")
17 |     end
18 |   end
19 | 
20 |   def gpx(conn, %{"date" => date, "id" => _timeline_item_id} = _params) do
21 |     path = @arc_export_path <> "GPX/Daily/" <> date <> ".gpx"
22 | 
23 |     path
24 |     |> File.read()
25 |     |> case do
26 |       {:ok, content} ->
27 |         conn
28 |         |> put_resp_content_type("application/gpx+xml", "utf-8")
29 |         |> send_resp(200, content)
30 | 
31 |       {:error, _error} ->
32 |         send_resp(conn, 404, "Not found")
33 |     end
34 |   end
35 | 
36 |   def geojson(conn, %{"date" => date, "id" => timeline_item_id} = _params) do
37 |     date
38 |     |> find_timeline_item(timeline_item_id)
39 |     |> case do
40 |       {:ok, item} ->
41 |         json(conn, %{
42 |           "type" => "Feature",
43 |           "properties" => %{},
44 |           "geometry" => %{
45 |             "type" => "LineString",
46 |             "coordinates" =>
47 |               item["samples"]
48 |               |> Enum.filter(fn sample -> not is_nil(sample["location"]["longitude"]) end)
49 |               |> Enum.map(fn sample -> [sample["location"]["longitude"], sample["location"]["latitude"]] end)
50 |           }
51 |         })
52 | 
53 |       {:error, _error} ->
54 |         send_resp(conn, 404, "Not found")
55 |     end
56 |   end
57 | 
58 |   defp find_timeline_item(date, "arc-" <> timeline_item_id) do
59 |     path = @arc_export_path <> "JSON/Daily/" <> date <> ".json.gz"
60 | 
61 |     path
62 |     |> File.stream!([:compressed])
63 |     |> Enum.into("")
64 |     |> Jason.decode()
65 |     |> case do
66 |       {:ok, content} ->
67 |         item = Enum.find(content["timelineItems"], fn item -> item["itemId"] == timeline_item_id end)
68 | 
69 |         {:ok, item}
70 | 
71 |       {:error, error} ->
72 |         {:error, error}
73 |     end
74 |   end
75 | end
76 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/controllers/error_html.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.ErrorHTML do
 2 |   use MemexWeb, :html
 3 |   # If you want to customize your error pages,
 4 |   # uncomment the embed_templates/1 call below
 5 |   # and add pages to the error directory:
 6 |   #
 7 |   #   * lib/my_app_web/controllers/error_html/404.html.heex
 8 |   #   * lib/my_app_web/controllers/error_html/500.html.heex
 9 |   #
10 |   # embed_templates "error_html/*"
11 |   # The default is to render a plain text page based on
12 |   # the template name. For example, "404.html" becomes
13 |   # "Not Found".
14 |   def render(template, _assigns) do
15 |     Phoenix.Controller.status_message_from_template(template)
16 |   end
17 | end
18 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/controllers/error_json.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.ErrorJSON do
 2 |   # If you want to customize a particular status code,
 3 |   # you may add your own clauses, such as:
 4 |   #
 5 |   # def render("500.json", _assigns) do
 6 |   #   %{errors: %{detail: "Internal Server Error"}}
 7 |   # end
 8 |   # By default, Phoenix returns the status message from
 9 |   # the template name. For example, "404.json" becomes
10 |   # "Not Found".
11 |   def render(template, _assigns) do
12 |     %{errors: %{detail: Phoenix.Controller.status_message_from_template(template)}}
13 |   end
14 | end
15 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/controllers/index_controller.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.IndexController do
 2 |   use MemexWeb, :controller
 3 | 
 4 |   alias Memex.Importer
 5 | 
 6 |   plug Plug.Parsers,
 7 |     parsers: [],
 8 |     pass: ["text/*"],
 9 |     body_reader: {}
10 | 
11 |   def upsert_documents(conn, %{"index" => _index} = params) do
12 |     with {:ok, documents} <- Importer.parse_body(params["_json"]),
13 |          {:ok} <- Importer.bulk_upsert_documents(documents) do
14 |       send_resp(conn, 200, "ok")
15 |     else
16 |       {:error, :no_data} -> send_resp(conn, 400, ~c"No content")
17 |       {:error, error} -> send_resp(conn, 400, error)
18 |     end
19 |   end
20 | end
21 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/controllers/photo_controller.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.PhotoController do
 2 |   use MemexWeb, :controller
 3 | 
 4 |   @photos_path "/Users/adrimbp/Pictures/Photos\ Library.photoslibrary/resources/derivatives/masters/"
 5 | 
 6 |   def image(conn, %{"path" => path} = _params) do
 7 |     new_path =
 8 |       @photos_path <>
 9 |         String.replace(path, [".jpeg", ".heic", ".mov", ".png"], "_4_5005_c.jpeg")
10 | 
11 |     case File.read(new_path) do
12 |       {:ok, contents} ->
13 |         conn
14 |         |> put_resp_content_type("image/jpeg", "utf-8")
15 |         |> send_resp(200, contents)
16 | 
17 |       {:error, _} ->
18 |         send_resp(conn, 404, "Not found")
19 |     end
20 |   end
21 | 
22 |   # Insecure but needed to avoid HTTPS on localhost
23 |   def https_proxy(conn, %{"url" => url} = _params) do
24 |     url
25 |     |> Tesla.get()
26 |     |> case do
27 |       {:ok, %{status: 200} = response} ->
28 |         conn
29 |         |> merge_resp_headers(
30 |           Enum.filter(response.headers, fn {key, _} ->
31 |             Enum.member?(["content-type", "content-length", "content-disposition", "ETag"], key)
32 |           end)
33 |         )
34 |         |> send_resp(200, response.body)
35 | 
36 |       _ ->
37 |         send_resp(conn, 404, "Not found")
38 |     end
39 |   end
40 | end
41 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/endpoint.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Endpoint do
 2 |   use Phoenix.Endpoint, otp_app: :memex
 3 | 
 4 |   # The session will be stored in the cookie and signed,
 5 |   # this means its contents can be read but not tampered with.
 6 |   # Set :encryption_salt if you would also like to encrypt it.
 7 |   @session_options [
 8 |     store: :cookie,
 9 |     key: "_memex_key",
10 |     signing_salt: "Yzxaa2KS"
11 |   ]
12 | 
13 |   socket "/socket", MemexWeb.UserSocket,
14 |     websocket: true,
15 |     longpoll: false
16 | 
17 |   socket "/live", Phoenix.LiveView.Socket, websocket: [connect_info: [session: @session_options]]
18 | 
19 |   # Serve at "/" the static files from "priv/static" directory.
20 |   #
21 |   # You should set gzip to true if you are running phx.digest
22 |   # when deploying your static files in production.
23 |   plug Plug.Static,
24 |     at: "/",
25 |     from: :memex,
26 |     gzip: false,
27 |     only: MemexWeb.static_paths()
28 | 
29 |   # Code reloading can be explicitly enabled under the
30 |   # :code_reloader configuration of your endpoint.
31 |   if code_reloading? do
32 |     socket "/phoenix/live_reload/socket", Phoenix.LiveReloader.Socket
33 |     plug Phoenix.LiveReloader
34 |     plug Phoenix.CodeReloader
35 |   end
36 | 
37 |   plug Plug.RequestId
38 |   plug Plug.Telemetry, event_prefix: [:phoenix, :endpoint]
39 | 
40 |   plug Plug.Parsers,
41 |     parsers: [:urlencoded, :multipart, :json],
42 |     pass: ["*/*"],
43 |     # Increase to 300MB max upload
44 |     length: 300_000_000,
45 |     json_decoder: Phoenix.json_library()
46 | 
47 |   plug Plug.MethodOverride
48 |   plug Plug.Head
49 |   plug Plug.Session, @session_options
50 |   plug MemexWeb.Router
51 | end
52 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/gettext.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Gettext do
 2 |   @moduledoc """
 3 |   A module providing Internationalization with a gettext-based API.
 4 | 
 5 |   By using [Gettext](https://hexdocs.pm/gettext),
 6 |   your module gains a set of macros for translations, for example:
 7 | 
 8 |       import MemexWeb.Gettext
 9 | 
10 |       # Simple translation
11 |       gettext("Here is the string to translate")
12 | 
13 |       # Plural translation
14 |       ngettext("Here is the string to translate",
15 |                "Here are the strings to translate",
16 |                3)
17 | 
18 |       # Domain-based translation
19 |       dgettext("errors", "Here is the error message to translate")
20 | 
21 |   See the [Gettext Docs](https://hexdocs.pm/gettext) for detailed usage.
22 |   """
23 |   use Gettext, otp_app: :memex
24 | end
25 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/live/page_live.ex:
--------------------------------------------------------------------------------
  1 | defmodule MemexWeb.PageLive do
  2 |   @moduledoc false
  3 |   use MemexWeb, :surface_live_view
  4 | 
  5 |   alias Memex.Search.LegacyQuery
  6 |   alias Memex.Search.Query
  7 |   alias Memex.Search.Sidebars
  8 |   alias MemexWeb.Components.Badge
  9 |   alias MemexWeb.Components.Icons.SettingsIcon
 10 |   alias MemexWeb.DatesFacet
 11 |   alias MemexWeb.SearchBar
 12 |   alias MemexWeb.SearchResultStats
 13 |   alias MemexWeb.SidebarsComponent
 14 |   alias MemexWeb.Timeline
 15 | 
 16 |   # todo: remove
 17 |   data(query, :string, default: "")
 18 |   data(filters, :list, default: [])
 19 |   data(items, :list, default: [])
 20 |   data(page, :number, default: 1)
 21 |   data(selected_index, :number, default: 0)
 22 |   data(dates, :list, default: [])
 23 |   data(total_hits, :number, default: 0)
 24 |   data(sidebars, :list, default: Sidebars.init())
 25 | 
 26 |   def mount(params, _session, socket) do
 27 |     Sidebars.subscribe()
 28 | 
 29 |     {:ok, assign(socket, debug: Map.has_key?(params, "debug"))}
 30 |   end
 31 | 
 32 |   @impl true
 33 |   def render(assigns) do
 34 |     ~F"""
 35 |     <div class="mx-auto mt-3">
 36 |       <SearchBar query={@query} />
 37 |       <div :if={@query == ""}>
 38 |         <Badge class="absolute right-2 top-6" click="open-sidebar" values={type: "settings"}>
 39 |           <:icon><SettingsIcon /></:icon>
 40 |         </Badge>
 41 |         <div class="flex items-start">
 42 |           <!-- <CloseCircles /> -->
 43 |           <!-- <Memex.Importers.Github.HomepageItem /> -->
 44 |           <div>
 45 |           </div>
 46 |         </div>
 47 |       </div>
 48 | 
 49 |       <div class="flex items-start">
 50 |         <div :if={@query != ""} class="w-4/5 mt-8">
 51 |           <SearchResultStats total_hits={@total_hits} />
 52 |           <Timeline
 53 |             query={@query}
 54 |             items={@items}
 55 |             selected_index={@selected_index}
 56 |             page={@page}
 57 |             debug={@debug}
 58 |             class="ml-12 md:ml-20"
 59 |             enable_load_more
 60 |           />
 61 |         </div>
 62 |         <div :if={@query != ""} class="w-1/5 overflow-hidden pl-5 text-white">
 63 |           <DatesFacet dates={@dates} loading={false} />
 64 |         </div>
 65 |         <SidebarsComponent sidebars={@sidebars} socket={assigns.socket} />
 66 |       </div>
 67 |     </div>
 68 |     """
 69 |   end
 70 | 
 71 |   def handle_event("search", %{"query" => ""}, socket) do
 72 |     {:noreply, assign(socket, query: "", filters: [], page: 1, selected_index: 0, items: [])}
 73 |   end
 74 | 
 75 |   def handle_event("search", %{"query" => query, "filters" => filters}, socket) do
 76 |     {:noreply, socket |> assign(query: query, filters: filters, selected_index: 0, page: 1) |> search()}
 77 |   end
 78 | 
 79 |   def handle_event("completion", %{"sources" => sources, "key" => key, "value" => value}, socket) do
 80 |     options =
 81 |       key
 82 |       |> Memex.Schema.Catalog.search(value, sources)
 83 |       |> Enum.map(fn %{"type" => type, "value" => value} ->
 84 |         %{
 85 |           "type" => type,
 86 |           "label" => value
 87 |         }
 88 |       end)
 89 | 
 90 |     {:reply, %{"options" => options}, socket}
 91 |   end
 92 | 
 93 |   def handle_event("load-more", _, %{assigns: assigns} = socket) do
 94 |     {:noreply, socket |> assign(page: assigns.page + 1) |> search()}
 95 |   end
 96 | 
 97 |   def handle_event("key-pressed", %{"key" => "Enter"}, %{assigns: assigns} = socket) do
 98 |     selected_item = Enum.at(assigns.items, assigns.selected_index)
 99 | 
100 |     cond do
101 |       selected_item == nil && assigns.total_hits == 0 ->
102 |         {:noreply, redirect(socket, external: "https://duckduckgo.com/?q=#{assigns.query}")}
103 | 
104 |       %{"website_url" => url} = selected_item ->
105 |         {:reply, %{}, redirect(socket, external: url)}
106 | 
107 |       true ->
108 |         {:noreply, socket}
109 |     end
110 |   end
111 | 
112 |   def handle_event("key-pressed", %{"key" => "ArrowDown"}, %{assigns: assigns} = socket) do
113 |     {:noreply, assign(socket, selected_index: min(assigns.selected_index + 1, length(assigns.items)))}
114 |   end
115 | 
116 |   def handle_event("key-pressed", %{"key" => "ArrowUp"}, %{assigns: assigns} = socket) do
117 |     {:noreply, assign(socket, selected_index: max(assigns.selected_index - 1, 0))}
118 |   end
119 | 
120 |   def handle_event("key-pressed", _, socket), do: {:noreply, socket}
121 | 
122 |   @impl true
123 |   def handle_event("open-sidebar", data, %{assigns: %{sidebars: sidebars}} = socket) do
124 |     {:noreply, assign(socket, sidebars: Sidebars.open(sidebars, data))}
125 |   end
126 | 
127 |   @impl true
128 |   def handle_event("close-last-sidebar", _key, %{assigns: %{sidebars: sidebars}} = socket) do
129 |     {:noreply, assign(socket, sidebars: Sidebars.close_last(sidebars))}
130 |   end
131 | 
132 |   defp search(%{assigns: %{page: page, filters: filters}} = socket) do
133 |     query = Query.from_filters(filters)
134 | 
135 |     socket
136 |     |> async_query(:items, [], %{
137 |       query
138 |       | select: :hits_with_highlights,
139 |         order_by: ["created_at_desc"],
140 |         limit: 10 * page
141 |     })
142 |     |> async_query(:dates, [], %{Query.remove_filter(query, "month") | select: [facet: "month"]})
143 |     |> async_query(:total_hits, nil, %{query | select: :total_hits})
144 |   end
145 | 
146 |   def handle_info(%{event: event, payload: payload}, socket) do
147 |     handle_event(event, payload, socket)
148 |   end
149 | end
150 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/live/sidebars/activity_live.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Sidebars.ActivityLive do
 2 |   @moduledoc false
 3 |   use MemexWeb, :surface_live_view
 4 | 
 5 |   alias Memex.Search.LegacyQuery, as: Query
 6 |   alias Memex.Search.Postgres
 7 |   alias MemexWeb.Map
 8 |   alias MemexWeb.TimeDuration
 9 |   alias MemexWeb.Timeline
10 |   alias MemexWeb.Timeline.Card
11 | 
12 |   def mount(params, session, socket) do
13 |     {:ok, doc} = Postgres.find(session["id"] || params["id"])
14 |     {:ok, previous} = Postgres.find(doc["id_previous"])
15 |     {:ok, next} = Postgres.find(doc["id_next"])
16 |     # doc |> IO.inspect(label: "15")
17 |     # previous |> IO.inspect(label: "16")
18 |     # next |> IO.inspect(label: "17")
19 | 
20 |     {:ok,
21 |      socket
22 |      |> assign(doc: doc, next: next, previous: previous)
23 |      |> async_query(:items, [], %Query{
24 |        limit: 20,
25 |        filters: %{"created_at_between" => [doc["timestamp_start_utc"], doc["timestamp_utc"]]},
26 |        order_by: ["created_at_desc"]
27 |      })}
28 |   end
29 | 
30 |   @impl true
31 |   def render(assigns) do
32 |     ~F"""
33 |     <div class="inline-flex items-center space-x-3 mb-2">
34 |       <!-- todo: Icon for activity type -->
35 |       <h2 class="text-xl flex-grow dark:text-white leading-7">
36 |         {String.capitalize(@doc["activity_type"] || "")}
37 |         {#unless is_nil(@previous["place_name"] || @previous["place_address"])}from <b>{@previous["place_name"] || @previous["place_address"]}</b>{/unless}
38 |         {#unless is_nil(@next["place_name"])}
39 |           to <b>{@next["place_name"]}</b>{/unless}
40 |       </h2>
41 |     </div>
42 | 
43 |     <Map
44 |       height={250}
45 |       items={@items}
46 |       geojson_path={Routes.arc_path(MemexWeb.Endpoint, :geojson,
47 |         date: String.slice(@doc["timestamp_utc"], 0..9),
48 |         id: @doc["id"]
49 |       )}
50 |     />
51 | 
52 |     <div class="flex w-full space-x-3">
53 |       <div class="w-8/12">
54 |         <Timeline items={@items} />
55 |       </div>
56 |       <div class="">
57 |         <TimeDuration start_time={@doc["timestamp_start_unix"]} end_time={@doc["timestamp_unix"]} />
58 |         <Card>
59 |           <:content>
60 |             Distance: xx<br>
61 |             Most common speed: xx km/h<br>
62 |             Steps: {@doc["activity_step_count"]}<br>
63 |             Flights climbed: {@doc["activity_floors_ascended"]} up, {@doc["activity_floors_descended"]} down<br>
64 |             Altitude: xx meters<br>
65 |           </:content>
66 |         </Card>
67 |         <!-- Chart Heartrate -->
68 |         <Card>
69 |           <:content>
70 |             Heartrate<br>
71 |             Average: {@doc["activity_heart_rate_average"]}<br>
72 |             Max: {@doc["activity_heart_rate_max"]}<br>
73 |           </:content>
74 |         </Card>
75 | 
76 |         <!-- Chart Trip speed -->
77 |       </div>
78 |     </div>
79 |     """
80 |   end
81 | end
82 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/live/sidebars/generic_live.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Sidebars.GenericLive do
 2 |   @moduledoc false
 3 |   use MemexWeb, :surface_live_view
 4 | 
 5 |   alias Memex.Search.LegacyQuery, as: Query
 6 |   alias Memex.Search.Postgres
 7 |   alias MemexWeb.Timeline
 8 | 
 9 |   def mount(params, session, socket) do
10 |     {:ok, doc} = Postgres.find(session["id"] || params["id"])
11 | 
12 |     within =
13 |       Postgres.query(%Query{
14 |         select: :hits_with_highlights,
15 |         limit: 3,
16 |         filters: %{"created_at_within" => DateTime.from_unix!(doc["timestamp_unix"])}
17 |       })
18 | 
19 |     {
20 |       :ok,
21 |       assign(socket, doc: doc, within: within)
22 |       #  |> async_query(:items, [], %Query{
23 |       #    limit: 20,
24 |       #    filters: %{"created_at_betweeenc" => doc["timestamp_utc"]},
25 |       #    order_by: ["created_at_desc"]
26 |       #  })
27 |     }
28 |   end
29 | 
30 |   @impl true
31 |   def render(assigns) do
32 |     ~F"""
33 |     <div class="text-white">Generic</div>
34 |     <Timeline :if={@within} items={@within ++ [@doc]} />
35 |     """
36 |   end
37 | end
38 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/live/sidebars/person_live.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Sidebars.PersonLive do
 2 |   @moduledoc false
 3 |   use MemexWeb, :live_view
 4 | 
 5 |   alias Memex.Search.LegacyQuery, as: Query
 6 | 
 7 |   @impl true
 8 |   def mount(params, session, socket) do
 9 |     hit = session["hit"]
10 |     name = hit["name"] || params["name"]
11 | 
12 |     {:ok,
13 |      socket
14 |      |> assign(name: name, hit: hit, last: [])
15 |      |> async_query(:last, [], %Query{
16 |        filters: %{"person_name" => name},
17 |        order_by: ["created_at_desc"],
18 |        limit: 10
19 |      })
20 |      |> async_query(:links, [], %Query{
21 |        query: "http",
22 |        filters: %{"person_name" => name},
23 |        order_by: ["created_at_desc"],
24 |        limit: 10
25 |      })
26 |      |> async_query(:photos, [], %Query{
27 |        filters: %{"provider" => "Photos", "person_name" => name},
28 |        order_by: ["created_at_desc"],
29 |        limit: 3
30 |      })
31 |      |> async_query(:counts, [], %Query{
32 |        select: [facet: "provider"],
33 |        filters: %{"person_name" => name}
34 |      })}
35 |   end
36 | 
37 |   @impl true
38 |   def render(assigns) do
39 |     ~L"""
40 |     <div class="flex place-items-start justify-between">
41 |       <h2 class="text-lg flex-grow dark:text-white leading-7"><%= @name %></h2>
42 |     </div>
43 | 
44 |     <div class="flex space-x-4">
45 |       <%= for {provider, count} <- @counts do %>
46 |         <div class="flex-1 rounded-md bg-white dark:bg-gray-900 hover:border-blue-100 transition-colors p-4 my-2 shadow-md overflow-hidden dark:text-white">
47 |           <span class="float-right rounded-full bg-gray-700 px-2 py-1 text-xs"><%= MemexWeb.TimelineView.number_short(count) %></span>
48 |           <p class="dark:text-white truncate"><%= provider %></p>
49 |           <%= if provider === "Photos" do %>
50 |             <div class="flex space-x-2 mt-2">
51 |             <%= for hit <- @photos do %>
52 |                 <img class="object-cover w-10 h-10 rounded inline-block" width="60" height="60" src="<%= Routes.photo_path(MemexWeb.Endpoint, :image, hit["photo_file_path"]) %>" />
53 |             <% end %>
54 |             </div>
55 |           <% end %>
56 |         </div>
57 |       <% end %>
58 |     </div>
59 | 
60 |     <div class="flex flex-col justify-between gap-4">
61 |         <h3 class="dark:text-white">Timeline</h3>
62 |         <%= for hit <- @last do %>
63 |           <div class="rounded-md bg-white dark:bg-gray-900 hover:border-blue-100 transition-colors p-4 shadow-md overflow-hidden dark:text-white">
64 |             <p class="dark:text-white truncate"><%= hit["message_text"] %></p>
65 |           </div>
66 |         <% end %>
67 | 
68 |         <h3 class="dark:text-white">Links</h3>
69 |         <%= for hit <- @links do %>
70 |           <div class="rounded-md bg-white dark:bg-gray-900 hover:border-blue-100 transition-colors p-4 shadow-md overflow-hidden dark:text-white">
71 |             <span class="dark:text-white truncate block"><%= hit["message_text"] %></span>
72 |           </div>
73 |         <% end %>
74 |     </div>
75 |     """
76 |   end
77 | end
78 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/live/sidebars/setttings_live.ex:
--------------------------------------------------------------------------------
  1 | defmodule MemexWeb.Sidebars.SettingsLive do
  2 |   @moduledoc false
  3 |   use MemexWeb, :surface_live_view
  4 | 
  5 |   alias Memex.Importer
  6 |   alias Memex.Schema.ImporterLog
  7 |   alias Memex.Search.LegacyQuery, as: Query
  8 |   alias Memex.Search.Sidebars
  9 |   alias MemexWeb.Components.Badge
 10 |   alias MemexWeb.Components.Icon
 11 |   alias MemexWeb.Components.Text
 12 |   alias MemexWeb.Timeline.Card
 13 | 
 14 |   def mount(_params, _session, socket) do
 15 |     if connected?(socket) do
 16 |       Importer.subscribe()
 17 |     end
 18 | 
 19 |     {:ok, fetch_data(socket)}
 20 |   end
 21 | 
 22 |   def fetch_data(socket) do
 23 |     importers = Importer.configured_importers()
 24 | 
 25 |     socket =
 26 |       Enum.reduce(importers, socket, fn importer, socket ->
 27 |         query = Query.from_string("provider:#{importer.provider}")
 28 | 
 29 |         async_query(socket, "total_hits_#{importer.id}", nil, %{query | select: :total_hits})
 30 |       end)
 31 | 
 32 |     assign(socket, importers: importers, logs: ImporterLog.last_imports())
 33 |   end
 34 | 
 35 |   @impl true
 36 |   def render(assigns) do
 37 |     ~F"""
 38 |     <div>
 39 |       <Text.H1>Settings</Text.H1>
 40 |       <Text.SubtTitle>Configure integrations and other settings.</Text.SubtTitle>
 41 | 
 42 |       <Text.H2>Importers</Text.H2>
 43 |       <Badge class="text-white bg-gray-900 float-right" click="open-sidebar" values={type: "settings"}>
 44 |         <:icon><Icon.Plus /></:icon>
 45 |         Add new
 46 |       </Badge>
 47 |       <Text.SubtTitle>Configure importers and what can be searched for.</Text.SubtTitle>
 48 | 
 49 |       <Card :for={importer <- @importers}>
 50 |         <:content>
 51 |           {importer.display_name}
 52 |           <p :if={@logs[importer.id]} class="text-xs dark:text-gray-500">{state_to_emoij(@logs[importer.id]["state"])} Last import at {@logs[importer.id]["inserted_at"]}, took {Postgrex.Interval.to_string(@logs[importer.id]["duration"])}
 53 |             <code :if={@logs[importer.id]["log"] != "[]"} class="text-xs dark:text-gray-500">{@logs[importer.id]["log"]}</code>
 54 |             <!-- imported document counts or barchards per week? -->
 55 |             <!-- average duration -->
 56 |             <!-- errors -->
 57 |             <!-- view documents [open another sidebar] -->
 58 |           </p>
 59 |         </:content>
 60 |         <:right>
 61 |           <button :if={assigns["total_hits_#{importer.id}"]}>
 62 |             {MemexWeb.TimelineView.number_short(assigns[String.to_atom("total_hits_#{importer.id}")])} docs
 63 |           </button>
 64 |           <Badge click="import" values={id: importer.id}>
 65 |             <:icon><Icon.Plus /></:icon>
 66 |             Import
 67 |           </Badge>
 68 |         </:right>
 69 |       </Card>
 70 |     </div>
 71 |     """
 72 |   end
 73 | 
 74 |   defp state_to_emoij(state) do
 75 |     case state do
 76 |       "success" -> "✅"
 77 |       "error" -> "❌"
 78 |       "running" -> "⏳"
 79 |       _ -> "❓"
 80 |     end
 81 |   end
 82 | 
 83 |   @impl true
 84 |   def handle_event("open-sidebar", sidebar, socket) do
 85 |     Sidebars.broadcast_open(sidebar)
 86 |     {:noreply, socket}
 87 |   end
 88 | 
 89 |   @impl true
 90 |   def handle_event("toggle_importer", _, socket) do
 91 |     {:noreply, socket}
 92 |   end
 93 | 
 94 |   @impl true
 95 |   def handle_event("import", %{"id" => id}, socket) do
 96 |     config = Enum.find(socket.assigns.importers, &(&1.id == id))
 97 |     Importer.import(config)
 98 | 
 99 |     {:noreply, socket}
100 |   end
101 | 
102 |   def handle_info({Importer, event}, socket) when not is_nil(event) do
103 |     {:noreply, fetch_data(socket)}
104 |   end
105 | 
106 |   def handle_info({Importer, _}, socket), do: {:noreply, socket}
107 | end
108 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/router.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Router do
 2 |   use MemexWeb, :router
 3 | 
 4 |   pipeline :browser do
 5 |     plug :accepts, ["html"]
 6 |     plug :fetch_session
 7 |     plug :fetch_live_flash
 8 |     plug :put_root_layout, {MemexWeb.Layouts, :root}
 9 |     plug :protect_from_forgery
10 |     plug :put_secure_browser_headers
11 |   end
12 | 
13 |   pipeline :api do
14 |     plug :accepts, ["json"]
15 |   end
16 | 
17 |   scope "/", MemexWeb do
18 |     pipe_through :browser
19 | 
20 |     live "/", PageLive, :index
21 |     get "/photos/:path", PhotoController, :image
22 |     get "/https_proxy/", PhotoController, :https_proxy
23 |     get "/arc/geopoints", ArcController, :geopoint
24 |     get "/arc/gpx", ArcController, :gpx
25 |     get "/arc/geojson", ArcController, :geojson
26 |     get "/alfred/search", AlfredController, :search
27 |   end
28 | 
29 |   scope "/sidebar", MemexWeb do
30 |     pipe_through :browser
31 |     live "/activity", Sidebars.ActivityLive, :index
32 |     live "/person", Sidebars.PersonLive, :index
33 |     live "/settings", Sidebars.SettingsLive, :index
34 |     live "/generic", Sidebars.GenericLive, :index
35 |   end
36 | 
37 |   # Other scopes may use custom stacks.
38 |   scope "/", MemexWeb do
39 |     pipe_through :api
40 |     post "/indexes/:index/documents", IndexController, :upsert_documents
41 |   end
42 | end
43 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/telemetry.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.Telemetry do
 2 |   @moduledoc false
 3 |   use Supervisor
 4 | 
 5 |   import Telemetry.Metrics
 6 | 
 7 |   def start_link(arg) do
 8 |     Supervisor.start_link(__MODULE__, arg, name: __MODULE__)
 9 |   end
10 | 
11 |   @impl true
12 |   def init(_arg) do
13 |     children = [
14 |       # Telemetry poller will execute the given period measurements
15 |       # every 10_000ms. Learn more here: https://hexdocs.pm/telemetry_metrics
16 |       {:telemetry_poller, measurements: periodic_measurements(), period: 10_000}
17 |       # Add reporters as children of your supervision tree.
18 |       # {Telemetry.Metrics.ConsoleReporter, metrics: metrics()}
19 |     ]
20 | 
21 |     Supervisor.init(children, strategy: :one_for_one)
22 |   end
23 | 
24 |   def metrics do
25 |     [
26 |       # Tesla Metrics
27 |       summary("tesla.request.stop.duration",
28 |         unit: {:native, :millisecond}
29 |       ),
30 |       counter("tesla.request.stop.duration"),
31 | 
32 |       # Phoenix Metrics
33 |       summary("phoenix.endpoint.stop.duration",
34 |         unit: {:native, :millisecond}
35 |       ),
36 |       summary("phoenix.router_dispatch.stop.duration",
37 |         tags: [:route],
38 |         unit: {:native, :millisecond}
39 |       ),
40 | 
41 |       # VM Metrics
42 |       summary("vm.memory.total", unit: {:byte, :kilobyte}),
43 |       summary("vm.total_run_queue_lengths.total"),
44 |       summary("vm.total_run_queue_lengths.cpu"),
45 |       summary("vm.total_run_queue_lengths.io")
46 |     ]
47 |   end
48 | 
49 |   defp periodic_measurements do
50 |     [
51 |       # A module, function and arguments to be invoked periodically.
52 |       # This function must call :telemetry.execute/3 and a metric must be added above.
53 |       # {MemexWeb, :count_users, []}
54 |     ]
55 |   end
56 | end
57 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/views/error_helpers.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.ErrorHelpers do
 2 |   @moduledoc """
 3 |   Conveniences for translating and building error messages.
 4 |   """
 5 | 
 6 |   use Phoenix.HTML
 7 | 
 8 |   @doc """
 9 |   Generates tag for inlined form input errors.
10 |   """
11 |   def error_tag(form, field) do
12 |     Enum.map(Keyword.get_values(form.errors, field), fn error ->
13 |       content_tag(:span, translate_error(error),
14 |         class: "invalid-feedback",
15 |         phx_feedback_for: input_id(form, field)
16 |       )
17 |     end)
18 |   end
19 | 
20 |   @doc """
21 |   Translates an error message using gettext.
22 |   """
23 |   def translate_error({msg, opts}) do
24 |     # When using gettext, we typically pass the strings we want
25 |     # to translate as a static argument:
26 |     #
27 |     #     # Translate "is invalid" in the "errors" domain
28 |     #     dgettext("errors", "is invalid")
29 |     #
30 |     #     # Translate the number of files with plural rules
31 |     #     dngettext("errors", "1 file", "%{count} files", count)
32 |     #
33 |     # Because the error messages we show in our forms and APIs
34 |     # are defined inside Ecto, we need to translate them dynamically.
35 |     # This requires us to call the Gettext module passing our gettext
36 |     # backend as first argument.
37 |     #
38 |     # Note we use the "errors" domain, which means translations
39 |     # should be written to the errors.po file. The :count option is
40 |     # set by Ecto and indicates we should also apply plural rules.
41 |     if count = opts[:count] do
42 |       Gettext.dngettext(MemexWeb.Gettext, "errors", msg, msg, count, opts)
43 |     else
44 |       Gettext.dgettext(MemexWeb.Gettext, "errors", msg, opts)
45 |     end
46 |   end
47 | end
48 | 


--------------------------------------------------------------------------------
/memex/lib/memex_web/views/timeline_view.ex:
--------------------------------------------------------------------------------
  1 | defmodule MemexWeb.TimelineView do
  2 |   @youtube_url_regex ~r/(?:youtube\.com\/(?:[^\/]+\/.+\/|(?:v|e(?:mbed)?)\/|.*[?&]v=)|youtu\.be\/)([^"&?\/\s]{11})/i
  3 |   @url_regex ~r/https\:\/\/[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(\/\S*)?/i
  4 | 
  5 |   def is_youtube_url(nil), do: false
  6 |   def is_youtube_url(url), do: Regex.match?(@youtube_url_regex, url)
  7 | 
  8 |   def parse_youtube_id(url), do: Enum.at(Regex.run(@youtube_url_regex, url), 1)
  9 | 
 10 |   def auto_link(text) do
 11 |     Regex.replace(@url_regex, text, "<a href=\"\\0\" target=\"_blank\">\\0</a>")
 12 |   end
 13 | 
 14 |   def days_between(timestamp1, timestamp2) do
 15 |     timestamp2
 16 |     |> date_from_timestamp()
 17 |     |> Date.diff(date_from_timestamp(timestamp1))
 18 |     |> max(0)
 19 |   end
 20 | 
 21 |   def time_between(timestamp1, timestamp2) do
 22 |     DateTime.diff(date_from_timestamp(timestamp2), date_from_timestamp(timestamp1))
 23 |   end
 24 | 
 25 |   def timestamp_start_between(results, time_range) do
 26 |     Enum.filter(results, fn hit ->
 27 |       hit["timestamp_start_unix"] && Enum.member?(time_range, hit["timestamp_start_unix"])
 28 |     end)
 29 |   end
 30 | 
 31 |   def count_results_between(results, timestamp) do
 32 |     Enum.count(results, fn hit ->
 33 |       hit["timestamp_start_unix"] && Enum.member?(hit["timestamp_start_unix"]..hit["timestamp_unix"], timestamp)
 34 |     end)
 35 |   end
 36 | 
 37 |   def timeline_classes(results, hit, index) do
 38 |     previous_hit = Enum.at(results, index - 1)["timestamp_unix"]
 39 |     previous_results_between = MemexWeb.TimelineView.count_results_between(results, previous_hit)
 40 |     results_between = MemexWeb.TimelineView.count_results_between(results, hit["timestamp_unix"])
 41 | 
 42 |     timeline_classes =
 43 |       if results_between > previous_results_between do
 44 |         " rounded-t"
 45 |       else
 46 |         ""
 47 |       end
 48 | 
 49 |     previous_timeline_classes =
 50 |       case previous_results_between do
 51 |         0 -> "bg-gray-400 dark:bg-gray-700 " <> timeline_classes
 52 |         1 -> "bg-gray-500 dark:bg-gray-500 " <> timeline_classes
 53 |         2 -> "bg-gray-600 dark:bg-gray-300 " <> timeline_classes
 54 |         _ -> "bg-gray-700 dark:bg-gray-100 " <> timeline_classes
 55 |       end
 56 | 
 57 |     timeline_classes =
 58 |       case results_between do
 59 |         0 -> "bg-gray-400 dark:bg-gray-700 " <> timeline_classes
 60 |         1 -> "bg-gray-500 dark:bg-gray-500 " <> timeline_classes
 61 |         2 -> "bg-gray-600 dark:bg-gray-300 " <> timeline_classes
 62 |         _ -> "bg-gray-700 dark:bg-gray-100 " <> timeline_classes
 63 |       end
 64 | 
 65 |     {previous_timeline_classes, timeline_classes}
 66 |   end
 67 | 
 68 |   def human_time_between(timestamp1, timestamp2) do
 69 |     diff = abs(time_between(timestamp1, timestamp2))
 70 | 
 71 |     cond do
 72 |       diff < 60 ->
 73 |         "less than a minute"
 74 | 
 75 |       diff > 60 && diff < 120 ->
 76 |         "1 minute"
 77 | 
 78 |       diff >= 120 && diff < 3600 ->
 79 |         "#{div(diff, 60)} minutes"
 80 | 
 81 |       diff >= 3600 && diff < 7200 ->
 82 |         min = diff |> rem(3600) |> div(60)
 83 |         "a 1 hour and #{min} minutes"
 84 | 
 85 |       diff >= 7200 ->
 86 |         min = diff |> rem(3600) |> div(60)
 87 |         "#{div(diff, 3600)} hours and #{min} minutes"
 88 |     end
 89 |   end
 90 | 
 91 |   def icon_by_provider(provider) do
 92 |     case provider do
 93 |       "Safari" ->
 94 |         "/images/safari-small.png"
 95 | 
 96 |       "MoneyMoney" ->
 97 |         "/images/MoneyMoney-small.png"
 98 | 
 99 |       "iMessage" ->
100 |         "/images/iMessage-small.png"
101 | 
102 |       "GitHub" ->
103 |         "/images/GitHub-small.png"
104 | 
105 |       "Photos" ->
106 |         "/images/Photos-small.png"
107 | 
108 |       "terminal" ->
109 |         "/images/terminal-small.png"
110 | 
111 |       "Twitter" ->
112 |         "/images/Twitter-small.png"
113 | 
114 |       "git-notes" ->
115 |         "/images/GitHub-small.png"
116 | 
117 |       "Arc" ->
118 |         "/images/Arc-small.png"
119 | 
120 |       "Podcasts" ->
121 |         "/images/Podcasts-small.png"
122 | 
123 |       _ ->
124 |         nil
125 |     end
126 |   end
127 | 
128 |   def date_from_timestamp(timestamp) do
129 |     cond_result =
130 |       cond do
131 |         is_integer(timestamp) ->
132 |           timestamp
133 | 
134 |         is_float(timestamp) ->
135 |           round(timestamp)
136 | 
137 |         true ->
138 |           String.to_integer(timestamp)
139 |       end
140 | 
141 |     DateTime.from_unix!(cond_result)
142 |   end
143 | 
144 |   def file_header(patch, status) do
145 |     from = patch.from
146 |     to = patch.to
147 | 
148 |     case status do
149 |       "changed" -> from
150 |       "renamed" -> "#{from} -> #{to}"
151 |       "removed" -> from
152 |       "added" -> to
153 |     end
154 |   end
155 | 
156 |   def patch_status(patch) do
157 |     from = patch.from
158 |     to = patch.to
159 | 
160 |     cond do
161 |       !from -> "added"
162 |       !to -> "removed"
163 |       from == to -> "changed"
164 |       true -> "renamed"
165 |     end
166 |   end
167 | 
168 |   def nl2br(nil), do: ""
169 | 
170 |   def nl2br(text) do
171 |     text
172 |     |> String.trim()
173 |     |> String.replace("\n", "<br />")
174 |   end
175 | 
176 |   def line_type(line), do: to_string(line.type)
177 | 
178 |   def line_text("+" <> text), do: "" <> nl2br(text)
179 |   def line_text("-" <> text), do: "" <> nl2br(text)
180 |   def line_text(text), do: " " <> nl2br(text)
181 | 
182 |   @highlight_regex ~r/<em>(.*)<\/em>/u
183 |   def highlight_line_text(text, _highlight_text) when is_nil(text), do: line_text(text)
184 | 
185 |   def highlight_line_text(text, highlight_text) do
186 |     with [_, highlights] <- Regex.run(@highlight_regex, highlight_text) do
187 |       text
188 |       |> String.replace(highlights, "<em>#{highlights}</em>")
189 |       |> line_text()
190 |     else
191 |       _e -> line_text(text)
192 |     end
193 |   end
194 | 
195 |   def number_short(number) do
196 |     cond do
197 |       number < 1000 ->
198 |         number
199 | 
200 |       number < 1_000_000 ->
201 |         "#{Float.round(number / 1000, 1)}K"
202 | 
203 |       number < 1_000_000_000 ->
204 |         "#{Float.round(number / 1_000_000, 1)}M"
205 | 
206 |       true ->
207 |         number
208 |     end
209 |   end
210 | 
211 |   def number_to_currency(_number, nil), do: nil
212 |   def number_to_currency(nil, _options), do: nil
213 | 
214 |   def number_to_currency(number, currency) do
215 |     {:ok, money} = Money.parse(number, currency)
216 | 
217 |     Money.to_string(money, symbol: true)
218 |   end
219 | end
220 | 


--------------------------------------------------------------------------------
/memex/mix.exs:
--------------------------------------------------------------------------------
 1 | defmodule Memex.MixProject do
 2 |   use Mix.Project
 3 | 
 4 |   def project do
 5 |     [
 6 |       app: :memex,
 7 |       version: "0.1.0",
 8 |       elixir: "~> 1.14",
 9 |       elixirc_paths: elixirc_paths(Mix.env()),
10 |       start_permanent: Mix.env() == :prod,
11 |       aliases: aliases(),
12 |       deps: deps()
13 |     ]
14 |   end
15 | 
16 |   # Configuration for the OTP application.
17 |   #
18 |   # Type `mix help compile.app` for more information.
19 |   def application do
20 |     [
21 |       mod: {Memex.Application, []},
22 |       extra_applications: [:logger, :con_cache, :runtime_tools]
23 |     ]
24 |   end
25 | 
26 |   # Specifies which paths to compile per environment.
27 |   defp elixirc_paths(:test), do: ["lib", "test/support"]
28 |   defp elixirc_paths(:dev), do: ["lib"]
29 |   defp elixirc_paths(_), do: ["lib"]
30 | 
31 |   # Specifies your project dependencies.
32 |   #
33 |   # Type `mix help deps` for examples and options.
34 |   defp deps do
35 |     [
36 |       {:bandit, "~> 1.0-pre"},
37 |       {:con_cache, "~> 0.13"},
38 |       {:credo, "~> 1.7", only: [:dev, :test], runtime: false},
39 |       {:earmark, "~> 1.4.29"},
40 |       {:cloak_ecto, "~> 1.2.0"},
41 |       {:ecto_sql, "~> 3.10"},
42 |       {:esbuild, "~> 0.5", runtime: Mix.env() == :dev},
43 |       {:ex_check, "~> 0.13.0", only: [:dev], runtime: false},
44 |       {:exqlite, "~> 0.19.0"},
45 |       {:file_system, "~> 0.2"},
46 |       {:finch, "~> 0.16.0"},
47 |       {:floki, ">= 0.27.0", only: :test},
48 |       {:gettext, "~> 0.20"},
49 |       {:git_diff, "~> 0.6.2"},
50 |       {:hackney, "~> 1.20.1"},
51 |       {:heroicons, "~> 0.5.3"},
52 |       {:jason, "~> 1.0"},
53 |       {:money, "~> 1.9"},
54 |       {:month, "~> 2.1"},
55 |       {:phoenix_ecto, "~> 4.4"},
56 |       {:phoenix_html, "~> 3.0"},
57 |       {:phoenix_live_reload, "~> 1.3", only: :dev},
58 |       {:phoenix_live_view, "~> 0.19.3"},
59 |       {:phoenix_view, "~> 2.0"},
60 |       {:phoenix, "~> 1.7.5"},
61 |       {:plug_cowboy, "~> 2.0"},
62 |       {:postgrex, "~> 0.17.1"},
63 |       {:surface, "~> 0.10"},
64 |       {:surface_formatter, "~> 0.7.5", only: :dev},
65 |       {:styler, "~> 0.7", only: [:dev, :test], runtime: false},
66 |       {:tailwind, "~> 0.1.8", runtime: Mix.env() == :dev},
67 |       {:telemetry_metrics, "~> 0.6"},
68 |       {:telemetry_poller, "~> 1.0.0"},
69 |       {:tesla, "~> 1.7.0"},
70 |       {:tzdata, "~> 1.0"}
71 |     ]
72 |   end
73 | 
74 |   # Aliases are shortcuts or tasks specific to the current project.
75 |   # For example, to install project dependencies and perform other setup tasks, run:
76 |   #
77 |   #     $ mix setup
78 |   #
79 |   # See the documentation for `Mix` for more info on aliases.
80 |   defp aliases do
81 |     [
82 |       setup: ["deps.get", "ecto.setup", "assets.setup", "assets.build"],
83 |       "ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"],
84 |       "ecto.reset": ["ecto.drop", "ecto.setup"],
85 |       test: ["ecto.create --quiet", "ecto.migrate --quiet", "test"],
86 |       "assets.setup": ["tailwind.install --if-missing", "esbuild.install --if-missing"],
87 |       "assets.build": ["tailwind default", "esbuild default"],
88 |       "assets.deploy": [
89 |         "cmd --cd assets npm run deploy",
90 |         "tailwind default --minify",
91 |         "esbuild default --minify",
92 |         "phx.digest"
93 |       ]
94 |     ]
95 |   end
96 | end
97 | 


--------------------------------------------------------------------------------
/memex/priv/gettext/en/LC_MESSAGES/errors.po:
--------------------------------------------------------------------------------
 1 | ## `msgid`s in this file come from POT (.pot) files.
 2 | ##
 3 | ## Do not add, change, or remove `msgid`s manually here as
 4 | ## they're tied to the ones in the corresponding POT file
 5 | ## (with the same domain).
 6 | ##
 7 | ## Use `mix gettext.extract --merge` or `mix gettext.merge`
 8 | ## to merge POT files into PO files.
 9 | msgid ""
10 | msgstr ""
11 | "Language: en\n"
12 | 


--------------------------------------------------------------------------------
/memex/priv/gettext/errors.pot:
--------------------------------------------------------------------------------
 1 | ## This is a PO Template file.
 2 | ##
 3 | ## `msgid`s here are often extracted from source code.
 4 | ## Add new translations manually only if they're dynamic
 5 | ## translations that can't be statically extracted.
 6 | ##
 7 | ## Run `mix gettext.extract` to bring this file up to
 8 | ## date. Leave `msgstr`s empty as changing them here has no
 9 | ## effect: edit them in PO (`.po`) files instead.
10 | 
11 | 


--------------------------------------------------------------------------------
/memex/priv/repo/migrations/20210619105629_add_documents.exs:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Repo.Migrations.AddDocuments do
 2 |   use Ecto.Migration
 3 | 
 4 |   def change do
 5 |     execute """
 6 |     CREATE TABLE "public"."documents" (
 7 |       "id" varchar(255) GENERATED ALWAYS AS (body ->> 'id') stored,
 8 |       "body" jsonb NOT NULL,
 9 |       "created_at" timestamptz GENERATED ALWAYS AS (to_timestamp((body ->> 'timestamp_unix')::int)) stored,
10 |       "updated_at" timestamptz NOT NULL DEFAULT now(),
11 |       "search" tsvector GENERATED ALWAYS AS (
12 |         to_tsvector('simple',
13 |                     COALESCE(body ->> 'place_name', '')
14 |           || ' ' || COALESCE(body ->> 'verb', '')
15 |           || ' ' || COALESCE(body ->> 'provider', '')
16 |           || ' ' || COALESCE(body ->> 'place_name', '')
17 |           || ' ' || COALESCE(body ->> 'place_address', '')
18 |           || ' ' || COALESCE(translate(body ->> 'website_url', '@./', '  '), '')
19 |           || ' ' || COALESCE(body ->> 'website_title', '')
20 |           || ' ' || COALESCE(body ->> 'device_name', '')
21 |           || ' ' || COALESCE(body ->> 'command', '')
22 |           || ' ' || COALESCE(translate(body ->> 'message_text', '@./', '  '), '')
23 |           || ' ' || COALESCE(body ->> 'message_service', '')
24 |           || ' ' || COALESCE(body ->> 'message_direction', '')
25 |           || ' ' || COALESCE(body ->> 'person_name', '')
26 |           || ' ' || COALESCE(body ->> 'repo_name', '')
27 |           || ' ' || COALESCE(body ->> 'repo_homepage', '')
28 |           || ' ' || COALESCE(body ->> 'repo_description', '')
29 |           || ' ' || COALESCE(body ->> 'repo_language', '')
30 |           || ' ' || COALESCE(body ->> 'transaction_amount', '')
31 |           || ' ' || COALESCE(body ->> 'transaction_purpose', '')
32 |           || ' ' || COALESCE(body ->> 'transaction_category', '')
33 |           || ' ' || COALESCE(body ->> 'transaction_recipient', '')
34 |           || ' ' || COALESCE(body ->> 'tweet_user_screen_name', '')
35 |           || ' ' || COALESCE(body ->> 'tweet_user_name', '')
36 |           || ' ' || COALESCE(translate(body ->> 'tweet_full_text', '@./', '  '), '')
37 |           || ' ' || COALESCE(translate(body ->> 'commit_diff', '@./', '  '), '')
38 |           || ' ' || COALESCE(body ->> 'photo_kind', '')
39 |           || ' ' || COALESCE(body ->> 'photo_labels', '')
40 |           || ' ' || COALESCE(body ->> 'photo_file_name', '')
41 |         )
42 |       ) STORED,
43 |       PRIMARY KEY ("id")
44 |     );
45 |     """
46 |     execute "CREATE INDEX idx_documents_created_at ON documents (created_at DESC)"
47 |     execute "CREATE INDEX idx_documents_search ON documents USING GIN(search)"
48 |     execute "CREATE INDEX idx_documents_body ON documents USING GIN(body jsonb_path_ops)"
49 |   end
50 | end
51 | 


--------------------------------------------------------------------------------
/memex/priv/repo/migrations/20210619110040_add_relations.exs:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Repo.Migrations.AddRelations do
 2 |   use Ecto.Migration
 3 | 
 4 |   def change do
 5 |     execute """
 6 |     CREATE TABLE "public"."relations" (
 7 |       "id" varchar(255) NOT NULL,
 8 |       "type" varchar(255) NOT NULL,
 9 |       "source_id" varchar(255) NOT NULL,
10 |       "metadata" jsonb NOT NULL DEFAULT '{}',
11 |       "created_at" timestamptz NOT NULL DEFAULT now(),
12 |       PRIMARY KEY ("id", "type", "source_id")
13 |     );
14 |     """
15 |   end
16 | end
17 | 


--------------------------------------------------------------------------------
/memex/priv/repo/migrations/20211009134648_create_importer_config.exs:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Repo.Migrations.CreateImporterConfig do
 2 |   use Ecto.Migration
 3 | 
 4 |   def change do
 5 |     create table(:importer_config, primary_key: false) do
 6 |       add(:id, :string, primary_key: true)
 7 |       add(:provider, :string)
 8 |       add(:display_name, :string)
 9 |       add(:encrypted_secrets, :binary)
10 |       add(:config_overwrite, :map)
11 |       timestamps()
12 |     end
13 |   end
14 | end
15 | 


--------------------------------------------------------------------------------
/memex/priv/repo/migrations/20221230125036_create_importer_log.exs:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Repo.Migrations.CreateImporterLog do
 2 |   use Ecto.Migration
 3 | 
 4 |   def change do
 5 |     create table(:importer_log, primary_key: false) do
 6 |       add(:id, :uuid, primary_key: true)
 7 |       add(:state, :string)
 8 |       add(:log, :text)
 9 |       add(:config_id, references(:importer_config, type: :string))
10 |       timestamps()
11 |     end
12 | 
13 |     alter table(:documents) do
14 |       add(:importer_log_id, references(:importer_log, type: :uuid))
15 |     end
16 |   end
17 | end
18 | 


--------------------------------------------------------------------------------
/memex/priv/repo/seeds.exs:
--------------------------------------------------------------------------------
1 | 
2 | 


--------------------------------------------------------------------------------
/memex/priv/static/images/Arc-small.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/Arc-small.png


--------------------------------------------------------------------------------
/memex/priv/static/images/Arc.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/Arc.png


--------------------------------------------------------------------------------
/memex/priv/static/images/GitHub-small.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/GitHub-small.png


--------------------------------------------------------------------------------
/memex/priv/static/images/GitHub.icns:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/GitHub.icns


--------------------------------------------------------------------------------
/memex/priv/static/images/MoneyMoney-small.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/MoneyMoney-small.png


--------------------------------------------------------------------------------
/memex/priv/static/images/MoneyMoney.icns:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/MoneyMoney.icns


--------------------------------------------------------------------------------
/memex/priv/static/images/Photos-small.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/Photos-small.png


--------------------------------------------------------------------------------
/memex/priv/static/images/Photos.icns:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/Photos.icns


--------------------------------------------------------------------------------
/memex/priv/static/images/Podcasts-small.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/Podcasts-small.png


--------------------------------------------------------------------------------
/memex/priv/static/images/Podcasts.icns:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/Podcasts.icns


--------------------------------------------------------------------------------
/memex/priv/static/images/Terminal.icns:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/Terminal.icns


--------------------------------------------------------------------------------
/memex/priv/static/images/Twitter-small.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/Twitter-small.png


--------------------------------------------------------------------------------
/memex/priv/static/images/Twitter.icns:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/Twitter.icns


--------------------------------------------------------------------------------
/memex/priv/static/images/iMessage-small.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/iMessage-small.png


--------------------------------------------------------------------------------
/memex/priv/static/images/iMessage.icns:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/iMessage.icns


--------------------------------------------------------------------------------
/memex/priv/static/images/memex.icns:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/memex.icns


--------------------------------------------------------------------------------
/memex/priv/static/images/memex.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/memex.png


--------------------------------------------------------------------------------
/memex/priv/static/images/memex/memex.afdesign:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/memex/memex.afdesign


--------------------------------------------------------------------------------
/memex/priv/static/images/phoenix.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/phoenix.png


--------------------------------------------------------------------------------
/memex/priv/static/images/safari-small.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/safari-small.png


--------------------------------------------------------------------------------
/memex/priv/static/images/safari.icns:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/safari.icns


--------------------------------------------------------------------------------
/memex/priv/static/images/terminal-small.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/adri/memex/28527259f28e95d6c1201eb257b8110d2030c98e/memex/priv/static/images/terminal-small.png


--------------------------------------------------------------------------------
/memex/test/memex/search/query_test.exs:
--------------------------------------------------------------------------------
 1 | defmodule Memex.Search.LegacyQueryTest do
 2 |   use ExUnit.Case, async: true
 3 | 
 4 |   alias Memex.Search.LegacyQuery, as: Query
 5 | 
 6 |   describe "date filter" do
 7 |     test "add filters" do
 8 |       query = Query.add_filter(%Query{}, "date", "2020")
 9 |       assert query.filters == %{"date" => "2020"}
10 |     end
11 | 
12 |     test "serialize to string" do
13 |       query =
14 |         %Query{}
15 |         |> Query.add_filter("date", "2020")
16 |         |> Query.to_string()
17 | 
18 |       assert query == "date:2020"
19 |     end
20 | 
21 |     test "from string" do
22 |       query = Query.from_string("test date:2020-12")
23 | 
24 |       assert query.query == "test"
25 |       assert query.filters == %{"date" => "2020-12"}
26 |     end
27 | 
28 |     test "from string with quotes" do
29 |       query = Query.from_string("test date:\"2020 12\"")
30 | 
31 |       assert query.query == "test"
32 |       assert query.filters == %{"date" => "2020 12"}
33 |     end
34 | 
35 |     test "from string broken" do
36 |       query = Query.from_string("test date:")
37 | 
38 |       assert query.query == "test date:"
39 |       assert query.filters == %{}
40 |     end
41 |   end
42 | end
43 | 


--------------------------------------------------------------------------------
/memex/test/memex_web/live/page_live_test.exs:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.PageLiveTest do
 2 |   use MemexWeb.ConnCase
 3 | 
 4 |   import Phoenix.LiveViewTest
 5 | 
 6 |   test "disconnected and connected render", %{conn: conn} do
 7 |     {:ok, page_live, disconnected_html} = live(conn, "/")
 8 |     assert disconnected_html =~ "Search..."
 9 |     assert render(page_live) =~ "Search..."
10 |   end
11 | end
12 | 


--------------------------------------------------------------------------------
/memex/test/memex_web/views/error_view_test.exs:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.ErrorViewTest do
 2 |   use MemexWeb.ConnCase, async: true
 3 | 
 4 |   # Bring render/3 and render_to_string/3 for testing custom views
 5 |   import Phoenix.View
 6 | 
 7 |   test "renders 404.html" do
 8 |     assert render_to_string(MemexWeb.ErrorView, "404.html", []) == "Not Found"
 9 |   end
10 | 
11 |   test "renders 500.html" do
12 |     assert render_to_string(MemexWeb.ErrorView, "500.html", []) == "Internal Server Error"
13 |   end
14 | end
15 | 


--------------------------------------------------------------------------------
/memex/test/memex_web/views/layout_view_test.exs:
--------------------------------------------------------------------------------
1 | defmodule MemexWeb.LayoutViewTest do
2 |   use MemexWeb.ConnCase, async: true
3 | 
4 |   # When testing helpers, you may want to import Phoenix.HTML and
5 |   # use functions such as safe_to_string() to convert the helper
6 |   # result into an HTML string.
7 |   # import Phoenix.HTML
8 | end
9 | 


--------------------------------------------------------------------------------
/memex/test/memex_web/views/timeline_view_test.exs:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.TimelineViewTest do
 2 |   use MemexWeb.ConnCase, async: true
 3 | 
 4 |   test "Return a short format of a number" do
 5 |     assert MemexWeb.TimelineView.number_short(12) == 12
 6 |     assert MemexWeb.TimelineView.number_short(1234) == "1.2K"
 7 |     assert MemexWeb.TimelineView.number_short(12_345) == "12.3K"
 8 |     assert MemexWeb.TimelineView.number_short(123_456) == "123.5K"
 9 |     assert MemexWeb.TimelineView.number_short(1_234_567) == "1.2M"
10 |     assert MemexWeb.TimelineView.number_short(12_345_678) == "12.3M"
11 |   end
12 | end
13 | 


--------------------------------------------------------------------------------
/memex/test/support/channel_case.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.ChannelCase do
 2 |   @moduledoc """
 3 |   This module defines the test case to be used by
 4 |   channel tests.
 5 | 
 6 |   Such tests rely on `Phoenix.ChannelTest` and also
 7 |   import other functionality to make it easier
 8 |   to build common data structures and query the data layer.
 9 | 
10 |   Finally, if the test case interacts with the database,
11 |   we enable the SQL sandbox, so changes done to the database
12 |   are reverted at the end of every test. If you are using
13 |   PostgreSQL, you can even run database tests asynchronously
14 |   by setting `use MemexWeb.ChannelCase, async: true`, although
15 |   this option is not recommended for other databases.
16 |   """
17 | 
18 |   use ExUnit.CaseTemplate
19 | 
20 |   using do
21 |     quote do
22 |       # Import conveniences for testing with channels
23 |       import MemexWeb.ChannelCase
24 |       import Phoenix.ChannelTest
25 | 
26 |       # The default endpoint for testing
27 |       @endpoint MemexWeb.Endpoint
28 |     end
29 |   end
30 | 
31 |   setup _tags do
32 |     :ok
33 |   end
34 | end
35 | 


--------------------------------------------------------------------------------
/memex/test/support/conn_case.ex:
--------------------------------------------------------------------------------
 1 | defmodule MemexWeb.ConnCase do
 2 |   @moduledoc """
 3 |   This module defines the test case to be used by
 4 |   tests that require setting up a connection.
 5 | 
 6 |   Such tests rely on `Phoenix.ConnTest` and also
 7 |   import other functionality to make it easier
 8 |   to build common data structures and query the data layer.
 9 | 
10 |   Finally, if the test case interacts with the database,
11 |   we enable the SQL sandbox, so changes done to the database
12 |   are reverted at the end of every test. If you are using
13 |   PostgreSQL, you can even run database tests asynchronously
14 |   by setting `use MemexWeb.ConnCase, async: true`, although
15 |   this option is not recommended for other databases.
16 |   """
17 | 
18 |   use ExUnit.CaseTemplate
19 | 
20 |   using do
21 |     quote do
22 |       # Import conveniences for testing with connections
23 |       use MemexWeb, :verified_routes
24 | 
25 |       import MemexWeb.ConnCase
26 |       import Phoenix.ConnTest
27 |       import Plug.Conn
28 | 
29 |       alias MemexWeb.Router.Helpers, as: Routes
30 | 
31 |       # The default endpoint for testing
32 |       @endpoint MemexWeb.Endpoint
33 |     end
34 |   end
35 | 
36 |   setup _tags do
37 |     {:ok, conn: Phoenix.ConnTest.build_conn()}
38 |   end
39 | end
40 | 


--------------------------------------------------------------------------------
/memex/test/test_helper.exs:
--------------------------------------------------------------------------------
1 | ExUnit.start()
2 | 


--------------------------------------------------------------------------------