├── .dockerignore
├── .gitignore
├── DECISIONS.md
├── Dockerfile
├── Dockerfile.dev
├── LICENSE
├── README.md
├── config
    ├── config.exs
    ├── dev.exs
    ├── prod.exs
    └── test.exs
├── frontend
    ├── about.html.eex
    ├── home.html.eex
    ├── retirement_message.html.eex
    ├── search.html.eex
    ├── search_bar.html.eex
    ├── static
    │   ├── background.jpg
    │   ├── retirement_message.css
    │   ├── search_bar.css
    │   ├── search_icon.svg
    │   └── warning_footer.css
    └── warning_footer.html.eex
├── lib
    ├── mix
    │   └── tasks
    │   │   ├── refresh_instance_api_non_regression_reference.ex
    │   │   ├── rescan.ex
    │   │   ├── seed_from_instances_joinpeertube_org.ex
    │   │   └── seed_from_instances_the_federation_info.ex
    ├── peertube_index.ex
    └── peertube_index
    │   ├── application.ex
    │   ├── frontend_helpers.ex
    │   ├── instance_scanner.ex
    │   ├── status_storage.ex
    │   ├── templates.ex
    │   ├── video_storage.ex
    │   └── web_server.ex
├── mix.exs
├── mix.lock
├── priv
    └── repo
    │   └── migrations
    │       └── 20190816201502_create_statuses_table.exs
├── scan_loop.sh
├── seed_loop.sh
└── test
    ├── peertube_index
        ├── frontend_helpers_test.exs
        ├── instance_scanner_non_regression_test.exs
        ├── instance_scanner_non_regression_test_data
        │   └── .gitignore
        ├── instance_scanner_test.exs
        ├── status_storage_test.exs
        ├── video_storage_test.exs
        └── web_server_test.exs
    ├── peertube_index_test.exs
    ├── support
        └── mocks.ex
    └── test_helper.exs


/.dockerignore:
--------------------------------------------------------------------------------
 1 | # The directory Mix will write compiled artifacts to.
 2 | /_build/
 3 | 
 4 | # If you run "mix test --cover", coverage assets end up here.
 5 | /cover/
 6 | 
 7 | # The directory Mix downloads your dependencies sources to.
 8 | /deps/
 9 | 
10 | # Where 3rd-party dependencies like ExDoc output generated docs.
11 | /doc/
12 | 
13 | # Ignore .fetch files in case you like to edit your project deps locally.
14 | /.fetch
15 | 
16 | # If the VM crashes, it generates a dump, let's ignore it too.
17 | erl_crash.dump
18 | 
19 | # Also ignore archive artifacts (built via "mix archive.build").
20 | *.ez
21 | 
22 | # Ignore package tarball (built via "mix hex.build").
23 | peertube_index-*.tar
24 | 
25 | /local_only/
26 | /.git/
27 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # The directory Mix will write compiled artifacts to.
 2 | /_build/
 3 | 
 4 | # If you run "mix test --cover", coverage assets end up here.
 5 | /cover/
 6 | 
 7 | # The directory Mix downloads your dependencies sources to.
 8 | /deps/
 9 | 
10 | # Where 3rd-party dependencies like ExDoc output generated docs.
11 | /doc/
12 | 
13 | # Ignore .fetch files in case you like to edit your project deps locally.
14 | /.fetch
15 | 
16 | # If the VM crashes, it generates a dump, let's ignore it too.
17 | erl_crash.dump
18 | 
19 | # Also ignore archive artifacts (built via "mix archive.build").
20 | *.ez
21 | 
22 | # Ignore package tarball (built via "mix hex.build").
23 | peertube_index-*.tar
24 | 


--------------------------------------------------------------------------------
/DECISIONS.md:
--------------------------------------------------------------------------------
 1 | # Project decisions
 2 | This file contains decisions that may help understanding the project architecture.
 3 | 
 4 | **Maintainability**
 5 | 
 6 | - We try to decouple use cases code from storage and user interface code (hexagonal architecture).
 7 | - We try to use [MaintainableCSS](https://maintainablecss.com/) for structuring CSS
 8 | 
 9 | **Instance scanning**
10 | 
11 | - We do not want to put too much pressure on instances when scanning them so we do not parallelize queries for one instance.
12 | - For the beginning, for the sake of simplicity, we do not retry on scanning error. An instance is marked as failed at 
13 | - Our validation of video documents expects fields that are not in PeerTube OpenAPI spec but that we found were provided by most instances.
14 | We should check monitor video document validation errors per PeerTube instance version to make sure our validation works correctly for newer versions.
15 | - When scanning an instance, we stream videos to a file on disk.
16 | Only after we successfully fetched and verified all videos, we insert them in video storage streaming from the disk. 
17 | This avoids storing all the videos for one instance in memory before inserting them in the video storage.
18 | We did this because we ran out of memory when storing all videos in memory before verifying and storing them.
19 | It turns out that the large number of videos some instances have consists mainly of non local videos that we do not need to store.
20 | As long as the number of local videos an instance have fits into memory, we do not need to stream to disk.
21 | Reducing the stream to memory by excluding non local videos is sufficient.
22 | As we discovered this after implementing the streaming to disk, we leave the code as is for the moment.
23 | We took a shortcut by hard coding the file name we use as temporary storage, which prevents running multiple scans concurrently.
24 | - We found that for some instances, some non local videos had an one validation error.
25 | We decided to add and exception that allows this specific validation error for non local videos.
26 | This means we only silent this error and any other validation error will make the scan fail.
27 | We do not want to silent unexpected errors.
28 | 
29 | **Status database**
30 | 
31 | - We use PostgreSQL for storing instances status because we need concurrent access without corrupted reads or writes.
32 | We chose to use the ecto library because we will need soon need transactions.
33 | 
34 | **Search database**
35 | 
36 | We use Elasticsearch as the search database because :
37 | - we need fast text search 
38 | - we found it was easier to setup and faster to get started with than other databases
39 | 
40 | **Status monitoring database**
41 | We use PostgreSQL version 10 because it is the highest available version in Grafana PostgreSQL data source setup page.
42 | 
43 | **Deployment**
44 | 
45 | We use Docker containers with Docker Compose to deploy the full stack on a single machine, it is enough for the beginning.
46 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM elixir:1.9.1-alpine
 2 | RUN apk update && apk add git
 3 | 
 4 | ENV MIX_ENV prod
 5 | WORKDIR /srv
 6 | RUN mix local.rebar --force
 7 | RUN mix local.hex --force
 8 | 
 9 | COPY . .
10 | RUN mix do deps.get, deps.compile, compile
11 | 
12 | CMD mix run --no-halt
13 | 


--------------------------------------------------------------------------------
/Dockerfile.dev:
--------------------------------------------------------------------------------
1 | FROM elixir:1.9.1-alpine
2 | RUN apk update && apk add git
3 | 
4 | WORKDIR /srv
5 | RUN mix local.rebar --force
6 | RUN mix local.hex --force
7 | 
8 | CMD /bin/sh -c "mix deps.get && mix run --no-halt"
9 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2019 silicium14
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # PeerTube Index
 2 | 
 3 | PeerTube Index is a centralized search engine for PeerTube videos.
 4 | It crawls PeerTube instances, index their videos, and discovers new instances.
 5 | It is hosted at [peertube-index.net](https://peertube-index.net).
 6 | 
 7 | This project is not affiliated to Framasoft, the maintainer of the [PeerTube](https://github.com/Chocobozzz/PeerTube) software.
 8 | 
 9 | ## How it works / what it does
10 | 
11 | - Scans instances if they have not been scanned for more than 24 hours
12 | - Respects robots.txt directives
13 | - Discovers new instances by adding the followed and following instances of a known instance being scanned
14 | - Periodically updates its list of known instances from thefederation.info and instances.joinpeertube.org
15 | - No scanning retry: at the first scanning error (network error, HTTP error, parsing error...) the instance is marked as failed for the next 24h
16 | 
17 | ## State of the project
18 | 
19 | **I will stop maintaining PeerTube Index, it will be retired soon.**
20 | 
21 | Framasoft, the maintainer of PeerTube, now has an official solution for global search on the PeerTube federation: [Sepia Search at https://search.joinpeertube.org/](https://search.joinpeertube.org/).
22 | People can use it instead of PeerTube Index.
23 | It is open source and people can use it to setup their own search engine with their own rules if they want.
24 | It does not automatically discovers new instances as PeerTube index does (by fetching following and followed instances).
25 | 


--------------------------------------------------------------------------------
/config/config.exs:
--------------------------------------------------------------------------------
 1 | use Mix.Config
 2 | 
 3 | config :peertube_index,
 4 |   elasticsearch_config: %{
 5 |     url: {:system, "ELASTICSEARCH_URL"}, # The URL to reach ElasticSearch, for example: http://localost:9200
 6 |     api: Elasticsearch.API.HTTP
 7 |   },
 8 |   status_storage_database_url: {:system, "STATUS_STORAGE_DATABASE_URL"},
 9 |   http_api_port: {:system, :integer, "HTTP_API_PORT"}, # The TCP port used to listen to incoming HTTP requests for the API, for example: 80
10 |   ecto_repos: [PeertubeIndex.StatusStorage.Repo]
11 | 
12 | import_config "#{Mix.env}.exs"
13 | 


--------------------------------------------------------------------------------
/config/dev.exs:
--------------------------------------------------------------------------------
1 | use Mix.Config
2 | 
3 | import_config "prod.exs"
4 | 


--------------------------------------------------------------------------------
/config/prod.exs:
--------------------------------------------------------------------------------
 1 | use Mix.Config
 2 | 
 3 | config :peertube_index,
 4 |   video_storage: PeertubeIndex.VideoStorage.Elasticsearch,
 5 |   instance_scanner: PeertubeIndex.InstanceScanner.Http,
 6 |   status_storage: PeertubeIndex.StatusStorage.Postgresql
 7 | 
 8 | config :gollum,
 9 |   refresh_secs: 10, # Amount of time before the robots.txt will be refetched
10 |   lazy_refresh: true, # Whether to setup a timer that auto-refetches, or to only refetch when requested
11 |   user_agent: "PeertubeIndex" # User agent to use when sending the GET request for the robots.txt
12 | 


--------------------------------------------------------------------------------
/config/test.exs:
--------------------------------------------------------------------------------
 1 | use Mix.Config
 2 | 
 3 | config :peertube_index,
 4 |   video_storage: PeertubeIndex.VideoStorage.Mock,
 5 |   instance_scanner: PeertubeIndex.InstanceScanner.Mock,
 6 |   status_storage: PeertubeIndex.StatusStorage.Mock
 7 | 
 8 | config :gollum,
 9 |   refresh_secs: 0, # Amount of time before the robots.txt will be refetched
10 |   lazy_refresh: true, # Whether to setup a timer that auto-refetches, or to only refetch when requested
11 |   user_agent: "PeertubeIndex" # User agent to use when sending the GET request for the robots.txt
12 | 
13 | config :logger,
14 |   level: :warn
15 | 


--------------------------------------------------------------------------------
/frontend/about.html.eex:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html>
  3 | 
  4 | <head>
  5 | <meta name="viewport" content="width=device-width, initial-scale=1">
  6 | <title>PeerTube Index</title>
  7 | <style>
  8 | html {
  9 |     -webkit-text-size-adjust: 100%; /* Prevent font scaling in landscape while allowing user zoom */
 10 | }
 11 | body {
 12 |     min-width: 320px;
 13 |     display: flex;
 14 |     flex-direction: column;
 15 |     align-items: center;
 16 |     margin: 20px auto 0px;
 17 |     font-family: Verdana, sans-serif;
 18 | }
 19 | @media (min-width: 900px) {
 20 |     body {
 21 |         width: auto;
 22 |     }
 23 | }
 24 | .aboutSection {
 25 |     box-sizing: border-box;
 26 |     width: 267px;
 27 |     padding: 10px;
 28 |     border-radius: 8px;
 29 | }
 30 | .aboutSection a {
 31 |     color: black;
 32 | }
 33 | @media (min-width: 900px) {
 34 |     .aboutSection {
 35 |         width: 768px;
 36 |     }
 37 | }
 38 | .aboutSection:target {
 39 |     color: white;
 40 |     background-color: #333333;
 41 | }
 42 | .aboutSection:target a {
 43 |     color: white;
 44 | }
 45 | .reportForm .reportForm-item:first-of-type {
 46 |     margin-top: 0;
 47 | }
 48 | .reportForm {
 49 |     max-width: 500px;
 50 | }
 51 | .reportForm-item {
 52 |     margin-top: 20px;
 53 | }
 54 | .reportForm-inputWithLabel {
 55 |     display: block;
 56 |     font-size: medium;
 57 |     box-sizing: border-box;
 58 |     width: 100%;
 59 |     margin-top: 10px;
 60 |     border-radius: 8px;
 61 |     border: 1px solid lightgray;
 62 |     padding: 0.8rem;
 63 | }
 64 | .reportForm-reason {
 65 |     height: 150px;
 66 |     resize: none;
 67 |     font-family: inherit;
 68 | }
 69 | .reportForm-submitButton {
 70 |     padding: 0.8rem;
 71 |     border-radius: 8px;
 72 |     border: 1px solid lightgray;
 73 |     font-size: medium;
 74 |     cursor: pointer;
 75 | }
 76 | .quefaireaparisLogo {
 77 |     display: flex;
 78 |     border-radius: 50%;
 79 |     background-color: white;
 80 |     height: 196px;
 81 |     width: 196px;
 82 |     margin: 16px 16px;
 83 | }
 84 | .quefaireaparisLogo img {
 85 |     width: 140px;
 86 |     object-fit: contain;
 87 |     margin: 28px;
 88 | }
 89 | .navigationLinks {
 90 |     text-align: center;
 91 |     margin-top: 30px;
 92 |     margin-bottom: 20px;
 93 | }
 94 | .navigationLinks a {
 95 |     color: black
 96 | }
 97 | </style>
 98 | <link rel="stylesheet" href="/static/retirement_message.css">
 99 | </head>
100 | 
101 | <body>
102 | 
103 | <article id="this-site" class="aboutSection">
104 |     <h2>About this site</h2>
105 |     <p>
106 |         PeerTube Index is a centralized search engine for PeerTube videos.
107 |         We run a crawler that visits PeerTube instances to find videos and new instances.
108 |         All found videos are made searchable by their name.
109 |         We exclude videos marked as Not Safe For Work (NSFW) for now.
110 |     </p>
111 |     <p>
112 | 
113 |     </p>
114 |     <p>
115 |         This site is not affiliated to Framasoft, the maintainer of the PeerTube software.
116 |     </p>
117 | </article>
118 | 
119 | <article id="unsafe-content" class="aboutSection">
120 |     <h2>⚠️ Unsafe content warning</h2>
121 |     <p>
122 |         Unsafe content, also known as Not Safe For Work or NSFW (such as pornography, violence...) may be displayed on search results, even with normal or "safe" search terms.
123 |     </p>
124 |     <p>
125 |         PeerTube allows video publishers to tag their videos either as safe or unsafe.
126 |         We get this information when we scan PeerTube instances, we are then able to exclude the videos marked as unsafe.
127 |         This tagging is the only information we have to exclude unsafe videos. However, this tagging is out of our control.
128 |         Because anybody can host a PeerTube instance (which is the very purpose of PeerTube), anybody can choose to ignore the tagging on their own instance and publish unsafe videos not tagged as such.
129 |     </p>
130 |     <p>
131 |         As PeerTube Index discovers PeerTube instances and scans them in an automatic fashion in order to find as many instances and videos as possible,
132 |         PeerTube Index cannot be responsible for the not properly tagged content that it discovers and displays on search results.
133 |     </p>
134 |     <p>
135 |         If you find a video on PeerTube Index that is unsafe and should not be displayed,
136 |         you should contact the corresponding instance's administrators to ask them to properly tag their videos.
137 |         You can also report it to us using the <a href="#report-form">form below</a> so we may exclude it from our search results.
138 |     </p>
139 | </article>
140 | 
141 | <article id="report-form" class="aboutSection">
142 |     <h2>Report a video</h2>
143 |     <p>
144 |         You can use the form below to report a video referenced on PeerTube Index that is illegal according to European Union law or that is legal but unsafe (Not Safe For Work).
145 |         There is no guarantee that we will remove a video from our search results by submitting this form, this is at our discretion.
146 |     </p>
147 |     <p>
148 |         For unsafe but legal videos, you should first contact the corresponding instance's administrators to ask them to properly tag their videos.
149 |     </p>
150 |     <form class="reportForm" accept-charset="UTF-8" action="https://usebasin.com/f/080553025681" method="POST" target="_blank">
151 |         <div class="reportForm-item">
152 |             <label for="reportform-videotitle">The title of the video</label>
153 |             <input class="reportForm-inputWithLabel" id="reportform-videolink" type="text" name="video_title" required>
154 |         </div>
155 |         <div class="reportForm-item">
156 |             <label for="reportform-videolink">The video URL</label>
157 |             <input class="reportForm-inputWithLabel" id="reportform-videolink" type="url" name="video_link" required>
158 |         </div>
159 |         <div class="reportForm-item">
160 |             <label for="reportform-searchlink">The URL of the PeerTube Index search showing the video</label>
161 |             <input class="reportForm-inputWithLabel" id="reportform-searchlink" type="url" name="search_link" placeholder="http://peertube-index.net/search?text=example" required>
162 |         </div>
163 |         <div class="reportForm-item">
164 |             <label for="reportform-reason">Why do you report this video?</label>
165 |             <textarea class="reportForm-inputWithLabel reportForm-reason" id="reportform-reason" name="reason" required></textarea>
166 |         </div>
167 |         <div class="reportForm-item">
168 |             <button class="reportForm-submitButton" type="submit">Submit</button>
169 |         </div>
170 |     </form>
171 | </article>
172 | 
173 | <article id="photo" class="aboutSection">
174 |     <h2>Photo credits</h2>
175 |     <p>
176 |         Backgound image published on <a href="https://quefaire.paris.fr/11510/reouverture-de-la-bnf-richelieu">quefaire.paris.fr</a>
177 |     </p>
178 |     <span class="quefaireaparisLogo"><img src="https://quefaire.paris.fr/logo.png"></span>
179 | </article>
180 | 
181 | <nav class="navigationLinks"><a href="/">home page</a> - <a href="javascript:history.back()">go back</a></nav>
182 | <%= PeertubeIndex.Templates.retirement_message() %>
183 | </body>
184 | 
185 | </html>
186 | 


--------------------------------------------------------------------------------
/frontend/home.html.eex:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html>
 3 | <head>
 4 | <meta name="viewport" content="width=device-width, initial-scale=1">
 5 | <style>
 6 | html {
 7 |     -webkit-text-size-adjust: 100%; /* Prevent font scaling in landscape while allowing user zoom */
 8 | }
 9 | 
10 | body {
11 |     margin: 0;
12 |     font-family: Verdana, sans-serif;
13 |     min-height: 100vh;
14 |     display: flex;
15 |     flex-direction: column;
16 | }
17 | 
18 | #background {
19 |     left: 0;
20 |     top: 0;
21 |     position: fixed;
22 |     width: 100%;
23 |     height: 100%;
24 |     background-image: url("/static/background.jpg");
25 |     background-size: cover;
26 |     z-index: -100;
27 |     opacity: 0.3;
28 | }
29 | 
30 | .searchForm {
31 |     width: 100%;
32 |     min-width: 320px;
33 |     margin-top: 25vh;
34 |     display: flex;
35 |     flex-direction: column;
36 |     align-items: center;
37 | }
38 | .searchForm-title {
39 |     font-weight: normal;
40 |     font-family: Bitter, serif;
41 |     margin-bottom: 5vh;
42 | }
43 | @media (min-width: 900px) {
44 |     .searchForm-title {
45 |         font-size: 60px;
46 |     }
47 | }
48 | .searchForm-button {
49 |     box-sizing: border-box;
50 |     margin-top: 15px;
51 |     width: 288px;
52 |     padding: 0.8rem;
53 |     border: none;
54 |     border-radius: 8px;
55 |     font-size: medium;
56 |     color: white;
57 |     background-color: #333333;
58 |     cursor: pointer;
59 | }
60 | @media (min-width: 900px) {
61 |     .searchForm-button {
62 |         width: 572px;
63 |         align-items: stretch;
64 |     }
65 | }
66 | .aboutLinks {
67 |     margin-top: 40px;
68 |     flex-grow: 1;
69 |     width: 100%;
70 |     text-align: center;
71 | }
72 | .aboutLinks a {
73 |     color: black;
74 | }
75 | </style>
76 | 
77 | <link rel="stylesheet" href="/static/search_bar.css">
78 | <link rel="stylesheet" href="/static/warning_footer.css">
79 | <link rel="stylesheet" href="/static/retirement_message.css">
80 | <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Bitter&display=swap">
81 | 
82 | <title>PeerTube Index</title>
83 | </head>
84 | 
85 | <body>
86 | <div id="background"></div>
87 | <form class="searchForm" action="search">
88 |     <h1 class="searchForm-title">PeerTube Index</h1>
89 |     <%= PeertubeIndex.Templates.search_bar("") %>
90 | </form>
91 | <div class="aboutLinks">
92 | <a href="/about#this-site">about this site</a> - <a href="/about#photo">photo credits</a>
93 | </div>
94 | <%= PeertubeIndex.Templates.warning_footer() %>
95 | <%= PeertubeIndex.Templates.retirement_message() %>
96 | </body>
97 | </html>
98 | 


--------------------------------------------------------------------------------
/frontend/retirement_message.html.eex:
--------------------------------------------------------------------------------
1 | <div class="retirementMessage">
2 |     <p>PeerTube now has an <a href="https://search.joinpeertube.org/">official solution for search</a> that people should use, PeerTube Index will be retired soon.</p>
3 | </div>
4 | 


--------------------------------------------------------------------------------
/frontend/search.html.eex:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html>
  3 | <head>
  4 | <meta name="viewport" content="width=device-width, initial-scale=1">
  5 | <style>
  6 | 
  7 | html {
  8 |     -webkit-text-size-adjust: 100%; /* Prevent font scaling in landscape while allowing user zoom */
  9 | }
 10 | 
 11 | body {
 12 |     font-family: Verdana, sans-serif;
 13 |     margin: 0;
 14 |     min-height: 100vh;
 15 |     display: flex;
 16 |     flex-direction: column;
 17 |     justify-content: space-between;
 18 | }
 19 | 
 20 | .searchForm {
 21 |     display: flex;
 22 |     justify-content: center;
 23 |     width: 100%;
 24 |     min-width: 320px;
 25 |     position: fixed;
 26 |     top: 20px;
 27 |     z-index: 100;
 28 | }
 29 | 
 30 | .homeButton {
 31 |     position: fixed;
 32 |     display: flex;
 33 |     justify-content: center;
 34 |     width: 100%;
 35 |     z-index: 100;
 36 |     visibility: hidden;
 37 | }
 38 | .homeButton span {
 39 |     position: relative;
 40 | }
 41 | .homeButton a {
 42 |     color: white;
 43 |     text-decoration: none;
 44 |     background-color: #333333;
 45 |     padding: 0.8rem 0.4rem;
 46 |     border-radius: 8px;
 47 |     box-shadow: 3px 3px 6px rgba(0, 0, 0, 0.3);
 48 |     position: absolute;
 49 | }
 50 | @media (min-width: 900px) {
 51 |     .homeButton--top {
 52 |         visibility: visible;
 53 |     }
 54 | }
 55 | .homeButton--top {
 56 |     top: 20px;
 57 | }
 58 | .homeButton--top span {
 59 |     width: 572px;
 60 | }
 61 | .homeButton--top a {
 62 |     right: 592px; /* width of search bar plus 20px */
 63 | }
 64 | @media (max-width: 899px) {
 65 |     .homeButton--bottom {
 66 |         visibility: visible;
 67 |     }
 68 | }
 69 | .homeButton--bottom {
 70 |     bottom: 0px;
 71 | }
 72 | .homeButton--bottom span {
 73 |     width: 288px;
 74 | }
 75 | .homeButton--bottom a {
 76 |     bottom: 140px;
 77 | }
 78 | 
 79 | .main {
 80 |     width: 267px;
 81 |     /* The bottom margin leaves room for the home button */
 82 |     margin: 80px auto 90px auto;
 83 |     flex-grow: 1;
 84 |     display: flex;
 85 |     flex-direction: column;
 86 |     align-items: center;
 87 | }
 88 | @media (min-width: 900px) {
 89 |     .main {
 90 |         margin-bottom: 70px;
 91 |         width: 768px;
 92 |         align-items: stretch;
 93 |     }
 94 | }
 95 | .resultsMessage {
 96 |     margin-bottom: 10px;
 97 |     font-size: small;
 98 |     color: gray;
 99 | }
100 | .resultsMessage--noresult {
101 |     text-align: center;
102 | }
103 | 
104 | .video {
105 |     display: flex;
106 |     flex-direction: column;
107 |     margin-bottom: 50px;
108 | }
109 | 
110 | .video:last-of-type {
111 |     margin-bottom: auto;
112 | }
113 | 
114 | @media (min-width: 900px) {
115 |     .video {
116 |         flex-direction: row;
117 |     }
118 | }
119 | .video-thumbnail {
120 |     position: relative;
121 |     z-index: 0;
122 | }
123 | @media (min-width: 900px) {
124 |     .video-thumbnail {
125 |         margin-right: 20px;
126 |     }
127 | }
128 | .video-thumbnail img {
129 |     width: 268px;
130 |     object-fit: contain;
131 |     border-radius: 10px;
132 |     border: none;
133 |     /* https://stackoverflow.com/a/11126701 */
134 |     display: block;
135 | }
136 | .video-duration {
137 |     position: absolute;
138 |     right: 5px;
139 |     bottom: 5px;
140 |     border: none;
141 |     border-radius: 5px;
142 |     background-color: rgba(0,0,0,.8);
143 |     color: white;
144 |     padding: 2px 5px;
145 |     font-size: 12px;
146 | }
147 | .video-details {
148 |     margin-left: 2px;
149 |     margin-right: 2px;
150 | }
151 | @media (min-width: 900px) {
152 |     .video {
153 |         flex-direction: row;
154 |     }
155 | }
156 | .video-title {
157 |     color: black;
158 |     text-decoration: none;
159 | }
160 | .video-title p {
161 |     margin-top: 5px;
162 |     margin-bottom: 5px;
163 | }
164 | .video-date,
165 | .video-views,
166 | .video-account {
167 |     color: gray;
168 |     font-size: smaller;
169 |     margin: 0;
170 |     text-decoration: none;
171 | }
172 | </style>
173 | <link rel="stylesheet" href="/static/search_bar.css">
174 | <link rel="stylesheet" href="/static/warning_footer.css">
175 | <link rel="stylesheet" href="/static/retirement_message.css">
176 | 
177 | <title>PeerTube Index</title>
178 | </head>
179 | 
180 | <body>
181 |     <form class="searchForm">
182 |         <%= PeertubeIndex.Templates.search_bar(query) %>
183 |     </form>
184 |     <div class="homeButton homeButton--top"><span><a href="/">home</a></span></div>
185 |     <div class="main">
186 | 
187 |         <div class="resultsMessage<%= if length(videos) == 0, do: " resultsMessage--noresult"%>">
188 |             <p>
189 |             <%= case length(videos) do
190 |                 0 ->
191 |                     "No result found for your query, sorry"
192 |                 1 ->
193 |                     "1 result found"
194 |                 n when n >= 100 ->
195 |                     "100 or more results found, showing only the first 100"
196 |                 n ->
197 |                     "#{n} results found"
198 |             end%>
199 |             </p>
200 |         </div>
201 |         <%= for video <- videos do %>
202 |         <article class="video">
203 |             <%=
204 |                 Phoenix.HTML.Tag.content_tag(
205 |                     :a,
206 |                     [
207 |                         Phoenix.HTML.Tag.content_tag(
208 |                             :div,
209 |                             [
210 |                                 Phoenix.HTML.Tag.img_tag("https://" <> video["account"]["host"] <> video["previewPath"]),
211 |                                 Phoenix.HTML.Tag.content_tag(
212 |                                     :span,
213 |                                     PeertubeIndex.FrontendHelpers.format_duration(video["duration"]),
214 |                                     class: "video-duration"
215 |                                 )
216 |                             ],
217 |                             class: "video-thumbnail"
218 |                         )
219 |                     ],
220 |                     href: "https://" <> video["account"]["host"] <> "/videos/watch/" <> video["uuid"]
221 |                 ) |> Phoenix.HTML.safe_to_string()
222 |             %>
223 |             <div class="video-details">
224 |                 <%=
225 |                     Phoenix.HTML.Tag.content_tag(
226 |                         :a,
227 |                         Phoenix.HTML.Tag.content_tag(
228 |                             :p,
229 |                             video["name"]
230 |                         ),
231 |                         href: "https://" <> video["account"]["host"] <> "/videos/watch/" <> video["uuid"],
232 |                         class: "video-title"
233 |                     ) |> Phoenix.HTML.safe_to_string()
234 |                 %>
235 |                 <%=
236 |                     Phoenix.HTML.Tag.content_tag(
237 |                         :p,
238 |                         "Published #{video["publishedAt"] |> NaiveDateTime.from_iso8601!() |> NaiveDateTime.truncate(:second)}",
239 |                         class: "video-date"
240 |                     ) |> Phoenix.HTML.safe_to_string()
241 |                 %>
242 |                 <%=
243 |                     Phoenix.HTML.Tag.content_tag(
244 |                         :p,
245 |                         "#{video["views"]} views",
246 |                         class: "video-views"
247 |                     ) |> Phoenix.HTML.safe_to_string()
248 |                 %>
249 |                 <%=
250 |                     Phoenix.HTML.Tag.content_tag(
251 |                         :a,
252 |                         "#{video["account"]["name"]}@#{video["account"]["host"]}",
253 |                         class: "video-account",
254 |                         href: video["account"]["url"]
255 |                     ) |> Phoenix.HTML.safe_to_string()
256 |                 %>
257 |             </div>
258 |         </article>
259 |     <%end%>
260 |     </div>
261 |     <div class="homeButton homeButton--bottom"><span><a href="/">home</a></span></div>
262 |     <%= PeertubeIndex.Templates.warning_footer() %>
263 |     <%= PeertubeIndex.Templates.retirement_message() %>
264 | </body>
265 | 
266 | </html>
267 | 


--------------------------------------------------------------------------------
/frontend/search_bar.html.eex:
--------------------------------------------------------------------------------
 1 | <div class="searchBar">
 2 | <%=
 3 | Phoenix.HTML.Tag.tag(
 4 |     "input",
 5 |     type: "text",
 6 |     name: "text",
 7 |     value: query,
 8 |     placeholder: "Search videos by name",
 9 |     required: true
10 | ) |> Phoenix.HTML.safe_to_string()
11 | %><button type="submit"><img src="/static/search_icon.svg"></button>
12 | </div>
13 | 


--------------------------------------------------------------------------------
/frontend/static/background.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/silicium14/peertube_index/241264a2a20f3ad9088f667bc8313a98b52901c2/frontend/static/background.jpg


--------------------------------------------------------------------------------
/frontend/static/retirement_message.css:
--------------------------------------------------------------------------------
 1 | .retirementMessage {
 2 |     position: -webkit-sticky;
 3 |     position: sticky;
 4 |     bottom: 0;
 5 |     margin: 0;
 6 |     width: 100%;
 7 |     z-index: 100;
 8 |     min-width: 320px;
 9 |     text-align: center;
10 |     background-color: salmon;
11 |     color: white;
12 |     font-size: 14px;
13 | }
14 | .retirementMessage p {
15 |     margin: 0;
16 |     padding: 10px 0;
17 | }
18 | .retirementMessage p a {
19 |     color: inherit;
20 | }
21 | 


--------------------------------------------------------------------------------
/frontend/static/search_bar.css:
--------------------------------------------------------------------------------
 1 | .searchBar {
 2 |     display: flex;
 3 |     box-sizing: border-box;
 4 |     width: 288px;
 5 |     border: none;
 6 |     border-radius: 8px;
 7 |     box-shadow: 3px 3px 6px rgba(0, 0, 0, 0.3);
 8 |     z-index: 100;
 9 | }
10 | @media (min-width: 900px) {
11 |     .searchBar {
12 |         width: 572px;
13 |     }
14 | }
15 | .searchBar input {
16 |     font-size: medium;
17 |     font-family: Verdana, sans-serif;
18 |     flex-grow: 1;
19 |     box-sizing: border-box;
20 |     margin: 0px;
21 |     padding: 0.8rem;
22 |     border: none;
23 |     border-radius: 8px 0 0 8px;
24 | }
25 | .searchBar button {
26 |     box-sizing: border-box;
27 |     margin: 0;
28 |     padding: 0;
29 |     width: 60px;
30 |     border: none;
31 |     border-radius: 0 8px 8px 0;
32 |     background-color: #333333;
33 |     cursor: pointer;
34 |     flex-shrink: 0;
35 | }
36 | .searchBar button img {
37 |     /* https://stackoverflow.com/a/11126701 */
38 |     display: block;
39 |     margin: auto;
40 | }
41 | 


--------------------------------------------------------------------------------
/frontend/static/search_icon.svg:
--------------------------------------------------------------------------------
1 | <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
2 | <svg width="24" height="24" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
3 |   <circle cx="10" cy="10" r="7" fill-opacity="0" stroke="#fff" stroke-width="2"/>
4 |   <line x1="15" y1="15" x2="21" y2="21" stroke="#fff" stroke-linecap="round" stroke-width="2"/>
5 | </svg>
6 | 


--------------------------------------------------------------------------------
/frontend/static/warning_footer.css:
--------------------------------------------------------------------------------
 1 | .warningFooter {
 2 |     position: -webkit-sticky;
 3 |     position: sticky;
 4 |     bottom: 0;
 5 |     margin: 0;
 6 |     width: 100%;
 7 |     z-index: 100;
 8 |     min-width: 320px;
 9 |     text-align: center;
10 |     background-color: #333333;
11 |     color: white;
12 |     font-size: 14px;
13 | }
14 | .warningFooter p {
15 |     margin: 0;
16 |     padding: 10px 0;
17 | }
18 | .warningFooter p a {
19 |     color: inherit;
20 | }
21 | 


--------------------------------------------------------------------------------
/frontend/warning_footer.html.eex:
--------------------------------------------------------------------------------
1 | <div class="warningFooter">
2 |     <p>⚠️ Unsafe content (violence, pornography...) may be displayed, <a href="about#unsafe-content">click here to see why.</a></p>
3 | </div>
4 | 


--------------------------------------------------------------------------------
/lib/mix/tasks/refresh_instance_api_non_regression_reference.ex:
--------------------------------------------------------------------------------
 1 | defmodule Mix.Tasks.RefreshInstanceScannerNonRegressionReference do
 2 |   use Mix.Task
 3 | 
 4 |   @moduledoc """
 5 |   Scans reference Peertube instance with the current version of the code
 6 |   and save result as the reference for instance scanner non regression tests
 7 |   """
 8 | 
 9 |   @shortdoc """
10 |   Update reference data for instance scanner non regression tests
11 |   """
12 |   def run(_) do
13 |     Application.ensure_all_started :gollum
14 |     {:ok, {videos, instances}} = PeertubeIndex.InstanceScanner.Http.scan("peertube.cpy.re", 100)
15 |     videos = Enum.to_list(videos)
16 | 
17 |     File.write!(
18 |       "test/peertube_index/instance_scanner_non_regression_test_data/reference_videos.json",
19 |       Poison.encode!(videos, pretty: true),
20 |       [:binary, :write]
21 |     )
22 |     File.write!(
23 |       "test/peertube_index/instance_scanner_non_regression_test_data/reference_instances.json",
24 |       Poison.encode!(MapSet.to_list(instances), pretty: true),
25 |       [:binary, :write]
26 |     )
27 |   end
28 | end
29 | 


--------------------------------------------------------------------------------
/lib/mix/tasks/rescan.ex:
--------------------------------------------------------------------------------
 1 | defmodule Mix.Tasks.Rescan do
 2 |   use Mix.Task
 3 | 
 4 |   @moduledoc """
 5 |   Rescans known instances
 6 |   """
 7 | 
 8 |   @shortdoc """
 9 |   Rescans known instances
10 |   """
11 |   def run(_) do
12 |     Application.ensure_all_started :elasticsearch
13 |     Application.ensure_all_started :gollum
14 |     Application.ensure_all_started :ecto_sql
15 |     Application.ensure_all_started :postgrex
16 |     PeertubeIndex.StatusStorage.Repo.start_link()
17 |     PeertubeIndex.rescan
18 |   end
19 | end
20 | 


--------------------------------------------------------------------------------
/lib/mix/tasks/seed_from_instances_joinpeertube_org.ex:
--------------------------------------------------------------------------------
 1 | defmodule Mix.Tasks.SeedFromInstancesJoinpeertubeOrg do
 2 |   use Mix.Task
 3 | 
 4 |   @moduledoc """
 5 |   Fetches instances from instances.joinpeertube.org and adds the not yet known instances to the instance storage
 6 |   """
 7 | 
 8 |   @shortdoc """
 9 |   Fetches instances from instances.joinpeertube.org and adds the not yet known instances to the instance storage
10 |   """
11 |   def run(_) do
12 |     Application.ensure_all_started :httpoison
13 |     Application.ensure_all_started :ecto_sql
14 |     Application.ensure_all_started :postgrex
15 |     PeertubeIndex.StatusStorage.Repo.start_link()
16 | 
17 |     {:ok, response} = HTTPoison.get "https://instances.joinpeertube.org/api/v1/instances?count=1000000000000000000"
18 |     response.body
19 |     |> Poison.decode!()
20 |     |> Map.get("data")
21 |     |> Enum.map(fn instance -> instance["host"] end)
22 |     |> MapSet.new()
23 |     |> Enum.to_list()
24 |     |> PeertubeIndex.add_instances()
25 |   end
26 | end
27 | 


--------------------------------------------------------------------------------
/lib/mix/tasks/seed_from_instances_the_federation_info.ex:
--------------------------------------------------------------------------------
 1 | defmodule Mix.Tasks.SeedFromTheFederationInfo do
 2 |   use Mix.Task
 3 | 
 4 |   @moduledoc """
 5 |   Fetches instances from the-federation.info and adds the not yet known instances to the instance storage
 6 |   """
 7 | 
 8 |   @shortdoc """
 9 |   Fetches instances from the-federation.info and adds the not yet known instances to the instance storage
10 |   """
11 |   def run(_) do
12 |     Application.ensure_all_started :httpoison
13 |     Application.ensure_all_started :ecto_sql
14 |     Application.ensure_all_started :postgrex
15 |     PeertubeIndex.StatusStorage.Repo.start_link()
16 | 
17 |     query = "{nodes(platform: \"peertube\") {host}}"
18 |     url = URI.encode("https://the-federation.info/graphql?query=" <> query)
19 |     {:ok, response} = HTTPoison.get(url)
20 | 
21 |     response.body
22 |     |> Poison.decode!()
23 |     |> Map.get("data")
24 |     |> Map.get("nodes")
25 |     |> Enum.map(fn instance -> instance["host"] end)
26 |     |> MapSet.new()
27 |     |> Enum.to_list()
28 |     |> PeertubeIndex.add_instances()
29 |   end
30 | end
31 | 


--------------------------------------------------------------------------------
/lib/peertube_index.ex:
--------------------------------------------------------------------------------
 1 | defmodule PeertubeIndex do
 2 |   @moduledoc """
 3 |   PeerTube Index use cases
 4 |   """
 5 | 
 6 |   require Logger
 7 | 
 8 |   @video_storage Application.fetch_env!(:peertube_index, :video_storage)
 9 |   @instance_scanner Application.fetch_env!(:peertube_index, :instance_scanner)
10 |   @status_storage Application.fetch_env!(:peertube_index, :status_storage)
11 | 
12 |   @spec search(String.t) :: [map]
13 |   def search(name) do
14 |     @video_storage.search(name, nsfw: false)
15 |   end
16 | 
17 |   @spec scan([String.t], (-> NaiveDateTime.t)) :: :ok
18 |   def scan(hostnames, get_local_time \\ &get_current_time_naivedatetime/0) do
19 |     banned_instances = @status_storage.find_instances(:banned)
20 |     for host <- hostnames do
21 |       if Enum.member?(banned_instances, host) do
22 |         Logger.info "Skipping banned instance #{host}"
23 |       else
24 |         Logger.info "Scanning instance #{host}"
25 |         result = @instance_scanner.scan(host)
26 |         scan_end = get_local_time.()
27 |         case result do
28 |           {:ok, {videos, found_instances}} ->
29 |             Logger.info "Scan successful for #{host}"
30 |             @video_storage.delete_instance_videos!(host)
31 |             @video_storage.insert_videos!(videos)
32 |             @status_storage.ok_instance(host, scan_end)
33 |             add_instances(found_instances, fn -> scan_end end)
34 |           {:error, reason} ->
35 |             Logger.info "Scan failed for #{host}, reason: #{inspect(reason)}"
36 |             @status_storage.failed_instance(host, reason, scan_end)
37 |             @video_storage.delete_instance_videos!(host)
38 |         end
39 |       end
40 |     end
41 | 
42 |     :ok
43 |   end
44 | 
45 |   @spec rescan((-> NaiveDateTime.t), ([String.t] -> :ok)) :: :ok
46 |   def rescan(get_local_time \\ &get_current_time_naivedatetime/0, scan_function \\ &scan/1) do
47 |     one_day_in_seconds = 24 * 60 * 60
48 |     maximum_date = NaiveDateTime.add(get_local_time.(), - one_day_in_seconds)
49 | 
50 |     instances_to_rescan =
51 |     @status_storage.find_instances(:discovered)
52 |     ++ @status_storage.find_instances(:ok, maximum_date)
53 |     ++ @status_storage.find_instances(:error, maximum_date)
54 | 
55 |     scan_function.(instances_to_rescan)
56 |   end
57 | 
58 |   @spec ban_instance(String.t, String.t, (-> NaiveDateTime.t)) :: :ok
59 |   def ban_instance(hostname, reason, get_local_time \\ &get_current_time_naivedatetime/0) do
60 |     @video_storage.delete_instance_videos!(hostname)
61 |     @status_storage.banned_instance(hostname, reason, get_local_time.())
62 |     :ok
63 |   end
64 | 
65 |   @spec remove_ban(String.t, (-> NaiveDateTime.t)) :: :ok
66 |   def remove_ban(hostname, get_local_time \\ &get_current_time_naivedatetime/0) do
67 |     @status_storage.discovered_instance(hostname, get_local_time.())
68 |     :ok
69 |   end
70 | 
71 |   @spec add_instances([String.t], (-> NaiveDateTime.t)) :: :ok
72 |   def add_instances(hostnames, get_local_time \\ &get_current_time_naivedatetime/0) do
73 |     current_time = get_local_time.()
74 |     for hostname <- hostnames do
75 |       if not @status_storage.has_a_status(hostname) do
76 |         @status_storage.discovered_instance(hostname, current_time)
77 |       end
78 |     end
79 |     :ok
80 |   end
81 | 
82 |   @spec get_current_time_naivedatetime :: NaiveDateTime.t
83 |   defp get_current_time_naivedatetime do
84 |     :calendar.local_time() |> NaiveDateTime.from_erl!()
85 |   end
86 | end
87 | 


--------------------------------------------------------------------------------
/lib/peertube_index/application.ex:
--------------------------------------------------------------------------------
 1 | defmodule PeertubeIndex.Application do
 2 |   @moduledoc false
 3 | 
 4 |   use Application
 5 | 
 6 |   def start(_type, _args) do
 7 |     children = [
 8 |       Plug.Cowboy.child_spec(
 9 |         scheme: :http,
10 |         plug: PeertubeIndex.WebServer,
11 |         options: [port: Confex.fetch_env!(:peertube_index, :http_api_port)]
12 |       )
13 |     ]
14 | 
15 |     opts = [strategy: :one_for_one, name: PeertubeIndex.WebServerSupervisor]
16 |     Supervisor.start_link(children, opts)
17 |   end
18 | end
19 | 


--------------------------------------------------------------------------------
/lib/peertube_index/frontend_helpers.ex:
--------------------------------------------------------------------------------
 1 | defmodule PeertubeIndex.FrontendHelpers do
 2 |   @moduledoc false
 3 | 
 4 |   def format_duration(seconds) do
 5 |     hours = div(seconds, 3600)
 6 |     if hours > 0 do
 7 |       formatted_minutes = seconds |> rem(3600) |> div(60) |> to_string |> String.pad_leading(2, "0")
 8 |       formatted_seconds = seconds |> rem(3600) |> rem(60) |> to_string |> String.pad_leading(2, "0")
 9 |       "#{hours}:#{formatted_minutes}:#{formatted_seconds}"
10 |     else
11 |       minutes = seconds |> div(60)
12 |       formatted_seconds = seconds |> rem(60) |> to_string |> String.pad_leading(2, "0")
13 |       "#{minutes}:#{formatted_seconds}"
14 |     end
15 |   end
16 | end
17 | 


--------------------------------------------------------------------------------
/lib/peertube_index/instance_scanner.ex:
--------------------------------------------------------------------------------
  1 | defmodule PeertubeIndex.InstanceScanner do
  2 |   @moduledoc false
  3 | 
  4 |   @callback scan(String.t, integer, boolean, integer) :: {:ok, {Enumerable.t, MapSet.t}} | {:error, any()}
  5 |   # With default arguments
  6 |   @callback scan(String.t) :: {:ok, {Enumerable.t, MapSet.t}} | {:error, any()}
  7 | end
  8 | 
  9 | defmodule PeertubeIndex.InstanceScanner.Http do
 10 |   @moduledoc false
 11 |   @behaviour PeertubeIndex.InstanceScanner
 12 |   require Logger
 13 |   use Params
 14 | 
 15 |   @user_agent "PeertubeIndex"
 16 | 
 17 |   @impl true
 18 |   def scan(host, page_size \\ 100, use_tls \\ true, request_timeout \\ 5000) do
 19 |     scheme = if use_tls, do: "https://", else: "http://"
 20 |     api_base_url = scheme <> host <> "/api/v1/"
 21 | 
 22 |     with true <- crawlable?(api_base_url <> "videos"),
 23 |          true <- crawlable?(api_base_url <> "server/followers"),
 24 |          true <- crawlable?(api_base_url <> "server/following"),
 25 |          {:ok, instances_from_followers} <- get_instances_from_followers(api_base_url <> "server/followers", page_size, request_timeout),
 26 |          {:ok, instances_from_following} <- get_instances_from_following(api_base_url <> "server/following", page_size, request_timeout),
 27 |          {:ok, videos, instances_from_videos} <- get_videos(api_base_url <> "videos", page_size, request_timeout) do
 28 | 
 29 |       instances =
 30 |       instances_from_videos
 31 |       |> MapSet.union(instances_from_followers)
 32 |       |> MapSet.union(instances_from_following)
 33 |       |> MapSet.delete(host)
 34 | 
 35 |       {:ok, {videos, instances}}
 36 |     else
 37 |       {:error, {:invalid_video_document, _faulty_video_document, validation_errors}} ->
 38 |         version =
 39 |         case get_json_page(api_base_url <> "config", request_timeout) do
 40 |           {:ok, %{"serverVersion" => fetched_version}} ->
 41 |             fetched_version
 42 |           _ ->
 43 |             nil
 44 |         end
 45 |         {
 46 |           :error, {
 47 |             :invalid_video_document,
 48 |             %{version: version},
 49 |             validation_errors
 50 |           }
 51 |         }
 52 |       other_error ->
 53 |         other_error
 54 |     end
 55 |   end
 56 | 
 57 |   defp crawlable?(url) do
 58 |     case Gollum.crawlable?(@user_agent, url) do
 59 |       :uncrawlable ->
 60 |         {:error, :robots_txt_disallowed}
 61 |       :crawlable ->
 62 |         true
 63 |       :undefined ->
 64 |         true
 65 |     end
 66 |   end
 67 | 
 68 |   defp get_instances_from_following(following_url, page_size, request_timeout) do
 69 |     following_url
 70 |     |> get_collection(page_size, request_timeout)
 71 |     |> Enum.reduce_while(
 72 |          {:ok, MapSet.new()},
 73 |          reducer_while_no_error(fn following, set_of_instances ->
 74 |            MapSet.put(set_of_instances, following["following"]["host"])
 75 |          end)
 76 |        )
 77 |   end
 78 | 
 79 |   defp get_instances_from_followers(followers_url, page_size, request_timeout) do
 80 |     followers_url
 81 |     |> get_collection(page_size, request_timeout)
 82 |     |> reduce_enum_while_no_error(MapSet.new(), fn follower, set_of_instances -> MapSet.put(set_of_instances, follower["follower"]["host"]) end)
 83 |   end
 84 | 
 85 |   #  Fetches videos with streaming, saves the result on a file on disk
 86 |   #  and returns a stream to read the videos from disk.
 87 |   #  Also returns a set instances found in the videos.
 88 |   @spec get_videos(String.t, integer, integer) :: {:ok, Enumerable.t, MapSet.t}
 89 |   defp get_videos(videos_url, page_size, request_timeout) do
 90 |     buffer_file_path = "video_buffer"
 91 |     buffer_file = File.open!(buffer_file_path, [:binary, :write])
 92 | 
 93 |     result_of_processing =
 94 |     videos_url
 95 |     |> get_collection(page_size, request_timeout)
 96 |     |> Stream.map(&validate_one_video_and_keep_errors/1)
 97 |     |> Stream.reject(&is_invalid_and_non_local_video?/1)
 98 |     |> reduce_enum_while_no_error({buffer_file, MapSet.new()}, &save_videos_and_reduce_instances/2)
 99 | 
100 |     File.close(buffer_file)
101 | 
102 |     case result_of_processing do
103 |       {:ok, {_buffer_file, instances}} ->
104 |         {
105 |           :ok,
106 |           Stream.resource(
107 |             fn -> File.open!(buffer_file_path, [:binary, :read]) end,
108 |             &read_next_video/1,
109 |             fn buffer_file ->
110 |               :ok = File.close(buffer_file)
111 |               :ok = File.rm(buffer_file_path)
112 |             end
113 |           ),
114 |           instances
115 |         }
116 |       {:error, reason} ->
117 |         File.rm(buffer_file_path)
118 |         {:error, reason}
119 |     end
120 |   end
121 | 
122 |   #  Iterate over the stream of videos and
123 |   #  - compute the set of instances found from the videos
124 |   #  - save videos to disk, excluding non local videos
125 |   #
126 |   #  The following format is used for serializing the collection of videos
127 |   #  start_of_file
128 |   #  [size_as_text,newline
129 |   #  term_as_binary,newline] repeated as many times as needed
130 |   #  end_of_file
131 |   #
132 |   #  size_as_text: the string representation of the binary size of term_as_binary,
133 |   #                if term_as_binary is 152 bytes long, then size_as_text is the string 152
134 |   #  newline: a line jump character, \n
135 |   #  term_as_binary: the bytes given by :erlang.term_to_binary
136 |   defp save_videos_and_reduce_instances(video = %{"isLocal" => true}, {buffer_file, instances}) do
137 |     # Save video to disk
138 |     term_binary = :erlang.term_to_binary(video)
139 |     size_field = term_binary |> byte_size() |> Integer.to_string()
140 |     IO.binwrite(buffer_file, size_field <> "\n" <> term_binary <> "\n")
141 | 
142 |     # Add instance
143 |     instances = MapSet.put(instances, video["account"]["host"])
144 | 
145 |     {buffer_file, instances}
146 |   end
147 | 
148 |   defp save_videos_and_reduce_instances(video = %{"isLocal" => false}, {buffer_file, instances}) do
149 |     # Just add instance
150 |     instances = MapSet.put(instances, video["account"]["host"])
151 |     {buffer_file, instances}
152 |   end
153 | 
154 |   defp read_next_video(buffer_file) do
155 |     case IO.binread(buffer_file, :line) do
156 |       line when is_binary(line) ->
157 |         {size, _} = Integer.parse(line)
158 |         item = buffer_file |> IO.binread(size) |> :erlang.binary_to_term()
159 |         IO.binread(buffer_file, 1) # Read newline
160 |         {[item], buffer_file}
161 |       :eof ->
162 |         {:halt, buffer_file}
163 |     end
164 |   end
165 | 
166 |   defp request_with_timeout(url, timeout) do
167 |     request = Task.async(fn ->
168 |       :httpc.request(
169 |         :get,
170 |         {String.to_charlist(url), [{String.to_charlist("User-Agent"), String.to_charlist(@user_agent)}]},
171 |         [],
172 |         body_format: :binary)
173 |     end)
174 |     case Task.yield(request, timeout) || Task.shutdown(request) do
175 |       {:ok, httpc_result} ->
176 |         {:ok, httpc_result}
177 |       nil ->
178 |         {:error, :timeout}
179 |     end
180 |   end
181 | 
182 |   defp request_successful(response) do
183 |     with {:ok, {
184 |       {_http_version, status_code, _reason_phrase}, _headers, body
185 |     }} <- response do
186 |       if status_code >= 400 do
187 |         {:error, :http_error}
188 |       else
189 |         {:ok, body}
190 |       end
191 |     end
192 |   end
193 | 
194 |   defp validate_page_data(page_data) do
195 |     if is_integer(Map.get(page_data, "total")) and is_list(Map.get(page_data, "data")) do
196 |       {:ok, page_data}
197 |     else
198 |       {:error, :page_invalid}
199 |     end
200 |   end
201 | 
202 |   defp get_json_page(url, request_timeout) do
203 |     Logger.debug fn -> "Getting #{url}" end
204 |     with {:ok, httpc_result} <- request_with_timeout(url, request_timeout),
205 |          {:ok, body} <- request_successful(httpc_result),
206 |          {:ok, parsed} <- Poison.decode(body) do
207 |       {:ok, parsed}
208 |     end
209 |   end
210 | 
211 |   defp get_page(url, request_timeout) do
212 |     with {:ok, parsed} <- get_json_page(url, request_timeout),
213 |          {:ok, parsed} <- validate_page_data(parsed) do
214 |       {:ok, parsed}
215 |     end
216 |   end
217 | 
218 |   defp url_with_params(url, params) do
219 |     params_fragment =
220 |       params
221 |       |> Map.to_list
222 |       |> Enum.map(fn {key, value} -> "#{key}=#{value}" end)
223 |       |> Enum.join("&")
224 | 
225 |     url <> "?" <> params_fragment
226 |   end
227 | 
228 |   defp generate_urls_after_first_page(paginated_collection_url, common_params, page_size, number_of_pages) do
229 |     for page_number <- 2..number_of_pages do
230 |       url_with_params(
231 |         paginated_collection_url,
232 |         Map.put(common_params, "start", (page_number - 1) * page_size)
233 |       )
234 |     end
235 |   end
236 | 
237 |   defp validate_one_video_and_keep_errors({:ok, video}) do
238 |     changeset = PeertubeIndex.InstanceScanner.VideoParams.from(video)
239 |     if changeset.valid? do
240 |       {:ok, video}
241 |     else
242 |       errors = Ecto.Changeset.traverse_errors(changeset, fn error -> error end)
243 |       {
244 |         :error, {
245 |           :invalid_video_document,
246 |           video,
247 |           errors
248 |         }
249 |       }
250 |     end
251 |   end
252 | 
253 |   defp validate_one_video_and_keep_errors({:error, reason}) do
254 |     {:error, reason}
255 |   end
256 | 
257 |   defp is_invalid_and_non_local_video?({:error, {:invalid_video_document, %{"isLocal" => false}, _errors}}) do
258 |     true
259 |   end
260 | 
261 |   defp is_invalid_and_non_local_video?(_) do
262 |     false
263 |   end
264 | 
265 |   #  Returns an stream of ok/error tuples for each item of the collection: {:ok, item} or {:error, reason}.
266 |   #  The errors that my be present are about the page fetching steps.
267 |   #  For example, with a page size of 2 items and 3 pages,
268 |   #  if there was an http error on the second page, the output would be :
269 |   #  `[{:ok, item}, {:ok, item}, {:error, :http_error}, {:ok, item}, {:ok, item}]`
270 |   @spec get_collection(String.t, integer(), integer()) :: Enumerable.t
271 |   defp get_collection(paginated_collection_url, page_size, request_timeout) do
272 |     paginated_collection_url
273 |     |> get_pages(page_size, request_timeout) # {:ok, page} or {:error, reason}
274 |     |> Stream.flat_map(&extract_page_items_and_keep_errors/1) # {:ok, video} or {:error, reason}
275 |   end
276 | 
277 |   #  Returns an enumerable of ok/error tuples for each page: {:ok, page_data} or {:error, reason}
278 |   #  If there is a single page the result is a list.
279 |   #  If there is more than one page the result is a stream.
280 |   @spec get_pages(String.t, integer(), integer()) :: Enumerable.t
281 |   defp get_pages(paginated_collection_url, page_size, request_timeout) do
282 |     common_params = %{
283 |       "count" => page_size,
284 |       "sort" => "createdAt"
285 |     }
286 |     case get_page(url_with_params(paginated_collection_url, common_params), request_timeout) do
287 |       {:ok, first_page} ->
288 |         # credo:disable-for-next-line Credo.Check.Refactor.PipeChainStart
289 |       number_of_pages = (first_page["total"] / page_size) |> Float.ceil() |> trunc()
290 |       Logger.debug fn -> "#{paginated_collection_url} has #{first_page["total"]} items, using #{number_of_pages} pages" end
291 |       if number_of_pages > 1 do
292 |         urls = generate_urls_after_first_page(paginated_collection_url, common_params, page_size, number_of_pages)
293 |         Stream.concat(
294 |           [{:ok, first_page}],
295 |           Stream.map(urls, fn url -> get_page(url, request_timeout) end)
296 |         )
297 |       else
298 |         [{:ok, first_page}]
299 |       end
300 |     {:error, reason} ->
301 |       [{:error, reason}]
302 |     end
303 |   end
304 | 
305 |   defp extract_page_items_and_keep_errors({:ok, page}) do
306 |     for item <- page["data"] do
307 |       {:ok, item}
308 |     end
309 |   end
310 | 
311 |   defp extract_page_items_and_keep_errors({:error, reason}) do
312 |     [{:error, reason}]
313 |   end
314 | 
315 |   #####
316 | 
317 |   # The accumulator is {:ok, real_accumulator} because the expected output is
318 |   # {:ok, value} or {:error, reason}
319 |   defp reduce_while_no_error({:ok, element}, {:ok, accumulator}, reducer) do
320 |     {
321 |       :cont,
322 |       {:ok, reducer.(element, accumulator)}
323 |     }
324 |   end
325 | 
326 |   defp reduce_while_no_error({:error, reason}, {:ok, _accumulator}, _reducer) do
327 |     {:halt, {:error, reason}}
328 |   end
329 | 
330 |   # reducer: (element, accumulator -> accumulator)
331 |   @spec reducer_while_no_error(
332 |           (any(), any() -> any())
333 |   ) :: ({:ok, any()} | {:error, any()}, {:ok, any()} -> {:cont, {:ok, any()}} | {:halt, {:error, any()}})
334 |   defp reducer_while_no_error(reducer) do
335 |     fn element, accumulator -> reduce_while_no_error(element, accumulator, reducer) end
336 |   end
337 | 
338 |   #  Reduces an enumerable whose elements are either {:ok, element} or {:error, reason}
339 |   #  and stops if at the first {:error, reason} found.
340 |   #
341 |   #  The reducer function receives the second element of each tuple.
342 |   #
343 |   #  If an error is found, is is returned as is.
344 |   #  If no error is found, the returned value is {:ok, accumulator}.
345 |   @spec reduce_enum_while_no_error(Enumerable.t, any(), (any(), any() -> any())) :: {:ok, any()} | {:error, any()}
346 |   defp reduce_enum_while_no_error(enum, acc, fun) do
347 |     Enum.reduce_while(enum, {:ok, acc}, reducer_while_no_error(fun))
348 |   end
349 | end
350 | 
351 | defmodule PeertubeIndex.InstanceScanner.VideoParams do
352 |   @moduledoc false
353 | 
354 |   use Params.Schema, %{
355 |     id!: :integer,
356 |     uuid!: :string,
357 |     name!: :string,
358 |     category!: %{
359 |       id: :integer,
360 |       label!: :string
361 |     },
362 |     licence!: %{
363 |       id: :integer,
364 |       label!: :string
365 |     },
366 |     language!: %{
367 | #      id: :integer before 1.4.0 and :string 1.4.0 onwards
368 |       label!: :string
369 |     },
370 |     privacy: %{ # we used to require this field but found it may be absent
371 |       id!: :integer,
372 |       label!: :string
373 |     },
374 |     nsfw!: :boolean,
375 |     description: :string,
376 |     isLocal!: :boolean,
377 |     duration!: :integer,
378 |     views!: :integer,
379 |     likes!: :integer,
380 |     dislikes!: :integer,
381 |     thumbnailPath!: :string,
382 |     previewPath!: :string,
383 |     embedPath!: :string,
384 |     createdAt!: :string, # Validate date format?
385 |     updatedAt!: :string, # Validate date format?
386 |     publishedAt!: :string, # Validate date format?
387 |     account!: %{
388 |       id: :integer, # we used to require this field but found it may be absent
389 |       uuid: :string, # we used to require this field but found it may be absent
390 |       name!: :string,
391 |       displayName!: :string,
392 |       url!: :string,
393 |       host!: :string,
394 |       avatar: %{
395 |         path: :string,
396 |         createdAt: :string, # Validate date format?
397 |         updatedAt: :string, # Validate date format?
398 |       }
399 |     },
400 |     channel: %{ # we used to require this field but found it may be absent
401 |       id: :integer,
402 |       uuid: :string,
403 |       name: :string,
404 |       displayName: :string,
405 |       url: :string,
406 |       host: :string,
407 |       avatar: %{
408 |         path: :string,
409 |         createdAt: :string, # Validate date format?
410 |         updatedAt: :string, # Validate date format?
411 |       }
412 |     }
413 |   }
414 | end
415 | 


--------------------------------------------------------------------------------
/lib/peertube_index/status_storage.ex:
--------------------------------------------------------------------------------
  1 | defmodule PeertubeIndex.StatusStorage do
  2 |   @moduledoc false
  3 | 
  4 |   @doc """
  5 |   Create an empty status storage for testing
  6 |   """
  7 |   @callback empty() :: :ok
  8 | 
  9 |   @doc """
 10 |   Create a status storage with given statuses for testing
 11 |   """
 12 |   @callback with_statuses([tuple()]) :: :ok
 13 | 
 14 |   @doc """
 15 |   Returns the list of all statuses
 16 |   """
 17 |   @callback all() :: [tuple()]
 18 | 
 19 |   @doc """
 20 |   Find instances that have the given status and with a status updated before the given date
 21 |   """
 22 |   @callback find_instances(:ok | :error | :discovered, NaiveDateTime.t) :: [String.t]
 23 | 
 24 |   @doc """
 25 |   Find instances that have the given status
 26 |   """
 27 |   @callback find_instances(:ok | :error | :discovered) :: [String.t]
 28 | 
 29 |   @doc """
 30 |   Notify a successful instance scan at the given datetime
 31 |   """
 32 |   @callback ok_instance(String.t, NaiveDateTime.t) :: :ok
 33 | 
 34 |   @doc """
 35 |   Notify a failed instance scan, with a reason, at the given datetime
 36 |   """
 37 |   @callback failed_instance(String.t, any(), NaiveDateTime.t) :: :ok
 38 | 
 39 |   @doc """
 40 |   Notify a discovered instance at the given datetime.
 41 |   This will not override any previously existing status for the same instance.
 42 |   """
 43 |   @callback discovered_instance(String.t, NaiveDateTime.t) :: :ok
 44 | 
 45 |   @doc """
 46 |   Notify a banned instance, with a reason, at the given datetime
 47 |   """
 48 |   @callback banned_instance(String.t, String.t, NaiveDateTime.t) :: :ok
 49 | 
 50 |   @doc """
 51 |   Returns true if a instance (identified by it's hostname) has a status in the database,
 52 |   and false otherwise
 53 |   """
 54 |   @callback has_a_status(String.t) :: boolean()
 55 | end
 56 | 
 57 | defmodule PeertubeIndex.StatusStorage.Postgresql do
 58 |   @moduledoc false
 59 | 
 60 |   @behaviour PeertubeIndex.StatusStorage
 61 | 
 62 |   @impl true
 63 |   def empty do
 64 |     Mix.Tasks.Ecto.Drop.run([])
 65 |     Mix.Tasks.Ecto.Create.run([])
 66 |     Mix.Tasks.Ecto.Migrate.run([])
 67 |     :ok
 68 |   end
 69 | 
 70 |   @impl true
 71 |   def with_statuses(statuses) do
 72 |     for status <- statuses do
 73 |       case status do
 74 |         {host, :ok, date} ->
 75 |           Ecto.Adapters.SQL.query(
 76 |             PeertubeIndex.StatusStorage.Repo,
 77 |             "INSERT INTO statuses (host, status, date) VALUES ('#{host}', 'ok', '#{NaiveDateTime.to_iso8601(date)}')"
 78 |           )
 79 |         {host, {:error, reason}, date} ->
 80 |           Ecto.Adapters.SQL.query(
 81 |             PeertubeIndex.StatusStorage.Repo,
 82 |             "INSERT INTO statuses (host, status, reason, date) VALUES ('#{host}', 'error', '#{inspect(reason)}', '#{NaiveDateTime.to_iso8601(date)}')"
 83 |           )
 84 |         {host, :discovered, date} ->
 85 |           Ecto.Adapters.SQL.query(
 86 |             PeertubeIndex.StatusStorage.Repo,
 87 |             "INSERT INTO statuses (host, status, date) VALUES ('#{host}', 'discovered', '#{NaiveDateTime.to_iso8601(date)}')"
 88 |           )
 89 |         {host, {:banned, reason}, date} ->
 90 |           Ecto.Adapters.SQL.query(
 91 |             PeertubeIndex.StatusStorage.Repo,
 92 |             "INSERT INTO statuses (host, status, reason, date) VALUES ('#{host}', 'banned', '#{reason}', '#{NaiveDateTime.to_iso8601(date)}')"
 93 |           )
 94 |       end
 95 |     end
 96 |     :ok
 97 |   end
 98 | 
 99 |   @impl true
100 |   def all do
101 |     {:ok, result} = Ecto.Adapters.SQL.query PeertubeIndex.StatusStorage.Repo, "select host, status, reason, date from statuses"
102 |     for row <- result.rows do
103 |       case List.to_tuple(row) do
104 |         {host, "discovered", nil, date} ->
105 |           {host, :discovered, date |> NaiveDateTime.truncate(:second)}
106 |         {host, "ok", nil, date} ->
107 |           {host, :ok, date |> NaiveDateTime.truncate(:second)}
108 |         {host, "error", reason_string, date} ->
109 |           {host, {:error, reason_string}, date |> NaiveDateTime.truncate(:second)}
110 |         {host, "banned", reason_string, date} ->
111 |           {host, {:banned, reason_string}, date |> NaiveDateTime.truncate(:second)}
112 |       end
113 |     end
114 |   end
115 | 
116 |   @impl true
117 |   def find_instances(wanted_status, maximum_date) do
118 |     {:ok, r} = Ecto.Adapters.SQL.query(
119 |       PeertubeIndex.StatusStorage.Repo,
120 |       "
121 |       SELECT host
122 |       FROM statuses
123 |       WHERE status = '#{wanted_status}'
124 |       AND date < '#{NaiveDateTime.to_iso8601(maximum_date)}'
125 |       "
126 |     )
127 |     r.rows
128 |     |> Enum.map(&Enum.at(&1, 0))
129 |   end
130 | 
131 |   @impl true
132 |   def find_instances(wanted_status) do
133 |     {:ok, r} = Ecto.Adapters.SQL.query(
134 |       PeertubeIndex.StatusStorage.Repo,
135 |       "SELECT host FROM statuses WHERE status = '#{wanted_status}'"
136 |     )
137 |     r.rows
138 |     |> Enum.map(&Enum.at(&1, 0))
139 |   end
140 | 
141 |   @impl true
142 |   def ok_instance(host, date) do
143 |     {:ok, _} = Ecto.Adapters.SQL.query(
144 |       PeertubeIndex.StatusStorage.Repo,
145 |       "
146 |       INSERT INTO statuses (host, status, date)
147 |       VALUES ($1, 'ok', $2)
148 |       ON CONFLICT (host)
149 |       DO UPDATE SET status = EXCLUDED.status, date = EXCLUDED.date
150 |       ",
151 |       [host, date]
152 |     )
153 |     :ok
154 |   end
155 | 
156 |   @impl true
157 |   def failed_instance(host, reason, date) do
158 |     {:ok, _} = Ecto.Adapters.SQL.query(
159 |       PeertubeIndex.StatusStorage.Repo,
160 |       "
161 |       INSERT INTO statuses (host, status, reason, date)
162 |       VALUES ($1, 'error', $2, $3)
163 |       ON CONFLICT (host)
164 |       DO UPDATE SET status = EXCLUDED.status, reason = EXCLUDED.reason, date = EXCLUDED.date
165 |       ",
166 |       [host, inspect(reason), date]
167 |     )
168 |     :ok
169 |   end
170 | 
171 |   @impl true
172 |   def discovered_instance(host, date) do
173 |    {:ok, _} = Ecto.Adapters.SQL.query(
174 |       PeertubeIndex.StatusStorage.Repo,
175 |       "
176 |       INSERT INTO statuses (host, status, reason, date)
177 |       VALUES ($1, 'discovered', null, $2)
178 |       ON CONFLICT (host)
179 |       DO UPDATE SET status = EXCLUDED.status, reason = EXCLUDED.reason, date = EXCLUDED.date
180 |       ",
181 |       [host, date]
182 |     )
183 |     :ok
184 |   end
185 | 
186 |   @impl true
187 |   def banned_instance(host, reason, date) do
188 |     {:ok, _} = Ecto.Adapters.SQL.query(
189 |       PeertubeIndex.StatusStorage.Repo,
190 |       "
191 |       INSERT INTO statuses (host, status, reason, date)
192 |       VALUES ($1, 'banned', $2, $3)
193 |       ON CONFLICT (host)
194 |       DO UPDATE SET status = EXCLUDED.status, reason = EXCLUDED.reason, date = EXCLUDED.date
195 |       ",
196 |       [host, reason, date]
197 |     )
198 |     :ok
199 |   end
200 | 
201 |   @impl true
202 |   def has_a_status(host) do
203 |     {:ok, r} = Ecto.Adapters.SQL.query(
204 |       PeertubeIndex.StatusStorage.Repo,
205 |       "SELECT count(*) FROM statuses WHERE host = '#{host}'"
206 |     )
207 |     count = r.rows |> Enum.at(0) |> Enum.at(0)
208 |     count == 1
209 |   end
210 | end
211 | 
212 | defmodule PeertubeIndex.StatusStorage.Repo do
213 |   use Ecto.Repo,
214 |     otp_app: :peertube_index,
215 |     adapter: Ecto.Adapters.Postgres
216 | 
217 |   def init(_, config) do
218 |     url = Confex.fetch_env!(:peertube_index, :status_storage_database_url)
219 |     {:ok, Keyword.put(config, :url, url)}
220 |   end
221 | end
222 | 


--------------------------------------------------------------------------------
/lib/peertube_index/templates.ex:
--------------------------------------------------------------------------------
 1 | defmodule PeertubeIndex.Templates do
 2 |   @moduledoc false
 3 |   require EEx
 4 | 
 5 |   for {template_name, arguments} <- [
 6 |     {:about, []},
 7 |     {:home, []},
 8 |     {:search, [:videos, :query]},
 9 |     {:search_bar, [:query]},
10 |     {:warning_footer, []},
11 |     {:retirement_message, []},
12 |   ] do
13 |     EEx.function_from_file(:def, template_name, "frontend/#{template_name}.html.eex", arguments)
14 |   end
15 | end
16 | 


--------------------------------------------------------------------------------
/lib/peertube_index/video_storage.ex:
--------------------------------------------------------------------------------
  1 | defmodule PeertubeIndex.VideoStorage do
  2 |   @moduledoc false
  3 | 
  4 |   @doc """
  5 |   Removes existing videos of the given instance
  6 |   """
  7 |   @callback delete_instance_videos!(String.t) :: :ok
  8 | 
  9 |   @doc """
 10 |   Inserts a list of videos.
 11 |   """
 12 |   @callback insert_videos!([map]) :: :ok
 13 | 
 14 |   @doc """
 15 |   Search for a video by its name
 16 |   Options:
 17 |     - nsfw:
 18 |       - missing: do not filter on safety, gets both safe and unsafe videos
 19 |       - true: only get unsafe for work videos
 20 |       - false: only get safe for work videos
 21 |   """
 22 |   @callback search(String.t, Keyword.t()) :: [map]
 23 | 
 24 |   @doc """
 25 |   Create a video storage with some videos for testing
 26 |   """
 27 |   @callback with_videos!([map]) :: :ok
 28 | 
 29 |   @doc """
 30 |   Create an empty video storage for testing
 31 |   """
 32 |   @callback empty!() :: :ok
 33 | end
 34 | 
 35 | defmodule PeertubeIndex.VideoStorage.Elasticsearch do
 36 |   @moduledoc false
 37 | 
 38 |   @behaviour PeertubeIndex.VideoStorage
 39 | 
 40 |   @index "videos"
 41 |   def elasticsearch_config, do: Confex.fetch_env!(:peertube_index, :elasticsearch_config)
 42 | 
 43 |   @impl true
 44 |   def delete_instance_videos!(hostname) do
 45 |     Elasticsearch.post!(
 46 |       elasticsearch_config(),
 47 |       "/#{@index}/_delete_by_query",
 48 |       %{"query" => %{"term" => %{"account.host" => hostname}}}
 49 |     )
 50 | 
 51 |     :ok
 52 |   end
 53 | 
 54 |   @impl true
 55 |   def insert_videos!(videos) do
 56 |     for video <- videos do
 57 |       Elasticsearch.post!(elasticsearch_config(), "/#{@index}/_doc", video)
 58 |     end
 59 | 
 60 |     :ok
 61 |   end
 62 | 
 63 |   @impl true
 64 |   def search(query, options \\ []) do
 65 |     elasticsearch_query = %{
 66 |       "from" => "0", "size" => 100,
 67 |       "query" => %{
 68 |         "bool" => %{
 69 |           "must" => [
 70 |             %{
 71 |               "match" => %{
 72 |                 "name" => %{
 73 |                   "query" => query,
 74 |                   "fuzziness" => "AUTO"
 75 |                 }
 76 |               }
 77 |             }
 78 |           ]
 79 |         }
 80 |       }
 81 |     }
 82 |     nsfw = options[:nsfw]
 83 |     elasticsearch_query =
 84 |     if is_nil(nsfw)  do
 85 |       elasticsearch_query
 86 |     else
 87 |       put_in(elasticsearch_query, ["query", "bool", "filter"], [%{"term" => %{"nsfw" => nsfw}}])
 88 |     end
 89 | 
 90 |     %{"hits" => %{"hits" => hits}} = Elasticsearch.post!(elasticsearch_config(), "/#{@index}/_search", elasticsearch_query)
 91 | 
 92 |     hits
 93 |     |> Enum.map(fn hit -> hit["_source"] end)
 94 |     |> Enum.to_list()
 95 |   end
 96 | 
 97 |   defp create_index! do
 98 |     :ok = Elasticsearch.Index.create(
 99 |       elasticsearch_config(),
100 |       @index,
101 |       %{
102 |         "mappings"=> %{
103 |           "properties"=> %{
104 |             "uuid"=> %{"type"=> "keyword"},
105 |             "name"=> %{"type"=> "text"},
106 |             "nsfw"=> %{"type"=> "boolean"},
107 |             "description"=> %{"type"=> "text"},
108 |             "duration"=> %{"type"=> "long"},
109 |             "views"=> %{"type"=> "long"},
110 |             "likes"=> %{"type"=> "long"},
111 |             "dislikes"=> %{"type"=> "long"},
112 |             "createdAt"=> %{"type"=> "date"},
113 |             "updatedAt"=> %{"type"=> "date"},
114 |             "publishedAt"=> %{"type"=> "date"},
115 |             "account"=> %{
116 |               "properties"=> %{
117 |                 "uuid"=> %{"type"=> "keyword"},
118 |                 "name"=> %{"type"=> "text"},
119 |                 "displayName"=> %{"type"=> "text"},
120 |                 "host"=> %{"type"=> "keyword"}
121 |               }
122 |             },
123 |             "channel"=> %{
124 |               "properties"=> %{
125 |                 "uuid"=> %{"type"=> "keyword"},
126 |                 "name"=> %{"type"=> "text"},
127 |                 "displayName"=> %{"type"=> "text"},
128 |                 "host"=> %{"type"=> "keyword"}
129 |               }
130 |             }
131 |           }
132 |         }
133 |       }
134 |     )
135 |   end
136 | 
137 |   @impl true
138 |   def with_videos!(videos) do
139 |     empty!()
140 |     videos
141 |     |> Enum.each(fn video -> Elasticsearch.post!(elasticsearch_config(), "/#{@index}/_doc", video) end)
142 |   end
143 | 
144 |   @impl true
145 |   def empty! do
146 |     delete_index_ignore_not_exising!()
147 |     create_index!()
148 |   end
149 | 
150 |   defp delete_index_ignore_not_exising! do
151 |     result = Elasticsearch.delete(elasticsearch_config(), "/#{@index}")
152 |     case result do
153 |       {:ok, _} ->
154 |         :ok
155 |       {:error, %Elasticsearch.Exception{type: "index_not_found_exception"}} ->
156 |         :ok
157 |       {:error, unexpected_error} ->
158 |         raise unexpected_error
159 |     end
160 |   end
161 | 
162 | end
163 | 


--------------------------------------------------------------------------------
/lib/peertube_index/web_server.ex:
--------------------------------------------------------------------------------
 1 | defmodule PeertubeIndex.WebServer do
 2 |   @moduledoc false
 3 | 
 4 |   use Plug.Router
 5 | 
 6 |   plug Plug.Logger
 7 |   plug Plug.Static,
 8 |     at: "/static",
 9 |     from: "frontend/static"
10 |   plug :match
11 |   plug :dispatch
12 | 
13 |   get "/ping" do
14 |     send_resp(conn, 200, "pong")
15 |   end
16 | 
17 |   get "/" do
18 |     conn = put_resp_content_type(conn, "text/html")
19 |     send_resp(conn, 200, PeertubeIndex.Templates.home())
20 |   end
21 | 
22 |   get "/search" do
23 |     search = conn.assigns[:search_usecase_function] || &PeertubeIndex.search/1
24 |     render = conn.assigns[:render_page_function] || &PeertubeIndex.Templates.search/2
25 | 
26 |     conn = fetch_query_params(conn)
27 |     query = conn.query_params["text"]
28 |     case query do
29 |       "" ->
30 |         conn = put_resp_content_type(conn, "text/html")
31 |         conn = put_resp_header(conn, "location", "/")
32 |         send_resp(conn, 302, "")
33 |       nil ->
34 |         conn = put_resp_content_type(conn, "text/html")
35 |         conn = put_resp_header(conn, "location", "/")
36 |         send_resp(conn, 302, "")
37 |       _ ->
38 |         videos = search.(query)
39 |         conn = put_resp_content_type(conn, "text/html")
40 |         send_resp(conn, 200, render.(videos, query))
41 |     end
42 |   end
43 | 
44 |   get "/api/search" do
45 |     search = conn.assigns[:search_usecase_function] || &PeertubeIndex.search/1
46 | 
47 |     conn = fetch_query_params(conn)
48 |     videos = search.(Map.get(conn.query_params, "text"))
49 |     conn = put_resp_content_type(conn, "application/json")
50 |     send_resp(conn, 200, Poison.encode!(videos))
51 |   end
52 | 
53 |   get "/about" do
54 |     conn = put_resp_content_type(conn, "text/html")
55 |     send_resp(conn, 200, PeertubeIndex.Templates.about())
56 |   end
57 | 
58 |   match _ do
59 |     send_resp(conn, 404, "Not found")
60 |   end
61 | end
62 | 


--------------------------------------------------------------------------------
/mix.exs:
--------------------------------------------------------------------------------
 1 | defmodule PeertubeIndex.MixProject do
 2 |   use Mix.Project
 3 | 
 4 |   def project do
 5 |     [
 6 |       app: :peertube_index,
 7 |       version: "0.1.0",
 8 |       elixir: "~> 1.9",
 9 |       start_permanent: Mix.env() == :prod,
10 |       deps: deps(),
11 |       elixirc_paths: elixirc_paths(Mix.env),
12 |     ]
13 |   end
14 | 
15 |   def elixirc_paths(:test), do: ["test/support", "lib"]
16 |   def elixirc_paths(_),     do: ["lib"]
17 | 
18 |   # Run "mix help compile.app" to learn about applications.
19 |   def application do
20 |     [
21 |       mod: {PeertubeIndex.Application, []}
22 |     ]
23 |   end
24 | 
25 |   # Run "mix help deps" to learn about dependencies.
26 |   defp deps do
27 |     [
28 |       {:confex, "~> 3.4"},
29 |       {:ecto_sql, "~> 3.1"},
30 |       {:postgrex, ">= 0.0.0"},
31 |       {:elasticsearch, "~> 1.0"},
32 |       {:gollum, github: "silicium14/gollum", ref: "ff84c9c00433ce0d5ff75697ec2f32d34750d6d8"},
33 |       {:httpoison, "~> 1.5"},
34 |       {:params, "~> 2.1"},
35 |       {:phoenix_html, "~> 2.13"},
36 |       {:plug, "~> 1.8"},
37 |       {:plug_cowboy, "~> 2.0"},
38 |       {:poison, "~> 4.0"},
39 |       # Dev only
40 |       {:bypass, "~> 1.0.0", only: :test},
41 |       {:credo, "~> 1.1", only: :dev, runtime: false},
42 |       {:dialyxir, "~> 1.0.0-rc.4", only: [:dev], runtime: false},
43 |       {:mox, "~> 0.5.1", only: :test},
44 |       {:remix, "~> 0.0.2", only: :dev}
45 |     ]
46 |   end
47 | end
48 | 


--------------------------------------------------------------------------------
/mix.lock:
--------------------------------------------------------------------------------
 1 | %{
 2 |   "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"},
 3 |   "bypass": {:hex, :bypass, "1.0.0", "b78b3dcb832a71aca5259c1a704b2e14b55fd4e1327ff942598b4e7d1a7ad83d", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}], "hexpm", "5a1dc855dfcc86160458c7a70d25f65d498bd8012bd4c06a8d3baa368dda3c45"},
 4 |   "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "805abd97539caf89ec6d4732c91e62ba9da0cda51ac462380bbd28ee697a8c42"},
 5 |   "confex": {:hex, :confex, "3.4.0", "8b1c3cc7a93320291abb31223a178df19d7f722ee816c05a8070c8c9a054560d", [:mix], [], "hexpm", "4a14e15185c772389979cf4c050ddcc7a25a4d62759da13a170e0ca7274a22c7"},
 6 |   "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm", "4a0850c9be22a43af9920a71ab17c051f5f7d45c209e40269a1938832510e4d9"},
 7 |   "cowboy": {:hex, :cowboy, "2.6.3", "99aa50e94e685557cad82e704457336a453d4abcb77839ad22dbe71f311fcc06", [:rebar3], [{:cowlib, "~> 2.7.3", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "e5580029080f3f1ad17436fb97b0d5ed2ed4e4815a96bac36b5a992e20f58db6"},
 8 |   "cowlib": {:hex, :cowlib, "2.7.3", "a7ffcd0917e6d50b4d5fb28e9e2085a0ceb3c97dea310505f7460ff5ed764ce9", [:rebar3], [], "hexpm", "1e1a3d176d52daebbecbbcdfd27c27726076567905c2a9d7398c54da9d225761"},
 9 |   "credo": {:hex, :credo, "1.1.0", "e0c07b2fd7e2109495f582430a1bc96b2c71b7d94c59dfad120529f65f19872f", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "7338d04b30026e30adbcaaedbf0eb7e4d749510d90c2708ff8cc100fa9c8291f"},
10 |   "db_connection": {:hex, :db_connection, "2.1.0", "122e2f62c4906bf2e49554f1e64db5030c19229aa40935f33088e7d543aa79d0", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "f398f3144db606de82ed42c2ddc69767f0607abdb796e8220de5f0fcf80f5ba4"},
11 |   "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm", "52694ef56e60108e5012f8af9673874c66ed58ac1c4fae9b5b7ded31786663f5"},
12 |   "dialyxir": {:hex, :dialyxir, "1.0.0-rc.4", "71b42f5ee1b7628f3e3a6565f4617dfb02d127a0499ab3e72750455e986df001", [:mix], [{:erlex, "~> 0.1", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "4bba10c6f267a0dd127d687d1295f6a11af6a7f160cc0e261c46f1962a98d7d8"},
13 |   "ecto": {:hex, :ecto, "3.1.7", "fa21d06ef56cdc2fdaa62574e8c3ba34a2751d44ea34c30bc65f0728421043e5", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "fd0f11a8454e490ae11b6f69aa1ed9e0352641242d014cc3d2f420d7743f6966"},
14 |   "ecto_sql": {:hex, :ecto_sql, "3.1.6", "1e80e30d16138a729c717f73dcb938590bcdb3a4502f3012414d0cbb261045d8", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0 or ~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cdb6a76a6d88b256fd1bfc37da66cfc96f0935591c5114c1123b04c150828b69"},
15 |   "elasticsearch": {:hex, :elasticsearch, "1.0.0", "626d3fb8e7554d9c93eb18817ae2a3d22c2a4191cc903c4644b1334469b15374", [:mix], [{:httpoison, ">= 0.0.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sigaws, "~> 0.7", [hex: :sigaws, repo: "hexpm", optional: true]}, {:vex, "~> 0.6.0", [hex: :vex, repo: "hexpm", optional: false]}], "hexpm", "9fa0b717ad57a54c28451b3eb10c5121211c29a7b33615d2bcc7e2f3c9418b2e"},
16 |   "erlex": {:hex, :erlex, "0.1.6", "c01c889363168d3fdd23f4211647d8a34c0f9a21ec726762312e08e083f3d47e", [:mix], [], "hexpm", "f9388f7d1a668bee6ebddc040422ed6340af74aced153e492330da4c39516d92"},
17 |   "gollum": {:git, "https://github.com/silicium14/gollum.git", "ff84c9c00433ce0d5ff75697ec2f32d34750d6d8", [ref: "ff84c9c00433ce0d5ff75697ec2f32d34750d6d8"]},
18 |   "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "c2790c9f0f7205f4a362512192dee8179097394400e745e4d20bab7226a8eaad"},
19 |   "httpoison": {:hex, :httpoison, "1.5.1", "0f55b5b673b03c5c327dac7015a67cb571b99b631acc0bc1b0b98dcd6b9f2104", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "191a3b6329c917de4e7ca68431919a59bf19e60694b313a69bc1f56a4cb160bf"},
20 |   "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "4bdd305eb64e18b0273864920695cb18d7a2021f31a11b9c5fbcd9a253f936e2"},
21 |   "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fdf843bca858203ae1de16da2ee206f53416bbda5dc8c9e78f43243de4bc3afe"},
22 |   "maybe": {:hex, :maybe, "1.0.0", "65311dd7e16659579116666b268d03d7e1d1b3da8776c81a6b199de7177b43d6", [:mix], [], "hexpm"},
23 |   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
24 |   "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm", "6cbe761d6a0ca5a31a0931bf4c63204bceb64538e664a8ecf784a9a6f3b875f1"},
25 |   "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
26 |   "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm", "052346cf322311c49a0f22789f3698eea030eec09b8c47367f0686ef2634ae14"},
27 |   "params": {:hex, :params, "2.1.1", "0b37c6bcc6bdba81241c546d79e47a18a42c45c22a0dd3dc0f6455303cf9187f", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "f93d0770f454728771fe6181552776da5a9d2f861a27d586955a40699f7afcfa"},
28 |   "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"},
29 |   "phoenix_html": {:hex, :phoenix_html, "2.13.3", "850e292ff6e204257f5f9c4c54a8cb1f6fbc16ed53d360c2b780a3d0ba333867", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "8b01b3d6d39731ab18aa548d928b5796166d2500755f553725cfe967bafba7d9"},
30 |   "plug": {:hex, :plug, "1.8.2", "0bcce1daa420f189a6491f3940cc77ea7fb1919761175c9c3b59800d897440fc", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "54c8bbd5062cb32880aa1afc9894a823b1c18a47a1821a552887310b561cd418"},
31 |   "plug_cowboy": {:hex, :plug_cowboy, "2.0.2", "6055f16868cc4882b24b6e1d63d2bada94fb4978413377a3b32ac16c18dffba2", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "44a58653dda2b8cf69757c7f14c0920d4df0f79305ed8571bb93b2ea0a31a895"},
32 |   "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm", "73c1682f0e414cfb5d9b95c8e8cd6ffcfdae699e3b05e1db744e58b7be857759"},
33 |   "poison": {:hex, :poison, "4.0.1", "bcb755a16fac91cad79bfe9fc3585bb07b9331e50cfe3420a24bcc2d735709ae", [:mix], [], "hexpm", "ba8836feea4b394bb718a161fc59a288fe0109b5006d6bdf97b6badfcf6f0f25"},
34 |   "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "0ec1f09319f29b4dfc2d0e08c776834d219faae0da3536f3d9460f6793e6af1f"},
35 |   "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"},
36 |   "remix": {:hex, :remix, "0.0.2", "f06115659d8ede8d725fae1708920ef73353a1b39efe6a232d2a38b1f2902109", [:mix], [], "hexpm", "5f5555646ed4fca83fab8620735150aa0bc408c5a17a70d28cfa7086bc6f497c"},
37 |   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm", "603561dc0fd62f4f2ea9b890f4e20e1a0d388746d6e20557cafb1b16950de88c"},
38 |   "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm", "e9e3cacfd37c1531c0ca70ca7c0c30ce2dbb02998a4f7719de180fe63f8d41e4"},
39 |   "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm", "1d1848c40487cdb0b30e8ed975e34e025860c02e419cb615d255849f3427439d"},
40 |   "vex": {:hex, :vex, "0.6.0", "4e79b396b2ec18cd909eed0450b19108d9631842598d46552dc05031100b7a56", [:mix], [], "hexpm", "7e4d9b50dd72cf931b52aba3470513686007f2ad54832de37cdb659cc85ba73e"},
41 | }
42 | 


--------------------------------------------------------------------------------
/priv/repo/migrations/20190816201502_create_statuses_table.exs:
--------------------------------------------------------------------------------
 1 | defmodule PeertubeIndex.StatusStorage.Repo.Migrations.CreateStatusesTable do
 2 |   use Ecto.Migration
 3 | 
 4 |   def change do
 5 |     execute(
 6 |       "CREATE TYPE status AS ENUM ('ok', 'error', 'discovered', 'banned')",
 7 |       "DROP TYPE status"
 8 |     )
 9 |     # TODO `:string` type should be postgres type `text` and not `character varying(255)`
10 |     create table(:statuses, primary_key: false) do
11 |       add :host, :text, primary_key: true
12 |       add :status, :status, null: false
13 |       add :reason, :text
14 |       add :date, :naive_datetime, null: false, default: fragment("now()")
15 |     end
16 |   end
17 | end
18 | 


--------------------------------------------------------------------------------
/scan_loop.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env sh
2 | 
3 | while [ 1 ]; do
4 |     echo "$(date) Start scanning"
5 |     { time mix rescan; }
6 |     echo "$(date) Finished scanning"
7 |     sleep 300
8 | done
9 | 


--------------------------------------------------------------------------------
/seed_loop.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env sh
 2 | 
 3 | PAUSE_DURATION=300
 4 | 
 5 | while [ 1 ]; do
 6 |     echo "$(date) Seeding from instances.joinpeertube.org"
 7 |     { time mix seed_from_instances_joinpeertube_org; }
 8 |     echo "$(date) Seeding from the-federation.info"
 9 |     { time mix seed_from_the_federation_info; }
10 |     echo "$(date) Waiting ${PAUSE_DURATION} seconds"
11 |     sleep ${PAUSE_DURATION}
12 | done
13 | 


--------------------------------------------------------------------------------
/test/peertube_index/frontend_helpers_test.exs:
--------------------------------------------------------------------------------
 1 | defmodule FrontendHelpersTest do
 2 |   use ExUnit.Case, async: true
 3 | 
 4 |   alias PeertubeIndex.FrontendHelpers
 5 | 
 6 |   @moduletag :capture_log
 7 | 
 8 |   doctest FrontendHelpers
 9 | 
10 |   for {seconds, expected_output} <- [
11 |     {10, "0:10"},
12 |     {70, "1:10"},
13 |     {3600 * 2 + 60 * 30 + 11, "2:30:11"},
14 |     {60, "1:00"},
15 |     {3600, "1:00:00"},
16 |     {3600 + 1, "1:00:01"},
17 |     {3600 * 25, "25:00:00"}
18 |   ] do
19 |     test "format_duration converts #{seconds} to #{expected_output}" do
20 |       assert FrontendHelpers.format_duration(unquote(seconds)) == unquote(expected_output)
21 |     end
22 |   end
23 | end
24 | 


--------------------------------------------------------------------------------
/test/peertube_index/instance_scanner_non_regression_test.exs:
--------------------------------------------------------------------------------
 1 | defmodule InstanceScannerNonRegressionTest do
 2 |   use ExUnit.Case, async: false
 3 |   @moduledoc """
 4 |   Non regression tests for PeerTube instance scanner module.
 5 |   The state of the reference PeerTube instance we use changes frequently,
 6 |   to take this this into account we suggest the following workflow:
 7 |   - checkout to a known working version of the instance scanner module
 8 |   - update reference dataset with `mix refresh_instance_scanner_non_regression_reference`
 9 |   - checkout to the instance scanner module version to test
10 |   - run this test
11 |   """
12 | 
13 |   @moduletag :nonregression
14 | 
15 |   @reference_videos_file "test/peertube_index/instance_scanner_non_regression_test_data/reference_videos.json"
16 |   @reference_instances_file "test/peertube_index/instance_scanner_non_regression_test_data/reference_instances.json"
17 |   @result_videos_file "test/peertube_index/instance_scanner_non_regression_test_data/videos.json"
18 |   @result_instances_file "test/peertube_index/instance_scanner_non_regression_test_data/instances.json"
19 | 
20 |   setup_all do
21 |     {:ok, {videos, instances}} = PeertubeIndex.InstanceScanner.Http.scan("peertube.cpy.re", 5)
22 |     videos = Enum.to_list(videos)
23 | 
24 |     # Save results for debugging
25 |     File.write!(
26 |       @result_videos_file,
27 |       Poison.encode!(videos, pretty: true),
28 |       [:binary, :write]
29 |     )
30 |     File.write!(
31 |       @result_instances_file,
32 |       Poison.encode!(MapSet.to_list(instances), pretty: true),
33 |       [:binary, :write]
34 |     )
35 | 
36 |     %{videos: videos, instances: instances}
37 |   end
38 | 
39 |   test "scan gives the same videos", %{videos: videos} do
40 |     expected = @reference_videos_file |> File.read!() |> Poison.decode!
41 |     assert videos == expected, "Results differ from reference, compare reference: #{@reference_videos_file} with results: #{@result_videos_file}"
42 |   end
43 | 
44 |   test "scan gives the same instances", %{instances: instances} do
45 |     expected = @reference_instances_file |> File.read!() |> Poison.decode! |> MapSet.new
46 |     assert MapSet.equal?(instances, expected), "Results differ from reference, compare reference: #{@reference_instances_file} with results: #{@result_instances_file}"
47 |   end
48 | end
49 | 


--------------------------------------------------------------------------------
/test/peertube_index/instance_scanner_non_regression_test_data/.gitignore:
--------------------------------------------------------------------------------
1 | *
2 | !.gitignore
3 | 


--------------------------------------------------------------------------------
/test/peertube_index/instance_scanner_test.exs:
--------------------------------------------------------------------------------
  1 | defmodule PeertubeIndex.InstanceScannerTest do
  2 |   use ExUnit.Case, async: true
  3 | 
  4 |   @valid_video %{
  5 |     "id"=> 1,
  6 |     "uuid"=> "00000000-0000-0000-0000-000000000000",
  7 |     "name"=> "This is the video name",
  8 |     "category"=> %{
  9 |       "id"=> 15,
 10 |       "label"=> "Science & Technology"
 11 |     },
 12 |     "licence"=> %{
 13 |       "id"=> 4,
 14 |       "label"=> "Attribution - Non Commercial"
 15 |     },
 16 |     "language"=> %{
 17 |       "id"=> nil,
 18 |       "label"=> "Unknown"
 19 |     },
 20 |     "privacy"=> %{
 21 |       "id"=> 1,
 22 |       "label"=> "Public"
 23 |     },
 24 |     "nsfw"=> false,
 25 |     "description"=> "This is the video description",
 26 |     "isLocal"=> true,
 27 |     "duration"=> 274,
 28 |     "views"=> 1696,
 29 |     "likes"=> 29,
 30 |     "dislikes"=> 0,
 31 |     "thumbnailPath"=> "/static/thumbnails/00000000-0000-0000-0000-000000000000.jpg",
 32 |     "previewPath"=> "/static/previews/00000000-0000-0000-0000-000000000000.jpg",
 33 |     "embedPath"=> "/videos/embed/00000000-0000-0000-0000-000000000000",
 34 |     "createdAt"=> "2018-08-02T13:47:17.515Z",
 35 |     "updatedAt"=> "2019-02-12T05:01:00.587Z",
 36 |     "publishedAt"=> "2018-08-02T13:55:13.338Z",
 37 |     "account"=> %{
 38 |       "id"=> 501,
 39 |       "uuid"=> "00000000-0000-0000-0000-000000000000",
 40 |       "name"=> "user",
 41 |       "displayName"=> "user",
 42 |       "url"=> "https://peertube.example.com/accounts/user",
 43 |       "host"=> "peertube.example.com",
 44 |       "avatar"=> %{
 45 |         "path"=> "/static/avatars/00000000-0000-0000-0000-000000000000.jpg",
 46 |         "createdAt"=> "2018-08-02T10:56:25.627Z",
 47 |         "updatedAt"=> "2018-08-02T10:56:25.627Z"
 48 |       }
 49 |     },
 50 |     "channel"=> %{
 51 |       "id"=> 23,
 52 |       "uuid"=> "00000000-0000-0000-0000-000000000000",
 53 |       "name"=> "00000000-0000-0000-0000-000000000000",
 54 |       "displayName"=> "Default user channel",
 55 |       "url"=> "https://peertube.example.com/video-channels/00000000-0000-0000-0000-000000000000",
 56 |       "host"=> "peertube.example.com",
 57 |       "avatar"=> %{
 58 |         "path"=> "/static/avatars/00000000-0000-0000-0000-000000000000.jpg",
 59 |         "createdAt"=> "2018-08-02T10:56:25.627Z",
 60 |         "updatedAt"=> "2018-08-02T10:56:25.627Z"
 61 |       }
 62 |     }
 63 |   }
 64 | 
 65 |   setup do
 66 |     bypass = Bypass.open
 67 |     {:ok, bypass: bypass}
 68 |   end
 69 | 
 70 |   test "unable to connect", %{bypass: bypass} do
 71 |     Bypass.down(bypass)
 72 |     result = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}")
 73 |     assert result == {:error, {
 74 |       :failed_connect, [
 75 |         {:to_address, {'localhost', bypass.port}},
 76 |         {:inet, [:inet], :econnrefused}
 77 |       ]
 78 |     }}
 79 |   end
 80 | 
 81 |   defp empty_instance do
 82 |     %{
 83 |       {"GET", "/robots.txt"} => {
 84 |         :stub, fn conn ->
 85 |           Plug.Conn.resp(conn, 200, "")
 86 |         end},
 87 |       {"GET", "/api/v1/videos"} => {
 88 |         :stub, fn conn ->
 89 |           Plug.Conn.resp(conn, 200, ~s<{"total": 0, "data": []}>)
 90 |         end},
 91 |       {"GET", "/api/v1/server/followers"} => {
 92 |           :stub, fn conn ->
 93 |             Plug.Conn.resp(conn, 200, ~s<{"total": 0, "data": []}>)
 94 |           end},
 95 |       {"GET", "/api/v1/server/following"} => {
 96 |           :stub, fn conn ->
 97 |             Plug.Conn.resp(conn, 200, ~s<{"total": 0, "data": []}>)
 98 |           end}
 99 |     }
100 |   end
101 | 
102 |   defp overwrite_expectation(routes, type, method, path, function) do
103 |     Map.put(routes, {method, path}, {type, function})
104 |   end
105 | 
106 |   defp apply_bypass(routes, bypass) do
107 |     for {{method, path}, {expect_function, function}} <- routes do
108 |       case expect_function do
109 |         :stub ->
110 |           Bypass.stub(bypass, method, path, function)
111 |         :expect ->
112 |           Bypass.expect(bypass, method, path, function)
113 |         :expect_once ->
114 |           Bypass.expect_once(bypass, method, path, function)
115 |         invalid ->
116 |           raise "Invalid expect function #{invalid}"
117 |       end
118 |     end
119 |   end
120 | 
121 |   # Create bypass responses for an empty instance and overwrite one route with the given arguments
122 |   defp empty_instance_but(bypass, expect_function, method, path, function) do
123 |     empty_instance()
124 |     |> overwrite_expectation(expect_function, method, path, function)
125 |     |> apply_bypass(bypass)
126 |   end
127 | 
128 |   test "bad HTTP status", %{bypass: bypass} do
129 |     empty_instance_but(bypass, :expect_once, "GET", "/api/v1/videos", &Plug.Conn.resp(&1, 400, "{}"))
130 |     result = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 100, false)
131 |     assert result == {:error, :http_error}
132 |   end
133 | 
134 |   test "JSON parse error", %{bypass: bypass} do
135 |     empty_instance_but(bypass, :expect_once, "GET", "/api/v1/videos", &Plug.Conn.resp(&1, 200, "invalid JSON document"))
136 |     result = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 100, false)
137 |     assert result == {:error, %Poison.ParseError{pos: 0, rest: nil, value: "i"}}
138 |   end
139 | 
140 |   test "error after fist page", %{bypass: bypass} do
141 |     empty_instance_but(bypass, :expect, "GET", "/api/v1/videos", fn conn ->
142 |       conn = Plug.Conn.fetch_query_params(conn)
143 |       response = if Map.has_key?(conn.query_params, "start") do
144 |         ~s<bad json>
145 |       else
146 |         ~s<{"total": 10, "data": []}>
147 |       end
148 |       Plug.Conn.resp(conn, 200, response)
149 |     end)
150 | 
151 |     {status, _} = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 5, false)
152 |     assert status == :error
153 |   end
154 | 
155 |   test "error on followers", %{bypass: bypass} do
156 |     empty_instance_but(bypass, :expect, "GET", "/api/v1/server/followers", &Plug.Conn.resp(&1, 500, "Error"))
157 |     result = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 100, false)
158 |     assert result == {:error, :http_error}
159 |   end
160 | 
161 |   test "error on following", %{bypass: bypass} do
162 |     empty_instance_but(bypass, :expect, "GET", "/api/v1/server/following", &Plug.Conn.resp(&1, 500, "Error"))
163 |     result = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 100, false)
164 |     assert result == {:error, :http_error}
165 |   end
166 | 
167 |   test "gets all videos correctly with a single page", %{bypass: bypass} do
168 |     a_video = @valid_video |> Map.put("id", 0)
169 |     another_video = @valid_video |> Map.put("id", 1)
170 | 
171 |     empty_instance_but(bypass, :expect, "GET", "/api/v1/videos", fn conn ->
172 |       conn = Plug.Conn.fetch_query_params(conn)
173 |       Plug.Conn.resp(conn, 200, ~s<{
174 |         "total": 2,
175 |         "data": [
176 |           #{a_video |> Poison.encode!()},
177 |           #{another_video |> Poison.encode!()}
178 |         ]
179 |       }>)
180 |     end)
181 | 
182 |     {:ok, {videos, _instances}} = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 10, false)
183 |     assert Enum.to_list(videos) == [a_video, another_video]
184 |   end
185 | 
186 |   test "gets all videos correctly with pagination", %{bypass: bypass} do
187 |     empty_instance_but(bypass, :expect, "GET", "/api/v1/videos", fn conn ->
188 |       conn = Plug.Conn.fetch_query_params(conn)
189 |       {start, ""} = conn.query_params |> Map.get("start", "0") |> Integer.parse()
190 |       Plug.Conn.resp(conn, 200, ~s<{
191 |         "total": 3,
192 |         "data": [
193 |           #{@valid_video |> Map.put("id", start) |> Poison.encode!()}
194 |         ]
195 |       }>)
196 |     end)
197 | 
198 |     {:ok, {videos, _instances}} = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 1, false)
199 |     assert Enum.to_list(videos) == [
200 |       @valid_video |> Map.put("id", 0),
201 |       @valid_video |> Map.put("id", 1),
202 |       @valid_video |> Map.put("id", 2)
203 |     ]
204 |   end
205 | 
206 |   test "wrong page format", %{bypass: bypass} do
207 |     empty_instance_but(bypass, :expect, "GET", "/api/v1/videos", &Plug.Conn.resp(&1, 200, "{\"not the correct format\": \"some value\"}"))
208 |     result = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 100, false)
209 |     assert result == {:error, :page_invalid}
210 |   end
211 | 
212 |   test "validates incoming video documents and returns validation errors with server version", %{bypass: bypass} do
213 |     valid_video = @valid_video |> Map.put("isLocal", true)
214 |     invalid_video = @valid_video |> Map.delete("account")
215 |     empty_instance_but(bypass, :expect, "GET", "/api/v1/videos", fn conn ->
216 |       Plug.Conn.resp(
217 |         conn,
218 |         200,
219 |         ~s<
220 |         {
221 |           "total": 2,
222 |           "data": [
223 |             #{Poison.encode!(valid_video)},
224 |             #{Poison.encode!(invalid_video)}
225 |           ]
226 |         }
227 |         >
228 |       )
229 |     end)
230 |     Bypass.expect_once(bypass, "GET", "/api/v1/config", fn conn ->
231 |       Plug.Conn.resp(conn, 200, ~s<{"serverVersion": "1.4.0"}>)
232 |     end)
233 |     result = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 100, false)
234 |     assert result == {
235 |              :error, {
236 |                :invalid_video_document,
237 |                %{version: "1.4.0"},
238 |                %{account: [{"can't be blank", [validation: :required]}]}
239 |              }
240 |            }
241 |   end
242 | 
243 |   test "does not fail if unable to fetch server version after a video document validation error", %{bypass: bypass} do
244 |     invalid_video = Map.delete(@valid_video, "uuid")
245 |     empty_instance_but(bypass, :expect, "GET", "/api/v1/videos", fn conn ->
246 |       Plug.Conn.resp(
247 |         conn,
248 |         200,
249 |         ~s<
250 |         {
251 |           "total": 2,
252 |           "data": [
253 |             #{Poison.encode!(@valid_video)},
254 |             #{Poison.encode!(invalid_video)}
255 |           ]
256 |         }
257 |         >
258 |       )
259 |     end)
260 |     Bypass.expect_once(bypass, "GET", "/api/v1/config", fn conn ->
261 |       Plug.Conn.resp(conn, 500, "error page")
262 |     end)
263 |     result = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 100, false)
264 |     assert result == {
265 |              :error, {
266 |                :invalid_video_document,
267 |                %{version: nil},
268 |                %{uuid: [{"can't be blank", [validation: :required]}]}
269 |              }
270 |            }
271 |   end
272 | 
273 |   test "allows validation errors on non local videos and discard these videos", %{bypass: bypass} do
274 |     invalid_video = @valid_video |> Map.delete("name") |> Map.put("isLocal", false) |> put_in(["account", "host"], "foreign-peertube.example.com")
275 |     empty_instance_but(bypass, :expect, "GET", "/api/v1/videos", fn conn ->
276 |       Plug.Conn.resp(
277 |         conn,
278 |         200,
279 |         ~s<
280 |         {
281 |           "total": 2,
282 |           "data": [
283 |             #{Poison.encode!(@valid_video)},
284 |             #{Poison.encode!(invalid_video)}
285 |           ]
286 |         }
287 |         >
288 |       )
289 |     end)
290 | 
291 |     {:ok, {videos, instances}} = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 10, false)
292 |     assert Enum.to_list(videos) == [@valid_video]
293 |     assert instances == MapSet.new([get_in(@valid_video, ["account", "host"])])
294 |   end
295 | 
296 |   # TODO
297 | #  test "video document without isLocal field" do
298 | #
299 | #  end
300 | 
301 |   test "can timeout on requests", %{bypass: bypass} do
302 |     reponse_delay = 600
303 |     empty_instance_but(bypass, :expect, "GET", "/api/v1/videos", fn conn ->
304 |       Process.sleep(reponse_delay)
305 |       Plug.Conn.resp(conn, 200, ~s<{"total": 0, "data": []}>)
306 |     end)
307 | 
308 |     result = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 100, false, reponse_delay - 100)
309 |     assert result == {:error, :timeout}
310 |   end
311 | 
312 |   test "discovers new instances from videos", %{bypass: bypass} do
313 |     hostname = "localhost:#{bypass.port}"
314 |     a_video =
315 |     @valid_video
316 |     |>Map.put("id", 0)
317 |     |> put_in(["account", "host"], hostname)
318 |     |> put_in(["channel", "host"], hostname)
319 | 
320 |     another_video =
321 |     @valid_video
322 |     |> Map.put("id", 1)
323 |     |> put_in(["account", "host"], "new-instance.example.com")
324 |     |> put_in(["channel", "host"], "new-instance.example.com")
325 | 
326 |     empty_instance_but(bypass, :expect, "GET", "/api/v1/videos", fn conn ->
327 |       conn = Plug.Conn.fetch_query_params(conn)
328 |       Plug.Conn.resp(conn, 200, ~s<{
329 |         "total": 2,
330 |         "data": [
331 |           #{a_video |> Poison.encode!()},
332 |           #{another_video |> Poison.encode!()}
333 |         ]
334 |       }>)
335 |     end)
336 | 
337 |     {:ok, {_videos, instances}} = PeertubeIndex.InstanceScanner.Http.scan(hostname, 10, false)
338 |     # The returned instances does not contain the instance being scanned
339 |     assert instances == MapSet.new(["new-instance.example.com"])
340 |   end
341 | 
342 |   test "discovers new instances from following", %{bypass: bypass} do
343 |     empty_instance_but(bypass, :expect, "GET", "/api/v1/server/following", fn conn ->
344 |       conn = Plug.Conn.fetch_query_params(conn)
345 |       Plug.Conn.resp(conn, 200, ~s<{
346 |         "total": 1,
347 |         "data": [
348 |           {"following": {"host": "new-instance.example.com"}},
349 |           {"following": {"host": "another-new-instance.example.com"}}
350 |         ]
351 |       }>)
352 |     end)
353 | 
354 |     {:ok, {_videos, instances}} = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 10, false)
355 |     assert instances == MapSet.new(["new-instance.example.com", "another-new-instance.example.com"])
356 |   end
357 | 
358 |   test "discovers new instances from followers", %{bypass: bypass} do
359 |     empty_instance_but(bypass, :expect, "GET", "/api/v1/server/followers", fn conn ->
360 |       conn = Plug.Conn.fetch_query_params(conn)
361 |       Plug.Conn.resp(conn, 200, ~s<{
362 |         "total": 1,
363 |         "data": [
364 |           {"follower": {"host": "new-instance.example.com"}},
365 |           {"follower": {"host": "another-new-instance.example.com"}}
366 |         ]
367 |       }>)
368 |     end)
369 | 
370 |     {:ok, {_videos, instances}} = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 10, false)
371 |     assert instances == MapSet.new(["new-instance.example.com", "another-new-instance.example.com"])
372 |   end
373 | 
374 |   test "excludes non local videos", %{bypass: bypass} do
375 |     local_video = @valid_video |> Map.put("id", 0)
376 |     non_local_video =
377 |     @valid_video
378 |     |> Map.put("id", 1)
379 |     |> Map.put("isLocal", false)
380 |     |> put_in(["channel", "host"], "other-instance.example.com")
381 |     |> put_in(["account", "host"], "other-instance.example.com")
382 | 
383 |     empty_instance_but(bypass, :expect, "GET", "/api/v1/videos", fn conn ->
384 |       conn = Plug.Conn.fetch_query_params(conn)
385 |       Plug.Conn.resp(conn, 200, ~s<{
386 |         "total": 2,
387 |         "data": [
388 |           #{local_video |> Poison.encode!()},
389 |           #{non_local_video |> Poison.encode!()}
390 |         ]
391 |       }>)
392 |     end)
393 | 
394 |     {:ok, {videos, _instances}} = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 10, false)
395 |     assert Enum.to_list(videos) == [local_video]
396 |   end
397 | 
398 |   describe "respects robots.txt file," do
399 |     # We only check simple cases to make sure we use the robots.txt parsing library correctly
400 |     test "checks videos endpoint", %{bypass: bypass} do
401 |       empty_instance_but(bypass, :expect, "GET", "/robots.txt", fn conn ->
402 |         Plug.Conn.resp(conn, 200, "User-agent: PeertubeIndex\nDisallow: /api/v1/videos\n")
403 |       end)
404 | 
405 |       result = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 100, false)
406 |       assert result == {:error, :robots_txt_disallowed}
407 |     end
408 | 
409 |     test "checks followers endpoint", %{bypass: bypass} do
410 |       empty_instance_but(bypass, :expect, "GET", "/robots.txt", fn conn ->
411 |         Plug.Conn.resp(conn, 200, "User-agent: PeertubeIndex\nDisallow: /api/v1/server/followers\n")
412 |       end)
413 | 
414 |       result = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 100, false)
415 |       assert result == {:error, :robots_txt_disallowed}
416 |     end
417 | 
418 |     test "checks following endpoint", %{bypass: bypass} do
419 |       empty_instance_but(bypass, :expect, "GET", "/robots.txt", fn conn ->
420 |         Plug.Conn.resp(conn, 200, "User-agent: PeertubeIndex\nDisallow: /api/v1/server/following\n")
421 |       end)
422 | 
423 |       result = PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 100, false)
424 |       assert result == {:error, :robots_txt_disallowed}
425 |     end
426 |   end
427 | 
428 |   test "presents the correct user agent", %{bypass: bypass} do
429 |     test_pid = self()
430 |     empty_instance_but(bypass, :stub, "GET", "/api/v1/videos", fn conn ->
431 |       user_agent = conn.req_headers |> List.keyfind("user-agent", 0) |> elem(1)
432 |       send test_pid, {:request_user_agent, user_agent}
433 | 
434 |       Plug.Conn.resp(conn, 200, ~s<{"total": 0, "data": []}>)
435 |     end)
436 | 
437 |     PeertubeIndex.InstanceScanner.Http.scan("localhost:#{bypass.port}", 100, false)
438 | 
439 |     assert_received {:request_user_agent, "PeertubeIndex"}
440 |   end
441 | end
442 | 


--------------------------------------------------------------------------------
/test/peertube_index/status_storage_test.exs:
--------------------------------------------------------------------------------
  1 | defmodule PeertubeIndex.StatusStorageTest do
  2 |   use ExUnit.Case, async: true
  3 | 
  4 |   @moduletag :integration
  5 |   @status_storage PeertubeIndex.StatusStorage.Postgresql
  6 | 
  7 |   setup do
  8 |     @status_storage.empty()
  9 |     PeertubeIndex.StatusStorage.Repo.start_link()
 10 |     :ok
 11 |   end
 12 | 
 13 |   test "status reporting functions create entries when instances have no status yet" do
 14 |     @status_storage.ok_instance("example.com", ~N[2018-01-02 15:50:00])
 15 |     @status_storage.failed_instance("other.example.com", {:some_error_reason, "arbitrary error data"}, ~N[2018-01-03 16:20:00])
 16 |     @status_storage.discovered_instance("newly-discovered.example.com", ~N[2018-02-05 10:00:00])
 17 |     @status_storage.banned_instance("banned-instance.example.com", "Reason for the ban", ~N[2019-03-03 19:04:00])
 18 | 
 19 |     assert MapSet.new(@status_storage.all()) == MapSet.new([
 20 |              {"example.com", :ok, ~N[2018-01-02 15:50:00]},
 21 |              {"other.example.com", {:error, inspect({:some_error_reason, "arbitrary error data"})}, ~N[2018-01-03 16:20:00]},
 22 |              {"newly-discovered.example.com", :discovered, ~N[2018-02-05 10:00:00]},
 23 |              {"banned-instance.example.com", {:banned, "Reason for the ban"}, ~N[2019-03-03 19:04:00]}
 24 |            ])
 25 |   end
 26 | 
 27 |   test "find_instances with status" do
 28 |     :ok = @status_storage.with_statuses([
 29 |       {"discovered1.example.com", :discovered, ~N[2018-03-11 20:10:31]},
 30 |       {"discovered2.example.com", :discovered, ~N[2018-03-12 21:10:31]},
 31 |       {"ok1.example.com", :ok, ~N[2018-03-11 20:10:31]},
 32 |       {"ok2.example.com", :ok, ~N[2018-03-12 21:10:31]},
 33 |       {"failed1.example.com", {:error, :some_reason}, ~N[2018-03-11 20:10:31]},
 34 |       {"failed2.example.com", {:error, :some_reason}, ~N[2018-03-12 21:10:31]},
 35 |       {"banned1.example.com", {:banned, "Some reason for a ban"}, ~N[2018-03-13 20:10:31]},
 36 |       {"banned2.example.com", {:banned, "Some reason for a ban"}, ~N[2018-03-14 21:10:31]}
 37 |     ])
 38 | 
 39 |     instances = @status_storage.find_instances(:ok)
 40 |     assert MapSet.new(instances) == MapSet.new(["ok1.example.com", "ok2.example.com"])
 41 | 
 42 |     instances = @status_storage.find_instances(:discovered)
 43 |     assert MapSet.new(instances) == MapSet.new(["discovered1.example.com", "discovered2.example.com"])
 44 | 
 45 |     instances = @status_storage.find_instances(:error)
 46 |     assert MapSet.new(instances) == MapSet.new(["failed1.example.com", "failed2.example.com"])
 47 | 
 48 |     instances = @status_storage.find_instances(:banned)
 49 |     assert MapSet.new(instances) == MapSet.new(["banned1.example.com", "banned2.example.com"])
 50 |   end
 51 | 
 52 |   test "find_instances with status and maximum date" do
 53 |     year = 2018
 54 |     month = 3
 55 |     day = 10
 56 |     hour = 12
 57 |     minute = 30
 58 |     second = 30
 59 |     {:ok, maximum_date} = NaiveDateTime.new(year, month, day, hour, minute, second)
 60 |     {:ok, old_enough} = NaiveDateTime.new(year, month, day , hour, minute, second - 1)
 61 |     @status_storage.with_statuses([
 62 |       {"ok-too-recent.example.com", :ok, maximum_date},
 63 |       {"ok-old-enough.example.com", :ok, old_enough},
 64 |       {"failed-too-recent.example.com", {:error, :some_reason}, maximum_date},
 65 |       {"failed-old-enough.example.com", {:error, :some_reason}, old_enough},
 66 |       {"discovered-too-recent.example.com", :discovered, maximum_date},
 67 |       {"discovered-old-enough.example.com", :discovered, old_enough},
 68 |     ])
 69 | 
 70 |     instances = @status_storage.find_instances(:discovered, maximum_date)
 71 |     assert MapSet.new(instances) == MapSet.new(["discovered-old-enough.example.com"])
 72 | 
 73 |     instances = @status_storage.find_instances(:ok, maximum_date)
 74 |     assert MapSet.new(instances) == MapSet.new(["ok-old-enough.example.com"])
 75 | 
 76 |     instances = @status_storage.find_instances(:error, maximum_date)
 77 |     assert MapSet.new(instances) == MapSet.new(["failed-old-enough.example.com"])
 78 |   end
 79 | 
 80 |   test "failed_instance overrides existing status" do
 81 |     :ok = @status_storage.with_statuses([{"example.com", :ok, ~N[2018-03-11 20:10:31]}])
 82 |     assert @status_storage.failed_instance("example.com", {:some_error_reason, "arbitrary error data"}, ~N[2018-04-01 12:40:00]) == :ok
 83 |     assert @status_storage.all() == [{"example.com", {:error, inspect({:some_error_reason, "arbitrary error data"})}, ~N[2018-04-01 12:40:00]}]
 84 |   end
 85 | 
 86 |   test "ok_instance overrides existing status" do
 87 |     :ok = @status_storage.with_statuses([{"example.com", :ok, ~N[2018-03-11 20:10:31]}])
 88 |     assert @status_storage.ok_instance("example.com", ~N[2018-04-01 12:40:00]) == :ok
 89 |     assert @status_storage.all() == [{"example.com", :ok, ~N[2018-04-01 12:40:00]}]
 90 |   end
 91 | 
 92 |   test "discovered_instance overrides existing status" do
 93 |     :ok = @status_storage.with_statuses([{"example.com", :ok, ~N[2019-03-03 21:23:44]}])
 94 |     assert @status_storage.discovered_instance("example.com", ~N[2019-03-04 04:05:29]) == :ok
 95 |     assert @status_storage.all() == [{"example.com", :discovered, ~N[2019-03-04 04:05:29]}]
 96 |   end
 97 | 
 98 |   test "banned_instance overrides existing status" do
 99 |     :ok = @status_storage.with_statuses([{"example.com", :ok, ~N[2019-03-03 21:23:44]}])
100 |     assert @status_storage.banned_instance("example.com", "Reason for the ban", ~N[2019-03-04 04:05:29]) == :ok
101 |     assert @status_storage.all() == [{"example.com", {:banned, "Reason for the ban"}, ~N[2019-03-04 04:05:29]}]
102 |   end
103 | 
104 |   test "can insert arbitrary strings" do
105 |     @status_storage.failed_instance("example.com", "A string with 'single quotes'", ~N[2019-08-23 17:16:33])
106 |     @status_storage.banned_instance("2.example.com", "A string with 'single quotes'", ~N[2019-08-23 17:16:33])
107 |     assert MapSet.new(@status_storage.all()) == MapSet.new([
108 |       {"example.com", {:error, inspect("A string with 'single quotes'")}, ~N[2019-08-23 17:16:33]},
109 |       {"2.example.com", {:banned, "A string with 'single quotes'"}, ~N[2019-08-23 17:16:33]},
110 |     ])
111 |   end
112 | 
113 |   test "has_a_status" do
114 |     known_instance = "known-instance.example.com"
115 |     :ok = @status_storage.with_statuses([{known_instance, :ok, ~N[2018-03-11 20:10:31]}])
116 |     assert @status_storage.has_a_status(known_instance) == true
117 |     assert @status_storage.has_a_status("unknown-instance.example.com") == false
118 |   end
119 | end
120 | 


--------------------------------------------------------------------------------
/test/peertube_index/video_storage_test.exs:
--------------------------------------------------------------------------------
 1 | defmodule PeertubeIndex.VideoStorageTest do
 2 |   use ExUnit.Case, async: true
 3 | 
 4 |   @moduletag :integration
 5 | 
 6 |   test "we can search videos" do
 7 |     videos = [
 8 |       %{"name" => "A cat video"},
 9 |       %{"name" => "Another video about a cat"}
10 |     ]
11 |     PeertubeIndex.VideoStorage.Elasticsearch.with_videos!(videos)
12 |     Process.sleep 1_000
13 |     assert MapSet.new(PeertubeIndex.VideoStorage.Elasticsearch.search("cat")) == MapSet.new(videos)
14 |   end
15 | 
16 |   test "search is fuzzy" do
17 |     videos = [%{"name" => "Cats best moments"}]
18 |     PeertubeIndex.VideoStorage.Elasticsearch.with_videos!(videos)
19 |     Process.sleep 1_000
20 |     assert MapSet.new(PeertubeIndex.VideoStorage.Elasticsearch.search("cat best momt")) == MapSet.new(videos)
21 |   end
22 | 
23 |   test "search can filter on NSFW" do
24 |     safe_video = %{"name" => "A safe video", "nsfw" => false}
25 |     unsafe_video = %{"name" => "An unsafe video", "nsfw" => true}
26 |     videos = [safe_video, unsafe_video]
27 |     PeertubeIndex.VideoStorage.Elasticsearch.with_videos!(videos)
28 |     Process.sleep 1_000
29 |     assert MapSet.new(PeertubeIndex.VideoStorage.Elasticsearch.search("video")) == MapSet.new([safe_video, unsafe_video])
30 |     assert PeertubeIndex.VideoStorage.Elasticsearch.search("video", nsfw: false) == [safe_video]
31 |     assert PeertubeIndex.VideoStorage.Elasticsearch.search("video", nsfw: true) == [unsafe_video]
32 |   end
33 | 
34 |   test "search gives the first 100 results" do
35 |     videos =
36 |     for index <- 1..110 do
37 |       %{"name" => "A cat video", "uuid" => "#{index}"}
38 |     end
39 |     PeertubeIndex.VideoStorage.Elasticsearch.with_videos!(videos)
40 |     Process.sleep 1_000
41 |     assert length(PeertubeIndex.VideoStorage.Elasticsearch.search("video")) == 100
42 |   end
43 | 
44 |   test "insert_videos! adds videos and we can search them" do
45 |     # Given I have an empty index
46 |     PeertubeIndex.VideoStorage.Elasticsearch.empty!()
47 |     # When I update an instance with some videos
48 |     a_video = %{"name" => "A dummy video"}
49 |     another_video = %{"name" => "An interesting video"}
50 |     PeertubeIndex.VideoStorage.Elasticsearch.insert_videos!([a_video, another_video])
51 |     Process.sleep 1_000
52 |     # Then
53 |     assert PeertubeIndex.VideoStorage.Elasticsearch.search("dummy") == [a_video]
54 |     assert PeertubeIndex.VideoStorage.Elasticsearch.search("interesting") == [another_video]
55 |   end
56 | 
57 |   test "delete_instance_videos! removes videos of the given instance" do
58 |     # Given We have videos
59 |     videos = [
60 |       %{"name" => "A dummy video", "account" => %{"host" => "example.com"}},
61 |       %{"name" => "Another dummy video", "account" => %{"host" => "example.com"}}
62 |     ]
63 |     PeertubeIndex.VideoStorage.Elasticsearch.with_videos!(videos)
64 |     Process.sleep 1_000
65 |     # When I delete videos of the instance
66 |     PeertubeIndex.VideoStorage.Elasticsearch.delete_instance_videos!("example.com")
67 |     Process.sleep 1_000
68 |     # Then
69 |     assert PeertubeIndex.VideoStorage.Elasticsearch.search("dummy") == []
70 |   end
71 | end
72 | 


--------------------------------------------------------------------------------
/test/peertube_index/web_server_test.exs:
--------------------------------------------------------------------------------
  1 | defmodule PeertubeIndex.WebServerTest do
  2 |   use ExUnit.Case, async: true
  3 |   use Plug.Test
  4 | 
  5 |   @moduletag :integration
  6 | 
  7 |   @opts PeertubeIndex.WebServer.init([])
  8 | 
  9 |   test "ping works" do
 10 |     # Create a test connection
 11 |     conn = conn(:get, "/ping")
 12 | 
 13 |     # Invoke the plug
 14 |     conn = PeertubeIndex.WebServer.call(conn, @opts)
 15 | 
 16 |     # Assert the response and status
 17 |     assert conn.state == :sent
 18 |     assert conn.status == 200
 19 |     assert conn.resp_body == "pong"
 20 |   end
 21 | 
 22 |   test "unknown URL returns not found page" do
 23 |     conn = conn(:get, "/not_existing")
 24 | 
 25 |     conn = PeertubeIndex.WebServer.call(conn, @opts)
 26 | 
 27 |     assert conn.state == :sent
 28 |     assert conn.status == 404
 29 |     assert conn.resp_body == "Not found"
 30 |   end
 31 | 
 32 |   test "user can see home page" do
 33 |     conn = conn(:get, "/")
 34 | 
 35 |     conn = PeertubeIndex.WebServer.call(conn, @opts)
 36 | 
 37 |     assert conn.state == :sent
 38 |     assert conn.status == 200
 39 |     assert List.keyfind(conn.resp_headers, "content-type", 0) == {"content-type", "text/html; charset=utf-8"}
 40 |   end
 41 | 
 42 |   test "user can search videos" do
 43 |     query = "the_user_search_text2"
 44 |     conn = conn(:get, "/search?text=#{query}")
 45 |     videos = []
 46 |     conn = assign(conn, :search_usecase_function, fn text ->
 47 |       send self(), {:search_function_called, text}
 48 |       videos
 49 |     end)
 50 |     conn = assign(conn, :render_page_function, fn videos, query ->
 51 |       send self(), {:render_function_called, videos, query}
 52 |       "Fake search result"
 53 |     end)
 54 | 
 55 |     # When a user does a search
 56 |     conn = PeertubeIndex.WebServer.call(conn, @opts)
 57 | 
 58 |     # Then the search use case is called with the user search text
 59 |     assert_received {:search_function_called, ^query}
 60 |     # And the search result is given to the page rendering
 61 |     assert_received {:render_function_called, ^videos, ^query}
 62 | 
 63 |     # Then he sees a response
 64 |     assert conn.state == :sent
 65 |     assert conn.status == 200
 66 |     assert List.keyfind(conn.resp_headers, "content-type", 0) == {"content-type", "text/html; charset=utf-8"}
 67 |     assert conn.resp_body == "Fake search result"
 68 |   end
 69 | 
 70 |   test "an empty search shows a validation error" do
 71 |     # When a user does a search with an empty text
 72 |     conn = conn(:get, "/search?text=")
 73 |     conn = PeertubeIndex.WebServer.call(conn, @opts)
 74 | 
 75 |     # Then the user is redirected to home page
 76 |     assert conn.state == :sent
 77 |     assert conn.status == 302
 78 |     assert List.keyfind(conn.resp_headers, "content-type", 0) == {"content-type", "text/html; charset=utf-8"}
 79 |     assert List.keyfind(conn.resp_headers, "location", 0) == {"location", "/"}
 80 |   end
 81 | 
 82 |   test "a missing search text query param redirects to home page" do
 83 |     # When a user does a search without the query parameter
 84 |     conn = conn(:get, "/search")
 85 |     conn = PeertubeIndex.WebServer.call(conn, @opts)
 86 | 
 87 |     # Then the user is redirected to the home page
 88 |     assert conn.state == :sent
 89 |     assert conn.status == 302
 90 |     assert List.keyfind(conn.resp_headers, "content-type", 0) == {"content-type", "text/html; charset=utf-8"}
 91 |     assert List.keyfind(conn.resp_headers, "location", 0) == {"location", "/"}
 92 |   end
 93 | 
 94 |   test "user can see about page" do
 95 |     conn = conn(:get, "/about")
 96 | 
 97 |     conn = PeertubeIndex.WebServer.call(conn, @opts)
 98 | 
 99 |     assert conn.state == :sent
100 |     assert conn.status == 200
101 |     assert List.keyfind(conn.resp_headers, "content-type", 0) == {"content-type", "text/html; charset=utf-8"}
102 |   end
103 | 
104 |   test "user can search videos as JSON" do
105 |     query = "yet another user search text"
106 |     conn = conn(:get, "/api/search?text=#{query}")
107 |     videos = [
108 |       %{"name" => "Some video"},
109 |       %{"name" => "Some other video"}
110 |     ]
111 |     conn = assign(conn, :search_usecase_function, fn text ->
112 |       send self(), {:search_function_called, text}
113 |       videos
114 |     end)
115 | 
116 |     # When a user does a search
117 |     conn = PeertubeIndex.WebServer.call(conn, @opts)
118 | 
119 |     # Then the search use case is called with the user search text
120 |     assert_received {:search_function_called, ^query}
121 | 
122 |     # Then he gets an ok reponse
123 |     assert conn.state == :sent
124 |     assert conn.status == 200
125 |     assert List.keyfind(conn.resp_headers, "content-type", 0) == {"content-type", "application/json; charset=utf-8"}
126 |     # And the response contains the search result
127 |     assert Poison.decode!(conn.resp_body) == videos
128 |   end
129 | end
130 | 


--------------------------------------------------------------------------------
/test/peertube_index_test.exs:
--------------------------------------------------------------------------------
  1 | defmodule PeertubeIndexTest do
  2 |   use ExUnit.Case, async: true
  3 | 
  4 |   def scan_mocks do
  5 |     Mox.stub(PeertubeIndex.InstanceScanner.Mock, :scan, fn _hostname -> {:ok, {[], MapSet.new()}} end)
  6 | 
  7 |     Mox.stub(PeertubeIndex.VideoStorage.Mock, :delete_instance_videos!, fn _hostname -> :ok end)
  8 |     Mox.stub(PeertubeIndex.VideoStorage.Mock, :insert_videos!, fn _videos -> :ok end)
  9 | 
 10 |     Mox.stub(PeertubeIndex.StatusStorage.Mock, :find_instances, fn _status -> [] end)
 11 |     Mox.stub(PeertubeIndex.StatusStorage.Mock, :ok_instance, fn _hostname, _datetime -> :ok end)
 12 |     Mox.stub(PeertubeIndex.StatusStorage.Mock, :discovered_instance, fn _hostname, _datetime -> :ok end)
 13 |     Mox.stub(PeertubeIndex.StatusStorage.Mock, :failed_instance, fn _hostname, _reason, _datetime -> :ok end)
 14 |     Mox.stub(PeertubeIndex.StatusStorage.Mock, :has_a_status, fn _hostname -> false end)
 15 |   end
 16 | 
 17 |   test "we can search safe videos by their name using storage" do
 18 |     # Given there are videos
 19 |     a_video = %{"name" => "A video about a cat"}
 20 |     another_video = %{"name" => "A video about a cats and dogs"}
 21 |     Mox.expect(
 22 |       PeertubeIndex.VideoStorage.Mock, :search,
 23 |       # Then The storage is asked for the correct term and safety
 24 |       fn "cat", nsfw: false ->
 25 |         [a_video, another_video]
 26 |       end
 27 |     )
 28 | 
 29 |     # When the user searches for a video name
 30 |     videos = PeertubeIndex.search("cat")
 31 | 
 32 |     # Then the storage is asked for matching videos
 33 |     Mox.verify!()
 34 |     # And the matching videos are returned in the same order as returned by the storage
 35 |     assert videos == [a_video, another_video]
 36 |   end
 37 | 
 38 |   test "scan uses instance scanner, deletes existing instance videos and inserts new ones in video storage" do
 39 |     scan_mocks()
 40 |     videos = [%{"name" => "some video"}]
 41 |     Mox.expect(PeertubeIndex.InstanceScanner.Mock, :scan, fn "some-instance.example.com" -> {:ok, {videos, MapSet.new()}} end)
 42 |     Mox.expect(PeertubeIndex.VideoStorage.Mock, :delete_instance_videos!, fn "some-instance.example.com" -> :ok end)
 43 |     Mox.expect(PeertubeIndex.VideoStorage.Mock, :insert_videos!, fn ^videos -> :ok end)
 44 | 
 45 |     videos = [%{"name" => "some other video"}]
 46 |     Mox.expect(PeertubeIndex.InstanceScanner.Mock, :scan, fn "some-other-instance.example.com" -> {:ok, {videos, MapSet.new()}} end)
 47 |     Mox.expect(PeertubeIndex.VideoStorage.Mock, :delete_instance_videos!, fn "some-other-instance.example.com" -> :ok end)
 48 |     Mox.expect(PeertubeIndex.VideoStorage.Mock, :insert_videos!, fn ^videos -> :ok end)
 49 | 
 50 |     PeertubeIndex.scan(["some-instance.example.com", "some-other-instance.example.com"])
 51 | 
 52 |     Mox.verify!()
 53 |   end
 54 | 
 55 |   test "scan updates instance status" do
 56 |     scan_mocks()
 57 |     {:ok, finishes_at} = NaiveDateTime.new(2018, 1, 1, 14, 15, 16)
 58 |     Mox.expect(PeertubeIndex.StatusStorage.Mock, :ok_instance, fn "some-instance.example.com", ^finishes_at -> :ok end)
 59 | 
 60 |     PeertubeIndex.scan(["some-instance.example.com"], fn -> finishes_at end)
 61 | 
 62 |     Mox.verify!()
 63 |   end
 64 | 
 65 |   test "scan reports the appropriate status for discovered instances" do
 66 |     scan_mocks()
 67 |     Mox.stub(
 68 |       PeertubeIndex.InstanceScanner.Mock, :scan,
 69 |       fn "some-instance.example.com" ->
 70 |         {:ok, {[], MapSet.new(["found-instance.example.com", "another-found-instance.example.com"])}}
 71 |       end
 72 |     )
 73 |     {:ok, finishes_at} = NaiveDateTime.new(2018, 1, 1, 14, 15, 16)
 74 |     # Then we set the status for the discovered instances
 75 |     Mox.expect(PeertubeIndex.StatusStorage.Mock, :discovered_instance, fn "another-found-instance.example.com", ^finishes_at -> :ok end)
 76 |     Mox.expect(PeertubeIndex.StatusStorage.Mock, :discovered_instance, fn "found-instance.example.com", ^finishes_at -> :ok end)
 77 | 
 78 |     PeertubeIndex.scan(["some-instance.example.com"], fn -> finishes_at end)
 79 | 
 80 |     Mox.verify!()
 81 |   end
 82 | 
 83 |   test "scan does not override an existing status with the discovered status" do
 84 |     scan_mocks()
 85 |     # Given we have some instances with a status
 86 |     instances_with_a_status = ["known-instance-1.example.com",  "known-instance-2.example.com"]
 87 |     Mox.expect(PeertubeIndex.StatusStorage.Mock, :has_a_status, 2, fn hostname -> Enum.member?(instances_with_a_status, hostname) end)
 88 | 
 89 |     # When we discover those instances during a scan
 90 |     Mox.expect(
 91 |       PeertubeIndex.InstanceScanner.Mock, :scan,
 92 |       fn "some-instance.example.com" -> {:ok, {[], MapSet.new(instances_with_a_status)}} end
 93 |     )
 94 | 
 95 |     # Then We must not change the status of instances discovered instances
 96 |     Mox.expect(PeertubeIndex.StatusStorage.Mock, :discovered_instance, 0, fn _hostname, _date -> :ok end)
 97 | 
 98 |     PeertubeIndex.scan(["some-instance.example.com"])
 99 |     Mox.verify!()
100 |   end
101 | 
102 |   test "scan handles failures, reports the corresponding statuses and deletes existing videos for the failed instance" do
103 |     scan_mocks()
104 |     Mox.stub(PeertubeIndex.InstanceScanner.Mock, :scan, fn "some-instance.example.com" -> {:error, :some_reason} end)
105 |     Mox.expect(PeertubeIndex.VideoStorage.Mock, :delete_instance_videos!, fn "some-instance.example.com" -> :ok end)
106 |     {:ok, finishes_at} = NaiveDateTime.new(2018, 1, 1, 14, 15, 16)
107 |     Mox.expect(PeertubeIndex.StatusStorage.Mock, :failed_instance, fn "some-instance.example.com", :some_reason, ^finishes_at -> :ok end)
108 | 
109 |     PeertubeIndex.scan(["some-instance.example.com"], fn -> finishes_at end)
110 | 
111 |     Mox.verify!()
112 |   end
113 | 
114 |   test "scan skips banned instances" do
115 |     Mox.expect(PeertubeIndex.StatusStorage.Mock, :find_instances, fn :banned -> ["banned-instance.example.com"] end)
116 |     # We should not scan
117 |     Mox.expect(PeertubeIndex.InstanceScanner.Mock, :scan, 0, fn _instance -> nil end)
118 | 
119 |     PeertubeIndex.scan(["banned-instance.example.com"])
120 | 
121 |     Mox.verify!()
122 |   end
123 | 
124 |   test "rescan" do
125 |     {:ok, current_time} = NaiveDateTime.new(2018, 2, 2, 14, 15, 16)
126 |     {:ok, maximum_date} = NaiveDateTime.new(2018, 2, 1, 14, 15, 16)
127 | 
128 |     discovered_instances = ["discovered1.example.com", "discovered2.example.com"]
129 |     ok_and_old_enough_instances = ["ok1.example.com", "ok2.example.com"]
130 |     failed_and_old_enough_instances = ["failed1.example.com", "failed2.example.com"]
131 | 
132 |     Mox.expect(PeertubeIndex.StatusStorage.Mock, :find_instances, fn :discovered -> discovered_instances end)
133 |     Mox.expect(PeertubeIndex.StatusStorage.Mock, :find_instances, fn :ok, ^maximum_date -> ok_and_old_enough_instances end)
134 |     Mox.expect(PeertubeIndex.StatusStorage.Mock, :find_instances, fn :error, ^maximum_date -> failed_and_old_enough_instances end)
135 | 
136 |     PeertubeIndex.rescan(fn -> current_time end, fn instances -> send self(), {:scan_function_called, instances} end)
137 |     insances_to_rescan = discovered_instances ++ ok_and_old_enough_instances ++ failed_and_old_enough_instances
138 | 
139 |     Mox.verify!()
140 |     assert_received {:scan_function_called, ^insances_to_rescan}
141 |   end
142 | 
143 |   test "banning an instance removes all videos for this instance and saves its banned status" do
144 |     {:ok, current_time} = NaiveDateTime.new(2019, 3, 1, 13, 14, 15)
145 | 
146 |     Mox.expect(PeertubeIndex.VideoStorage.Mock, :delete_instance_videos!, fn "instance-to-ban.example.com" -> :ok end)
147 |     Mox.expect(PeertubeIndex.StatusStorage.Mock, :banned_instance, fn "instance-to-ban.example.com", "Provides illegal content", ^current_time -> :ok end)
148 | 
149 |     PeertubeIndex.ban_instance("instance-to-ban.example.com", "Provides illegal content", fn -> current_time end)
150 | 
151 |     Mox.verify!()
152 |   end
153 | 
154 |   test "removing a ban on an instance changes its status to discovered" do
155 |     {:ok, current_time} = NaiveDateTime.new(2019, 3, 3, 12, 13, 14)
156 |     Mox.expect(PeertubeIndex.StatusStorage.Mock, :discovered_instance, fn "unbanned-instance.example.com", ^current_time -> :ok end)
157 |     PeertubeIndex.remove_ban("unbanned-instance.example.com", fn -> current_time end)
158 |     Mox.verify!()
159 |   end
160 | 
161 |   test "add_instances adds not yet known instances with the discovered status" do
162 |     {:ok, current_time} = NaiveDateTime.new(2019, 5, 1, 17, 41, 55)
163 | 
164 |     # Given We do no know the instances
165 |     Mox.stub(PeertubeIndex.StatusStorage.Mock, :has_a_status, fn _hostname -> false end)
166 | 
167 |     # Then We set the status for the discovered instances
168 |     Mox.expect(PeertubeIndex.StatusStorage.Mock, :discovered_instance, fn "an-instance.example.com", ^current_time -> :ok end)
169 |     Mox.expect(PeertubeIndex.StatusStorage.Mock, :discovered_instance, fn "another-instance.example.com", ^current_time -> :ok end)
170 | 
171 |     # When We add the instances
172 |     PeertubeIndex.add_instances(["an-instance.example.com", "another-instance.example.com"], fn -> current_time end)
173 | 
174 |     Mox.verify!()
175 |   end
176 | 
177 |   test "add_instances does not override an existing status" do
178 |     {:ok, current_time} = NaiveDateTime.new(2019, 5, 1, 17, 48, 55)
179 | 
180 |     # Given We already have a status for an instance
181 |     Mox.stub(PeertubeIndex.StatusStorage.Mock, :has_a_status, fn "already-known-instance.example.com" -> true end)
182 | 
183 |     # Then We do not change the status of the existing instance
184 |     Mox.expect(PeertubeIndex.StatusStorage.Mock, :discovered_instance, 0, fn _hostname, _date -> :ok end)
185 | 
186 |     # When We try to add the instance
187 |     PeertubeIndex.add_instances(["already-known-instance.example.com"], fn -> current_time end)
188 | 
189 |     Mox.verify!()
190 |   end
191 | end
192 | 


--------------------------------------------------------------------------------
/test/support/mocks.ex:
--------------------------------------------------------------------------------
1 | Mox.defmock(PeertubeIndex.VideoStorage.Mock, for: PeertubeIndex.VideoStorage)
2 | Mox.defmock(PeertubeIndex.InstanceScanner.Mock, for: PeertubeIndex.InstanceScanner)
3 | Mox.defmock(PeertubeIndex.StatusStorage.Mock, for: PeertubeIndex.StatusStorage)
4 | 


--------------------------------------------------------------------------------
/test/test_helper.exs:
--------------------------------------------------------------------------------
1 | ExUnit.configure exclude: [:integration, :nonregression]
2 | ExUnit.start()
3 | Application.ensure_all_started(:bypass)
4 | 


--------------------------------------------------------------------------------