├── .env.example
├── .env.travis
├── .gitattributes
├── .gitignore
├── .travis.yml
├── LICENSE
├── README.md
├── app
├── Console
│ ├── Commands
│ │ ├── BuildCommand.php
│ │ └── CrawlCommand.php
│ └── Kernel.php
├── Email.php
├── Exceptions
│ └── Handler.php
├── Http
│ ├── Controllers
│ │ ├── Api
│ │ │ ├── ApiController.php
│ │ │ ├── SearchController.php
│ │ │ ├── SearchStatisticsController.php
│ │ │ ├── UserController.php
│ │ │ └── UserSearchController.php
│ │ ├── Auth
│ │ │ ├── ForgotPasswordController.php
│ │ │ ├── LoginController.php
│ │ │ ├── RegisterController.php
│ │ │ └── ResetPasswordController.php
│ │ ├── Controller.php
│ │ ├── DashboardController.php
│ │ ├── HomeController.php
│ │ ├── SearchEmailController.php
│ │ └── UserEmailController.php
│ ├── Kernel.php
│ ├── Middleware
│ │ ├── EncryptCookies.php
│ │ ├── RedirectIfAuthenticated.php
│ │ ├── TrimStrings.php
│ │ ├── TrustProxies.php
│ │ └── VerifyCsrfToken.php
│ ├── Requests
│ │ ├── SearchRequest.php
│ │ ├── UserRequest.php
│ │ └── UserSearchRequest.php
│ └── Resources
│ │ ├── SearchCollection.php
│ │ ├── SearchResource.php
│ │ └── UserResource.php
├── Jobs
│ └── CrawlJob.php
├── Policies
│ ├── SearchPolicy.php
│ └── UserPolicy.php
├── Providers
│ ├── AppServiceProvider.php
│ ├── AuthServiceProvider.php
│ ├── BroadcastServiceProvider.php
│ ├── EventServiceProvider.php
│ └── RouteServiceProvider.php
├── Search.php
├── Services
│ └── Crawler
│ │ └── Crawler.php
├── Url.php
├── User.php
└── helpers.php
├── artisan
├── bootstrap
├── app.php
└── cache
│ └── .gitignore
├── composer.json
├── composer.lock
├── config
├── app.php
├── auth.php
├── broadcasting.php
├── cache.php
├── database.php
├── filesystems.php
├── hashing.php
├── logging.php
├── mail.php
├── queue.php
├── services.php
├── session.php
└── view.php
├── database
├── .gitignore
├── factories
│ ├── EmailFactory.php
│ ├── SearchFactory.php
│ ├── UrlFactory.php
│ └── UserFactory.php
├── migrations
│ ├── 2014_10_12_000000_create_users_table.php
│ ├── 2014_10_12_100000_create_password_resets_table.php
│ ├── 2016_06_01_000001_create_oauth_auth_codes_table.php
│ ├── 2016_06_01_000002_create_oauth_access_tokens_table.php
│ ├── 2016_06_01_000003_create_oauth_refresh_tokens_table.php
│ ├── 2016_06_01_000004_create_oauth_clients_table.php
│ ├── 2016_06_01_000005_create_oauth_personal_access_clients_table.php
│ ├── 2018_01_19_203819_create_sessions_table.php
│ ├── 2018_01_19_205205_create_searches_table.php
│ ├── 2018_01_19_212151_create_urls_table.php
│ └── 2018_01_19_213937_create_emails_table.php
└── seeds
│ ├── DatabaseSeeder.php
│ └── UsersTableSeeder.php
├── package-lock.json
├── package.json
├── phpunit.xml
├── public
├── .htaccess
├── css
│ └── app.css
├── favicon.ico
├── index.php
├── js
│ └── app.js
├── mix-manifest.json
├── robots.txt
└── svg
│ ├── 403.svg
│ ├── 404.svg
│ ├── 500.svg
│ └── 503.svg
├── resources
├── js
│ ├── api
│ │ ├── client.js
│ │ ├── index.js
│ │ ├── search_statistics.js
│ │ ├── searches.js
│ │ ├── user_searches.js
│ │ └── users.js
│ ├── app.js
│ ├── components
│ │ ├── ConfirmDeleteModal.vue
│ │ ├── Dashboard.vue
│ │ ├── Pagination.vue
│ │ ├── ProgressBar.vue
│ │ ├── forms
│ │ │ ├── FormCheckbox.vue
│ │ │ ├── FormInput.vue
│ │ │ ├── SearchForm.vue
│ │ │ └── UserForm.vue
│ │ ├── search
│ │ │ ├── SearchCreate.vue
│ │ │ ├── SearchIndex.vue
│ │ │ └── SearchShow.vue
│ │ └── user
│ │ │ └── UserEdit.vue
│ ├── helpers
│ │ ├── route.js
│ │ └── vue-filters.js
│ └── router
│ │ ├── index.js
│ │ └── routes.js
├── lang
│ └── en
│ │ ├── auth.php
│ │ ├── pagination.php
│ │ ├── passwords.php
│ │ └── validation.php
├── sass
│ ├── _general.scss
│ ├── _transitions.scss
│ └── app.scss
└── views
│ ├── auth
│ ├── login.blade.php
│ ├── passwords
│ │ ├── email.blade.php
│ │ └── reset.blade.php
│ └── register.blade.php
│ ├── dashboard.blade.php
│ ├── home.blade.php
│ ├── layouts
│ └── app.blade.php
│ └── partials
│ ├── footer.blade.php
│ ├── header.blade.php
│ └── main.blade.php
├── routes
├── api.php
├── channels.php
├── console.php
└── web.php
├── server.php
├── storage
├── app
│ ├── .gitignore
│ └── public
│ │ └── .gitignore
├── framework
│ ├── .gitignore
│ ├── cache
│ │ ├── .gitignore
│ │ └── data
│ │ │ └── .gitignore
│ ├── sessions
│ │ └── .gitignore
│ ├── testing
│ │ └── .gitignore
│ └── views
│ │ └── .gitignore
└── logs
│ └── .gitignore
├── tests
├── CreatesApplication.php
├── Feature
│ ├── Api
│ │ ├── Search
│ │ │ ├── DestroySearchTest.php
│ │ │ ├── ShowSearchStatisticsTest.php
│ │ │ ├── ShowSearchTest.php
│ │ │ └── UpdateSearchTest.php
│ │ ├── User
│ │ │ ├── MeUserTest.php
│ │ │ └── UpdateUserTest.php
│ │ └── UserSearch
│ │ │ ├── IndexUserSearchTest.php
│ │ │ └── StoreUserSearchTest.php
│ ├── SearchEmailTest.php
│ └── UserEmailTest.php
├── TestCase.php
└── Unit
│ ├── EmailTest.php
│ ├── SearchTest.php
│ ├── UrlTest.php
│ └── UserTest.php
└── webpack.mix.js
/.env.example:
--------------------------------------------------------------------------------
1 | APP_NAME=Crawler
2 | APP_ENV=local
3 | APP_KEY=
4 | APP_DEBUG=true
5 | APP_URL=http://crawler.localhost
6 |
7 | LOG_CHANNEL=stack
8 |
9 | DB_CONNECTION=mysql
10 | DB_HOST=127.0.0.1
11 | DB_PORT=3306
12 | DB_DATABASE=crawler
13 | DB_USERNAME=root
14 | DB_PASSWORD=
15 |
16 | BROADCAST_DRIVER=log
17 | CACHE_DRIVER=file
18 | QUEUE_CONNECTION=sync
19 | SESSION_DRIVER=file
20 | SESSION_LIFETIME=120
21 |
22 | REDIS_HOST=127.0.0.1
23 | REDIS_PASSWORD=null
24 | REDIS_PORT=6379
25 |
26 | MAIL_DRIVER=smtp
27 | MAIL_HOST=smtp.mailtrap.io
28 | MAIL_PORT=2525
29 | MAIL_USERNAME=null
30 | MAIL_PASSWORD=null
31 | MAIL_ENCRYPTION=null
32 |
33 | PUSHER_APP_ID=
34 | PUSHER_APP_KEY=
35 | PUSHER_APP_SECRET=
36 | PUSHER_APP_CLUSTER=mt1
37 |
38 | MIX_PUSHER_APP_KEY="${PUSHER_APP_KEY}"
39 | MIX_PUSHER_APP_CLUSTER="${PUSHER_APP_CLUSTER}"
40 |
--------------------------------------------------------------------------------
/.env.travis:
--------------------------------------------------------------------------------
1 | APP_NAME=Crawler
2 | APP_ENV=testing
3 | APP_KEY=
4 |
5 | DB_CONNECTION=sqlite
6 | CACHE_DRIVER=array
7 | SESSION_DRIVER=array
8 | QUEUE_DRIVER=sync
9 | DB_CONNECTION=sqlite
10 | DB_DATABASE=":memory:"
11 |
--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | * text=auto
2 | *.css linguist-vendored
3 | *.scss linguist-vendored
4 | *.js linguist-vendored
5 | CHANGELOG.md export-ignore
6 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /node_modules
2 | /public/hot
3 | /public/storage
4 | /storage/*.key
5 | /vendor
6 | /.idea
7 | /.vagrant
8 | Homestead.json
9 | Homestead.yaml
10 | npm-debug.log
11 | yarn-error.log
12 | .env
13 | /.phpstorm.meta.php
14 | /_ide_helper.php
15 | /_ide_helper_models.php
16 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: php
2 | php:
3 | - 7.2
4 | before_script:
5 | - cp .env.travis .env
6 | - composer self-update
7 | - composer install --no-interaction
8 | - php artisan key:generate
9 | - php artisan crawler:build
10 | script:
11 | - ./vendor/bin/phpunit
12 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2018 hedii
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # php-crawler
2 |
3 | ***This repository has not been updated since a long time and I am not using it anymore. Please feel free to fork it and do whatever you want with it.***
4 |
5 | [](https://travis-ci.org/hedii/php-crawler)
6 |
7 | A crawler application with a php backend using Laravel, and a js frontend using vuejs, that finds email addresses on the internets.
8 |
9 | Given an entry point url, the crawler will search for emails in all the urls available from this entry point domain name.
10 | The emails are downloadable as a text file.
11 |
12 | Multiple users can start searching for emails without being able see the other users' searches (searches are related to a single user).
13 |
14 | ## Screenshots
15 |
16 | 
17 |
18 | 
19 |
20 | 
21 |
22 | 
23 |
24 | ## Server requirements
25 |
26 | - PHP >= 7.2.0
27 | - OpenSSL PHP Extension
28 | - PDO PHP Extension
29 | - Mbstring PHP Extension
30 | - Tokenizer PHP Extension
31 | - XML PHP Extension
32 |
33 | ## Installation
34 |
35 | - Create a mysql database (default name: `crawler`)
36 | - Install the project with [composer](https://getcomposer.org/):
37 | ```bash
38 | composer create-project hedii/php-crawler crawler
39 | cd crawler
40 | ```
41 | - Open the `.env` file, check the database credentials, and modify it if needed:
42 | ```
43 | DB_CONNECTION=mysql
44 | DB_HOST=127.0.0.1
45 | DB_PORT=3306
46 | DB_DATABASE=crawler
47 | DB_USERNAME=root
48 | DB_PASSWORD=your_password_here
49 | ```
50 | - In the `.env` file, set the application url:
51 | ```
52 | APP_URL=http://crawler.localhost
53 | ```
54 | - Build the crawler application
55 | ```bash
56 | php artisan crawler:build
57 | ```
58 | - Point your web server's document / web root to be the public directory: `/some/path/crawler/public`. The index.php in this directory serves as the front controller for all HTTP requests entering your application. [See Laravel documentation](https://laravel.com/docs/master/installation). I highly recommend using [Laravel Valet](https://laravel.com/docs/master/valet) if you are using a Mac. Otherwise, check [Laravel Homestead](https://laravel.com/docs/master/homestead).
59 | - Done
60 |
61 | ## Usage
62 |
63 | - Navigate to your php-crawler website
64 | - Register a new account
65 | - Create a new search
66 | - Create more searches
67 | - Download the found emails
68 |
69 | ## Testing
70 |
71 | ```
72 | composer test
73 | ```
74 |
75 | ## Contributing
76 |
77 | **All** contributions are welcome :)
78 |
79 | Please write some tests if you are adding or modifying features.
80 |
81 | ## License
82 |
83 | php-crawler is open-sourced software licensed under the [MIT license](https://opensource.org/licenses/MIT).
84 |
--------------------------------------------------------------------------------
/app/Console/Commands/BuildCommand.php:
--------------------------------------------------------------------------------
1 | comment('Running fresh application migrations');
31 |
32 | $returnCode = $this->call('migrate:fresh');
33 |
34 | if ($returnCode !== 0) {
35 | $this->error('Unable run the fresh application migrations');
36 |
37 | return false;
38 | } else {
39 | $this->info('Application fresh migrations ok');
40 | $this->line('---');
41 | }
42 |
43 | $this->comment('Installing Laravel Passport');
44 |
45 | $returnCode = $this->callSilent('passport:install');
46 |
47 | if ($returnCode !== 0) {
48 | $this->error('Unable to install Laravel Passport');
49 |
50 | return false;
51 | } else {
52 | $this->info('Laravel Passport installed');
53 | $this->line('---');
54 | }
55 |
56 | $this->info('Crawler app ready!');
57 |
58 | return true;
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/app/Console/Commands/CrawlCommand.php:
--------------------------------------------------------------------------------
1 | argument('search_id'))) {
33 | $this->error("No search with the id `{$this->argument('search_id')}`");
34 |
35 | return false;
36 | }
37 |
38 | $crawler = new Crawler($search);
39 |
40 | return $crawler->run();
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/app/Console/Kernel.php:
--------------------------------------------------------------------------------
1 | command('inspire')
29 | // ->hourly();
30 | }
31 |
32 | /**
33 | * Register the commands for the application.
34 | *
35 | * @return void
36 | */
37 | protected function commands()
38 | {
39 | $this->load(__DIR__.'/Commands');
40 |
41 | require base_path('routes/console.php');
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/app/Email.php:
--------------------------------------------------------------------------------
1 | 'integer'
31 | ];
32 |
33 | /**
34 | * Get the search that owns the email.
35 | *
36 | * @return \Illuminate\Database\Eloquent\Relations\BelongsTo
37 | */
38 | public function search(): BelongsTo
39 | {
40 | return $this->belongsTo(Search::class);
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/app/Exceptions/Handler.php:
--------------------------------------------------------------------------------
1 | 'v1']);
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/app/Http/Controllers/Api/SearchController.php:
--------------------------------------------------------------------------------
1 | update(array_filter($request->validated()));
34 |
35 | return new SearchResource($search->fresh());
36 | }
37 |
38 | /**
39 | * Remove the specified resource from storage.
40 | *
41 | * @param \App\Search $search
42 | * @return \Illuminate\Http\JsonResponse
43 | */
44 | public function destroy(Search $search): JsonResponse
45 | {
46 | $search->delete();
47 |
48 | return new JsonResponse(null, Response::HTTP_NO_CONTENT);
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/app/Http/Controllers/Api/SearchStatisticsController.php:
--------------------------------------------------------------------------------
1 | function (Builder $query) {
24 | $query->where('is_crawled', true);
25 | },
26 | 'urls as not_crawled_urls_count' => function (Builder $query) {
27 | $query->where('is_crawled', false);
28 | }
29 | ])->find($search->id);
30 |
31 | return new SearchResource($search);
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/app/Http/Controllers/Api/UserController.php:
--------------------------------------------------------------------------------
1 | user());
21 | }
22 |
23 | /**
24 | * Update the specified resource in storage.
25 | *
26 | * @param \App\Http\Requests\UserRequest $request
27 | * @param \App\User $user
28 | * @return \App\Http\Resources\UserResource
29 | */
30 | public function update(UserRequest $request, User $user): UserResource
31 | {
32 | $user->update($request->validated());
33 |
34 | return new UserResource($user->fresh());
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/app/Http/Controllers/Api/UserSearchController.php:
--------------------------------------------------------------------------------
1 | searches()->paginate());
22 | }
23 |
24 | /**
25 | * Store a newly created resource in storage.
26 | *
27 | * @param \App\Http\Requests\UserSearchRequest $request
28 | * @param \App\User $user
29 | * @return \App\Http\Resources\SearchResource
30 | */
31 | public function store(UserSearchRequest $request, User $user): SearchResource
32 | {
33 | $search = $user->searches()->create($request->validated());
34 |
35 | CrawlJob::dispatch($search);
36 |
37 | return new SearchResource($search->fresh());
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/app/Http/Controllers/Auth/ForgotPasswordController.php:
--------------------------------------------------------------------------------
1 | middleware('guest');
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/app/Http/Controllers/Auth/LoginController.php:
--------------------------------------------------------------------------------
1 | middleware('guest')->except('logout');
38 |
39 | $this->redirectTo = route('dashboard');
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/app/Http/Controllers/Auth/RegisterController.php:
--------------------------------------------------------------------------------
1 | middleware('guest');
40 |
41 | $this->redirectTo = route('dashboard');
42 | }
43 |
44 | /**
45 | * Get a validator for an incoming registration request.
46 | *
47 | * @param array $data
48 | * @return \Illuminate\Contracts\Validation\Validator
49 | */
50 | protected function validator(array $data)
51 | {
52 | return Validator::make($data, [
53 | 'name' => 'required|string|max:255',
54 | 'email' => 'required|string|email|max:255|unique:users',
55 | 'password' => 'required|string|min:6|confirmed',
56 | ]);
57 | }
58 |
59 | /**
60 | * Create a new user instance after a valid registration.
61 | *
62 | * @param array $data
63 | * @return \App\User
64 | */
65 | protected function create(array $data)
66 | {
67 | return User::create([
68 | 'name' => $data['name'],
69 | 'email' => $data['email'],
70 | 'password' => bcrypt($data['password']),
71 | ]);
72 | }
73 | }
74 |
--------------------------------------------------------------------------------
/app/Http/Controllers/Auth/ResetPasswordController.php:
--------------------------------------------------------------------------------
1 | middleware('guest');
38 |
39 | $this->redirectTo = route('dashboard');
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/app/Http/Controllers/Controller.php:
--------------------------------------------------------------------------------
1 | emails->pluck('name')->unique()->all();
20 |
21 | $tempFileName = str_random() . '.txt';
22 |
23 | Storage::put($tempFileName, implode("\r\n", $emails));
24 |
25 | return response()
26 | ->download(storage_path("app/{$tempFileName}"))
27 | ->deleteFileAfterSend(true);
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/app/Http/Controllers/UserEmailController.php:
--------------------------------------------------------------------------------
1 | emails->pluck('name')->unique()->all();
22 |
23 | $tempFileName = str_random() . '.txt';
24 |
25 | Storage::put($tempFileName, implode("\r\n", $emails));
26 |
27 | return response()
28 | ->download(storage_path("app/{$tempFileName}"))
29 | ->deleteFileAfterSend(true);
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/app/Http/Kernel.php:
--------------------------------------------------------------------------------
1 | [
31 | \App\Http\Middleware\EncryptCookies::class,
32 | \Illuminate\Cookie\Middleware\AddQueuedCookiesToResponse::class,
33 | \Illuminate\Session\Middleware\StartSession::class,
34 | // \Illuminate\Session\Middleware\AuthenticateSession::class,
35 | \Illuminate\View\Middleware\ShareErrorsFromSession::class,
36 | \App\Http\Middleware\VerifyCsrfToken::class,
37 | \Illuminate\Routing\Middleware\SubstituteBindings::class,
38 | \Laravel\Passport\Http\Middleware\CreateFreshApiToken::class,
39 | ],
40 |
41 | 'api' => [
42 | 'throttle:60,1',
43 | 'bindings',
44 | ],
45 | ];
46 |
47 | /**
48 | * The application's route middleware.
49 | *
50 | * These middleware may be assigned to groups or used individually.
51 | *
52 | * @var array
53 | */
54 | protected $routeMiddleware = [
55 | 'auth' => \Illuminate\Auth\Middleware\Authenticate::class,
56 | 'auth.basic' => \Illuminate\Auth\Middleware\AuthenticateWithBasicAuth::class,
57 | 'bindings' => \Illuminate\Routing\Middleware\SubstituteBindings::class,
58 | 'can' => \Illuminate\Auth\Middleware\Authorize::class,
59 | 'guest' => \App\Http\Middleware\RedirectIfAuthenticated::class,
60 | 'throttle' => \Illuminate\Routing\Middleware\ThrottleRequests::class,
61 | ];
62 | }
63 |
--------------------------------------------------------------------------------
/app/Http/Middleware/EncryptCookies.php:
--------------------------------------------------------------------------------
1 | check()) {
21 | return redirect(route('dashboard'));
22 | }
23 |
24 | return $next($request);
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/app/Http/Middleware/TrimStrings.php:
--------------------------------------------------------------------------------
1 | 'nullable|boolean',
30 | 'status' => ['nullable', Rule::in([
31 | Search::STATUS_FINISHED,
32 | Search::STATUS_PAUSED,
33 | ])]
34 | ];
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/app/Http/Requests/UserRequest.php:
--------------------------------------------------------------------------------
1 | 'required|string|max:255',
29 | 'email' => [
30 | 'required',
31 | 'string',
32 | 'email',
33 | 'max:255',
34 | Rule::unique('users')->ignore($this->route('user')->id),
35 | ],
36 | 'password' => 'nullable|string|min:6|confirmed'
37 | ];
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/app/Http/Requests/UserSearchRequest.php:
--------------------------------------------------------------------------------
1 | 'required|url',
28 | 'is_limited' => 'required|boolean'
29 | ];
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/app/Http/Resources/SearchCollection.php:
--------------------------------------------------------------------------------
1 | search = $search;
34 | }
35 |
36 | /**
37 | * Execute the job.
38 | */
39 | public function handle(): void
40 | {
41 | if (! App::environment('testing')) {
42 | $php = (new PhpExecutableFinder())->find();
43 |
44 | $process = Process::fromShellCommandline(
45 | "{$php} artisan crawler:crawl {$this->search->id} > /dev/null 2>&1 &",
46 | realpath(base_path())
47 | );
48 |
49 | $process->run();
50 | }
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/app/Policies/SearchPolicy.php:
--------------------------------------------------------------------------------
1 | id === $search->user_id;
23 | }
24 |
25 | /**
26 | * Determine whether the user can create searches.
27 | *
28 | * @param \App\User $user
29 | * @return bool
30 | */
31 | public function create(User $user): bool
32 | {
33 | return true;
34 | }
35 |
36 | /**
37 | * Determine whether the user can update the search.
38 | *
39 | * @param \App\User $user
40 | * @param \App\Search $search
41 | * @return bool
42 | */
43 | public function update(User $user, Search $search): bool
44 | {
45 | return $user->id === $search->user_id;
46 | }
47 |
48 | /**
49 | * Determine whether the user can delete the search.
50 | *
51 | * @param \App\User $user
52 | * @param \App\Search $search
53 | * @return bool
54 | */
55 | public function delete(User $user, Search $search): bool
56 | {
57 | return $user->id === $search->user_id;
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/app/Policies/UserPolicy.php:
--------------------------------------------------------------------------------
1 | id === $model->id;
22 | }
23 |
24 | /**
25 | * Determine whether the user can update the model.
26 | *
27 | * @param \App\User $user
28 | * @param \App\User $model
29 | * @return bool
30 | */
31 | public function update(User $user, User $model): bool
32 | {
33 | return $user->id === $model->id;
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/app/Providers/AppServiceProvider.php:
--------------------------------------------------------------------------------
1 | UserPolicy::class,
21 | Search::class => SearchPolicy::class
22 | ];
23 |
24 | /**
25 | * Register any authentication / authorization services.
26 | *
27 | * @return void
28 | */
29 | public function boot()
30 | {
31 | $this->registerPolicies();
32 |
33 | Passport::routes();
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/app/Providers/BroadcastServiceProvider.php:
--------------------------------------------------------------------------------
1 | [
17 | 'App\Listeners\EventListener',
18 | ],
19 | ];
20 |
21 | /**
22 | * Register any events for your application.
23 | *
24 | * @return void
25 | */
26 | public function boot()
27 | {
28 | parent::boot();
29 |
30 | //
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/app/Providers/RouteServiceProvider.php:
--------------------------------------------------------------------------------
1 | mapApiRoutes();
39 |
40 | $this->mapWebRoutes();
41 |
42 | //
43 | }
44 |
45 | /**
46 | * Define the "web" routes for the application.
47 | *
48 | * These routes all receive session state, CSRF protection, etc.
49 | *
50 | * @return void
51 | */
52 | protected function mapWebRoutes()
53 | {
54 | Route::middleware('web')
55 | ->namespace($this->namespace)
56 | ->group(base_path('routes/web.php'));
57 | }
58 |
59 | /**
60 | * Define the "api" routes for the application.
61 | *
62 | * These routes are typically stateless.
63 | *
64 | * @return void
65 | */
66 | protected function mapApiRoutes()
67 | {
68 | Route::prefix('api')
69 | ->prefix('api/v1')
70 | ->middleware('api')
71 | ->namespace("{$this->namespace}\Api")
72 | ->group(base_path('routes/api.php'));
73 | }
74 | }
75 |
--------------------------------------------------------------------------------
/app/Search.php:
--------------------------------------------------------------------------------
1 | 'integer',
31 | 'is_limited' => 'boolean'
32 | ];
33 |
34 | /**
35 | * Get the user that owns the search.
36 | *
37 | * @return \Illuminate\Database\Eloquent\Relations\BelongsTo
38 | */
39 | public function user(): BelongsTo
40 | {
41 | return $this->belongsTo(User::class);
42 | }
43 |
44 | /**
45 | * Get the urls for the search.
46 | *
47 | * @return \Illuminate\Database\Eloquent\Relations\HasMany
48 | */
49 | public function urls(): HasMany
50 | {
51 | return $this->hasMany(Url::class);
52 | }
53 |
54 | /**
55 | * Get the emails for the search.
56 | *
57 | * @return \Illuminate\Database\Eloquent\Relations\HasMany
58 | */
59 | public function emails(): HasMany
60 | {
61 | return $this->hasMany(Email::class);
62 | }
63 |
64 | /**
65 | * Whether the search is in "created" state.
66 | *
67 | * @return bool
68 | */
69 | public function getIsCreatedAttribute(): bool
70 | {
71 | return $this->status === self::STATUS_CREATED;
72 | }
73 |
74 | /**
75 | * Whether the search is in "running" state.
76 | *
77 | * @return bool
78 | */
79 | public function getIsRunningAttribute(): bool
80 | {
81 | return $this->status === self::STATUS_RUNNING;
82 | }
83 |
84 | /**
85 | * Whether the search is in "finished" state.
86 | *
87 | * @return bool
88 | */
89 | public function getIsFinishedAttribute(): bool
90 | {
91 | return $this->status === self::STATUS_FINISHED;
92 | }
93 |
94 | /**
95 | * Whether the search is in "failed" state.
96 | *
97 | * @return bool
98 | */
99 | public function getIsFailedAttribute(): bool
100 | {
101 | return $this->status === self::STATUS_FAILED;
102 | }
103 |
104 | /**
105 | * Get the search's base domain.
106 | *
107 | * @return string
108 | */
109 | public function getDomainAttribute(): string
110 | {
111 | $parsed = parse_url($this->url);
112 |
113 | return "{$parsed['scheme']}://{$parsed['host']}";
114 | }
115 | }
116 |
--------------------------------------------------------------------------------
/app/Url.php:
--------------------------------------------------------------------------------
1 | 'integer',
32 | 'is_crawled' => 'boolean'
33 | ];
34 |
35 | /**
36 | * Get the search that owns the url.
37 | *
38 | * @return \Illuminate\Database\Eloquent\Relations\BelongsTo
39 | */
40 | public function search(): BelongsTo
41 | {
42 | return $this->belongsTo(Search::class);
43 | }
44 |
45 | /**
46 | * Whether the url is not crawled yet.
47 | *
48 | * @return bool
49 | */
50 | public function getIsNotCrawledAttribute(): bool
51 | {
52 | return ! $this->is_crawled;
53 | }
54 |
55 | /**
56 | * Scope a query to only include not crawled urls.
57 | *
58 | * @param \Illuminate\Database\Eloquent\Builder $query
59 | * @return \Illuminate\Database\Eloquent\Builder
60 | */
61 | public function scopeNotCrawled(Builder $query): Builder
62 | {
63 | return $query->where('is_crawled', false);
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/app/User.php:
--------------------------------------------------------------------------------
1 | hasMany(Search::class);
41 | }
42 |
43 | /**
44 | * Get all of the emails for the user.
45 | *
46 | * @return \Illuminate\Database\Eloquent\Relations\HasManyThrough
47 | */
48 | public function emails(): HasManyThrough
49 | {
50 | return $this->hasManyThrough(Email::class, Search::class);
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/app/helpers.php:
--------------------------------------------------------------------------------
1 | getName(), $namespace)) {
21 | $routes[$route->getName()] = config('app.url') . '/' . $route->uri();
22 | }
23 | }
24 |
25 | return $routes;
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/artisan:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env php
2 | make(Illuminate\Contracts\Console\Kernel::class);
34 |
35 | $status = $kernel->handle(
36 | $input = new Symfony\Component\Console\Input\ArgvInput,
37 | new Symfony\Component\Console\Output\ConsoleOutput
38 | );
39 |
40 | /*
41 | |--------------------------------------------------------------------------
42 | | Shutdown The Application
43 | |--------------------------------------------------------------------------
44 | |
45 | | Once Artisan has finished running, we will fire off the shutdown events
46 | | so that any final work may be done by the application before we shut
47 | | down the process. This is the last thing to happen to the request.
48 | |
49 | */
50 |
51 | $kernel->terminate($input, $status);
52 |
53 | exit($status);
54 |
--------------------------------------------------------------------------------
/bootstrap/app.php:
--------------------------------------------------------------------------------
1 | singleton(
30 | Illuminate\Contracts\Http\Kernel::class,
31 | App\Http\Kernel::class
32 | );
33 |
34 | $app->singleton(
35 | Illuminate\Contracts\Console\Kernel::class,
36 | App\Console\Kernel::class
37 | );
38 |
39 | $app->singleton(
40 | Illuminate\Contracts\Debug\ExceptionHandler::class,
41 | App\Exceptions\Handler::class
42 | );
43 |
44 | /*
45 | |--------------------------------------------------------------------------
46 | | Return The Application
47 | |--------------------------------------------------------------------------
48 | |
49 | | This script returns the application instance. The instance is given to
50 | | the calling script so we can separate the building of the instances
51 | | from the actual running of the application and sending responses.
52 | |
53 | */
54 |
55 | return $app;
56 |
--------------------------------------------------------------------------------
/bootstrap/cache/.gitignore:
--------------------------------------------------------------------------------
1 | *
2 | !.gitignore
3 |
--------------------------------------------------------------------------------
/composer.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hedii/php-crawler",
3 | "description": "A crawler application written with php and Laravel that finds email addresses on the internets.",
4 | "keywords": ["php-crawler", "hedii", "crawler", "php", "laravel", "email"],
5 | "homepage": "https://github.com/hedii/php-crawler",
6 | "authors": [
7 | {
8 | "name": "Hedi Chaibi",
9 | "email": "contact@hedichaibi.com",
10 | "homepage": "https://hedichaibi.com"
11 | }
12 | ],
13 | "license": "MIT",
14 | "type": "project",
15 | "require": {
16 | "php": ">=7.2.0",
17 | "doctrine/dbal": "^2.6",
18 | "fideloper/proxy": "~4.0",
19 | "guzzlehttp/guzzle": "^6.3",
20 | "hedii/extractors": "^2.0",
21 | "laravel/framework": "5.7.*",
22 | "laravel/passport": "^7.0",
23 | "laravel/tinker": "~1.0"
24 | },
25 | "require-dev": {
26 | "barryvdh/laravel-ide-helper": "^2.4",
27 | "filp/whoops": "~2.0",
28 | "fzaninotto/faker": "~1.4",
29 | "mockery/mockery": "~1.0",
30 | "nunomaduro/collision": "~1.1",
31 | "phpunit/phpunit": "~7.0"
32 | },
33 | "autoload": {
34 | "classmap": [
35 | "database/seeds",
36 | "database/factories"
37 | ],
38 | "psr-4": {
39 | "App\\": "app/"
40 | },
41 | "files": [
42 | "app/helpers.php"
43 | ]
44 | },
45 | "autoload-dev": {
46 | "psr-4": {
47 | "Tests\\": "tests/"
48 | }
49 | },
50 | "extra": {
51 | "laravel": {
52 | "dont-discover": [
53 | ]
54 | }
55 | },
56 | "scripts": {
57 | "test": "vendor/bin/phpunit",
58 | "post-root-package-install": [
59 | "@php -r \"file_exists('.env') || copy('.env.example', '.env');\""
60 | ],
61 | "post-create-project-cmd": [
62 | "@php artisan key:generate"
63 | ],
64 | "post-autoload-dump": [
65 | "Illuminate\\Foundation\\ComposerScripts::postAutoloadDump",
66 | "@php artisan package:discover"
67 | ],
68 | "post-update-cmd": [
69 | "Illuminate\\Foundation\\ComposerScripts::postUpdate",
70 | "php artisan ide-helper:generate",
71 | "php artisan ide-helper:meta"
72 | ]
73 | },
74 | "config": {
75 | "preferred-install": "dist",
76 | "sort-packages": true,
77 | "optimize-autoloader": true
78 | }
79 | }
80 |
--------------------------------------------------------------------------------
/config/auth.php:
--------------------------------------------------------------------------------
1 | [
17 | 'guard' => 'web',
18 | 'passwords' => 'users',
19 | ],
20 |
21 | /*
22 | |--------------------------------------------------------------------------
23 | | Authentication Guards
24 | |--------------------------------------------------------------------------
25 | |
26 | | Next, you may define every authentication guard for your application.
27 | | Of course, a great default configuration has been defined for you
28 | | here which uses session storage and the Eloquent user provider.
29 | |
30 | | All authentication drivers have a user provider. This defines how the
31 | | users are actually retrieved out of your database or other storage
32 | | mechanisms used by this application to persist your user's data.
33 | |
34 | | Supported: "session", "token"
35 | |
36 | */
37 |
38 | 'guards' => [
39 | 'web' => [
40 | 'driver' => 'session',
41 | 'provider' => 'users',
42 | ],
43 |
44 | 'api' => [
45 | 'driver' => 'passport',
46 | 'provider' => 'users',
47 | ],
48 | ],
49 |
50 | /*
51 | |--------------------------------------------------------------------------
52 | | User Providers
53 | |--------------------------------------------------------------------------
54 | |
55 | | All authentication drivers have a user provider. This defines how the
56 | | users are actually retrieved out of your database or other storage
57 | | mechanisms used by this application to persist your user's data.
58 | |
59 | | If you have multiple user tables or models you may configure multiple
60 | | sources which represent each model / table. These sources may then
61 | | be assigned to any extra authentication guards you have defined.
62 | |
63 | | Supported: "database", "eloquent"
64 | |
65 | */
66 |
67 | 'providers' => [
68 | 'users' => [
69 | 'driver' => 'eloquent',
70 | 'model' => App\User::class,
71 | ],
72 |
73 | // 'users' => [
74 | // 'driver' => 'database',
75 | // 'table' => 'users',
76 | // ],
77 | ],
78 |
79 | /*
80 | |--------------------------------------------------------------------------
81 | | Resetting Passwords
82 | |--------------------------------------------------------------------------
83 | |
84 | | You may specify multiple password reset configurations if you have more
85 | | than one user table or model in the application and you want to have
86 | | separate password reset settings based on the specific user types.
87 | |
88 | | The expire time is the number of minutes that the reset token should be
89 | | considered valid. This security feature keeps tokens short-lived so
90 | | they have less time to be guessed. You may change this as needed.
91 | |
92 | */
93 |
94 | 'passwords' => [
95 | 'users' => [
96 | 'provider' => 'users',
97 | 'table' => 'password_resets',
98 | 'expire' => 60,
99 | ],
100 | ],
101 |
102 | ];
103 |
--------------------------------------------------------------------------------
/config/broadcasting.php:
--------------------------------------------------------------------------------
1 | env('BROADCAST_DRIVER', 'null'),
19 |
20 | /*
21 | |--------------------------------------------------------------------------
22 | | Broadcast Connections
23 | |--------------------------------------------------------------------------
24 | |
25 | | Here you may define all of the broadcast connections that will be used
26 | | to broadcast events to other systems or over websockets. Samples of
27 | | each available type of connection are provided inside this array.
28 | |
29 | */
30 |
31 | 'connections' => [
32 |
33 | 'pusher' => [
34 | 'driver' => 'pusher',
35 | 'key' => env('PUSHER_APP_KEY'),
36 | 'secret' => env('PUSHER_APP_SECRET'),
37 | 'app_id' => env('PUSHER_APP_ID'),
38 | 'options' => [
39 | 'cluster' => env('PUSHER_APP_CLUSTER'),
40 | 'encrypted' => true,
41 | ],
42 | ],
43 |
44 | 'redis' => [
45 | 'driver' => 'redis',
46 | 'connection' => 'default',
47 | ],
48 |
49 | 'log' => [
50 | 'driver' => 'log',
51 | ],
52 |
53 | 'null' => [
54 | 'driver' => 'null',
55 | ],
56 |
57 | ],
58 |
59 | ];
60 |
--------------------------------------------------------------------------------
/config/cache.php:
--------------------------------------------------------------------------------
1 | env('CACHE_DRIVER', 'file'),
21 |
22 | /*
23 | |--------------------------------------------------------------------------
24 | | Cache Stores
25 | |--------------------------------------------------------------------------
26 | |
27 | | Here you may define all of the cache "stores" for your application as
28 | | well as their drivers. You may even define multiple stores for the
29 | | same cache driver to group types of items stored in your caches.
30 | |
31 | */
32 |
33 | 'stores' => [
34 |
35 | 'apc' => [
36 | 'driver' => 'apc',
37 | ],
38 |
39 | 'array' => [
40 | 'driver' => 'array',
41 | ],
42 |
43 | 'database' => [
44 | 'driver' => 'database',
45 | 'table' => 'cache',
46 | 'connection' => null,
47 | ],
48 |
49 | 'file' => [
50 | 'driver' => 'file',
51 | 'path' => storage_path('framework/cache/data'),
52 | ],
53 |
54 | 'memcached' => [
55 | 'driver' => 'memcached',
56 | 'persistent_id' => env('MEMCACHED_PERSISTENT_ID'),
57 | 'sasl' => [
58 | env('MEMCACHED_USERNAME'),
59 | env('MEMCACHED_PASSWORD'),
60 | ],
61 | 'options' => [
62 | // Memcached::OPT_CONNECT_TIMEOUT => 2000,
63 | ],
64 | 'servers' => [
65 | [
66 | 'host' => env('MEMCACHED_HOST', '127.0.0.1'),
67 | 'port' => env('MEMCACHED_PORT', 11211),
68 | 'weight' => 100,
69 | ],
70 | ],
71 | ],
72 |
73 | 'redis' => [
74 | 'driver' => 'redis',
75 | 'connection' => 'cache',
76 | ],
77 |
78 | ],
79 |
80 | /*
81 | |--------------------------------------------------------------------------
82 | | Cache Key Prefix
83 | |--------------------------------------------------------------------------
84 | |
85 | | When utilizing a RAM based store such as APC or Memcached, there might
86 | | be other applications utilizing the same cache. So, we'll specify a
87 | | value to get prefixed to all our keys so we can avoid collisions.
88 | |
89 | */
90 |
91 | 'prefix' => env('CACHE_PREFIX', Str::slug(env('APP_NAME', 'laravel'), '_').'_cache'),
92 |
93 | ];
94 |
--------------------------------------------------------------------------------
/config/database.php:
--------------------------------------------------------------------------------
1 | env('DB_CONNECTION', 'mysql'),
17 |
18 | /*
19 | |--------------------------------------------------------------------------
20 | | Database Connections
21 | |--------------------------------------------------------------------------
22 | |
23 | | Here are each of the database connections setup for your application.
24 | | Of course, examples of configuring each database platform that is
25 | | supported by Laravel is shown below to make development simple.
26 | |
27 | |
28 | | All database work in Laravel is done through the PHP PDO facilities
29 | | so make sure you have the driver for your particular database of
30 | | choice installed on your machine before you begin development.
31 | |
32 | */
33 |
34 | 'connections' => [
35 |
36 | 'sqlite' => [
37 | 'driver' => 'sqlite',
38 | 'database' => env('DB_DATABASE', database_path('database.sqlite')),
39 | 'prefix' => '',
40 | 'foreign_key_constraints' => env('DB_FOREIGN_KEYS', true),
41 | ],
42 |
43 | 'mysql' => [
44 | 'driver' => 'mysql',
45 | 'host' => env('DB_HOST', '127.0.0.1'),
46 | 'port' => env('DB_PORT', '3306'),
47 | 'database' => env('DB_DATABASE', 'forge'),
48 | 'username' => env('DB_USERNAME', 'forge'),
49 | 'password' => env('DB_PASSWORD', ''),
50 | 'unix_socket' => env('DB_SOCKET', ''),
51 | 'charset' => 'utf8mb4',
52 | 'collation' => 'utf8mb4_unicode_ci',
53 | 'prefix' => '',
54 | 'prefix_indexes' => true,
55 | 'strict' => true,
56 | 'engine' => null,
57 | ],
58 |
59 | 'pgsql' => [
60 | 'driver' => 'pgsql',
61 | 'host' => env('DB_HOST', '127.0.0.1'),
62 | 'port' => env('DB_PORT', '5432'),
63 | 'database' => env('DB_DATABASE', 'forge'),
64 | 'username' => env('DB_USERNAME', 'forge'),
65 | 'password' => env('DB_PASSWORD', ''),
66 | 'charset' => 'utf8',
67 | 'prefix' => '',
68 | 'prefix_indexes' => true,
69 | 'schema' => 'public',
70 | 'sslmode' => 'prefer',
71 | ],
72 |
73 | 'sqlsrv' => [
74 | 'driver' => 'sqlsrv',
75 | 'host' => env('DB_HOST', 'localhost'),
76 | 'port' => env('DB_PORT', '1433'),
77 | 'database' => env('DB_DATABASE', 'forge'),
78 | 'username' => env('DB_USERNAME', 'forge'),
79 | 'password' => env('DB_PASSWORD', ''),
80 | 'charset' => 'utf8',
81 | 'prefix' => '',
82 | 'prefix_indexes' => true,
83 | ],
84 |
85 | ],
86 |
87 | /*
88 | |--------------------------------------------------------------------------
89 | | Migration Repository Table
90 | |--------------------------------------------------------------------------
91 | |
92 | | This table keeps track of all the migrations that have already run for
93 | | your application. Using this information, we can determine which of
94 | | the migrations on disk haven't actually been run in the database.
95 | |
96 | */
97 |
98 | 'migrations' => 'migrations',
99 |
100 | /*
101 | |--------------------------------------------------------------------------
102 | | Redis Databases
103 | |--------------------------------------------------------------------------
104 | |
105 | | Redis is an open source, fast, and advanced key-value store that also
106 | | provides a richer body of commands than a typical key-value system
107 | | such as APC or Memcached. Laravel makes it easy to dig right in.
108 | |
109 | */
110 |
111 | 'redis' => [
112 |
113 | 'client' => 'predis',
114 |
115 | 'default' => [
116 | 'host' => env('REDIS_HOST', '127.0.0.1'),
117 | 'password' => env('REDIS_PASSWORD', null),
118 | 'port' => env('REDIS_PORT', 6379),
119 | 'database' => env('REDIS_DB', 0),
120 | ],
121 |
122 | 'cache' => [
123 | 'host' => env('REDIS_HOST', '127.0.0.1'),
124 | 'password' => env('REDIS_PASSWORD', null),
125 | 'port' => env('REDIS_PORT', 6379),
126 | 'database' => env('REDIS_CACHE_DB', 1),
127 | ],
128 |
129 | ],
130 |
131 | ];
132 |
--------------------------------------------------------------------------------
/config/filesystems.php:
--------------------------------------------------------------------------------
1 | env('FILESYSTEM_DRIVER', 'local'),
17 |
18 | /*
19 | |--------------------------------------------------------------------------
20 | | Default Cloud Filesystem Disk
21 | |--------------------------------------------------------------------------
22 | |
23 | | Many applications store files both locally and in the cloud. For this
24 | | reason, you may specify a default "cloud" driver here. This driver
25 | | will be bound as the Cloud disk implementation in the container.
26 | |
27 | */
28 |
29 | 'cloud' => env('FILESYSTEM_CLOUD', 's3'),
30 |
31 | /*
32 | |--------------------------------------------------------------------------
33 | | Filesystem Disks
34 | |--------------------------------------------------------------------------
35 | |
36 | | Here you may configure as many filesystem "disks" as you wish, and you
37 | | may even configure multiple disks of the same driver. Defaults have
38 | | been setup for each driver as an example of the required options.
39 | |
40 | | Supported Drivers: "local", "ftp", "sftp", "s3", "rackspace"
41 | |
42 | */
43 |
44 | 'disks' => [
45 |
46 | 'local' => [
47 | 'driver' => 'local',
48 | 'root' => storage_path('app'),
49 | ],
50 |
51 | 'public' => [
52 | 'driver' => 'local',
53 | 'root' => storage_path('app/public'),
54 | 'url' => env('APP_URL').'/storage',
55 | 'visibility' => 'public',
56 | ],
57 |
58 | 's3' => [
59 | 'driver' => 's3',
60 | 'key' => env('AWS_ACCESS_KEY_ID'),
61 | 'secret' => env('AWS_SECRET_ACCESS_KEY'),
62 | 'region' => env('AWS_DEFAULT_REGION'),
63 | 'bucket' => env('AWS_BUCKET'),
64 | 'url' => env('AWS_URL'),
65 | ],
66 |
67 | ],
68 |
69 | ];
70 |
--------------------------------------------------------------------------------
/config/hashing.php:
--------------------------------------------------------------------------------
1 | 'bcrypt',
19 |
20 | /*
21 | |--------------------------------------------------------------------------
22 | | Bcrypt Options
23 | |--------------------------------------------------------------------------
24 | |
25 | | Here you may specify the configuration options that should be used when
26 | | passwords are hashed using the Bcrypt algorithm. This will allow you
27 | | to control the amount of time it takes to hash the given password.
28 | |
29 | */
30 |
31 | 'bcrypt' => [
32 | 'rounds' => env('BCRYPT_ROUNDS', 10),
33 | ],
34 |
35 | /*
36 | |--------------------------------------------------------------------------
37 | | Argon Options
38 | |--------------------------------------------------------------------------
39 | |
40 | | Here you may specify the configuration options that should be used when
41 | | passwords are hashed using the Argon algorithm. These will allow you
42 | | to control the amount of time it takes to hash the given password.
43 | |
44 | */
45 |
46 | 'argon' => [
47 | 'memory' => 1024,
48 | 'threads' => 2,
49 | 'time' => 2,
50 | ],
51 |
52 | ];
53 |
--------------------------------------------------------------------------------
/config/logging.php:
--------------------------------------------------------------------------------
1 | env('LOG_CHANNEL', 'stack'),
20 |
21 | /*
22 | |--------------------------------------------------------------------------
23 | | Log Channels
24 | |--------------------------------------------------------------------------
25 | |
26 | | Here you may configure the log channels for your application. Out of
27 | | the box, Laravel uses the Monolog PHP logging library. This gives
28 | | you a variety of powerful log handlers / formatters to utilize.
29 | |
30 | | Available Drivers: "single", "daily", "slack", "syslog",
31 | | "errorlog", "monolog",
32 | | "custom", "stack"
33 | |
34 | */
35 |
36 | 'channels' => [
37 | 'stack' => [
38 | 'driver' => 'stack',
39 | 'channels' => ['daily'],
40 | ],
41 |
42 | 'single' => [
43 | 'driver' => 'single',
44 | 'path' => storage_path('logs/laravel.log'),
45 | 'level' => 'debug',
46 | ],
47 |
48 | 'daily' => [
49 | 'driver' => 'daily',
50 | 'path' => storage_path('logs/laravel.log'),
51 | 'level' => 'debug',
52 | 'days' => 14,
53 | ],
54 |
55 | 'slack' => [
56 | 'driver' => 'slack',
57 | 'url' => env('LOG_SLACK_WEBHOOK_URL'),
58 | 'username' => 'Laravel Log',
59 | 'emoji' => ':boom:',
60 | 'level' => 'critical',
61 | ],
62 |
63 | 'papertrail' => [
64 | 'driver' => 'monolog',
65 | 'level' => 'debug',
66 | 'handler' => SyslogUdpHandler::class,
67 | 'handler_with' => [
68 | 'host' => env('PAPERTRAIL_URL'),
69 | 'port' => env('PAPERTRAIL_PORT'),
70 | ],
71 | ],
72 |
73 | 'stderr' => [
74 | 'driver' => 'monolog',
75 | 'handler' => StreamHandler::class,
76 | 'formatter' => env('LOG_STDERR_FORMATTER'),
77 | 'with' => [
78 | 'stream' => 'php://stderr',
79 | ],
80 | ],
81 |
82 | 'syslog' => [
83 | 'driver' => 'syslog',
84 | 'level' => 'debug',
85 | ],
86 |
87 | 'errorlog' => [
88 | 'driver' => 'errorlog',
89 | 'level' => 'debug',
90 | ],
91 | ],
92 |
93 | ];
94 |
--------------------------------------------------------------------------------
/config/mail.php:
--------------------------------------------------------------------------------
1 | env('MAIL_DRIVER', 'smtp'),
20 |
21 | /*
22 | |--------------------------------------------------------------------------
23 | | SMTP Host Address
24 | |--------------------------------------------------------------------------
25 | |
26 | | Here you may provide the host address of the SMTP server used by your
27 | | applications. A default option is provided that is compatible with
28 | | the Mailgun mail service which will provide reliable deliveries.
29 | |
30 | */
31 |
32 | 'host' => env('MAIL_HOST', 'smtp.mailgun.org'),
33 |
34 | /*
35 | |--------------------------------------------------------------------------
36 | | SMTP Host Port
37 | |--------------------------------------------------------------------------
38 | |
39 | | This is the SMTP port used by your application to deliver e-mails to
40 | | users of the application. Like the host we have set this value to
41 | | stay compatible with the Mailgun e-mail application by default.
42 | |
43 | */
44 |
45 | 'port' => env('MAIL_PORT', 587),
46 |
47 | /*
48 | |--------------------------------------------------------------------------
49 | | Global "From" Address
50 | |--------------------------------------------------------------------------
51 | |
52 | | You may wish for all e-mails sent by your application to be sent from
53 | | the same address. Here, you may specify a name and address that is
54 | | used globally for all e-mails that are sent by your application.
55 | |
56 | */
57 |
58 | 'from' => [
59 | 'address' => env('MAIL_FROM_ADDRESS', 'hello@example.com'),
60 | 'name' => env('MAIL_FROM_NAME', 'Example'),
61 | ],
62 |
63 | /*
64 | |--------------------------------------------------------------------------
65 | | E-Mail Encryption Protocol
66 | |--------------------------------------------------------------------------
67 | |
68 | | Here you may specify the encryption protocol that should be used when
69 | | the application send e-mail messages. A sensible default using the
70 | | transport layer security protocol should provide great security.
71 | |
72 | */
73 |
74 | 'encryption' => env('MAIL_ENCRYPTION', 'tls'),
75 |
76 | /*
77 | |--------------------------------------------------------------------------
78 | | SMTP Server Username
79 | |--------------------------------------------------------------------------
80 | |
81 | | If your SMTP server requires a username for authentication, you should
82 | | set it here. This will get used to authenticate with your server on
83 | | connection. You may also set the "password" value below this one.
84 | |
85 | */
86 |
87 | 'username' => env('MAIL_USERNAME'),
88 |
89 | 'password' => env('MAIL_PASSWORD'),
90 |
91 | /*
92 | |--------------------------------------------------------------------------
93 | | Sendmail System Path
94 | |--------------------------------------------------------------------------
95 | |
96 | | When using the "sendmail" driver to send e-mails, we will need to know
97 | | the path to where Sendmail lives on this server. A default path has
98 | | been provided here, which will work well on most of your systems.
99 | |
100 | */
101 |
102 | 'sendmail' => '/usr/sbin/sendmail -bs',
103 |
104 | /*
105 | |--------------------------------------------------------------------------
106 | | Markdown Mail Settings
107 | |--------------------------------------------------------------------------
108 | |
109 | | If you are using Markdown based email rendering, you may configure your
110 | | theme and component paths here, allowing you to customize the design
111 | | of the emails. Or, you may simply stick with the Laravel defaults!
112 | |
113 | */
114 |
115 | 'markdown' => [
116 | 'theme' => 'default',
117 |
118 | 'paths' => [
119 | resource_path('views/vendor/mail'),
120 | ],
121 | ],
122 |
123 | /*
124 | |--------------------------------------------------------------------------
125 | | Log Channel
126 | |--------------------------------------------------------------------------
127 | |
128 | | If you are using the "log" driver, you may specify the logging channel
129 | | if you prefer to keep mail messages separate from other log entries
130 | | for simpler reading. Otherwise, the default channel will be used.
131 | |
132 | */
133 |
134 | 'log_channel' => env('MAIL_LOG_CHANNEL'),
135 |
136 | ];
137 |
--------------------------------------------------------------------------------
/config/queue.php:
--------------------------------------------------------------------------------
1 | env('QUEUE_CONNECTION', 'sync'),
17 |
18 | /*
19 | |--------------------------------------------------------------------------
20 | | Queue Connections
21 | |--------------------------------------------------------------------------
22 | |
23 | | Here you may configure the connection information for each server that
24 | | is used by your application. A default configuration has been added
25 | | for each back-end shipped with Laravel. You are free to add more.
26 | |
27 | | Drivers: "sync", "database", "beanstalkd", "sqs", "redis", "null"
28 | |
29 | */
30 |
31 | 'connections' => [
32 |
33 | 'sync' => [
34 | 'driver' => 'sync',
35 | ],
36 |
37 | 'database' => [
38 | 'driver' => 'database',
39 | 'table' => 'jobs',
40 | 'queue' => 'default',
41 | 'retry_after' => 90,
42 | ],
43 |
44 | 'beanstalkd' => [
45 | 'driver' => 'beanstalkd',
46 | 'host' => 'localhost',
47 | 'queue' => 'default',
48 | 'retry_after' => 90,
49 | ],
50 |
51 | 'sqs' => [
52 | 'driver' => 'sqs',
53 | 'key' => env('SQS_KEY', 'your-public-key'),
54 | 'secret' => env('SQS_SECRET', 'your-secret-key'),
55 | 'prefix' => env('SQS_PREFIX', 'https://sqs.us-east-1.amazonaws.com/your-account-id'),
56 | 'queue' => env('SQS_QUEUE', 'your-queue-name'),
57 | 'region' => env('SQS_REGION', 'us-east-1'),
58 | ],
59 |
60 | 'redis' => [
61 | 'driver' => 'redis',
62 | 'connection' => 'default',
63 | 'queue' => env('REDIS_QUEUE', 'default'),
64 | 'retry_after' => 90,
65 | 'block_for' => null,
66 | ],
67 |
68 | ],
69 |
70 | /*
71 | |--------------------------------------------------------------------------
72 | | Failed Queue Jobs
73 | |--------------------------------------------------------------------------
74 | |
75 | | These options configure the behavior of failed queue job logging so you
76 | | can control which database and table are used to store the jobs that
77 | | have failed. You may change them to any database / table you wish.
78 | |
79 | */
80 |
81 | 'failed' => [
82 | 'database' => env('DB_CONNECTION', 'mysql'),
83 | 'table' => 'failed_jobs',
84 | ],
85 |
86 | ];
87 |
--------------------------------------------------------------------------------
/config/services.php:
--------------------------------------------------------------------------------
1 | [
18 | 'domain' => env('MAILGUN_DOMAIN'),
19 | 'secret' => env('MAILGUN_SECRET'),
20 | 'endpoint' => env('MAILGUN_ENDPOINT', 'api.mailgun.net'),
21 | ],
22 |
23 | 'ses' => [
24 | 'key' => env('SES_KEY'),
25 | 'secret' => env('SES_SECRET'),
26 | 'region' => env('SES_REGION', 'us-east-1'),
27 | ],
28 |
29 | 'sparkpost' => [
30 | 'secret' => env('SPARKPOST_SECRET'),
31 | ],
32 |
33 | 'stripe' => [
34 | 'model' => App\User::class,
35 | 'key' => env('STRIPE_KEY'),
36 | 'secret' => env('STRIPE_SECRET'),
37 | 'webhook' => [
38 | 'secret' => env('STRIPE_WEBHOOK_SECRET'),
39 | 'tolerance' => env('STRIPE_WEBHOOK_TOLERANCE', 300),
40 | ],
41 | ],
42 |
43 | ];
44 |
--------------------------------------------------------------------------------
/config/view.php:
--------------------------------------------------------------------------------
1 | [
17 | resource_path('views'),
18 | ],
19 |
20 | /*
21 | |--------------------------------------------------------------------------
22 | | Compiled View Path
23 | |--------------------------------------------------------------------------
24 | |
25 | | This option determines where all the compiled Blade templates will be
26 | | stored for your application. Typically, this is within the storage
27 | | directory. However, as usual, you are free to change this value.
28 | |
29 | */
30 |
31 | 'compiled' => env(
32 | 'VIEW_COMPILED_PATH',
33 | realpath(storage_path('framework/views'))
34 | ),
35 |
36 | ];
37 |
--------------------------------------------------------------------------------
/database/.gitignore:
--------------------------------------------------------------------------------
1 | *.sqlite
2 |
--------------------------------------------------------------------------------
/database/factories/EmailFactory.php:
--------------------------------------------------------------------------------
1 | define(Email::class, function (Faker $faker) {
8 | return [
9 | 'name' => $faker->safeEmail,
10 | 'search_id' => function () {
11 | return factory(Search::class)->create()->id;
12 | }
13 | ];
14 | });
15 |
--------------------------------------------------------------------------------
/database/factories/SearchFactory.php:
--------------------------------------------------------------------------------
1 | define(Search::class, function (Faker $faker) {
8 | return [
9 | 'url' => $faker->url,
10 | 'user_id' => function () {
11 | return factory(User::class)->create()->id;
12 | },
13 | 'is_limited' => $faker->boolean,
14 | 'pid' => null
15 | ];
16 | });
17 |
18 | $factory->state(Search::class, 'created', ['status' => Search::STATUS_CREATED]);
19 | $factory->state(Search::class, 'running', ['status' => Search::STATUS_RUNNING]);
20 | $factory->state(Search::class, 'finished', ['status' => Search::STATUS_FINISHED]);
21 | $factory->state(Search::class, 'paused', ['status' => Search::STATUS_PAUSED]);
22 | $factory->state(Search::class, 'failed', ['status' => Search::STATUS_FAILED]);
23 | $factory->state(Search::class, 'limited', ['is_limited' => true]);
24 | $factory->state(Search::class, 'unlimited', ['is_limited' => false]);
25 |
--------------------------------------------------------------------------------
/database/factories/UrlFactory.php:
--------------------------------------------------------------------------------
1 | define(Url::class, function (Faker $faker) {
8 | return [
9 | 'name' => $faker->url,
10 | 'search_id' => function () {
11 | return factory(Search::class)->create()->id;
12 | },
13 | 'is_crawled' => $faker->boolean
14 | ];
15 | });
16 |
17 | $factory->state(Url::class, 'crawled', ['is_crawled' => true]);
18 | $factory->state(Url::class, 'not_crawled', ['is_crawled' => false]);
--------------------------------------------------------------------------------
/database/factories/UserFactory.php:
--------------------------------------------------------------------------------
1 | define(User::class, function (Faker $faker) {
18 | return [
19 | 'name' => $faker->name,
20 | 'email' => $faker->unique()->safeEmail,
21 | 'password' => '$2y$10$TKh8H1.PfQx37YgCzwiKb.KjNyWgaHb9cbcoQgdIVFlYg7B77UdFm', // secret
22 | 'remember_token' => str_random(10)
23 | ];
24 | });
25 |
--------------------------------------------------------------------------------
/database/migrations/2014_10_12_000000_create_users_table.php:
--------------------------------------------------------------------------------
1 | increments('id');
18 | $table->string('name');
19 | $table->string('email')->unique();
20 | $table->string('password');
21 | $table->rememberToken();
22 | $table->timestamps();
23 | });
24 | }
25 |
26 | /**
27 | * Reverse the migrations.
28 | *
29 | * @return void
30 | */
31 | public function down()
32 | {
33 | Schema::dropIfExists('users');
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/database/migrations/2014_10_12_100000_create_password_resets_table.php:
--------------------------------------------------------------------------------
1 | string('email')->index();
18 | $table->string('token');
19 | $table->timestamp('created_at')->nullable();
20 | });
21 | }
22 |
23 | /**
24 | * Reverse the migrations.
25 | *
26 | * @return void
27 | */
28 | public function down()
29 | {
30 | Schema::dropIfExists('password_resets');
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/database/migrations/2016_06_01_000001_create_oauth_auth_codes_table.php:
--------------------------------------------------------------------------------
1 | string('id', 100)->primary();
18 | $table->integer('user_id');
19 | $table->integer('client_id');
20 | $table->text('scopes')->nullable();
21 | $table->boolean('revoked');
22 | $table->dateTime('expires_at')->nullable();
23 | });
24 | }
25 |
26 | /**
27 | * Reverse the migrations.
28 | *
29 | * @return void
30 | */
31 | public function down()
32 | {
33 | Schema::drop('oauth_auth_codes');
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/database/migrations/2016_06_01_000002_create_oauth_access_tokens_table.php:
--------------------------------------------------------------------------------
1 | string('id', 100)->primary();
18 | $table->integer('user_id')->index()->nullable();
19 | $table->integer('client_id');
20 | $table->string('name')->nullable();
21 | $table->text('scopes')->nullable();
22 | $table->boolean('revoked');
23 | $table->timestamps();
24 | $table->dateTime('expires_at')->nullable();
25 | });
26 | }
27 |
28 | /**
29 | * Reverse the migrations.
30 | *
31 | * @return void
32 | */
33 | public function down()
34 | {
35 | Schema::drop('oauth_access_tokens');
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/database/migrations/2016_06_01_000003_create_oauth_refresh_tokens_table.php:
--------------------------------------------------------------------------------
1 | string('id', 100)->primary();
18 | $table->string('access_token_id', 100)->index();
19 | $table->boolean('revoked');
20 | $table->dateTime('expires_at')->nullable();
21 | });
22 | }
23 |
24 | /**
25 | * Reverse the migrations.
26 | *
27 | * @return void
28 | */
29 | public function down()
30 | {
31 | Schema::drop('oauth_refresh_tokens');
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/database/migrations/2016_06_01_000004_create_oauth_clients_table.php:
--------------------------------------------------------------------------------
1 | increments('id');
18 | $table->integer('user_id')->index()->nullable();
19 | $table->string('name');
20 | $table->string('secret', 100);
21 | $table->text('redirect');
22 | $table->boolean('personal_access_client');
23 | $table->boolean('password_client');
24 | $table->boolean('revoked');
25 | $table->timestamps();
26 | });
27 | }
28 |
29 | /**
30 | * Reverse the migrations.
31 | *
32 | * @return void
33 | */
34 | public function down()
35 | {
36 | Schema::drop('oauth_clients');
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/database/migrations/2016_06_01_000005_create_oauth_personal_access_clients_table.php:
--------------------------------------------------------------------------------
1 | increments('id');
18 | $table->integer('client_id')->index();
19 | $table->timestamps();
20 | });
21 | }
22 |
23 | /**
24 | * Reverse the migrations.
25 | *
26 | * @return void
27 | */
28 | public function down()
29 | {
30 | Schema::drop('oauth_personal_access_clients');
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/database/migrations/2018_01_19_203819_create_sessions_table.php:
--------------------------------------------------------------------------------
1 | string('id')->unique();
18 | $table->unsignedInteger('user_id')->nullable();
19 | $table->string('ip_address', 45)->nullable();
20 | $table->text('user_agent')->nullable();
21 | $table->text('payload');
22 | $table->integer('last_activity');
23 | });
24 | }
25 |
26 | /**
27 | * Reverse the migrations.
28 | *
29 | * @return void
30 | */
31 | public function down()
32 | {
33 | Schema::dropIfExists('sessions');
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/database/migrations/2018_01_19_205205_create_searches_table.php:
--------------------------------------------------------------------------------
1 | increments('id');
19 | $table->text('url');
20 | $table->unsignedInteger('user_id');
21 | $table->enum('status', [
22 | Search::STATUS_CREATED,
23 | Search::STATUS_RUNNING,
24 | Search::STATUS_FINISHED,
25 | Search::STATUS_PAUSED,
26 | Search::STATUS_FAILED
27 | ])->default(Search::STATUS_CREATED);
28 | $table->boolean('is_limited')->default(true);
29 | $table->string('pid')->nullable();
30 | $table->timestamps();
31 |
32 | $table->foreign('user_id')
33 | ->references('id')->on('users')
34 | ->onDelete('cascade');
35 | });
36 | }
37 |
38 | /**
39 | * Reverse the migrations.
40 | *
41 | * @return void
42 | */
43 | public function down()
44 | {
45 | Schema::dropIfExists('searches');
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/database/migrations/2018_01_19_212151_create_urls_table.php:
--------------------------------------------------------------------------------
1 | increments('id');
18 | $table->string('name')->index();
19 | $table->unsignedInteger('search_id');
20 | $table->boolean('is_crawled')->default(false);
21 |
22 | $table->unique(['name', 'search_id']);
23 |
24 | $table->foreign('search_id')
25 | ->references('id')->on('searches')
26 | ->onDelete('cascade');
27 | });
28 | }
29 |
30 | /**
31 | * Reverse the migrations.
32 | *
33 | * @return void
34 | */
35 | public function down()
36 | {
37 | Schema::dropIfExists('urls');
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/database/migrations/2018_01_19_213937_create_emails_table.php:
--------------------------------------------------------------------------------
1 | increments('id');
18 | $table->string('name')->index();
19 | $table->unsignedInteger('search_id');
20 |
21 | $table->unique(['name', 'search_id']);
22 |
23 | $table->foreign('search_id')
24 | ->references('id')->on('searches')
25 | ->onDelete('cascade');
26 | });
27 | }
28 |
29 | /**
30 | * Reverse the migrations.
31 | *
32 | * @return void
33 | */
34 | public function down()
35 | {
36 | Schema::dropIfExists('emails');
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/database/seeds/DatabaseSeeder.php:
--------------------------------------------------------------------------------
1 | call(UsersTableSeeder::class);
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/database/seeds/UsersTableSeeder.php:
--------------------------------------------------------------------------------
1 | create([
16 | 'name' => 'test',
17 | 'email' => 'test@example.com'
18 | ]);
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "private": true,
3 | "scripts": {
4 | "dev": "npm run development",
5 | "development": "cross-env NODE_ENV=development node_modules/webpack/bin/webpack.js --progress --hide-modules --config=node_modules/laravel-mix/setup/webpack.config.js",
6 | "watch": "npm run development -- --watch",
7 | "watch-poll": "npm run watch -- --watch-poll",
8 | "hot": "cross-env NODE_ENV=development node_modules/webpack-dev-server/bin/webpack-dev-server.js --inline --hot --config=node_modules/laravel-mix/setup/webpack.config.js",
9 | "prod": "npm run production",
10 | "production": "cross-env NODE_ENV=production node_modules/webpack/bin/webpack.js --no-progress --hide-modules --config=node_modules/laravel-mix/setup/webpack.config.js"
11 | },
12 | "devDependencies": {
13 | "cross-env": "^5.1",
14 | "laravel-mix": "^4.0.7",
15 | "sass": "^1.15.2",
16 | "sass-loader": "^7.1.0",
17 | "vue-template-compiler": "^2.5.22"
18 | },
19 | "dependencies": {
20 | "axios": "^0.18",
21 | "babel-polyfill": "^6.26.0",
22 | "bootstrap": "^4.2.1",
23 | "jquery": "^3.3.1",
24 | "popper.js": "^1.14.6",
25 | "vue": "^2.5.22",
26 | "vue-router": "^3.0.2"
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/phpunit.xml:
--------------------------------------------------------------------------------
1 |
2 | You are now logged in.
17 |
17 | Welcome {{ $root.user.name }}!
8 |
23 |
23 |
25 | Status:
26 |
27 |
28 | {{ search.url }}
29 |
30 |
32 |
33 | Emails Statistics
34 |
35 |
37 |
43 |
44 | Urls Statistics
45 |
46 |
50 | Url crawled: {{ search.crawled_urls_count }}
51 | Url not crawled: {{ search.not_crawled_urls_count }}
52 |