├── .github └── workflows │ └── ci.yml ├── .gitignore ├── CLOUDFRONT.md ├── MIGRATION.md ├── README.md ├── TAGGING.md ├── TESTING.md ├── classes ├── azure_blob_storage_file_system.php ├── azure_file_system.php ├── check │ ├── proxy_range_request.php │ ├── tagging_migration_status.php │ ├── tagging_status.php │ ├── tagging_sync_status.php │ └── token_expiry.php ├── digitalocean_file_system.php ├── local │ ├── manager.php │ ├── object_manipulator │ │ ├── candidates │ │ │ ├── candidates_factory.php │ │ │ ├── candidates_finder.php │ │ │ ├── checker_candidates.php │ │ │ ├── deleter_candidates.php │ │ │ ├── manipulator_candidates.php │ │ │ ├── manipulator_candidates_base.php │ │ │ ├── orphaner_candidates.php │ │ │ ├── puller_candidates.php │ │ │ ├── pusher_candidates.php │ │ │ └── recoverer_candidates.php │ │ ├── checker.php │ │ ├── deleter.php │ │ ├── logger.php │ │ ├── manipulator.php │ │ ├── manipulator_builder.php │ │ ├── object_manipulator.php │ │ ├── orphaner.php │ │ ├── puller.php │ │ ├── pusher.php │ │ └── recoverer.php │ ├── report │ │ ├── location_report_builder.php │ │ ├── log_size_report_builder.php │ │ ├── mime_type_report_builder.php │ │ ├── object_location_history_table.php │ │ ├── object_status_history_table.php │ │ ├── objectfs_report.php │ │ ├── objectfs_report_builder.php │ │ └── tag_count_report_builder.php │ ├── store │ │ ├── azure │ │ │ ├── client.php │ │ │ ├── file_system.php │ │ │ └── stream_wrapper.php │ │ ├── azure_blob_storage │ │ │ ├── client.php │ │ │ └── file_system.php │ │ ├── digitalocean │ │ │ ├── client.php │ │ │ └── file_system.php │ │ ├── object_client.php │ │ ├── object_client_base.php │ │ ├── object_file_system.php │ │ ├── s3 │ │ │ ├── admin_settings_aws_region.php │ │ │ ├── client.php │ │ │ └── file_system.php │ │ ├── signed_url.php │ │ └── swift │ │ │ ├── client.php │ │ │ ├── file_system.php │ │ │ └── stream_wrapper.php │ ├── table │ │ └── files_table.php │ ├── tag │ │ ├── environment_source.php │ │ ├── location_source.php │ │ ├── tag_manager.php │ │ └── tag_source.php │ └── tag_sync_count_result.php ├── log │ ├── aggregate_logger.php │ ├── null_logger.php │ ├── objectfs_logger.php │ ├── objectfs_statistic.php │ └── real_time_logger.php ├── privacy │ └── provider.php ├── s3_file_system.php ├── swift_file_system.php ├── task │ ├── check_objects_location.php │ ├── delete_local_empty_directories.php │ ├── delete_local_objects.php │ ├── delete_orphaned_object_metadata.php │ ├── generate_status_report.php │ ├── objectfs_task.php │ ├── orphan_objects.php │ ├── populate_objects_filesize.php │ ├── pull_objects_from_storage.php │ ├── push_objects_to_storage.php │ ├── recover_error_objects.php │ ├── task.php │ ├── trigger_update_object_tags.php │ └── update_object_tags.php └── tests │ ├── test_azure_blob_storage_integration_client.php │ ├── test_azure_integration_client.php │ ├── test_client.php │ ├── test_digitalocean_integration_client.php │ ├── test_file_system.php │ ├── test_s3_integration_client.php │ ├── test_swift_integration_client.php │ └── testcase.php ├── db ├── install.xml ├── tasks.php └── upgrade.php ├── lang └── en │ └── tool_objectfs.php ├── lib.php ├── missing_files.php ├── object_location.php ├── object_status.php ├── patch ├── core27.diff ├── core31.diff ├── core33.diff └── core38.diff ├── phpunit.xml ├── pix ├── Microsoft-logo_rgb_c-gray.png └── catalyst-logo.png ├── presignedurl_tests.php ├── renderer.php ├── settings.php ├── styles.css ├── tests ├── check │ ├── tagging_migration_status_test.php │ └── tagging_sync_status_test.php ├── fixtures │ ├── test.txt │ ├── testimage.png │ ├── testlarge.pdf │ ├── testrelativeurls.html │ ├── testsmall.pdf │ ├── testspreadsheet.ods │ ├── teststyle.css │ ├── testvideo.mp4 │ ├── testwriter.odt │ └── 😀.txt ├── local │ ├── manager_test.php │ ├── object_manipulator │ │ ├── checker_test.php │ │ ├── deleter_test.php │ │ ├── orphaner_test.php │ │ ├── puller_test.php │ │ ├── pusher_test.php │ │ └── recoverer_test.php │ ├── report │ │ └── object_status_test.php │ ├── store │ │ └── clients_test.php │ ├── tagging_test.php │ └── tasks_test.php ├── object_file_system_test.php ├── privacy │ └── privacy_test.php ├── task │ ├── populate_objects_filesize_test.php │ ├── trigger_update_object_tags_test.php │ └── update_object_tags_test.php └── token_expiry_test.php ├── thirdpartylibs.xml └── version.php /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # .github/workflows/ci.yml 2 | name: ci 3 | 4 | on: [push, pull_request] 5 | 6 | jobs: 7 | ci: 8 | uses: catalyst/catalyst-moodle-workflows/.github/workflows/ci.yml@main 9 | secrets: 10 | moodle_org_token: ${{ secrets.MOODLE_ORG_TOKEN }} 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tests/classes/integration_test_config.php 2 | -------------------------------------------------------------------------------- /MIGRATION.md: -------------------------------------------------------------------------------- 1 | # Migration guides 2 | 3 | ## Migrating from local_azure_storage to local_azureblobstorage 4 | 5 | Since March 2024, Microsoft officially discontinued support for the PHP SDK for Azure storage. This means the `local_azure_storage` plugin which is a wrapper of the SDK will no longer be updated, and is already out of date with newer php versions. 6 | 7 | The plugin `local_azureblobstorage` was created to replace this, with a simpler and cleaner API for interacting with the Azure blob storage service via REST APIs. Objectfs has been updated with a new client handler class to enable you to cut over to the new storage system as easily as possible. 8 | 9 | This new library is only supported in higher PHP versions. 10 | 11 | ### Steps 12 | 1. If you are on Moodle 4.2, ensure you have updated the previous `local_azure_storage` to the `MOODLE_42_STABLE` branch. This fixes some fatal errors caused by Guzzle namespace conflicts. 13 | 2. Install `local_azureblobstorage` https://github.com/catalyst/moodle-local_azureblobstorage 14 | 3. In the objectfs settings, change the `filesystem` config variable to `\tool_objectfs\azure_blob_storage_file_system` and save. ObjectFS will now be using the new API to communicate with Azure. You do not need to enter new credentials, the credentials are shared with the old client. 15 | 4. Test and ensure the site works as expected. 16 | 5. If you encounter any issues and wish to revert back, simply change the `filesystem` configuration back to the old client. This will immediately begin to use the old libraries again. 17 | 6. Once you are happy, simply uninstall the `local_azure_storage` plugin. The migration is now complete. -------------------------------------------------------------------------------- /TAGGING.md: -------------------------------------------------------------------------------- 1 | # Tagging 2 | Tagging allows extra metadata about your files to be send to the external object store. These sources are defined in code, and currently cannot be configured on/off from the UI. 3 | 4 | Currently, this is only implemented for the S3 file system client. 5 | **Tagging vs metadata** 6 | 7 | Note object tags are different from object metadata. 8 | 9 | Object metadata is immutable, and attached to the object on upload. With metadata, if you wish to update it (for example during a migration, or the sources changed), you have to copy the object with the new metadata, and delete the old object. This is not ideal, since deletion is optional in objectfs. 10 | 11 | Object tags are more suitable, since their permissions can be managed separately (e.g. a client can be allowed to modify tags, but not delete objects). 12 | 13 | ## File system setup 14 | ### S3 15 | [See the S3 docs for more information about tagging](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html). 16 | 17 | You must allow `s3:GetObjectTagging` and `s3:PutObjectTagging` permission to the objectfs client. 18 | 19 | ## Sources 20 | The following sources are implemented currently: 21 | ### Environment 22 | What environment the file was uploaded in. Configure the environment using `taggingenvironment` in the objectfs plugin settings. 23 | 24 | This tag is also used by objectfs to determine if tags can be overwritten. See [Multiple environments setup](#multiple-environments-setup) for more information. 25 | 26 | ### Location 27 | Either `orphan` if the file no longer exists in the `files` table in Moodle, otherwise `active`. 28 | 29 | ## Multiple environments setup 30 | This feature is designed to work in situations where multiple environments (e.g. prod, staging) points to the same bucket, however, some setup is needed: 31 | 32 | 1. Turn off `overwriteobjecttags` in every environment except the production environment. 33 | 2. Configure `taggingenvironment` to be unique for all environments. 34 | 35 | By doing the above two steps, it will allow the production environment to always set its own tags, even if a file was first uploaded to staging and then to production. 36 | 37 | Lower environments can still update tags, but only if the `environment` matches theirs. This allows staging to manage object tags on objects only it knows about, but as soon as the file is uploaded from production (and therefore have it's environment tag replaced with `prod`), staging will no longer touch it. 38 | 39 | ## Migration 40 | Only new objects uploaded after enabling this feature will have tags added. To backfill tags for previously uploaded objects, you must do the following: 41 | 42 | - Manually run `trigger_update_object_tags` scheduled task from the UI, which queues a `update_object_tags` adhoc task that will process all objects marked as needing sync. 43 | or 44 | - Call the CLI to execute a `update_object_tags` adhoc task manually. 45 | 46 | You may need to update the DB to mark objects tag sync status as needing sync if the object has previously been synced before. 47 | ## Reporting 48 | There is an additional graph added to the object summary report showing the tag value combinations and counts of each. 49 | 50 | Note, this is only for files that have been uploaded from the respective environment, and may not be consistent for environments where `overwriteobjecttags` is disabled (because the site does not know if a file was overwritten in the external store by another client). 51 | 52 | ## For developers 53 | 54 | ### Adding a new source 55 | Note the rules about sources: 56 | - Identifier must be < 32 chars long. 57 | - Value must be < 128 chars long. 58 | 59 | While external providers allow longer key/values, we intentionally limit it to reserve space for future use. These limits may change in the future as the feature matures. 60 | 61 | To add a new source: 62 | - Implement `tag_source` 63 | - Add to the `tag_manager` class 64 | - As part of an upgrade step, mark all objects `tagsyncstatus` to needing sync (using `tag_manager` class, or manually in the DB) 65 | - As part of an upgrade step, queue a `update_object_tags` adhoc task to process the tag migration. -------------------------------------------------------------------------------- /TESTING.md: -------------------------------------------------------------------------------- 1 | # Integration testing 2 | Objectfs supports integration testing with S3, Azure and Swift storage ([Test client initialisation](https://github.com/catalyst/moodle-tool_objectfs/blob/master/tests/classes/test_file_system.php)). 3 | 4 | To run integration tests follow steps below: 5 | 1. Install `tool_objectfs` plugin and required SDK libraries. 6 | 2. Initialise php unit environment: 7 | `php admin/tool/phpunit/cli/init.php` 8 | 3. Set integration credentials in `config.php` for one of the supported storage: 9 | * S3: 10 | ```php 11 | $CFG->phpunit_objectfs_s3_integration_test_credentials = array( 12 | 's3_key' => 'Your key', 13 | 's3_secret' => 'Your secret', 14 | 's3_bucket' => 'Your bucket', 15 | 's3_region' => 'Your region', 16 | ); 17 | ``` 18 | * Azure (deprecated API): 19 | ```php 20 | $CFG->phpunit_objectfs_azure_integration_test_credentials = array( 21 | 'azure_accountname' => 'Your account name', 22 | 'azure_container' => 'Your container', 23 | 'azure_sastoken' => 'Your sas token', 24 | ); 25 | ``` 26 | * Azure Blob Storage: 27 | ```php 28 | $CFG->phpunit_objectfs_azure_blob_storage_integration_test_credentials = [ 29 | 'azure_accountname' => 'Your account name', 30 | 'azure_container' => 'Your container', 31 | 'azure_sastoken' => 'Your sas token', 32 | ]; 33 | ``` 34 | * Swift: 35 | ```php 36 | $CFG->phpunit_objectfs_swift_integration_test_credentials = array( 37 | 'openstack_authurl' => 'Your auth URL', 38 | 'openstack_region' => 'Your region', 39 | 'openstack_container' => 'Your container', 40 | 'openstack_username' => 'Your username', 41 | 'openstack_password' => 'Your password', 42 | 'openstack_tenantname' => 'Your tenantname', 43 | 'openstack_projectid' => 'Your project ID', 44 | ); 45 | ``` 46 | * Run Objectfs tests: 47 | ```php 48 | vendor/bin/phpunit -c admin/tool/objectfs/ 49 | ``` 50 | * To run tests for core subsystems or other plugins that make use of the filesystem against Objectfs, add the following to config.php: 51 | ```php 52 | $CFG->alternative_file_system_class = '\tool_objectfs\tests\test_file_system'; 53 | ``` 54 | -------------------------------------------------------------------------------- /classes/azure_blob_storage_file_system.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs; 18 | 19 | use tool_objectfs\local\store\azure_blob_storage\file_system; 20 | 21 | /** 22 | * File system for Azure Blob Storage. 23 | * 24 | * This file tells objectfs that this storage system is available for use. 25 | * E.g. via $CFG->alternative_file_system_class 26 | * 27 | * @package tool_objectfs 28 | * @author Matthew Hilton 29 | * @copyright Catalyst IT 30 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 31 | */ 32 | class azure_blob_storage_file_system extends file_system { 33 | } 34 | -------------------------------------------------------------------------------- /classes/azure_file_system.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * File system for Azure Blob Storage. 19 | * 20 | * @package tool_objectfs 21 | * @author Nicholas Hoobin 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs; 27 | 28 | use tool_objectfs\local\store\azure\file_system; 29 | 30 | /** 31 | * Unknown? 32 | */ 33 | class azure_file_system extends file_system { 34 | 35 | } 36 | -------------------------------------------------------------------------------- /classes/check/proxy_range_request.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\check; 18 | use core\check\check; 19 | use core\check\result; 20 | 21 | /** 22 | * Status check for objectFS proxied range requests. 23 | * 24 | * @package tool_objectfs 25 | * @author Peter Burnett 26 | * @copyright Catalyst IT 27 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 28 | */ 29 | class proxy_range_request extends check { 30 | /** 31 | * Link to ObjectFS settings page. 32 | * 33 | * @return \action_link|null 34 | */ 35 | public function get_action_link(): ?\action_link { 36 | $url = new \moodle_url('/admin/category.php', ['category' => 'tool_objectfs']); 37 | return new \action_link($url, get_string('pluginname', 'tool_objectfs')); 38 | } 39 | 40 | /** 41 | * Check for the success of a proxied range request, if the setting is enabled. 42 | * 43 | * @return result 44 | */ 45 | public function get_result(): result { 46 | $config = \tool_objectfs\local\manager::get_objectfs_config(); 47 | $client = \tool_objectfs\local\manager::get_client($config); 48 | 49 | $signingsupport = false; 50 | if (!empty($config->filesystem)) { 51 | $signingsupport = (new $config->filesystem())->supports_presigned_urls(); 52 | } 53 | 54 | $testconn = $client->test_connection(); 55 | $connstatus = $testconn->success; 56 | 57 | if ($connstatus && $signingsupport) { 58 | $range = $client->test_range_request(new $config->filesystem()); 59 | 60 | if ($range->result) { 61 | return new result(result::OK, get_string('settings:presignedurl:testrangeok', 'tool_objectfs')); 62 | } else { 63 | return new result(result::WARNING, get_string('settings:presignedurl:testrangeerror', 'tool_objectfs')); 64 | } 65 | } 66 | 67 | return new result(result::UNKNOWN, get_string('check:proxyrangerequestsdisabled', 'tool_objectfs')); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /classes/check/tagging_migration_status.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\check; 18 | 19 | use core\check\check; 20 | use core\check\result; 21 | use core\task\manager; 22 | use html_table; 23 | use html_writer; 24 | use tool_objectfs\task\update_object_tags; 25 | 26 | /** 27 | * Tagging migration status check 28 | * 29 | * @package tool_objectfs 30 | * @author Matthew Hilton 31 | * @copyright Catalyst IT 32 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 33 | */ 34 | class tagging_migration_status extends check { 35 | /** 36 | * Link to ObjectFS settings page. 37 | * 38 | * @return \action_link|null 39 | */ 40 | public function get_action_link(): ?\action_link { 41 | $url = new \moodle_url('/admin/category.php', ['category' => 'tool_objectfs']); 42 | return new \action_link($url, get_string('pluginname', 'tool_objectfs')); 43 | } 44 | 45 | /** 46 | * Get result 47 | * @return result 48 | */ 49 | public function get_result(): result { 50 | // We want to check this regardless if enabled or supported and not exit early. 51 | // Because it may have been turned off accidentally thus causing the migration to fail. 52 | $tasks = manager::get_adhoc_tasks(update_object_tags::class); 53 | 54 | if (empty($tasks)) { 55 | return new result(result::NA, get_string('tagging:migration:nothingrunning', 'tool_objectfs')); 56 | } 57 | 58 | $table = new html_table(); 59 | $table->head = [ 60 | get_string('table:taskid', 'tool_objectfs'), 61 | get_string('table:iteration', 'tool_objectfs'), 62 | get_string('table:status', 'tool_objectfs'), 63 | ]; 64 | 65 | foreach ($tasks as $task) { 66 | $table->data[$task->get_id()] = [$task->get_id(), $task->get_iteration(), $task->get_status_badge()]; 67 | } 68 | $html = html_writer::table($table); 69 | 70 | $ataskisfailing = !empty(array_filter($tasks, function($task) { 71 | return $task->get_fail_delay() > 0; 72 | })); 73 | 74 | if ($ataskisfailing) { 75 | return new result(result::WARNING, get_string('check:tagging:migrationerror', 'tool_objectfs'), $html); 76 | } 77 | 78 | return new result(result::OK, get_string('check:tagging:migrationok', 'tool_objectfs'), $html); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /classes/check/tagging_status.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\check; 18 | 19 | use core\check\check; 20 | use core\check\result; 21 | use tool_objectfs\local\tag\tag_manager; 22 | 23 | /** 24 | * Tagging status check 25 | * 26 | * @package tool_objectfs 27 | * @author Matthew Hilton 28 | * @copyright Catalyst IT 29 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 30 | */ 31 | class tagging_status extends check { 32 | /** 33 | * Link to ObjectFS settings page. 34 | * 35 | * @return \action_link|null 36 | */ 37 | public function get_action_link(): ?\action_link { 38 | $url = new \moodle_url('/admin/category.php', ['category' => 'tool_objectfs']); 39 | return new \action_link($url, get_string('pluginname', 'tool_objectfs')); 40 | } 41 | 42 | /** 43 | * Get result 44 | * @return result 45 | */ 46 | public function get_result(): result { 47 | if (!tag_manager::is_tagging_enabled_and_supported()) { 48 | return new result(result::NA, get_string('check:tagging:na', 'tool_objectfs')); 49 | } 50 | 51 | // Do a tag set test. 52 | $config = \tool_objectfs\local\manager::get_objectfs_config(); 53 | $client = \tool_objectfs\local\manager::get_client($config); 54 | $result = $client->test_set_object_tag(); 55 | 56 | if ($result->success) { 57 | return new result(result::OK, get_string('check:tagging:ok', 'tool_objectfs'), $result->details); 58 | } else { 59 | return new result(result::ERROR, get_string('check:tagging:error', 'tool_objectfs'), $result->details); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /classes/check/tagging_sync_status.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\check; 18 | 19 | use core\check\check; 20 | use core\check\result; 21 | use tool_objectfs\local\tag\tag_manager; 22 | use tool_objectfs\local\tag_sync_count_result; 23 | 24 | /** 25 | * Tagging sync status check 26 | * 27 | * @package tool_objectfs 28 | * @author Matthew Hilton 29 | * @copyright Catalyst IT 30 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 31 | */ 32 | class tagging_sync_status extends check { 33 | /** 34 | * Link to ObjectFS settings page. 35 | * 36 | * @return \action_link|null 37 | */ 38 | public function get_action_link(): ?\action_link { 39 | $url = new \moodle_url('/admin/category.php', ['category' => 'tool_objectfs']); 40 | return new \action_link($url, get_string('pluginname', 'tool_objectfs')); 41 | } 42 | 43 | /** 44 | * Get result 45 | * @return result 46 | */ 47 | public function get_result(): result { 48 | if (!tag_manager::is_tagging_enabled_and_supported()) { 49 | return new tag_sync_count_result(result::NA, get_string('check:tagging:na', 'tool_objectfs')); 50 | } 51 | 52 | // We only do a lightweight check here, the get_details is overwritten in tag_sync_status_result 53 | // to provide more information that is more computationally expensive to calculate. 54 | if (tag_manager::tag_sync_errors_exist()) { 55 | return new tag_sync_count_result(result::WARNING, get_string('check:tagging:syncerror', 'tool_objectfs')); 56 | } 57 | 58 | return new tag_sync_count_result(result::OK, get_string('check:tagging:syncok', 'tool_objectfs')); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /classes/check/token_expiry.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\check; 18 | 19 | use core\check\check; 20 | use core\check\result; 21 | 22 | /** 23 | * Token expiry check. 24 | * 25 | * @package tool_objectfs 26 | * @author Matthew Hilton 27 | * @copyright Catalyst IT 28 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 29 | */ 30 | class token_expiry extends check { 31 | /** 32 | * Link to ObjectFS settings page. 33 | * 34 | * @return \action_link|null 35 | */ 36 | public function get_action_link(): ?\action_link { 37 | $url = new \moodle_url('/admin/category.php', ['category' => 'tool_objectfs']); 38 | return new \action_link($url, get_string('pluginname', 'tool_objectfs')); 39 | } 40 | 41 | /** 42 | * Checks the token expiry time against thresholds 43 | * @return result 44 | */ 45 | public function get_result(): result { 46 | $config = \tool_objectfs\local\manager::get_objectfs_config(); 47 | $client = \tool_objectfs\local\manager::get_client($config); 48 | 49 | // No client set - n/a. 50 | if (empty($client)) { 51 | return new result(result::NA, get_string('check:tokenexpiry:na', 'tool_objectfs')); 52 | } 53 | 54 | $expirytime = $client->get_token_expiry_time(); 55 | $secondsleft = $expirytime - time(); 56 | 57 | $strparams = [ 58 | 'dayssince' => abs(round($secondsleft / DAYSECS)), 59 | 'time' => userdate($expirytime), 60 | ]; 61 | 62 | // Not implemented or token not set - n/a. 63 | if ($expirytime == -1) { 64 | return new result(result::NA, get_string('check:tokenexpiry:na', 'tool_objectfs')); 65 | } 66 | 67 | // Is in past - token has expired. 68 | if ($secondsleft < 0) { 69 | return new result(result::CRITICAL, get_string('check:tokenexpiry:expired', 'tool_objectfs', $strparams)); 70 | } 71 | 72 | // Is in warning period - warn. 73 | $warnthreshold = (int) $config->tokenexpirywarnperiod; 74 | if ($secondsleft < $warnthreshold) { 75 | return new result(result::WARNING, get_string('check:tokenexpiry:expiresin', 'tool_objectfs', $strparams)); 76 | } 77 | 78 | // Else ok. 79 | return new result(result::OK, get_string('check:tokenexpiry:expiresin', 'tool_objectfs', $strparams)); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /classes/digitalocean_file_system.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * File system for Digital Ocean Storage. 19 | * 20 | * @package tool_objectfs 21 | * @author Brian Yanosik 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs; 27 | 28 | use tool_objectfs\local\store\digitalocean\file_system; 29 | 30 | /** 31 | * Unknown? 32 | */ 33 | class digitalocean_file_system extends file_system { 34 | 35 | } 36 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/candidates/candidates_factory.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Class candidates_factory 19 | * @package tool_objectfs 20 | * @author Gleimer Mora 21 | * @copyright Catalyst IT 22 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 23 | */ 24 | 25 | namespace tool_objectfs\local\object_manipulator\candidates; 26 | 27 | use moodle_exception; 28 | use stdClass; 29 | use tool_objectfs\local\object_manipulator\checker; 30 | use tool_objectfs\local\object_manipulator\deleter; 31 | use tool_objectfs\local\object_manipulator\puller; 32 | use tool_objectfs\local\object_manipulator\pusher; 33 | use tool_objectfs\local\object_manipulator\recoverer; 34 | use tool_objectfs\local\object_manipulator\orphaner; 35 | 36 | /** 37 | * Candidates Factory 38 | */ 39 | class candidates_factory { 40 | 41 | /** @var array $manipulatormap */ 42 | private static $manipulatormap = [ 43 | checker::class => checker_candidates::class, 44 | deleter::class => deleter_candidates::class, 45 | puller::class => puller_candidates::class, 46 | pusher::class => pusher_candidates::class, 47 | recoverer::class => recoverer_candidates::class, 48 | orphaner::class => orphaner_candidates::class, 49 | ]; 50 | 51 | /** 52 | * Finder 53 | * @param mixed $manipulator 54 | * @param stdClass $config 55 | * 56 | * @return mixed 57 | * @throws moodle_exception 58 | */ 59 | public static function finder($manipulator, stdClass $config) { 60 | if (isset(self::$manipulatormap[$manipulator])) { 61 | $classname = self::$manipulatormap[$manipulator]; 62 | return new $classname($config); 63 | } 64 | throw new moodle_exception('invalidclass', 'error', '', 'Invalid manipulator class'); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/candidates/candidates_finder.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Class candidates_finder 19 | * @package tool_objectfs 20 | * @author Gleimer Mora 21 | * @copyright Catalyst IT 22 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 23 | */ 24 | 25 | namespace tool_objectfs\local\object_manipulator\candidates; 26 | 27 | use moodle_exception; 28 | use stdClass; 29 | 30 | /** 31 | * Candidates Finder 32 | */ 33 | class candidates_finder { 34 | 35 | /** @var string $finder */ 36 | private $finder = ''; 37 | 38 | /** 39 | * candidates_finder constructor. 40 | * @param string $manipulator 41 | * @param stdClass $config 42 | * @throws moodle_exception 43 | */ 44 | public function __construct($manipulator, stdClass $config) { 45 | $this->finder = candidates_factory::finder($manipulator, $config); 46 | } 47 | 48 | /** 49 | * get 50 | * @return array 51 | */ 52 | public function get() { 53 | return $this->finder->get(); 54 | } 55 | 56 | /** 57 | * get_query_name 58 | * @return string 59 | */ 60 | public function get_query_name() { 61 | return $this->finder->get_query_name(); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/candidates/checker_candidates.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Class checker_candidates 19 | * @package tool_objectfs 20 | * @author Gleimer Mora 21 | * @copyright Catalyst IT 22 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 23 | */ 24 | 25 | namespace tool_objectfs\local\object_manipulator\candidates; 26 | 27 | /** 28 | * chcker_candiates 29 | */ 30 | class checker_candidates extends manipulator_candidates_base { 31 | 32 | /** 33 | * queryname 34 | * @var string 35 | */ 36 | protected $queryname = 'get_check_candidates'; 37 | 38 | /** 39 | * get_candiates_sql 40 | * @return string 41 | */ 42 | public function get_candidates_sql() { 43 | return 'SELECT f.contenthash 44 | FROM {files} f 45 | LEFT JOIN {tool_objectfs_objects} o ON f.contenthash = o.contenthash 46 | WHERE f.filesize > 0 47 | AND o.location is NULL 48 | GROUP BY f.contenthash'; 49 | } 50 | 51 | /** 52 | * get_candidates_sql_params 53 | * @return array 54 | */ 55 | public function get_candidates_sql_params() { 56 | return []; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/candidates/deleter_candidates.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Class deleter_candidates 19 | * @package tool_objectfs 20 | * @author Gleimer Mora 21 | * @copyright Catalyst IT 22 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 23 | */ 24 | 25 | namespace tool_objectfs\local\object_manipulator\candidates; 26 | 27 | /** 28 | * deleter_candidates 29 | */ 30 | class deleter_candidates extends manipulator_candidates_base { 31 | 32 | /** 33 | * queryname 34 | * @var string 35 | */ 36 | protected $queryname = 'get_delete_candidates'; 37 | 38 | /** 39 | * get_candiates_sql 40 | * @return string 41 | */ 42 | public function get_candidates_sql() { 43 | return 'SELECT contenthash, 44 | filesize 45 | FROM {tool_objectfs_objects} 46 | WHERE timeduplicated <= :consistancythreshold 47 | AND location = :location 48 | AND filesize > :sizethreshold'; 49 | } 50 | 51 | /** 52 | * get_candiates_sql_params 53 | * @return array 54 | */ 55 | public function get_candidates_sql_params() { 56 | $consistancythreshold = time() - $this->config->consistencydelay; 57 | return [ 58 | 'consistancythreshold' => $consistancythreshold, 59 | 'location' => OBJECT_LOCATION_DUPLICATED, 60 | 'sizethreshold' => $this->config->sizethreshold, 61 | ]; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/candidates/manipulator_candidates.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\local\object_manipulator\candidates; 18 | 19 | use dml_exception; 20 | 21 | /** 22 | * Interface manipulator_candidates 23 | * @package tool_objectfs 24 | * @author Gleimer Mora 25 | * @copyright Catalyst IT 26 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 27 | */ 28 | interface manipulator_candidates { 29 | 30 | /** 31 | * Returns a manipulator query name for logging. 32 | * 33 | * @return string 34 | */ 35 | public function get_query_name(); 36 | 37 | /** 38 | * Returns SQL to retrieve objects for manipulation. 39 | * 40 | * @return string 41 | */ 42 | public function get_candidates_sql(); 43 | 44 | /** 45 | * Returns a list of parameters for SQL from get_candidates_sql. 46 | * 47 | * @return array 48 | */ 49 | public function get_candidates_sql_params(); 50 | 51 | /** 52 | * get 53 | * @return array 54 | * @throws dml_exception 55 | */ 56 | public function get(); 57 | } 58 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/candidates/manipulator_candidates_base.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Class candidates_factory 19 | * @package tool_objectfs 20 | * @author Gleimer Mora 21 | * @copyright Catalyst IT 22 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 23 | */ 24 | 25 | namespace tool_objectfs\local\object_manipulator\candidates; 26 | 27 | use dml_exception; 28 | use stdClass; 29 | 30 | /** 31 | * manipulator_candidates_base 32 | */ 33 | abstract class manipulator_candidates_base implements manipulator_candidates { 34 | 35 | /** @var stdClass $config */ 36 | protected $config; 37 | 38 | /** 39 | * manipulator_candidates_base constructor. 40 | * @param stdClass $config 41 | */ 42 | public function __construct(stdClass $config) { 43 | $this->config = $config; 44 | } 45 | 46 | /** 47 | * get_query_name 48 | * @return string 49 | */ 50 | public function get_query_name() { 51 | return $this->queryname; 52 | } 53 | 54 | /** 55 | * get 56 | * @return array 57 | * @throws dml_exception 58 | */ 59 | public function get() { 60 | global $DB; 61 | return $DB->get_records_sql( 62 | $this->get_candidates_sql(), 63 | $this->get_candidates_sql_params(), 64 | 0, 65 | $this->config->batchsize 66 | ); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/candidates/orphaner_candidates.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Class orphaner_candidates 19 | * @package tool_objectfs 20 | * @author Nathan Mares 21 | * @copyright Catalyst IT 22 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 23 | */ 24 | 25 | namespace tool_objectfs\local\object_manipulator\candidates; 26 | 27 | /** 28 | * orphaner_candidates 29 | */ 30 | class orphaner_candidates extends manipulator_candidates_base { 31 | 32 | /** 33 | * queryname 34 | * @var string 35 | */ 36 | protected $queryname = 'get_orphan_candidates'; 37 | 38 | /** 39 | * get_candidates_sql 40 | * @return string 41 | */ 42 | public function get_candidates_sql() { 43 | return 'SELECT o.id, o.contenthash, o.location 44 | FROM {tool_objectfs_objects} o 45 | LEFT JOIN {files} f ON o.contenthash = f.contenthash 46 | WHERE f.id is null 47 | AND o.location != :location'; 48 | } 49 | 50 | /** 51 | * get_candidates_sql_params 52 | * @return array 53 | */ 54 | public function get_candidates_sql_params() { 55 | return [ 56 | 'location' => OBJECT_LOCATION_ORPHANED, 57 | ]; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/candidates/puller_candidates.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Class puller_candidates 19 | * @package tool_objectfs 20 | * @author Gleimer Mora 21 | * @copyright Catalyst IT 22 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 23 | */ 24 | 25 | namespace tool_objectfs\local\object_manipulator\candidates; 26 | 27 | /** 28 | * puller_candidates 29 | */ 30 | class puller_candidates extends manipulator_candidates_base { 31 | 32 | /** 33 | * queryname 34 | * @var string 35 | */ 36 | protected $queryname = 'get_pull_candidates'; 37 | 38 | /** 39 | * get_candidates_sql 40 | * @return string 41 | */ 42 | public function get_candidates_sql() { 43 | return 'SELECT contenthash, 44 | filesize 45 | FROM {tool_objectfs_objects} 46 | WHERE filesize <= :sizethreshold 47 | AND location = :location'; 48 | } 49 | 50 | /** 51 | * get_candidates_sql_params 52 | * @return array 53 | */ 54 | public function get_candidates_sql_params() { 55 | return ['sizethreshold' => $this->config->sizethreshold, 'location' => OBJECT_LOCATION_EXTERNAL]; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/candidates/pusher_candidates.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Class pusher_candidates 19 | * @package tool_objectfs 20 | * @author Gleimer Mora 21 | * @copyright Catalyst IT 22 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 23 | */ 24 | 25 | namespace tool_objectfs\local\object_manipulator\candidates; 26 | 27 | /** 28 | * pusher_candidates 29 | */ 30 | class pusher_candidates extends manipulator_candidates_base { 31 | 32 | /** 33 | * queryname 34 | * @var string 35 | */ 36 | protected $queryname = 'get_push_candidates'; 37 | 38 | /** 39 | * get_candidates_sql 40 | * @return string 41 | */ 42 | public function get_candidates_sql() { 43 | return 'SELECT contenthash, 44 | filesize 45 | FROM {tool_objectfs_objects} 46 | WHERE filesize > :threshold 47 | AND filesize < :maximum_file_size 48 | AND timeduplicated <= :maxcreatedtimestamp 49 | AND location = :object_location'; 50 | } 51 | 52 | /** 53 | * get_candidates_sql_params 54 | * @return array 55 | */ 56 | public function get_candidates_sql_params() { 57 | $filesystem = new $this->config->filesystem; 58 | return [ 59 | 'maxcreatedtimestamp' => time() - $this->config->minimumage, 60 | 'threshold' => $this->config->sizethreshold, 61 | 'maximum_file_size' => $filesystem->get_maximum_upload_filesize(), 62 | 'object_location' => OBJECT_LOCATION_LOCAL, 63 | ]; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/candidates/recoverer_candidates.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Class recoverer_candidates 19 | * @package tool_objectfs 20 | * @author Gleimer Mora 21 | * @copyright Catalyst IT 22 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 23 | */ 24 | 25 | namespace tool_objectfs\local\object_manipulator\candidates; 26 | 27 | /** 28 | * recoverer_candidates 29 | */ 30 | class recoverer_candidates extends manipulator_candidates_base { 31 | 32 | /** 33 | * queryname 34 | * @var string 35 | */ 36 | protected $queryname = 'get_recover_candidates'; 37 | 38 | /** 39 | * get_candidates_sql 40 | * @return string 41 | */ 42 | public function get_candidates_sql() { 43 | return 'SELECT contenthash, 44 | filesize 45 | FROM {tool_objectfs_objects} 46 | WHERE location = :location'; 47 | } 48 | 49 | /** 50 | * get_candidates_sql_params 51 | * @return array 52 | */ 53 | public function get_candidates_sql_params() { 54 | return ['location' => OBJECT_LOCATION_ERROR]; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/checker.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Checks files to have their location stored in database. 19 | * 20 | * @package tool_objectfs 21 | * @author Mikhail Golenkov 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\local\object_manipulator; 27 | 28 | use stdClass; 29 | use tool_objectfs\local\store\object_file_system; 30 | use tool_objectfs\log\aggregate_logger; 31 | 32 | /** 33 | * checker 34 | */ 35 | class checker extends manipulator { 36 | 37 | /** 38 | * Checker constructor. 39 | * This manipulator adds location for files that do not have records in {tool_objectfs_objects} table. 40 | * 41 | * @param object_file_system $filesystem objectfs file system 42 | * @param stdClass $config objectfs config. 43 | * @param aggregate_logger $logger 44 | */ 45 | public function __construct(object_file_system $filesystem, stdClass $config, aggregate_logger $logger) { 46 | parent::__construct($filesystem, $config, $logger); 47 | $this->batchsize = $this->batchsize * 10; 48 | } 49 | 50 | /** 51 | * manipulate_object 52 | * @param stdClass $objectrecord 53 | * @return int 54 | */ 55 | public function manipulate_object(stdClass $objectrecord) { 56 | return $this->filesystem->get_object_location_from_hash($objectrecord->contenthash); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/deleter.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Deletes files that are old enough and are in S3. 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\local\object_manipulator; 27 | 28 | use stdClass; 29 | use tool_objectfs\local\store\object_file_system; 30 | use tool_objectfs\log\aggregate_logger; 31 | 32 | /** 33 | * deleter 34 | */ 35 | class deleter extends manipulator { 36 | 37 | /** 38 | * How long file must exist after 39 | * duplication before it can be deleted. 40 | * 41 | * @var int 42 | */ 43 | private $consistencydelay; 44 | 45 | /** 46 | * Whether to delete local files 47 | * once they are in remote. 48 | * 49 | * @var bool 50 | */ 51 | private $deletelocal; 52 | 53 | /** 54 | * deleter constructor. 55 | * @param object_file_system $filesystem 56 | * @param stdClass $config 57 | * @param aggregate_logger $logger 58 | */ 59 | public function __construct(object_file_system $filesystem, stdClass $config, aggregate_logger $logger) { 60 | parent::__construct($filesystem, $config, $logger); 61 | $this->consistencydelay = $config->consistencydelay; 62 | $this->deletelocal = $config->deletelocal; 63 | $this->sizethreshold = $config->sizethreshold; 64 | } 65 | 66 | /** 67 | * manipulate_object 68 | * @param stdClass $objectrecord 69 | * @return int 70 | */ 71 | public function manipulate_object(stdClass $objectrecord) { 72 | $newlocation = $this->filesystem->delete_object_from_local_by_hash($objectrecord->contenthash, $objectrecord->filesize); 73 | return $newlocation; 74 | } 75 | 76 | /** 77 | * manipulator_can_execute 78 | * @return bool 79 | */ 80 | protected function manipulator_can_execute() { 81 | if ($this->deletelocal == 0) { 82 | mtrace("Delete local disabled \n"); 83 | return false; 84 | } 85 | 86 | return true; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/logger.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** Logs manipulator actions 18 | * 19 | * @package tool_objectfs 20 | * @author Kenneth Hendricks 21 | * @copyright Catalyst IT 22 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 23 | */ 24 | 25 | namespace tool_objectfs\local\object_manipulator; 26 | 27 | defined('MOODLE_INTERNAL') || die(); 28 | 29 | require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); 30 | 31 | /** 32 | * logger 33 | */ 34 | class logger { 35 | 36 | /** 37 | * @var [type] 38 | */ 39 | private $action; // Which action to log. 40 | /** 41 | * @var int 42 | */ 43 | private $timestart; 44 | /** 45 | * @var int 46 | */ 47 | private $timeend; 48 | /** 49 | * @var int 50 | */ 51 | private $totalfilesize; 52 | /** 53 | * @var int 54 | */ 55 | private $totalfilecount; 56 | 57 | /** 58 | * construct 59 | */ 60 | public function __construct() { 61 | $this->totalfilecount = 0; 62 | $this->totalfilesize = 0; 63 | } 64 | 65 | /** 66 | * start_timing 67 | * @return void 68 | */ 69 | public function start_timing() { 70 | $this->timestart = time(); 71 | } 72 | 73 | /** 74 | * end_timing 75 | * @return void 76 | */ 77 | public function end_timing() { 78 | $this->timeend = time(); 79 | } 80 | 81 | /** 82 | * set_action 83 | * @param mixed $action 84 | * 85 | * @return void 86 | */ 87 | public function set_action($action) { 88 | $this->action = $action; 89 | } 90 | 91 | /** 92 | * add_object_manipulation 93 | * @param int $filesize 94 | * 95 | * @return void 96 | */ 97 | public function add_object_manipulation($filesize) { 98 | $this->totalfilesize += $filesize; 99 | $this->totalfilecount++; 100 | } 101 | 102 | /** 103 | * log_object_manipulation 104 | * @return void 105 | */ 106 | public function log_object_manipulation() { 107 | $duration = $this->timestart - $this->timeend; 108 | $totalfilesize = display_size($this->totalfilesize); 109 | $logstring = "Objectsfs $this->action manipulator took $duration seconds "; 110 | $logstring .= "to $this->action $this->totalfilecount objects. "; 111 | $logstring .= "Total size: $totalfilesize Total time: $duration seconds"; 112 | mtrace($logstring); 113 | } 114 | 115 | /** 116 | * log_object_manipulation_query 117 | * @param mixed $totalobjectsfound 118 | * 119 | * @return void 120 | */ 121 | public function log_object_manipulation_query($totalobjectsfound) { 122 | $duration = $this->timeend - $this->timestart; 123 | $logstring = "Objectsfs $this->action manipulator took $duration seconds "; 124 | $logstring .= "to find $totalobjectsfound potential $this->action objects. "; 125 | $logstring .= "Total time: $duration seconds"; 126 | mtrace($logstring); 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/manipulator.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * File manipulator abstract class. 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\local\object_manipulator; 27 | 28 | use dml_exception; 29 | use stdClass; 30 | use tool_objectfs\local\manager; 31 | use tool_objectfs\local\store\object_file_system; 32 | use tool_objectfs\log\aggregate_logger; 33 | 34 | defined('MOODLE_INTERNAL') || die(); 35 | 36 | require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); 37 | 38 | /** 39 | * manipulator 40 | */ 41 | abstract class manipulator implements object_manipulator { 42 | 43 | /** 44 | * object file system 45 | * 46 | * @var object_file_system 47 | */ 48 | protected $filesystem; 49 | 50 | /** 51 | * What time the file manipulator should finish execution by. 52 | * 53 | * @var int 54 | */ 55 | protected $finishtime; 56 | 57 | /** @var aggregate_logger $logger */ 58 | protected $logger; 59 | 60 | /** @var int $batchsize */ 61 | protected $batchsize; 62 | 63 | /** 64 | * Size threshold for pulling files from remote in bytes. 65 | * 66 | * @var int 67 | */ 68 | protected $sizethreshold; 69 | 70 | /** 71 | * Manipulator constructor 72 | * 73 | * @param object_file_system $filesystem object file system 74 | * @param stdClass $config 75 | * @param aggregate_logger $logger 76 | */ 77 | public function __construct(object_file_system $filesystem, stdClass $config, aggregate_logger $logger) { 78 | $this->finishtime = time() + $config->maxtaskruntime; 79 | $this->filesystem = $filesystem; 80 | $this->batchsize = $config->batchsize; 81 | $this->sizethreshold = $config->sizethreshold; 82 | $this->logger = $logger; 83 | // Inject our logger into the filesystem. 84 | $this->filesystem->set_logger($this->logger); 85 | } 86 | 87 | /** 88 | * execute 89 | * @param array $objectrecords 90 | * @return mixed|void 91 | * @throws dml_exception 92 | */ 93 | public function execute(array $objectrecords) { 94 | if (!$this->manipulator_can_execute()) { 95 | mtrace('Objectfs manipulator exiting early'); 96 | return; 97 | } 98 | $this->logger->start_timing(); 99 | 100 | foreach ($objectrecords as $objectrecord) { 101 | if (time() >= $this->finishtime) { 102 | break; 103 | } 104 | 105 | $objectlock = $this->filesystem->acquire_object_lock($objectrecord->contenthash); 106 | 107 | // Object is currently being manipulated elsewhere. 108 | if (!$objectlock) { 109 | continue; 110 | } 111 | 112 | $newlocation = $this->manipulate_object($objectrecord); 113 | if (!empty($objectrecord->id)) { 114 | manager::upsert_object($objectrecord, $newlocation); 115 | } else { 116 | manager::update_object_by_hash($objectrecord->contenthash, $newlocation); 117 | } 118 | $objectlock->release(); 119 | } 120 | 121 | $this->logger->end_timing(); 122 | $this->logger->output_move_statistics(); 123 | } 124 | 125 | /** 126 | * Given an object record, the class implementing this will be able to manipulate 127 | * the object, and return the new location of the object. 128 | * @see examples in lib.php (OBJECT_LOCATION_*) 129 | * 130 | * @param stdClass $objectrecord 131 | * @return int OBJECT_LOCATION_* 132 | */ 133 | abstract public function manipulate_object(stdClass $objectrecord); 134 | 135 | /** 136 | * Returns whether or not the particular manipulator will manipulate the 137 | * object when execute is called. 138 | * 139 | * @return bool 140 | */ 141 | protected function manipulator_can_execute() { 142 | return true; 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/manipulator_builder.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * File manipulator abstract class. 19 | * 20 | * @package tool_objectfs 21 | * @author Gleimer Mora 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\local\object_manipulator; 27 | 28 | use coding_exception; 29 | use moodle_exception; 30 | use stdClass; 31 | use tool_objectfs\local\manager; 32 | use tool_objectfs\local\object_manipulator\candidates\candidates_finder; 33 | use tool_objectfs\log\aggregate_logger; 34 | 35 | defined('MOODLE_INTERNAL') || die(); 36 | 37 | require_once(__DIR__ . '/../../../lib.php'); 38 | 39 | /** 40 | * manipulator_builder 41 | */ 42 | class manipulator_builder { 43 | 44 | /** @var array $manipulators */ 45 | private $manipulators = [ 46 | deleter::class, 47 | puller::class, 48 | pusher::class, 49 | recoverer::class, 50 | checker::class, 51 | orphaner::class, 52 | ]; 53 | 54 | /** @var string $manipulatorclass */ 55 | private $manipulatorclass; 56 | 57 | /** @var candidates_finder $finder */ 58 | private $finder; 59 | 60 | /** @var stdClass $config */ 61 | private $config; 62 | 63 | /** @var aggregate_logger $logger */ 64 | private $logger; 65 | 66 | /** @var array $candidates */ 67 | private $candidates = []; 68 | 69 | /** 70 | * execute 71 | * @param string $manipulator 72 | * @throws coding_exception 73 | * @throws moodle_exception 74 | */ 75 | public function execute($manipulator) { 76 | $this->build($manipulator); 77 | if (empty($this->candidates)) { 78 | return; 79 | } 80 | $filesystem = new $this->config->filesystem(); 81 | if (!$filesystem->get_client_availability()) { 82 | mtrace(get_string('client_not_available', 'tool_objectfs')); 83 | return; 84 | } 85 | $manipulator = new $this->manipulatorclass($filesystem, $this->config, $this->logger); 86 | $manipulator->execute($this->candidates); 87 | } 88 | 89 | /** 90 | * execute_all 91 | * @throws coding_exception 92 | * @throws moodle_exception 93 | */ 94 | public function execute_all() { 95 | foreach ($this->manipulators as $manipulator) { 96 | mtrace("Executing objectfs $manipulator"); 97 | $this->execute($manipulator); 98 | mtrace("Objectfs $manipulator successfully executed"); 99 | } 100 | } 101 | 102 | /** 103 | * build 104 | * @param string $manipulator 105 | * @throws moodle_exception 106 | */ 107 | private function build($manipulator) { 108 | $this->config = manager::get_objectfs_config(); 109 | $this->manipulatorclass = $manipulator; 110 | $this->logger = new aggregate_logger(); 111 | $this->finder = new candidates_finder($manipulator, $this->config); 112 | $this->candidates = $this->finder->get(); 113 | $countcandidates = count($this->candidates); 114 | $this->logger->log_object_query($this->finder->get_query_name(), $countcandidates); 115 | if ($countcandidates === 0) { 116 | mtrace('No candidate objects found.'); 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/object_manipulator.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\local\object_manipulator; 18 | 19 | use stdClass; 20 | 21 | /** 22 | * Object manipulator interface class. 23 | * 24 | * @package tool_objectfs 25 | * @author Gleimer Mora 26 | * @copyright Catalyst IT 27 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 28 | */ 29 | interface object_manipulator { 30 | 31 | 32 | /** 33 | * execute 34 | * @param array $objects 35 | * @return mixed 36 | */ 37 | public function execute(array $objects); 38 | 39 | /** 40 | * manipulate_object 41 | * @param stdClass $objectrecord 42 | * @return int 43 | */ 44 | public function manipulate_object(stdClass $objectrecord); 45 | } 46 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/orphaner.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Orphans records for files deleted 19 | * 20 | * Orphans {tool_objectfs_objects} records for files that have been 21 | * deleted from the core {files} table. 22 | * 23 | * @package tool_objectfs 24 | * @author Nathan Mares 25 | * @author Kevin Pham 26 | * @copyright Catalyst IT 27 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 28 | */ 29 | 30 | namespace tool_objectfs\local\object_manipulator; 31 | 32 | use stdClass; 33 | 34 | /** 35 | * orphaner 36 | */ 37 | class orphaner extends manipulator { 38 | 39 | /** 40 | * Updates the location of {tool_objectfs_objects} records for files that 41 | * have been deleted from the core {files} table. 42 | * 43 | * @param \stdClass $objectrecord 44 | * @return int 45 | */ 46 | public function manipulate_object(stdClass $objectrecord): int { 47 | return OBJECT_LOCATION_ORPHANED; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/puller.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Pulls files from remote storage if they meet the configured criterea. 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\local\object_manipulator; 27 | 28 | use stdClass; 29 | 30 | /** 31 | * puller 32 | */ 33 | class puller extends manipulator { 34 | 35 | /** 36 | * manipulate_object 37 | * @param stdClass $objectrecord 38 | * @return int 39 | */ 40 | public function manipulate_object(stdClass $objectrecord) { 41 | $contenthash = $objectrecord->contenthash; 42 | $filesize = $objectrecord->filesize; 43 | return $this->filesystem->copy_object_from_external_to_local_by_hash($contenthash, $filesize); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/pusher.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Pushes files to remote storage if they meet the configured criterea. 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\local\object_manipulator; 27 | 28 | use stdClass; 29 | use tool_objectfs\local\store\object_file_system; 30 | use tool_objectfs\log\aggregate_logger; 31 | 32 | /** 33 | * pusher 34 | */ 35 | class pusher extends manipulator { 36 | 37 | /** 38 | * Minimum age of a file to be pushed to remote in seconds. 39 | * 40 | * @var int 41 | */ 42 | private $minimumage; 43 | 44 | /** 45 | * The maximum upload file size in bytes. 46 | * 47 | * @var int 48 | */ 49 | private $maximumfilesize; 50 | 51 | /** 52 | * pusher constructor. 53 | * @param object_file_system $filesystem 54 | * @param stdClass $config 55 | * @param aggregate_logger $logger 56 | */ 57 | public function __construct(object_file_system $filesystem, stdClass $config, aggregate_logger $logger) { 58 | parent::__construct($filesystem, $config, $logger); 59 | $this->sizethreshold = $config->sizethreshold; 60 | $this->minimumage = $config->minimumage; 61 | $this->maximumfilesize = $this->filesystem->get_maximum_upload_filesize(); 62 | } 63 | 64 | /** 65 | * manipulate_object 66 | * @param stdClass $objectrecord 67 | * @return int 68 | */ 69 | public function manipulate_object(stdClass $objectrecord) { 70 | $contenthash = $objectrecord->contenthash; 71 | $filesize = $objectrecord->filesize; 72 | return $this->filesystem->copy_object_from_local_to_external_by_hash($contenthash, $filesize); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /classes/local/object_manipulator/recoverer.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Recovers objects that are in the error state if it can. 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\local\object_manipulator; 27 | 28 | use stdClass; 29 | 30 | /** 31 | * recoverer 32 | */ 33 | class recoverer extends manipulator { 34 | 35 | /** 36 | * manipulate_object 37 | * @param stdClass $objectrecord 38 | * @return int 39 | */ 40 | public function manipulate_object(stdClass $objectrecord) { 41 | return $this->filesystem->get_object_location_from_hash($objectrecord->contenthash); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /classes/local/report/log_size_report_builder.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Log size report 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\local\report; 27 | 28 | /** 29 | * log_size_report_builder 30 | */ 31 | class log_size_report_builder extends objectfs_report_builder { 32 | 33 | /** 34 | * build_report 35 | * @param int $reportid 36 | * 37 | * @return objectfs_report 38 | */ 39 | public function build_report($reportid) { 40 | global $DB; 41 | 42 | $report = new objectfs_report('log_size', $reportid); 43 | 44 | $sql = 'SELECT floor (log(2, filesize)) as datakey, 45 | sum(filesize) as objectsum, 46 | count(*) as objectcount 47 | FROM (SELECT DISTINCT contenthash, filesize 48 | FROM {files} 49 | WHERE filesize > 0) d 50 | GROUP BY datakey ORDER BY datakey'; 51 | 52 | $stats = $DB->get_records_sql($sql); 53 | 54 | $this->compress_small_log_sizes($stats); 55 | 56 | $report->add_rows($stats); 57 | 58 | return $report; 59 | } 60 | 61 | /** 62 | * compress_small_log_sizes 63 | * @param mixed $stats 64 | * 65 | * @return void 66 | */ 67 | public function compress_small_log_sizes(&$stats) { 68 | $smallstats = new \stdClass(); 69 | $smallstats->datakey = 1; 70 | $smallstats->objectsum = 0; 71 | $smallstats->objectcount = 0; 72 | 73 | foreach ($stats as $key => $stat) { 74 | 75 | // Logsize of <= 9 means that files are smaller than 1 KB. 76 | if ($stat->datakey <= 9) { 77 | $smallstats->objectcount += $stat->objectcount; 78 | $smallstats->objectsum += $stat->objectsum; 79 | unset($stats[$key]); 80 | } 81 | 82 | } 83 | // Add to the beginning of the array. 84 | array_unshift($stats, $smallstats); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /classes/local/report/mime_type_report_builder.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Mime type report 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\local\report; 27 | 28 | /** 29 | * mime_type_report_builder 30 | */ 31 | class mime_type_report_builder extends objectfs_report_builder { 32 | 33 | /** 34 | * build_report 35 | * @param int $reportid 36 | * 37 | * @return objectfs_report 38 | */ 39 | public function build_report($reportid) { 40 | global $DB; 41 | 42 | $report = new objectfs_report('mime_type', $reportid); 43 | 44 | $sql = "SELECT sum(filesize) as objectsum, filetype as datakey, count(*) as objectcount 45 | FROM (SELECT distinct filesize, 46 | CASE 47 | WHEN mimetype = 'application/pdf' THEN 'pdf' 48 | WHEN mimetype = 'application/epub+zip' THEN 'epub' 49 | WHEN mimetype = 'application/vnd.moodle.backup' THEN 'moodlebackup' 50 | WHEN mimetype = 'application/msword' THEN 'document' 51 | WHEN mimetype = 'application/x-mspublisher' THEN 'document' 52 | WHEN mimetype like 'application/vnd.ms-word%' THEN 'document' 53 | WHEN mimetype like 'application/vnd.oasis.opendocument.text%' THEN 'document' 54 | WHEN mimetype like 'application/vnd.openxmlformats-officedocument%' THEN 'document' 55 | WHEN mimetype like 'application/vnd.ms-powerpoint%' THEN 'document' 56 | WHEN mimetype = 'application/vnd.oasis.opendocument.presentation' THEN 'document' 57 | WHEN mimetype = 'application/vnd.oasis.opendocument.spreadsheet' THEN 'spreadsheet' 58 | WHEN mimetype like 'application/vnd.ms-excel%' THEN 'spreadsheet' 59 | WHEN mimetype = 'application/g-zip' THEN 'archive' 60 | WHEN mimetype = 'application/x-7z-compressed' THEN 'archive' 61 | WHEN mimetype = 'application/x-rar-compressed' THEN 'archive' 62 | WHEN mimetype like 'application/%' THEN 'other' 63 | ELSE substr(mimetype,0,position('/' IN mimetype)) 64 | END AS filetype 65 | FROM {files} 66 | WHERE mimetype IS NOT NULL) stats 67 | GROUP BY datakey 68 | ORDER BY 69 | sum(filesize) / 1024, datakey"; 70 | 71 | $result = $DB->get_records_sql($sql); 72 | 73 | $report->add_rows($result); 74 | 75 | return $report; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /classes/local/report/objectfs_report_builder.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * sss report abstract class. 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\local\report; 27 | 28 | /** 29 | * objectfs_report_builder 30 | */ 31 | abstract class objectfs_report_builder { 32 | 33 | /** 34 | * build_report 35 | * @param int $reportid 36 | * 37 | * @return objectfs_report 38 | */ 39 | abstract public function build_report($reportid); 40 | } 41 | -------------------------------------------------------------------------------- /classes/local/report/tag_count_report_builder.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\local\report; 18 | 19 | /** 20 | * Tag count report builder. 21 | * 22 | * @package tool_objectfs 23 | * @author Matthew Hilton 24 | * @copyright Catalyst IT 25 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 26 | */ 27 | class tag_count_report_builder extends objectfs_report_builder { 28 | /** 29 | * Builds report 30 | * @param int $reportid 31 | * @return objectfs_report 32 | */ 33 | public function build_report($reportid) { 34 | global $DB; 35 | $report = new objectfs_report('tag_count', $reportid); 36 | 37 | // Returns counts of key:value. 38 | $sql = " 39 | SELECT CONCAT(COALESCE(object_tags.tagkey, '(untagged)'), ': ', COALESCE(object_tags.tagvalue, '')) as datakey, 40 | COUNT(DISTINCT object_tags.objectid) as objectcount 41 | FROM {tool_objectfs_object_tags} object_tags 42 | GROUP BY object_tags.tagkey, object_tags.tagvalue 43 | "; 44 | $result = $DB->get_records_sql($sql); 45 | $report->add_rows($result); 46 | return $report; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /classes/local/store/azure/file_system.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * File system for Azure Blob Storage. 19 | * 20 | * @package tool_objectfs 21 | * @author Nicholas Hoobin 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\local\store\azure; 27 | 28 | defined('MOODLE_INTERNAL') || die(); 29 | 30 | use tool_objectfs\local\store\object_file_system; 31 | 32 | require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); 33 | 34 | /** 35 | * file_system 36 | * @deprecated Since Moodle 4.2 - Please see the README about updating to new azure_blob_storage client. 37 | */ 38 | class file_system extends object_file_system { 39 | 40 | /** 41 | * initialise_external_client 42 | * @param mixed $config 43 | * 44 | * @return client 45 | */ 46 | protected function initialise_external_client($config) { 47 | $asclient = new client($config); 48 | return $asclient; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /classes/local/store/azure_blob_storage/file_system.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\local\store\azure_blob_storage; 18 | 19 | use tool_objectfs\local\store\azure_blob_storage\client; 20 | use tool_objectfs\local\store\object_file_system; 21 | 22 | /** 23 | * Azure blob store file system 24 | * 25 | * @package tool_objectfs 26 | * @author Matthew Hilton 27 | * @copyright 2024 Catalyst IT 28 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 29 | */ 30 | class file_system extends object_file_system { 31 | /** 32 | * Initialise client 33 | * @param mixed $config 34 | * @return client 35 | */ 36 | protected function initialise_external_client($config) { 37 | return new client($config); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /classes/local/store/digitalocean/client.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * DigitalOcean Spaces client. 19 | * 20 | * @package tool_objectfs 21 | * @author Brian Yanosik 22 | * @copyright Brian Yanosik 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\local\store\digitalocean; 27 | 28 | use tool_objectfs\local\store\s3\client as s3_client; 29 | 30 | /** 31 | * client 32 | */ 33 | class client extends s3_client { 34 | 35 | /** 36 | * construct 37 | * @param \stdClass $config 38 | * @return void 39 | */ 40 | public function __construct($config) { 41 | if ($this->get_availability() && !empty($config)) { 42 | $this->bucket = $config->do_space; 43 | $this->set_client($config); 44 | } else { 45 | parent::__construct($config); 46 | } 47 | } 48 | 49 | /** 50 | * We do not need to check for the autoloader as AWS SDK is integrated in to Moodle 4.4 51 | * 52 | * @return bool 53 | */ 54 | public function get_availability() { 55 | return true; 56 | } 57 | 58 | /** 59 | * Check if the client configured properly. 60 | * 61 | * @param \stdClass $config Client config. 62 | * @return bool 63 | */ 64 | protected function is_configured($config) { 65 | if (empty($config->do_key) || empty($config->do_secret) || empty($config->do_region)) { 66 | return false; 67 | } 68 | 69 | return true; 70 | } 71 | 72 | /** 73 | * set_client 74 | * @param \stdClass $config 75 | * 76 | * @return void 77 | */ 78 | public function set_client($config) { 79 | if (!$this->is_configured($config)) { 80 | $this->client = null; 81 | return; 82 | } 83 | 84 | $this->client = \Aws\S3\S3Client::factory([ 85 | 'credentials' => ['key' => $config->do_key, 'secret' => $config->do_secret], 86 | 'region' => $config->do_region, 87 | 'endpoint' => 'https://' . $config->do_region . '.digitaloceanspaces.com', 88 | 'version' => AWS_API_VERSION, 89 | ]); 90 | } 91 | 92 | /** 93 | * define_client_section 94 | * @param admin_settingpage $settings 95 | * @param \stdClass $config 96 | * @return admin_settingpage 97 | */ 98 | public function define_client_section($settings, $config) { 99 | 100 | $regionoptions = [ 101 | 'sfo2' => 'sfo2 (San Fransisco)', 102 | 'nyc3' => 'nyc3 (New York City)', 103 | 'ams3' => 'ams3 (Amsterdam)', 104 | 'sgp1' => 'spg1 (Singapore)', 105 | 'fra1' => 'fra1 (Frankfurt)', 106 | ]; 107 | 108 | $settings->add(new \admin_setting_heading('tool_objectfs/do', 109 | new \lang_string('settings:do:header', 'tool_objectfs'), '')); 110 | 111 | $settings->add(new \admin_setting_configtext('tool_objectfs/do_key', 112 | new \lang_string('settings:do:key', 'tool_objectfs'), 113 | new \lang_string('settings:do:key_help', 'tool_objectfs'), '')); 114 | 115 | $settings->add(new \admin_setting_configpasswordunmask('tool_objectfs/do_secret', 116 | new \lang_string('settings:do:secret', 'tool_objectfs'), 117 | new \lang_string('settings:do:secret_help', 'tool_objectfs'), '')); 118 | 119 | $settings->add(new \admin_setting_configtext('tool_objectfs/do_space', 120 | new \lang_string('settings:do:space', 'tool_objectfs'), 121 | new \lang_string('settings:do:space_help', 'tool_objectfs'), '')); 122 | 123 | $settings->add(new \admin_setting_configselect('tool_objectfs/do_region', 124 | new \lang_string('settings:do:region', 'tool_objectfs'), 125 | new \lang_string('settings:do:region_help', 'tool_objectfs'), '', $regionoptions)); 126 | 127 | return $settings; 128 | } 129 | 130 | } 131 | -------------------------------------------------------------------------------- /classes/local/store/digitalocean/file_system.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * object_file_system abstract class. 19 | * 20 | * Remote object storage providers extent this class. 21 | * At minimum you need to implement get_remote_client. 22 | * 23 | * @package tool_objectfs 24 | * @author Brian Yanosik 25 | * @copyright Brian Yanosik 26 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 27 | */ 28 | 29 | namespace tool_objectfs\local\store\digitalocean; 30 | 31 | defined('MOODLE_INTERNAL') || die(); 32 | 33 | use tool_objectfs\local\store\s3\file_system as s3_file_system; 34 | 35 | require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); 36 | 37 | /** 38 | * [Description file_system] 39 | */ 40 | class file_system extends s3_file_system { 41 | 42 | /** 43 | * initialise_external_client 44 | * @param \stdClass $config 45 | * 46 | * @return client 47 | */ 48 | protected function initialise_external_client($config) { 49 | $doclient = new client($config); 50 | 51 | return $doclient; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /classes/local/store/s3/admin_settings_aws_region.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Admin setting for AWS regions. 19 | * 20 | * @package tool_objectfs 21 | * @author Dmitrii Metelkin 22 | * @copyright 2020 Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\local\store\s3; 27 | 28 | defined('MOODLE_INTERNAL') || die(); 29 | 30 | require_once($CFG->dirroot . '/lib/adminlib.php'); 31 | 32 | /** 33 | * Admin setting for a list of AWS regions. 34 | * 35 | * @package tool_objectfs 36 | * @copyright 2020 Catalyst IT 37 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 38 | */ 39 | class admin_settings_aws_region extends \admin_setting_configtext { 40 | 41 | /** 42 | * Return part of form with setting. 43 | * 44 | * @param mixed $data array or string depending on setting 45 | * @param string $query 46 | * @return string 47 | */ 48 | public function output_html($data, $query='') { 49 | global $CFG; 50 | 51 | $default = $this->get_defaultsetting(); 52 | 53 | $options = []; 54 | 55 | $all = require($CFG->dirroot . '/lib/aws-sdk/src/data/endpoints.json.php'); 56 | $ends = $all['partitions'][0]['regions']; 57 | if ($ends) { 58 | foreach ($ends as $key => $value) { 59 | $options[] = [ 60 | 'value' => $key, 61 | 'label' => $key . ' - ' . $value['description'], 62 | ]; 63 | } 64 | } 65 | 66 | $inputparams = array( 67 | 'type' => 'text', 68 | 'list' => $this->get_full_name(), 69 | 'name' => $this->get_full_name(), 70 | 'value' => $data, 71 | 'size' => $this->size, 72 | 'id' => $this->get_id(), 73 | 'class' => 'form-control text-ltr', 74 | ); 75 | 76 | $element = \html_writer::start_tag('div', array('class' => 'form-text defaultsnext')); 77 | $element .= \html_writer::empty_tag('input', $inputparams); 78 | $element .= \html_writer::start_tag('datalist', array('id' => $this->get_full_name())); 79 | foreach ($options as $option) { 80 | $element .= \html_writer::tag('option', $option['label'], array('value' => $option['value'])); 81 | } 82 | $element .= \html_writer::end_tag('datalist'); 83 | $element .= \html_writer::end_tag('div'); 84 | 85 | return format_admin_setting($this, $this->visiblename, $element, $this->description, true, '', $default, $query); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /classes/local/store/s3/file_system.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * object_file_system abstract class. 19 | * 20 | * Remote object storage providers extent this class. 21 | * At minimum you need to implement get_remote_client. 22 | * 23 | * @package tool_objectfs 24 | * @author Kenneth Hendricks 25 | * @copyright Catalyst IT 26 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 27 | */ 28 | 29 | namespace tool_objectfs\local\store\s3; 30 | 31 | defined('MOODLE_INTERNAL') || die(); 32 | 33 | use tool_objectfs\local\manager; 34 | use tool_objectfs\local\store\object_file_system; 35 | 36 | require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); 37 | 38 | /** 39 | * [Description file_system] 40 | */ 41 | class file_system extends object_file_system { 42 | 43 | /** 44 | * initialise_external_client 45 | * @param \stdClass $config 46 | * 47 | * @return client 48 | */ 49 | protected function initialise_external_client($config) { 50 | $s3client = new client($config); 51 | 52 | return $s3client; 53 | } 54 | 55 | /** 56 | * readfile 57 | * @param \stored_file $file 58 | * @return void 59 | * @throws \file_exception 60 | */ 61 | public function readfile(\stored_file $file) { 62 | $path = $this->get_remote_path_from_storedfile($file); 63 | 64 | $this->get_logger()->start_timing(); 65 | if ($path == $this->get_external_client()->get_fullpath_from_hash($file->get_contenthash())) { 66 | // There is an issue using core readfile_allow_large() for the big (more than 1G) files from s3. 67 | $success = readfile($path); 68 | } else { 69 | $success = readfile_allow_large($path, $file->get_filesize()); 70 | } 71 | $this->get_logger()->end_timing(); 72 | $this->get_logger()->log_object_read('readfile', $path, $file->get_filesize()); 73 | 74 | if ($success === false) { 75 | manager::update_object_by_hash($file->get_contenthash(), OBJECT_LOCATION_ERROR); 76 | throw new \file_exception('storedfilecannotreadfile', $file->get_filename()); 77 | } 78 | } 79 | 80 | /** 81 | * copy_from_local_to_external 82 | * @param mixed $contenthash 83 | * 84 | * @return bool 85 | */ 86 | public function copy_from_local_to_external($contenthash) { 87 | $localpath = $this->get_local_path_from_hash($contenthash); 88 | $mime = $this->get_mimetype_from_hash($contenthash); 89 | 90 | try { 91 | $this->get_external_client()->upload_to_s3($localpath, $contenthash, $mime); 92 | return true; 93 | } catch (\Exception $e) { 94 | $this->get_logger()->error_log( 95 | 'ERROR: copy ' . $localpath . ' to ' . $this->get_external_path_from_hash($contenthash) . ': ' . $e->getMessage() 96 | ); 97 | return false; 98 | } 99 | } 100 | 101 | /** 102 | * supports_presigned_urls 103 | * @return bool 104 | */ 105 | public function supports_presigned_urls() { 106 | return true; 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /classes/local/store/signed_url.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\local\store; 18 | 19 | /** 20 | * A signed URL for direct downloads 21 | * 22 | * A signed URL which can be used by a user to directly download a file from object store, rather 23 | * than from the Moodle server. 24 | * 25 | * Signed URLs are valid for a limited time, indicated by the $expiresat value. 26 | * 27 | * This can be obtained using the object_client::generate_presigned_url function. 28 | * 29 | * @package tool_objectfs 30 | * @copyright 2022 The Open University 31 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 32 | */ 33 | class signed_url { 34 | /** @var \moodle_url URL to redirect to */ 35 | public $url; 36 | 37 | /** @var int Expiry timestamp (Unix epoch) after which this URL will stop working */ 38 | public $expiresat; 39 | 40 | /** 41 | * construct 42 | * @param \moodle_url $url URL to redirect to 43 | * @param int $expiresat Expiry timestamp (Unix epoch) after which this URL will stop working 44 | */ 45 | public function __construct($url, $expiresat) { 46 | $this->url = $url; 47 | $this->expiresat = $expiresat; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /classes/local/store/swift/file_system.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * File system for Openstack Object Storage 19 | * 20 | * @package tool_objectfs 21 | * @author Matt Clarkson 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\local\store\swift; 27 | 28 | defined('MOODLE_INTERNAL') || die(); 29 | 30 | use tool_objectfs\local\store\object_file_system; 31 | 32 | require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); 33 | 34 | /** 35 | * [Description file_system] 36 | */ 37 | class file_system extends object_file_system { 38 | 39 | /** 40 | * initialise_external_client 41 | * @param \stdClass $config 42 | * 43 | * @return client 44 | */ 45 | protected function initialise_external_client($config) { 46 | $client = new client($config); 47 | return $client; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /classes/local/tag/environment_source.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\local\tag; 18 | 19 | use moodle_exception; 20 | 21 | /** 22 | * Provides current environment to file. 23 | * 24 | * @package tool_objectfs 25 | * @author Matthew Hilton 26 | * @copyright Catalyst IT 27 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 28 | */ 29 | class environment_source implements tag_source { 30 | /** 31 | * Identifier used in tagging file. Is the 'key' of the tag. 32 | * @return string 33 | */ 34 | public static function get_identifier(): string { 35 | return 'environment'; 36 | } 37 | 38 | /** 39 | * Description for source displayed in the admin settings. 40 | * @return string 41 | */ 42 | public static function get_description(): string { 43 | return get_string('tagsource:environment', 'tool_objectfs', self::get_env()); 44 | } 45 | 46 | /** 47 | * Returns current env value from $CFG 48 | * @return string|null string if set, else null 49 | */ 50 | private static function get_env(): ?string { 51 | $value = get_config('tool_objectfs', 'taggingenvironment'); 52 | 53 | if (empty($value)) { 54 | return null; 55 | } 56 | 57 | // Must never be greater than 128, unlikely, but we must enforce this. 58 | if (strlen($value) > 128) { 59 | throw new moodle_exception('tagsource:environment:toolong', 'tool_objectfs'); 60 | } 61 | 62 | return $value; 63 | } 64 | 65 | /** 66 | * Returns the tag value for the given file contenthash 67 | * @param string $contenthash 68 | * @return string|null environment value. 69 | */ 70 | public function get_value_for_contenthash(string $contenthash): ?string { 71 | return self::get_env(); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /classes/local/tag/location_source.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\local\tag; 18 | 19 | /** 20 | * Provides location status for a file. 21 | * 22 | * @package tool_objectfs 23 | * @author Matthew Hilton 24 | * @copyright Catalyst IT 25 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 26 | */ 27 | class location_source implements tag_source { 28 | /** 29 | * Identifier used in tagging file. Is the 'key' of the tag. 30 | * @return string 31 | */ 32 | public static function get_identifier(): string { 33 | return 'location'; 34 | } 35 | 36 | /** 37 | * Description for source displayed in the admin settings. 38 | * @return string 39 | */ 40 | public static function get_description(): string { 41 | return get_string('tagsource:location', 'tool_objectfs'); 42 | } 43 | 44 | /** 45 | * Returns the tag value for the given file contenthash 46 | * @param string $contenthash 47 | * @return string|null mime type for file. 48 | */ 49 | public function get_value_for_contenthash(string $contenthash): ?string { 50 | global $DB; 51 | 52 | $isorphaned = $DB->record_exists('tool_objectfs_objects', ['contenthash' => $contenthash, 53 | 'location' => OBJECT_LOCATION_ORPHANED]); 54 | 55 | return $isorphaned ? 'orphan' : 'active'; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /classes/local/tag/tag_source.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\local\tag; 18 | 19 | /** 20 | * Tag source interface 21 | * 22 | * @package tool_objectfs 23 | * @author Matthew Hilton 24 | * @copyright Catalyst IT 25 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 26 | */ 27 | interface tag_source { 28 | /** 29 | * Returns an unchanging identifier for this source. 30 | * Must never change, otherwise it will lose connection with the tags replicated to objects. 31 | * If it ever must change, a migration step must be completed to trigger all objects to recalculate their tags. 32 | * Must not exceed 128 chars. 33 | * @return string 34 | */ 35 | public static function get_identifier(): string; 36 | 37 | /** 38 | * Description for source displayed in the admin settings. 39 | * @return string 40 | */ 41 | public static function get_description(): string; 42 | 43 | /** 44 | * Returns the value of this tag for the file with the given content hash. 45 | * This must be deterministic, and should never exceed 128 chars. 46 | * @param string $contenthash 47 | * @return string 48 | */ 49 | public function get_value_for_contenthash(string $contenthash): ?string; 50 | } 51 | -------------------------------------------------------------------------------- /classes/local/tag_sync_count_result.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\local; 18 | 19 | use core\check\result; 20 | use html_table; 21 | use html_writer; 22 | use tool_objectfs\local\tag\tag_manager; 23 | 24 | /** 25 | * Tagging sync count result 26 | * 27 | * @package tool_objectfs 28 | * @author Matthew Hilton 29 | * @copyright Catalyst IT 30 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 31 | */ 32 | class tag_sync_count_result extends result { 33 | /** 34 | * Returns the details, which is a table that displays the count for each object status possibility. 35 | * On larger sites, this can take several seconds to execute. 36 | * @return string 37 | */ 38 | public function get_details(): string { 39 | $statuses = tag_manager::get_tag_sync_status_summary(); 40 | $table = new html_table(); 41 | $table->head = [ 42 | get_string('table:status', 'tool_objectfs'), 43 | get_string('table:objectcount', 'tool_objectfs'), 44 | ]; 45 | 46 | foreach (tag_manager::SYNC_STATUSES as $status) { 47 | // If no objects have a status, they won't appear in the SQL above. 48 | // In this case, just show zero (so the use knows it exists, but is zero). 49 | $count = isset($statuses[$status]->statuscount) ? $statuses[$status]->statuscount : 0; 50 | $table->data[$status] = [tag_manager::get_sync_status_string($status), $count]; 51 | } 52 | $table = html_writer::table($table); 53 | return $table; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /classes/log/null_logger.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * objectfs null logger class. 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\log; 27 | 28 | defined('MOODLE_INTERNAL') || die(); 29 | 30 | require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); 31 | 32 | /** 33 | * [Description null_logger] 34 | */ 35 | class null_logger extends objectfs_logger { 36 | 37 | /** 38 | * log_object_read 39 | * @param mixed $readname 40 | * @param mixed $objectpath 41 | * @param int $objectsize 42 | * 43 | * @return void 44 | */ 45 | public function log_object_read($readname, $objectpath, $objectsize = 0) { 46 | return; 47 | } 48 | 49 | /** 50 | * log_object_move 51 | * @param mixed $movename 52 | * @param mixed $initallocation 53 | * @param mixed $finallocation 54 | * @param mixed $objecthash 55 | * @param int $objectsize 56 | * 57 | * @return void 58 | */ 59 | public function log_object_move($movename, $initallocation, $finallocation, $objecthash, $objectsize = 0) { 60 | return; 61 | } 62 | 63 | /** 64 | * log_object_query 65 | * @param mixed $queryname 66 | * @param mixed $objectcount 67 | * @param int $objectsum 68 | * 69 | * @return void 70 | */ 71 | public function log_object_query($queryname, $objectcount, $objectsum = 0) { 72 | return; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /classes/log/objectfs_logger.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * objectfs logger abstract class. 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\log; 27 | 28 | defined('MOODLE_INTERNAL') || die(); 29 | 30 | require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); 31 | 32 | /** 33 | * [Description objectfs_logger] 34 | */ 35 | abstract class objectfs_logger { 36 | /** 37 | * @var float 38 | */ 39 | protected $timestart; 40 | /** 41 | * @var float 42 | */ 43 | protected $timeend; 44 | 45 | /** 46 | * construct 47 | */ 48 | public function __construct() { 49 | $this->timestart = 0; 50 | $this->timeend = 0; 51 | } 52 | 53 | /** 54 | * start_timing 55 | * @return float 56 | */ 57 | public function start_timing() { 58 | $this->timestart = microtime(true); 59 | return $this->timestart; 60 | } 61 | 62 | /** 63 | * end_timing 64 | * @return float 65 | */ 66 | public function end_timing() { 67 | $this->timeend = microtime(true); 68 | return $this->timeend; 69 | } 70 | 71 | /** 72 | * get_timing 73 | * @return float 74 | */ 75 | protected function get_timing() { 76 | return $this->timeend - $this->timestart; 77 | } 78 | 79 | /** 80 | * error_log 81 | * @param mixed $error 82 | * 83 | * @return void 84 | */ 85 | public function error_log($error) { 86 | // @codingStandardsIgnoreStart 87 | error_log($error); 88 | // @codingStandardsIgnoreEnd 89 | } 90 | 91 | /** 92 | * log_lock_timing 93 | * @param mixed $lock 94 | * 95 | * @return void 96 | */ 97 | public function log_lock_timing($lock) { 98 | return; 99 | } 100 | 101 | /** 102 | * log_object_read 103 | * @param string $readname 104 | * @param string $objectpath 105 | * @param int $objectsize 106 | * 107 | * @return void 108 | */ 109 | abstract public function log_object_read($readname, $objectpath, $objectsize = 0); 110 | 111 | /** 112 | * log_object_move 113 | * @param mixed $movename 114 | * @param string $initallocation 115 | * @param string $finallocation 116 | * @param string $objecthash 117 | * @param int $objectsize 118 | * 119 | * @return void 120 | */ 121 | abstract public function log_object_move($movename, $initallocation, $finallocation, $objecthash, $objectsize = 0); 122 | 123 | /** 124 | * log_object_query 125 | * @param string $queryname 126 | * @param int $objectcount 127 | * @param int $objectsum 128 | * 129 | * @return void 130 | */ 131 | abstract public function log_object_query($queryname, $objectcount, $objectsum = 0); 132 | } 133 | -------------------------------------------------------------------------------- /classes/log/objectfs_statistic.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * objectfs statistic container class. 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\log; 27 | 28 | defined('MOODLE_INTERNAL') || die(); 29 | 30 | require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); 31 | 32 | /** 33 | * [Description objectfs_statistic] 34 | */ 35 | class objectfs_statistic { 36 | 37 | /** 38 | * @var string 39 | */ 40 | private $key; 41 | 42 | /** 43 | * @var int 44 | */ 45 | private $objectcount; 46 | 47 | /** 48 | * @var int 49 | */ 50 | private $objectsum; 51 | 52 | /** 53 | * construct 54 | * @param string $key 55 | */ 56 | public function __construct($key) { 57 | $this->key = $key; 58 | $this->objectcount = 0; 59 | $this->objectsum = 0; 60 | } 61 | 62 | /** 63 | * get_objectcount 64 | * @return int 65 | */ 66 | public function get_objectcount() { 67 | return $this->objectcount; 68 | } 69 | 70 | /** 71 | * get_objectsum 72 | * @return int 73 | */ 74 | public function get_objectsum() { 75 | return $this->objectsum; 76 | } 77 | 78 | /** 79 | * get_key 80 | * @return string 81 | */ 82 | public function get_key() { 83 | return $this->key; 84 | } 85 | 86 | /** 87 | * add_statistic 88 | * @param objectfs_statistic $statistic 89 | * 90 | * @return void 91 | */ 92 | public function add_statistic(objectfs_statistic $statistic) { 93 | $this->objectcount += $statistic->get_objectcount(); 94 | $this->objectsum += $statistic->get_objectsum(); 95 | } 96 | 97 | /** 98 | * add_object_data 99 | * @param int $objectcount 100 | * @param int $objectsum 101 | * 102 | * @return void 103 | */ 104 | public function add_object_data($objectcount, $objectsum) { 105 | $this->objectcount += $objectcount; 106 | $this->objectsum += $objectsum; 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /classes/privacy/provider.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Privacy provider. 19 | * 20 | * @package tool_objectfs 21 | * @author Ilya Tregubov (ilyatregubov@catalyst-au.net) 22 | * @copyright 2018 Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | namespace tool_objectfs\privacy; 26 | use core_privacy\local\metadata\null_provider; 27 | /** 28 | * Class provider 29 | * @package tool_objectfs 30 | */ 31 | class provider implements null_provider { 32 | /** 33 | * Get the language string identifier with the component's language 34 | * file to explain why this plugin stores no data. 35 | * 36 | * @return string 37 | */ 38 | public static function get_reason(): string { 39 | return 'privacy:metadata'; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /classes/s3_file_system.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * object_file_system abstract class. 19 | * 20 | * Remote object storage providers extent this class. 21 | * At minimum you need to implement get_remote_client. 22 | * 23 | * @package tool_objectfs 24 | * @author Kenneth Hendricks 25 | * @copyright Catalyst IT 26 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 27 | */ 28 | 29 | namespace tool_objectfs; 30 | 31 | use tool_objectfs\local\store\s3\file_system; 32 | 33 | /** 34 | * [Description s3_file_system] 35 | */ 36 | class s3_file_system extends file_system { 37 | 38 | } 39 | -------------------------------------------------------------------------------- /classes/swift_file_system.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * File system for Openstack Object Storage 19 | * 20 | * @package tool_objectfs 21 | * @author Matt Clarkson 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs; 27 | 28 | use tool_objectfs\local\store\swift\file_system; 29 | 30 | /** 31 | * [Description swift_file_system] 32 | */ 33 | class swift_file_system extends file_system { 34 | 35 | } 36 | -------------------------------------------------------------------------------- /classes/task/check_objects_location.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Task that adds missing objects locations. 19 | * 20 | * @package tool_objectfs 21 | * @author Mikhail Golenkov 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\task; 27 | 28 | use tool_objectfs\local\object_manipulator\checker; 29 | 30 | /** 31 | * [Description check_objects_location] 32 | */ 33 | class check_objects_location extends task { 34 | 35 | /** @var string $manipulator */ 36 | protected $manipulator = checker::class; 37 | 38 | /** @var string $stringname */ 39 | protected $stringname = 'check_objects_location_task'; 40 | } 41 | -------------------------------------------------------------------------------- /classes/task/delete_local_empty_directories.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Task that deletes empty dirs from $CFG->filedir. 19 | * 20 | * @package tool_objectfs 21 | * @author Gleimer Mora 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\task; 27 | 28 | use coding_exception; 29 | use tool_objectfs\local\manager; 30 | 31 | /** 32 | * [Description delete_local_empty_directories] 33 | */ 34 | class delete_local_empty_directories extends task { 35 | 36 | /** @var string $stringname */ 37 | protected $stringname = 'delete_local_empty_directories_task'; 38 | 39 | /** 40 | * Execute task 41 | * @throws coding_exception 42 | */ 43 | public function execute() { 44 | if (!$this->enabled_tasks()) { 45 | return; 46 | } 47 | // If config is set to not deletelocal objects, don't run directory clean up either. 48 | $config = manager::get_objectfs_config(); 49 | if (!$config->deletelocal) { 50 | return; 51 | } 52 | $filesystem = new $this->config->filesystem(); 53 | $filesystem->delete_empty_dirs(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /classes/task/delete_local_objects.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Task that pushes files to S3. 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\task; 27 | 28 | use tool_objectfs\local\object_manipulator\deleter; 29 | 30 | 31 | /** 32 | * [Description delete_local_objects] 33 | */ 34 | class delete_local_objects extends task { 35 | 36 | /** @var string $manipulator */ 37 | protected $manipulator = deleter::class; 38 | 39 | /** @var string $stringname */ 40 | protected $stringname = 'delete_local_objects_task'; 41 | } 42 | -------------------------------------------------------------------------------- /classes/task/delete_orphaned_object_metadata.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Task that checks for old orphaned objects 19 | * 20 | * And removes their metadata (record) 21 | * and external file (if delete external enabled) as it is no longer useful/relevant. 22 | * 23 | * @package tool_objectfs 24 | * @author Kevin Pham 25 | * @copyright Catalyst IT 26 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 27 | */ 28 | 29 | namespace tool_objectfs\task; 30 | 31 | defined('MOODLE_INTERNAL') || die(); 32 | 33 | require_once(__DIR__ . '/../../lib.php'); 34 | 35 | /** 36 | * [Description delete_orphaned_object_metadata] 37 | */ 38 | class delete_orphaned_object_metadata extends task { 39 | 40 | /** @var string $stringname */ 41 | protected $stringname = 'delete_orphaned_object_metadata_task'; 42 | 43 | /** 44 | * Execute task 45 | */ 46 | public function execute() { 47 | global $DB; 48 | 49 | $ageforremoval = $this->config->maxorphanedage; 50 | if (empty($ageforremoval)) { 51 | mtrace('Skipping deletion of orphaned object metadata as maxorphanedage is set to an empty value.'); 52 | return; 53 | } 54 | 55 | $params = [ 56 | 'location' => OBJECT_LOCATION_ORPHANED, 57 | 'ageforremoval' => time() - $ageforremoval, 58 | ]; 59 | 60 | if (!empty($this->config->deleteexternal) && $this->config->deleteexternal == TOOL_OBJECTFS_DELETE_EXTERNAL_TRASH) { 61 | // We need to delete the external files as well as the orphaned data. 62 | $filesystem = new $this->config->filesystem(); 63 | 64 | // Join with files table to make extra sure we aren't deleting something that already exists. 65 | $sql = 'SELECT o.* 66 | FROM {tool_objectfs_objects} o 67 | LEFT JOIN {files} f ON o.contenthash = f.contenthash 68 | WHERE f.id is null AND o.location = :location AND timeduplicated < :ageforremoval'; 69 | 70 | $objects = $DB->get_recordset_sql($sql, $params); 71 | $count = 0; 72 | foreach ($objects as $object) { 73 | // Delete the external file. 74 | $filesystem->delete_external_file_from_hash($object->contenthash, true); 75 | // Delete the metadata in the object table. 76 | $DB->delete_records('tool_objectfs_objects', ['id' => $object->id]); 77 | $count++; 78 | } 79 | $objects->close(); 80 | mtrace("Deleted $count orphaned files and their metadata (orphaned tool_objectfs_objects)"); 81 | } else { 82 | // Delete external files is turned off, we only delete the metadata. 83 | $wheresql = 'location = :location and timeduplicated < :ageforremoval'; 84 | $count = $DB->count_records_select('tool_objectfs_objects', $wheresql, $params); 85 | if (!empty($count)) { 86 | mtrace("Deleting $count records with orphaned metadata (orphaned tool_objectfs_objects)"); 87 | $DB->delete_records_select('tool_objectfs_objects', $wheresql, $params); 88 | } 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /classes/task/generate_status_report.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Task that pushes files to S3. 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\task; 27 | 28 | use tool_objectfs\local\report\objectfs_report; 29 | 30 | defined('MOODLE_INTERNAL') || die(); 31 | 32 | require_once(__DIR__ . '/../../lib.php'); 33 | 34 | /** 35 | * [Description generate_status_report] 36 | */ 37 | class generate_status_report extends task { 38 | 39 | /** @var string $stringname */ 40 | protected $stringname = 'generate_status_report_task'; 41 | 42 | /** 43 | * Execute task 44 | * @return void 45 | */ 46 | public function execute() { 47 | objectfs_report::cleanup_reports(); 48 | objectfs_report::generate_status_report(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /classes/task/objectfs_task.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\task; 18 | 19 | /** 20 | * Base abstract class for objectfs tasks. 21 | * 22 | * @package tool_objectfs 23 | * @author Gleimer Mora 24 | * @copyright Catalyst IT 25 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 26 | */ 27 | interface objectfs_task { 28 | 29 | /** 30 | * Get a descriptive name for this task (shown to admins). 31 | * 32 | * @return string 33 | */ 34 | public function get_name(); 35 | 36 | /** 37 | * Do the job. 38 | * Throw exceptions on errors (the job will be retried). 39 | */ 40 | public function execute(); 41 | } 42 | -------------------------------------------------------------------------------- /classes/task/orphan_objects.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Task that orphans {tool_objectfs_object} records for deleted {files} records. 19 | * 20 | * @package tool_objectfs 21 | * @author Nathan Mares 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\task; 27 | 28 | use tool_objectfs\local\object_manipulator\orphaner; 29 | 30 | 31 | /** 32 | * [Description orphan_objects] 33 | */ 34 | class orphan_objects extends task { 35 | 36 | /** @var string $manipulator */ 37 | protected $manipulator = orphaner::class; 38 | 39 | /** @var string $stringname */ 40 | protected $stringname = 'orphan_objects_task'; 41 | } 42 | -------------------------------------------------------------------------------- /classes/task/populate_objects_filesize.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\task; 18 | 19 | use core\task\adhoc_task; 20 | use core\task\manager; 21 | 22 | /** 23 | * Ad-hoc task to update objects table. Used for async upgrade. 24 | * 25 | * @package tool_objectfs 26 | * @author Andrew Madden 27 | * @copyright 2022 Catalyst IT 28 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 29 | */ 30 | class populate_objects_filesize extends adhoc_task { 31 | 32 | /** @var int Max number of updates that task should trigger before scheduling a new task. */ 33 | private const MAX_UPDATES = 100000; 34 | 35 | /** 36 | * Action of task. 37 | */ 38 | public function execute() { 39 | global $DB; 40 | 41 | // Get the custom data. 42 | $data = $this->get_custom_data(); 43 | $maxupdates = !empty($data->maxupdates) ? $data->maxupdates : self::MAX_UPDATES; 44 | 45 | // Get all objects without a filesize and join them to a filesize from the files table. 46 | // Values less than 0 for object's location indicate an error for the object. 47 | $sql = "SELECT o.id, o.contenthash, o.timeduplicated, o.location, f.filesize 48 | FROM {tool_objectfs_objects} o 49 | JOIN {files} f ON o.contenthash = f.contenthash 50 | WHERE o.filesize IS NULL 51 | AND o.location >= 0 52 | GROUP BY o.id, 53 | o.contenthash, 54 | f.filesize"; 55 | $records = $DB->get_recordset_sql($sql, null, 0, $maxupdates + 1); 56 | 57 | // If more records found than the max number of updates, only process max updates then queue new task. 58 | $queueadditionaltask = false; 59 | 60 | $updatecount = 0; 61 | foreach ($records as $record) { 62 | if ($updatecount >= $maxupdates) { 63 | $queueadditionaltask = true; 64 | break; 65 | } 66 | $DB->update_record('tool_objectfs_objects', $record, true); 67 | $updatecount += 1; 68 | } 69 | $records->close(); 70 | if (!PHPUNIT_TEST) { 71 | mtrace("$updatecount records updated"); 72 | } 73 | 74 | if ($queueadditionaltask) { 75 | if (!PHPUNIT_TEST) { 76 | mtrace("Queueing additional task"); 77 | } 78 | manager::queue_adhoc_task(new populate_objects_filesize()); 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /classes/task/pull_objects_from_storage.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Task that pulls files from S3. 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\task; 27 | 28 | use tool_objectfs\local\object_manipulator\puller; 29 | 30 | /** 31 | * [Description pull_objects_from_storage] 32 | */ 33 | class pull_objects_from_storage extends task { 34 | 35 | /** @var string $manipulator */ 36 | protected $manipulator = puller::class; 37 | 38 | /** @var string $stringname */ 39 | protected $stringname = 'pull_objects_from_storage_task'; 40 | } 41 | -------------------------------------------------------------------------------- /classes/task/push_objects_to_storage.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Task that pushes files to S3. 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\task; 27 | 28 | use tool_objectfs\local\object_manipulator\pusher; 29 | 30 | /** 31 | * [Description push_objects_to_storage] 32 | */ 33 | class push_objects_to_storage extends task { 34 | 35 | /** @var string $manipulator */ 36 | protected $manipulator = pusher::class; 37 | 38 | /** @var string $stringname */ 39 | protected $stringname = 'push_objects_to_storage_task'; 40 | } 41 | -------------------------------------------------------------------------------- /classes/task/recover_error_objects.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Task that attempts to recover error state objects. 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\task; 27 | 28 | use tool_objectfs\local\object_manipulator\recoverer; 29 | 30 | /** 31 | * [Description recover_error_objects] 32 | */ 33 | class recover_error_objects extends task { 34 | 35 | /** @var string $manipulator */ 36 | protected $manipulator = recoverer::class; 37 | 38 | /** @var string $stringname */ 39 | protected $stringname = 'recover_error_objects_task'; 40 | } 41 | -------------------------------------------------------------------------------- /classes/task/task.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Base abstract class for objectfs tasks. 19 | * 20 | * @package tool_objectfs 21 | * @author Gleimer Mora 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | namespace tool_objectfs\task; 27 | 28 | use coding_exception; 29 | use moodle_exception; 30 | use stdClass; 31 | use tool_objectfs\local\manager; 32 | use tool_objectfs\local\object_manipulator\manipulator_builder; 33 | 34 | defined('MOODLE_INTERNAL') || die(); 35 | 36 | require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); 37 | 38 | /** 39 | * [Description task] 40 | */ 41 | abstract class task extends \core\task\scheduled_task implements objectfs_task { 42 | 43 | /** @var stdClass $config */ 44 | protected $config; 45 | 46 | /** 47 | * task constructor. 48 | */ 49 | public function __construct() { 50 | $this->config = manager::get_objectfs_config(); 51 | } 52 | 53 | /** 54 | * Get task name 55 | * @return string 56 | * @throws coding_exception 57 | */ 58 | public function get_name() { 59 | return get_string($this->stringname, 'tool_objectfs'); 60 | } 61 | 62 | /** 63 | * Execute task 64 | * @throws coding_exception 65 | * @throws moodle_exception 66 | */ 67 | public function execute() { 68 | if ($this->enabled_tasks()) { 69 | (new manipulator_builder())->execute($this->manipulator); 70 | } 71 | } 72 | 73 | /** 74 | * enabled_tasks 75 | * @return bool 76 | * @throws coding_exception 77 | */ 78 | protected function enabled_tasks() { 79 | $enabletasks = (bool)$this->config->enabletasks; 80 | if (!$enabletasks) { 81 | mtrace(get_string('not_enabled', 'tool_objectfs')); 82 | } 83 | return $enabletasks; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /classes/task/trigger_update_object_tags.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\task; 18 | 19 | use core\task\manager; 20 | use core\task\scheduled_task; 21 | 22 | /** 23 | * Queues update_object_tags adhoc task periodically, or manually from the frontend. 24 | * 25 | * @package tool_objectfs 26 | * @author Matthew Hilton 27 | * @copyright Catalyst IT 28 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 29 | */ 30 | class trigger_update_object_tags extends scheduled_task { 31 | /** 32 | * Task name 33 | */ 34 | public function get_name() { 35 | return get_string('task:triggerupdateobjecttags', 'tool_objectfs'); 36 | } 37 | /** 38 | * Execute task 39 | */ 40 | public function execute() { 41 | // Only schedule up to the max amount, less any that are already scheduled. 42 | $alreadyexist = count(manager::get_adhoc_tasks(update_object_tags::class)); 43 | $maxtoschedule = get_config('tool_objectfs', 'maxtaggingtaskstospawn'); 44 | $toschedule = max(0, $maxtoschedule - $alreadyexist); 45 | 46 | for ($i = 0; $i < $toschedule; $i++) { 47 | // Queue adhoc task, nothing else. 48 | $task = new update_object_tags(); 49 | $task->set_custom_data([ 50 | 'iteration' => 1, 51 | ]); 52 | manager::queue_adhoc_task($task); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /classes/tests/test_azure_blob_storage_integration_client.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\tests; 18 | 19 | use tool_objectfs\local\store\azure_blob_storage\client; 20 | 21 | /** 22 | * Client used for integration testing azure blob storage client 23 | * 24 | * @package tool_objectfs 25 | * @author Matthew Hilton 26 | * @copyright 2024 Catalyst IT 27 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 28 | */ 29 | class test_azure_blob_storage_integration_client extends client { 30 | /** 31 | * @var string 32 | */ 33 | private $runidentifier; 34 | 35 | /** 36 | * construct 37 | * @param mixed $config 38 | * @return void 39 | */ 40 | public function __construct($config) { 41 | parent::__construct($config); 42 | $time = microtime(); 43 | $this->runidentifier = md5($time); 44 | } 45 | 46 | /** 47 | * get_filepath_from_hash 48 | * @param mixed $contenthash 49 | * 50 | * @return string 51 | */ 52 | protected function get_filepath_from_hash($contenthash): string { 53 | $l1 = $contenthash[0] . $contenthash[1]; 54 | $l2 = $contenthash[2] . $contenthash[3]; 55 | $runidentifier = $this->runidentifier; 56 | return "test/$runidentifier/$l1/$l2/$contenthash"; 57 | } 58 | 59 | } 60 | 61 | -------------------------------------------------------------------------------- /classes/tests/test_azure_integration_client.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\tests; 18 | 19 | use tool_objectfs\local\store\azure\client; 20 | 21 | /** 22 | * Client used for integration testing azure client 23 | * 24 | * @package tool_objectfs 25 | * @copyright Catalyst IT 26 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 27 | * @deprecated Since Moodle 4.2 - Please see the README about updating to new azure_blob_storage client. 28 | */ 29 | class test_azure_integration_client extends client { 30 | 31 | /** 32 | * @var string 33 | */ 34 | private $runidentifier; 35 | 36 | /** 37 | * construct 38 | * @param mixed $config 39 | * @return void 40 | */ 41 | public function __construct($config) { 42 | // Set config directly. Calling __construct will do nothing 43 | // since unit tests do not have the azure sdk installed. 44 | $this->config = $config; 45 | 46 | $time = microtime(); 47 | $this->runidentifier = md5($time); 48 | } 49 | 50 | /** 51 | * get_filepath_from_hash 52 | * @param mixed $contenthash 53 | * 54 | * @return string 55 | */ 56 | protected function get_filepath_from_hash($contenthash) { 57 | $l1 = $contenthash[0] . $contenthash[1]; 58 | $l2 = $contenthash[2] . $contenthash[3]; 59 | $runidentifier = $this->runidentifier; 60 | return "test/$runidentifier/$l1/$l2/$contenthash"; 61 | } 62 | 63 | } 64 | 65 | -------------------------------------------------------------------------------- /classes/tests/test_digitalocean_integration_client.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\tests; 18 | 19 | use tool_objectfs\local\store\digitalocean\client; 20 | 21 | /** 22 | * Client used for integration testing digitalocean client 23 | * 24 | * @package tool_objectfs 25 | * @copyright Catalyst IT 26 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 27 | */ 28 | class test_digitalocean_integration_client extends client { 29 | 30 | /** 31 | * @var string 32 | */ 33 | private $runidentifier; 34 | 35 | /** 36 | * construct 37 | * @param mixed $config 38 | * @return void 39 | */ 40 | public function __construct($config) { 41 | parent::__construct($config); 42 | $time = microtime(); 43 | $this->runidentifier = md5($time); 44 | } 45 | 46 | /** 47 | * get_filepath_from_hash 48 | * @param mixed $contenthash 49 | * 50 | * @return string 51 | */ 52 | protected function get_filepath_from_hash($contenthash) { 53 | $l1 = $contenthash[0] . $contenthash[1]; 54 | $l2 = $contenthash[2] . $contenthash[3]; 55 | $runidentifier = $this->runidentifier; 56 | return "test/$runidentifier/$l1/$l2/$contenthash"; 57 | } 58 | 59 | } 60 | 61 | -------------------------------------------------------------------------------- /classes/tests/test_file_system.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * object_file_system abstract class. 19 | * 20 | * Remote object storage providers extent this class. 21 | * At minimum you need to impletment get_remote_client. 22 | * 23 | * @package tool_objectfs 24 | * @author Kenneth Hendricks 25 | * @copyright Catalyst IT 26 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 27 | */ 28 | 29 | namespace tool_objectfs\tests; 30 | 31 | use tool_objectfs\local\manager; 32 | use tool_objectfs\local\store\object_file_system; 33 | 34 | /** 35 | * [Description test_file_system] 36 | */ 37 | class test_file_system extends object_file_system { 38 | 39 | /** 40 | * @var int 41 | */ 42 | private $maxupload; 43 | 44 | /** 45 | * initialise_external_client 46 | * @param \stdClass $config 47 | * 48 | * @return mixed 49 | */ 50 | protected function initialise_external_client($config) { 51 | global $CFG; 52 | if (isset($CFG->phpunit_objectfs_s3_integration_test_credentials)) { 53 | $credentials = $CFG->phpunit_objectfs_s3_integration_test_credentials; 54 | $config->s3_key = $credentials['s3_key']; 55 | $config->s3_secret = $credentials['s3_secret']; 56 | $config->s3_bucket = $credentials['s3_bucket']; 57 | $config->s3_region = $credentials['s3_region']; 58 | manager::set_objectfs_config($config); 59 | $client = new test_s3_integration_client($config); 60 | } else if (isset($CFG->phpunit_objectfs_azure_integration_test_credentials)) { 61 | $credentials = $CFG->phpunit_objectfs_azure_integration_test_credentials; 62 | $config->azure_accountname = $credentials['azure_accountname']; 63 | $config->azure_container = $credentials['azure_container']; 64 | $config->azure_sastoken = $credentials['azure_sastoken']; 65 | manager::set_objectfs_config($config); 66 | $client = new test_azure_integration_client($config); 67 | } else if (isset($CFG->phpunit_objectfs_azure_blob_storage_integration_test_credentials)) { 68 | $credentials = $CFG->phpunit_objectfs_azure_blob_storage_integration_test_credentials; 69 | $config->azure_accountname = $credentials['azure_accountname']; 70 | $config->azure_container = $credentials['azure_container']; 71 | $config->azure_sastoken = $credentials['azure_sastoken']; 72 | manager::set_objectfs_config($config); 73 | $client = new test_azure_blob_storage_integration_client($config); 74 | } else if (isset($CFG->phpunit_objectfs_swift_integration_test_credentials)) { 75 | $credentials = $CFG->phpunit_objectfs_swift_integration_test_credentials; 76 | $config->openstack_authurl = $credentials['openstack_authurl']; 77 | $config->openstack_region = $credentials['openstack_region']; 78 | $config->openstack_container = $credentials['openstack_container']; 79 | $config->openstack_username = $credentials['openstack_username']; 80 | $config->openstack_password = $credentials['openstack_password']; 81 | $config->openstack_tenantname = $credentials['openstack_tenantname']; 82 | $config->openstack_projectid = $credentials['openstack_projectid']; 83 | manager::set_objectfs_config($config); 84 | $client = new test_swift_integration_client($config); 85 | } else { 86 | $client = new test_client($config); 87 | } 88 | $this->maxupload = $client->get_maximum_upload_size(); 89 | return $client; 90 | } 91 | 92 | /** 93 | * get_maximum_upload_size 94 | * @return float|int 95 | */ 96 | public function get_maximum_upload_size() { 97 | return $this->maxupload; 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /classes/tests/test_s3_integration_client.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\tests; 18 | 19 | use tool_objectfs\local\store\s3\client; 20 | 21 | /** 22 | * Client used for integration testing aws client 23 | * 24 | * @package tool_objectfs 25 | * @copyright Catalyst IT 26 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 27 | */ 28 | class test_s3_integration_client extends client { 29 | 30 | /** 31 | * @var string 32 | */ 33 | private $runidentifier; 34 | 35 | /** 36 | * construct 37 | * @param \stdClass $config 38 | */ 39 | public function __construct($config) { 40 | parent::__construct($config); 41 | $time = microtime(); 42 | $this->runidentifier = md5($time); 43 | } 44 | 45 | /** 46 | * get_filepath_from_hash 47 | * @param string $contenthash 48 | * 49 | * @return string 50 | */ 51 | protected function get_filepath_from_hash($contenthash) { 52 | $l1 = $contenthash[0] . $contenthash[1]; 53 | $l2 = $contenthash[2] . $contenthash[3]; 54 | $runidentifier = $this->runidentifier; 55 | return "test/$runidentifier/$l1/$l2/$contenthash"; 56 | } 57 | 58 | } 59 | 60 | -------------------------------------------------------------------------------- /classes/tests/test_swift_integration_client.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\tests; 18 | 19 | use tool_objectfs\local\store\swift\client; 20 | 21 | /** 22 | * Client used for integration testing swift client 23 | * 24 | * @package tool_objectfs 25 | * @copyright Catalyst IT 26 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 27 | */ 28 | class test_swift_integration_client extends client { 29 | 30 | /** 31 | * @var string 32 | */ 33 | private $runidentifier; 34 | 35 | /** 36 | * string 37 | * @param \stdClass $config 38 | */ 39 | public function __construct($config) { 40 | parent::__construct($config); 41 | $time = microtime(); 42 | $this->runidentifier = md5($time); 43 | } 44 | 45 | /** 46 | * get_filepath_from_hash 47 | * @param string $contenthash 48 | * 49 | * @return string 50 | */ 51 | protected function get_filepath_from_hash($contenthash) { 52 | $l1 = $contenthash[0] . $contenthash[1]; 53 | $l2 = $contenthash[2] . $contenthash[3]; 54 | $runidentifier = $this->runidentifier; 55 | return "test/$runidentifier/$l1/$l2/$contenthash"; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /db/install.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 |
25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 |
37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 |
54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 |
68 | 69 | 70 | -------------------------------------------------------------------------------- /db/tasks.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * tool_objectfs tasks 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | defined('MOODLE_INTERNAL') || die(); 27 | 28 | $tasks = [ 29 | [ 30 | 'classname' => 'tool_objectfs\task\push_objects_to_storage', 31 | 'blocking' => 0, 32 | 'minute' => '*', 33 | 'hour' => '*', 34 | 'day' => '*', 35 | 'dayofweek' => '*', 36 | 'month' => '*', 37 | ], 38 | [ 39 | 'classname' => 'tool_objectfs\task\generate_status_report', 40 | 'blocking' => 0, 41 | 'minute' => '17', 42 | 'hour' => '*', 43 | 'day' => '*', 44 | 'dayofweek' => '*', 45 | 'month' => '*', 46 | ], 47 | [ 48 | 'classname' => 'tool_objectfs\task\delete_local_objects', 49 | 'blocking' => 0, 50 | 'minute' => '*', 51 | 'hour' => '*', 52 | 'day' => '*', 53 | 'dayofweek' => '*', 54 | 'month' => '*', 55 | ], 56 | [ 57 | 'classname' => 'tool_objectfs\task\orphan_objects', 58 | 'blocking' => 0, 59 | 'minute' => 'R', 60 | 'hour' => 'R', 61 | 'day' => '*', 62 | 'dayofweek' => '*', 63 | 'month' => '*', 64 | ], 65 | [ 66 | 'classname' => 'tool_objectfs\task\delete_orphaned_object_metadata', 67 | 'blocking' => 0, 68 | 'minute' => 'R', 69 | 'hour' => 'R', 70 | 'day' => '*', 71 | 'dayofweek' => '*', 72 | 'month' => '*', 73 | ], 74 | [ 75 | 'classname' => 'tool_objectfs\task\delete_local_empty_directories', 76 | 'blocking' => 0, 77 | 'minute' => '0', 78 | 'hour' => '1', 79 | 'day' => '*', 80 | 'dayofweek' => '*', 81 | 'month' => '*', 82 | ], 83 | [ 84 | 'classname' => 'tool_objectfs\task\pull_objects_from_storage', 85 | 'blocking' => 0, 86 | 'minute' => '*', 87 | 'hour' => '*', 88 | 'day' => '*', 89 | 'dayofweek' => '*', 90 | 'month' => '*', 91 | ], 92 | [ 93 | 'classname' => 'tool_objectfs\task\recover_error_objects', 94 | 'blocking' => 0, 95 | 'minute' => '34', 96 | 'hour' => '*/12', 97 | 'day' => '*', 98 | 'dayofweek' => '*', 99 | 'month' => '*', 100 | ], 101 | [ 102 | 'classname' => 'tool_objectfs\task\check_objects_location', 103 | 'blocking' => 0, 104 | 'minute' => 'R', 105 | 'hour' => '*', 106 | 'day' => '*', 107 | 'dayofweek' => '*', 108 | 'month' => '*', 109 | ], 110 | [ 111 | 'classname' => 'tool_objectfs\task\trigger_update_object_tags', 112 | 'blocking' => 0, 113 | 'minute' => 'R', 114 | 'hour' => '*', 115 | 'day' => '*', 116 | 'dayofweek' => '*', 117 | 'month' => '*', 118 | // Default disabled - intended to be manually run. 119 | // Also, objectfs tagging support is default off. 120 | 'disabled' => true, 121 | ], 122 | ]; 123 | 124 | -------------------------------------------------------------------------------- /lib.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * S3 file system lib 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | use tool_objectfs\local\object_manipulator\manipulator_builder; 27 | use tool_objectfs\local\tag\tag_manager; 28 | 29 | define('OBJECTFS_PLUGIN_NAME', 'tool_objectfs'); 30 | 31 | /** 32 | * Location enum of the object 33 | * ORPHANED is when the {objectfs_objects} table contains a record linking to a 34 | * moodle {files} record which is no longer present. 35 | */ 36 | define('OBJECT_LOCATION_ORPHANED', -2); 37 | 38 | /** 39 | * Location enum of the object 40 | * ERROR is when the file is missing when it is expected to be there. 41 | * @see tests/object_file_system_test.php for examples. 42 | */ 43 | define('OBJECT_LOCATION_ERROR', -1); 44 | 45 | /** 46 | * Location enum of the object 47 | * LOCAL is when the object exists locally only. 48 | */ 49 | define('OBJECT_LOCATION_LOCAL', 0); 50 | 51 | /** 52 | * Location enum of the object 53 | * DUPLICATED is when the object exists both locally, and remotely. 54 | */ 55 | define('OBJECT_LOCATION_DUPLICATED', 1); 56 | 57 | /** 58 | * Location enum of the object 59 | * EXTERNAL is when when the object lives remotely only. 60 | */ 61 | define('OBJECT_LOCATION_EXTERNAL', 2); 62 | 63 | define('OBJECTFS_REPORT_OBJECT_LOCATION', 0); 64 | define('OBJECTFS_REPORT_LOG_SIZE', 1); 65 | define('OBJECTFS_REPORT_MIME_TYPE', 2); 66 | 67 | define('OBJECTFS_BYTES_IN_TERABYTE', 1099511627776); 68 | 69 | define('TOOL_OBJECTFS_DELETE_EXTERNAL_NO', 0); 70 | define('TOOL_OBJECTFS_DELETE_EXTERNAL_TRASH', 1); 71 | define('TOOL_OBJECTFS_DELETE_EXTERNAL_FULL', 2); 72 | 73 | /** 74 | * Sends a plugin file to the browser. 75 | * @param mixed $course 76 | * @param mixed $cm 77 | * @param \context $context 78 | * @param string $filearea 79 | * @param array $args 80 | * @param bool $forcedownload 81 | * @param array $options 82 | * @return bool 83 | * @throws coding_exception 84 | */ 85 | function tool_objectfs_pluginfile($course, $cm, context $context, $filearea, array $args, bool $forcedownload, 86 | array $options = []) { 87 | 88 | $fs = get_file_storage(); 89 | $file = $fs->get_file($context->id, OBJECTFS_PLUGIN_NAME, $filearea, $args[0], '/', $args[1]); 90 | if (!$file || (is_object($file) && $file->is_directory())) { 91 | send_file_not_found(); 92 | } 93 | $lifetime = optional_param('expires', null, PARAM_INT); 94 | \core\session\manager::write_close(); 95 | send_stored_file($file, $lifetime, 0, $forcedownload, $options); 96 | return true; 97 | } 98 | 99 | /** 100 | * Get status checks for tool_objectfs. 101 | * 102 | * @return array 103 | */ 104 | function tool_objectfs_status_checks() { 105 | $checks = [ 106 | new tool_objectfs\check\token_expiry(), 107 | new tool_objectfs\check\tagging_status(), 108 | new tool_objectfs\check\tagging_sync_status(), 109 | new tool_objectfs\check\tagging_migration_status(), 110 | ]; 111 | 112 | if (get_config('tool_objectfs', 'proxyrangerequests')) { 113 | $checks[] = new tool_objectfs\check\proxy_range_request(); 114 | } 115 | 116 | return $checks; 117 | } 118 | -------------------------------------------------------------------------------- /missing_files.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Missing files page. 19 | * 20 | * @package tool_objectfs 21 | * @author Dmitrii Metelkin 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | require_once(__DIR__ . '/../../../config.php'); 27 | require_once(__DIR__ . '/lib.php'); 28 | require_once($CFG->dirroot . '/lib/adminlib.php'); 29 | 30 | admin_externalpage_setup('tool_objectfs_missing_files'); 31 | 32 | use tool_objectfs\local\table\files_table; 33 | 34 | $download = optional_param('download', '', PARAM_ALPHA); 35 | 36 | $PAGE->set_url('/admin/tool/objectfs/missing_files.php'); 37 | $PAGE->set_context(context_system::instance()); 38 | $PAGE->set_cacheable(false); 39 | $output = $PAGE->get_renderer('tool_objectfs'); 40 | $table = new files_table('missing-files', OBJECT_LOCATION_ERROR); 41 | $table->define_baseurl('/admin/tool/objectfs/missing_files.php'); 42 | 43 | if ($table->is_downloading($download, get_string('filename:missingfiles', 'tool_objectfs'))) { 44 | $table->out(200, false); 45 | die(); 46 | } 47 | 48 | echo $output->header(); 49 | echo $output->heading(get_string('page:missingfiles', 'tool_objectfs')); 50 | 51 | $table->out(200, false); 52 | 53 | echo $output->footer(); 54 | -------------------------------------------------------------------------------- /object_location.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Object location history page. 19 | * 20 | * @package tool_objectfs 21 | * @author Mikhail Golenkov 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | define('NO_OUTPUT_BUFFERING', true); 27 | require_once(__DIR__ . '/../../../config.php'); 28 | require_once($CFG->dirroot . '/lib/adminlib.php'); 29 | require_once($CFG->libdir.'/tablelib.php'); 30 | 31 | admin_externalpage_setup('tool_objectfs_object_location_history'); 32 | 33 | $logformat = optional_param('download', '', PARAM_ALPHA); 34 | $params = []; 35 | if ($logformat) { 36 | $params['download'] = $logformat; 37 | } 38 | 39 | $pageurl = new \moodle_url('/admin/tool/objectfs/object_location.php', $params); 40 | $heading = get_string('object_status:locationhistory', 'tool_objectfs'); 41 | $PAGE->set_url($pageurl); 42 | $PAGE->set_context(context_system::instance()); 43 | $PAGE->set_pagelayout('report'); 44 | $PAGE->set_title($heading); 45 | $PAGE->set_heading($heading); 46 | 47 | $OUTPUT = $PAGE->get_renderer('tool_objectfs'); 48 | 49 | $table = new tool_objectfs\local\report\object_location_history_table(); 50 | $table->baseurl = $pageurl; 51 | 52 | if (empty($logformat)) { 53 | echo $OUTPUT->header(); 54 | $table->out(0, false); 55 | echo $OUTPUT->footer(); 56 | } else { 57 | $filename = 'object_location_history_' . userdate(time(), get_string('backupnameformat', 'langconfig'), 99, false); 58 | $table->is_downloading($logformat, $filename); 59 | $table->out(0, false); 60 | } 61 | -------------------------------------------------------------------------------- /object_status.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * File status history page. 19 | * 20 | * @package tool_objectfs 21 | * @author Mikhail Golenkov 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | require_once(__DIR__ . '/../../../config.php'); 27 | require_once($CFG->dirroot . '/lib/adminlib.php'); 28 | require_once($CFG->libdir.'/tablelib.php'); 29 | 30 | admin_externalpage_setup('tool_objectfs_object_status'); 31 | 32 | use tool_objectfs\local\report\objectfs_report; 33 | use tool_objectfs\local\report\object_status_history_table; 34 | 35 | $reportid = optional_param('reportid', 0, PARAM_INT); 36 | 37 | $baseurl = '/admin/tool/objectfs/object_status.php'; 38 | $pageurl = new \moodle_url($baseurl, ['reportid' => $reportid]); 39 | $heading = get_string('object_status:page', 'tool_objectfs'); 40 | $PAGE->set_url($pageurl); 41 | $PAGE->set_context(context_system::instance()); 42 | $PAGE->set_pagelayout('report'); 43 | $PAGE->set_title($heading); 44 | $PAGE->set_heading($heading); 45 | 46 | $OUTPUT = $PAGE->get_renderer('tool_objectfs'); 47 | echo $OUTPUT->header(); 48 | 49 | if ($reports = objectfs_report::get_report_ids()) { 50 | if (empty($reportid) || !array_key_exists($reportid, $reports)) { 51 | $reportid = key($reports); 52 | } 53 | echo $OUTPUT->object_status_history_page_header($reports, $reportid); 54 | 55 | $reporttypes = objectfs_report::get_report_types(); 56 | foreach ($reporttypes as $reporttype) { 57 | echo $OUTPUT->box_start(); 58 | $table = new object_status_history_table($reporttype, $reportid); 59 | $table->baseurl = $pageurl; 60 | 61 | $heading = get_string('object_status:' . $reporttype, 'tool_objectfs'); 62 | echo $OUTPUT->heading($heading, 2); 63 | $table->out(0, false); 64 | echo $OUTPUT->box_end(); 65 | } 66 | } else { 67 | echo $OUTPUT->heading(get_string('nothingtodisplay')); 68 | } 69 | 70 | echo $OUTPUT->footer(); 71 | -------------------------------------------------------------------------------- /phpunit.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | tests 21 | 22 | 23 | ../../../lib/filestorage/tests 24 | ../../../files/tests 25 | 26 | 27 | ../../../privacy/tests 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /pix/Microsoft-logo_rgb_c-gray.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/catalyst/moodle-tool_objectfs/48cab1182137b8f54039b950ae8b9eca1801fd2e/pix/Microsoft-logo_rgb_c-gray.png -------------------------------------------------------------------------------- /pix/catalyst-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/catalyst/moodle-tool_objectfs/48cab1182137b8f54039b950ae8b9eca1801fd2e/pix/catalyst-logo.png -------------------------------------------------------------------------------- /presignedurl_tests.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Pre-Signed URL testing page 19 | * 20 | * @package tool_objectfs 21 | * @author Mikhail Golenkov 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | use tool_objectfs\local\manager; 27 | 28 | require_once(__DIR__ . '/../../../config.php'); 29 | require_once(__DIR__ . '/lib.php'); 30 | require_once($CFG->dirroot . '/lib/adminlib.php'); 31 | 32 | admin_externalpage_setup('tool_objectfs_presignedurl_testing'); 33 | 34 | $output = $PAGE->get_renderer('tool_objectfs'); 35 | 36 | $delete = optional_param('delete', 0, PARAM_BOOL); 37 | $deletedsuccess = ''; 38 | if ($delete) { 39 | require_sesskey(); 40 | $output->delete_presignedurl_tests_files(); 41 | $deletedstring = get_string('settings:presignedurl:deletedsuccess', OBJECTFS_PLUGIN_NAME); 42 | $deletedsuccess = $output->notification($deletedstring, 'success'); 43 | } 44 | 45 | echo $output->header(); 46 | echo $output->heading(get_string('presignedurl_testing:page', 'tool_objectfs')); 47 | $settingslink = \html_writer::link(new \moodle_url('/admin/settings.php?section=tool_objectfs_settings'), 48 | get_string('presignedurl_testing:objectfssettings', 'tool_objectfs')); 49 | 50 | $config = manager::get_objectfs_config(); 51 | $support = false; 52 | if (!empty($config->filesystem)) { 53 | $fs = new $config->filesystem(); 54 | $support = $fs->supports_presigned_urls(); 55 | } 56 | if ($support) { 57 | $deleteurl = new \moodle_url('/admin/tool/objectfs/presignedurl_tests.php', ['delete' => 1, 'sesskey' => sesskey()]); 58 | $deletelinktext = get_string('settings:presignedurl:deletefiles', OBJECTFS_PLUGIN_NAME); 59 | echo $output->heading(html_writer::link($deleteurl, $deletelinktext) . $deletedsuccess, 6); 60 | $client = manager::get_client($config); 61 | if ($client && $client->get_availability()) { 62 | $connection = $client->test_connection(); 63 | if ($connection->success) { 64 | $testfiles = $output->presignedurl_tests_load_files($fs); 65 | echo $output->presignedurl_tests_content($fs, $testfiles); 66 | } else { 67 | echo $output->notification(get_string('settings:connectionfailure', 'tool_objectfs', $connection->details), 68 | 'notifyproblem'); 69 | echo $output->heading(get_string('presignedurl_testing:checkconnectionsettings', 'tool_objectfs').$settingslink, 5); 70 | } 71 | 72 | } else { 73 | echo $output->notification(get_string('settings:clientnotavailable', 'tool_objectfs'), 'notifyproblem'); 74 | echo $output->heading(get_string('presignedurl_testing:checkclientsettings', 'tool_objectfs').$settingslink, 5); 75 | } 76 | 77 | } else { 78 | echo $output->notification(get_string('presignedurl_testing:presignedurlsnotsupported', 'tool_objectfs'), 'notifyproblem'); 79 | echo $output->heading(get_string('presignedurl_testing:checkfssettings', 'tool_objectfs').$settingslink, 5); 80 | } 81 | 82 | echo $output->footer(); 83 | -------------------------------------------------------------------------------- /styles.css: -------------------------------------------------------------------------------- 1 | .ofs-bar { 2 | white-space: nowrap; 3 | } 4 | 5 | .ofs-bar-column { 6 | min-width: 12vw; 7 | } 8 | -------------------------------------------------------------------------------- /tests/check/tagging_migration_status_test.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\check; 18 | 19 | use core\check\result; 20 | use core\task\manager; 21 | use tool_objectfs\check\tagging_migration_status; 22 | use tool_objectfs\task\update_object_tags; 23 | use tool_objectfs\tests\testcase; 24 | 25 | /** 26 | * Tagging migration status check tests 27 | * 28 | * @package tool_objectfs 29 | * @author Matthew Hilton 30 | * @copyright Catalyst IT 31 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 32 | * @covers \tool_objectfs\check\tagging_migration_status 33 | */ 34 | class tagging_migration_status_test extends testcase { 35 | /** 36 | * Tests scenario that returns N/A 37 | */ 38 | public function test_get_result_na() { 39 | // Regardless if this is disabled, the check should still return a non n/a status. 40 | $this->enable_filesystem_and_set_tagging(false); 41 | $check = new tagging_migration_status(); 42 | $this->assertEquals(result::NA, $check->get_result()->get_status()); 43 | } 44 | 45 | /* 46 | * Test scenario that returns WARNING 47 | */ 48 | public function test_get_result_warning() { 49 | // Regardless if this is disabled, the check should still return a non n/a status. 50 | $this->enable_filesystem_and_set_tagging(false); 51 | 52 | $task = new update_object_tags(); 53 | $task->set_fail_delay(64); 54 | manager::queue_adhoc_task($task); 55 | 56 | $check = new tagging_migration_status(); 57 | $this->assertEquals(result::WARNING, $check->get_result()->get_status()); 58 | } 59 | 60 | /* 61 | * Test scenario that returns OK 62 | */ 63 | public function test_get_result_ok() { 64 | // Regardless if this is disabled, the check should still return a non n/a status. 65 | $this->enable_filesystem_and_set_tagging(false); 66 | 67 | $task = new update_object_tags(); 68 | manager::queue_adhoc_task($task); 69 | 70 | $check = new tagging_migration_status(); 71 | $this->assertEquals(result::OK, $check->get_result()->get_status()); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /tests/check/tagging_sync_status_test.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\check; 18 | 19 | use core\check\result; 20 | use tool_objectfs\check\tagging_sync_status; 21 | use tool_objectfs\local\tag\tag_manager; 22 | use tool_objectfs\tests\testcase; 23 | 24 | /** 25 | * Tagging sync status check tests 26 | * 27 | * @package tool_objectfs 28 | * @author Matthew Hilton 29 | * @copyright Catalyst IT 30 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 31 | * @covers \tool_objectfs\check\tagging_sync_status 32 | */ 33 | class tagging_sync_status_test extends testcase { 34 | /** 35 | * Tests scenario that returns N/A 36 | */ 37 | public function test_get_result_na() { 38 | // Not enabled by default, should return N/A. 39 | $check = new tagging_sync_status(); 40 | $this->assertEquals(result::NA, $check->get_result()->get_status()); 41 | } 42 | 43 | /** 44 | * Test scenario that returns OK 45 | */ 46 | public function test_get_result_ok() { 47 | $this->enable_filesystem_and_set_tagging(true); 48 | $object = $this->create_remote_object(); 49 | tag_manager::mark_object_tag_sync_status($object->contenthash, tag_manager::SYNC_STATUS_COMPLETE); 50 | 51 | // All objects OK, should return ok. 52 | $check = new tagging_sync_status(); 53 | $this->assertEquals(result::OK, $check->get_result()->get_status()); 54 | } 55 | 56 | /** 57 | * Tests scenario that returns WARNING 58 | */ 59 | public function test_get_result_warning() { 60 | $this->enable_filesystem_and_set_tagging(true); 61 | $object = $this->create_remote_object(); 62 | tag_manager::mark_object_tag_sync_status($object->contenthash, tag_manager::SYNC_STATUS_ERROR); 63 | 64 | // An object has error, should return warning. 65 | $check = new tagging_sync_status(); 66 | $this->assertEquals(result::WARNING, $check->get_result()->get_status()); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /tests/fixtures/test.txt: -------------------------------------------------------------------------------- 1 | This is a test text file -------------------------------------------------------------------------------- /tests/fixtures/testimage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/catalyst/moodle-tool_objectfs/48cab1182137b8f54039b950ae8b9eca1801fd2e/tests/fixtures/testimage.png -------------------------------------------------------------------------------- /tests/fixtures/testlarge.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/catalyst/moodle-tool_objectfs/48cab1182137b8f54039b950ae8b9eca1801fd2e/tests/fixtures/testlarge.pdf -------------------------------------------------------------------------------- /tests/fixtures/testrelativeurls.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | HTML testing relative URLs 5 | 6 | 7 | 8 |
9 | test.txt 10 | 13 | 14 |

This is a heading

15 |

This is a paragraph.

16 | 17 | -------------------------------------------------------------------------------- /tests/fixtures/testsmall.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/catalyst/moodle-tool_objectfs/48cab1182137b8f54039b950ae8b9eca1801fd2e/tests/fixtures/testsmall.pdf -------------------------------------------------------------------------------- /tests/fixtures/testspreadsheet.ods: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/catalyst/moodle-tool_objectfs/48cab1182137b8f54039b950ae8b9eca1801fd2e/tests/fixtures/testspreadsheet.ods -------------------------------------------------------------------------------- /tests/fixtures/teststyle.css: -------------------------------------------------------------------------------- 1 | body { 2 | background-color: lightblue; 3 | } 4 | 5 | h1 { 6 | color: navy; 7 | margin-left: 20px; 8 | } 9 | -------------------------------------------------------------------------------- /tests/fixtures/testvideo.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/catalyst/moodle-tool_objectfs/48cab1182137b8f54039b950ae8b9eca1801fd2e/tests/fixtures/testvideo.mp4 -------------------------------------------------------------------------------- /tests/fixtures/testwriter.odt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/catalyst/moodle-tool_objectfs/48cab1182137b8f54039b950ae8b9eca1801fd2e/tests/fixtures/testwriter.odt -------------------------------------------------------------------------------- /tests/fixtures/😀.txt: -------------------------------------------------------------------------------- 1 | This is a test text file with a unicode character in the file name 2 | -------------------------------------------------------------------------------- /tests/local/object_manipulator/puller_test.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\local\object_manipulator; 18 | 19 | use tool_objectfs\local\manager; 20 | 21 | /** 22 | * Tests for object puller. 23 | * 24 | * @covers \tool_objectfs\local\object_manipulator\puller 25 | * @package tool_objectfs 26 | * @copyright Catalyst IT 27 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 28 | */ 29 | class puller_test extends \tool_objectfs\tests\testcase { 30 | 31 | /** @var string $manipulator */ 32 | protected $manipulator = puller::class; 33 | 34 | /** @var puller Puller object */ 35 | protected $puller; 36 | 37 | protected function setUp(): void { 38 | parent::setUp(); 39 | $config = manager::get_objectfs_config(); 40 | $config->sizethreshold = 100; 41 | manager::set_objectfs_config($config); 42 | $this->logger = new \tool_objectfs\log\aggregate_logger(); 43 | $this->puller = new puller($this->filesystem, $config, $this->logger); 44 | ob_start(); 45 | } 46 | 47 | protected function tearDown(): void { 48 | ob_end_clean(); 49 | } 50 | 51 | /** 52 | * set_puller_config 53 | * @param mixed $key 54 | * @param mixed $value 55 | * 56 | * @return void 57 | */ 58 | protected function set_puller_config($key, $value) { 59 | $config = manager::get_objectfs_config(); 60 | $config->$key = $value; 61 | manager::set_objectfs_config($config); 62 | $this->puller = new puller($this->filesystem, $config, $this->logger); 63 | } 64 | 65 | public function test_puller_get_candidate_objects_will_get_remote_objects() { 66 | $remoteobject = $this->create_remote_object(); 67 | 68 | self::assertTrue($this->objects_contain_hash($remoteobject->contenthash)); 69 | } 70 | 71 | public function test_puller_get_candidate_objects_will_not_get_duplicated_or_local_objects() { 72 | $localobject = $this->create_local_object(); 73 | $duplicatedobject = $this->create_duplicated_object(); 74 | 75 | self::assertFalse($this->objects_contain_hash($localobject->contenthash)); 76 | self::assertFalse($this->objects_contain_hash($duplicatedobject->contenthash)); 77 | } 78 | 79 | public function test_puller_get_candidate_objects_will_not_get_objects_over_sizethreshold() { 80 | global $DB; 81 | $remoteobject = $this->create_remote_object(); 82 | $DB->set_field('files', 'filesize', 10, ['contenthash' => $remoteobject->contenthash]); 83 | $this->set_puller_config('sizethreshold', 0); 84 | 85 | self::assertFalse($this->objects_contain_hash($remoteobject->contenthash)); 86 | } 87 | 88 | public function test_puller_can_pull_remote_file() { 89 | global $DB; 90 | $object = $this->create_remote_object(); 91 | 92 | $this->puller->execute([$object]); 93 | 94 | $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); 95 | $this->assertEquals(OBJECT_LOCATION_DUPLICATED, $location); 96 | $this->assertTrue($this->is_locally_readable_by_hash($object->contenthash)); 97 | $this->assertTrue($this->is_externally_readable_by_hash($object->contenthash)); 98 | } 99 | 100 | public function test_puller_can_handle_duplicated_file() { 101 | global $DB; 102 | $object = $this->create_duplicated_object(); 103 | 104 | $this->puller->execute([$object]); 105 | 106 | $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); 107 | $this->assertEquals(OBJECT_LOCATION_DUPLICATED, $location); 108 | $this->assertTrue($this->is_locally_readable_by_hash($object->contenthash)); 109 | $this->assertTrue($this->is_externally_readable_by_hash($object->contenthash)); 110 | } 111 | 112 | public function test_puller_can_handle_local_file() { 113 | global $DB; 114 | $object = $this->create_local_object(); 115 | 116 | $this->puller->execute([$object]); 117 | 118 | $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); 119 | $this->assertEquals(OBJECT_LOCATION_LOCAL, $location); 120 | $this->assertTrue($this->is_locally_readable_by_hash($object->contenthash)); 121 | $this->assertFalse($this->is_externally_readable_by_hash($object->contenthash)); 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /tests/local/object_manipulator/recoverer_test.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\local\object_manipulator; 18 | 19 | use tool_objectfs\local\manager; 20 | use tool_objectfs\local\object_manipulator\candidates\candidates_finder; 21 | 22 | /** 23 | * Tests for object recoverer. 24 | * 25 | * @covers \tool_objectfs\local\object_manipulator\recoverer 26 | * @package tool_objectfs 27 | * @copyright Catalyst IT 28 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 29 | */ 30 | class recoverer_test extends \tool_objectfs\tests\testcase { 31 | 32 | /** @var candidates_finder Candidates finder object */ 33 | protected $candidatesfinder; 34 | 35 | /** @var recoverer Recoverer object */ 36 | protected $recoverer; 37 | 38 | protected function setUp(): void { 39 | parent::setUp(); 40 | $config = manager::get_objectfs_config(); 41 | $this->candidatesfinder = new candidates_finder(recoverer::class, $config); 42 | manager::set_objectfs_config($config); 43 | $this->logger = new \tool_objectfs\log\aggregate_logger(); 44 | $this->recoverer = new recoverer($this->filesystem, $config, $this->logger); 45 | ob_start(); 46 | } 47 | 48 | protected function tearDown(): void { 49 | ob_end_clean(); 50 | } 51 | 52 | public function test_recoverer_get_candidate_objects_will_get_error_objects() { 53 | $recovererobject = $this->create_error_object(); 54 | $candidateobjects = $this->candidatesfinder->get(); 55 | 56 | foreach ($candidateobjects as $candidate) { 57 | $this->assertEquals($recovererobject->contenthash, $candidate->contenthash); 58 | } 59 | } 60 | 61 | public function test_recoverer_will_recover_local_objects() { 62 | global $DB; 63 | $object = $this->create_local_object(); 64 | $DB->set_field('tool_objectfs_objects', 'location', OBJECT_LOCATION_ERROR, ['contenthash' => $object->contenthash]); 65 | 66 | $this->recoverer->execute([$object]); 67 | 68 | $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); 69 | $this->assertEquals(OBJECT_LOCATION_LOCAL, $location); 70 | } 71 | 72 | public function test_recoverer_will_recover_duplicated_objects() { 73 | global $DB; 74 | $object = $this->create_duplicated_object(); 75 | $DB->set_field('tool_objectfs_objects', 'location', OBJECT_LOCATION_ERROR, ['contenthash' => $object->contenthash]); 76 | 77 | $this->recoverer->execute([$object]); 78 | 79 | $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); 80 | $this->assertEquals(OBJECT_LOCATION_DUPLICATED, $location); 81 | } 82 | 83 | public function test_recoverer_will_recover_remote_objects() { 84 | global $DB; 85 | $object = $this->create_remote_object(); 86 | $DB->set_field('tool_objectfs_objects', 'location', OBJECT_LOCATION_ERROR, ['contenthash' => $object->contenthash]); 87 | 88 | $this->recoverer->execute([$object]); 89 | 90 | $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); 91 | $this->assertEquals(OBJECT_LOCATION_EXTERNAL, $location); 92 | } 93 | 94 | public function test_recoverer_will_not_recover_error_objects() { 95 | global $DB; 96 | $object = $this->create_error_object(); 97 | $DB->set_field('tool_objectfs_objects', 'location', OBJECT_LOCATION_ERROR, ['contenthash' => $object->contenthash]); 98 | 99 | $this->recoverer->execute([$object]); 100 | 101 | $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); 102 | $this->assertEquals(OBJECT_LOCATION_ERROR, $location); 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /tests/local/tasks_test.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\local; 18 | 19 | /** 20 | * End to end tests for tasks. Make sure all the plumbing is ok. 21 | * 22 | * @covers \tool_objectfs\local\manager 23 | * @package tool_objectfs 24 | * @copyright Catalyst IT 25 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 26 | */ 27 | class tasks_test extends \tool_objectfs\tests\testcase { 28 | 29 | protected function setUp(): void { 30 | parent::setUp(); 31 | ob_start(); 32 | } 33 | 34 | protected function tearDown(): void { 35 | ob_end_clean(); 36 | } 37 | 38 | public function test_run_scheduled_tasks() { 39 | global $CFG; 40 | // If tasks not implemented. 41 | if ($CFG->branch <= 26) { 42 | return true; 43 | } 44 | 45 | $config = manager::get_objectfs_config(); 46 | $config->enabletasks = true; 47 | $config->filesystem = '\\tool_objectfs\\tests\\test_file_system'; 48 | manager::set_objectfs_config($config); 49 | 50 | $scheduledtasknames = [ 51 | 'delete_local_objects', 52 | 'delete_local_empty_directories', 53 | 'generate_status_report', 54 | 'pull_objects_from_storage', 55 | 'push_objects_to_storage', 56 | 'recover_error_objects', 57 | 'check_objects_location', 58 | 'delete_orphaned_object_metadata', 59 | ]; 60 | 61 | foreach ($scheduledtasknames as $taskname) { 62 | $task = \core\task\manager::get_scheduled_task('\\tool_objectfs\\task\\' . $taskname); 63 | $task->execute(); 64 | } 65 | $this->expectNotToPerformAssertions(); // Just check we get this far without any exceptions. 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /tests/privacy/privacy_test.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\privacy; 18 | 19 | /** 20 | * Privacy test for Objectfs. 21 | * 22 | * @package tool_objectfs 23 | * @category test 24 | * @copyright 2020 Mikhail Golenkov 25 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 26 | * @covers \tool_objectfs\privacy\provider 27 | */ 28 | class privacy_test extends \advanced_testcase { 29 | 30 | /** 31 | * Check the privacy provider implements null_provider. 32 | */ 33 | public function test_provider_implements_null_provider() { 34 | // Privacy classes may not exist in older Moodles/Totara. 35 | if (interface_exists('\core_privacy\local\metadata\null_provider')) { 36 | $provider = new provider(); 37 | $this->assertInstanceOf('\core_privacy\local\metadata\null_provider', $provider); 38 | } else { 39 | $this->markTestSkipped('Interface not found: \core_privacy\local\metadata\null_provider'); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /tests/task/trigger_update_object_tags_test.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs\task; 18 | 19 | use advanced_testcase; 20 | use core\task\manager; 21 | 22 | /** 23 | * Tests trigger_update_object_tags 24 | * 25 | * @package tool_objectfs 26 | * @author Matthew Hilton 27 | * @copyright Catalyst IT 28 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 29 | */ 30 | class trigger_update_object_tags_test extends advanced_testcase { 31 | /** 32 | * Tests executing scheduled task. 33 | * @covers \tool_objectfs\task\trigger_update_object_tags::execute 34 | */ 35 | public function test_execute() { 36 | $this->resetAfterTest(); 37 | 38 | $task = new trigger_update_object_tags(); 39 | $task->execute(); 40 | 41 | // Ensure it spawned an adhoc task. 42 | $queuedadhoctasks = manager::get_adhoc_tasks(update_object_tags::class); 43 | $this->assertCount(1, $queuedadhoctasks); 44 | 45 | // Ensure the adhoc task spawned has an iteration of 1. 46 | $adhoctask = current($queuedadhoctasks); 47 | $this->assertNotEmpty($adhoctask->get_custom_data()); 48 | $this->assertEquals(1, $adhoctask->get_custom_data()->iteration); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /tests/token_expiry_test.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | namespace tool_objectfs; 18 | 19 | use core\check\result; 20 | use tool_objectfs\check\token_expiry; 21 | use tool_objectfs\local\manager; 22 | use tool_objectfs\tests\testcase; 23 | 24 | /** 25 | * Token expiry check test. 26 | * 27 | * @covers \tool_objectfs\check\token_expiry 28 | * @package tool_objectfs 29 | * @copyright Catalyst IT 30 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 31 | */ 32 | class token_expiry_test extends testcase { 33 | /** 34 | * Provides to test_get_result 35 | * @return array 36 | */ 37 | public static function get_result_provider(): array { 38 | return [ 39 | 'ok' => [ 40 | 'expirytime' => time() + 10 * DAYSECS, 41 | 'warnperiod' => 5 * DAYSECS, 42 | 'expectedresult' => result::OK, 43 | ], 44 | 'warning' => [ 45 | 'expirytime' => time() + DAYSECS, 46 | 'warnperiod' => 5 * DAYSECS, 47 | 'expectedresult' => result::WARNING, 48 | ], 49 | 'expired' => [ 50 | 'expirytime' => time() - DAYSECS, 51 | 'warnperiod' => 5 * DAYSECS, 52 | 'expectedresult' => result::CRITICAL, 53 | ], 54 | ]; 55 | } 56 | 57 | /** 58 | * Tests getting check result 59 | * @param int $expirytime time to use as the tokens expiry 60 | * @param int $warnperiod period to set for warning about token expiry 61 | * @param string $expectedresult one of the result:: constants that is expected to be returned. 62 | * @dataProvider get_result_provider 63 | */ 64 | public function test_get_result(int $expirytime, int $warnperiod, string $expectedresult) { 65 | global $CFG; 66 | $config = manager::get_objectfs_config(); 67 | $config->filesystem = '\\tool_objectfs\\tests\\test_file_system'; 68 | manager::set_objectfs_config($config); 69 | 70 | $CFG->objectfs_phpunit_token_expiry_time = $expirytime; 71 | set_config('tokenexpirywarnperiod', $warnperiod, 'tool_objectfs'); 72 | 73 | $check = new token_expiry(); 74 | $result = $check->get_result(); 75 | $this->assertEquals($expectedresult, $result->get_status()); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /thirdpartylibs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | classes/local/store/azure/stream_wrapper.php 5 | Azure Blob Storage stream wrapper 6 | custom 7 | 8 | 9 | 10 | 11 | classes/local/store/swift/stream_wrapper.php 12 | Swift storage stream wrapper 13 | custom 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /version.php: -------------------------------------------------------------------------------- 1 | . 16 | 17 | /** 18 | * Version information. 19 | * 20 | * @package tool_objectfs 21 | * @author Kenneth Hendricks 22 | * @copyright Catalyst IT 23 | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later 24 | */ 25 | 26 | defined('MOODLE_INTERNAL') || die(); 27 | 28 | $plugin->version = 2025040900; // The current plugin version (Date: YYYYMMDDXX). 29 | $plugin->release = 2025040900; // Same as version. 30 | $plugin->requires = 2024042200; // Requires 4.4. 31 | $plugin->component = "tool_objectfs"; 32 | $plugin->maturity = MATURITY_STABLE; 33 | $plugin->supported = [404, 405]; 34 | --------------------------------------------------------------------------------