├── .dockerignore ├── .github ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml └── workflows │ ├── actionlint.yml │ ├── ci.yml │ ├── deploy.yml │ ├── release.yml │ └── rspec.yml ├── .gitignore ├── .govuk_dependabot_merger.yml ├── .rspec ├── .rubocop.yml ├── .ruby-version ├── .travis.yml ├── Dockerfile ├── Gemfile ├── Gemfile.lock ├── LICENCE ├── Procfile ├── README.md ├── Rakefile ├── app ├── assets │ └── config │ │ └── manifest.js ├── controllers │ ├── api │ │ ├── base_controller.rb │ │ ├── document_types_controller.rb │ │ ├── documents_controller.rb │ │ ├── healthcheck_controller.rb │ │ ├── metrics_controller.rb │ │ └── organisations_controller.rb │ ├── application_controller.rb │ ├── content_controller.rb │ └── single_item_controller.rb ├── domain │ ├── api │ │ ├── base_request.rb │ │ ├── content_request.rb │ │ ├── document_children_request.rb │ │ └── single_page_request.rb │ ├── etl │ │ ├── aggregations │ │ │ ├── monthly.rb │ │ │ └── search.rb │ │ ├── edition │ │ │ ├── content │ │ │ │ ├── parser.rb │ │ │ │ ├── parsers │ │ │ │ │ ├── body_content.rb │ │ │ │ │ ├── contact.rb │ │ │ │ │ ├── email_alert_signup.rb │ │ │ │ │ ├── finder.rb │ │ │ │ │ ├── finder_email_signup.rb │ │ │ │ │ ├── generic_with_links.rb │ │ │ │ │ ├── hmrc_manual.rb │ │ │ │ │ ├── licence.rb │ │ │ │ │ ├── local_transaction.rb │ │ │ │ │ ├── mainstream.rb │ │ │ │ │ ├── need.rb │ │ │ │ │ ├── no_content.rb │ │ │ │ │ ├── parts.rb │ │ │ │ │ ├── place.rb │ │ │ │ │ ├── service_manual_homepage.rb │ │ │ │ │ ├── service_manual_service_toolkit.rb │ │ │ │ │ ├── service_manual_standard.rb │ │ │ │ │ ├── service_manual_topic.rb │ │ │ │ │ ├── service_sign_in.rb │ │ │ │ │ ├── statistics_announcement.rb │ │ │ │ │ ├── step_by_step.rb │ │ │ │ │ ├── taxon.rb │ │ │ │ │ ├── transaction.rb │ │ │ │ │ ├── travel_advice.rb │ │ │ │ │ ├── travel_advice_index.rb │ │ │ │ │ └── unpublished.rb │ │ │ │ └── reading_time.rb │ │ │ ├── metadata │ │ │ │ ├── number_of_files.rb │ │ │ │ ├── number_of_pdfs.rb │ │ │ │ └── number_of_word_files.rb │ │ │ └── processor.rb │ │ ├── feedex │ │ │ ├── processor.rb │ │ │ └── service.rb │ │ ├── ga │ │ │ ├── bigquery.rb │ │ │ ├── concerns │ │ │ │ └── transform_path.rb │ │ │ ├── internal_search_processor.rb │ │ │ ├── internal_search_service.rb │ │ │ ├── user_feedback_processor.rb │ │ │ ├── user_feedback_service.rb │ │ │ ├── views_and_navigation_processor.rb │ │ │ └── views_and_navigation_service.rb │ │ └── main │ │ │ ├── main_processor.rb │ │ │ └── metrics_processor.rb │ ├── finders │ │ ├── aggregations.rb │ │ ├── all_document_types.rb │ │ ├── all_organisations.rb │ │ ├── content.rb │ │ ├── document_children.rb │ │ ├── find_series.rb │ │ ├── select_view.rb │ │ └── series.rb │ ├── healthchecks │ │ ├── concerns │ │ │ └── deactivable.rb │ │ ├── daily_metrics_check.rb │ │ ├── etl_metric_values.rb │ │ ├── monthly_aggregations.rb │ │ └── search_aggregations.rb │ ├── monitor │ │ ├── aggregations.rb │ │ ├── dimensions.rb │ │ ├── etl.rb │ │ ├── facts.rb │ │ └── messages.rb │ ├── specific_months.rb │ └── streams │ │ ├── consumer.rb │ │ ├── grow_dimension.rb │ │ ├── handlers │ │ ├── base_handler.rb │ │ ├── multipart_handler.rb │ │ └── single_item_handler.rb │ │ ├── message_processor_job.rb │ │ ├── messages │ │ ├── base_message.rb │ │ ├── factory.rb │ │ ├── multipart_message.rb │ │ └── single_item_message.rb │ │ └── parent_child │ │ ├── base_parser.rb │ │ ├── links_processor.rb │ │ ├── manual_section_manual_parser.rb │ │ ├── manual_sections_parser.rb │ │ ├── parent_child_links_parser.rb │ │ └── parser.rb ├── models │ ├── aggregations.rb │ ├── aggregations │ │ ├── materialized_view.rb │ │ ├── monthly_metric.rb │ │ ├── search_last_month.rb │ │ ├── search_last_six_months.rb │ │ ├── search_last_thirty_days.rb │ │ ├── search_last_three_months.rb │ │ └── search_last_twelve_months.rb │ ├── application_record.rb │ ├── concerns │ │ ├── trace_and_recoverable.rb │ │ └── traceable.rb │ ├── dimensions.rb │ ├── dimensions │ │ ├── date.rb │ │ ├── edition.rb │ │ └── month.rb │ ├── document_type.rb │ ├── events.rb │ ├── events │ │ ├── feedex.rb │ │ ├── ga.rb │ │ └── publishing_api.rb │ ├── facts.rb │ ├── facts │ │ ├── edition.rb │ │ └── metric.rb │ ├── metric.rb │ ├── organisation.rb │ └── user.rb ├── serializers │ ├── document_type_serializer.rb │ └── organisation_serializer.rb └── views │ ├── api │ └── documents │ │ └── children.json.jbuilder │ ├── content │ └── show.json.jbuilder │ ├── document_type │ └── index.json.jbuilder │ ├── organisation │ └── index.json.jbuilder │ └── single_item │ └── show.json.jbuilder ├── bin ├── brakeman ├── rails ├── rake ├── rspec ├── rubocop ├── setup ├── spring └── yarn ├── config.ru ├── config ├── application.rb ├── boot.rb ├── brakeman.ignore ├── database.yml ├── environment.rb ├── environments │ ├── development.rb │ ├── production.rb │ └── test.rb ├── initializers │ ├── ams.rb │ ├── application_controller_renderer.rb │ ├── assets.rb │ ├── backtrace_silencers.rb │ ├── content_security_policy.rb │ ├── cookies_serializer.rb │ ├── date_formats.rb │ ├── filter_parameter_logging.rb │ ├── govuk_date_formats.rb │ ├── inflections.rb │ ├── mime_types.rb │ ├── permissions_policy.rb │ ├── prometheus.rb │ └── wrap_parameters.rb ├── locales │ └── en.yml ├── metrics.yml ├── puma.rb ├── routes.rb ├── secrets.yml ├── sidekiq.yml └── spring.rb ├── db ├── migrate │ ├── 20161122145459_create_organisations.rb │ ├── 20161123141301_create_content_items.rb │ ├── 20161130153206_add_public_updated_at_to_content_items.rb │ ├── 20161201111152_add_link_to_content_items.rb │ ├── 20161202101420_add_title_to_content_items.rb │ ├── 20161212124434_add_index_to_content_items.rb │ ├── 20161212124642_add_index_to_organisations.rb │ ├── 20161214104355_add_title_to_organisation.rb │ ├── 20161215145222_add_document_type_to_content_items.rb │ ├── 20161221134137_add_description_to_content_items.rb │ ├── 20170105100634_add_number_of_views_to_content_items.rb │ ├── 20170111104312_rename_link_to_base_path.rb │ ├── 20170112173524_set_default_value_for_number_of_views.rb │ ├── 20170117171347_rename_pageviews_to_uniquepageviews.rb │ ├── 20170203113501_create_content_items_organisations_join_table.rb │ ├── 20170206153248_migrate_many_to_many_content_items_organisation.rb │ ├── 20170206153437_remove_organisation_reference_from_content_items.rb │ ├── 20170221150721_create_taxonomies.rb │ ├── 20170227153415_add_pdf_count_to_content_item.rb │ ├── 20170301154536_rename_page_views_to_one_month_page_views.rb │ ├── 20170301165728_add_six_months_column.rb │ ├── 20170307151842_add_index_to_content_items_title.rb │ ├── 20170314110425_create_join_table_content_item_taxonomy.rb │ ├── 20170316110649_create_groups.rb │ ├── 20170318110819_add_content_ids_to_groups.rb │ ├── 20170327105838_add_content_id_to_organisations.rb │ ├── 20170515134126_create_users.rb │ ├── 20170517133236_create_links.rb │ ├── 20170518121026_create_audits.rb │ ├── 20170518123000_create_questions.rb │ ├── 20170518125955_create_responses.rb │ ├── 20170526095145_seed_questions.rb │ ├── 20170605161124_rename_taxonomies_to_taxons.rb │ ├── 20170606105748_create_join_table_group_content_item.rb │ ├── 20170606110152_remove_content_ids_list_from_groups.rb │ ├── 20170606163123_create_taxonomy_projects.rb │ ├── 20170606163951_create_taxonomy_todos.rb │ ├── 20170607101716_create_themes.rb │ ├── 20170607101811_create_subthemes.rb │ ├── 20170607102042_create_inventory_rules.rb │ ├── 20170607125158_create_terms.rb │ ├── 20170626100039_add_publishing_app_to_content_item.rb │ ├── 20170626133926_create_report_rows.rb │ ├── 20170626153519_re_seed_questions.rb │ ├── 20170706143916_create_join_table_for_todos_terms.rb │ ├── 20170706143917_add_status_to_taxonomy_todos.rb │ ├── 20170728202520_update_type_column_for_questions.rb │ ├── 20170804092444_add_locale_to_content_items.rb │ ├── 20170808081001_add_qualitative_measures_to_audits.rb │ ├── 20170810074052_create_allocations.rb │ ├── 20170824142643_drop_groups.rb │ ├── 20170824143605_drop_join_table_groups_content_items.rb │ ├── 20170824145426_drop_organisations.rb │ ├── 20170824145545_drop_organisations_items_join_table.rb │ ├── 20170824145959_drop_taxons_items_join_table.rb │ ├── 20170824150019_drop_taxons.rb │ ├── 20170909211813_add_redirect_urls_to_audits.rb │ ├── 20170912052906_drop_inventory_rules.rb │ ├── 20170912052917_drop_subthemes.rb │ ├── 20170912052927_drop_themes.rb │ ├── 20171207164824_add_foreign_keys_to_allocations.rb │ ├── 20171218120450_add_word_files_count_to_content_item.rb │ ├── 20171220091851_create_dimensions_dates.rb │ ├── 20171222101252_create_dimensions_organisations.rb │ ├── 20171228132858_create_dimensions_items.rb │ ├── 20171229163406_create_facts_metrics.rb │ ├── 20180103115527_delete_content_from_dimensions_items.rb │ ├── 20180103141736_create_dimensions_items_temps.rb │ ├── 20180103161730_add_latest_to_dimensions_items.rb │ ├── 20180105112615_add_index_to_items_dimension.rb │ ├── 20180118132842_drop_terms.rb │ ├── 20180118133606_drop_taxonomy_todos_terms.rb │ ├── 20180118133918_drop_taxonomy_todos.rb │ ├── 20180118134135_drop_taxonomy_projects.rb │ ├── 20180119144707_drop_allocations.rb │ ├── 20180119145749_drop_audits.rb │ ├── 20180119150347_drop_questions.rb │ ├── 20180119150908_drop_responses.rb │ ├── 20180119151925_drop_report_rows.rb │ ├── 20180128183042_add_pageviews_to_facts_metrics.rb │ ├── 20180131111547_remove_dimensions_organisation_id_from_facts_metrics.rb │ ├── 20180131112209_drop_dimensions_organisations.rb │ ├── 20180131112642_change_dimensions_items_fields.rb │ ├── 20180131114516_change_dimensions_items_temps_fields.rb │ ├── 20180205162236_remove_organisation_id_from_dimension_items_and_dimension_items_temps.rb │ ├── 20180207103905_add_raw_json_to_dimensions_item.rb │ ├── 20180210123458_create_events_gas.rb │ ├── 20180210154836_add_index_dimension_items_lates_base_path.rb │ ├── 20180210231645_add_default_values_for_ga.rb │ ├── 20180214145821_add_number_of_pdfs_to_dimensions_items.rb │ ├── 20180216114348_add_dirty_to_dimension_items.rb │ ├── 20180219151742_add_metadata_fields_to_dimensions_item.rb │ ├── 20180220163110_add_number_of_word_files_to_dimensions_item.rb │ ├── 20180221151608_add_status_to_dimensions_items.rb │ ├── 20180223142258_create_events_feedexes.rb │ ├── 20180226111742_add_number_of_issues_to_facts_metrics.rb │ ├── 20180301230712_add_quality_metrics_to_dimensions_items.rb │ ├── 20180305111459_drop_unused_indexes_to_reduce_space.rb │ ├── 20180305112605_remove_table_dimensions_items_temp.rb │ ├── 20180305161014_add_odyssey_metrics.rb │ ├── 20180307113805_rename_column_dimensions_items_content_purpose_supertype.rb │ ├── 20180309120223_add_primary_org_fields_to_dimensions_item.rb │ ├── 20180309144813_add_index_dimensions_items_organisation.rb │ ├── 20180309144814_rename_column_dirty_to_outdated.rb │ ├── 20180309144815_remove_links.rb │ ├── 20180309144816_remove_content_items.rb │ ├── 20180315144816_rename_number_of_issues_to_feedex_comments.rb │ ├── 20180319161814_add_content_hash_to_dimensions_item.rb │ ├── 20180321152146_add_locale_to_dimensions_items.rb │ ├── 20180322134606_drop_indexes_created_without_migration.rb │ ├── 20180322140016_add_indexes_for_master_process.rb │ ├── 20180323170720_add_oudated_at_to_dimensions_items.rb │ ├── 20180417102741_add_payload_version_to_dimensions_items.rb │ ├── 20180418135558_add_is_this_useful_columns_to_events_gas.rb │ ├── 20180418135632_add_is_this_useful_columns_to_facts_metrics.rb │ ├── 20180425092626_create_facts_editions.rb │ ├── 20180425100640_remove_outdated_from_items.rb │ ├── 20180425131954_add_process_name_to_ga_events.rb │ ├── 20180425132405_process_name_is_not_null.rb │ ├── 20180430100333_add_internal_search_to_events_gas.rb │ ├── 20180430100349_add_internal_search_to_facts_metrics.rb │ ├── 20180430104334_add_unique_index_items.rb │ ├── 20180430110607_delete_latest_index_items.rb │ ├── 20180508161332_remove_quality_metrics_from_dimensions_items.rb │ ├── 20180509100804_remove_metadata_from_facts_editions.rb │ ├── 20180514085910_add_additional_metrics_to_events_gas_and_facts_metrics.rb │ ├── 20180514152920_add_constraint_on_duplicate_events.rb │ ├── 20180515104926_add_content_purpose_supergroup_to_dimensions_items.rb │ ├── 20180515105428_add_content_purpose_subgroup_to_dimensions_items.rb │ ├── 20180518133900_change_events_gas_page_views_to_default_to_zero.rb │ ├── 20180522103512_add_index_on_base_path_to_dimensions_items.rb │ ├── 20180607203406_remove_content_hash_from_dimensions_item.rb │ ├── 20180607204735_remove_status_from_dimensions_item.rb │ ├── 20180607205553_add_schema_name_to_dimensions_item.rb │ ├── 20180608124756_remove_default_values_in_editions.rb │ ├── 20180608125059_remove_default_values_for_locale_in_items.rb │ ├── 20180610073937_add_not_null_constratint_to_facts_editions.rb │ ├── 20180610074451_add_not_null_constratint_to_facts_metrics.rb │ ├── 20180610075700_add_unique_index_to_facts_editions.rb │ ├── 20180610080335_add_unique_index_to_facts_metrics.rb │ ├── 20180610081321_add_default_not_null_to_dimensions_items.rb │ ├── 20180610081625_update_indexes_dimensions_items.rb │ ├── 20180625120718_add_content_to_dimensions_items.rb │ ├── 20180626085042_create_publishing_api_events.rb │ ├── 20180626085229_add_reference_to_events_from_dimensions_items.rb │ ├── 20180711100526_rename_column_dimensions_items_event_id.rb │ ├── 20180712115111_rename_dimensions_item_content_to_document_text.rb │ ├── 20180718161302_remove_publishing_api_events.rb │ ├── 20180719133916_add_columns_to_dimensions_items.rb │ ├── 20180725102859_rename_dimensions_items_links_column.rb │ ├── 20180725103453_add_default_to_dimensions_items_expanded_links.rb │ ├── 20180725104938_remove_index_base_path_latest.rb │ ├── 20180801075913_add_bounces_and_time_on_page_to_events_gas_and_facts_metrics.rb │ ├── 20180820115550_remove_raw_json_from_dimensions_item.rb │ ├── 20180821135700_remove_expanded_links_column.rb │ ├── 20180822144907_add_satisfaction_score_to_facts_metric.rb │ ├── 20180824105759_delete_old_edition_metrics.rb │ ├── 20180831101255_add_index_to_dimensions_items_on_latest.rb │ ├── 20180920150009_add_content_uuid_to_dimensions_item.rb │ ├── 20180921122451_add_index_dimensions_items_content_uuid.rb │ ├── 20180921155800_add_index_for_content_list.rb │ ├── 20180924093138_restore_raw_json_to_dimensions_item.rb │ ├── 20180926091756_add_default_value_to_feedback_metrics.rb │ ├── 20180926100046_rename_content_uuid_to_warehouse_item_id.rb │ ├── 20180926105207_add_constraints_to_satisfaction_score_columns.rb │ ├── 20180928141301_populate_warehouse_item_id.rb │ ├── 20180928150637_add_not_null_constraint_to_warehouse_item_id.rb │ ├── 20181001125635_rename_facts_metrics_columns.rb │ ├── 20181001174718_rename_facts_edition_columns.rb │ ├── 20181002110036_rename_events_gas_columns.rb │ ├── 20181004070753_rename_primary_organisation_content_id_to_content_id.rb │ ├── 20181004113705_rename_dimensions_item_to_dimensions_editions.rb │ ├── 20181004115155_rename_fact_edition_dimensions_item_id_to_dimensions_edition_id.rb │ ├── 20181004120457_rename_facts_metrics_dimensions_item_id_to_dimensions_edition_id.rb │ ├── 20181009133439_add_index_for_organisation_list.rb │ ├── 20181015093842_add_index_dimensions_editions_document_type.rb │ ├── 20181025132726_add_index_to_facts_metrics_on_edition_id.rb │ ├── 20181025133812_remove_duplicated_warehouse_item_ids_in_metrics.rb │ ├── 20181025134740_add_unique_index_warehouse_id_latest.rb │ ├── 20181025135539_remove_duplicated_base_paths_metrics.rb │ ├── 20181025135947_add_unique_index_on_latest_basepath.rb │ ├── 20181029112946_add_withdrawn_and_historical.rb │ ├── 20181030120956_add_withdrawn_and_historical_constraints.rb │ ├── 20181031112754_create_dimensions_months.rb │ ├── 20181031124930_create_aggregations_monthly_metrics.rb │ ├── 20181101123329_add_foreign_key_to_monthly_aggregation.rb │ ├── 20181101163401_add_withdrawn_constraints.rb │ ├── 20181101163411_add_historical_constraints.rb │ ├── 20181105152020_create_aggregations_search_last_thirty_days.rb │ ├── 20181105160339_add_index_to_last_thirty_days.rb │ ├── 20181106114625_create_aggregations_search_last_months.rb │ ├── 20181106121337_add_index_to_last_month_aggregations.rb │ ├── 20181106170339_create_aggregations_search_last_three_months.rb │ ├── 20181106171806_add_index_to_last_three_months_aggregations.rb │ ├── 20181107165445_create_aggregations_search_last_six_months.rb │ ├── 20181107165733_add_index_to_last_six_months_aggregations.rb │ ├── 20181107170231_create_aggregations_search_last_twelve_months.rb │ ├── 20181107170246_add_index_to_last_twelve_months_aggregations.rb │ ├── 20181120164728_add_not_null_constraint_locale.rb │ ├── 20181121135744_add_full_text_index_title.rb │ ├── 20181122105601_add_full_text_index_base_path.rb │ ├── 20181122134132_remove_default_zeros_from_facts_metrics.rb │ ├── 20181122142744_remove_default_zeros_from_events_gas.rb │ ├── 20181122152546_remove_default_zeros_from_aggregations_monthly_metrics.rb │ ├── 20181126095403_recreate_publishing_api_events.rb │ ├── 20181126120610_update_aggregations_search_last_thirty_days_to_version_2.rb │ ├── 20181126135420_update_aggregations_search_last_six_months_to_version_2.rb │ ├── 20181126151050_update_aggregations_search_last_three_months_to_version_2.rb │ ├── 20181126152543_remove_raw_json_from_dimension_editions.rb │ ├── 20181126152658_update_aggregations_search_last_twelve_months_to_version_2.rb │ ├── 20181129111315_add_default_zeros_to_events_gas.rb │ ├── 20181129115927_add_default_to_events_feedex.rb │ ├── 20181129125513_add_not_null_constraint_to_facts_metrics.rb │ ├── 20181204140034_add_unique_index_to_last_thirty_days.rb │ ├── 20181204140917_refresh_view_last_thirty_days.rb │ ├── 20181204142533_add_unique_index_to_last_month.rb │ ├── 20181204142554_refresh_view_last_month.rb │ ├── 20181204143719_add_unique_index_to_last_three_months.rb │ ├── 20181204143802_refresh_view_last_three_months.rb │ ├── 20181204144322_add_unique_index_to_last_six_months.rb │ ├── 20181204144330_refresh_view_last_six_months.rb │ ├── 20181204145152_add_unique_index_to_last_twelve_months.rb │ ├── 20181204145159_refresh_view_last_twelve_months.rb │ ├── 20181205154240_update_aggregations_search_last_thirty_days_to_version_3.rb │ ├── 20181205165620_update_aggregations_search_last_three_months_to_version_3.rb │ ├── 20181206115804_update_aggregations_search_last_six_months_to_version_3.rb │ ├── 20181206120629_update_aggregations_search_last_twelve_months_to_version_3.rb │ ├── 20181211114147_update_aggregations_search_last_thirty_days_to_version_4.rb │ ├── 20181211115100_update_aggregations_search_last_months_to_version_2.rb │ ├── 20181211115357_update_aggregations_search_last_three_months_to_version_4.rb │ ├── 20181211115944_update_aggregations_search_last_six_months_to_version_4.rb │ ├── 20181211120306_update_aggregations_search_last_twelve_months_to_version_4.rb │ ├── 20190104120051_add_not_null_constraint_and_index_to_document_types.rb │ ├── 20190116142924_remove_content_purpose_document_supertype_column_from_dimensions_editions.rb │ ├── 20190117123913_remove_bounce_rate.rb │ ├── 20190117125756_remove_avg_page_time.rb │ ├── 20190118140551_add_acronym_to_dimensions_editions.rb │ ├── 20190118145538_update_acronym_for_organisations.rb │ ├── 20190204102543_remove_content_supergroups_and_subgroups.rb │ ├── 20190204150122_add_reading_time_to_facts_edition.rb │ ├── 20190204171432_update_reading_time.rb │ ├── 20190207143213_add_index_facts_metrics_pviews.rb │ ├── 20190207143221_add_index_facts_metrics_upviews.rb │ ├── 20190207143240_add_index_facts_metrics_searches.rb │ ├── 20190207143301_add_index_facts_metrics_useful_no.rb │ ├── 20190207143313_add_index_facts_metrics_useful_feedex.rb │ ├── 20190213115655_update_aggregations_search_last_thirty_days_to_version_5.rb │ ├── 20190213141732_update_aggregations_search_last_months_to_version_3.rb │ ├── 20190213141743_update_aggregations_search_last_three_months_to_version_5.rb │ ├── 20190213141753_update_aggregations_search_last_six_months_to_version_5.rb │ ├── 20190213141801_update_aggregations_search_last_twelve_months_to_version_5.rb │ ├── 20190213153521_clean_up_unused_indexes.rb │ ├── 20190213154809_add_indexes_view_aggregations_search_last_thirty_days.rb │ ├── 20190213155742_add_indexes_view_aggregations_search_last_month.rb │ ├── 20190213155920_add_indexes_view_aggregations_search_last_three_months.rb │ ├── 20190213155931_add_indexes_view_aggregations_search_last_six_months.rb │ ├── 20190213155940_add_indexes_view_aggregations_search_last_twelve_months.rb │ ├── 20190213162703_add_unique_indexes_view_aggregations_search_views.rb │ ├── 20190214113305_add_index_views_using_warehouse_item_id.rb │ ├── 20190218113118_update_aggregations_search_last_thirty_days_to_version_6.rb │ ├── 20190218121734_update_aggregations_search_last_months_to_version_4.rb │ ├── 20190218122820_update_aggregations_search_last_three_months_to_version_6.rb │ ├── 20190218123800_update_aggregations_search_last_six_months_to_version_6.rb │ ├── 20190218124214_update_aggregations_search_last_twelve_months_to_version_6.rb │ ├── 20190220161718_change_upviews_and_warehouse_item_id_indexes.rb │ ├── 20190225113933_add_index_to_monthly_aggregation_created_at.rb │ ├── 20190304151614_remove_default_satisfaction.rb │ ├── 20190311163406_update_aggregations_search_last_twelve_months_to_version_7.rb │ ├── 20190311164157_update_aggregations_search_last_six_months_to_version_7.rb │ ├── 20190311164340_update_aggregations_search_last_three_months_to_version_7.rb │ ├── 20190311164541_update_aggregations_search_last_months_to_version_5.rb │ ├── 20190311164701_update_aggregations_search_last_thirty_days_to_version_7.rb │ ├── 20190312140803_update_aggregations_search_last_thirty_days_to_version_8.rb │ ├── 20190312141411_update_aggregations_search_last_months_to_version_6.rb │ ├── 20190312141423_update_aggregations_search_last_six_months_to_version_8.rb │ ├── 20190312141431_update_aggregations_search_last_three_months_to_version_8.rb │ ├── 20190312141439_update_aggregations_search_last_twelve_months_to_version_8.rb │ ├── 20190402143148_rename_latest_column_to_live.rb │ ├── 20190603080748_rename_dimensions_edition_organisation_id.rb │ ├── 20190603083348_update_aggregations_search_last_months_to_version_7.rb │ ├── 20190603083456_update_aggregations_search_last_thirty_days_to_version_9.rb │ ├── 20190603083504_update_aggregations_search_last_three_months_to_version_9.rb │ ├── 20190603083515_update_aggregations_search_last_twelve_months_to_version_9.rb │ ├── 20190603083643_update_aggregations_search_last_six_months_to_version_9.rb │ ├── 20190603111746_restore_organisation_related_view_indexes.rb │ ├── 20190603133313_add_organisation_ids_to_dimensions_edition.rb │ ├── 20190603145810_update_aggregations_search_last_months_to_version_8.rb │ ├── 20190603145828_update_aggregations_search_last_thirty_days_to_version_10.rb │ ├── 20190603145838_update_aggregations_search_last_three_months_to_version_10.rb │ ├── 20190603145851_update_aggregations_search_last_twelve_months_to_version_10.rb │ ├── 20190603145901_update_aggregations_search_last_six_months_to_version_10.rb │ ├── 20190604105034_create_lower_index_on_base_path.rb │ ├── 20190612150721_add_parent_id_to_dimension_edition.rb │ ├── 20190613082656_add_sibling_order_to_dimensions_edition.rb │ └── 20190613112120_add_child_sort_order_to_dimensions_editions.rb ├── schema.rb └── views │ ├── aggregations_search_last_months_v01.sql │ ├── aggregations_search_last_months_v02.sql │ ├── aggregations_search_last_months_v03.sql │ ├── aggregations_search_last_months_v04.sql │ ├── aggregations_search_last_months_v05.sql │ ├── aggregations_search_last_months_v06.sql │ ├── aggregations_search_last_months_v07.sql │ ├── aggregations_search_last_months_v08.sql │ ├── aggregations_search_last_six_months_v01.sql │ ├── aggregations_search_last_six_months_v02.sql │ ├── aggregations_search_last_six_months_v03.sql │ ├── aggregations_search_last_six_months_v04.sql │ ├── aggregations_search_last_six_months_v05.sql │ ├── aggregations_search_last_six_months_v06.sql │ ├── aggregations_search_last_six_months_v07.sql │ ├── aggregations_search_last_six_months_v08.sql │ ├── aggregations_search_last_six_months_v09.sql │ ├── aggregations_search_last_six_months_v10.sql │ ├── aggregations_search_last_thirty_days_v01.sql │ ├── aggregations_search_last_thirty_days_v02.sql │ ├── aggregations_search_last_thirty_days_v03.sql │ ├── aggregations_search_last_thirty_days_v04.sql │ ├── aggregations_search_last_thirty_days_v05.sql │ ├── aggregations_search_last_thirty_days_v06.sql │ ├── aggregations_search_last_thirty_days_v07.sql │ ├── aggregations_search_last_thirty_days_v08.sql │ ├── aggregations_search_last_thirty_days_v09.sql │ ├── aggregations_search_last_thirty_days_v10.sql │ ├── aggregations_search_last_three_months_v01.sql │ ├── aggregations_search_last_three_months_v02.sql │ ├── aggregations_search_last_three_months_v03.sql │ ├── aggregations_search_last_three_months_v04.sql │ ├── aggregations_search_last_three_months_v05.sql │ ├── aggregations_search_last_three_months_v06.sql │ ├── aggregations_search_last_three_months_v07.sql │ ├── aggregations_search_last_three_months_v08.sql │ ├── aggregations_search_last_three_months_v09.sql │ ├── aggregations_search_last_three_months_v10.sql │ ├── aggregations_search_last_twelve_months_v01.sql │ ├── aggregations_search_last_twelve_months_v02.sql │ ├── aggregations_search_last_twelve_months_v03.sql │ ├── aggregations_search_last_twelve_months_v04.sql │ ├── aggregations_search_last_twelve_months_v05.sql │ ├── aggregations_search_last_twelve_months_v06.sql │ ├── aggregations_search_last_twelve_months_v07.sql │ ├── aggregations_search_last_twelve_months_v08.sql │ ├── aggregations_search_last_twelve_months_v09.sql │ └── aggregations_search_last_twelve_months_v10.sql ├── docs ├── arch │ ├── adr-000-document-architectural-decisions.md │ ├── adr-003-use-google-tag-manager.md │ ├── adr-005-split-audit-tool-cpm.md │ ├── adr-006-track-metrics-via-time-dimension.md │ ├── adr-007-etl-publishing-api-content-store.md │ ├── adr-008-focus-on-english-content.md │ ├── adr-009-consume-from-publishing-api.md │ ├── adr-009-track-metrics-by-basepath.md │ ├── adr-010-track-content-items-in-all-languages.md │ ├── adr-011-new-app-for-ui-management.md │ ├── adr-012-use-sidekiq-for-proessing-rabbitmq-messages.md │ ├── adr-013-store-json-from-events-that-carry-content-changes.md │ ├── adr-014-track-attribute-changes-per-basepath.md │ ├── adr-015-convention-for-data-warehouse-unique-id.md │ ├── adr-016-rename-content-performance-manager.md │ └── images │ │ └── content_etl.png ├── data-warehouse-schema.md ├── data-warehouse-what-and-why.md ├── etl-process.md ├── google_analytics_setup.md ├── how_to_handle_errors.md ├── images │ ├── fig-01-no-trends.png │ ├── fig-02-with-trends.png │ ├── fig-03-comparisons.png │ ├── fig-04-predictions.png │ └── schema.png ├── import_production_data.md ├── processing_publishing_api_messages.md └── requeue_all_content.md ├── lib └── tasks │ ├── .keep │ ├── data_migrations.rake │ ├── editions.rake │ ├── etl.rake │ └── publishing_api_consumer.rake ├── log └── .keep ├── openapi.yaml ├── public ├── apple-touch-icon-precomposed.png ├── apple-touch-icon.png ├── favicon.ico └── robots.txt ├── spec ├── domain │ ├── api │ │ ├── content_request_spec.rb │ │ └── document_children_request_spec.rb │ ├── etl │ │ ├── aggregations │ │ │ ├── monthly_spec.rb │ │ │ └── search_spec.rb │ │ ├── edition │ │ │ ├── content │ │ │ │ ├── body_content_spec.rb │ │ │ │ ├── contact_spec.rb │ │ │ │ ├── email_alert_signup_spec.rb │ │ │ │ ├── finder_email_signup_spec.rb │ │ │ │ ├── finder_spec.rb │ │ │ │ ├── generic_with_links_spec.rb │ │ │ │ ├── hmrc_manual_spec.rb │ │ │ │ ├── licence_spec.rb │ │ │ │ ├── local_transaction_spec.rb │ │ │ │ ├── mainstream_spec.rb │ │ │ │ ├── need_spec.rb │ │ │ │ ├── no_content_spec.rb │ │ │ │ ├── parser_spec.rb │ │ │ │ ├── place_spec.rb │ │ │ │ ├── reading_time_spec.rb │ │ │ │ ├── service_manual_homepage_spec.rb │ │ │ │ ├── service_manual_service_toolkit_spec.rb │ │ │ │ ├── service_manual_standard_spec.rb │ │ │ │ ├── service_manual_topic_spec.rb │ │ │ │ ├── service_sign_in_spec.rb │ │ │ │ ├── statistics_announcement_spec.rb │ │ │ │ ├── step_by_step_spec.rb │ │ │ │ ├── taxon_spec.rb │ │ │ │ ├── transaction_spec.rb │ │ │ │ ├── travel_advice_index_spec.rb │ │ │ │ └── unpublished_spec.rb │ │ │ └── metadata │ │ │ │ ├── number_of_pdfs_spec.rb │ │ │ │ └── number_of_word_files_spec.rb │ │ ├── feedex │ │ │ ├── processor_spec.rb │ │ │ └── service_spec.rb │ │ ├── ga │ │ │ ├── bigquery_spec.rb │ │ │ ├── concerns │ │ │ │ └── transform_path_spec.rb │ │ │ ├── internal_search_processor_spec.rb │ │ │ ├── internal_search_service_spec.rb │ │ │ ├── user_feedback_processor_spec.rb │ │ │ ├── user_feedback_service_spec.rb │ │ │ ├── views_and_navigation_processor_spec.rb │ │ │ └── views_and_navigation_service_spec.rb │ │ └── main │ │ │ ├── main_processor_spec.rb │ │ │ └── metrics_spec.rb │ ├── finders │ │ ├── aggregations_spec.rb │ │ ├── all_document_types_spec.rb │ │ ├── all_organisations_spec.rb │ │ ├── content_spec.rb │ │ ├── document_children_spec.rb │ │ ├── find_series_spec.rb │ │ ├── select_view_spec.rb │ │ └── series_spec.rb │ ├── healthchecks │ │ ├── daily_metrics_check_spec.rb │ │ ├── etl_metric_values_spec.rb │ │ ├── monthly_aggregations_spec.rb │ │ └── search_aggregations_spec.rb │ ├── monitor │ │ ├── aggregations_spec.rb │ │ ├── dimensions_spec.rb │ │ ├── etl_spec.rb │ │ ├── facts_spec.rb │ │ └── messages_spec.rb │ └── streams │ │ ├── consumer_spec.rb │ │ ├── grow_dimension_spec.rb │ │ ├── handlers │ │ └── single_item_handler_spec.rb │ │ ├── message_processor_job_spec.rb │ │ └── messages │ │ ├── multipart_message_spec.rb │ │ └── single_item_message_spec.rb ├── factories │ ├── dimensions_date.rb │ ├── dimensions_months.rb │ ├── editions.rb │ ├── facts_editions.rb │ ├── feedexes.rb │ ├── ga_events.rb │ ├── messages.rb │ ├── metrics.rb │ ├── monthly_metrics.rb │ ├── publishing_api_events.rb │ └── user.rb ├── integration │ ├── item │ │ ├── extract_content_all_schemas_spec.rb │ │ └── import_edition_metrics_spec.rb │ ├── master │ │ └── daily_metrics_spec.rb │ └── streams │ │ ├── all_schemas_spec.rb │ │ ├── errors_spec.rb │ │ ├── multiple │ │ ├── grow_dimension_spec.rb │ │ └── parent_child_spec.rb │ │ ├── parent_child │ │ └── links_processor_spec.rb │ │ └── single │ │ ├── grow_dimension_spec.rb │ │ ├── html_publication_parents_spec.rb │ │ ├── manuals_parent_child_spec.rb │ │ └── publishing_editions_spec.rb ├── models │ ├── aggregations │ │ ├── monthly_metric_spec.rb │ │ ├── search_aggregations_spec.rb │ │ ├── search_last_six_months_spec.rb │ │ ├── search_last_thirty_days_spec.rb │ │ ├── search_last_three_months_spec.rb │ │ └── search_last_twelve_months_spec.rb │ ├── dimensions │ │ ├── date_spec.rb │ │ ├── edition_spec.rb │ │ └── month_spec.rb │ ├── document_type_spec.rb │ ├── facts │ │ ├── edition_spec.rb │ │ └── metric_spec.rb │ ├── metric_spec.rb │ └── organisation_spec.rb ├── rails_helper.rb ├── requests │ ├── api │ │ ├── single_page_spec.rb │ │ └── v1 │ │ │ ├── content_spec.rb │ │ │ ├── documents_spec.rb │ │ │ ├── healthcheck_spec.rb │ │ │ └── metrics_spec.rb │ ├── document_types_spec.rb │ ├── healthcheck_metrics_spec.rb │ ├── healthcheck_search_spec.rb │ └── organisations_spec.rb ├── routing │ ├── documents_routing_spec.rb │ └── single_page_routing_spec.rb ├── support │ ├── aggregations_support.rb │ ├── authentication.rb │ ├── google_analytics_requests.rb │ ├── govuk_test.rb │ ├── matchers │ │ └── be_sorted_by_attribute_matcher.rb │ ├── publishing_event_processing_spec_helper.rb │ ├── schemas_iterator.rb │ └── shared │ │ ├── shared_aggregation_view.rb │ │ ├── shared_api_response.rb │ │ ├── shared_healthcheck_enabled.rb │ │ ├── shared_masterialized_views.rb │ │ ├── shared_messages.rb │ │ └── shared_traps_and_logs.rb └── tasks │ ├── editions_spec.rb │ ├── etl_spec.rb │ └── publishing_api_spec.rb ├── tmp └── .keep └── vendor └── assets ├── javascripts └── .keep └── stylesheets └── .keep /.dockerignore: -------------------------------------------------------------------------------- 1 | .dockerignore 2 | .git 3 | .gitignore 4 | .github 5 | Dockerfile 6 | Procfile 7 | README.md 8 | coverage 9 | docs 10 | features 11 | log 12 | node_modules 13 | script 14 | spec 15 | test 16 | tmp 17 | vendor 18 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # Description 2 | What are the changes? Why are you making these changes? 3 | 4 | --- 5 | # Review Checklist 6 | * [ ] Changes in scope. 7 | * [ ] Added/updated unit tests. 8 | * [ ] Added/updated feature tests. 9 | * [ ] Added/updated relevant documentation. 10 | * [ ] Added to Trello card. 11 | 12 | ⚠️ This repo is Continuously Deployed: make sure you [follow the guidance](https://docs.publishing.service.gov.uk/manual/development-pipeline.html#merge-your-own-pull-request) ⚠️ 13 | 14 | Follow [these steps](https://guides.rubyonrails.org/upgrading_ruby_on_rails.html) if you are doing a Rails upgrade. 15 | 16 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: bundler 4 | directory: / 5 | schedule: 6 | interval: daily 7 | -------------------------------------------------------------------------------- /.github/workflows/actionlint.yml: -------------------------------------------------------------------------------- 1 | name: Lint GitHub Actions 2 | on: 3 | push: 4 | paths: ['.github/**'] 5 | jobs: 6 | actionlint: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | with: 11 | show-progress: false 12 | - uses: alphagov/govuk-infrastructure/.github/actions/actionlint@main 13 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | workflow_dispatch: 5 | workflow_run: 6 | workflows: [CI] 7 | types: [completed] 8 | branches: [main] 9 | 10 | jobs: 11 | release: 12 | if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' 13 | name: Release 14 | uses: alphagov/govuk-infrastructure/.github/workflows/release.yml@main 15 | secrets: 16 | GH_TOKEN: ${{ secrets.GOVUK_CI_GITHUB_API_TOKEN }} 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files for more about ignoring files. 2 | # 3 | # If you find yourself ignoring temporary files generated by your text editor 4 | # or operating system, you probably want to add a global ignore instead: 5 | # git config --global core.excludesfile '~/.gitignore_global' 6 | 7 | # Ignore bundler config. 8 | 9 | .bundle 10 | .byebug_history 11 | .env 12 | .DS_Store 13 | config/local_env.yml 14 | coverage 15 | 16 | /log/* 17 | /tmp/* 18 | !/log/.keep 19 | !/tmp/.keep 20 | -------------------------------------------------------------------------------- /.govuk_dependabot_merger.yml: -------------------------------------------------------------------------------- 1 | api_version: 2 2 | defaults: 3 | auto_merge: true 4 | update_external_dependencies: true 5 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --require rails_helper 3 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | inherit_gem: 2 | rubocop-govuk: 3 | - "config/default.yml" 4 | - "config/rails.yml" 5 | 6 | inherit_mode: 7 | merge: 8 | - Exclude 9 | 10 | # ************************************************************** 11 | # TRY NOT TO ADD OVERRIDES IN THIS FILE 12 | # 13 | # This repo is configured to follow the RuboCop GOV.UK styleguide. 14 | # Any rules you override here will cause this repo to diverge from 15 | # the way we write code in all other GOV.UK repos. 16 | # 17 | # See https://github.com/alphagov/rubocop-govuk/blob/main/CONTRIBUTING.md 18 | # ************************************************************** 19 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 3.3.1 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | services: 3 | - postgresql 4 | rvm: 5 | - 2.3.1 6 | script: 7 | - bundle exec govuk-lint-ruby app config db lib spec --format clang 8 | - bundle exec rake db:create db:migrate db:test:prepare 9 | - bundle exec rspec 10 | addons: 11 | postgresql: "9.3" 12 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG ruby_version=3.3 2 | ARG base_image=ghcr.io/alphagov/govuk-ruby-base:$ruby_version 3 | ARG builder_image=ghcr.io/alphagov/govuk-ruby-builder:$ruby_version 4 | 5 | 6 | FROM --platform=$TARGETPLATFORM $builder_image AS builder 7 | 8 | WORKDIR $APP_HOME 9 | COPY Gemfile* .ruby-version ./ 10 | RUN bundle install 11 | COPY . . 12 | RUN bootsnap precompile --gemfile . 13 | 14 | 15 | FROM --platform=$TARGETPLATFORM $base_image 16 | 17 | ENV GOVUK_APP_NAME=content-data-api 18 | 19 | WORKDIR $APP_HOME 20 | COPY --from=builder $BUNDLE_PATH $BUNDLE_PATH 21 | COPY --from=builder $BOOTSNAP_CACHE_DIR $BOOTSNAP_CACHE_DIR 22 | COPY --from=builder $APP_HOME . 23 | 24 | USER app 25 | CMD ["puma"] 26 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | default-worker: bundle exec sidekiq -C config/sidekiq.yml 2 | publishing-api-consumer: bundle exec rake publishing_api:consumer 3 | bulk-import-publishing-api-consumer: bundle exec rake publishing_api:bulk_import_consumer 4 | web: bundle exec puma -t 5:5 -p ${PORT:-3000} -e ${RACK_ENV:-development} 5 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # Add your own tasks in files placed in lib/tasks ending in .rake, 2 | # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. 3 | 4 | require_relative "config/application" 5 | 6 | Rails.application.load_tasks 7 | 8 | desc "Lint ruby files" 9 | task lint: :environment do 10 | sh "bundle exec rubocop --parallel app config lib spec" 11 | end 12 | 13 | task default: %i[spec lint] 14 | -------------------------------------------------------------------------------- /app/assets/config/manifest.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphagov/content-data-api/a0f2ff6f1f4f664344e2ecb5c991ac2cbf0bd5e5/app/assets/config/manifest.js -------------------------------------------------------------------------------- /app/controllers/api/document_types_controller.rb: -------------------------------------------------------------------------------- 1 | class Api::DocumentTypesController < Api::BaseController 2 | def index 3 | @document_types = Finders::AllDocumentTypes.run 4 | render root: "document_types", json: @document_types 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /app/controllers/api/healthcheck_controller.rb: -------------------------------------------------------------------------------- 1 | class Api::HealthcheckController < Api::BaseController 2 | skip_before_action :authenticate_user! 3 | skip_before_action :set_cache_headers 4 | 5 | def index 6 | render json: { status: "ok" } 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /app/controllers/api/metrics_controller.rb: -------------------------------------------------------------------------------- 1 | class Api::MetricsController < Api::BaseController 2 | def index 3 | items = Metric.find_all 4 | render json: items 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /app/controllers/api/organisations_controller.rb: -------------------------------------------------------------------------------- 1 | class Api::OrganisationsController < Api::BaseController 2 | def index 3 | @organisations = Finders::AllOrganisations.run 4 | render root: "organisations", json: @organisations 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /app/controllers/application_controller.rb: -------------------------------------------------------------------------------- 1 | class ApplicationController < ActionController::Base 2 | include GDS::SSO::ControllerMethods 3 | 4 | protect_from_forgery with: :exception 5 | before_action :authenticate_user! 6 | 7 | # Raise ActionController::UnpermittedParameters if there are unpermitted 8 | # parameters, so we can return better errors for invalid API requests 9 | ActionController::Parameters.action_on_unpermitted_parameters = :raise 10 | end 11 | -------------------------------------------------------------------------------- /app/domain/api/single_page_request.rb: -------------------------------------------------------------------------------- 1 | class Api::SinglePageRequest < Api::BaseRequest 2 | attr_reader :metrics, :base_path 3 | 4 | def initialize(params) 5 | super(params) 6 | @base_path = params[:base_path] 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /app/domain/etl/aggregations/search.rb: -------------------------------------------------------------------------------- 1 | class Etl::Aggregations::Search 2 | include Traceable 3 | 4 | def self.process(*args) 5 | new(*args).process 6 | end 7 | 8 | def process 9 | do_process(::Aggregations::SearchLastThirtyDays) 10 | do_process(::Aggregations::SearchLastMonth) 11 | do_process(::Aggregations::SearchLastThreeMonths) 12 | do_process(::Aggregations::SearchLastSixMonths) 13 | do_process(::Aggregations::SearchLastTwelveMonths) 14 | end 15 | 16 | private 17 | 18 | def do_process(klass) 19 | name = klass.name.demodulize.underscore 20 | time(process: name) do 21 | ActiveRecord::Base.transaction do 22 | klass.refresh 23 | end 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/email_alert_signup.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::EmailAlertSignup 2 | def parse(json) 3 | html = [] 4 | breadcrumbs = json.dig("details", "breadcrumbs") 5 | unless breadcrumbs.nil? 6 | breadcrumbs.each do |crumb| 7 | html << crumb["title"] 8 | end 9 | end 10 | html << json.dig("details", "summary") 11 | html.join(" ") 12 | end 13 | 14 | def schemas 15 | %w[email_alert_signup] 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/finder.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::Finder 2 | def parse(json) 3 | html = [] 4 | html << json["title"] unless json["title"].nil? 5 | children = json.dig("links", "children") 6 | unless children.nil? 7 | children.each do |child| 8 | html << child["title"] 9 | html << child["description"] 10 | end 11 | end 12 | html.join(" ") unless html.empty? 13 | end 14 | 15 | def schemas 16 | %w[finder] 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/finder_email_signup.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::FinderEmailSignup 2 | def parse(json) 3 | html = [] 4 | choices = json.dig("details", "email_signup_choice") 5 | unless choices.nil? 6 | choices.each do |choice| 7 | html << choice["radio_button_name"] 8 | end 9 | end 10 | html << json["description"] 11 | html.join(" ") 12 | end 13 | 14 | def schemas 15 | %w[finder_email_signup] 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/generic_with_links.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::GenericWithLinks 2 | def parse(json) 3 | html = [] 4 | links = json.dig("details", "external_related_links") 5 | unless links.nil? 6 | links.each do |link| 7 | html << link["title"] 8 | end 9 | html.join(" ") 10 | end 11 | end 12 | 13 | def schemas 14 | %w[generic_with_external_related_links] 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/hmrc_manual.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::HMRCManual 2 | def parse(json) 3 | html = [] 4 | html << json["title"] unless json["title"].nil? 5 | html << json["description"] unless json["description"].nil? 6 | groups = json.dig("details", "child_section_groups") 7 | unless groups.nil? 8 | groups.each do |group| 9 | html << group["title"] unless group["title"].nil? 10 | sections = group["child_sections"] 11 | next if sections.nil? 12 | 13 | sections.each do |section| 14 | html << section["section_id"] 15 | html << section["title"] 16 | end 17 | end 18 | end 19 | html.join(" ") unless html.empty? 20 | end 21 | 22 | def schemas 23 | %w[hmrc_manual] 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/licence.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::Licence 2 | def parse(json) 3 | body = json.dig("details", "licence_overview") 4 | return if body.blank? 5 | 6 | if body.is_a?(Array) 7 | body_by_content_type = body.map(&:values).to_h 8 | body = body_by_content_type.fetch("text/html", nil) 9 | end 10 | 11 | body 12 | end 13 | 14 | def schemas 15 | %w[licence] 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/local_transaction.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::LocalTransaction 2 | def parse(json) 3 | html = [] 4 | html << json.dig("details", "introduction") 5 | html << json.dig("details", "need_to_know") 6 | html << json.dig("details", "more_information") 7 | html.join(" ") 8 | end 9 | 10 | def schemas 11 | %w[local_transaction] 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/mainstream.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::Mainstream 2 | def parse(json) 3 | html = [] 4 | html << json["title"] unless json["title"].nil? 5 | html << json["description"] unless json["description"].nil? 6 | 7 | children = json.dig("links", "children") 8 | if children.present? 9 | children.each do |child| 10 | html << child["title"] 11 | end 12 | end 13 | 14 | related_topics = json.dig("links", "related_topics") 15 | if related_topics.present? 16 | related_topics.each do |topic| 17 | html << topic["title"] 18 | end 19 | end 20 | 21 | html.compact.join(" ") 22 | end 23 | 24 | def schemas 25 | %w[mainstream_browse_page] 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/need.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::Need 2 | def parse(json) 3 | html = [] 4 | html << json.dig("details", "role") 5 | html << json.dig("details", "goal") 6 | html << json.dig("details", "benefit") 7 | html.join(" ") 8 | end 9 | 10 | def schemas 11 | %w[need] 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/parts.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::Parts 2 | def parse_subpage(json, subpage_path) 3 | parts = json.dig("details", "parts") 4 | return if parts.nil? 5 | 6 | current_part = parts.find { |part| part["slug"] == subpage_path } 7 | return if current_part.nil? 8 | 9 | body = current_part["body"] 10 | 11 | if body.is_a?(Array) 12 | body_by_content_type = body.map(&:values).to_h 13 | body = body_by_content_type.fetch("text/html", nil) 14 | end 15 | 16 | body 17 | end 18 | 19 | def schemas 20 | %w[guide] 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/place.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::Place 2 | def parse(json) 3 | html = [] 4 | html << json.dig("details", "introduction") 5 | html << json.dig("details", "more_information") 6 | html.join(" ") 7 | end 8 | 9 | def schemas 10 | %w[place] 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/service_manual_homepage.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::ServiceManualHomepage 2 | def parse(json) 3 | html = [] 4 | html << json["title"] 5 | html << json["description"] 6 | 7 | children = json.dig("links", "children") 8 | if children.present? 9 | children.each do |child| 10 | html << child["title"] 11 | html << child["description"] 12 | end 13 | end 14 | 15 | html.compact.join(" ") 16 | end 17 | 18 | def schemas 19 | %w[service_manual_homepage] 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/service_manual_service_toolkit.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::ServiceManualServiceToolkit 2 | def parse(json) 3 | html = [] 4 | items = json.dig("details", "collections") 5 | unless items.nil? 6 | items.each do |item| 7 | html << item["title"] 8 | html << item["description"] 9 | links = item["links"] 10 | next if links.nil? 11 | 12 | links.each do |link| 13 | html << link["title"] 14 | html << link["description"] 15 | end 16 | end 17 | html.join(" ") 18 | end 19 | end 20 | 21 | def schemas 22 | %w[service_manual_service_toolkit] 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/service_manual_standard.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::ServiceManualStandard 2 | def parse(json) 3 | html = [] 4 | html << json["title"] 5 | html << json.dig("details", "body") 6 | children = json.dig("links", "children") 7 | unless children.nil? 8 | children.each do |child| 9 | html << child["title"] 10 | html << child["description"] 11 | end 12 | end 13 | html.join(" ") 14 | end 15 | 16 | def schemas 17 | %w[service_manual_service_standard] 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/service_manual_topic.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::ServiceManualTopic 2 | def parse(json) 3 | html = [] 4 | html << json["description"] 5 | groups = json.dig("details", "groups") 6 | unless groups.nil? 7 | groups.each do |group| 8 | html << group["name"] 9 | html << group["description"] 10 | end 11 | end 12 | html.join(" ") 13 | end 14 | 15 | def schemas 16 | %w[service_manual_topic] 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/service_sign_in.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::ServiceSignIn 2 | def parse(json) 3 | html = [] 4 | html << json.dig("details", "choose_sign_in", "title") 5 | options = json.dig("details", "choose_sign_in", "options") 6 | unless options.nil? 7 | options.each do |option| 8 | html << option["text"] 9 | html << option["hint_text"] 10 | end 11 | end 12 | html << json.dig("details", "create_new_account", "title") 13 | html << json.dig("details", "create_new_account", "body") 14 | html.join(" ") 15 | end 16 | 17 | def schemas 18 | %w[service_sign_in] 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/statistics_announcement.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::StatisticsAnnouncement 2 | def parse(json) 3 | html = [] 4 | html << json["description"] 5 | html << json.dig("details", "display_date") 6 | html << json.dig("details", "state") 7 | html.join(" ") 8 | end 9 | 10 | def schemas 11 | %w[statistics_announcement] 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/taxon.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::Taxon 2 | def parse(json) 3 | html = [] 4 | html << json["description"] 5 | children = json.dig("links", "child_taxons") 6 | unless children.nil? 7 | children.each do |child| 8 | html << child["title"] 9 | html << child["description"] 10 | end 11 | end 12 | html.join(" ") 13 | end 14 | 15 | def schemas 16 | %w[taxon] 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/transaction.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::Transaction 2 | def parse(json) 3 | html = [] 4 | html << json.dig("details", "introductory_paragraph") 5 | html << json.dig("details", "start_button_text") 6 | html << json.dig("details", "will_continue_on") 7 | html << json.dig("details", "more_information") 8 | html.join(" ") 9 | end 10 | 11 | def schemas 12 | %w[transaction] 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/travel_advice.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::TravelAdvice 2 | def parse_subpage(json, subpage_path) 3 | if summary_page?(json, subpage_path) 4 | json.dig("details", "summary").find { |x| x["content_type"] == "text/html" }.try(:dig, "content") 5 | else 6 | Etl::Edition::Content::Parsers::Parts.new.parse_subpage(json, subpage_path) 7 | end 8 | end 9 | 10 | def schemas 11 | %w[travel_advice] 12 | end 13 | 14 | private 15 | 16 | def summary_page?(json, subpage_path) 17 | json["base_path"] == subpage_path 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/travel_advice_index.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::TravelAdviceIndex 2 | def parse(json) 3 | html = [] 4 | children = json.dig("links", "children") 5 | return if children.nil? 6 | 7 | children.each do |child| 8 | country = child.dig("country", "name") 9 | html << country unless country.nil? 10 | end 11 | html.join(" ") 12 | end 13 | 14 | def schemas 15 | %w[travel_advice_index] 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/parsers/unpublished.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::Parsers::Unpublished 2 | def parse(json) 3 | json.dig("details", "explanation") 4 | end 5 | 6 | def schemas 7 | %w[unpublishing gone] 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /app/domain/etl/edition/content/reading_time.rb: -------------------------------------------------------------------------------- 1 | class Etl::Edition::Content::ReadingTime 2 | def self.calculate(words) 3 | # 200 words per minute. Rounds up to 1 anything less than 1 minute. 4 | (words / 200.00).ceil 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /app/domain/etl/edition/metadata/number_of_pdfs.rb: -------------------------------------------------------------------------------- 1 | module Etl::Edition::Metadata 2 | class NumberOfPdfs 3 | def self.parse(raw_json) 4 | NumberOfFiles.number_of_files(raw_json, %w[pdf]) 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /app/domain/etl/edition/metadata/number_of_word_files.rb: -------------------------------------------------------------------------------- 1 | module Etl::Edition::Metadata 2 | class NumberOfWordFiles 3 | def self.parse(raw_json) 4 | NumberOfFiles.number_of_files(raw_json, %w[doc docx docm]) 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /app/domain/etl/ga/bigquery.rb: -------------------------------------------------------------------------------- 1 | require "google/cloud/bigquery" 2 | require "googleauth" 3 | 4 | class Etl::GA::Bigquery 5 | include Google::Auth 6 | 7 | def self.build 8 | new.build 9 | end 10 | 11 | def build 12 | credentials = { 13 | "client_email" => ENV["BIGQUERY_CLIENT_EMAIL"], 14 | "private_key" => ENV["BIGQUERY_PRIVATE_KEY"], 15 | } 16 | 17 | Google::Cloud::Bigquery.new( 18 | project_id: ENV["BIGQUERY_PROJECT"], 19 | credentials: Google::Auth::ServiceAccountCredentials.make_creds( 20 | json_key_io: StringIO.new(credentials.to_json), 21 | scope: ["https://www.googleapis.com/auth/bigquery"], 22 | ), 23 | ) 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /app/domain/finders/all_organisations.rb: -------------------------------------------------------------------------------- 1 | class Finders::AllOrganisations 2 | def self.run(locale: "en") 3 | new.run(locale) 4 | end 5 | 6 | def run(locale) 7 | editions = find_all(locale) 8 | editions.map { |edition| new_organisation(edition) } 9 | end 10 | 11 | private 12 | 13 | def new_organisation(org) 14 | Organisation.new( 15 | id: org[:content_id], 16 | name: org[:title], 17 | acronym: org[:acronym], 18 | ) 19 | end 20 | 21 | def find_all(locale) 22 | Dimensions::Edition.live 23 | .select(:content_id, :title, :locale, :acronym) 24 | .where(document_type: "organisation", locale:) 25 | .order(:title) 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /app/domain/finders/series.rb: -------------------------------------------------------------------------------- 1 | class Finders::Series 2 | attr_reader :metric_name 3 | 4 | def initialize(metric_name, all_metrics) 5 | @metric_name = metric_name 6 | @all_metrics = all_metrics 7 | end 8 | 9 | def total 10 | time_series.reduce(0) { |total, time_point| total + time_point[:value] } 11 | end 12 | 13 | def time_series 14 | time_series = @all_metrics.map do |metric| 15 | { 16 | date: metric.dimensions_date_id.to_s, 17 | value: metric.public_send(metric_name), 18 | } 19 | end 20 | time_series.sort_by { |point| point[:date] } 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /app/domain/healthchecks/concerns/deactivable.rb: -------------------------------------------------------------------------------- 1 | require "active_support/concern" 2 | 3 | module Healthchecks::Concerns::Deactivable 4 | extend ActiveSupport::Concern 5 | 6 | included do 7 | def enabled? 8 | healthchecks_enabled? && within_time_range? 9 | end 10 | 11 | private 12 | 13 | def healthchecks_enabled? 14 | ENV["ETL_HEALTHCHECK_ENABLED"] == "1" 15 | end 16 | 17 | def within_time_range? 18 | time = Time.zone.now 19 | 20 | time.hour >= Integer(ENV["ETL_HEALTHCHECK_ENABLED_FROM_HOUR"]) 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /app/domain/healthchecks/daily_metrics_check.rb: -------------------------------------------------------------------------------- 1 | module Healthchecks 2 | class DailyMetricsCheck 3 | include Concerns::Deactivable 4 | 5 | def name 6 | :daily_metrics 7 | end 8 | 9 | def status 10 | metrics.any? ? :ok : :critical 11 | end 12 | 13 | def message 14 | "ETL :: no daily metrics for yesterday" if status == :critical 15 | end 16 | 17 | private 18 | 19 | def metrics 20 | @metrics ||= Facts::Metric.where(dimensions_date_id: Date.yesterday) 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /app/domain/healthchecks/etl_metric_values.rb: -------------------------------------------------------------------------------- 1 | module Healthchecks 2 | class EtlMetricValues 3 | include ActiveModel::Model 4 | include Concerns::Deactivable 5 | 6 | attr_accessor :metric 7 | 8 | def self.build(metric) 9 | new(metric:) 10 | end 11 | 12 | def name 13 | "etl_metric_values_#{metric}".to_sym 14 | end 15 | 16 | def status 17 | if number_of_metric_values.positive? 18 | :ok 19 | else 20 | :critical 21 | end 22 | end 23 | 24 | def message 25 | "ETL :: no #{metric} for yesterday" if status == :critical 26 | end 27 | 28 | private 29 | 30 | def number_of_metric_values 31 | Facts::Metric.for_yesterday.where("#{metric} > 0").count 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /app/domain/healthchecks/monthly_aggregations.rb: -------------------------------------------------------------------------------- 1 | module Healthchecks 2 | class MonthlyAggregations 3 | include Concerns::Deactivable 4 | 5 | def name 6 | :aggregations 7 | end 8 | 9 | def status 10 | aggregations.positive? ? :ok : :critical 11 | end 12 | 13 | def message 14 | "ETL :: no monthly aggregations of metrics for yesterday" if status == :critical 15 | end 16 | 17 | private 18 | 19 | def aggregations 20 | @aggregations ||= Aggregations::MonthlyMetric.where("created_at > ?", Time.zone.today).count 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /app/domain/healthchecks/search_aggregations.rb: -------------------------------------------------------------------------------- 1 | module Healthchecks 2 | class SearchAggregations 3 | include ActiveModel::Model 4 | include Concerns::Deactivable 5 | 6 | attr_accessor :range 7 | 8 | def self.build(range) 9 | new(range:) 10 | end 11 | 12 | def name 13 | "search_#{range}".to_sym 14 | end 15 | 16 | def status 17 | searches.positive? ? :ok : :critical 18 | end 19 | 20 | def message 21 | "ETL :: no #{range.to_s.humanize} searches updated from yesterday" if status == :critical 22 | end 23 | 24 | private 25 | 26 | def klass_name 27 | "Aggregations::Search#{range.to_s.camelize}".constantize 28 | end 29 | 30 | def searches 31 | @searches ||= klass_name.where(updated_at: Time.zone.today).count 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /app/domain/monitor/messages.rb: -------------------------------------------------------------------------------- 1 | class Monitor::Messages 2 | def self.run(routing_key) 3 | new.run(routing_key) 4 | end 5 | 6 | def run(routing_key) 7 | statsd_for_messages!(routing_key) 8 | end 9 | 10 | def self.increment_discarded 11 | GovukStatsd.increment("monitor.messages.discarded") 12 | end 13 | 14 | private 15 | 16 | def statsd_for_messages!(routing_key) 17 | GovukStatsd.increment(monitoring_code(routing_key)) 18 | end 19 | 20 | def monitoring_code(routing_key) 21 | routing_key_type = routing_key.split(".").last 22 | "monitor.messages.#{routing_key_type}" 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /app/domain/specific_months.rb: -------------------------------------------------------------------------------- 1 | module SpecificMonths 2 | YEARS = (2018..Time.zone.today.year).to_a.freeze 3 | MONTHS = Date::MONTHNAMES.compact.freeze 4 | 5 | VALID_SPECIFIC_MONTHS = YEARS.flat_map do |year| 6 | MONTHS.map do |month| 7 | "#{month.downcase}-#{year}" 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /app/domain/streams/consumer.rb: -------------------------------------------------------------------------------- 1 | module Streams 2 | class Consumer 3 | def process(message) 4 | if valid_routing_key?(message) 5 | Streams::MessageProcessorJob.perform_later(message.payload, message.delivery_info.routing_key) 6 | else 7 | Monitor::Messages.increment_discarded 8 | end 9 | 10 | message.ack 11 | rescue StandardError => e 12 | GovukError.notify(e) 13 | message.discard 14 | end 15 | 16 | private 17 | 18 | def valid_routing_key?(message) 19 | routing_key = message.delivery_info.routing_key 20 | 21 | ROUTING_KEYS.any? { |suffix| routing_key.ends_with?(suffix) } 22 | end 23 | 24 | ROUTING_KEYS = %w[links major minor unpublish bulk.data-warehouse].freeze 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /app/domain/streams/message_processor_job.rb: -------------------------------------------------------------------------------- 1 | require_dependency("streams/messages/factory") 2 | 3 | module Streams 4 | class MessageProcessorJob < ActiveJob::Base # rubocop:disable Rails/ApplicationJob 5 | retry_on ActiveRecord::RecordNotUnique, wait: 5.seconds, attempts: 3 6 | 7 | def perform(payload, routing_key) 8 | message = Streams::Messages::Factory.build(payload, routing_key) 9 | 10 | if message.invalid? 11 | Monitor::Messages.increment_discarded 12 | return 13 | end 14 | 15 | Monitor::Messages.run(routing_key) 16 | 17 | ActiveRecord::Base.transaction do 18 | message.handler.process 19 | end 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /app/domain/streams/messages/factory.rb: -------------------------------------------------------------------------------- 1 | module Streams::Messages 2 | class Factory 3 | def self.build(payload, routing_key) 4 | if MultipartMessage.is_multipart?(payload) 5 | MultipartMessage.new(payload, routing_key) 6 | else 7 | SingleItemMessage.new(payload, routing_key) 8 | end 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /app/domain/streams/parent_child/base_parser.rb: -------------------------------------------------------------------------------- 1 | class Streams::ParentChild::BaseParser 2 | def self.to_warehouse_id(content_id, locale) 3 | "#{content_id}:#{locale}" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /app/domain/streams/parent_child/manual_section_manual_parser.rb: -------------------------------------------------------------------------------- 1 | class Streams::ParentChild::ManualSectionManualParser < Streams::ParentChild::BaseParser 2 | DOCUMENT_TYPES = %w[manual_section].freeze 3 | 4 | def self.get_parent_id(payload) 5 | manuals = payload.dig("expanded_links", "manual") 6 | return nil if manuals.blank? 7 | 8 | to_warehouse_id(manuals.first["content_id"], manuals.first["locale"]) 9 | end 10 | 11 | def self.get_children_ids(_) 12 | [] 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /app/domain/streams/parent_child/manual_sections_parser.rb: -------------------------------------------------------------------------------- 1 | class Streams::ParentChild::ManualSectionsParser < Streams::ParentChild::BaseParser 2 | DOCUMENT_TYPES = %w[manual].freeze 3 | 4 | def self.get_children_ids(payload) 5 | sections = payload.dig("expanded_links", "sections") || [] 6 | sections.map { |h| to_warehouse_id(h["content_id"], h["locale"]) } 7 | end 8 | 9 | def self.get_parent_id(_) 10 | nil 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /app/models/aggregations.rb: -------------------------------------------------------------------------------- 1 | module Aggregations 2 | def self.table_name_prefix 3 | "aggregations_" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /app/models/aggregations/monthly_metric.rb: -------------------------------------------------------------------------------- 1 | class Aggregations::MonthlyMetric < ApplicationRecord 2 | belongs_to :dimensions_month, class_name: "Dimensions::Month" 3 | belongs_to :dimensions_edition, class_name: "Dimensions::Edition" 4 | 5 | validates :dimensions_month, presence: true 6 | validates :dimensions_edition, presence: true 7 | end 8 | -------------------------------------------------------------------------------- /app/models/aggregations/search_last_month.rb: -------------------------------------------------------------------------------- 1 | class Aggregations::SearchLastMonth < Aggregations::MaterializedView 2 | end 3 | -------------------------------------------------------------------------------- /app/models/aggregations/search_last_six_months.rb: -------------------------------------------------------------------------------- 1 | class Aggregations::SearchLastSixMonths < Aggregations::MaterializedView 2 | end 3 | -------------------------------------------------------------------------------- /app/models/aggregations/search_last_thirty_days.rb: -------------------------------------------------------------------------------- 1 | class Aggregations::SearchLastThirtyDays < Aggregations::MaterializedView 2 | end 3 | -------------------------------------------------------------------------------- /app/models/aggregations/search_last_three_months.rb: -------------------------------------------------------------------------------- 1 | class Aggregations::SearchLastThreeMonths < Aggregations::MaterializedView 2 | end 3 | -------------------------------------------------------------------------------- /app/models/aggregations/search_last_twelve_months.rb: -------------------------------------------------------------------------------- 1 | class Aggregations::SearchLastTwelveMonths < Aggregations::MaterializedView 2 | end 3 | -------------------------------------------------------------------------------- /app/models/application_record.rb: -------------------------------------------------------------------------------- 1 | class ApplicationRecord < ActiveRecord::Base 2 | self.abstract_class = true 3 | end 4 | -------------------------------------------------------------------------------- /app/models/dimensions.rb: -------------------------------------------------------------------------------- 1 | module Dimensions 2 | def self.table_name_prefix 3 | "dimensions_" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /app/models/document_type.rb: -------------------------------------------------------------------------------- 1 | class DocumentType 2 | include ActiveModel::Model 3 | include ActiveModel::Serialization 4 | 5 | IGNORED_TYPES = %w[redirect gone vanish unpublishing need].freeze 6 | 7 | attr_accessor :id, :name 8 | end 9 | -------------------------------------------------------------------------------- /app/models/events.rb: -------------------------------------------------------------------------------- 1 | module Events 2 | def self.table_name_prefix 3 | "events_" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /app/models/events/feedex.rb: -------------------------------------------------------------------------------- 1 | class Events::Feedex < ApplicationRecord 2 | end 3 | -------------------------------------------------------------------------------- /app/models/events/ga.rb: -------------------------------------------------------------------------------- 1 | class Events::GA < ApplicationRecord 2 | enum process_name: { user_feedback: 0, views: 1, searches: 2 } 3 | end 4 | -------------------------------------------------------------------------------- /app/models/events/publishing_api.rb: -------------------------------------------------------------------------------- 1 | class Events::PublishingApi < ApplicationRecord 2 | self.table_name = "publishing_api_events" 3 | has_many :dimensions_editions, class_name: "Dimensions::Edition" 4 | end 5 | -------------------------------------------------------------------------------- /app/models/facts.rb: -------------------------------------------------------------------------------- 1 | module Facts 2 | def self.table_name_prefix 3 | "facts_" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /app/models/facts/edition.rb: -------------------------------------------------------------------------------- 1 | class Facts::Edition < ApplicationRecord 2 | belongs_to :dimensions_date, class_name: "Dimensions::Date" 3 | belongs_to :dimensions_edition, class_name: "Dimensions::Edition" 4 | 5 | validates :dimensions_date, presence: true 6 | validates :dimensions_edition, presence: true 7 | 8 | scope :between, 9 | lambda { |from, to| 10 | joins(:dimensions_date) 11 | .where("dimensions_dates.date BETWEEN ? AND ?", from, to) 12 | } 13 | 14 | def clone_for!(new_dim_edition, new_date) 15 | new_facts_edition = dup 16 | new_facts_edition.assign_attributes(dimensions_edition: new_dim_edition, dimensions_date: new_date) 17 | new_facts_edition.save! 18 | new_facts_edition 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /app/models/organisation.rb: -------------------------------------------------------------------------------- 1 | class Organisation 2 | include ActiveModel::Model 3 | include ActiveModel::Serialization 4 | 5 | attr_accessor :id, :name, :acronym 6 | end 7 | -------------------------------------------------------------------------------- /app/models/user.rb: -------------------------------------------------------------------------------- 1 | class User < ApplicationRecord 2 | include GDS::SSO::User 3 | 4 | serialize :permissions, coder: YAML, type: Array 5 | 6 | def organisation 7 | @organisation ||= item.find_by(content_id: organisation_content_id) 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /app/serializers/document_type_serializer.rb: -------------------------------------------------------------------------------- 1 | class DocumentTypeSerializer < ActiveModel::Serializer 2 | attributes :id, :name 3 | end 4 | -------------------------------------------------------------------------------- /app/serializers/organisation_serializer.rb: -------------------------------------------------------------------------------- 1 | class OrganisationSerializer < ActiveModel::Serializer 2 | attributes :id, :name, :acronym 3 | end 4 | -------------------------------------------------------------------------------- /app/views/api/documents/children.json.jbuilder: -------------------------------------------------------------------------------- 1 | json.parent_base_path @parent.base_path 2 | json.documents @documents do |document| 3 | json.base_path document.base_path 4 | json.title document.title 5 | json.primary_organisation_id document.primary_organisation_id 6 | json.document_type document.document_type 7 | json.sibling_order document.sibling_order 8 | json.upviews document.upviews 9 | json.pviews document.pviews 10 | json.feedex document.feedex 11 | json.useful_yes document.useful_yes 12 | json.useful_no document.useful_no 13 | json.satisfaction document.satisfaction 14 | json.searches document.searches 15 | end 16 | -------------------------------------------------------------------------------- /app/views/document_type/index.json.jbuilder: -------------------------------------------------------------------------------- 1 | json.document_types @document_types.map(&:document_type) 2 | -------------------------------------------------------------------------------- /app/views/organisation/index.json.jbuilder: -------------------------------------------------------------------------------- 1 | json.organisations @organisations do |organisation| 2 | json.title organisation[:title] 3 | json.organisation_id organisation[:organisation_id] 4 | end 5 | -------------------------------------------------------------------------------- /app/views/single_item/show.json.jbuilder: -------------------------------------------------------------------------------- 1 | json.metadata do 2 | json.merge! @live_edition.metadata 3 | end 4 | 5 | json.number_of_related_content @live_edition.related_content_count 6 | 7 | json.time_period do 8 | json.from @from 9 | json.to @to 10 | end 11 | 12 | json.time_series_metrics @time_series_metrics do |series| 13 | json.name series.metric_name 14 | json.total @aggregations.fetch(series.metric_name.to_sym) 15 | json.time_series series.time_series do |time_point| 16 | json.date time_point[:date] 17 | json.value time_point[:value] 18 | end 19 | end 20 | 21 | json.edition_metrics @edition_metrics do |metric_name| 22 | json.name metric_name 23 | json.value @live_edition.facts_edition.attributes[metric_name] 24 | end 25 | -------------------------------------------------------------------------------- /bin/brakeman: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require "rubygems" 3 | require "bundler/setup" 4 | 5 | ARGV.unshift("--ensure-latest") 6 | 7 | load Gem.bin_path("brakeman", "brakeman") 8 | -------------------------------------------------------------------------------- /bin/rails: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | APP_PATH = File.expand_path("../config/application", __dir__) 3 | require_relative "../config/boot" 4 | require "rails/commands" 5 | -------------------------------------------------------------------------------- /bin/rake: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require_relative "../config/boot" 3 | require "rake" 4 | Rake.application.run 5 | -------------------------------------------------------------------------------- /bin/rspec: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | begin 3 | load File.expand_path("spring", __dir__) 4 | rescue LoadError => e 5 | raise unless e.message.include?("spring") 6 | end 7 | require "bundler/setup" 8 | load Gem.bin_path("rspec-core", "rspec") 9 | -------------------------------------------------------------------------------- /bin/rubocop: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require "rubygems" 3 | require "bundler/setup" 4 | 5 | # explicit rubocop config increases performance slightly while avoiding config confusion. 6 | ARGV.unshift("--config", File.expand_path("../.rubocop.yml", __dir__)) 7 | 8 | load Gem.bin_path("rubocop", "rubocop") 9 | -------------------------------------------------------------------------------- /bin/spring: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | if !defined?(Spring) && [nil, "development", "test"].include?(ENV["RAILS_ENV"]) 3 | gem "bundler" 4 | require "bundler" 5 | 6 | # Load Spring without loading other gems in the Gemfile, for speed. 7 | Bundler.locked_gems&.specs&.find { |spec| spec.name == "spring" }&.tap do |spring| 8 | Gem.use_paths Gem.dir, Bundler.bundle_path.to_s, *Gem.path 9 | gem "spring", spring.version 10 | require "spring/binstub" 11 | rescue Gem::LoadError 12 | # Ignore when Spring is not installed. 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /bin/yarn: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | APP_ROOT = File.expand_path("..", __dir__) 3 | Dir.chdir(APP_ROOT) do 4 | yarn = ENV["PATH"].split(File::PATH_SEPARATOR) 5 | .reject { |dir| File.expand_path(dir) == __dir__ } 6 | .product(["yarn", "yarn.cmd", "yarn.ps1"]) 7 | .map { |dir, file| File.expand_path(file, dir) } 8 | .find { |file| File.executable?(file) } 9 | 10 | if yarn 11 | exec yarn, *ARGV 12 | else 13 | warn "Yarn executable was not detected in the system." 14 | warn "Download Yarn at https://yarnpkg.com/en/docs/install" 15 | exit 1 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | # This file is used by Rack-based servers to start the application. 2 | 3 | require_relative "config/environment" 4 | 5 | run Rails.application 6 | Rails.application.load_server 7 | -------------------------------------------------------------------------------- /config/boot.rb: -------------------------------------------------------------------------------- 1 | ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__) 2 | 3 | require "bundler/setup" # Set up gems listed in the Gemfile. 4 | require "bootsnap/setup" # Speed up boot time by caching expensive operations. 5 | -------------------------------------------------------------------------------- /config/database.yml: -------------------------------------------------------------------------------- 1 | default: &default 2 | adapter: postgresql 3 | encoding: unicode 4 | pool: 12 5 | 6 | development: 7 | <<: *default 8 | database: content_performance_manager_development 9 | url: <%= ENV['DEVELOPMENT_DATABASE_URL'] || ENV['DATABASE_URL'] %> 10 | 11 | test: 12 | <<: *default 13 | database: content_performance_manager_test 14 | url: <%= ENV['TEST_DATABASE_URL'] || ENV['DATABASE_URL'] %> 15 | 16 | production: 17 | <<: *default 18 | # Set using the DATABASE_URL environment variable 19 | -------------------------------------------------------------------------------- /config/environment.rb: -------------------------------------------------------------------------------- 1 | # Load the Rails application. 2 | require_relative "application" 3 | 4 | # Initialize the Rails application. 5 | Rails.application.initialize! 6 | -------------------------------------------------------------------------------- /config/initializers/ams.rb: -------------------------------------------------------------------------------- 1 | require "active_model_serializers" 2 | 3 | ActiveModel::Serializer.config.adapter = :json 4 | -------------------------------------------------------------------------------- /config/initializers/application_controller_renderer.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # ActiveSupport::Reloader.to_prepare do 4 | # ApplicationController.renderer.defaults.merge!( 5 | # http_host: 'example.org', 6 | # https: false 7 | # ) 8 | # end 9 | -------------------------------------------------------------------------------- /config/initializers/assets.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Version of your assets, change this if you want to expire all your assets. 4 | Rails.application.config.assets.version = "1.0" 5 | 6 | # Add additional assets to the asset load path. 7 | # Rails.application.config.assets.paths << Emoji.images_path 8 | 9 | # Precompile additional assets. 10 | # application.js, application.css, and all non-JS/CSS in the app/assets 11 | # folder are already added. 12 | # Rails.application.config.assets.precompile += %w( admin.js admin.css ) 13 | -------------------------------------------------------------------------------- /config/initializers/backtrace_silencers.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces. 4 | # Rails.backtrace_cleaner.add_silencer { |line| /my_noisy_library/.match?(line) } 5 | 6 | # You can also remove all the silencers if you're trying to debug a problem that might stem from framework code 7 | # by setting BACKTRACE=1 before calling your invocation, like "BACKTRACE=1 ./bin/rails runner 'MyClass.perform'". 8 | Rails.backtrace_cleaner.remove_silencers! if ENV["BACKTRACE"] 9 | -------------------------------------------------------------------------------- /config/initializers/cookies_serializer.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Specify a serializer for the signed and encrypted cookie jars. 4 | # Valid options are :json, :marshal, and :hybrid. 5 | Rails.application.config.action_dispatch.cookies_serializer = :json 6 | -------------------------------------------------------------------------------- /config/initializers/date_formats.rb: -------------------------------------------------------------------------------- 1 | Date::DATE_FORMATS[:short] = "%d/%m/%y" 2 | 3 | # 2018-12 4 | Date::DATE_FORMATS[:month_edition] = "%04d-%02d" 5 | -------------------------------------------------------------------------------- /config/initializers/filter_parameter_logging.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Configure parameters to be partially matched (e.g. passw matches password) and filtered from the log file. 4 | # Use this to limit dissemination of sensitive information. 5 | # See the ActiveSupport::ParameterFilter documentation for supported notations and behaviors. 6 | Rails.application.config.filter_parameters += %i[ 7 | passw secret token _key crypt salt certificate otp ssn 8 | ] 9 | -------------------------------------------------------------------------------- /config/initializers/govuk_date_formats.rb: -------------------------------------------------------------------------------- 1 | # 1 January 2013 2 | Date::DATE_FORMATS[:govuk_date] = "%-e %B %Y" 3 | 4 | # 1 Jan 2013 5 | Date::DATE_FORMATS[:govuk_date_short] = "%-e %b %Y" 6 | 7 | # 1:15pm, 1 January 2013 8 | Time::DATE_FORMATS[:govuk_date] = "%-I:%M%P, %-e %B %Y" 9 | 10 | # 1:15pm, 1 Jan 2013 11 | Time::DATE_FORMATS[:govuk_date_short] = "%-I:%M%P, %-e %b %Y" 12 | -------------------------------------------------------------------------------- /config/initializers/inflections.rb: -------------------------------------------------------------------------------- 1 | ActiveSupport::Inflector.inflections(:en) do |inflect| 2 | inflect.acronym "AAIB" 3 | inflect.acronym "CMA" 4 | inflect.acronym "DFID" 5 | inflect.acronym "ESI" 6 | inflect.acronym "FOI" 7 | inflect.acronym "HMRC" 8 | inflect.acronym "HTML" 9 | inflect.acronym "MAIB" 10 | inflect.acronym "RAIB" 11 | inflect.acronym "UTAAC" 12 | inflect.acronym "GA" 13 | end 14 | -------------------------------------------------------------------------------- /config/initializers/mime_types.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Add new mime types for use in respond_to blocks: 4 | # Mime::Type.register "text/richtext", :rtf 5 | -------------------------------------------------------------------------------- /config/initializers/permissions_policy.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Define an application-wide HTTP permissions policy. For further 4 | # information see: https://developers.google.com/web/updates/2018/06/feature-policy 5 | 6 | # Rails.application.config.permissions_policy do |policy| 7 | # policy.camera :none 8 | # policy.gyroscope :none 9 | # policy.microphone :none 10 | # policy.usb :none 11 | # policy.fullscreen :self 12 | # policy.payment :self, "https://secure.example.com" 13 | # end 14 | -------------------------------------------------------------------------------- /config/initializers/prometheus.rb: -------------------------------------------------------------------------------- 1 | require "govuk_app_config/govuk_prometheus_exporter" 2 | GovukPrometheusExporter.configure 3 | -------------------------------------------------------------------------------- /config/initializers/wrap_parameters.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # This file contains settings for ActionController::ParamsWrapper which 4 | # is enabled by default. 5 | 6 | # Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array. 7 | ActiveSupport.on_load(:action_controller) do 8 | wrap_parameters format: [:json] 9 | end 10 | 11 | # To enable root element in JSON for ActiveRecord objects. 12 | # ActiveSupport.on_load(:active_record) do 13 | # self.include_root_in_json = true 14 | # end 15 | -------------------------------------------------------------------------------- /config/puma.rb: -------------------------------------------------------------------------------- 1 | require "govuk_app_config/govuk_puma" 2 | GovukPuma.configure_rails(self) 3 | -------------------------------------------------------------------------------- /config/sidekiq.yml: -------------------------------------------------------------------------------- 1 | --- 2 | :concurrency: 6 3 | :queues: 4 | - default 5 | <% if ENV.key?('SIDEKIQ_LOGFILE') %> 6 | :logfile: <%= ENV['SIDEKIQ_LOGFILE'] %> 7 | <% end %> 8 | -------------------------------------------------------------------------------- /config/spring.rb: -------------------------------------------------------------------------------- 1 | Spring.watch( 2 | ".ruby-version", 3 | ".rbenv-vars", 4 | "tmp/restart.txt", 5 | "tmp/caching-dev.txt", 6 | ) 7 | -------------------------------------------------------------------------------- /db/migrate/20161122145459_create_organisations.rb: -------------------------------------------------------------------------------- 1 | class CreateOrganisations < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :organisations do |t| 4 | t.string :slug 5 | 6 | t.timestamps 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20161123141301_create_content_items.rb: -------------------------------------------------------------------------------- 1 | class CreateContentItems < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :content_items do |t| 4 | t.string :content_id 5 | t.references :organisation, foreign_key: true 6 | 7 | t.timestamps 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20161130153206_add_public_updated_at_to_content_items.rb: -------------------------------------------------------------------------------- 1 | class AddPublicUpdatedAtToContentItems < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :content_items, :public_updated_at, :datetime 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20161201111152_add_link_to_content_items.rb: -------------------------------------------------------------------------------- 1 | class AddLinkToContentItems < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :content_items, :link, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20161202101420_add_title_to_content_items.rb: -------------------------------------------------------------------------------- 1 | class AddTitleToContentItems < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :content_items, :title, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20161212124434_add_index_to_content_items.rb: -------------------------------------------------------------------------------- 1 | class AddIndexToContentItems < ActiveRecord::Migration[5.0] 2 | def change 3 | add_index :content_items, :content_id, unique: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20161212124642_add_index_to_organisations.rb: -------------------------------------------------------------------------------- 1 | class AddIndexToOrganisations < ActiveRecord::Migration[5.0] 2 | def change 3 | add_index :organisations, :slug, unique: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20161214104355_add_title_to_organisation.rb: -------------------------------------------------------------------------------- 1 | class AddTitleToOrganisation < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :organisations, :title, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20161215145222_add_document_type_to_content_items.rb: -------------------------------------------------------------------------------- 1 | class AddDocumentTypeToContentItems < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :content_items, :document_type, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20161221134137_add_description_to_content_items.rb: -------------------------------------------------------------------------------- 1 | class AddDescriptionToContentItems < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :content_items, :description, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170105100634_add_number_of_views_to_content_items.rb: -------------------------------------------------------------------------------- 1 | class AddNumberOfViewsToContentItems < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :content_items, :number_of_views, :integer, default: 0 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170111104312_rename_link_to_base_path.rb: -------------------------------------------------------------------------------- 1 | class RenameLinkToBasePath < ActiveRecord::Migration[5.0] 2 | def change 3 | rename_column :content_items, :link, :base_path 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170112173524_set_default_value_for_number_of_views.rb: -------------------------------------------------------------------------------- 1 | class SetDefaultValueForNumberOfViews < ActiveRecord::Migration[5.0] 2 | def up 3 | execute "UPDATE content_items SET number_of_views = 0;" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170117171347_rename_pageviews_to_uniquepageviews.rb: -------------------------------------------------------------------------------- 1 | class RenamePageviewsToUniquepageviews < ActiveRecord::Migration[5.0] 2 | def change 3 | rename_column :content_items, :number_of_views, :unique_page_views 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170203113501_create_content_items_organisations_join_table.rb: -------------------------------------------------------------------------------- 1 | class CreateContentItemsOrganisationsJoinTable < ActiveRecord::Migration[5.0] 2 | def change 3 | create_join_table :content_items, :organisations 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170206153248_migrate_many_to_many_content_items_organisation.rb: -------------------------------------------------------------------------------- 1 | class MigrateManyToManyContentItemsOrganisation < ActiveRecord::Migration[5.0] 2 | class ContentItem < ActiveRecord::Base 3 | belongs_to :organisation 4 | has_and_belongs_to_many :organisations 5 | end 6 | 7 | def change 8 | MigrateManyToManyContentItemsOrganisation::ContentItem.find_each do |content_item| 9 | content_item.organisations << content_item.organisation 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /db/migrate/20170206153437_remove_organisation_reference_from_content_items.rb: -------------------------------------------------------------------------------- 1 | class RemoveOrganisationReferenceFromContentItems < ActiveRecord::Migration[5.0] 2 | def change 3 | remove_reference :content_items, :organisation, foreign_key: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170221150721_create_taxonomies.rb: -------------------------------------------------------------------------------- 1 | class CreateTaxonomies < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :taxonomies do |t| 4 | t.string :content_id 5 | t.string :title 6 | 7 | t.timestamps 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20170227153415_add_pdf_count_to_content_item.rb: -------------------------------------------------------------------------------- 1 | class AddPdfCountToContentItem < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :content_items, :number_of_pdfs, :integer, default: 0 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170301154536_rename_page_views_to_one_month_page_views.rb: -------------------------------------------------------------------------------- 1 | class RenamePageViewsToOneMonthPageViews < ActiveRecord::Migration[5.0] 2 | def change 3 | rename_column :content_items, :unique_page_views, :one_month_page_views 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170301165728_add_six_months_column.rb: -------------------------------------------------------------------------------- 1 | class AddSixMonthsColumn < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :content_items, :six_months_page_views, :integer, default: 0 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170307151842_add_index_to_content_items_title.rb: -------------------------------------------------------------------------------- 1 | class AddIndexToContentItemsTitle < ActiveRecord::Migration[5.0] 2 | def change 3 | add_index :content_items, :title 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170314110425_create_join_table_content_item_taxonomy.rb: -------------------------------------------------------------------------------- 1 | class CreateJoinTableContentItemTaxonomy < ActiveRecord::Migration[5.0] 2 | def change 3 | create_join_table :content_items, :taxonomies do |t| 4 | t.index [:content_item_id] 5 | t.index %i[taxonomy_id content_item_id], name: "index_content_item_taxonomies", unique: true 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20170316110649_create_groups.rb: -------------------------------------------------------------------------------- 1 | class CreateGroups < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :groups do |t| 4 | t.string :slug 5 | t.string :name 6 | t.string :group_type 7 | t.integer :parent_group_id, foreign_key: true, null: true 8 | 9 | t.timestamps 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /db/migrate/20170318110819_add_content_ids_to_groups.rb: -------------------------------------------------------------------------------- 1 | class AddContentIdsToGroups < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :groups, :content_item_ids, :text, array: true, default: [] 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170327105838_add_content_id_to_organisations.rb: -------------------------------------------------------------------------------- 1 | class AddContentIdToOrganisations < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :organisations, :content_id, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170515134126_create_users.rb: -------------------------------------------------------------------------------- 1 | class CreateUsers < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :users do |t| 4 | t.string :name 5 | t.string :email 6 | t.string :uid 7 | t.string :organisation_slug 8 | t.string :organisation_content_id 9 | t.text :permissions 10 | t.boolean :remotely_signed_out, default: false 11 | t.boolean :disabled, default: false 12 | t.timestamps 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /db/migrate/20170517133236_create_links.rb: -------------------------------------------------------------------------------- 1 | class CreateLinks < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :links do |t| 4 | t.string :source_content_id 5 | t.string :link_type 6 | t.string :target_content_id 7 | t.timestamps 8 | end 9 | 10 | add_index :links, :source_content_id 11 | add_index :links, :target_content_id 12 | add_index :links, :link_type 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /db/migrate/20170518121026_create_audits.rb: -------------------------------------------------------------------------------- 1 | class CreateAudits < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :audits do |t| 4 | t.string :content_id, null: false 5 | t.string :uid, null: false 6 | 7 | t.timestamps 8 | end 9 | 10 | add_index :audits, :content_id, unique: true 11 | add_index :audits, :uid 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /db/migrate/20170518123000_create_questions.rb: -------------------------------------------------------------------------------- 1 | class CreateQuestions < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :questions do |t| 4 | t.string :type, null: false 5 | t.text :text, null: false 6 | 7 | t.timestamps 8 | end 9 | 10 | add_index :questions, :type 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /db/migrate/20170518125955_create_responses.rb: -------------------------------------------------------------------------------- 1 | class CreateResponses < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :responses do |t| 4 | t.belongs_to :audit 5 | t.belongs_to :question 6 | t.text :value 7 | 8 | t.timestamps 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrate/20170526095145_seed_questions.rb: -------------------------------------------------------------------------------- 1 | class SeedQuestions < ActiveRecord::Migration[5.0] 2 | def change 3 | # noop 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170605161124_rename_taxonomies_to_taxons.rb: -------------------------------------------------------------------------------- 1 | class RenameTaxonomiesToTaxons < ActiveRecord::Migration[5.1] 2 | def change 3 | rename_table :taxonomies, :taxons 4 | rename_table :content_items_taxonomies, :content_items_taxons 5 | rename_column :content_items_taxons, :taxonomy_id, :taxon_id 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20170606105748_create_join_table_group_content_item.rb: -------------------------------------------------------------------------------- 1 | class CreateJoinTableGroupContentItem < ActiveRecord::Migration[5.1] 2 | def change 3 | create_join_table :groups, :content_items do |t| 4 | t.index [:content_item_id] 5 | t.index [:group_id] 6 | t.index %i[group_id content_item_id], name: "index_group_content_items", unique: true 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20170606110152_remove_content_ids_list_from_groups.rb: -------------------------------------------------------------------------------- 1 | class RemoveContentIdsListFromGroups < ActiveRecord::Migration[5.1] 2 | def change 3 | remove_column :groups, :content_item_ids 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170606163123_create_taxonomy_projects.rb: -------------------------------------------------------------------------------- 1 | class CreateTaxonomyProjects < ActiveRecord::Migration[5.1] 2 | def change 3 | create_table :taxonomy_projects do |t| 4 | t.string :name 5 | 6 | t.timestamps 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20170606163951_create_taxonomy_todos.rb: -------------------------------------------------------------------------------- 1 | class CreateTaxonomyTodos < ActiveRecord::Migration[5.1] 2 | def change 3 | create_table :taxonomy_todos do |t| 4 | t.belongs_to :content_item, foreign_key: true 5 | t.belongs_to :taxonomy_project, foreign_key: true 6 | t.datetime :completed_at 7 | t.string :completed_by 8 | 9 | t.timestamps 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /db/migrate/20170607101716_create_themes.rb: -------------------------------------------------------------------------------- 1 | class CreateThemes < ActiveRecord::Migration[5.1] 2 | def change 3 | create_table :themes do |t| 4 | t.string :name, null: false 5 | t.timestamps 6 | end 7 | 8 | add_index :themes, :name, unique: true 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20170607101811_create_subthemes.rb: -------------------------------------------------------------------------------- 1 | class CreateSubthemes < ActiveRecord::Migration[5.1] 2 | def change 3 | create_table :subthemes do |t| 4 | t.belongs_to :theme 5 | t.string :name, null: false 6 | t.timestamps 7 | end 8 | 9 | add_index :subthemes, %i(theme_id name), unique: true 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrate/20170607102042_create_inventory_rules.rb: -------------------------------------------------------------------------------- 1 | class CreateInventoryRules < ActiveRecord::Migration[5.1] 2 | def change 3 | create_table :inventory_rules do |t| 4 | t.belongs_to :subtheme 5 | t.string :link_type, null: false 6 | t.string :target_content_id, null: false 7 | t.timestamps 8 | end 9 | 10 | add_index :inventory_rules, 11 | %i(subtheme_id link_type target_content_id), 12 | unique: true, 13 | name: "index_subtheme_link_type_content_id" 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /db/migrate/20170607125158_create_terms.rb: -------------------------------------------------------------------------------- 1 | class CreateTerms < ActiveRecord::Migration[5.1] 2 | def change 3 | create_table :terms do |t| 4 | t.string :name 5 | t.belongs_to :taxonomy_project 6 | t.timestamps 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20170626100039_add_publishing_app_to_content_item.rb: -------------------------------------------------------------------------------- 1 | class AddPublishingAppToContentItem < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :content_items, :publishing_app, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170626133926_create_report_rows.rb: -------------------------------------------------------------------------------- 1 | class CreateReportRows < ActiveRecord::Migration[5.1] 2 | def change 3 | create_table :report_rows do |t| 4 | t.string :content_id 5 | t.json :data 6 | end 7 | 8 | add_index :report_rows, :content_id, unique: true 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20170626153519_re_seed_questions.rb: -------------------------------------------------------------------------------- 1 | class ReSeedQuestions < ActiveRecord::Migration[5.1] 2 | def change 3 | # noop 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170706143916_create_join_table_for_todos_terms.rb: -------------------------------------------------------------------------------- 1 | class CreateJoinTableForTodosTerms < ActiveRecord::Migration[5.1] 2 | def change 3 | create_join_table :terms, :taxonomy_todos do |t| 4 | t.index [:taxonomy_todo_id] 5 | t.index [:term_id] 6 | t.index %i[term_id taxonomy_todo_id], name: "index_terms_taxonomy_todos", unique: true 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20170706143917_add_status_to_taxonomy_todos.rb: -------------------------------------------------------------------------------- 1 | class AddStatusToTaxonomyTodos < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :taxonomy_todos, :status, :string, default: "todo" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170728202520_update_type_column_for_questions.rb: -------------------------------------------------------------------------------- 1 | class UpdateTypeColumnForQuestions < ActiveRecord::Migration[5.1] 2 | def change 3 | # noop 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170804092444_add_locale_to_content_items.rb: -------------------------------------------------------------------------------- 1 | class AddLocaleToContentItems < ActiveRecord::Migration[5.1] 2 | def up 3 | add_column :content_items, :locale, :string 4 | ActiveRecord::Base.connection.execute("UPDATE content_items SET locale = 'en'") 5 | change_column_null :content_items, :locale, false 6 | end 7 | 8 | def down 9 | remove_column :content_items, :locale 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrate/20170808081001_add_qualitative_measures_to_audits.rb: -------------------------------------------------------------------------------- 1 | class AddQualitativeMeasuresToAudits < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :audits, :change_title, :boolean 4 | add_column :audits, :change_description, :boolean 5 | add_column :audits, :change_body, :boolean 6 | add_column :audits, :change_attachments, :boolean 7 | add_column :audits, :outdated, :boolean 8 | add_column :audits, :redundant, :boolean 9 | add_column :audits, :reformat, :boolean 10 | add_column :audits, :similar, :boolean 11 | add_column :audits, :similar_urls, :text 12 | add_column :audits, :notes, :text 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /db/migrate/20170810074052_create_allocations.rb: -------------------------------------------------------------------------------- 1 | class CreateAllocations < ActiveRecord::Migration[5.1] 2 | def change 3 | create_table :allocations do |t| 4 | t.string :content_id, null: false 5 | t.string :uid, null: false 6 | 7 | t.timestamps 8 | end 9 | 10 | add_index :allocations, :content_id, unique: true 11 | add_index :allocations, :uid 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /db/migrate/20170824142643_drop_groups.rb: -------------------------------------------------------------------------------- 1 | class DropGroups < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :groups do |t| 4 | t.string :slug 5 | t.string :name 6 | t.string :group_type 7 | t.integer :parent_group_id, foreign_key: true, null: true 8 | 9 | t.timestamps 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /db/migrate/20170824143605_drop_join_table_groups_content_items.rb: -------------------------------------------------------------------------------- 1 | class DropJoinTableGroupsContentItems < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_join_table :groups, :content_items do |t| 4 | t.index [:content_item_id] 5 | t.index [:group_id] 6 | t.index %i[group_id content_item_id], name: "index_group_content_items", unique: true 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20170824145426_drop_organisations.rb: -------------------------------------------------------------------------------- 1 | class DropOrganisations < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :organisations do |t| 4 | t.string "slug" 5 | t.datetime "created_at", null: false 6 | t.datetime "updated_at", null: false 7 | t.string "title" 8 | t.string "content_id" 9 | t.index %w[slug], name: "index_organisations_on_slug", unique: true 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /db/migrate/20170824145545_drop_organisations_items_join_table.rb: -------------------------------------------------------------------------------- 1 | class DropOrganisationsItemsJoinTable < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_join_table :content_items, :organisations 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170824145959_drop_taxons_items_join_table.rb: -------------------------------------------------------------------------------- 1 | class DropTaxonsItemsJoinTable < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_join_table :content_items, :taxons do |t| 4 | t.index [:content_item_id] 5 | t.index %i[taxon_id content_item_id], name: "index_content_item_taxonomies", unique: true 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20170824150019_drop_taxons.rb: -------------------------------------------------------------------------------- 1 | class DropTaxons < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :taxons do |t| 4 | t.string :content_id 5 | t.string :title 6 | 7 | t.timestamps 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20170909211813_add_redirect_urls_to_audits.rb: -------------------------------------------------------------------------------- 1 | class AddRedirectUrlsToAudits < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :audits, :redirect_urls, :text 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170912052906_drop_inventory_rules.rb: -------------------------------------------------------------------------------- 1 | class DropInventoryRules < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :inventory_rules do |t| 4 | t.belongs_to :subtheme 5 | t.string :link_type, null: false 6 | t.string :target_content_id, null: false 7 | t.timestamps 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20170912052917_drop_subthemes.rb: -------------------------------------------------------------------------------- 1 | class DropSubthemes < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :subthemes do |t| 4 | t.bigint "theme_id" 5 | t.string "name", null: false 6 | t.timestamps 7 | 8 | t.index %w[theme_id name], name: "index_subthemes_on_theme_id_and_name", unique: true 9 | t.index %w[theme_id], name: "index_subthemes_on_theme_id" 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /db/migrate/20170912052927_drop_themes.rb: -------------------------------------------------------------------------------- 1 | class DropThemes < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :themes do |t| 4 | t.string "name", null: false 5 | t.timestamps 6 | 7 | t.index %w[name], name: "index_themes_on_name", unique: true 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20171207164824_add_foreign_keys_to_allocations.rb: -------------------------------------------------------------------------------- 1 | class AddForeignKeysToAllocations < ActiveRecord::Migration[5.1] 2 | def change 3 | add_index "users", :uid, unique: true 4 | 5 | add_foreign_key "allocations", "content_items", column: :content_id, primary_key: :content_id 6 | add_foreign_key "allocations", "users", column: :uid, primary_key: :uid 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20171218120450_add_word_files_count_to_content_item.rb: -------------------------------------------------------------------------------- 1 | class AddWordFilesCountToContentItem < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :content_items, :number_of_word_files, :integer, default: 0 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20171222101252_create_dimensions_organisations.rb: -------------------------------------------------------------------------------- 1 | class CreateDimensionsOrganisations < ActiveRecord::Migration[5.1] 2 | def change 3 | create_table :dimensions_organisations do |t| 4 | t.string :title 5 | t.string :slug 6 | t.string :description 7 | t.string :link 8 | t.string :organisation_id 9 | t.string :state 10 | t.string :content_id 11 | 12 | t.timestamps 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /db/migrate/20171228132858_create_dimensions_items.rb: -------------------------------------------------------------------------------- 1 | class CreateDimensionsItems < ActiveRecord::Migration[5.1] 2 | def change 3 | create_table :dimensions_items do |t| 4 | t.string :content_id 5 | t.string :title 6 | t.string :link 7 | t.string :description 8 | t.string :organisation_id 9 | t.text :content 10 | 11 | t.timestamps 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /db/migrate/20171229163406_create_facts_metrics.rb: -------------------------------------------------------------------------------- 1 | class CreateFactsMetrics < ActiveRecord::Migration[5.1] 2 | def change 3 | create_table :facts_metrics do |t| 4 | t.date :dimensions_date_id, index: true 5 | t.references :dimensions_item, foreign_key: true 6 | t.references :dimensions_organisation, foreign_key: true 7 | 8 | t.timestamps 9 | end 10 | 11 | add_foreign_key :facts_metrics, :dimensions_dates, foreign_key: :dimensions_date_id, primary_key: :date 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /db/migrate/20180103115527_delete_content_from_dimensions_items.rb: -------------------------------------------------------------------------------- 1 | class DeleteContentFromDimensionsItems < ActiveRecord::Migration[5.1] 2 | def change 3 | remove_column :dimensions_items, :content 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180103141736_create_dimensions_items_temps.rb: -------------------------------------------------------------------------------- 1 | class CreateDimensionsItemsTemps < ActiveRecord::Migration[5.1] 2 | def change 3 | create_table :dimensions_items_temps, id: false do |t| 4 | t.string :content_id 5 | t.string :title 6 | t.string :link 7 | t.string :description 8 | t.string :organisation_id 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrate/20180103161730_add_latest_to_dimensions_items.rb: -------------------------------------------------------------------------------- 1 | class AddLatestToDimensionsItems < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :latest, :boolean 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180105112615_add_index_to_items_dimension.rb: -------------------------------------------------------------------------------- 1 | class AddIndexToItemsDimension < ActiveRecord::Migration[5.1] 2 | def change 3 | add_index :dimensions_items, %i[content_id link organisation_id], name: :dimensions_items_natural_key 4 | add_index :dimensions_items_temps, %i[content_id link organisation_id], name: :dimensions_items_temps_natual_key 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20180118132842_drop_terms.rb: -------------------------------------------------------------------------------- 1 | class DropTerms < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :terms do |t| 4 | t.string "name" 5 | t.bigint "taxonomy_project_id" 6 | t.timestamps null: false 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20180118133606_drop_taxonomy_todos_terms.rb: -------------------------------------------------------------------------------- 1 | class DropTaxonomyTodosTerms < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :taxonomy_todos_terms do |t| 4 | t.bigint "term_id", null: false 5 | t.bigint "taxonomy_todo_id", null: false 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20180118133918_drop_taxonomy_todos.rb: -------------------------------------------------------------------------------- 1 | class DropTaxonomyTodos < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :taxonomy_todos do |t| 4 | t.bigint "content_item_id" 5 | t.bigint "taxonomy_project_id" 6 | t.datetime "completed_at" 7 | t.string "completed_by" 8 | t.string "status", default: "todo" 9 | t.timestamps null: false 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /db/migrate/20180118134135_drop_taxonomy_projects.rb: -------------------------------------------------------------------------------- 1 | class DropTaxonomyProjects < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :taxonomy_projects do |t| 4 | t.string "name" 5 | t.timestamps null: false 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20180119144707_drop_allocations.rb: -------------------------------------------------------------------------------- 1 | class DropAllocations < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :allocations do |t| 4 | t.string "content_id", null: false 5 | t.string "uid", null: false 6 | t.timestamps null: false 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20180119145749_drop_audits.rb: -------------------------------------------------------------------------------- 1 | class DropAudits < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :audits do |t| 4 | t.string "content_id", null: false 5 | t.string "uid", null: false 6 | t.boolean "change_title" 7 | t.boolean "change_description" 8 | t.boolean "change_body" 9 | t.boolean "change_attachments" 10 | t.boolean "outdated" 11 | t.boolean "redundant" 12 | t.boolean "reformat" 13 | t.boolean "similar" 14 | t.text "similar_urls" 15 | t.text "notes" 16 | t.text "redirect_urls" 17 | t.timestamps null: false 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /db/migrate/20180119150347_drop_questions.rb: -------------------------------------------------------------------------------- 1 | class DropQuestions < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :questions do |t| 4 | t.string "type", null: false 5 | t.text "text", null: false 6 | t.timestamps null: false 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20180119150908_drop_responses.rb: -------------------------------------------------------------------------------- 1 | class DropResponses < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :responses do |t| 4 | t.integer "audit_id" 5 | t.integer "question_id" 6 | t.text "value" 7 | t.timestamps null: false 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20180119151925_drop_report_rows.rb: -------------------------------------------------------------------------------- 1 | class DropReportRows < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :report_rows do |t| 4 | t.string "content_id" 5 | t.json "data" 6 | t.timestamps null: false 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20180128183042_add_pageviews_to_facts_metrics.rb: -------------------------------------------------------------------------------- 1 | class AddPageviewsToFactsMetrics < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :facts_metrics, :pageviews, :integer 4 | add_column :facts_metrics, :unique_pageviews, :integer 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20180131111547_remove_dimensions_organisation_id_from_facts_metrics.rb: -------------------------------------------------------------------------------- 1 | class RemoveDimensionsOrganisationIdFromFactsMetrics < ActiveRecord::Migration[5.1] 2 | def change 3 | remove_column :facts_metrics, :dimensions_organisation_id, :bigint 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180131112209_drop_dimensions_organisations.rb: -------------------------------------------------------------------------------- 1 | class DropDimensionsOrganisations < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :dimensions_organisations do |t| 4 | t.string "title" 5 | t.string "slug" 6 | t.string "description" 7 | t.string "link" 8 | t.string "organisation_id" 9 | t.string "state" 10 | t.string "content_id" 11 | t.timestamps null: false 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /db/migrate/20180131112642_change_dimensions_items_fields.rb: -------------------------------------------------------------------------------- 1 | class ChangeDimensionsItemsFields < ActiveRecord::Migration[5.1] 2 | def change 3 | rename_column :dimensions_items, :link, :base_path 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180131114516_change_dimensions_items_temps_fields.rb: -------------------------------------------------------------------------------- 1 | class ChangeDimensionsItemsTempsFields < ActiveRecord::Migration[5.1] 2 | def change 3 | rename_column :dimensions_items_temps, :link, :base_path 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180205162236_remove_organisation_id_from_dimension_items_and_dimension_items_temps.rb: -------------------------------------------------------------------------------- 1 | class RemoveOrganisationIdFromDimensionItemsAndDimensionItemsTemps < ActiveRecord::Migration[5.1] 2 | def change 3 | remove_column :dimensions_items, :organisation_id, :string 4 | remove_column :dimensions_items_temps, :organisation_id, :string 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20180207103905_add_raw_json_to_dimensions_item.rb: -------------------------------------------------------------------------------- 1 | class AddRawJsonToDimensionsItem < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :raw_json, :json 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180210123458_create_events_gas.rb: -------------------------------------------------------------------------------- 1 | class CreateEventsGas < ActiveRecord::Migration[5.1] 2 | def change 3 | create_table :events_gas do |t| 4 | t.date :date 5 | t.string :page_path 6 | t.integer :pageviews 7 | t.integer :unique_pageviews 8 | 9 | t.timestamps 10 | t.index %w[page_path date] 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /db/migrate/20180210154836_add_index_dimension_items_lates_base_path.rb: -------------------------------------------------------------------------------- 1 | class AddIndexDimensionItemsLatesBasePath < ActiveRecord::Migration[5.1] 2 | def change 3 | add_index :dimensions_items, %i(latest base_path) 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180210231645_add_default_values_for_ga.rb: -------------------------------------------------------------------------------- 1 | class AddDefaultValuesForGa < ActiveRecord::Migration[5.1] 2 | class Facts::Metric < ApplicationRecord 3 | end 4 | 5 | def up 6 | Facts::Metric.update_all(pageviews: 0, unique_pageviews: 0) 7 | 8 | change_column :facts_metrics, :pageviews, :integer, default: 0 9 | change_column :facts_metrics, :unique_pageviews, :integer, default: 0 10 | end 11 | 12 | def down; end 13 | end 14 | -------------------------------------------------------------------------------- /db/migrate/20180214145821_add_number_of_pdfs_to_dimensions_items.rb: -------------------------------------------------------------------------------- 1 | class AddNumberOfPdfsToDimensionsItems < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :number_of_pdfs, :integer 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180216114348_add_dirty_to_dimension_items.rb: -------------------------------------------------------------------------------- 1 | class AddDirtyToDimensionItems < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :dirty, :boolean, default: false 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180219151742_add_metadata_fields_to_dimensions_item.rb: -------------------------------------------------------------------------------- 1 | class AddMetadataFieldsToDimensionsItem < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :document_type, :string 4 | add_column :dimensions_items, :content_purpose_supertype, :string 5 | add_column :dimensions_items, :first_published_at, :datetime 6 | add_column :dimensions_items, :public_updated_at, :datetime 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20180220163110_add_number_of_word_files_to_dimensions_item.rb: -------------------------------------------------------------------------------- 1 | class AddNumberOfWordFilesToDimensionsItem < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :number_of_word_files, :integer 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180221151608_add_status_to_dimensions_items.rb: -------------------------------------------------------------------------------- 1 | class AddStatusToDimensionsItems < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :status, :string, default: "live" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180223142258_create_events_feedexes.rb: -------------------------------------------------------------------------------- 1 | class CreateEventsFeedexes < ActiveRecord::Migration[5.1] 2 | def change 3 | create_table :events_feedexes do |t| 4 | t.date :date 5 | t.string :page_path 6 | t.integer :number_of_issues 7 | t.index %w[page_path date] 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20180226111742_add_number_of_issues_to_facts_metrics.rb: -------------------------------------------------------------------------------- 1 | class AddNumberOfIssuesToFactsMetrics < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :facts_metrics, :number_of_issues, :integer, default: 0 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180305111459_drop_unused_indexes_to_reduce_space.rb: -------------------------------------------------------------------------------- 1 | class DropUnusedIndexesToReduceSpace < ActiveRecord::Migration[5.1] 2 | def change 3 | # duplicated of [:dimensions_date_id, dimensions_item_id] 4 | remove_index :facts_metrics, :dimensions_date_id 5 | 6 | # This is no longer in used 7 | remove_index :content_items, :title 8 | 9 | # This index is no longer in use 10 | remove_index :dimensions_dates, :date_name_abbreviated 11 | 12 | # We don't need an index on both fields, only on `latest` 13 | remove_index :dimensions_items, %i[latest base_path] 14 | add_index :dimensions_items, :latest 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /db/migrate/20180305112605_remove_table_dimensions_items_temp.rb: -------------------------------------------------------------------------------- 1 | class RemoveTableDimensionsItemsTemp < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :dimensions_items_temps do |t| 4 | t.string :content_id 5 | t.string :title 6 | t.string :link 7 | t.string :description 8 | t.string :organisation_id 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrate/20180305161014_add_odyssey_metrics.rb: -------------------------------------------------------------------------------- 1 | class AddOdysseyMetrics < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :string_length, :integer, default: 0 4 | add_column :dimensions_items, :sentence_count, :integer, default: 0 5 | add_column :dimensions_items, :word_count, :integer, default: 0 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20180307113805_rename_column_dimensions_items_content_purpose_supertype.rb: -------------------------------------------------------------------------------- 1 | class RenameColumnDimensionsItemsContentPurposeSupertype < ActiveRecord::Migration[5.1] 2 | def change 3 | rename_column :dimensions_items, :content_purpose_supertype, :content_purpose_document_supertype 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180309120223_add_primary_org_fields_to_dimensions_item.rb: -------------------------------------------------------------------------------- 1 | class AddPrimaryOrgFieldsToDimensionsItem < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :primary_organisation_title, :string 4 | add_column :dimensions_items, :primary_organisation_content_id, :string 5 | add_column :dimensions_items, :primary_organisation_withdrawn, :boolean 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20180309144813_add_index_dimensions_items_organisation.rb: -------------------------------------------------------------------------------- 1 | class AddIndexDimensionsItemsOrganisation < ActiveRecord::Migration[5.1] 2 | def change 3 | add_index :dimensions_items, :primary_organisation_content_id, 4 | name: :index_dimensions_items_primary_organisation_content_id 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20180309144814_rename_column_dirty_to_outdated.rb: -------------------------------------------------------------------------------- 1 | class RenameColumnDirtyToOutdated < ActiveRecord::Migration[5.1] 2 | def change 3 | rename_column :dimensions_items, :dirty, :outdated 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180309144815_remove_links.rb: -------------------------------------------------------------------------------- 1 | class RemoveLinks < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :links do |t| 4 | t.string :source_content_id 5 | t.string :link_type 6 | t.string :target_content_id 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20180309144816_remove_content_items.rb: -------------------------------------------------------------------------------- 1 | class RemoveContentItems < ActiveRecord::Migration[5.1] 2 | def change 3 | drop_table :content_items do |t| 4 | t.string :content_id 5 | t.datetime :public_updated_at 6 | t.string :base_path 7 | t.string :title 8 | t.string :document_type 9 | t.string :description 10 | t.integer :one_month_page_views, default: 0 11 | t.integer :number_of_pdfs, default: 0 12 | t.integer :six_months_page_views, default: 0 13 | t.string :publishing_app 14 | t.string :locale, null: false 15 | t.integer :number_of_word_files, default: 0 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /db/migrate/20180315144816_rename_number_of_issues_to_feedex_comments.rb: -------------------------------------------------------------------------------- 1 | class RenameNumberOfIssuesToFeedexComments < ActiveRecord::Migration[5.1] 2 | def change 3 | rename_column :facts_metrics, :number_of_issues, :feedex_comments 4 | rename_column :events_feedexes, :number_of_issues, :feedex_comments 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20180319161814_add_content_hash_to_dimensions_item.rb: -------------------------------------------------------------------------------- 1 | class AddContentHashToDimensionsItem < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :content_hash, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180321152146_add_locale_to_dimensions_items.rb: -------------------------------------------------------------------------------- 1 | class AddLocaleToDimensionsItems < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :locale, :string, default: "en", null: false 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180322134606_drop_indexes_created_without_migration.rb: -------------------------------------------------------------------------------- 1 | class DropIndexesCreatedWithoutMigration < ActiveRecord::Migration[5.1] 2 | def up 3 | if index_exists?(:dimensions_items, %i[latest content_id], name: "idx_latest_content_id") 4 | remove_index :dimensions_items, name: "idx_latest_content_id" 5 | end 6 | 7 | if index_exists?(:facts_metrics, %i[dimensions_date_id dimensions_item_id], name: "idx_facts_metrics_on_dimensions_date_id_dimensios_items_id") 8 | remove_index :facts_metrics, name: "idx_facts_metrics_on_dimensions_date_id_dimensios_items_id" 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrate/20180322140016_add_indexes_for_master_process.rb: -------------------------------------------------------------------------------- 1 | class AddIndexesForMasterProcess < ActiveRecord::Migration[5.1] 2 | def change 3 | add_index :dimensions_items, %i[latest content_id] 4 | add_index :facts_metrics, %i[dimensions_date_id dimensions_item_id], name: "index_facts_metrics_date_item_id" 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20180323170720_add_oudated_at_to_dimensions_items.rb: -------------------------------------------------------------------------------- 1 | class AddOudatedAtToDimensionsItems < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :outdated_at, :datetime 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180417102741_add_payload_version_to_dimensions_items.rb: -------------------------------------------------------------------------------- 1 | class AddPayloadVersionToDimensionsItems < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :publishing_api_payload_version, :bigint, default: 0, null: false 4 | change_column_default :dimensions_items, :publishing_api_payload_version, nil 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20180418135558_add_is_this_useful_columns_to_events_gas.rb: -------------------------------------------------------------------------------- 1 | class AddIsThisUsefulColumnsToEventsGas < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :events_gas, :is_this_useful_yes, :integer, default: 0 4 | add_column :events_gas, :is_this_useful_no, :integer, default: 0 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20180418135632_add_is_this_useful_columns_to_facts_metrics.rb: -------------------------------------------------------------------------------- 1 | class AddIsThisUsefulColumnsToFactsMetrics < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :facts_metrics, :is_this_useful_yes, :integer, default: 0 4 | add_column :facts_metrics, :is_this_useful_no, :integer, default: 0 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20180425100640_remove_outdated_from_items.rb: -------------------------------------------------------------------------------- 1 | class RemoveOutdatedFromItems < ActiveRecord::Migration[5.1] 2 | def change 3 | remove_column :dimensions_items, :outdated, :boolean 4 | remove_column :dimensions_items, :outdated_at, :datetime 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20180425131954_add_process_name_to_ga_events.rb: -------------------------------------------------------------------------------- 1 | class AddProcessNameToGaEvents < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :events_gas, :process_name, :integer 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180425132405_process_name_is_not_null.rb: -------------------------------------------------------------------------------- 1 | class ProcessNameIsNotNull < ActiveRecord::Migration[5.1] 2 | def up 3 | Events::GA.update_all(process_name: 1) 4 | change_column :events_gas, :process_name, :integer, null: false 5 | end 6 | 7 | def down 8 | change_column :events_gas, :process_name, :integer, null: true 9 | Events::GA.update_all(process_name: nil) 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrate/20180430100333_add_internal_search_to_events_gas.rb: -------------------------------------------------------------------------------- 1 | class AddInternalSearchToEventsGas < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :events_gas, :number_of_internal_searches, :integer, default: 0 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180430100349_add_internal_search_to_facts_metrics.rb: -------------------------------------------------------------------------------- 1 | class AddInternalSearchToFactsMetrics < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :facts_metrics, :number_of_internal_searches, :integer, default: 0 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180430104334_add_unique_index_items.rb: -------------------------------------------------------------------------------- 1 | class AddUniqueIndexItems < ActiveRecord::Migration[5.1] 2 | def change 3 | add_index :dimensions_items, %i(latest base_path), unique: true, where: "latest = 'true'" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180430110607_delete_latest_index_items.rb: -------------------------------------------------------------------------------- 1 | class DeleteLatestIndexItems < ActiveRecord::Migration[5.1] 2 | def change 3 | remove_index :dimensions_items, :latest 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180509100804_remove_metadata_from_facts_editions.rb: -------------------------------------------------------------------------------- 1 | class RemoveMetadataFromFactsEditions < ActiveRecord::Migration[5.1] 2 | def change 3 | remove_column :facts_editions, :raw_json, :json 4 | remove_column :facts_editions, :status, :string 5 | remove_column :facts_editions, :description, :string 6 | remove_column :facts_editions, :first_published_at, :datetime 7 | remove_column :facts_editions, :public_updated_at, :datetime 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20180514085910_add_additional_metrics_to_events_gas_and_facts_metrics.rb: -------------------------------------------------------------------------------- 1 | class AddAdditionalMetricsToEventsGasAndFactsMetrics < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :events_gas, :exits, :integer, default: 0 4 | add_column :events_gas, :entrances, :integer, default: 0 5 | add_column :events_gas, :bounce_rate, :integer, default: 0 6 | add_column :events_gas, :avg_time_on_page, :integer, default: 0 7 | add_column :facts_metrics, :exits, :integer, default: 0 8 | add_column :facts_metrics, :entrances, :integer, default: 0 9 | add_column :facts_metrics, :bounce_rate, :integer, default: 0 10 | add_column :facts_metrics, :avg_time_on_page, :integer, default: 0 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /db/migrate/20180514152920_add_constraint_on_duplicate_events.rb: -------------------------------------------------------------------------------- 1 | class AddConstraintOnDuplicateEvents < ActiveRecord::Migration[5.1] 2 | def change 3 | add_index :events_gas, %i[process_name date page_path], unique: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180515104926_add_content_purpose_supergroup_to_dimensions_items.rb: -------------------------------------------------------------------------------- 1 | class AddContentPurposeSupergroupToDimensionsItems < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :content_purpose_supergroup, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180515105428_add_content_purpose_subgroup_to_dimensions_items.rb: -------------------------------------------------------------------------------- 1 | class AddContentPurposeSubgroupToDimensionsItems < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :content_purpose_subgroup, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180518133900_change_events_gas_page_views_to_default_to_zero.rb: -------------------------------------------------------------------------------- 1 | class ChangeEventsGasPageViewsToDefaultToZero < ActiveRecord::Migration[5.1] 2 | def up 3 | change_column :events_gas, :pageviews, :integer, default: 0 4 | change_column :events_gas, :unique_pageviews, :integer, default: 0 5 | end 6 | 7 | def down 8 | change_column :events_gas, :pageviews, :integer, default: nil 9 | change_column :events_gas, :unique_pageviews, :integer, default: nil 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrate/20180522103512_add_index_on_base_path_to_dimensions_items.rb: -------------------------------------------------------------------------------- 1 | class AddIndexOnBasePathToDimensionsItems < ActiveRecord::Migration[5.1] 2 | def change 3 | add_index :dimensions_items, :base_path 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180607203406_remove_content_hash_from_dimensions_item.rb: -------------------------------------------------------------------------------- 1 | class RemoveContentHashFromDimensionsItem < ActiveRecord::Migration[5.1] 2 | def change 3 | remove_column :dimensions_items, :content_hash, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180607204735_remove_status_from_dimensions_item.rb: -------------------------------------------------------------------------------- 1 | class RemoveStatusFromDimensionsItem < ActiveRecord::Migration[5.1] 2 | def change 3 | remove_column :dimensions_items, :status, :boolean 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180607205553_add_schema_name_to_dimensions_item.rb: -------------------------------------------------------------------------------- 1 | class AddSchemaNameToDimensionsItem < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :schema_name, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180608124756_remove_default_values_in_editions.rb: -------------------------------------------------------------------------------- 1 | class RemoveDefaultValuesInEditions < ActiveRecord::Migration[5.1] 2 | def change 3 | change_column_default :facts_editions, :string_length, nil 4 | change_column_default :facts_editions, :sentence_count, nil 5 | change_column_default :facts_editions, :word_count, nil 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20180608125059_remove_default_values_for_locale_in_items.rb: -------------------------------------------------------------------------------- 1 | class RemoveDefaultValuesForLocaleInItems < ActiveRecord::Migration[5.1] 2 | def change 3 | change_column(:dimensions_items, :locale, :string, null: true) 4 | change_column_default :dimensions_items, :locale, nil 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20180610073937_add_not_null_constratint_to_facts_editions.rb: -------------------------------------------------------------------------------- 1 | class AddNotNullConstratintToFactsEditions < ActiveRecord::Migration[5.1] 2 | def change 3 | change_column_null :facts_editions, :dimensions_item_id, false 4 | change_column_null :facts_editions, :dimensions_date_id, false 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20180610074451_add_not_null_constratint_to_facts_metrics.rb: -------------------------------------------------------------------------------- 1 | class AddNotNullConstratintToFactsMetrics < ActiveRecord::Migration[5.1] 2 | def change 3 | change_column_null :facts_metrics, :dimensions_item_id, false 4 | change_column_null :facts_metrics, :dimensions_date_id, false 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20180610075700_add_unique_index_to_facts_editions.rb: -------------------------------------------------------------------------------- 1 | class AddUniqueIndexToFactsEditions < ActiveRecord::Migration[5.1] 2 | def change 3 | remove_index :facts_editions, :dimensions_item_id 4 | remove_index :facts_editions, :dimensions_date_id 5 | add_index :facts_editions, %i[dimensions_item_id dimensions_date_id], name: :editions_item_id_date_id, unique: true 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20180610080335_add_unique_index_to_facts_metrics.rb: -------------------------------------------------------------------------------- 1 | class AddUniqueIndexToFactsMetrics < ActiveRecord::Migration[5.1] 2 | def change 3 | remove_index :facts_metrics, name: "index_facts_metrics_date_item_id" 4 | remove_index :facts_metrics, name: "index_facts_metrics_on_dimensions_item_id" 5 | add_index :facts_metrics, %i[dimensions_date_id dimensions_item_id], name: :metrics_item_id_date_id, unique: true 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20180610081321_add_default_not_null_to_dimensions_items.rb: -------------------------------------------------------------------------------- 1 | class AddDefaultNotNullToDimensionsItems < ActiveRecord::Migration[5.1] 2 | def change 3 | change_column_null :dimensions_items, :content_id, false 4 | change_column_null :dimensions_items, :base_path, false 5 | change_column_null :dimensions_items, :schema_name, false 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20180610081625_update_indexes_dimensions_items.rb: -------------------------------------------------------------------------------- 1 | class UpdateIndexesDimensionsItems < ActiveRecord::Migration[5.1] 2 | def change 3 | remove_index :dimensions_items, %i(latest base_path) 4 | add_index :dimensions_items, %i(base_path latest), unique: true, where: "latest = 'true'" 5 | 6 | remove_index :dimensions_items, name: "index_dimensions_items_on_latest_and_content_id" 7 | add_index :dimensions_items, %i(content_id latest) 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20180625120718_add_content_to_dimensions_items.rb: -------------------------------------------------------------------------------- 1 | class AddContentToDimensionsItems < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :content, :text 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180626085042_create_publishing_api_events.rb: -------------------------------------------------------------------------------- 1 | class CreatePublishingApiEvents < ActiveRecord::Migration[5.1] 2 | def change 3 | create_table :publishing_api_events do |t| 4 | t.string :routing_key 5 | t.json :payload 6 | 7 | t.timestamps 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20180626085229_add_reference_to_events_from_dimensions_items.rb: -------------------------------------------------------------------------------- 1 | class AddReferenceToEventsFromDimensionsItems < ActiveRecord::Migration[5.1] 2 | def change 3 | add_reference :dimensions_items, :publishing_api_events, foreign_key: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180711100526_rename_column_dimensions_items_event_id.rb: -------------------------------------------------------------------------------- 1 | class RenameColumnDimensionsItemsEventId < ActiveRecord::Migration[5.1] 2 | def change 3 | rename_column :dimensions_items, :publishing_api_events_id, :publishing_api_event_id 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180712115111_rename_dimensions_item_content_to_document_text.rb: -------------------------------------------------------------------------------- 1 | class RenameDimensionsItemContentToDocumentText < ActiveRecord::Migration[5.1] 2 | def change 3 | rename_column :dimensions_items, :content, :document_text 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180718161302_remove_publishing_api_events.rb: -------------------------------------------------------------------------------- 1 | class RemovePublishingApiEvents < ActiveRecord::Migration[5.1] 2 | def change 3 | remove_reference :dimensions_items, :publishing_api_event, foreign_key: true 4 | 5 | drop_table :publishing_api_events do |t| 6 | t.string :routing_key 7 | t.json :payload 8 | 9 | t.timestamps 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /db/migrate/20180719133916_add_columns_to_dimensions_items.rb: -------------------------------------------------------------------------------- 1 | class AddColumnsToDimensionsItems < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :dimensions_items, :publishing_app, :string 4 | add_column :dimensions_items, :rendering_app, :string 5 | add_column :dimensions_items, :analytics_identifier, :string 6 | add_column :dimensions_items, :phase, :string 7 | add_column :dimensions_items, :previous_version, :string 8 | add_column :dimensions_items, :update_type, :string 9 | add_column :dimensions_items, :last_edited_at, :datetime 10 | add_column :dimensions_items, :links, :json 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /db/migrate/20180725102859_rename_dimensions_items_links_column.rb: -------------------------------------------------------------------------------- 1 | class RenameDimensionsItemsLinksColumn < ActiveRecord::Migration[5.1] 2 | def change 3 | rename_column :dimensions_items, :links, :expanded_links 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180725103453_add_default_to_dimensions_items_expanded_links.rb: -------------------------------------------------------------------------------- 1 | class AddDefaultToDimensionsItemsExpandedLinks < ActiveRecord::Migration[5.1] 2 | def change 3 | change_column_default :dimensions_items, :expanded_links, {} 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180725104938_remove_index_base_path_latest.rb: -------------------------------------------------------------------------------- 1 | class RemoveIndexBasePathLatest < ActiveRecord::Migration[5.1] 2 | def change 3 | remove_index :dimensions_items, name: "index_dimensions_items_on_base_path_and_latest" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180801075913_add_bounces_and_time_on_page_to_events_gas_and_facts_metrics.rb: -------------------------------------------------------------------------------- 1 | class AddBouncesAndTimeOnPageToEventsGasAndFactsMetrics < ActiveRecord::Migration[5.1] 2 | def change 3 | add_column :events_gas, :bounces, :integer, default: 0 4 | add_column :events_gas, :time_on_page, :integer, default: 0 5 | add_column :facts_metrics, :bounces, :integer, default: 0 6 | add_column :facts_metrics, :time_on_page, :integer, default: 0 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20180820115550_remove_raw_json_from_dimensions_item.rb: -------------------------------------------------------------------------------- 1 | class RemoveRawJsonFromDimensionsItem < ActiveRecord::Migration[5.1] 2 | def change 3 | remove_column :dimensions_items, :raw_json, :json 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180821135700_remove_expanded_links_column.rb: -------------------------------------------------------------------------------- 1 | class RemoveExpandedLinksColumn < ActiveRecord::Migration[5.1] 2 | def change 3 | remove_column :dimensions_items, :expanded_links, :json 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180822144907_add_satisfaction_score_to_facts_metric.rb: -------------------------------------------------------------------------------- 1 | class AddSatisfactionScoreToFactsMetric < ActiveRecord::Migration[5.2] 2 | def change 3 | add_column :facts_metrics, :satisfaction_score, :float 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180824105759_delete_old_edition_metrics.rb: -------------------------------------------------------------------------------- 1 | class DeleteOldEditionMetrics < ActiveRecord::Migration[5.2] 2 | def change 3 | remove_column :facts_editions, :contractions_count 4 | remove_column :facts_editions, :equality_count 5 | remove_column :facts_editions, :indefinite_article_count 6 | remove_column :facts_editions, :passive_count 7 | remove_column :facts_editions, :profanities_count 8 | remove_column :facts_editions, :redundant_acronyms_count 9 | remove_column :facts_editions, :repeated_words_count 10 | remove_column :facts_editions, :simplify_count 11 | remove_column :facts_editions, :spell_count 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /db/migrate/20180831101255_add_index_to_dimensions_items_on_latest.rb: -------------------------------------------------------------------------------- 1 | class AddIndexToDimensionsItemsOnLatest < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :dimensions_items, :latest 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180920150009_add_content_uuid_to_dimensions_item.rb: -------------------------------------------------------------------------------- 1 | class AddContentUuidToDimensionsItem < ActiveRecord::Migration[5.2] 2 | def change 3 | add_column :dimensions_items, :content_uuid, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180921122451_add_index_dimensions_items_content_uuid.rb: -------------------------------------------------------------------------------- 1 | class AddIndexDimensionsItemsContentUuid < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :dimensions_items, :content_uuid, name: "index_dimensions_items_content_uuid" 4 | add_index :dimensions_items, %i[content_uuid latest], name: "index_dimensions_items_content_uuid_latest" 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20180921155800_add_index_for_content_list.rb: -------------------------------------------------------------------------------- 1 | class AddIndexForContentList < ActiveRecord::Migration[5.2] 2 | def change 3 | # This index is required for the query we run to get the data 4 | # for the /content endpoint. We GROUP_BY these columns. 5 | add_index :dimensions_items, %i[content_uuid base_path title document_type], name: "index_for_content_query" 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20180924093138_restore_raw_json_to_dimensions_item.rb: -------------------------------------------------------------------------------- 1 | class RestoreRawJsonToDimensionsItem < ActiveRecord::Migration[5.2] 2 | def change 3 | add_column :dimensions_items, :raw_json, :json 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180926100046_rename_content_uuid_to_warehouse_item_id.rb: -------------------------------------------------------------------------------- 1 | class RenameContentUuidToWarehouseItemId < ActiveRecord::Migration[5.2] 2 | def change 3 | rename_column :dimensions_items, :content_uuid, :warehouse_item_id 4 | 5 | rename_index :dimensions_items, "index_dimensions_items_content_uuid_latest", "index_dimensions_items_warehouse_item_id_latest" 6 | rename_index :dimensions_items, "index_dimensions_items_content_uuid", "index_dimensions_items_warehouse_item_id" 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20180926105207_add_constraints_to_satisfaction_score_columns.rb: -------------------------------------------------------------------------------- 1 | class AddConstraintsToSatisfactionScoreColumns < ActiveRecord::Migration[5.2] 2 | def change 3 | change_column :facts_metrics, :satisfaction_score, :float, default: 0.0, null: false 4 | change_column :facts_metrics, :is_this_useful_yes, :integer, default: 0, null: false 5 | change_column :facts_metrics, :is_this_useful_no, :integer, default: 0, null: false 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20180928150637_add_not_null_constraint_to_warehouse_item_id.rb: -------------------------------------------------------------------------------- 1 | class AddNotNullConstraintToWarehouseItemId < ActiveRecord::Migration[5.2] 2 | def change 3 | say "setting not null constraint on `warehouse_item_id`" 4 | change_column_null :dimensions_items, :warehouse_item_id, false 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20181001125635_rename_facts_metrics_columns.rb: -------------------------------------------------------------------------------- 1 | class RenameFactsMetricsColumns < ActiveRecord::Migration[5.2] 2 | def change 3 | rename_column :facts_metrics, :pageviews, :pviews 4 | rename_column :facts_metrics, :unique_pageviews, :upviews 5 | rename_column :facts_metrics, :is_this_useful_yes, :useful_yes 6 | rename_column :facts_metrics, :is_this_useful_no, :useful_no 7 | rename_column :facts_metrics, :number_of_internal_searches, :searches 8 | rename_column :facts_metrics, :feedex_comments, :feedex 9 | rename_column :facts_metrics, :avg_time_on_page, :avg_page_time 10 | rename_column :facts_metrics, :time_on_page, :page_time 11 | rename_column :facts_metrics, :satisfaction_score, :satisfaction 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /db/migrate/20181001174718_rename_facts_edition_columns.rb: -------------------------------------------------------------------------------- 1 | class RenameFactsEditionColumns < ActiveRecord::Migration[5.2] 2 | def change 3 | rename_column :facts_editions, :number_of_pdfs, :pdf_count 4 | rename_column :facts_editions, :number_of_word_files, :doc_count 5 | rename_column :facts_editions, :readability_score, :readability 6 | rename_column :facts_editions, :string_length, :chars 7 | rename_column :facts_editions, :sentence_count, :sentences 8 | rename_column :facts_editions, :word_count, :words 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20181002110036_rename_events_gas_columns.rb: -------------------------------------------------------------------------------- 1 | class RenameEventsGasColumns < ActiveRecord::Migration[5.2] 2 | def change 3 | rename_column :events_gas, :number_of_internal_searches, :searches 4 | rename_column :events_gas, :pageviews, :pviews 5 | rename_column :events_gas, :unique_pageviews, :upviews 6 | rename_column :events_gas, :is_this_useful_yes, :useful_yes 7 | rename_column :events_gas, :is_this_useful_no, :useful_no 8 | rename_column :events_gas, :avg_time_on_page, :avg_page_time 9 | rename_column :events_gas, :time_on_page, :page_time 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrate/20181004070753_rename_primary_organisation_content_id_to_content_id.rb: -------------------------------------------------------------------------------- 1 | class RenamePrimaryOrganisationContentIdToContentId < ActiveRecord::Migration[5.2] 2 | def change 3 | rename_column :dimensions_items, :primary_organisation_content_id, :organisation_id 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181004113705_rename_dimensions_item_to_dimensions_editions.rb: -------------------------------------------------------------------------------- 1 | class RenameDimensionsItemToDimensionsEditions < ActiveRecord::Migration[5.2] 2 | def change 3 | rename_table :dimensions_items, :dimensions_editions 4 | 5 | rename_index :dimensions_editions, "index_dimensions_items_primary_organisation_content_id", "index_dimensions_editions_organisation_id" 6 | rename_index :dimensions_editions, "index_dimensions_items_warehouse_item_id_latest", "index_dimensions_editions_warehouse_item_id_latest" 7 | rename_index :dimensions_editions, "index_dimensions_items_warehouse_item_id", "index_dimensions_editions_warehouse_item_id" 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20181004115155_rename_fact_edition_dimensions_item_id_to_dimensions_edition_id.rb: -------------------------------------------------------------------------------- 1 | class RenameFactEditionDimensionsItemIdToDimensionsEditionId < ActiveRecord::Migration[5.2] 2 | def change 3 | rename_column :facts_editions, :dimensions_item_id, :dimensions_edition_id 4 | 5 | rename_index :facts_editions, "editions_item_id_date_id", "facts_editions_edition_id_date_id" 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20181004120457_rename_facts_metrics_dimensions_item_id_to_dimensions_edition_id.rb: -------------------------------------------------------------------------------- 1 | class RenameFactsMetricsDimensionsItemIdToDimensionsEditionId < ActiveRecord::Migration[5.2] 2 | def change 3 | rename_column :facts_metrics, :dimensions_item_id, :dimensions_edition_id 4 | 5 | rename_index :facts_metrics, "metrics_item_id_date_id", "metrics_edition_id_date_id" 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20181009133439_add_index_for_organisation_list.rb: -------------------------------------------------------------------------------- 1 | class AddIndexForOrganisationList < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :dimensions_editions, %i[latest organisation_id primary_organisation_title], 4 | name: "index_dimensions_editions_on_latest_org_id_org_title" 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20181015093842_add_index_dimensions_editions_document_type.rb: -------------------------------------------------------------------------------- 1 | class AddIndexDimensionsEditionsDocumentType < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :dimensions_editions, %i[latest document_type] 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181025132726_add_index_to_facts_metrics_on_edition_id.rb: -------------------------------------------------------------------------------- 1 | class AddIndexToFactsMetricsOnEditionId < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :facts_metrics, :dimensions_edition_id 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181025134740_add_unique_index_warehouse_id_latest.rb: -------------------------------------------------------------------------------- 1 | class AddUniqueIndexWarehouseIdLatest < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :dimensions_editions, %i(latest warehouse_item_id), unique: true, where: "latest = 'true'" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181025135947_add_unique_index_on_latest_basepath.rb: -------------------------------------------------------------------------------- 1 | class AddUniqueIndexOnLatestBasepath < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :dimensions_editions, %i(latest base_path), unique: true, where: "latest = 'true'" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181029112946_add_withdrawn_and_historical.rb: -------------------------------------------------------------------------------- 1 | class AddWithdrawnAndHistorical < ActiveRecord::Migration[5.2] 2 | def change 3 | add_column :dimensions_editions, :withdrawn, :boolean 4 | add_column :dimensions_editions, :historical, :boolean 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20181030120956_add_withdrawn_and_historical_constraints.rb: -------------------------------------------------------------------------------- 1 | class AddWithdrawnAndHistoricalConstraints < ActiveRecord::Migration[5.2] 2 | def up 3 | say "Updating withdrawn for all editions" 4 | Dimensions::Edition.update_all(withdrawn: false) 5 | 6 | say "Updating historical for all editions" 7 | Dimensions::Edition.update_all(historical: false) 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20181031112754_create_dimensions_months.rb: -------------------------------------------------------------------------------- 1 | class CreateDimensionsMonths < ActiveRecord::Migration[5.2] 2 | def change 3 | create_table :dimensions_months, id: false do |t| 4 | t.string :id, null: false 5 | t.string :month_name, null: false 6 | t.string :month_name_abbreviated, null: false 7 | t.integer :month_number, null: false 8 | t.integer :quarter, null: false 9 | t.integer :year, null: false 10 | 11 | t.timestamps null: false 12 | t.index :id, unique: true 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /db/migrate/20181101123329_add_foreign_key_to_monthly_aggregation.rb: -------------------------------------------------------------------------------- 1 | class AddForeignKeyToMonthlyAggregation < ActiveRecord::Migration[5.2] 2 | def change 3 | add_foreign_key :aggregations_monthly_metrics, :dimensions_months, foreign_key: :dimensions_month_id, primary_key: :id 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181101163401_add_withdrawn_constraints.rb: -------------------------------------------------------------------------------- 1 | class AddWithdrawnConstraints < ActiveRecord::Migration[5.2] 2 | def up 3 | change_column_null :dimensions_editions, :withdrawn, false 4 | end 5 | 6 | def down 7 | change_column_null :dimensions_editions, :withdrawn, true 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20181101163411_add_historical_constraints.rb: -------------------------------------------------------------------------------- 1 | class AddHistoricalConstraints < ActiveRecord::Migration[5.2] 2 | def up 3 | change_column_null :dimensions_editions, :historical, false 4 | end 5 | 6 | def down 7 | change_column_null :dimensions_editions, :historical, true 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20181105152020_create_aggregations_search_last_thirty_days.rb: -------------------------------------------------------------------------------- 1 | class CreateAggregationsSearchLastThirtyDays < ActiveRecord::Migration[5.2] 2 | def change 3 | Aggregations::MaterializedView.prepare 4 | create_view :aggregations_search_last_thirty_days, materialized: true 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20181105160339_add_index_to_last_thirty_days.rb: -------------------------------------------------------------------------------- 1 | class AddIndexToLastThirtyDays < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :aggregations_search_last_thirty_days, %w(dimensions_edition_id upviews), name: "index_on_last_thirty_days_edition_id_upviews" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181106114625_create_aggregations_search_last_months.rb: -------------------------------------------------------------------------------- 1 | class CreateAggregationsSearchLastMonths < ActiveRecord::Migration[5.2] 2 | def change 3 | Aggregations::MaterializedView.prepare 4 | create_view :aggregations_search_last_months, materialized: true 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20181106121337_add_index_to_last_month_aggregations.rb: -------------------------------------------------------------------------------- 1 | class AddIndexToLastMonthAggregations < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :aggregations_search_last_months, %w(dimensions_edition_id upviews), name: "index_on_last_month_edition_id_upviews" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181106170339_create_aggregations_search_last_three_months.rb: -------------------------------------------------------------------------------- 1 | class CreateAggregationsSearchLastThreeMonths < ActiveRecord::Migration[5.2] 2 | def change 3 | Aggregations::MaterializedView.prepare 4 | create_view :aggregations_search_last_three_months, materialized: true 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20181106171806_add_index_to_last_three_months_aggregations.rb: -------------------------------------------------------------------------------- 1 | class AddIndexToLastThreeMonthsAggregations < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :aggregations_search_last_three_months, %w(dimensions_edition_id upviews), name: "index_on_last_three_months_edition_id_upviews" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181107165445_create_aggregations_search_last_six_months.rb: -------------------------------------------------------------------------------- 1 | class CreateAggregationsSearchLastSixMonths < ActiveRecord::Migration[5.2] 2 | def change 3 | create_view :aggregations_search_last_six_months, materialized: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181107165733_add_index_to_last_six_months_aggregations.rb: -------------------------------------------------------------------------------- 1 | class AddIndexToLastSixMonthsAggregations < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :aggregations_search_last_six_months, %w(dimensions_edition_id upviews), name: "index_on_last_six_months_edition_id_upviews" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181107170231_create_aggregations_search_last_twelve_months.rb: -------------------------------------------------------------------------------- 1 | class CreateAggregationsSearchLastTwelveMonths < ActiveRecord::Migration[5.2] 2 | def change 3 | create_view :aggregations_search_last_twelve_months, materialized: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181107170246_add_index_to_last_twelve_months_aggregations.rb: -------------------------------------------------------------------------------- 1 | class AddIndexToLastTwelveMonthsAggregations < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :aggregations_search_last_twelve_months, %w(dimensions_edition_id upviews), name: "index_on_last_twelve_months_edition_id_upviews" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181120164728_add_not_null_constraint_locale.rb: -------------------------------------------------------------------------------- 1 | class AddNotNullConstraintLocale < ActiveRecord::Migration[5.2] 2 | def change 3 | change_column_null :dimensions_editions, :locale, false 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181121135744_add_full_text_index_title.rb: -------------------------------------------------------------------------------- 1 | class AddFullTextIndexTitle < ActiveRecord::Migration[5.2] 2 | def up 3 | execute "create index dimensions_editions_title on dimensions_editions using gin(to_tsvector('english', title ))" 4 | end 5 | 6 | def down 7 | execute "drop index dimensions_editions_title" 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20181122105601_add_full_text_index_base_path.rb: -------------------------------------------------------------------------------- 1 | class AddFullTextIndexBasePath < ActiveRecord::Migration[5.2] 2 | def up 3 | execute "create index dimensions_editions_base_path on dimensions_editions using gin(to_tsvector('english'::regconfig, replace((base_path)::text, '/'::text, ' '::text)))" 4 | end 5 | 6 | def down 7 | execute "drop index dimensions_editions_base_path" 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20181122134132_remove_default_zeros_from_facts_metrics.rb: -------------------------------------------------------------------------------- 1 | class RemoveDefaultZerosFromFactsMetrics < ActiveRecord::Migration[5.2] 2 | COLS = %i[pviews upviews searches feedex exits entrances bounces bounce_rate avg_page_time page_time].freeze 3 | 4 | def up 5 | COLS.each do |col| 6 | change_column_default(:facts_metrics, col, nil) 7 | end 8 | end 9 | 10 | def down 11 | COLS.each do |col| 12 | change_column_default(:facts_metrics, col, 0) 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /db/migrate/20181122142744_remove_default_zeros_from_events_gas.rb: -------------------------------------------------------------------------------- 1 | class RemoveDefaultZerosFromEventsGas < ActiveRecord::Migration[5.2] 2 | COLS = %i[pviews upviews searches exits entrances bounces bounce_rate avg_page_time page_time].freeze 3 | 4 | def up 5 | COLS.each do |col| 6 | change_column_default(:events_gas, col, nil) 7 | end 8 | end 9 | 10 | def down 11 | COLS.each do |col| 12 | change_column_default(:events_gas, col, 0) 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /db/migrate/20181122152546_remove_default_zeros_from_aggregations_monthly_metrics.rb: -------------------------------------------------------------------------------- 1 | class RemoveDefaultZerosFromAggregationsMonthlyMetrics < ActiveRecord::Migration[5.2] 2 | COLS = %i[pviews upviews feedex searches exits entrances bounces bounce_rate avg_page_time page_time].freeze 3 | 4 | def up 5 | COLS.each do |col| 6 | change_column_default(:aggregations_monthly_metrics, col, nil) 7 | change_column_null(:aggregations_monthly_metrics, col, true) 8 | end 9 | end 10 | 11 | def down 12 | COLS.each do |col| 13 | change_column_default(:aggregations_monthly_metrics, col, 0) 14 | change_column_null(:aggregations_monthly_metrics, col, false) 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /db/migrate/20181126095403_recreate_publishing_api_events.rb: -------------------------------------------------------------------------------- 1 | class RecreatePublishingApiEvents < ActiveRecord::Migration[5.2] 2 | def change 3 | create_table :publishing_api_events do |t| 4 | t.string :routing_key 5 | t.jsonb :payload 6 | 7 | t.timestamps 8 | end 9 | 10 | add_reference :dimensions_editions, :publishing_api_event, foreign_key: true, index: true 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /db/migrate/20181126120610_update_aggregations_search_last_thirty_days_to_version_2.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThirtyDaysToVersion2 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_thirty_days, version: 2, revert_to_version: 1, materialized: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181126135420_update_aggregations_search_last_six_months_to_version_2.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastSixMonthsToVersion2 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_six_months, version: 2, revert_to_version: 1, materialized: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181126151050_update_aggregations_search_last_three_months_to_version_2.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThreeMonthsToVersion2 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_three_months, version: 2, revert_to_version: 1, materialized: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181126152543_remove_raw_json_from_dimension_editions.rb: -------------------------------------------------------------------------------- 1 | class RemoveRawJsonFromDimensionEditions < ActiveRecord::Migration[5.2] 2 | def change 3 | remove_column :dimensions_editions, :raw_json 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181126152658_update_aggregations_search_last_twelve_months_to_version_2.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastTwelveMonthsToVersion2 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_twelve_months, version: 2, revert_to_version: 1, materialized: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181129111315_add_default_zeros_to_events_gas.rb: -------------------------------------------------------------------------------- 1 | class AddDefaultZerosToEventsGas < ActiveRecord::Migration[5.2] 2 | COLS = %i[pviews upviews searches exits entrances bounces bounce_rate avg_page_time page_time].freeze 3 | 4 | def up 5 | COLS.each do |col| 6 | change_column_default(:events_gas, col, 0) 7 | end 8 | end 9 | 10 | def down 11 | COLS.each do |col| 12 | change_column_default(:events_gas, col, nil) 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /db/migrate/20181129115927_add_default_to_events_feedex.rb: -------------------------------------------------------------------------------- 1 | class AddDefaultToEventsFeedex < ActiveRecord::Migration[5.2] 2 | def change 3 | change_column_default :events_feedexes, :feedex_comments, from: nil, to: 0 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181129125513_add_not_null_constraint_to_facts_metrics.rb: -------------------------------------------------------------------------------- 1 | class AddNotNullConstraintToFactsMetrics < ActiveRecord::Migration[5.2] 2 | COLS = %i[pviews upviews feedex searches exits entrances bounce_rate avg_page_time bounces page_time].freeze 3 | 4 | def up 5 | COLS.each do |col| 6 | change_column_null :facts_metrics, col, false 7 | change_column_default :facts_metrics, col, 0 8 | end 9 | end 10 | 11 | def down 12 | COLS.each do |col| 13 | change_column_null :facts_metrics, col, true 14 | change_column_default :facts_metrics, col, nil 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /db/migrate/20181204140034_add_unique_index_to_last_thirty_days.rb: -------------------------------------------------------------------------------- 1 | class AddUniqueIndexToLastThirtyDays < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :aggregations_search_last_thirty_days, %i(dimensions_edition_id warehouse_item_id), unique: true, name: "index_on_search_last_thirty_days_unique" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181204140917_refresh_view_last_thirty_days.rb: -------------------------------------------------------------------------------- 1 | class RefreshViewLastThirtyDays < ActiveRecord::Migration[5.2] 2 | def change 3 | Aggregations::SearchLastThirtyDays.refresh 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181204142533_add_unique_index_to_last_month.rb: -------------------------------------------------------------------------------- 1 | class AddUniqueIndexToLastMonth < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :aggregations_search_last_months, %i(dimensions_edition_id warehouse_item_id), unique: true, name: "index_on_search_last_monthunique" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181204142554_refresh_view_last_month.rb: -------------------------------------------------------------------------------- 1 | class RefreshViewLastMonth < ActiveRecord::Migration[5.2] 2 | def change 3 | Aggregations::SearchLastMonth.refresh 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181204143719_add_unique_index_to_last_three_months.rb: -------------------------------------------------------------------------------- 1 | class AddUniqueIndexToLastThreeMonths < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :aggregations_search_last_three_months, %i(dimensions_edition_id warehouse_item_id), unique: true, name: "index_on_search_last_three_months_unique" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181204143802_refresh_view_last_three_months.rb: -------------------------------------------------------------------------------- 1 | class RefreshViewLastThreeMonths < ActiveRecord::Migration[5.2] 2 | def change 3 | Aggregations::SearchLastThreeMonths.refresh 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181204144322_add_unique_index_to_last_six_months.rb: -------------------------------------------------------------------------------- 1 | class AddUniqueIndexToLastSixMonths < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :aggregations_search_last_six_months, %i(dimensions_edition_id warehouse_item_id), unique: true, name: "index_on_search_last_six_months_unique" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181204144330_refresh_view_last_six_months.rb: -------------------------------------------------------------------------------- 1 | class RefreshViewLastSixMonths < ActiveRecord::Migration[5.2] 2 | def change 3 | Aggregations::SearchLastSixMonths.refresh 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181204145152_add_unique_index_to_last_twelve_months.rb: -------------------------------------------------------------------------------- 1 | class AddUniqueIndexToLastTwelveMonths < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :aggregations_search_last_twelve_months, %i(dimensions_edition_id warehouse_item_id), unique: true, name: "index_on_search_last_twelve_months_unique" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181204145159_refresh_view_last_twelve_months.rb: -------------------------------------------------------------------------------- 1 | class RefreshViewLastTwelveMonths < ActiveRecord::Migration[5.2] 2 | def change 3 | Aggregations::SearchLastTwelveMonths.refresh 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181205154240_update_aggregations_search_last_thirty_days_to_version_3.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThirtyDaysToVersion3 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_thirty_days, version: 3, revert_to_version: 2, materialized: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181205165620_update_aggregations_search_last_three_months_to_version_3.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThreeMonthsToVersion3 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_three_months, version: 3, revert_to_version: 2, materialized: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181206115804_update_aggregations_search_last_six_months_to_version_3.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastSixMonthsToVersion3 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_six_months, version: 3, revert_to_version: 2, materialized: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181206120629_update_aggregations_search_last_twelve_months_to_version_3.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastTwelveMonthsToVersion3 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_twelve_months, version: 3, revert_to_version: 2, materialized: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181211114147_update_aggregations_search_last_thirty_days_to_version_4.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThirtyDaysToVersion4 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_thirty_days, version: 4, revert_to_version: 3, materialized: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181211115100_update_aggregations_search_last_months_to_version_2.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastMonthsToVersion2 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_months, 4 | version: 2, 5 | revert_to_version: 1, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20181211115357_update_aggregations_search_last_three_months_to_version_4.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThreeMonthsToVersion4 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_three_months, 4 | version: 4, 5 | revert_to_version: 3, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20181211115944_update_aggregations_search_last_six_months_to_version_4.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastSixMonthsToVersion4 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_six_months, 4 | version: 4, 5 | revert_to_version: 3, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20181211120306_update_aggregations_search_last_twelve_months_to_version_4.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastTwelveMonthsToVersion4 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_twelve_months, 4 | version: 4, 5 | revert_to_version: 3, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20190104120051_add_not_null_constraint_and_index_to_document_types.rb: -------------------------------------------------------------------------------- 1 | class AddNotNullConstraintAndIndexToDocumentTypes < ActiveRecord::Migration[5.2] 2 | def change 3 | change_column_null :dimensions_editions, :document_type, false 4 | add_index :dimensions_editions, :document_type 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20190116142924_remove_content_purpose_document_supertype_column_from_dimensions_editions.rb: -------------------------------------------------------------------------------- 1 | class RemoveContentPurposeDocumentSupertypeColumnFromDimensionsEditions < ActiveRecord::Migration[5.2] 2 | def change 3 | remove_column :dimensions_editions, :content_purpose_document_supertype, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20190117123913_remove_bounce_rate.rb: -------------------------------------------------------------------------------- 1 | class RemoveBounceRate < ActiveRecord::Migration[5.2] 2 | def change 3 | remove_column :aggregations_monthly_metrics, :bounce_rate 4 | remove_column :events_gas, :bounce_rate 5 | remove_column :facts_metrics, :bounce_rate 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20190117125756_remove_avg_page_time.rb: -------------------------------------------------------------------------------- 1 | class RemoveAvgPageTime < ActiveRecord::Migration[5.2] 2 | def change 3 | remove_column :aggregations_monthly_metrics, :avg_page_time 4 | remove_column :events_gas, :avg_page_time 5 | remove_column :facts_metrics, :avg_page_time 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20190118140551_add_acronym_to_dimensions_editions.rb: -------------------------------------------------------------------------------- 1 | class AddAcronymToDimensionsEditions < ActiveRecord::Migration[5.2] 2 | def change 3 | add_column :dimensions_editions, :acronym, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20190118145538_update_acronym_for_organisations.rb: -------------------------------------------------------------------------------- 1 | class UpdateAcronymForOrganisations < ActiveRecord::Migration[5.2] 2 | def up 3 | Dimensions::Edition.where(document_type: "organisation").find_each do |edition| 4 | event = edition.publishing_api_event 5 | acronym = event.payload.dig("details", "acronym") 6 | 7 | edition.update(acronym: acronym.blank? ? nil : acronym) 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20190204102543_remove_content_supergroups_and_subgroups.rb: -------------------------------------------------------------------------------- 1 | class RemoveContentSupergroupsAndSubgroups < ActiveRecord::Migration[5.2] 2 | def change 3 | remove_column :dimensions_editions, :content_purpose_supergroup 4 | remove_column :dimensions_editions, :content_purpose_subgroup 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20190204150122_add_reading_time_to_facts_edition.rb: -------------------------------------------------------------------------------- 1 | class AddReadingTimeToFactsEdition < ActiveRecord::Migration[5.2] 2 | def change 3 | add_column :facts_editions, :reading_time, :integer 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20190204171432_update_reading_time.rb: -------------------------------------------------------------------------------- 1 | class UpdateReadingTime < ActiveRecord::Migration[5.2] 2 | class Facts::Edition < ApplicationRecord 3 | end 4 | 5 | class Dimensions::Edition < ApplicationRecord 6 | has_one :facts_edition, class_name: "Facts::Edition", foreign_key: :dimensions_edition_id 7 | end 8 | 9 | def up 10 | Dimensions::Edition.where(latest: true).find_each do |edition| 11 | facts = edition.facts_edition 12 | if facts.words.present? && facts.words.positive? 13 | reading_time = Etl::Edition::Content::ReadingTime.calculate(facts.words) 14 | facts.update(reading_time: reading_time) 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /db/migrate/20190207143213_add_index_facts_metrics_pviews.rb: -------------------------------------------------------------------------------- 1 | class AddIndexFactsMetricsPviews < ActiveRecord::Migration[5.2] 2 | disable_ddl_transaction! 3 | 4 | def change 5 | add_index :facts_metrics, %i(dimensions_date_id pviews), algorithm: :concurrently 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20190207143221_add_index_facts_metrics_upviews.rb: -------------------------------------------------------------------------------- 1 | class AddIndexFactsMetricsUpviews < ActiveRecord::Migration[5.2] 2 | disable_ddl_transaction! 3 | 4 | def change 5 | add_index :facts_metrics, %i(dimensions_date_id upviews), algorithm: :concurrently 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20190207143240_add_index_facts_metrics_searches.rb: -------------------------------------------------------------------------------- 1 | class AddIndexFactsMetricsSearches < ActiveRecord::Migration[5.2] 2 | disable_ddl_transaction! 3 | 4 | def change 5 | add_index :facts_metrics, %i(dimensions_date_id searches), algorithm: :concurrently 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20190207143301_add_index_facts_metrics_useful_no.rb: -------------------------------------------------------------------------------- 1 | class AddIndexFactsMetricsUsefulNo < ActiveRecord::Migration[5.2] 2 | disable_ddl_transaction! 3 | 4 | def change 5 | add_index :facts_metrics, %i(dimensions_date_id useful_no), algorithm: :concurrently 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20190207143313_add_index_facts_metrics_useful_feedex.rb: -------------------------------------------------------------------------------- 1 | class AddIndexFactsMetricsUsefulFeedex < ActiveRecord::Migration[5.2] 2 | disable_ddl_transaction! 3 | 4 | def change 5 | add_index :facts_metrics, %i(dimensions_date_id feedex), algorithm: :concurrently 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20190213115655_update_aggregations_search_last_thirty_days_to_version_5.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThirtyDaysToVersion5 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_thirty_days, version: 5, revert_to_version: 4, materialized: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20190213141732_update_aggregations_search_last_months_to_version_3.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastMonthsToVersion3 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_months, version: 3, revert_to_version: 2, materialized: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20190213141743_update_aggregations_search_last_three_months_to_version_5.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThreeMonthsToVersion5 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_three_months, version: 5, revert_to_version: 4, materialized: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20190213141753_update_aggregations_search_last_six_months_to_version_5.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastSixMonthsToVersion5 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_six_months, version: 5, revert_to_version: 4, materialized: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20190213141801_update_aggregations_search_last_twelve_months_to_version_5.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastTwelveMonthsToVersion5 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_twelve_months, version: 5, revert_to_version: 4, materialized: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20190218113118_update_aggregations_search_last_thirty_days_to_version_6.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThirtyDaysToVersion6 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_thirty_days, 4 | version: 6, 5 | revert_to_version: 5, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20190218121734_update_aggregations_search_last_months_to_version_4.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastMonthsToVersion4 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_months, 4 | version: 4, 5 | revert_to_version: 3, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20190218122820_update_aggregations_search_last_three_months_to_version_6.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThreeMonthsToVersion6 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_three_months, 4 | version: 6, 5 | revert_to_version: 5, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20190218123800_update_aggregations_search_last_six_months_to_version_6.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastSixMonthsToVersion6 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_six_months, 4 | version: 6, 5 | revert_to_version: 5, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20190218124214_update_aggregations_search_last_twelve_months_to_version_6.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastTwelveMonthsToVersion6 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_twelve_months, 4 | version: 6, 5 | revert_to_version: 5, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20190225113933_add_index_to_monthly_aggregation_created_at.rb: -------------------------------------------------------------------------------- 1 | class AddIndexToMonthlyAggregationCreatedAt < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :aggregations_monthly_metrics, :created_at 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20190304151614_remove_default_satisfaction.rb: -------------------------------------------------------------------------------- 1 | class RemoveDefaultSatisfaction < ActiveRecord::Migration[5.2] 2 | def up 3 | change_column :facts_metrics, :satisfaction, :float, null: true 4 | change_column_default :facts_metrics, :satisfaction, nil 5 | 6 | change_column :aggregations_monthly_metrics, :satisfaction, :float, null: true 7 | change_column_default :aggregations_monthly_metrics, :satisfaction, nil 8 | end 9 | 10 | def down 11 | change_column :facts_metrics, :satisfaction, :float, null: false 12 | change_column_default :facts_metrics, :satisfaction, 0.0 13 | 14 | change_column :aggregations_monthly_metrics, :satisfaction, :float, null: false 15 | change_column_default :aggregations_monthly_metrics, :satisfaction, 0.0 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /db/migrate/20190311163406_update_aggregations_search_last_twelve_months_to_version_7.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastTwelveMonthsToVersion7 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_twelve_months, 4 | version: 7, 5 | revert_to_version: 6, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20190311164157_update_aggregations_search_last_six_months_to_version_7.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastSixMonthsToVersion7 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_six_months, 4 | version: 7, 5 | revert_to_version: 6, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20190311164340_update_aggregations_search_last_three_months_to_version_7.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThreeMonthsToVersion7 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_three_months, 4 | version: 7, 5 | revert_to_version: 6, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20190311164541_update_aggregations_search_last_months_to_version_5.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastMonthsToVersion5 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_months, 4 | version: 5, 5 | revert_to_version: 4, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20190311164701_update_aggregations_search_last_thirty_days_to_version_7.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThirtyDaysToVersion7 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_thirty_days, 4 | version: 7, 5 | revert_to_version: 6, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20190312140803_update_aggregations_search_last_thirty_days_to_version_8.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThirtyDaysToVersion8 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_thirty_days, 4 | version: 8, 5 | revert_to_version: 7, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20190312141411_update_aggregations_search_last_months_to_version_6.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastMonthsToVersion6 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_months, 4 | version: 6, 5 | revert_to_version: 5, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20190312141423_update_aggregations_search_last_six_months_to_version_8.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastSixMonthsToVersion8 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_six_months, 4 | version: 8, 5 | revert_to_version: 7, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20190312141431_update_aggregations_search_last_three_months_to_version_8.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThreeMonthsToVersion8 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_three_months, 4 | version: 8, 5 | revert_to_version: 7, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20190312141439_update_aggregations_search_last_twelve_months_to_version_8.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastTwelveMonthsToVersion8 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view :aggregations_search_last_twelve_months, 4 | version: 8, 5 | revert_to_version: 7, 6 | materialized: true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20190402143148_rename_latest_column_to_live.rb: -------------------------------------------------------------------------------- 1 | class RenameLatestColumnToLive < ActiveRecord::Migration[5.2] 2 | def up 3 | rename_column :dimensions_editions, :latest, :live 4 | end 5 | 6 | def down 7 | rename_column :dimensions_editions, :live, :latest 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20190603080748_rename_dimensions_edition_organisation_id.rb: -------------------------------------------------------------------------------- 1 | class RenameDimensionsEditionOrganisationId < ActiveRecord::Migration[5.2] 2 | def up 3 | rename_column :dimensions_editions, :organisation_id, :primary_organisation_id 4 | end 5 | 6 | def down 7 | rename_column :dimensions_editions, :primary_organisation_id, :organisation_id 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20190603083348_update_aggregations_search_last_months_to_version_7.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastMonthsToVersion7 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view( 4 | :aggregations_search_last_months, 5 | version: 7, 6 | revert_to_version: 6, 7 | materialized: true, 8 | ) 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20190603083456_update_aggregations_search_last_thirty_days_to_version_9.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThirtyDaysToVersion9 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view( 4 | :aggregations_search_last_thirty_days, 5 | version: 9, 6 | revert_to_version: 8, 7 | materialized: true, 8 | ) 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20190603083504_update_aggregations_search_last_three_months_to_version_9.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThreeMonthsToVersion9 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view( 4 | :aggregations_search_last_three_months, 5 | version: 9, 6 | revert_to_version: 8, 7 | materialized: true, 8 | ) 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20190603083515_update_aggregations_search_last_twelve_months_to_version_9.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastTwelveMonthsToVersion9 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view( 4 | :aggregations_search_last_twelve_months, 5 | version: 9, 6 | revert_to_version: 8, 7 | materialized: true, 8 | ) 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20190603083643_update_aggregations_search_last_six_months_to_version_9.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastSixMonthsToVersion9 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view( 4 | :aggregations_search_last_six_months, 5 | version: 9, 6 | revert_to_version: 8, 7 | materialized: true, 8 | ) 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20190603133313_add_organisation_ids_to_dimensions_edition.rb: -------------------------------------------------------------------------------- 1 | class AddOrganisationIdsToDimensionsEdition < ActiveRecord::Migration[5.2] 2 | def change 3 | add_column :dimensions_editions, :organisation_ids, :string, array: true, default: [] 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20190603145810_update_aggregations_search_last_months_to_version_8.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastMonthsToVersion8 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view( 4 | :aggregations_search_last_months, 5 | version: 8, 6 | revert_to_version: 7, 7 | materialized: true, 8 | ) 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20190603145828_update_aggregations_search_last_thirty_days_to_version_10.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThirtyDaysToVersion10 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view( 4 | :aggregations_search_last_thirty_days, 5 | version: 10, 6 | revert_to_version: 9, 7 | materialized: true, 8 | ) 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20190603145838_update_aggregations_search_last_three_months_to_version_10.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastThreeMonthsToVersion10 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view( 4 | :aggregations_search_last_three_months, 5 | version: 10, 6 | revert_to_version: 9, 7 | materialized: true, 8 | ) 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20190603145851_update_aggregations_search_last_twelve_months_to_version_10.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastTwelveMonthsToVersion10 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view( 4 | :aggregations_search_last_twelve_months, 5 | version: 10, 6 | revert_to_version: 9, 7 | materialized: true, 8 | ) 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20190603145901_update_aggregations_search_last_six_months_to_version_10.rb: -------------------------------------------------------------------------------- 1 | class UpdateAggregationsSearchLastSixMonthsToVersion10 < ActiveRecord::Migration[5.2] 2 | def change 3 | update_view( 4 | :aggregations_search_last_six_months, 5 | version: 10, 6 | revert_to_version: 9, 7 | materialized: true, 8 | ) 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20190604105034_create_lower_index_on_base_path.rb: -------------------------------------------------------------------------------- 1 | class CreateLowerIndexOnBasePath < ActiveRecord::Migration[5.2] 2 | def up 3 | execute <<-SQL 4 | CREATE INDEX index_lower_base_path ON dimensions_editions (lower(base_path)); 5 | SQL 6 | end 7 | 8 | def down 9 | execute <<-SQL 10 | DROP INDEX index_lower_base_path; 11 | SQL 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /db/migrate/20190612150721_add_parent_id_to_dimension_edition.rb: -------------------------------------------------------------------------------- 1 | class AddParentIdToDimensionEdition < ActiveRecord::Migration[5.2] 2 | def change 3 | add_column :dimensions_editions, :parent_id, :integer, null: true 4 | 5 | add_index :dimensions_editions, :parent_id, name: "index_dim_edition_parent_id" 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrate/20190613082656_add_sibling_order_to_dimensions_edition.rb: -------------------------------------------------------------------------------- 1 | class AddSiblingOrderToDimensionsEdition < ActiveRecord::Migration[5.2] 2 | def change 3 | add_column :dimensions_editions, :sibling_order, :integer, null: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20190613112120_add_child_sort_order_to_dimensions_editions.rb: -------------------------------------------------------------------------------- 1 | class AddChildSortOrderToDimensionsEditions < ActiveRecord::Migration[5.2] 2 | def change 3 | add_column :dimensions_editions, :child_sort_order, :string, array: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/views/aggregations_search_last_months_v01.sql: -------------------------------------------------------------------------------- 1 | SELECT warehouse_item_id, 2 | max(aggregations_monthly_metrics.dimensions_edition_id) AS dimensions_edition_id, 3 | sum(aggregations_monthly_metrics.upviews) AS upviews, 4 | sum(aggregations_monthly_metrics.useful_yes) AS useful_yes, 5 | sum(aggregations_monthly_metrics.useful_no) AS useful_no, 6 | sum(aggregations_monthly_metrics.searches) AS searches 7 | FROM aggregations_monthly_metrics 8 | INNER JOIN dimensions_months ON dimensions_months.id = aggregations_monthly_metrics.dimensions_month_id 9 | INNER JOIN dimensions_editions ON dimensions_editions.id = aggregations_monthly_metrics.dimensions_edition_id 10 | WHERE dimensions_months.id = to_char(NOW() - interval '1 month','YYYY-MM') 11 | GROUP BY warehouse_item_id -------------------------------------------------------------------------------- /db/views/aggregations_search_last_thirty_days_v01.sql: -------------------------------------------------------------------------------- 1 | SELECT warehouse_item_id, 2 | max(facts_metrics.dimensions_edition_id) AS dimensions_edition_id, 3 | sum(facts_metrics.upviews) AS upviews, 4 | sum(facts_metrics.useful_yes) AS useful_yes, 5 | sum(facts_metrics.useful_no) AS useful_no, 6 | sum(facts_metrics.searches) AS searches 7 | FROM facts_metrics 8 | JOIN dimensions_dates ON dimensions_dates.date = facts_metrics.dimensions_date_id 9 | JOIN dimensions_editions ON dimensions_editions.id = facts_metrics.dimensions_edition_id 10 | WHERE facts_metrics.dimensions_date_id >= (current_date - '30 days'::interval day) 11 | GROUP BY warehouse_item_id -------------------------------------------------------------------------------- /db/views/aggregations_search_last_thirty_days_v02.sql: -------------------------------------------------------------------------------- 1 | SELECT warehouse_item_id, 2 | max(facts_metrics.dimensions_edition_id) AS dimensions_edition_id, 3 | sum(facts_metrics.upviews) AS upviews, 4 | sum(facts_metrics.useful_yes) AS useful_yes, 5 | sum(facts_metrics.useful_no) AS useful_no, 6 | sum(facts_metrics.searches) AS searches 7 | FROM facts_metrics 8 | JOIN dimensions_dates ON dimensions_dates.date = facts_metrics.dimensions_date_id 9 | JOIN dimensions_editions ON dimensions_editions.id = facts_metrics.dimensions_edition_id 10 | WHERE (facts_metrics.dimensions_date_id >= (('yesterday'::text)::date - '30 days'::interval day)) 11 | AND (facts_metrics.dimensions_date_id < ('now'::text)::date) 12 | GROUP BY warehouse_item_id -------------------------------------------------------------------------------- /db/views/aggregations_search_last_thirty_days_v03.sql: -------------------------------------------------------------------------------- 1 | SELECT warehouse_item_id, 2 | max(facts_metrics.dimensions_edition_id) AS dimensions_edition_id, 3 | sum(facts_metrics.upviews) AS upviews, 4 | sum(facts_metrics.useful_yes) AS useful_yes, 5 | sum(facts_metrics.useful_no) AS useful_no, 6 | sum(facts_metrics.searches) AS searches 7 | FROM facts_metrics 8 | JOIN dimensions_dates ON dimensions_dates.date = facts_metrics.dimensions_date_id 9 | JOIN dimensions_editions ON dimensions_editions.id = facts_metrics.dimensions_edition_id 10 | WHERE (facts_metrics.dimensions_date_id > (('yesterday'::text)::date - '30 days'::interval day)) 11 | AND (facts_metrics.dimensions_date_id < ('now'::text)::date) 12 | GROUP BY warehouse_item_id -------------------------------------------------------------------------------- /db/views/aggregations_search_last_thirty_days_v04.sql: -------------------------------------------------------------------------------- 1 | SELECT warehouse_item_id, 2 | max(facts_metrics.dimensions_edition_id) AS dimensions_edition_id, 3 | sum(facts_metrics.upviews) AS upviews, 4 | sum(facts_metrics.pviews) AS pviews, 5 | sum(facts_metrics.useful_yes) AS useful_yes, 6 | sum(facts_metrics.useful_no) AS useful_no, 7 | sum(facts_metrics.feedex) AS feedex, 8 | sum(facts_metrics.searches) AS searches 9 | FROM facts_metrics 10 | JOIN dimensions_dates ON dimensions_dates.date = facts_metrics.dimensions_date_id 11 | JOIN dimensions_editions ON dimensions_editions.id = facts_metrics.dimensions_edition_id 12 | WHERE (facts_metrics.dimensions_date_id > (('yesterday'::text)::date - '30 days'::interval day)) 13 | AND (facts_metrics.dimensions_date_id < ('now'::text)::date) 14 | GROUP BY warehouse_item_id -------------------------------------------------------------------------------- /docs/arch/adr-000-document-architectural-decisions.md: -------------------------------------------------------------------------------- 1 | # ADR 000: Document architectural decisions 2 | 3 | ## Context 4 | 5 | We aim to: 6 | 7 | - Make it easier to understand the codebase and its status 8 | - Reduce the number of meetings to handover information across teams 9 | - Facilitate team rotations across GOV.UK 10 | 11 | ## Decision 12 | 13 | Track architectural decision that impact the status of the CPM, [following a lightweight format: ADR][1] 14 | 15 | ## Status 16 | 17 | Accepted. 18 | 19 | [1]: http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions 20 | -------------------------------------------------------------------------------- /docs/arch/adr-008-focus-on-english-content.md: -------------------------------------------------------------------------------- 1 | # ADR 008: Focus on english content in the first iterations. 2 | 3 | 28-01-2018 4 | 5 | ## Context 6 | 7 | Some Content Items are written in different languages, so [Publishing-api][1] will return the `content_id` along with all locales assigned to the Content Item. 8 | 9 | ## Decision 10 | 11 | Focus on content written in English. 12 | 13 | The main reason is that we would need different algorithms and libraries to make our application consistent among all the languages / locales. 14 | If this is a real need, we will support it in future iterations of the Data Warehouse. 15 | 16 | ### Benefits: 17 | 18 | This makes the codebase simpler. 19 | 20 | ## Status 21 | 22 | Accepted. 23 | 24 | [1]: http://github.com/alphagov/publishing-api 25 | -------------------------------------------------------------------------------- /docs/arch/adr-016-rename-content-performance-manager.md: -------------------------------------------------------------------------------- 1 | # ADR 016: Rename Content Performance Manager to Content Data API 2 | 3 | 01-04-2019 4 | 5 | ## Context 6 | 7 | The data warehouse is no longer an application with an UI. It is an API that is 8 | consumed by other applications in order to get metrics through time for content 9 | items of GOV.UK. 10 | 11 | The name of the application is not compliant with GOV.UK guidelines for naming. 12 | 13 | ## Decision 14 | 15 | Rename Content Performance Manager to Content Data Admin 16 | 17 | ## Consequences 18 | 19 | The Github repo, code, servers names and puppet configuration needs to be 20 | updated. It is extra work but we think that in the long run it will be 21 | benefitial for GOV.UK and the product. 22 | 23 | ## Status 24 | 25 | Accepted. 26 | -------------------------------------------------------------------------------- /docs/arch/images/content_etl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphagov/content-data-api/a0f2ff6f1f4f664344e2ecb5c991ac2cbf0bd5e5/docs/arch/images/content_etl.png -------------------------------------------------------------------------------- /docs/images/fig-01-no-trends.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphagov/content-data-api/a0f2ff6f1f4f664344e2ecb5c991ac2cbf0bd5e5/docs/images/fig-01-no-trends.png -------------------------------------------------------------------------------- /docs/images/fig-02-with-trends.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphagov/content-data-api/a0f2ff6f1f4f664344e2ecb5c991ac2cbf0bd5e5/docs/images/fig-02-with-trends.png -------------------------------------------------------------------------------- /docs/images/fig-03-comparisons.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphagov/content-data-api/a0f2ff6f1f4f664344e2ecb5c991ac2cbf0bd5e5/docs/images/fig-03-comparisons.png -------------------------------------------------------------------------------- /docs/images/fig-04-predictions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphagov/content-data-api/a0f2ff6f1f4f664344e2ecb5c991ac2cbf0bd5e5/docs/images/fig-04-predictions.png -------------------------------------------------------------------------------- /docs/images/schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphagov/content-data-api/a0f2ff6f1f4f664344e2ecb5c991ac2cbf0bd5e5/docs/images/schema.png -------------------------------------------------------------------------------- /docs/import_production_data.md: -------------------------------------------------------------------------------- 1 | # Import Production Data 2 | 3 | To import production data, follow the instructions for 4 | [govuk-docker](https://github.com/alphagov/govuk-docker#how-to-replicate-data-locally) 5 | -------------------------------------------------------------------------------- /docs/requeue_all_content.md: -------------------------------------------------------------------------------- 1 | # Re-queue all content from the Publishing API 2 | 3 | This is a rake task in the Publishing API. It will queue messages to RabbitMQ that the Content Data API will consume. This was initially created to bootstrap the data in the Content Data API, and shouldn't need re-running. 4 | 5 | To run the rake task, use the `run_rake_task` job on the deployment Jenkins and provide the routing key as below: 6 | 7 | ```bash 8 | queue:requeue_all_the_things[bulk.data-warehouse] 9 | ``` 10 | 11 | ## Details 12 | 13 | * PR to add [Task to queue messages for all content](https://github.com/alphagov/publishing-api/pull/1242) 14 | * Source for the [rake task](https://github.com/alphagov/publishing-api/blob/master/lib/tasks/queue.rake#L53) -------------------------------------------------------------------------------- /lib/tasks/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphagov/content-data-api/a0f2ff6f1f4f664344e2ecb5c991ac2cbf0bd5e5/lib/tasks/.keep -------------------------------------------------------------------------------- /lib/tasks/publishing_api_consumer.rake: -------------------------------------------------------------------------------- 1 | namespace :publishing_api do 2 | desc "Run worker to publishing API from rabbitmq" 3 | task consumer: :environment do 4 | GovukMessageQueueConsumer::Consumer.new( 5 | queue_name: ENV["RABBITMQ_QUEUE"], 6 | processor: Streams::Consumer.new, 7 | ).run 8 | rescue SignalException 9 | logger.info "SignalException suppressed" 10 | end 11 | 12 | desc "Run worker to publishing API from rabbitmq for bulk import" 13 | task bulk_import_consumer: :environment do 14 | GovukMessageQueueConsumer::Consumer.new( 15 | queue_name: ENV["RABBITMQ_QUEUE_BULK"], 16 | processor: Streams::Consumer.new, 17 | ).run 18 | rescue SignalException 19 | logger.info "SignalException suppressed" 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /log/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphagov/content-data-api/a0f2ff6f1f4f664344e2ecb5c991ac2cbf0bd5e5/log/.keep -------------------------------------------------------------------------------- /public/apple-touch-icon-precomposed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphagov/content-data-api/a0f2ff6f1f4f664344e2ecb5c991ac2cbf0bd5e5/public/apple-touch-icon-precomposed.png -------------------------------------------------------------------------------- /public/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphagov/content-data-api/a0f2ff6f1f4f664344e2ecb5c991ac2cbf0bd5e5/public/apple-touch-icon.png -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphagov/content-data-api/a0f2ff6f1f4f664344e2ecb5c991ac2cbf0bd5e5/public/favicon.ico -------------------------------------------------------------------------------- /public/robots.txt: -------------------------------------------------------------------------------- 1 | # See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file 2 | -------------------------------------------------------------------------------- /spec/domain/etl/edition/content/email_alert_signup_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Etl::Edition::Content::Parser do 2 | subject { described_class } 3 | 4 | describe "EmailAlertSignup" do 5 | it "returns summary if json does not have 'breadcrumbs' key" do 6 | json = { schema_name: "email_alert_signup", 7 | details: { summary: "Summary" } } 8 | expect(subject.extract_content(json.deep_stringify_keys)).to eq("Summary") 9 | end 10 | 11 | it "returns content json if schema_name is 'email_alert_signup'" do 12 | json = { schema_name: "email_alert_signup", 13 | details: { breadcrumbs: [{ title: "The title" }], 14 | summary: "Summary" } } 15 | expect(subject.extract_content(json.deep_stringify_keys)).to eq("The title Summary") 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /spec/domain/etl/edition/content/finder_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Etl::Edition::Content::Parser do 2 | subject { described_class } 3 | 4 | it "returns content json if schema_name is 'finder'" do 5 | json = { schema_name: "finder", 6 | title: "Contact HMRC", 7 | links: { children: [ 8 | { title: "Personal Tax", description: "Email, write or phone us" }, 9 | { title: "Child Benefit", description: "Tweet us" }, 10 | ] } } 11 | expected = "Contact HMRC Personal Tax Email, write or phone us Child Benefit Tweet us" 12 | expect(subject.extract_content(json.deep_stringify_keys)).to eql(expected) 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /spec/domain/etl/edition/content/generic_with_links_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Etl::Edition::Content::Parser do 2 | subject { described_class } 3 | 4 | it "returns nil if json does not have 'external_related_links' key" do 5 | json = { schema_name: "generic_with_external_related_links", 6 | details: {} } 7 | expect(subject.extract_content(json.deep_stringify_keys)).to eq(nil) 8 | end 9 | 10 | it "returns content json if schema_name is 'generic_with_external_related_links'" do 11 | json = { schema_name: "generic_with_external_related_links", 12 | details: { external_related_links: [ 13 | { title: "Check your Council Tax band" }, 14 | ] } } 15 | expect(subject.extract_content(json.deep_stringify_keys)).to eq("Check your Council Tax band") 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /spec/domain/etl/edition/content/local_transaction_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Etl::Edition::Content::Parser do 2 | subject { described_class } 3 | 4 | it "returns content json if schema_name is 'local_transaction'" do 5 | json = { schema_name: "local_transaction", 6 | details: { introduction: "Greetings", 7 | need_to_know: "A Name", 8 | more_information: "An Address" } } 9 | expect(subject.extract_content(json.deep_stringify_keys)).to eq("Greetings A Name An Address") 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /spec/domain/etl/edition/content/need_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Etl::Edition::Content::Parser do 2 | subject { described_class } 3 | 4 | it "returns content if schema name is 'need'" do 5 | json = { 6 | schema_name: "need", 7 | details: { 8 | role: "the role", 9 | goal: "the goal", 10 | benefit: "the benefit", 11 | }, 12 | } 13 | expect(subject.extract_content(json.deep_stringify_keys)).to eq("the role the goal the benefit") 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /spec/domain/etl/edition/content/place_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Etl::Edition::Content::Parser do 2 | subject { described_class } 3 | 4 | it "returns content json if schema is 'place'" do 5 | json = { schema_name: "place", 6 | details: { introduction: "Introduction", 7 | more_information: "Enter your postcode" } } 8 | expect(subject.extract_content(json.deep_stringify_keys)).to eq("Introduction Enter your postcode") 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /spec/domain/etl/edition/content/reading_time_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Etl::Edition::Content::ReadingTime do 2 | subject { described_class } 3 | 4 | describe "#calculate" do 5 | it "calculate readability based on 200 words per minute" do 6 | expected_reading_time = 400 / 200 7 | 8 | expect(subject.calculate(400)).to eq(expected_reading_time) 9 | end 10 | 11 | it "rounds up to 1 reading times that are between 0 and 1 minute" do 12 | expect(subject.calculate(20)).to eq(1) 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /spec/domain/etl/edition/content/service_manual_homepage_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Etl::Edition::Content::Parser do 2 | subject { described_class } 3 | 4 | it "returns content json if schema_name is 'service_manual_homepage'" do 5 | json = { schema_name: "service_manual_homepage", 6 | title: "Service Manual", 7 | description: "Digital Service Standard", 8 | links: { children: [ 9 | { title: "Design", description: "Naming your service" }, 10 | { title: "Technology", description: "Security and Maintenance" }, 11 | ] } } 12 | 13 | expected = "Service Manual Digital Service Standard Design Naming your service Technology Security and Maintenance" 14 | expect(subject.extract_content(json.deep_stringify_keys)).to eql(expected) 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /spec/domain/etl/edition/content/statistics_announcement_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Etl::Edition::Content::Parser do 2 | subject { described_class } 3 | 4 | it "returns content json if schema_name is 'statistics_announcement'" do 5 | json = { schema_name: "statistics_announcement", 6 | description: "Announcement", 7 | details: { display_date: "25 December 2017", state: "closed" } } 8 | expect(subject.extract_content(json.deep_stringify_keys)).to eq("Announcement 25 December 2017 closed") 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /spec/domain/etl/edition/content/transaction_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Etl::Edition::Content::Parser do 2 | subject { described_class } 3 | 4 | it "returns content json if schema_name is 'transaction'" do 5 | json = { schema_name: "transaction", 6 | details: { introductory_paragraph: "Report changes", 7 | start_button_text: "Start", 8 | will_continue_on: "Carer's Allowance service", 9 | more_information: "Facts" } } 10 | expected = "Report changes Start Carer's Allowance service Facts" 11 | expect(subject.extract_content(json.deep_stringify_keys)).to eq(expected) 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /spec/domain/etl/edition/content/travel_advice_index_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Etl::Edition::Content::Parser do 2 | subject { described_class } 3 | 4 | it "returns nil if json does not have children array" do 5 | json = { schema_name: "travel_advice_index", 6 | links: {} } 7 | expect(subject.extract_content(json.deep_stringify_keys)).to eq(nil) 8 | end 9 | 10 | it "returns content json" do 11 | json = { schema_name: "travel_advice_index", 12 | links: { children: [ 13 | { country: { name: "Portugal" } }, 14 | { country: { name: "Brazil" } }, 15 | ] } } 16 | expect(subject.extract_content(json.deep_stringify_keys)).to eq("Portugal Brazil") 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /spec/domain/etl/edition/content/unpublished_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Etl::Edition::Content::Parser do 2 | subject { described_class } 3 | 4 | it "returns content json if schema_name is 'gone'" do 5 | json = { schema_name: "gone", 6 | details: { explanation: "No page here" } } 7 | expect(subject.extract_content(json.deep_stringify_keys)).to eq("No page here") 8 | end 9 | 10 | it "returns content json if schema_name is 'unpublishing'" do 11 | json = { schema_name: "unpublishing", 12 | details: { explanation: "This content has been removed" } } 13 | expect(subject.extract_content(json.deep_stringify_keys)).to eq("This content has been removed") 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /spec/domain/healthchecks/daily_metrics_check_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Healthchecks::DailyMetricsCheck do 2 | include_examples "Healthcheck enabled/disabled within time range" 3 | 4 | describe "#status" do 5 | context "When there are metrics" do 6 | before { create :metric, dimensions_date: Dimensions::Date.build(Date.yesterday) } 7 | 8 | it "returns status :ok" do 9 | expect(subject.status).to eq(:ok) 10 | end 11 | end 12 | 13 | context "When there are no metrics" do 14 | it "returns status :critical" do 15 | expect(subject.status).to eq(:critical) 16 | end 17 | 18 | it "returns a detailed message" do 19 | expect(subject.message).to eq("ETL :: no daily metrics for yesterday") 20 | end 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/domain/streams/consumer_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Streams::Consumer do 2 | subject { described_class.new } 3 | it "increments `discard` in statsd when routing key is not valid" do 4 | message = build(:message, routing_key: "invalid_routing_key") 5 | expect(GovukStatsd).to receive(:increment).with("monitor.messages.discarded") 6 | 7 | subject.process(message) 8 | end 9 | 10 | it "sends an error to GovukError" do 11 | error = StandardError.new 12 | allow(Streams::MessageProcessorJob).to receive(:perform_later).and_raise(error) 13 | expect(GovukError).to receive(:notify).with(error) 14 | 15 | subject.process(build(:message)) 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /spec/domain/streams/message_processor_job_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Streams::MessageProcessorJob do 2 | it "increments `monitor.messages.*` " do 3 | message = build(:message, routing_key: "news_story.major") 4 | expect(GovukStatsd).to receive(:increment).with("monitor.messages.major") 5 | 6 | subject.perform(message.payload, message.delivery_info.routing_key) 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /spec/factories/dimensions_date.rb: -------------------------------------------------------------------------------- 1 | FactoryBot.define do 2 | factory :dimensions_date, class: Dimensions::Date do 3 | sequence(:date) { |i| i.days.ago.to_date } 4 | 5 | initialize_with { Dimensions::Date.find_existing_or_create(date.to_date) } 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /spec/factories/dimensions_months.rb: -------------------------------------------------------------------------------- 1 | FactoryBot.define do 2 | factory :dimensions_month, class: "Dimensions::Month" do 3 | end 4 | end 5 | -------------------------------------------------------------------------------- /spec/factories/facts_editions.rb: -------------------------------------------------------------------------------- 1 | FactoryBot.define do 2 | factory :facts_edition, class: Facts::Edition do 3 | dimensions_date 4 | pdf_count { 0 } 5 | doc_count { 0 } 6 | sentences { 0 } 7 | words { 0 } 8 | chars { 0 } 9 | readability { 0 } 10 | reading_time { 0 } 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /spec/factories/feedexes.rb: -------------------------------------------------------------------------------- 1 | FactoryBot.define do 2 | factory :feedex, class: Events::Feedex do 3 | sequence(:page_path) { |i| "/path/#{i}" } 4 | sequence(:date) { |i| i.days.ago.to_date } 5 | feedex_comments { 1 } 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /spec/factories/ga_events.rb: -------------------------------------------------------------------------------- 1 | FactoryBot.define do 2 | factory :ga_event, class: Events::GA do 3 | trait :with_views do 4 | pviews { 10 } 5 | upviews { 5 } 6 | process_name { "views" } 7 | end 8 | 9 | trait :with_user_feedback do 10 | useful_yes { 3 } 11 | useful_no { 6 } 12 | process_name { "user_feedback" } 13 | end 14 | 15 | trait :with_searches do 16 | searches { 100 } 17 | process_name { "searches" } 18 | end 19 | 20 | sequence(:page_path) { |i| "/path/#{i}" } 21 | sequence(:date) { |i| i.days.ago.to_date } 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/factories/metrics.rb: -------------------------------------------------------------------------------- 1 | FactoryBot.define do 2 | factory :metric, class: Facts::Metric do 3 | dimensions_date { Dimensions::Date.find_existing_or_create(date.to_date) } 4 | dimensions_edition { edition } 5 | pviews { 10 } 6 | upviews { 5 } 7 | transient do 8 | date { Time.zone.today } 9 | edition { create :edition } 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /spec/factories/monthly_metrics.rb: -------------------------------------------------------------------------------- 1 | FactoryBot.define do 2 | factory :monthly_metric, class: "Aggregations::MonthlyMetric" do 3 | transient do 4 | month { Dimensions::Month.build(Time.zone.today).id } 5 | edition { create :edition } 6 | end 7 | 8 | dimensions_month do 9 | y, m = *month.split("-").map(&:to_i) 10 | 11 | Dimensions::Month.find_existing_or_create(Date.new(y, m, 1)) 12 | end 13 | 14 | dimensions_edition { edition } 15 | 16 | pviews { 10 } 17 | upviews { 10 } 18 | feedex { 10 } 19 | useful_yes { 10 } 20 | useful_no { 10 } 21 | searches { 10 } 22 | exits { 10 } 23 | entrances { 10 } 24 | bounces { 10 } 25 | page_time { 10 } 26 | satisfaction { 10.0 } 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /spec/factories/publishing_api_events.rb: -------------------------------------------------------------------------------- 1 | FactoryBot.define do 2 | factory :publishing_api_event, class: Events::PublishingApi do 3 | payload { {} } 4 | routing_key { "routing_key" } 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /spec/factories/user.rb: -------------------------------------------------------------------------------- 1 | FactoryBot.define do 2 | factory :user do 3 | transient do 4 | organisation { nil } 5 | end 6 | 7 | sequence(:uid) { |i| "user-#{i}" } 8 | sequence(:name) { |i| "Test User #{i}" } 9 | email { "user@example.com" } 10 | permissions { %w[signin] } 11 | organisation_slug { "government-digital-service" } 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /spec/integration/item/extract_content_all_schemas_spec.rb: -------------------------------------------------------------------------------- 1 | require "support/schemas_iterator" 2 | 3 | RSpec.describe "Process parser", type: :integration do 4 | include SchemasIterator 5 | 6 | context "with a list of content schemas" do 7 | SchemasIterator.each_schema do |schema_name, schema| 8 | it "extracts the content for: #{schema_name}" do 9 | payload = SchemasIterator.payload_for(schema_name, schema) 10 | expect(Etl::Edition::Content::Parser.extract_content(payload)).to be_a(String).or eq(nil) 11 | end 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /spec/models/aggregations/monthly_metric_spec.rb: -------------------------------------------------------------------------------- 1 | require "rails_helper" 2 | 3 | RSpec.describe Aggregations::MonthlyMetric, type: :model do 4 | it { is_expected.to validate_presence_of(:dimensions_month) } 5 | it { is_expected.to validate_presence_of(:dimensions_edition) } 6 | end 7 | -------------------------------------------------------------------------------- /spec/models/document_type_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe DocumentType do 2 | end 3 | -------------------------------------------------------------------------------- /spec/models/organisation_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Organisation do 2 | end 3 | -------------------------------------------------------------------------------- /spec/requests/api/v1/healthcheck_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe "/api/v1/healthcheck/", type: :request do 2 | it "is not cacheable" do 3 | get "/api/v1/healthcheck/" 4 | 5 | expect(response.headers["Cache-Control"]).to eq "max-age=0, private, must-revalidate" 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /spec/requests/api/v1/metrics_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe "/api/v1/metrics/", type: :request do 2 | before { create(:user) } 3 | 4 | include_examples "API response", "/api/v1/metrics" 5 | 6 | describe "metrics index" do 7 | it "describes the available metrics" do 8 | get "/api/v1/metrics" 9 | 10 | json = JSON.parse(response.body) 11 | 12 | expect(json.count).to eq(::Metric.find_all.length) 13 | 14 | expect(json).to include( 15 | "name" => "pviews", 16 | "description" => "Number of pageviews", 17 | "source" => "Google Analytics", 18 | ) 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /spec/requests/healthcheck_metrics_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe "/healthcheck/metrics" do 2 | before do 3 | get "/healthcheck/metrics" 4 | end 5 | 6 | it "returns distinct organisations ordered by title" do 7 | json = JSON.parse(response.body) 8 | 9 | expect(json["checks"]).to include("daily_metrics") 10 | .and(include("etl_metric_values_pviews")) 11 | .and(include("etl_metric_values_searches")) 12 | .and(include("etl_metric_values_upviews")) 13 | .and(include("etl_metric_values_feedex")) 14 | end 15 | 16 | it "is not cacheable" do 17 | expect(response.headers["Cache-Control"]).to eq "max-age=0, private, must-revalidate" 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /spec/requests/healthcheck_search_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe "/healthcheck/search" do 2 | before do 3 | get "/healthcheck/search" 4 | end 5 | 6 | it "returns distinct organisations ordered by title" do 7 | json = JSON.parse(response.body) 8 | 9 | expect(json["checks"]).to include("aggregations") 10 | .and(include("search_last_month")) 11 | .and(include("search_last_six_months")) 12 | .and(include("search_last_thirty_days")) 13 | .and(include("search_last_three_months")) 14 | .and(include("search_last_twelve_months")) 15 | end 16 | 17 | it "is not cacheable" do 18 | expect(response.headers["Cache-Control"]).to eq "max-age=0, private, must-revalidate" 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /spec/routing/documents_routing_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe "documents endpoint routing" do 2 | it "routes /documents/1234/children" do 3 | expect(get: "api/v1/documents/1234/children").to route_to( 4 | controller: "api/documents", 5 | action: "children", 6 | format: :json, 7 | document_id: "1234", 8 | ) 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /spec/support/aggregations_support.rb: -------------------------------------------------------------------------------- 1 | module AggregationsSupport 2 | def recalculate_aggregations! 3 | refresh_last_year_of_monthly_aggregations 4 | refresh_views 5 | end 6 | 7 | private 8 | 9 | def refresh_last_year_of_monthly_aggregations 10 | Etl::Aggregations::Monthly.process(date: Time.zone.today) 11 | 12 | 13.times { |index| Etl::Aggregations::Monthly.process(date: index.month.ago) } 13 | end 14 | 15 | def refresh_views 16 | ::Aggregations::SearchLastThirtyDays.refresh 17 | ::Aggregations::SearchLastMonth.refresh 18 | ::Aggregations::SearchLastThreeMonths.refresh 19 | ::Aggregations::SearchLastSixMonths.refresh 20 | ::Aggregations::SearchLastTwelveMonths.refresh 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /spec/support/authentication.rb: -------------------------------------------------------------------------------- 1 | module AuthenticationControllerHelpers 2 | def login_as(user) 3 | request.env["warden"] = double( 4 | authenticate!: true, 5 | authenticated?: true, 6 | user:, 7 | ) 8 | end 9 | 10 | def stub_user 11 | create(:user) 12 | end 13 | 14 | def login_as_stub_user 15 | login_as stub_user 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /spec/support/google_analytics_requests.rb: -------------------------------------------------------------------------------- 1 | module GoogleAnalyticsRequests 2 | def build_report_data(*report_rows) 3 | Google::Apis::AnalyticsreportingV4::ReportData.new(rows: report_rows) 4 | end 5 | 6 | def build_report_row(dimensions:, metrics:) 7 | Google::Apis::AnalyticsreportingV4::ReportRow.new( 8 | dimensions:, 9 | metrics: metrics.map do |metric| 10 | Google::Apis::AnalyticsreportingV4::DateRangeValues.new( 11 | values: Array(metric), 12 | ) 13 | end, 14 | ) 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /spec/support/govuk_test.rb: -------------------------------------------------------------------------------- 1 | GovukTest.configure 2 | -------------------------------------------------------------------------------- /spec/support/matchers/be_sorted_by_attribute_matcher.rb: -------------------------------------------------------------------------------- 1 | RSpec::Matchers.define :be_sorted_by do 2 | match do |array| 3 | array.each_cons(2).all? do |a, b| 4 | (block_arg.call(a) <=> block_arg.call(b)) <= 0 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /spec/support/schemas_iterator.rb: -------------------------------------------------------------------------------- 1 | module SchemasIterator 2 | def self.each_schema 3 | all_schemas.each do |schema_name, value| 4 | schema_name = schema_name.split("/")[-3] 5 | yield schema_name, value 6 | end 7 | end 8 | 9 | def self.payload_for(schema_name, schema) 10 | all_payloads[schema_name] ||= GovukSchemas::RandomExample.new(schema:).payload 11 | end 12 | 13 | def self.all_payloads 14 | @all_payloads ||= {} 15 | end 16 | 17 | def self.all_schemas 18 | @all_schemas ||= GovukSchemas::Schema.all(schema_type: "notification") 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /spec/support/shared/shared_masterialized_views.rb: -------------------------------------------------------------------------------- 1 | RSpec.shared_examples "a materialized view" do |table_name| 2 | it "refresh the materialized view" do 3 | expect(Scenic.database).to receive(:refresh_materialized_view).with(table_name, concurrently: true, cascade: false) 4 | 5 | subject.refresh 6 | end 7 | 8 | it "prepares the environment to refresh the view" do 9 | allow(Scenic.database).to receive(:refresh_materialized_view) 10 | expect(Aggregations::MaterializedView).to receive(:prepare) 11 | 12 | subject.refresh 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /tmp/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphagov/content-data-api/a0f2ff6f1f4f664344e2ecb5c991ac2cbf0bd5e5/tmp/.keep -------------------------------------------------------------------------------- /vendor/assets/javascripts/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphagov/content-data-api/a0f2ff6f1f4f664344e2ecb5c991ac2cbf0bd5e5/vendor/assets/javascripts/.keep -------------------------------------------------------------------------------- /vendor/assets/stylesheets/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphagov/content-data-api/a0f2ff6f1f4f664344e2ecb5c991ac2cbf0bd5e5/vendor/assets/stylesheets/.keep --------------------------------------------------------------------------------