diff --git a/db/migrate/20200807132726_add_delete_original_index_at_to_reindexing_tasks.rb b/db/migrate/20200807132726_add_delete_original_index_at_to_reindexing_tasks.rb new file mode 100644 index 0000000000000000000000000000000000000000..435b2a45e859a76beb001824960f80f24b5ead80 --- /dev/null +++ b/db/migrate/20200807132726_add_delete_original_index_at_to_reindexing_tasks.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddDeleteOriginalIndexAtToReindexingTasks < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + with_lock_retries do + add_column :elastic_reindexing_tasks, :delete_original_index_at, :datetime_with_timezone + end + end + + def down + with_lock_retries do + remove_column :elastic_reindexing_tasks, :delete_original_index_at + end + end +end diff --git a/db/schema_migrations/20200807132726 b/db/schema_migrations/20200807132726 new file mode 100644 index 0000000000000000000000000000000000000000..a74b1ab23b69c43eea6e162ea1367a48475942e2 --- /dev/null +++ b/db/schema_migrations/20200807132726 @@ -0,0 +1 @@ +867cea94f966c1ad3d9e02f7fa5b641ceac5a71667426330c2c96d6181164f66 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 25af1ab11d6e22dfeed95655a5ed1877eb163d3a..3e479558706f079ec2b67d9bb97a975c4c8a61ad 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -11353,6 +11353,7 @@ CREATE TABLE public.elastic_reindexing_tasks ( elastic_task text, error_message text, documents_count_target integer, + delete_original_index_at timestamp with time zone, CONSTRAINT check_04151aca42 CHECK ((char_length(index_name_from) <= 255)), CONSTRAINT check_7f64acda8e CHECK ((char_length(error_message) <= 255)), CONSTRAINT check_85ebff7124 CHECK ((char_length(index_name_to) <= 255)), diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index 8d4c8b3cf97cc4a8b1ed4c5e2bf8c7cc05b6398f..f971d2db0669ddc33d6beec592119e76eb327d3e 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -548,13 +548,17 @@ To trigger the re-index from `primary` index: ### Trigger the reindex via the Elasticsearch administration -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34069) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.2. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34069) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.2. +> - A scheduled index deletion and the ability to cancel it was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38914) in GitLab Starter 13.3. Under **Admin Area > Integration > Elasticsearch zero-downtime reindexing**, click on **Trigger cluster reindexing**. NOTE: **Note:** Reindexing can be a lengthy process depending on the size of your Elasticsearch cluster. +CAUTION: **Caution:** +After the reindexing is completed, the original index will be scheduled to be deleted after 14 days. You can cancel this action by pressing the cancel button. + While the reindexing is running, you will be able to follow its progress under that same section. ## GitLab Elasticsearch Rake tasks diff --git a/ee/app/controllers/admin/elasticsearch_controller.rb b/ee/app/controllers/admin/elasticsearch_controller.rb index 3d6d0a291e266347206fd07f1b31d095f9fe32f3..fe96b56110cacb90aa916b2b8b373555e9fe8cf1 100644 --- a/ee/app/controllers/admin/elasticsearch_controller.rb +++ b/ee/app/controllers/admin/elasticsearch_controller.rb @@ -30,6 +30,17 @@ def trigger_reindexing redirect_to redirect_path end + # POST + # Cancel index deletion after a successful reindexing operation + def cancel_index_deletion + task = Elastic::ReindexingTask.find(params[:task_id]) + task.update!(delete_original_index_at: nil) + + flash[:notice] = _('Index deletion is canceled') + + redirect_to redirect_path + end + private def redirect_path diff --git a/ee/app/controllers/ee/admin/application_settings_controller.rb b/ee/app/controllers/ee/admin/application_settings_controller.rb index c9a9864eb25cd830c5fecb8542b5f213a5cbc7fc..30836f3f0dd5ca5aff24b3b4dfbb70919a799f46 100644 --- a/ee/app/controllers/ee/admin/application_settings_controller.rb +++ b/ee/app/controllers/ee/admin/application_settings_controller.rb @@ -9,7 +9,7 @@ module ApplicationSettingsController include ::Admin::MergeRequestApprovalSettingsHelper prepended do - before_action :elasticsearch_reindexing_task, only: [:integrations] + before_action :elasticsearch_reindexing_task, only: [:general] def elasticsearch_reindexing_task @elasticsearch_reindexing_task = Elastic::ReindexingTask.last diff --git a/ee/app/models/elastic/reindexing_task.rb b/ee/app/models/elastic/reindexing_task.rb index f87a8565b44ac1c4a09ceee27de5c47845528182..3fc05ee5011ef3a05cbbffdd4af4de51cda439d7 100644 --- a/ee/app/models/elastic/reindexing_task.rb +++ b/ee/app/models/elastic/reindexing_task.rb @@ -4,13 +4,17 @@ class Elastic::ReindexingTask < ApplicationRecord self.table_name = 'elastic_reindexing_tasks' enum state: { - initial: 0, - indexing_paused: 1, - reindexing: 2, - success: 10, # states less than 10 are considered in_progress - failure: 11 + initial: 0, + indexing_paused: 1, + reindexing: 2, + success: 10, # states less than 10 are considered in_progress + failure: 11, + original_index_deleted: 12 } + scope :old_indices_scheduled_for_deletion, -> { where(state: :success).where('delete_original_index_at IS NOT NULL') } + scope :old_indices_to_be_deleted, -> { old_indices_scheduled_for_deletion.where('delete_original_index_at < NOW()') } + before_save :set_in_progress_flag def self.current @@ -21,6 +25,14 @@ def self.running? current.present? end + def self.drop_old_indices! + old_indices_to_be_deleted.find_each do |task| + next unless Gitlab::Elastic::Helper.default.delete_index(index_name: task.index_name_from) + + task.update!(state: :original_index_deleted) + end + end + private def set_in_progress_flag diff --git a/ee/app/services/elastic/cluster_reindexing_service.rb b/ee/app/services/elastic/cluster_reindexing_service.rb index bb0d55e5f0aae7157e87641155a2561060a4a971..fa07f26f743e260364f8dfebc13e291417dcbcb9 100644 --- a/ee/app/services/elastic/cluster_reindexing_service.rb +++ b/ee/app/services/elastic/cluster_reindexing_service.rb @@ -10,6 +10,8 @@ class ClusterReindexingService translog: { durability: 'async' } }.freeze + DELETE_ORIGINAL_INDEX_AFTER = 14.days + def execute case current_task.state.to_sym when :initial @@ -124,7 +126,7 @@ def switch_alias_to_new_index def finalize_reindexing Gitlab::CurrentSettings.update!(elasticsearch_pause_indexing: false) - current_task.update!(state: :success) + current_task.update!(state: :success, delete_original_index_at: DELETE_ORIGINAL_INDEX_AFTER.from_now) end def reindexing! diff --git a/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml b/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml index c7094646243a3a5a0073ef4c7151b405595f8f85..b3b092778490a0a5baa4c25883029c67cd8bea1b 100644 --- a/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml +++ b/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml @@ -98,6 +98,10 @@ = link_to _('Trigger cluster reindexing'), admin_elasticsearch_trigger_reindexing_path, class: 'btn btn-success', method: :post, disabled: @elasticsearch_reindexing_task&.in_progress? .form-text.text-muted = _('This feature should be used with an index that was created after 13.0') + - Elastic::ReindexingTask.old_indices_scheduled_for_deletion.each do |task| + .form-text.text-danger + = _("Unused, previous index '%{index_name}' will be deleted after %{time} automatically.") % { index_name: task.index_name_from, time: task.delete_original_index_at } + = link_to _('Cancel index deletion'), admin_elasticsearch_cancel_index_deletion_path(task_id: task.id), method: :post - if @elasticsearch_reindexing_task - expected_documents = @elasticsearch_reindexing_task.documents_count - processed_documents = @elasticsearch_reindexing_task.documents_count_target diff --git a/ee/app/workers/elastic_cluster_reindexing_cron_worker.rb b/ee/app/workers/elastic_cluster_reindexing_cron_worker.rb index 527d7bd15023d67b95f7a762b4f4d2e023231ef7..6dd3719c763bec3822a0a8896febcaac242bab86 100644 --- a/ee/app/workers/elastic_cluster_reindexing_cron_worker.rb +++ b/ee/app/workers/elastic_cluster_reindexing_cron_worker.rb @@ -12,10 +12,12 @@ class ElasticClusterReindexingCronWorker idempotent! def perform - task = Elastic::ReindexingTask.current - return false unless task - in_lock(self.class.name.underscore, ttl: 1.hour, retries: 10, sleep_sec: 1) do + Elastic::ReindexingTask.drop_old_indices! + + task = Elastic::ReindexingTask.current + break false unless task + service.execute end end diff --git a/ee/changelogs/unreleased/225436-reindex-delete-original-index.yml b/ee/changelogs/unreleased/225436-reindex-delete-original-index.yml new file mode 100644 index 0000000000000000000000000000000000000000..22a783887b6bb8db2f47ab1df25cecf77cdcfdd6 --- /dev/null +++ b/ee/changelogs/unreleased/225436-reindex-delete-original-index.yml @@ -0,0 +1,5 @@ +--- +title: Automate the deletion of the old Index after a reindex +merge_request: 38914 +author: +type: changed diff --git a/ee/config/routes/admin.rb b/ee/config/routes/admin.rb index cca433a44e5973fd79f6d5a3082d56ea2aed6211..76ee096c04a8c58db3a38ae0bb3a6fb6485f7e31 100644 --- a/ee/config/routes/admin.rb +++ b/ee/config/routes/admin.rb @@ -72,5 +72,6 @@ namespace :elasticsearch do post :enqueue_index post :trigger_reindexing + post :cancel_index_deletion end end diff --git a/ee/spec/controllers/admin/elasticsearch_controller_spec.rb b/ee/spec/controllers/admin/elasticsearch_controller_spec.rb index 4a27d7a50fdb5abf31e96cbe692be560a479ea35..647eeaae4674c548f8d101859f4d97bc57b49b3e 100644 --- a/ee/spec/controllers/admin/elasticsearch_controller_spec.rb +++ b/ee/spec/controllers/admin/elasticsearch_controller_spec.rb @@ -62,4 +62,20 @@ expect(response).to redirect_to general_admin_application_settings_path(anchor: 'js-elasticsearch-settings') end end + + describe 'POST #cancel_index_deletion' do + before do + sign_in(admin) + end + + let(:task) { create(:elastic_reindexing_task, state: :success, delete_original_index_at: Time.current) } + + it 'sets delete_original_index_at to nil' do + post :cancel_index_deletion, params: { task_id: task.id } + + expect(task.reload.delete_original_index_at).to be_nil + expect(controller).to set_flash[:notice].to include('deletion is canceled') + expect(response).to redirect_to general_admin_application_settings_path(anchor: 'js-elasticsearch-settings') + end + end end diff --git a/ee/spec/models/elastic/reindexing_task_spec.rb b/ee/spec/models/elastic/reindexing_task_spec.rb index cc4109c33b3746729d3b0092f7eabef8897e2def..b096cebf011d352212a1401f97155ce6391fba36 100644 --- a/ee/spec/models/elastic/reindexing_task_spec.rb +++ b/ee/spec/models/elastic/reindexing_task_spec.rb @@ -16,4 +16,30 @@ task.update!(state: :reindexing) expect(task.in_progress).to eq(true) end + + describe '.drop_old_indices!' do + let(:task_1) { create(:elastic_reindexing_task, index_name_from: 'original_index_1', state: :reindexing, delete_original_index_at: 1.day.ago) } + let(:task_2) { create(:elastic_reindexing_task, index_name_from: 'original_index_2', state: :success, delete_original_index_at: nil) } + let(:task_3) { create(:elastic_reindexing_task, index_name_from: 'original_index_3', state: :success, delete_original_index_at: 1.day.ago) } + let(:task_4) { create(:elastic_reindexing_task, index_name_from: 'original_index_4', state: :success, delete_original_index_at: 5.days.ago) } + let(:task_5) { create(:elastic_reindexing_task, index_name_from: 'original_index_5', state: :success, delete_original_index_at: 14.days.from_now) } + let(:tasks_for_deletion) { [task_3, task_4] } + let(:other_tasks) { [task_1, task_2, task_5] } + + it 'deletes the correct indices' do + other_tasks.each do |task| + expect(Gitlab::Elastic::Helper.default).not_to receive(:delete_index).with(index_name: task.index_name_from) + end + + tasks_for_deletion.each do |task| + expect(Gitlab::Elastic::Helper.default).to receive(:delete_index).with(index_name: task.index_name_from).and_return(true) + end + + described_class.drop_old_indices! + + tasks_for_deletion.each do |task| + expect(task.reload.state).to eq('original_index_deleted') + end + end + end end diff --git a/ee/spec/services/elastic/cluster_reindexing_service_spec.rb b/ee/spec/services/elastic/cluster_reindexing_service_spec.rb index 8f23cb3c1079fb0244b6daa48cd0020818d2b087..fad195f7682c5871fa0c9cf64cd00eebf92cdf9f 100644 --- a/ee/spec/services/elastic/cluster_reindexing_service_spec.rb +++ b/ee/spec/services/elastic/cluster_reindexing_service_spec.rb @@ -84,6 +84,7 @@ expect(Gitlab::CurrentSettings).to receive(:update!).with(elasticsearch_pause_indexing: false) expect { subject.execute }.to change { task.reload.state }.from('reindexing').to('success') + expect(task.reload.delete_original_index_at).to be_within(1.minute).of(described_class::DELETE_ORIGINAL_INDEX_AFTER.from_now) end end end diff --git a/ee/spec/workers/elastic_cluster_reindexing_cron_worker_spec.rb b/ee/spec/workers/elastic_cluster_reindexing_cron_worker_spec.rb index eb22a00ceaa01b2ae9ef56366e8f23036f562aa2..077a4410b0d4bd21bd9ef6059cbecda3bf2fcedc 100644 --- a/ee/spec/workers/elastic_cluster_reindexing_cron_worker_spec.rb +++ b/ee/spec/workers/elastic_cluster_reindexing_cron_worker_spec.rb @@ -16,8 +16,9 @@ subject.perform end - it 'does nothing if no task is found' do + it 'removes old indices if no task is found' do expect(Elastic::ReindexingTask).to receive(:current).and_return(nil) + expect(Elastic::ReindexingTask).to receive(:drop_old_indices!) expect(subject.perform).to eq(false) end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 23ae5515f1197ad4312237aa9a181f7babbc40f8..e40c4aaa23ff81045013bb38e30d3e662867d7e8 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -4319,6 +4319,9 @@ msgstr "" msgid "Cancel" msgstr "" +msgid "Cancel index deletion" +msgstr "" + msgid "Cancel running" msgstr "" @@ -13010,6 +13013,9 @@ msgstr "" msgid "Index all projects" msgstr "" +msgid "Index deletion is canceled" +msgstr "" + msgid "Indicates whether this runner can pick jobs without tags" msgstr "" @@ -26253,6 +26259,9 @@ msgstr "" msgid "Until" msgstr "" +msgid "Unused, previous index '%{index_name}' will be deleted after %{time} automatically." +msgstr "" + msgid "Unverified" msgstr ""