From 264297be13f0ea3f1776adee82f0c94110da8cb9 Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Mon, 4 Aug 2025 16:43:34 -0400 Subject: [PATCH 01/13] Incremental updates the code embedding index --- ...context_code_incremental_index_project.yml | 10 +++ .../code/incremental_indexing_service.rb | 26 +++++++ .../code/indexing_service_base.rb | 76 +++++++++++++++++++ .../code/initial_indexing_service.rb | 67 +--------------- ee/app/services/ee/git/branch_push_service.rb | 13 +++- .../code/repository_index_worker.rb | 16 +++- .../code/incremental_indexing_service_spec.rb | 72 ++++++++++++++++++ .../ee/git/branch_push_service_spec.rb | 45 +++++++++++ .../code/repository_index_worker_spec.rb | 7 +- 9 files changed, 264 insertions(+), 68 deletions(-) create mode 100644 config/feature_flags/gitlab_com_derisk/active_context_code_incremental_index_project.yml create mode 100644 ee/app/services/ai/active_context/code/incremental_indexing_service.rb create mode 100644 ee/app/services/ai/active_context/code/indexing_service_base.rb create mode 100644 ee/spec/services/ai/active_context/code/incremental_indexing_service_spec.rb diff --git a/config/feature_flags/gitlab_com_derisk/active_context_code_incremental_index_project.yml b/config/feature_flags/gitlab_com_derisk/active_context_code_incremental_index_project.yml new file mode 100644 index 00000000000000..036ef22d4fa736 --- /dev/null +++ b/config/feature_flags/gitlab_com_derisk/active_context_code_incremental_index_project.yml @@ -0,0 +1,10 @@ +--- +name: active_context_code_incremental_index_project +description: +feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/554445 +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/201128 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/561020 +milestone: '18.3' +group: group::code creation +type: gitlab_com_derisk +default_enabled: false diff --git a/ee/app/services/ai/active_context/code/incremental_indexing_service.rb b/ee/app/services/ai/active_context/code/incremental_indexing_service.rb new file mode 100644 index 00000000000000..e4a7367726018f --- /dev/null +++ b/ee/app/services/ai/active_context/code/incremental_indexing_service.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Ai + module ActiveContext + module Code + class IncrementalIndexingService < IndexingServiceBase + def execute + run_indexer_and_enqueue_ref + rescue StandardError => e + # only record the error message, as the state should always be ready after initial indexing + update_repository_last_error(e.message) + + raise e + end + + private + + def update_repository_last_error(message) + repository.update!(last_error: message) + + log_error("incremental indexing failed", last_error: message) + end + end + end + end +end diff --git a/ee/app/services/ai/active_context/code/indexing_service_base.rb b/ee/app/services/ai/active_context/code/indexing_service_base.rb new file mode 100644 index 00000000000000..03cb1312edc860 --- /dev/null +++ b/ee/app/services/ai/active_context/code/indexing_service_base.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Ai + module ActiveContext + module Code + class IndexingServiceBase + include Gitlab::Loggable + + def self.execute(repository) + new(repository).execute + end + + def initialize(repository) + @repository = repository + end + + private + + attr_reader :repository + + def run_indexer_and_enqueue_ref + last_indexed_id = nil + + run_indexer! do |indexed_id| + enqueue_refs!([indexed_id]) + last_indexed_id = indexed_id + end + + set_highest_enqueued_item!(last_indexed_id) if last_indexed_id + end + + def run_indexer!(&block) + Indexer.run!(repository, &block) + end + + def enqueue_refs!(ids) + ::Ai::ActiveContext::Collections::Code.track_refs!(hashes: ids, routing: repository.project_id) + end + + def set_highest_enqueued_item!(item_id) + repository.update!( + initial_indexing_last_queued_item: item_id, + indexed_at: Time.current + ) + + log_info( + 'set_highest_enqueued_item', + initial_indexing_last_queued_item: item_id + ) + end + + def log_info(message, extra_params = {}) + logger.info(build_log_payload(message, extra_params)) + end + + def log_error(message, extra_params = {}) + logger.error(build_log_payload(message, extra_params)) + end + + def build_log_payload(message, extra_params = {}) + params = { + message: message, + ai_active_context_code_repository_id: repository.id, + project_id: repository.project_id + }.merge(extra_params) + + build_structured_payload(**params) + end + + def logger + @logger ||= ::ActiveContext::Config.logger + end + end + end + end +end diff --git a/ee/app/services/ai/active_context/code/initial_indexing_service.rb b/ee/app/services/ai/active_context/code/initial_indexing_service.rb index 679377da301558..5a46e003bac832 100644 --- a/ee/app/services/ai/active_context/code/initial_indexing_service.rb +++ b/ee/app/services/ai/active_context/code/initial_indexing_service.rb @@ -3,30 +3,13 @@ module Ai module ActiveContext module Code - class InitialIndexingService - include Gitlab::Loggable - - def self.execute(repository) - new(repository).execute - end - - def initialize(repository) - @repository = repository - end - + class InitialIndexingService < IndexingServiceBase def execute update_repository_state!(:code_indexing_in_progress) - last_indexed_id = nil - - run_indexer! do |indexed_id| - enqueue_refs!([indexed_id]) - last_indexed_id = indexed_id - end + run_indexer_and_enqueue_ref update_repository_state!(:embedding_indexing_in_progress) - - set_highest_enqueued_item!(last_indexed_id) if last_indexed_id rescue StandardError => e update_repository_state!(:failed, last_error: e.message) @@ -35,13 +18,9 @@ def execute private - attr_reader :repository - - def run_indexer!(&block) - Indexer.run!(repository, &block) - end - def update_repository_state!(state, extra_params = {}) + # only InitialIndexingService can update the repository state + # IncrementalIndexingService can't update the repository state repository.update!(state: state, **extra_params) if state == :failed @@ -50,44 +29,6 @@ def update_repository_state!(state, extra_params = {}) log_info(state.to_s) end end - - def enqueue_refs!(ids) - ::Ai::ActiveContext::Collections::Code.track_refs!(hashes: ids, routing: repository.project_id) - end - - def set_highest_enqueued_item!(item_id) - repository.update!( - initial_indexing_last_queued_item: item_id, - indexed_at: Time.current - ) - - log_info( - 'set_highest_enqueued_item', - initial_indexing_last_queued_item: item_id - ) - end - - def log_info(message, extra_params = {}) - logger.info(build_log_payload(message, extra_params)) - end - - def log_error(message, extra_params = {}) - logger.error(build_log_payload(message, extra_params)) - end - - def build_log_payload(message, extra_params = {}) - params = { - message: message, - ai_active_context_code_repository_id: repository.id, - project_id: repository.project_id - }.merge(extra_params) - - build_structured_payload(**params) - end - - def logger - @logger ||= ::ActiveContext::Config.logger - end end end end diff --git a/ee/app/services/ee/git/branch_push_service.rb b/ee/app/services/ee/git/branch_push_service.rb index b2ac36650ee3cf..a5b116511797f5 100644 --- a/ee/app/services/ee/git/branch_push_service.rb +++ b/ee/app/services/ee/git/branch_push_service.rb @@ -10,6 +10,7 @@ def execute enqueue_elasticsearch_indexing enqueue_zoekt_indexing enqueue_knowledge_graph_indexing + enqueue_code_embedding_indexing enqueue_update_external_pull_requests enqueue_product_analytics_event_metrics enqueue_repository_xray @@ -49,8 +50,18 @@ def enqueue_zoekt_indexing project.repository.async_update_zoekt_index end + def enqueue_code_embedding_indexing + return false unless ::Feature.enabled?(:active_context_code_incremental_index_project, project) + return false unless default_branch? + + repository = Ai::ActiveContext::Code::Repository.find_by_project_id(project.id) + return false unless repository&.ready? + + ::Ai::ActiveContext::Code::RepositoryIndexWorker.perform_async(repository.id) + end + def enqueue_knowledge_graph_indexing - return false unless ::Feature.enabled?(:knowledge_graph_indexing, project.project_namespace) + return false unless ::Feature.enabled?(:knowledge_graph_indexing, project) return false unless default_branch? return false unless ::GitlabSubscriptions::AddOnPurchase diff --git a/ee/app/workers/ai/active_context/code/repository_index_worker.rb b/ee/app/workers/ai/active_context/code/repository_index_worker.rb index 5476223ee77486..04160469a7515a 100644 --- a/ee/app/workers/ai/active_context/code/repository_index_worker.rb +++ b/ee/app/workers/ai/active_context/code/repository_index_worker.rb @@ -28,10 +28,18 @@ def perform(id) repository = Ai::ActiveContext::Code::Repository.find_by_id(id) - return false unless repository&.pending? + return false unless repository&.pending? || repository&.ready? in_lock(lease_key(id), ttl: LEASE_TTL, sleep_sec: LEASE_TRY_AFTER, retries: LEASE_RETRIES) do - InitialIndexingService.execute(repository) + if repository.pending? + logger.info("Initial code indexing started...") + + InitialIndexingService.execute(repository) + elsif repository.ready? + logger.info("Incremental code indexing started...") + + IncrementalIndexingService.execute(repository) + end end rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError self.class.perform_in(RETRY_IN_IF_LOCKED, id) @@ -40,6 +48,10 @@ def perform(id) def lease_key(id) "#{self.class.name}/#{id}" end + + def logger + @logger ||= ::ActiveContext::Config.logger + end end end end diff --git a/ee/spec/services/ai/active_context/code/incremental_indexing_service_spec.rb b/ee/spec/services/ai/active_context/code/incremental_indexing_service_spec.rb new file mode 100644 index 00000000000000..0485684e891b9e --- /dev/null +++ b/ee/spec/services/ai/active_context/code/incremental_indexing_service_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ai::ActiveContext::Code::IncrementalIndexingService, feature_category: :global_search do + let_it_be(:collection) { create(:ai_active_context_collection) } + let_it_be(:project) { create(:project) } + let_it_be_with_reload(:repository) { create(:ai_active_context_code_repository, state: :ready, project: project) } + + let(:logger) { instance_double(::Gitlab::ActiveContext::Logger, info: nil, error: nil) } + + before do + allow(ActiveContext::CollectionCache).to receive(:fetch).and_return(collection) + allow(::ActiveContext::Config).to receive(:logger).and_return(logger) + end + + def build_log_payload(message, extra_params = {}) + { + class: described_class.to_s, + message: message, + ai_active_context_code_repository_id: repository.id, + project_id: project.id + }.merge(extra_params).stringify_keys + end + + describe '.execute' do + subject(:execute) { described_class.execute(repository) } + + it 'calls the indexer with a block and tracks refs for each id as it processes' do + expect(repository.state).to eq('ready') + + expect(Ai::ActiveContext::Code::Indexer).to receive(:run!) do |repo, &block| + expect(repo).to eq(repository) + block.call('hash1') + block.call('hash2') + end + + expect(::Ai::ActiveContext::Collections::Code).to receive(:track_refs!) + .with(hashes: ['hash1'], routing: repository.project_id) + expect(::Ai::ActiveContext::Collections::Code).to receive(:track_refs!) + .with(hashes: ['hash2'], routing: repository.project_id) + + expect(logger).to receive(:info).with( + build_log_payload('set_highest_enqueued_item', initial_indexing_last_queued_item: 'hash2') + ).ordered + + execute + + expect(repository.state).to eq('ready') + expect(repository.initial_indexing_last_queued_item).to eq('hash2') + end + + context 'when indexing fails' do + let(:error) { StandardError.new('Indexing failed') } + + before do + allow(Ai::ActiveContext::Code::Indexer).to receive(:run!).and_raise(error) + end + + it 'sets the repository to failed and passes the error on' do + expect(logger).to receive(:error).with( + build_log_payload('ready', last_error: error.message) + ) + + expect { execute }.to raise_error(error) + + expect(repository.state).to eq('ready') + expect(repository.last_error).to eq(error.message) + end + end + end +end diff --git a/ee/spec/services/ee/git/branch_push_service_spec.rb b/ee/spec/services/ee/git/branch_push_service_spec.rb index 42715c1105d2cc..64da1946d68c94 100644 --- a/ee/spec/services/ee/git/branch_push_service_spec.rb +++ b/ee/spec/services/ee/git/branch_push_service_spec.rb @@ -309,6 +309,51 @@ end end + describe 'trigger enqueue_code_embedding_indexing', feature_category: :code_suggestions do + shared_examples 'does not trigger enqueue_code_embedding_indexing' do + it 'does not schedule a IncrementalRepositoryIndexWorker' do + expect(::Ai::ActiveContext::Code::RepositoryIndexWorker).not_to receive(:perform_async) + + branch_push_service.execute + end + end + + context 'when ff active_context_code_incremental_index_project is disabled' do + before do + stub_feature_flags(run_code_embedding_indexing: false) + end + + it_behaves_like 'does not trigger enqueue_code_embedding_indexing' + end + + context 'when pushing to a non-default branch' do + let(:ref) { 'refs/heads/other' } + + it_behaves_like 'does not trigger enqueue_code_embedding_indexing' + end + + context 'when there is no Ai::ActiveContext::Code::Repository' do + it_behaves_like 'does not trigger enqueue_code_embedding_indexing' + end + + context 'when the Ai::ActiveContext::Code::Repository is not ready' do + let_it_be(:repository) { create(:ai_active_context_code_repository, state: :pending, project: project) } + + it_behaves_like 'does not trigger enqueue_code_embedding_indexing' + + context 'when the Ai::ActiveContext::Code::Repository is ready' do + before do + repository.update!(state: :ready) + end + + it 'triggers perform_async on RepositoryIndexWorker' do + expect(::Ai::ActiveContext::Code::RepositoryIndexWorker).to receive(:perform_async).with(project.id) + branch_push_service.execute + end + end + end + end + describe 'Pipeline execution policy metadata sync' do let(:feature_licensed) { true } diff --git a/ee/spec/workers/ai/active_context/code/repository_index_worker_spec.rb b/ee/spec/workers/ai/active_context/code/repository_index_worker_spec.rb index 334333db6d8f76..5eb37f80fc830c 100644 --- a/ee/spec/workers/ai/active_context/code/repository_index_worker_spec.rb +++ b/ee/spec/workers/ai/active_context/code/repository_index_worker_spec.rb @@ -15,6 +15,7 @@ before do allow(::Ai::ActiveContext::Collections::Code).to receive(:indexing?).and_return(true) allow(Ai::ActiveContext::Code::InitialIndexingService).to receive(:execute) + allow(Ai::ActiveContext::Code::IncrementalIndexingService).to receive(:execute) end context 'when ActiveContext indexing is enabled' do @@ -23,16 +24,18 @@ worker.perform(repository.id) expect(Ai::ActiveContext::Code::InitialIndexingService).to have_received(:execute).with(repository) + expect(Ai::ActiveContext::Code::IncrementalIndexingService).not_to have_received(:execute).with(repository) end end - context 'with a repository that is not pending' do + context 'with a valid ready repository' do let_it_be(:ready_repository) { create(:ai_active_context_code_repository, state: :ready) } - it 'does not call InitialIndexingService.execute' do + it 'calls IncrementalIndexingService.execute' do worker.perform(ready_repository.id) expect(Ai::ActiveContext::Code::InitialIndexingService).not_to have_received(:execute) + expect(Ai::ActiveContext::Code::IncrementalIndexingService).to have_received(:execute).with(ready_repository) end end -- GitLab From 099ccf589034e193d41b9f2fef4face0aa7828f9 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 14 Aug 2025 12:44:38 -0400 Subject: [PATCH 02/13] Edit active_context_code_incremental_index_project.yml --- .../active_context_code_incremental_index_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/feature_flags/gitlab_com_derisk/active_context_code_incremental_index_project.yml b/config/feature_flags/gitlab_com_derisk/active_context_code_incremental_index_project.yml index 036ef22d4fa736..3c01cd7c6965c1 100644 --- a/config/feature_flags/gitlab_com_derisk/active_context_code_incremental_index_project.yml +++ b/config/feature_flags/gitlab_com_derisk/active_context_code_incremental_index_project.yml @@ -5,6 +5,6 @@ feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/554445 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/201128 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/561020 milestone: '18.3' -group: group::code creation +group: group::global search type: gitlab_com_derisk default_enabled: false -- GitLab From bc921ec73b08042ec7f54042729366d21746a6c0 Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Thu, 14 Aug 2025 14:45:56 -0400 Subject: [PATCH 03/13] update unit tests --- .../ai/active_context/code/incremental_indexing_service_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/spec/services/ai/active_context/code/incremental_indexing_service_spec.rb b/ee/spec/services/ai/active_context/code/incremental_indexing_service_spec.rb index 0485684e891b9e..f6d5ad46e80e8b 100644 --- a/ee/spec/services/ai/active_context/code/incremental_indexing_service_spec.rb +++ b/ee/spec/services/ai/active_context/code/incremental_indexing_service_spec.rb @@ -59,7 +59,7 @@ def build_log_payload(message, extra_params = {}) it 'sets the repository to failed and passes the error on' do expect(logger).to receive(:error).with( - build_log_payload('ready', last_error: error.message) + build_log_payload('incremental indexing failed', last_error: error.message) ) expect { execute }.to raise_error(error) -- GitLab From a87e631306be5133a9d26c8667e6468613ae02a9 Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Thu, 14 Aug 2025 17:01:22 -0400 Subject: [PATCH 04/13] update unit tests --- .../ai/active_context/code/initial_indexing_service_spec.rb | 2 +- ee/spec/services/ee/git/branch_push_service_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ee/spec/services/ai/active_context/code/initial_indexing_service_spec.rb b/ee/spec/services/ai/active_context/code/initial_indexing_service_spec.rb index 29e775a54b0bb2..25d65583bede73 100644 --- a/ee/spec/services/ai/active_context/code/initial_indexing_service_spec.rb +++ b/ee/spec/services/ai/active_context/code/initial_indexing_service_spec.rb @@ -41,10 +41,10 @@ def build_log_payload(message, extra_params = {}) .with(hashes: ['hash2'], routing: repository.project_id) expect(logger).to receive(:info).with(build_log_payload('code_indexing_in_progress')).ordered - expect(logger).to receive(:info).with(build_log_payload('embedding_indexing_in_progress')).ordered expect(logger).to receive(:info).with( build_log_payload('set_highest_enqueued_item', initial_indexing_last_queued_item: 'hash2') ).ordered + expect(logger).to receive(:info).with(build_log_payload('embedding_indexing_in_progress')).ordered execute diff --git a/ee/spec/services/ee/git/branch_push_service_spec.rb b/ee/spec/services/ee/git/branch_push_service_spec.rb index 64da1946d68c94..a6dc22c05562a1 100644 --- a/ee/spec/services/ee/git/branch_push_service_spec.rb +++ b/ee/spec/services/ee/git/branch_push_service_spec.rb @@ -347,7 +347,7 @@ end it 'triggers perform_async on RepositoryIndexWorker' do - expect(::Ai::ActiveContext::Code::RepositoryIndexWorker).to receive(:perform_async).with(project.id) + expect(::Ai::ActiveContext::Code::RepositoryIndexWorker).to receive(:perform_async).with(repository.id) branch_push_service.execute end end -- GitLab From 89b93de5333ee4f285c3199ee58e5de81906cfa7 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 14 Aug 2025 20:46:15 -0400 Subject: [PATCH 05/13] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: Pam Artiaga --- .../services/ai/active_context/code/initial_indexing_service.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/ee/app/services/ai/active_context/code/initial_indexing_service.rb b/ee/app/services/ai/active_context/code/initial_indexing_service.rb index 5a46e003bac832..786d97dc1d7171 100644 --- a/ee/app/services/ai/active_context/code/initial_indexing_service.rb +++ b/ee/app/services/ai/active_context/code/initial_indexing_service.rb @@ -19,8 +19,6 @@ def execute private def update_repository_state!(state, extra_params = {}) - # only InitialIndexingService can update the repository state - # IncrementalIndexingService can't update the repository state repository.update!(state: state, **extra_params) if state == :failed -- GitLab From e3212c9b88abc1e701294dc3150feaf1c3ae2b53 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 14 Aug 2025 20:47:52 -0400 Subject: [PATCH 06/13] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: Pam Artiaga --- .../workers/ai/active_context/code/repository_index_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/app/workers/ai/active_context/code/repository_index_worker.rb b/ee/app/workers/ai/active_context/code/repository_index_worker.rb index 04160469a7515a..362cb1ec404c42 100644 --- a/ee/app/workers/ai/active_context/code/repository_index_worker.rb +++ b/ee/app/workers/ai/active_context/code/repository_index_worker.rb @@ -32,7 +32,7 @@ def perform(id) in_lock(lease_key(id), ttl: LEASE_TTL, sleep_sec: LEASE_TRY_AFTER, retries: LEASE_RETRIES) do if repository.pending? - logger.info("Initial code indexing started...") + logger.info("Indexing started", project_id: id, indexing_mode: 'initial') InitialIndexingService.execute(repository) elsif repository.ready? -- GitLab From 72b3b29b06edbd03451dc3b1f436c9bf0743fad1 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 14 Aug 2025 20:48:05 -0400 Subject: [PATCH 07/13] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: Pam Artiaga --- .../ai/active_context/code/incremental_indexing_service.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/ee/app/services/ai/active_context/code/incremental_indexing_service.rb b/ee/app/services/ai/active_context/code/incremental_indexing_service.rb index e4a7367726018f..32b75b25e58420 100644 --- a/ee/app/services/ai/active_context/code/incremental_indexing_service.rb +++ b/ee/app/services/ai/active_context/code/incremental_indexing_service.rb @@ -7,7 +7,6 @@ class IncrementalIndexingService < IndexingServiceBase def execute run_indexer_and_enqueue_ref rescue StandardError => e - # only record the error message, as the state should always be ready after initial indexing update_repository_last_error(e.message) raise e -- GitLab From 30ae1ee5b7a8bd7fcf99f7610898d92af057af52 Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Thu, 14 Aug 2025 22:12:50 -0400 Subject: [PATCH 08/13] update unit tests --- .../code/repository_index_worker.rb | 11 +++++-- .../code/repository_index_worker_spec.rb | 32 +++++++++++++++++-- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/ee/app/workers/ai/active_context/code/repository_index_worker.rb b/ee/app/workers/ai/active_context/code/repository_index_worker.rb index 362cb1ec404c42..28d50eeb5c59de 100644 --- a/ee/app/workers/ai/active_context/code/repository_index_worker.rb +++ b/ee/app/workers/ai/active_context/code/repository_index_worker.rb @@ -5,6 +5,7 @@ module ActiveContext module Code class RepositoryIndexWorker include ApplicationWorker + include Gitlab::Loggable include Gitlab::Utils::StrongMemoize include Gitlab::ExclusiveLeaseHelpers prepend ::Geo::SkipSecondary @@ -32,11 +33,17 @@ def perform(id) in_lock(lease_key(id), ttl: LEASE_TTL, sleep_sec: LEASE_TRY_AFTER, retries: LEASE_RETRIES) do if repository.pending? - logger.info("Indexing started", project_id: id, indexing_mode: 'initial') + logger.info(build_structured_payload( + message: "Indexing started", + project_id: id, + indexing_mode: 'initial')) InitialIndexingService.execute(repository) elsif repository.ready? - logger.info("Incremental code indexing started...") + logger.info(build_structured_payload( + message: "Indexing started", + project_id: id, + indexing_mode: 'incremental')) IncrementalIndexingService.execute(repository) end diff --git a/ee/spec/workers/ai/active_context/code/repository_index_worker_spec.rb b/ee/spec/workers/ai/active_context/code/repository_index_worker_spec.rb index 5eb37f80fc830c..85f77b16bd3488 100644 --- a/ee/spec/workers/ai/active_context/code/repository_index_worker_spec.rb +++ b/ee/spec/workers/ai/active_context/code/repository_index_worker_spec.rb @@ -20,6 +20,10 @@ context 'when ActiveContext indexing is enabled' do context 'with a valid pending repository' do + before do + repository.pending! + end + it 'calls InitialIndexingService.execute with the repository' do worker.perform(repository.id) @@ -29,13 +33,32 @@ end context 'with a valid ready repository' do - let_it_be(:ready_repository) { create(:ai_active_context_code_repository, state: :ready) } + before do + repository.ready! + end it 'calls IncrementalIndexingService.execute' do - worker.perform(ready_repository.id) + worker.perform(repository.id) expect(Ai::ActiveContext::Code::InitialIndexingService).not_to have_received(:execute) - expect(Ai::ActiveContext::Code::IncrementalIndexingService).to have_received(:execute).with(ready_repository) + expect(Ai::ActiveContext::Code::IncrementalIndexingService).to have_received(:execute).with(repository) + end + end + + context 'with a valid repository that is not pending or ready' do + # assuming we follow the suggestion regarding test performance + before do + repository.failed! + end + + # or we can also create a new repository + let(:failed_repository) { create(:ai_active_context_code_repository, state: :failed) } + + it 'does not call any indexing service' do + expect(Ai::ActiveContext::Code::InitialIndexingService).not_to receive(:execute) + expect(Ai::ActiveContext::Code::IncrementalIndexingService).not_to receive(:execute) + + worker.perform(repository.id) end end @@ -51,6 +74,8 @@ context 'when indexing is disabled' do before do allow(::Ai::ActiveContext::Collections::Code).to receive(:indexing?).and_return(false) + + repository.pending! end it 'does not call InitialIndexingService.execute' do @@ -66,6 +91,7 @@ let(:lease_key) { "Ai::ActiveContext::Code::RepositoryIndexWorker/#{repository.id}" } before do + repository.pending! stub_exclusive_lease_taken(lease_key, timeout: described_class::LEASE_TTL) end -- GitLab From f1f6d663f9a9d9db415cc32395b37b4b731dcc84 Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Thu, 14 Aug 2025 22:19:37 -0400 Subject: [PATCH 09/13] update unit tests --- .../workers/ai/active_context/code/repository_index_worker.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ee/app/workers/ai/active_context/code/repository_index_worker.rb b/ee/app/workers/ai/active_context/code/repository_index_worker.rb index 28d50eeb5c59de..8fc50869adb01b 100644 --- a/ee/app/workers/ai/active_context/code/repository_index_worker.rb +++ b/ee/app/workers/ai/active_context/code/repository_index_worker.rb @@ -34,14 +34,14 @@ def perform(id) in_lock(lease_key(id), ttl: LEASE_TTL, sleep_sec: LEASE_TRY_AFTER, retries: LEASE_RETRIES) do if repository.pending? logger.info(build_structured_payload( - message: "Indexing started", + message: 'Indexing started', project_id: id, indexing_mode: 'initial')) InitialIndexingService.execute(repository) elsif repository.ready? logger.info(build_structured_payload( - message: "Indexing started", + message: 'Indexing started', project_id: id, indexing_mode: 'incremental')) -- GitLab From 5911d69830e6268077a02416e8a11026a3ab200f Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Thu, 14 Aug 2025 22:39:14 -0400 Subject: [PATCH 10/13] update unit tests --- .../ai/active_context/code/repository_index_worker_spec.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ee/spec/workers/ai/active_context/code/repository_index_worker_spec.rb b/ee/spec/workers/ai/active_context/code/repository_index_worker_spec.rb index 85f77b16bd3488..a5b8af545db325 100644 --- a/ee/spec/workers/ai/active_context/code/repository_index_worker_spec.rb +++ b/ee/spec/workers/ai/active_context/code/repository_index_worker_spec.rb @@ -46,14 +46,10 @@ end context 'with a valid repository that is not pending or ready' do - # assuming we follow the suggestion regarding test performance before do repository.failed! end - # or we can also create a new repository - let(:failed_repository) { create(:ai_active_context_code_repository, state: :failed) } - it 'does not call any indexing service' do expect(Ai::ActiveContext::Code::InitialIndexingService).not_to receive(:execute) expect(Ai::ActiveContext::Code::IncrementalIndexingService).not_to receive(:execute) -- GitLab From 25ac7dac44ae79f65a6ff5655d687b1eb51ca476 Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Thu, 14 Aug 2025 22:41:11 -0400 Subject: [PATCH 11/13] update unit tests --- .../ai/active_context/code/repository_index_worker_spec.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/ee/spec/workers/ai/active_context/code/repository_index_worker_spec.rb b/ee/spec/workers/ai/active_context/code/repository_index_worker_spec.rb index a5b8af545db325..502e53793103f9 100644 --- a/ee/spec/workers/ai/active_context/code/repository_index_worker_spec.rb +++ b/ee/spec/workers/ai/active_context/code/repository_index_worker_spec.rb @@ -70,7 +70,6 @@ context 'when indexing is disabled' do before do allow(::Ai::ActiveContext::Collections::Code).to receive(:indexing?).and_return(false) - repository.pending! end -- GitLab From abcedc1633f38bd9caf0644d9aab495d191318d5 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 18 Aug 2025 11:22:47 -0400 Subject: [PATCH 12/13] Apply 5 suggestion(s) to 2 file(s) Co-authored-by: Pam Artiaga --- ...context_code_incremental_index_project.yml | 2 +- .../code/repository_index_worker.rb | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/config/feature_flags/gitlab_com_derisk/active_context_code_incremental_index_project.yml b/config/feature_flags/gitlab_com_derisk/active_context_code_incremental_index_project.yml index 3c01cd7c6965c1..d4143e18afd303 100644 --- a/config/feature_flags/gitlab_com_derisk/active_context_code_incremental_index_project.yml +++ b/config/feature_flags/gitlab_com_derisk/active_context_code_incremental_index_project.yml @@ -4,7 +4,7 @@ description: feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/554445 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/201128 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/561020 -milestone: '18.3' +milestone: '18.4' group: group::global search type: gitlab_com_derisk default_enabled: false diff --git a/ee/app/workers/ai/active_context/code/repository_index_worker.rb b/ee/app/workers/ai/active_context/code/repository_index_worker.rb index 8fc50869adb01b..2df0bc93f8aa66 100644 --- a/ee/app/workers/ai/active_context/code/repository_index_worker.rb +++ b/ee/app/workers/ai/active_context/code/repository_index_worker.rb @@ -35,17 +35,32 @@ def perform(id) if repository.pending? logger.info(build_structured_payload( message: 'Indexing started', - project_id: id, + ai_active_context_code_repository_id: id, + project_id: repository.project_id, indexing_mode: 'initial')) InitialIndexingService.execute(repository) + + logger.info(build_structure_payload( + message: 'Indexing done', + ai_active_context_code_repository_id: id, + project_id: repository.project_id, + indexing_mode: 'initial')) elsif repository.ready? logger.info(build_structured_payload( message: 'Indexing started', - project_id: id, + ai_active_context_code_repository_id: id, + project_id: repository.project_id, indexing_mode: 'incremental')) IncrementalIndexingService.execute(repository) + + + logger.info(build_structure_payload( + message: 'Indexing done', + ai_active_context_code_repository_id: id, + project_id: repository.project_id, + indexing_mode: 'incremental')) end end rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError -- GitLab From 88d754a836fdbdedffbd21bd7e7f8a3f3b214244 Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Mon, 18 Aug 2025 11:27:30 -0400 Subject: [PATCH 13/13] create the log_indexing method --- .../code/repository_index_worker.rb | 49 +++++++++---------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/ee/app/workers/ai/active_context/code/repository_index_worker.rb b/ee/app/workers/ai/active_context/code/repository_index_worker.rb index 2df0bc93f8aa66..0e900422fae489 100644 --- a/ee/app/workers/ai/active_context/code/repository_index_worker.rb +++ b/ee/app/workers/ai/active_context/code/repository_index_worker.rb @@ -33,34 +33,13 @@ def perform(id) in_lock(lease_key(id), ttl: LEASE_TTL, sleep_sec: LEASE_TRY_AFTER, retries: LEASE_RETRIES) do if repository.pending? - logger.info(build_structured_payload( - message: 'Indexing started', - ai_active_context_code_repository_id: id, - project_id: repository.project_id, - indexing_mode: 'initial')) - - InitialIndexingService.execute(repository) - - logger.info(build_structure_payload( - message: 'Indexing done', - ai_active_context_code_repository_id: id, - project_id: repository.project_id, - indexing_mode: 'initial')) + log_indexing(repository, 'initial') do + InitialIndexingService.execute(repository) + end elsif repository.ready? - logger.info(build_structured_payload( - message: 'Indexing started', - ai_active_context_code_repository_id: id, - project_id: repository.project_id, - indexing_mode: 'incremental')) - - IncrementalIndexingService.execute(repository) - - - logger.info(build_structure_payload( - message: 'Indexing done', - ai_active_context_code_repository_id: id, - project_id: repository.project_id, - indexing_mode: 'incremental')) + log_indexing(repository, 'incremental') do + IncrementalIndexingService.execute(repository) + end end end rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError @@ -74,6 +53,22 @@ def lease_key(id) def logger @logger ||= ::ActiveContext::Config.logger end + + def log_indexing(repository, indexing_mode) + logger.info(build_structured_payload( + message: 'Indexing started', + ai_active_context_code_repository_id: repository.id, + project_id: repository.project_id, + indexing_mode: indexing_mode)) + + yield + + logger.info(build_structured_payload( + message: 'Indexing done', + ai_active_context_code_repository_id: repository.id, + project_id: repository.project_id, + indexing_mode: indexing_mode)) + end end end end -- GitLab