From d46514cd18ac3e8a0e9992ba41a425b4bcc109fe Mon Sep 17 00:00:00 2001 From: Shivam Sharma <192242476+ShivamSharma43@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:13:23 +0530 Subject: [PATCH] fix(taiga): page through epics, issues, tasks and user stories The four collectors sent only the project filter. Taiga list endpoints are paginated (30 items per page by default) and return page 1 when no page is given, so only the first 30 items of a project were collected, and the parallel page fetches stored the same page ten times. Send page and page_size, and stop when Taiga reports no next page, since a page past the last one returns 404. Closes #9340 Generated-by: Claude Code (Claude Opus 5.5) Co-Authored-By: Claude Opus 5.5 Signed-off-by: Shivam Sharma <192242476+ShivamSharma43@users.noreply.github.com> --- backend/plugins/taiga/tasks/epic_collector.go | 15 ++---- .../plugins/taiga/tasks/issue_collector.go | 15 ++---- backend/plugins/taiga/tasks/shared.go | 53 +++++++++++++++++++ backend/plugins/taiga/tasks/shared_test.go | 50 +++++++++++++++++ backend/plugins/taiga/tasks/task_collector.go | 15 ++---- .../taiga/tasks/user_story_collector.go | 16 ++---- 6 files changed, 123 insertions(+), 41 deletions(-) create mode 100644 backend/plugins/taiga/tasks/shared.go create mode 100644 backend/plugins/taiga/tasks/shared_test.go diff --git a/backend/plugins/taiga/tasks/epic_collector.go b/backend/plugins/taiga/tasks/epic_collector.go index 4deec5c097d..40a5445f7af 100644 --- a/backend/plugins/taiga/tasks/epic_collector.go +++ b/backend/plugins/taiga/tasks/epic_collector.go @@ -19,9 +19,7 @@ package tasks import ( "encoding/json" - "fmt" "net/http" - "net/url" "github.com/apache/devlake/core/errors" "github.com/apache/devlake/core/plugin" @@ -54,14 +52,11 @@ func CollectEpics(taskCtx plugin.SubTaskContext) errors.Error { }, Table: RAW_EPIC_TABLE, }, - ApiClient: data.ApiClient, - PageSize: 1000, - UrlTemplate: "epics", - Query: func(reqData *api.RequestData) (url.Values, errors.Error) { - query := url.Values{} - query.Set("project", fmt.Sprintf("%d", data.Options.ProjectId)) - return query, nil - }, + ApiClient: data.ApiClient, + PageSize: taigaPageSize, + UrlTemplate: "epics", + Query: taigaProjectListQuery(data.Options.ProjectId), + GetNextPageCustomData: getNextTaigaPage, ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { var result []json.RawMessage err := api.UnmarshalResponse(res, &result) diff --git a/backend/plugins/taiga/tasks/issue_collector.go b/backend/plugins/taiga/tasks/issue_collector.go index 937baeab79f..99e4f2d664b 100644 --- a/backend/plugins/taiga/tasks/issue_collector.go +++ b/backend/plugins/taiga/tasks/issue_collector.go @@ -19,9 +19,7 @@ package tasks import ( "encoding/json" - "fmt" "net/http" - "net/url" "github.com/apache/devlake/core/errors" "github.com/apache/devlake/core/plugin" @@ -54,14 +52,11 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { }, Table: RAW_ISSUE_TABLE, }, - ApiClient: data.ApiClient, - PageSize: 1000, - UrlTemplate: "issues", - Query: func(reqData *api.RequestData) (url.Values, errors.Error) { - query := url.Values{} - query.Set("project", fmt.Sprintf("%d", data.Options.ProjectId)) - return query, nil - }, + ApiClient: data.ApiClient, + PageSize: taigaPageSize, + UrlTemplate: "issues", + Query: taigaProjectListQuery(data.Options.ProjectId), + GetNextPageCustomData: getNextTaigaPage, ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { var result []json.RawMessage err := api.UnmarshalResponse(res, &result) diff --git a/backend/plugins/taiga/tasks/shared.go b/backend/plugins/taiga/tasks/shared.go new file mode 100644 index 00000000000..8e3a96dee93 --- /dev/null +++ b/backend/plugins/taiga/tasks/shared.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "net/http" + "net/url" + + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/helpers/pluginhelper/api" +) + +// taigaPageSize is the largest page size Taiga accepts (MAX_PAGINATE_BY). +const taigaPageSize = 1000 + +// taigaProjectListQuery builds the query for a Taiga list endpoint filtered by +// project. Taiga paginates list endpoints (30 items per page by default), so +// the page and page size are always sent. +func taigaProjectListQuery(projectId uint64) func(reqData *api.RequestData) (url.Values, errors.Error) { + return func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("project", fmt.Sprintf("%d", projectId)) + query.Set("page", fmt.Sprintf("%d", reqData.Pager.Page)) + query.Set("page_size", fmt.Sprintf("%d", reqData.Pager.Size)) + return query, nil + } +} + +// getNextTaigaPage stops the collection when Taiga reports no next page. +// Taiga answers a page past the last one with 404, so a full last page must +// not lead to another request. +func getNextTaigaPage(_ *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { + if prevPageResponse.Header.Get("X-Pagination-Next") == "" { + return nil, api.ErrFinishCollect + } + return nil, nil +} diff --git a/backend/plugins/taiga/tasks/shared_test.go b/backend/plugins/taiga/tasks/shared_test.go new file mode 100644 index 00000000000..2d8dd645fa8 --- /dev/null +++ b/backend/plugins/taiga/tasks/shared_test.go @@ -0,0 +1,50 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + "testing" + + "github.com/apache/devlake/helpers/pluginhelper/api" + "github.com/stretchr/testify/assert" +) + +func TestTaigaProjectListQueryRequestsThePage(t *testing.T) { + query := taigaProjectListQuery(42) + + got, err := query(&api.RequestData{Pager: &api.Pager{Page: 3, Size: 1000}}) + + assert.Nil(t, err) + assert.Equal(t, "42", got.Get("project")) + // Taiga paginates list endpoints (30 items by default) and only returns + // the page asked for, so the collector must send page and page_size. + assert.Equal(t, "3", got.Get("page")) + assert.Equal(t, "1000", got.Get("page_size")) +} + +func TestGetNextTaigaPage(t *testing.T) { + more := &http.Response{Header: http.Header{}} + more.Header.Set("X-Pagination-Next", "https://taiga.example/api/v1/issues?page=2&page_size=1000&project=42") + _, err := getNextTaigaPage(nil, more) + assert.Nil(t, err) + + last := &http.Response{Header: http.Header{}} + _, err = getNextTaigaPage(nil, last) + assert.ErrorIs(t, err, api.ErrFinishCollect) +} diff --git a/backend/plugins/taiga/tasks/task_collector.go b/backend/plugins/taiga/tasks/task_collector.go index 4d0f6a6f41d..ca5a905574c 100644 --- a/backend/plugins/taiga/tasks/task_collector.go +++ b/backend/plugins/taiga/tasks/task_collector.go @@ -19,9 +19,7 @@ package tasks import ( "encoding/json" - "fmt" "net/http" - "net/url" "github.com/apache/devlake/core/errors" "github.com/apache/devlake/core/plugin" @@ -54,14 +52,11 @@ func CollectTasks(taskCtx plugin.SubTaskContext) errors.Error { }, Table: RAW_TASK_TABLE, }, - ApiClient: data.ApiClient, - PageSize: 1000, - UrlTemplate: "tasks", - Query: func(reqData *api.RequestData) (url.Values, errors.Error) { - query := url.Values{} - query.Set("project", fmt.Sprintf("%d", data.Options.ProjectId)) - return query, nil - }, + ApiClient: data.ApiClient, + PageSize: taigaPageSize, + UrlTemplate: "tasks", + Query: taigaProjectListQuery(data.Options.ProjectId), + GetNextPageCustomData: getNextTaigaPage, ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { var result []json.RawMessage err := api.UnmarshalResponse(res, &result) diff --git a/backend/plugins/taiga/tasks/user_story_collector.go b/backend/plugins/taiga/tasks/user_story_collector.go index 1fa69f86ca7..206ae72c460 100644 --- a/backend/plugins/taiga/tasks/user_story_collector.go +++ b/backend/plugins/taiga/tasks/user_story_collector.go @@ -19,9 +19,7 @@ package tasks import ( "encoding/json" - "fmt" "net/http" - "net/url" "github.com/apache/devlake/core/errors" "github.com/apache/devlake/core/plugin" @@ -54,15 +52,11 @@ func CollectUserStories(taskCtx plugin.SubTaskContext) errors.Error { }, Table: RAW_USER_STORY_TABLE, }, - ApiClient: data.ApiClient, - PageSize: 1000, // Fetch all in one page - Taiga returns all user stories for a project - UrlTemplate: "userstories", - Query: func(reqData *api.RequestData) (url.Values, errors.Error) { - query := url.Values{} - query.Set("project", fmt.Sprintf("%d", data.Options.ProjectId)) - // Don't specify page - get all results - return query, nil - }, + ApiClient: data.ApiClient, + PageSize: taigaPageSize, + UrlTemplate: "userstories", + Query: taigaProjectListQuery(data.Options.ProjectId), + GetNextPageCustomData: getNextTaigaPage, ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { var result []json.RawMessage err := api.UnmarshalResponse(res, &result)