fix(taiga): page through epics, issues, tasks and user stories

backend/plugins/taiga/tasks/epic_collector.go

@@ -19,9 +19,7 @@ package tasks
 
 import (
 	"encoding/json"
-	"fmt"
 	"net/http"
-	"net/url"
 
 	"github.com/apache/devlake/core/errors"
 	"github.com/apache/devlake/core/plugin"
@@ -54,14 +52,11 @@ func CollectEpics(taskCtx plugin.SubTaskContext) errors.Error {
 			},
 			Table: RAW_EPIC_TABLE,
 		},
-		ApiClient:   data.ApiClient,
-		PageSize:    1000,
-		UrlTemplate: "epics",
-		Query: func(reqData *api.RequestData) (url.Values, errors.Error) {
-			query := url.Values{}
-			query.Set("project", fmt.Sprintf("%d", data.Options.ProjectId))
-			return query, nil
-		},
+		ApiClient:             data.ApiClient,
+		PageSize:              taigaPageSize,
+		UrlTemplate:           "epics",
+		Query:                 taigaProjectListQuery(data.Options.ProjectId),
+		GetNextPageCustomData: getNextTaigaPage,
 		ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
 			var result []json.RawMessage
 			err := api.UnmarshalResponse(res, &result)

backend/plugins/taiga/tasks/issue_collector.go

@@ -19,9 +19,7 @@ package tasks
 
 import (
 	"encoding/json"
-	"fmt"
 	"net/http"
-	"net/url"
 
 	"github.com/apache/devlake/core/errors"
 	"github.com/apache/devlake/core/plugin"
@@ -54,14 +52,11 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error {
 			},
 			Table: RAW_ISSUE_TABLE,
 		},
-		ApiClient:   data.ApiClient,
-		PageSize:    1000,
-		UrlTemplate: "issues",
-		Query: func(reqData *api.RequestData) (url.Values, errors.Error) {
-			query := url.Values{}
-			query.Set("project", fmt.Sprintf("%d", data.Options.ProjectId))
-			return query, nil
-		},
+		ApiClient:             data.ApiClient,
+		PageSize:              taigaPageSize,
+		UrlTemplate:           "issues",
+		Query:                 taigaProjectListQuery(data.Options.ProjectId),
+		GetNextPageCustomData: getNextTaigaPage,
 		ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
 			var result []json.RawMessage
 			err := api.UnmarshalResponse(res, &result)

backend/plugins/taiga/tasks/shared.go

@@ -0,0 +1,53 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"github.com/apache/devlake/core/errors"
+	"github.com/apache/devlake/helpers/pluginhelper/api"
+)
+
+// taigaPageSize is the largest page size Taiga accepts (MAX_PAGINATE_BY).
+const taigaPageSize = 1000
+
+// taigaProjectListQuery builds the query for a Taiga list endpoint filtered by
+// project. Taiga paginates list endpoints (30 items per page by default), so
+// the page and page size are always sent.
+func taigaProjectListQuery(projectId uint64) func(reqData *api.RequestData) (url.Values, errors.Error) {
+	return func(reqData *api.RequestData) (url.Values, errors.Error) {
+		query := url.Values{}
+		query.Set("project", fmt.Sprintf("%d", projectId))
+		query.Set("page", fmt.Sprintf("%d", reqData.Pager.Page))
+		query.Set("page_size", fmt.Sprintf("%d", reqData.Pager.Size))
+		return query, nil
+	}
+}
+
+// getNextTaigaPage stops the collection when Taiga reports no next page.
+// Taiga answers a page past the last one with 404, so a full last page must
+// not lead to another request.
+func getNextTaigaPage(_ *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) {
+	if prevPageResponse.Header.Get("X-Pagination-Next") == "" {
+		return nil, api.ErrFinishCollect
+	}
+	return nil, nil
+}

backend/plugins/taiga/tasks/shared_test.go

@@ -0,0 +1,50 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/apache/devlake/helpers/pluginhelper/api"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTaigaProjectListQueryRequestsThePage(t *testing.T) {
+	query := taigaProjectListQuery(42)
+
+	got, err := query(&api.RequestData{Pager: &api.Pager{Page: 3, Size: 1000}})
+
+	assert.Nil(t, err)
+	assert.Equal(t, "42", got.Get("project"))
+	// Taiga paginates list endpoints (30 items by default) and only returns
+	// the page asked for, so the collector must send page and page_size.
+	assert.Equal(t, "3", got.Get("page"))
+	assert.Equal(t, "1000", got.Get("page_size"))
+}
+
+func TestGetNextTaigaPage(t *testing.T) {
+	more := &http.Response{Header: http.Header{}}
+	more.Header.Set("X-Pagination-Next", "https://taiga.example/api/v1/issues?page=2&page_size=1000&project=42")
+	_, err := getNextTaigaPage(nil, more)
+	assert.Nil(t, err)
+
+	last := &http.Response{Header: http.Header{}}
+	_, err = getNextTaigaPage(nil, last)
+	assert.ErrorIs(t, err, api.ErrFinishCollect)
+}

backend/plugins/taiga/tasks/task_collector.go

@@ -19,9 +19,7 @@ package tasks
 
 import (
 	"encoding/json"
-	"fmt"
 	"net/http"
-	"net/url"
 
 	"github.com/apache/devlake/core/errors"
 	"github.com/apache/devlake/core/plugin"
@@ -54,14 +52,11 @@ func CollectTasks(taskCtx plugin.SubTaskContext) errors.Error {
 			},
 			Table: RAW_TASK_TABLE,
 		},
-		ApiClient:   data.ApiClient,
-		PageSize:    1000,
-		UrlTemplate: "tasks",
-		Query: func(reqData *api.RequestData) (url.Values, errors.Error) {
-			query := url.Values{}
-			query.Set("project", fmt.Sprintf("%d", data.Options.ProjectId))
-			return query, nil
-		},
+		ApiClient:             data.ApiClient,
+		PageSize:              taigaPageSize,
+		UrlTemplate:           "tasks",
+		Query:                 taigaProjectListQuery(data.Options.ProjectId),
+		GetNextPageCustomData: getNextTaigaPage,
 		ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
 			var result []json.RawMessage
 			err := api.UnmarshalResponse(res, &result)

backend/plugins/taiga/tasks/user_story_collector.go

@@ -19,9 +19,7 @@ package tasks
 
 import (
 	"encoding/json"
-	"fmt"
 	"net/http"
-	"net/url"
 
 	"github.com/apache/devlake/core/errors"
 	"github.com/apache/devlake/core/plugin"
@@ -54,15 +52,11 @@ func CollectUserStories(taskCtx plugin.SubTaskContext) errors.Error {
 			},
 			Table: RAW_USER_STORY_TABLE,
 		},
-		ApiClient:   data.ApiClient,
-		PageSize:    1000, // Fetch all in one page - Taiga returns all user stories for a project
-		UrlTemplate: "userstories",
-		Query: func(reqData *api.RequestData) (url.Values, errors.Error) {
-			query := url.Values{}
-			query.Set("project", fmt.Sprintf("%d", data.Options.ProjectId))
-			// Don't specify page - get all results
-			return query, nil
-		},
+		ApiClient:             data.ApiClient,
+		PageSize:              taigaPageSize,
+		UrlTemplate:           "userstories",
+		Query:                 taigaProjectListQuery(data.Options.ProjectId),
+		GetNextPageCustomData: getNextTaigaPage,
 		ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
 			var result []json.RawMessage
 			err := api.UnmarshalResponse(res, &result)