From db18be6527cd06b18250a562ca0632f3cda59799 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 10:58:27 +1000 Subject: [PATCH 01/28] feat(linear): add tool-layer models and init migration Add the Linear plugin's tool-layer data models (connection, team scope, scope config, account, issue, comment, issue label, workflow state, cycle, issue history) and the initial schema migration with archived snapshots. The connection authenticates with a personal API key passed verbatim in the Authorization header (Linear uses no Bearer prefix). Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- backend/plugins/linear/models/account.go | 38 ++++ backend/plugins/linear/models/connection.go | 73 ++++++++ backend/plugins/linear/models/cycle.go | 41 +++++ backend/plugins/linear/models/issue.go | 57 ++++++ .../plugins/linear/models/issue_comment.go | 40 +++++ .../plugins/linear/models/issue_history.go | 43 +++++ backend/plugins/linear/models/issue_label.go | 35 ++++ .../20260601_add_init_tables.go | 51 ++++++ .../migrationscripts/archived/models.go | 165 ++++++++++++++++++ .../models/migrationscripts/register.go | 29 +++ backend/plugins/linear/models/scope_config.go | 38 ++++ backend/plugins/linear/models/team.go | 66 +++++++ .../plugins/linear/models/workflow_state.go | 39 +++++ 13 files changed, 715 insertions(+) create mode 100644 backend/plugins/linear/models/account.go create mode 100644 backend/plugins/linear/models/connection.go create mode 100644 backend/plugins/linear/models/cycle.go create mode 100644 backend/plugins/linear/models/issue.go create mode 100644 backend/plugins/linear/models/issue_comment.go create mode 100644 backend/plugins/linear/models/issue_history.go create mode 100644 backend/plugins/linear/models/issue_label.go create mode 100644 backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go create mode 100644 backend/plugins/linear/models/migrationscripts/archived/models.go create mode 100644 backend/plugins/linear/models/migrationscripts/register.go create mode 100644 backend/plugins/linear/models/scope_config.go create mode 100644 backend/plugins/linear/models/team.go create mode 100644 backend/plugins/linear/models/workflow_state.go diff --git a/backend/plugins/linear/models/account.go b/backend/plugins/linear/models/account.go new file mode 100644 index 00000000000..7d1c9376f98 --- /dev/null +++ b/backend/plugins/linear/models/account.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearAccount is a Linear user (tool layer), converted to crossdomain.Account. +type LinearAccount struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + Name string `gorm:"type:varchar(255)" json:"name"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Email string `gorm:"type:varchar(255)" json:"email"` + AvatarUrl string `gorm:"type:varchar(255)" json:"avatarUrl"` + Active bool `json:"active"` + common.NoPKModel +} + +func (LinearAccount) TableName() string { + return "_tool_linear_accounts" +} diff --git a/backend/plugins/linear/models/connection.go b/backend/plugins/linear/models/connection.go new file mode 100644 index 00000000000..6f63e431a11 --- /dev/null +++ b/backend/plugins/linear/models/connection.go @@ -0,0 +1,73 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// LinearConn holds the essential information to connect to the Linear API. +// Linear authenticates with a personal API key passed verbatim in the +// `Authorization` header (NO `Bearer` prefix), so we implement our own +// SetupAuthentication instead of reusing helper.AccessToken. +type LinearConn struct { + helper.RestConnection `mapstructure:",squash"` + Token string `mapstructure:"token" validate:"required" json:"token" gorm:"serializer:encdec"` +} + +// SetupAuthentication sets up the HTTP request authentication for the Linear API. +func (lc *LinearConn) SetupAuthentication(req *http.Request) errors.Error { + req.Header.Set("Authorization", lc.Token) + return nil +} + +func (lc *LinearConn) Sanitize() LinearConn { + lc.Token = utils.SanitizeString(lc.Token) + return *lc +} + +// LinearConnection holds LinearConn plus ID/Name for database storage. +type LinearConnection struct { + helper.BaseConnection `mapstructure:",squash"` + LinearConn `mapstructure:",squash"` +} + +func (connection LinearConnection) Sanitize() LinearConnection { + connection.LinearConn = connection.LinearConn.Sanitize() + return connection +} + +func (connection *LinearConnection) MergeFromRequest(target *LinearConnection, body map[string]interface{}) error { + token := target.Token + if err := helper.DecodeMapStruct(body, target, true); err != nil { + return err + } + modifiedToken := target.Token + if modifiedToken == "" || modifiedToken == utils.SanitizeString(token) { + target.Token = token + } + return nil +} + +func (LinearConnection) TableName() string { + return "_tool_linear_connections" +} diff --git a/backend/plugins/linear/models/cycle.go b/backend/plugins/linear/models/cycle.go new file mode 100644 index 00000000000..4f61370dbb2 --- /dev/null +++ b/backend/plugins/linear/models/cycle.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearCycle is a Linear cycle (sprint-equivalent), converted to ticket.Sprint. +type LinearCycle struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` + Number int `json:"number"` + Name string `gorm:"type:varchar(255)" json:"name"` + StartsAt *time.Time `json:"startsAt"` + EndsAt *time.Time `json:"endsAt"` + CompletedAt *time.Time `json:"completedAt"` + common.NoPKModel +} + +func (LinearCycle) TableName() string { + return "_tool_linear_cycles" +} diff --git a/backend/plugins/linear/models/issue.go b/backend/plugins/linear/models/issue.go new file mode 100644 index 00000000000..b3f8a15206e --- /dev/null +++ b/backend/plugins/linear/models/issue.go @@ -0,0 +1,57 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearIssue is the tool-layer representation of a Linear issue. +type LinearIssue struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` + Identifier string `gorm:"type:varchar(255)" json:"identifier"` + Number int `json:"number"` + Title string `gorm:"type:varchar(255)" json:"title"` + Description string `json:"description"` + Url string `gorm:"type:varchar(255)" json:"url"` + Priority int `json:"priority"` + PriorityLabel string `gorm:"type:varchar(100)" json:"priorityLabel"` + Estimate *float64 `json:"estimate"` + StateId string `gorm:"index;type:varchar(255)" json:"stateId"` + StateName string `gorm:"type:varchar(255)" json:"stateName"` + StateType string `gorm:"type:varchar(100)" json:"stateType"` + CreatorId string `gorm:"type:varchar(255)" json:"creatorId"` + AssigneeId string `gorm:"type:varchar(255)" json:"assigneeId"` + CycleId string `gorm:"index;type:varchar(255)" json:"cycleId"` + ParentId string `gorm:"type:varchar(255)" json:"parentId"` + LeadTimeMinutes *uint `json:"leadTimeMinutes"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `gorm:"index" json:"updatedAt"` + StartedAt *time.Time `json:"startedAt"` + CompletedAt *time.Time `json:"completedAt"` + CanceledAt *time.Time `json:"canceledAt"` + common.NoPKModel +} + +func (LinearIssue) TableName() string { + return "_tool_linear_issues" +} diff --git a/backend/plugins/linear/models/issue_comment.go b/backend/plugins/linear/models/issue_comment.go new file mode 100644 index 00000000000..8ba18505038 --- /dev/null +++ b/backend/plugins/linear/models/issue_comment.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearComment is the tool-layer representation of a comment on a Linear issue. +type LinearComment struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + IssueId string `gorm:"index;type:varchar(255)" json:"issueId"` + Body string `json:"body"` + AuthorId string `gorm:"type:varchar(255)" json:"authorId"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `gorm:"index" json:"updatedAt"` + common.NoPKModel +} + +func (LinearComment) TableName() string { + return "_tool_linear_comments" +} diff --git a/backend/plugins/linear/models/issue_history.go b/backend/plugins/linear/models/issue_history.go new file mode 100644 index 00000000000..d4b6cc818bf --- /dev/null +++ b/backend/plugins/linear/models/issue_history.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearIssueHistory is a single entry in a Linear issue's history, used to +// build domain-layer changelogs and derive lead/cycle time. +type LinearIssueHistory struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + IssueId string `gorm:"index;type:varchar(255)" json:"issueId"` + ActorId string `gorm:"type:varchar(255)" json:"actorId"` + FromStateId string `gorm:"type:varchar(255)" json:"fromStateId"` + FromStateName string `gorm:"type:varchar(255)" json:"fromStateName"` + ToStateId string `gorm:"type:varchar(255)" json:"toStateId"` + ToStateName string `gorm:"type:varchar(255)" json:"toStateName"` + CreatedAt time.Time `gorm:"index" json:"createdAt"` + common.NoPKModel +} + +func (LinearIssueHistory) TableName() string { + return "_tool_linear_issue_history" +} diff --git a/backend/plugins/linear/models/issue_label.go b/backend/plugins/linear/models/issue_label.go new file mode 100644 index 00000000000..76a4517cb7b --- /dev/null +++ b/backend/plugins/linear/models/issue_label.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearIssueLabel joins a Linear issue to one of its labels. Labels are +// collected inline with issues, so there is no separate label collector. +type LinearIssueLabel struct { + ConnectionId uint64 `gorm:"primaryKey"` + IssueId string `gorm:"primaryKey;type:varchar(255)" json:"issueId"` + LabelName string `gorm:"primaryKey;type:varchar(255)" json:"labelName"` + common.NoPKModel +} + +func (LinearIssueLabel) TableName() string { + return "_tool_linear_issue_labels" +} diff --git a/backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go b/backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go new file mode 100644 index 00000000000..9bb577343be --- /dev/null +++ b/backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/linear/models/migrationscripts/archived" +) + +type addInitTables struct{} + +func (*addInitTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.LinearConnection{}, + &archived.LinearTeam{}, + &archived.LinearScopeConfig{}, + &archived.LinearAccount{}, + &archived.LinearIssue{}, + &archived.LinearComment{}, + &archived.LinearIssueLabel{}, + &archived.LinearWorkflowState{}, + &archived.LinearCycle{}, + &archived.LinearIssueHistory{}, + ) +} + +func (*addInitTables) Version() uint64 { + return 20260601000001 +} + +func (*addInitTables) Name() string { + return "linear init schemas" +} diff --git a/backend/plugins/linear/models/migrationscripts/archived/models.go b/backend/plugins/linear/models/migrationscripts/archived/models.go new file mode 100644 index 00000000000..a544b2afb5e --- /dev/null +++ b/backend/plugins/linear/models/migrationscripts/archived/models.go @@ -0,0 +1,165 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package archived holds frozen snapshots of the tool-layer models as they +// existed at each migration. The live models in plugins/linear/models may +// evolve; these snapshots keep historical migrations stable. +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type LinearConnection struct { + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name"` + archived.Model + Endpoint string `mapstructure:"endpoint" json:"endpoint"` + Proxy string `mapstructure:"proxy" json:"proxy"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Token string `mapstructure:"token" json:"token" gorm:"serializer:encdec"` +} + +func (LinearConnection) TableName() string { return "_tool_linear_connections" } + +type LinearTeam struct { + archived.NoPKModel + ConnectionId uint64 `json:"connectionId" gorm:"primaryKey"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty"` + TeamId string `json:"teamId" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" gorm:"type:varchar(255)"` + Key string `json:"key" gorm:"type:varchar(255)"` + Description string `json:"description"` +} + +func (LinearTeam) TableName() string { return "_tool_linear_teams" } + +type LinearScopeConfig struct { + archived.ScopeConfig + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `gorm:"type:varchar(255);uniqueIndex" json:"name"` +} + +func (LinearScopeConfig) TableName() string { return "_tool_linear_scope_configs" } + +type LinearAccount struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + DisplayName string `gorm:"type:varchar(255)"` + Email string `gorm:"type:varchar(255)"` + AvatarUrl string `gorm:"type:varchar(255)"` + Active bool + archived.NoPKModel +} + +func (LinearAccount) TableName() string { return "_tool_linear_accounts" } + +type LinearIssue struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"index;type:varchar(255)"` + Identifier string `gorm:"type:varchar(255)"` + Number int + Title string `gorm:"type:varchar(255)"` + Description string + Url string `gorm:"type:varchar(255)"` + Priority int + PriorityLabel string `gorm:"type:varchar(100)"` + Estimate *float64 + StateId string `gorm:"index;type:varchar(255)"` + StateName string `gorm:"type:varchar(255)"` + StateType string `gorm:"type:varchar(100)"` + CreatorId string `gorm:"type:varchar(255)"` + AssigneeId string `gorm:"type:varchar(255)"` + CycleId string `gorm:"index;type:varchar(255)"` + ParentId string `gorm:"type:varchar(255)"` + LeadTimeMinutes *uint + CreatedAt time.Time + UpdatedAt time.Time `gorm:"index"` + StartedAt *time.Time + CompletedAt *time.Time + CanceledAt *time.Time + archived.NoPKModel +} + +func (LinearIssue) TableName() string { return "_tool_linear_issues" } + +type LinearComment struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + IssueId string `gorm:"index;type:varchar(255)"` + Body string + AuthorId string `gorm:"type:varchar(255)"` + CreatedAt time.Time + UpdatedAt time.Time `gorm:"index"` + archived.NoPKModel +} + +func (LinearComment) TableName() string { return "_tool_linear_comments" } + +type LinearIssueLabel struct { + ConnectionId uint64 `gorm:"primaryKey"` + IssueId string `gorm:"primaryKey;type:varchar(255)"` + LabelName string `gorm:"primaryKey;type:varchar(255)"` + archived.NoPKModel +} + +func (LinearIssueLabel) TableName() string { return "_tool_linear_issue_labels" } + +type LinearWorkflowState struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"index;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Type string `gorm:"type:varchar(100)"` + Color string `gorm:"type:varchar(50)"` + Position float64 + archived.NoPKModel +} + +func (LinearWorkflowState) TableName() string { return "_tool_linear_workflow_states" } + +type LinearCycle struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"index;type:varchar(255)"` + Number int + Name string `gorm:"type:varchar(255)"` + StartsAt *time.Time + EndsAt *time.Time + CompletedAt *time.Time + archived.NoPKModel +} + +func (LinearCycle) TableName() string { return "_tool_linear_cycles" } + +type LinearIssueHistory struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + IssueId string `gorm:"index;type:varchar(255)"` + ActorId string `gorm:"type:varchar(255)"` + FromStateId string `gorm:"type:varchar(255)"` + FromStateName string `gorm:"type:varchar(255)"` + ToStateId string `gorm:"type:varchar(255)"` + ToStateName string `gorm:"type:varchar(255)"` + CreatedAt time.Time `gorm:"index"` + archived.NoPKModel +} + +func (LinearIssueHistory) TableName() string { return "_tool_linear_issue_history" } diff --git a/backend/plugins/linear/models/migrationscripts/register.go b/backend/plugins/linear/models/migrationscripts/register.go new file mode 100644 index 00000000000..ec054748c27 --- /dev/null +++ b/backend/plugins/linear/models/migrationscripts/register.go @@ -0,0 +1,29 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All return all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addInitTables), + } +} diff --git a/backend/plugins/linear/models/scope_config.go b/backend/plugins/linear/models/scope_config.go new file mode 100644 index 00000000000..1af91db58b0 --- /dev/null +++ b/backend/plugins/linear/models/scope_config.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearScopeConfig is intentionally minimal: Linear's WorkflowState.type maps +// deterministically to TODO/IN_PROGRESS/DONE, so no user status mapping is +// required. It is reserved for future label-based issue-type mapping. +type LinearScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` +} + +func (LinearScopeConfig) TableName() string { + return "_tool_linear_scope_configs" +} + +func (sc *LinearScopeConfig) SetConnectionId(c *LinearScopeConfig, connectionId uint64) { + c.ConnectionId = connectionId + c.ScopeConfig.ConnectionId = connectionId +} diff --git a/backend/plugins/linear/models/team.go b/backend/plugins/linear/models/team.go new file mode 100644 index 00000000000..d6226208266 --- /dev/null +++ b/backend/plugins/linear/models/team.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.ToolLayerScope = (*LinearTeam)(nil) + +// LinearTeam is the data-source scope for the Linear plugin. A Linear Team +// owns issues, cycles, workflow states and labels, mapping cleanly to a +// DevLake domain-layer ticket.Board. +type LinearTeam struct { + common.Scope `mapstructure:",squash"` + TeamId string `json:"teamId" mapstructure:"teamId" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + Key string `json:"key" mapstructure:"key" gorm:"type:varchar(255)"` + Description string `json:"description" mapstructure:"description"` +} + +func (t LinearTeam) ScopeId() string { + return t.TeamId +} + +func (t LinearTeam) ScopeName() string { + return t.Name +} + +func (t LinearTeam) ScopeFullName() string { + return t.Name +} + +func (t LinearTeam) ScopeParams() interface{} { + return &LinearApiParams{ + ConnectionId: t.ConnectionId, + TeamId: t.TeamId, + } +} + +func (LinearTeam) TableName() string { + return "_tool_linear_teams" +} + +// LinearApiParams identifies the scope a raw row belongs to. It is stored in +// the `params` column of every _raw_linear_* table. +type LinearApiParams struct { + ConnectionId uint64 + TeamId string +} diff --git a/backend/plugins/linear/models/workflow_state.go b/backend/plugins/linear/models/workflow_state.go new file mode 100644 index 00000000000..9273f183c91 --- /dev/null +++ b/backend/plugins/linear/models/workflow_state.go @@ -0,0 +1,39 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearWorkflowState is a Linear team's workflow state. Its Type +// (backlog|unstarted|started|completed|canceled) drives issue status mapping. +type LinearWorkflowState struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` + Name string `gorm:"type:varchar(255)" json:"name"` + Type string `gorm:"type:varchar(100)" json:"type"` + Color string `gorm:"type:varchar(50)" json:"color"` + Position float64 `json:"position"` + common.NoPKModel +} + +func (LinearWorkflowState) TableName() string { + return "_tool_linear_workflow_states" +} From 1009c4e4021eb03efb740ec322755f8fc5fed3cc Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 11:28:32 +1000 Subject: [PATCH 02/28] feat(linear): add plugin skeleton, connection API and GraphQL client Wire the Linear plugin entry point and implement all required plugin interfaces (meta, init, task, api, model, source, migration, blueprint v200, closeable). Add connection/scope/scope-config CRUD via the data-source helper, a test-connection endpoint that runs a GraphQL viewer query, and a rate-limited async GraphQL client that injects the API key via a bare Authorization header. SubTaskMetas is intentionally empty; collectors are added per entity in following commits. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- backend/plugins/linear/api/blueprint_v200.go | 98 +++++++++ backend/plugins/linear/api/connection_api.go | 177 ++++++++++++++++ backend/plugins/linear/api/init.go | 47 +++++ backend/plugins/linear/api/scope_api.go | 105 ++++++++++ .../plugins/linear/api/scope_config_api.go | 93 +++++++++ backend/plugins/linear/impl/impl.go | 194 ++++++++++++++++++ backend/plugins/linear/linear.go | 43 ++++ backend/plugins/linear/tasks/api_client.go | 108 ++++++++++ backend/plugins/linear/tasks/task_data.go | 43 ++++ 9 files changed, 908 insertions(+) create mode 100644 backend/plugins/linear/api/blueprint_v200.go create mode 100644 backend/plugins/linear/api/connection_api.go create mode 100644 backend/plugins/linear/api/init.go create mode 100644 backend/plugins/linear/api/scope_api.go create mode 100644 backend/plugins/linear/api/scope_config_api.go create mode 100644 backend/plugins/linear/impl/impl.go create mode 100644 backend/plugins/linear/linear.go create mode 100644 backend/plugins/linear/tasks/api_client.go create mode 100644 backend/plugins/linear/tasks/task_data.go diff --git a/backend/plugins/linear/api/blueprint_v200.go b/backend/plugins/linear/api/blueprint_v200.go new file mode 100644 index 00000000000..281d018cefb --- /dev/null +++ b/backend/plugins/linear/api/blueprint_v200.go @@ -0,0 +1,98 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func MakePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + plan, err := makePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + scopes, err := makeScopesV200(scopeDetails, connection) + return plan, scopes, err +} + +func makePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig], + connection *models.LinearConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + task, err := helper.MakePipelinePlanTask( + "linear", + subtaskMetas, + scopeConfig.Entities, + tasks.LinearOptions{ + ConnectionId: connection.ID, + TeamId: scope.TeamId, + }, + ) + if err != nil { + return nil, err + } + stage = append(stage, task) + plan[i] = stage + } + return plan, nil +} + +func makeScopesV200( + scopeDetails []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig], + connection *models.LinearConnection, +) ([]plugin.Scope, errors.Error) { + scopes := make([]plugin.Scope, 0, len(scopeDetails)) + idgen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + for _, scopeDetail := range scopeDetails { + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + id := idgen.Generate(connection.ID, scope.TeamId) + if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { + scopes = append(scopes, ticket.NewBoard(id, scope.Name)) + } + } + return scopes, nil +} diff --git a/backend/plugins/linear/api/connection_api.go b/backend/plugins/linear/api/connection_api.go new file mode 100644 index 00000000000..c2c95c54d64 --- /dev/null +++ b/backend/plugins/linear/api/connection_api.go @@ -0,0 +1,177 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/server/api/shared" +) + +const defaultEndpoint = "https://api.linear.app/graphql" + +type LinearTestConnResponse struct { + shared.ApiBody + Connection *models.LinearConn +} + +func testConnection(ctx context.Context, connection models.LinearConn) (*LinearTestConnResponse, errors.Error) { + if vld != nil { + if err := vld.Struct(connection); err != nil { + return nil, errors.Default.Wrap(err, "error validating target") + } + } + if connection.Endpoint == "" { + connection.Endpoint = defaultEndpoint + } + apiClient, err := helper.NewApiClientFromConnection(ctx, basicRes, &connection) + if err != nil { + return nil, err + } + // Linear is GraphQL-over-HTTP-POST; a minimal viewer query verifies the key. + reqBody := map[string]interface{}{"query": "{ viewer { id name } }"} + res, err := apiClient.Post("", nil, reqBody, nil) + if err != nil { + return nil, errors.BadInput.Wrap(err, "verify token failed") + } + if res.StatusCode == http.StatusUnauthorized || res.StatusCode == http.StatusForbidden { + return nil, errors.HttpStatus(http.StatusBadRequest).New("authentication failed, please check your API key") + } + if res.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection") + } + connection = connection.Sanitize() + body := LinearTestConnResponse{} + body.Success = true + body.Message = "success" + body.Connection = &connection + return &body, nil +} + +// TestConnection test linear connection +// @Summary test linear connection +// @Description Test linear Connection +// @Tags plugins/linear +// @Param body body models.LinearConn true "json body" +// @Success 200 {object} LinearTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/test [POST] +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connection models.LinearConn + if err := helper.Decode(input.Body, &connection, vld); err != nil { + return nil, err + } + result, err := testConnection(context.TODO(), connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// TestExistingConnection test linear connection by ID +// @Summary test linear connection +// @Description Test linear Connection +// @Tags plugins/linear +// @Param connectionId path int true "connection ID" +// @Success 200 {object} LinearTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/test [POST] +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.GetMergedConnection(input) + if err != nil { + return nil, errors.BadInput.Wrap(err, "find connection from db") + } + if err := helper.DecodeMapStruct(input.Body, connection, false); err != nil { + return nil, err + } + result, testErr := testConnection(context.TODO(), connection.LinearConn) + if testErr != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, testErr) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// PostConnections create linear connection +// @Summary create linear connection +// @Description Create linear connection +// @Tags plugins/linear +// @Param body body models.LinearConnection true "json body" +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections [POST] +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Post(input) +} + +// PatchConnection patch linear connection +// @Summary patch linear connection +// @Description Patch linear connection +// @Tags plugins/linear +// @Param body body models.LinearConnection true "json body" +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId} [PATCH] +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Patch(input) +} + +// DeleteConnection delete a linear connection +// @Summary delete a linear connection +// @Description Delete a linear connection +// @Tags plugins/linear +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 409 {object} services.BlueprintProjectPairs "References exist to this connection" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId} [DELETE] +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Delete(input) +} + +// ListConnections get all linear connections +// @Summary get all linear connections +// @Description Get all linear connections +// @Tags plugins/linear +// @Success 200 {object} []models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections [GET] +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetAll(input) +} + +// GetConnection get linear connection detail +// @Summary get linear connection detail +// @Description Get linear connection detail +// @Tags plugins/linear +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId} [GET] +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetDetail(input) +} diff --git a/backend/plugins/linear/api/init.go b/backend/plugins/linear/api/init.go new file mode 100644 index 00000000000..2dc7da5943a --- /dev/null +++ b/backend/plugins/linear/api/init.go @@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/go-playground/validator/v10" +) + +var vld *validator.Validate +var basicRes context.BasicRes +var dsHelper *api.DsHelper[models.LinearConnection, models.LinearTeam, models.LinearScopeConfig] + +func Init(br context.BasicRes, p plugin.PluginMeta) { + basicRes = br + vld = validator.New() + dsHelper = api.NewDataSourceHelper[ + models.LinearConnection, models.LinearTeam, models.LinearScopeConfig, + ]( + br, + p.Name(), + []string{"name"}, + func(c models.LinearConnection) models.LinearConnection { + return c.Sanitize() + }, + nil, + nil, + ) +} diff --git a/backend/plugins/linear/api/scope_api.go b/backend/plugins/linear/api/scope_api.go new file mode 100644 index 00000000000..9871d1d2bfe --- /dev/null +++ b/backend/plugins/linear/api/scope_api.go @@ -0,0 +1,105 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +type PutScopesReqBody api.PutScopesReqBody[models.LinearTeam] +type ScopeDetail api.ScopeDetail[models.LinearTeam, models.LinearScopeConfig] + +// PutScopes create or update linear teams +// @Summary create or update linear teams +// @Description Create or update linear teams +// @Tags plugins/linear +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scope body PutScopesReqBody true "json" +// @Success 200 {object} []models.LinearTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes [PUT] +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// PatchScope patch to linear team +// @Summary patch to linear team +// @Description patch to linear team +// @Tags plugins/linear +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scopeId path string false "team ID" +// @Param scope body models.LinearTeam true "json" +// @Success 200 {object} models.LinearTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/{scopeId} [PATCH] +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// GetScopeList get linear teams +// @Summary get linear teams +// @Description get linear teams +// @Tags plugins/linear +// @Param connectionId path int false "connection ID" +// @Param searchTerm query string false "search term for scope name" +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/ [GET] +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// GetScope get one linear team +// @Summary get one linear team +// @Description get one linear team +// @Tags plugins/linear +// @Param connectionId path int false "connection ID" +// @Param scopeId path string false "team ID" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/{scopeId} [GET] +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// DeleteScope delete plugin data associated with the scope and optionally the scope itself +// @Summary delete plugin data associated with the scope and optionally the scope itself +// @Description delete data associated with plugin scope +// @Tags plugins/linear +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Param delete_data_only query bool false "Only delete the scope data, not the scope itself" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/{scopeId} [DELETE] +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} diff --git a/backend/plugins/linear/api/scope_config_api.go b/backend/plugins/linear/api/scope_config_api.go new file mode 100644 index 00000000000..15be82a4dff --- /dev/null +++ b/backend/plugins/linear/api/scope_config_api.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// PostScopeConfig create scope config for Linear +// @Summary create scope config for Linear +// @Description create scope config for Linear +// @Tags plugins/linear +// @Accept application/json +// @Param scopeConfig body models.LinearScopeConfig true "scope config" +// @Success 200 {object} models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs [POST] +func PostScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) +} + +// PatchScopeConfig update scope config for Linear +// @Summary update scope config for Linear +// @Description update scope config for Linear +// @Tags plugins/linear +// @Accept application/json +// @Param scopeConfigId path int true "scopeConfigId" +// @Param scopeConfig body models.LinearScopeConfig true "scope config" +// @Success 200 {object} models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs/{scopeConfigId} [PATCH] +func PatchScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) +} + +// GetScopeConfig return one scope config +// @Summary return one scope config +// @Description return one scope config +// @Tags plugins/linear +// @Param scopeConfigId path int true "scopeConfigId" +// @Success 200 {object} models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs/{scopeConfigId} [GET] +func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetDetail(input) +} + +// GetScopeConfigList return all scope configs +// @Summary return all scope configs +// @Description return all scope configs +// @Tags plugins/linear +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs [GET] +func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetAll(input) +} + +// DeleteScopeConfig delete a scope config +// @Summary delete a scope config +// @Description delete a scope config +// @Tags plugins/linear +// @Param scopeConfigId path int true "scopeConfigId" +// @Param connectionId path int true "connectionId" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs/{scopeConfigId} [DELETE] +func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Delete(input) +} diff --git a/backend/plugins/linear/impl/impl.go b/backend/plugins/linear/impl/impl.go new file mode 100644 index 00000000000..8b2508dab01 --- /dev/null +++ b/backend/plugins/linear/impl/impl.go @@ -0,0 +1,194 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.PluginSource + plugin.PluginMigration + plugin.CloseablePluginTask + plugin.DataSourcePluginBlueprintV200 +} = (*Linear)(nil) + +type Linear struct{} + +func (p Linear) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Linear) Description() string { + return "To collect and enrich data from Linear" +} + +func (p Linear) Name() string { + return "linear" +} + +func (p Linear) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/linear" +} + +func (p Linear) Connection() dal.Tabler { + return &models.LinearConnection{} +} + +func (p Linear) Scope() plugin.ToolLayerScope { + return &models.LinearTeam{} +} + +func (p Linear) ScopeConfig() dal.Tabler { + return &models.LinearScopeConfig{} +} + +func (p Linear) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Linear) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.LinearConnection{}, + &models.LinearTeam{}, + &models.LinearScopeConfig{}, + &models.LinearAccount{}, + &models.LinearIssue{}, + &models.LinearComment{}, + &models.LinearIssueLabel{}, + &models.LinearWorkflowState{}, + &models.LinearCycle{}, + &models.LinearIssueHistory{}, + } +} + +func (p Linear) SubTaskMetas() []plugin.SubTaskMeta { + // Subtasks are registered incrementally as each entity is implemented. + return []plugin.SubTaskMeta{} +} + +func (p Linear) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.LinearOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, errors.Default.Wrap(err, "could not decode Linear options") + } + if op.ConnectionId == 0 { + return nil, errors.BadInput.New("linear connectionId is invalid") + } + if op.TeamId == "" { + return nil, errors.BadInput.New("linear teamId is required") + } + + connection := &models.LinearConnection{} + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + if err := connectionHelper.FirstById(connection, op.ConnectionId); err != nil { + return nil, errors.Default.Wrap(err, "error getting connection for Linear plugin") + } + + graphqlClient, err := tasks.NewLinearGraphqlClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to create Linear GraphQL client") + } + + taskData := &tasks.LinearTaskData{ + Options: &op, + GraphqlClient: graphqlClient, + } + if op.TimeAfter != "" { + timeAfter, errConv := errors.Convert01(time.Parse(time.RFC3339, op.TimeAfter)) + if errConv != nil { + return nil, errors.BadInput.Wrap(errConv, "invalid timeAfter") + } + taskData.TimeAfter = &timeAfter + } + return taskData, nil +} + +func (p Linear) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + "GET": api.GetConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/scope-configs": { + "POST": api.PostScopeConfig, + "GET": api.GetScopeConfigList, + }, + "connections/:connectionId/scope-configs/:scopeConfigId": { + "PATCH": api.PatchScopeConfig, + "GET": api.GetScopeConfig, + "DELETE": api.DeleteScopeConfig, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + } +} + +func (p Linear) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Linear) Close(taskCtx plugin.TaskContext) errors.Error { + data, ok := taskCtx.GetData().(*tasks.LinearTaskData) + if !ok { + return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) + } + if data.GraphqlClient != nil { + data.GraphqlClient.Release() + } + return nil +} diff --git a/backend/plugins/linear/linear.go b/backend/plugins/linear/linear.go new file mode 100644 index 00000000000..2cb64a49470 --- /dev/null +++ b/backend/plugins/linear/linear.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main // must be main for plugin entry point + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/spf13/cobra" +) + +var PluginEntry impl.Linear //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "linear"} + connectionId := cmd.Flags().Uint64P("connection", "c", 0, "linear connection id") + teamId := cmd.Flags().StringP("team", "t", "", "linear team id") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + _ = cmd.MarkFlagRequired("connection") + _ = cmd.MarkFlagRequired("team") + cmd.Run = func(c *cobra.Command, args []string) { + runner.DirectRun(c, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + "teamId": *teamId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/linear/tasks/api_client.go b/backend/plugins/linear/tasks/api_client.go new file mode 100644 index 00000000000..0ca6b8da25a --- /dev/null +++ b/backend/plugins/linear/tasks/api_client.go @@ -0,0 +1,108 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + gocontext "context" + "net/http" + "net/url" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/merico-ai/graphql" +) + +// linearTransport injects the Linear personal API key into every request. +// Linear expects the key verbatim in the Authorization header (no Bearer prefix). +type linearTransport struct { + token string + base http.RoundTripper +} + +func (t *linearTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set("Authorization", t.token) + return t.base.RoundTrip(req) +} + +// graphqlQueryViewer is a tiny probe used to validate connectivity / liveness. +type graphqlQueryViewer struct { + Viewer struct { + Id graphql.String + } +} + +// defaultRateLimitPerHour is Linear's documented per-API-key request budget. +// Used when the connection does not override RateLimitPerHour. +const defaultRateLimitPerHour = 1500 + +// NewLinearGraphqlClient builds a rate-limited async GraphQL client for the +// Linear API from the given connection. +func NewLinearGraphqlClient(taskCtx plugin.TaskContext, connection *models.LinearConnection) (*helper.GraphqlAsyncClient, errors.Error) { + httpClient, err := newLinearHttpClient(connection) + if err != nil { + return nil, err + } + + endpoint := connection.Endpoint + if endpoint == "" { + endpoint = "https://api.linear.app/graphql" + } + client := graphql.NewClient(endpoint, httpClient) + + rateLimitPerHour := connection.RateLimitPerHour + if rateLimitPerHour <= 0 { + rateLimitPerHour = defaultRateLimitPerHour + } + + return helper.CreateAsyncGraphqlClient(taskCtx, client, taskCtx.GetLogger(), + func(ctx gocontext.Context, c *graphql.Client, logger log.Logger) (rateRemaining int, resetAt *time.Time, e errors.Error) { + // Linear does not expose rate-limit info in the GraphQL body (it uses + // HTTP response headers), so we probe liveness and pace against the + // configured hourly budget. The async client self-throttles from here. + var q graphqlQueryViewer + dataErrors, queryErr := errors.Convert01(c.Query(ctx, &q, nil)) + if queryErr != nil { + return 0, nil, queryErr + } + if len(dataErrors) > 0 { + return 0, nil, errors.Default.Wrap(dataErrors[0], "linear graphql viewer query failed") + } + reset := time.Now().Add(1 * time.Hour) + logger.Info("linear graphql client initialized, pacing against %d req/hour", rateLimitPerHour) + return rateLimitPerHour, &reset, nil + }) +} + +func newLinearHttpClient(connection *models.LinearConnection) (*http.Client, errors.Error) { + base := http.DefaultTransport + if proxy := connection.Proxy; proxy != "" { + pu, err := url.Parse(proxy) + if err != nil { + return nil, errors.BadInput.Wrap(err, "malformed proxy url") + } + base = &http.Transport{Proxy: http.ProxyURL(pu)} + } + return &http.Client{ + Timeout: 60 * time.Second, + Transport: &linearTransport{token: connection.Token, base: base}, + }, nil +} diff --git a/backend/plugins/linear/tasks/task_data.go b/backend/plugins/linear/tasks/task_data.go new file mode 100644 index 00000000000..cd72916d074 --- /dev/null +++ b/backend/plugins/linear/tasks/task_data.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// LinearOptions are the per-scope options passed to a pipeline task. +type LinearOptions struct { + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` + TeamId string `json:"teamId" mapstructure:"teamId,omitempty"` + ScopeConfigId uint64 `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"` + // TimeAfter limits collection to data created/updated after this time. + TimeAfter string `json:"timeAfter" mapstructure:"timeAfter,omitempty"` +} + +// LinearTaskData is the shared context handed to every Linear subtask. +type LinearTaskData struct { + Options *LinearOptions + GraphqlClient *api.GraphqlAsyncClient + TimeAfter *time.Time +} + +type LinearApiParams models.LinearApiParams From e1e65e161602b9a91ee19b5765b7a2d5d2fd39dc Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 11:52:55 +1000 Subject: [PATCH 03/28] feat(linear): collect, extract and convert users to accounts Add the users GraphQL collector (paginated), extractor to _tool_linear_accounts, and convertor to the domain crossdomain.Account table, wired as the first three subtasks. Includes an e2e dataflow test with raw fixtures and verified snapshots. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- backend/plugins/linear/e2e/account_test.go | 58 +++++++++++ .../e2e/raw_tables/_raw_linear_accounts.csv | 4 + .../snapshot_tables/_tool_linear_accounts.csv | 4 + .../linear/e2e/snapshot_tables/accounts.csv | 4 + backend/plugins/linear/impl/impl.go | 7 +- .../plugins/linear/tasks/account_collector.go | 98 +++++++++++++++++++ .../plugins/linear/tasks/account_convertor.go | 97 ++++++++++++++++++ .../plugins/linear/tasks/account_extractor.go | 74 ++++++++++++++ backend/plugins/linear/tasks/shared.go | 47 +++++++++ 9 files changed, 391 insertions(+), 2 deletions(-) create mode 100644 backend/plugins/linear/e2e/account_test.go create mode 100644 backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/accounts.csv create mode 100644 backend/plugins/linear/tasks/account_collector.go create mode 100644 backend/plugins/linear/tasks/account_convertor.go create mode 100644 backend/plugins/linear/tasks/account_extractor.go create mode 100644 backend/plugins/linear/tasks/shared.go diff --git a/backend/plugins/linear/e2e/account_test.go b/backend/plugins/linear/e2e/account_test.go new file mode 100644 index 00000000000..9cb8f3a8e15 --- /dev/null +++ b/backend/plugins/linear/e2e/account_test.go @@ -0,0 +1,58 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearAccountDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_accounts.csv", "_raw_linear_accounts") + dataflowTester.FlushTabler(&models.LinearAccount{}) + dataflowTester.Subtask(tasks.ExtractAccountsMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearAccount{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_accounts.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&crossdomain.Account{}) + dataflowTester.Subtask(tasks.ConvertAccountsMeta, taskData) + dataflowTester.VerifyTableWithOptions(crossdomain.Account{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/accounts.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv new file mode 100644 index 00000000000..785c47d21b2 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv @@ -0,0 +1,4 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""user-1"",""Name"":""alice"",""DisplayName"":""Alice Anderson"",""Email"":""alice@example.com"",""AvatarUrl"":""https://linear.app/avatars/alice.png"",""Active"":true}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""user-2"",""Name"":""bob"",""DisplayName"":""Bob Brown"",""Email"":""bob@example.com"",""AvatarUrl"":""https://linear.app/avatars/bob.png"",""Active"":true}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""user-3"",""Name"":""carol"",""DisplayName"":""Carol Clark"",""Email"":""carol@example.com"",""AvatarUrl"":""https://linear.app/avatars/carol.png"",""Active"":false}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv new file mode 100644 index 00000000000..db07f6b4cf7 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv @@ -0,0 +1,4 @@ +connection_id,id,name,display_name,email,avatar_url,active +1,user-1,alice,Alice Anderson,alice@example.com,https://linear.app/avatars/alice.png,1 +1,user-2,bob,Bob Brown,bob@example.com,https://linear.app/avatars/bob.png,1 +1,user-3,carol,Carol Clark,carol@example.com,https://linear.app/avatars/carol.png,0 diff --git a/backend/plugins/linear/e2e/snapshot_tables/accounts.csv b/backend/plugins/linear/e2e/snapshot_tables/accounts.csv new file mode 100644 index 00000000000..12c2066d191 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/accounts.csv @@ -0,0 +1,4 @@ +id,email,full_name,user_name,avatar_url,organization,created_date,status +linear:LinearAccount:1:user-1,alice@example.com,Alice Anderson,alice,https://linear.app/avatars/alice.png,,,1 +linear:LinearAccount:1:user-2,bob@example.com,Bob Brown,bob,https://linear.app/avatars/bob.png,,,1 +linear:LinearAccount:1:user-3,carol@example.com,Carol Clark,carol,https://linear.app/avatars/carol.png,,,0 diff --git a/backend/plugins/linear/impl/impl.go b/backend/plugins/linear/impl/impl.go index 8b2508dab01..2f0af31172c 100644 --- a/backend/plugins/linear/impl/impl.go +++ b/backend/plugins/linear/impl/impl.go @@ -96,8 +96,11 @@ func (p Linear) GetTablesInfo() []dal.Tabler { } func (p Linear) SubTaskMetas() []plugin.SubTaskMeta { - // Subtasks are registered incrementally as each entity is implemented. - return []plugin.SubTaskMeta{} + return []plugin.SubTaskMeta{ + tasks.CollectAccountsMeta, + tasks.ExtractAccountsMeta, + tasks.ConvertAccountsMeta, + } } func (p Linear) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { diff --git a/backend/plugins/linear/tasks/account_collector.go b/backend/plugins/linear/tasks/account_collector.go new file mode 100644 index 00000000000..88fa181f002 --- /dev/null +++ b/backend/plugins/linear/tasks/account_collector.go @@ -0,0 +1,98 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_ACCOUNTS_TABLE = "linear_accounts" + +// GraphqlQueryAccountWrapper is the paginated `users` query envelope. +type GraphqlQueryAccountWrapper struct { + Users struct { + Nodes []GraphqlQueryAccount + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"users(first: $pageSize, after: $skipCursor)"` +} + +type GraphqlQueryAccount struct { + Id string + Name string + DisplayName string + Email string + AvatarUrl string + Active bool +} + +var CollectAccountsMeta = plugin.SubTaskMeta{ + Name: "Collect Users", + EntryPoint: CollectAccounts, + EnabledByDefault: true, + Description: "Collect workspace users from the Linear GraphQL API", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +var _ plugin.SubTaskEntryPoint = CollectAccounts + +func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ACCOUNTS_TABLE, + }, + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryAccountWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryAccountWrapper) + return query.Users.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryAccountWrapper) + for _, account := range query.Users.Nodes { + messages = append(messages, errors.Must1(json.Marshal(account))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/account_convertor.go b/backend/plugins/linear/tasks/account_convertor.go new file mode 100644 index 00000000000..d3c8efd3686 --- /dev/null +++ b/backend/plugins/linear/tasks/account_convertor.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertAccountsMeta = plugin.SubTaskMeta{ + Name: "Convert Users", + EntryPoint: ConvertAccounts, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_accounts into domain layer table accounts", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + DependencyTables: []string{models.LinearAccount{}.TableName()}, + ProductTables: []string{crossdomain.Account{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertAccounts + +func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + + cursor, err := db.Cursor( + dal.From(&models.LinearAccount{}), + dal.Where("connection_id = ?", data.Options.ConnectionId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ACCOUNTS_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearAccount{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + account := inputRow.(*models.LinearAccount) + status := 1 + if !account.Active { + status = 0 + } + fullName := account.Name + if account.DisplayName != "" { + fullName = account.DisplayName + } + domainAccount := &crossdomain.Account{ + DomainEntity: domainlayer.DomainEntity{ + Id: accountIdGen.Generate(data.Options.ConnectionId, account.Id), + }, + UserName: account.Name, + FullName: fullName, + Email: account.Email, + AvatarUrl: account.AvatarUrl, + Status: status, + } + return []interface{}{domainAccount}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/account_extractor.go b/backend/plugins/linear/tasks/account_extractor.go new file mode 100644 index 00000000000..19cc86a04cc --- /dev/null +++ b/backend/plugins/linear/tasks/account_extractor.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractAccountsMeta = plugin.SubTaskMeta{ + Name: "Extract Users", + EntryPoint: ExtractAccounts, + EnabledByDefault: true, + Description: "Extract raw user data into tool layer table _tool_linear_accounts", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +var _ plugin.SubTaskEntryPoint = ExtractAccounts + +func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ACCOUNTS_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiAccount := &GraphqlQueryAccount{} + if err := errors.Convert(json.Unmarshal(row.Data, apiAccount)); err != nil { + return nil, err + } + if apiAccount.Id == "" { + return nil, nil + } + account := &models.LinearAccount{ + ConnectionId: data.Options.ConnectionId, + Id: apiAccount.Id, + Name: apiAccount.Name, + DisplayName: apiAccount.DisplayName, + Email: apiAccount.Email, + AvatarUrl: apiAccount.AvatarUrl, + Active: apiAccount.Active, + } + return []interface{}{account}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/shared.go b/backend/plugins/linear/tasks/shared.go new file mode 100644 index 00000000000..9ba5a0ec1d4 --- /dev/null +++ b/backend/plugins/linear/tasks/shared.go @@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +// GraphqlInlineAccount is the shared shape used to collect a Linear user that +// is referenced inline on another entity (issue creator/assignee, comment +// author, history actor). +type GraphqlInlineAccount struct { + Id string + Name string + DisplayName string + Email string + AvatarUrl string +} + +// priorityLabels maps Linear's integer priority to its human-readable label. +// Linear: 0 = No priority, 1 = Urgent, 2 = High, 3 = Medium, 4 = Low. +var priorityLabels = map[int]string{ + 0: "No priority", + 1: "Urgent", + 2: "High", + 3: "Medium", + 4: "Low", +} + +// PriorityLabel returns the human-readable label for a Linear priority value. +func PriorityLabel(priority int) string { + if label, ok := priorityLabels[priority]; ok { + return label + } + return "No priority" +} From a076472bdfa7e8dcb4280f01abc3965346979db4 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 11:55:49 +1000 Subject: [PATCH 04/28] feat(linear): collect and extract workflow states Add the team-scoped workflow states GraphQL collector and extractor into _tool_linear_workflow_states. These states (backlog/unstarted/started/ completed/canceled) drive deterministic issue status mapping. Includes an e2e test covering all five state types. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- .../_raw_linear_workflow_states.csv | 6 ++ .../_tool_linear_workflow_states.csv | 6 ++ .../plugins/linear/e2e/workflow_state_test.go | 48 +++++++++ backend/plugins/linear/impl/impl.go | 2 + .../linear/tasks/workflow_state_collector.go | 100 ++++++++++++++++++ .../linear/tasks/workflow_state_extractor.go | 71 +++++++++++++ 6 files changed, 233 insertions(+) create mode 100644 backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv create mode 100644 backend/plugins/linear/e2e/workflow_state_test.go create mode 100644 backend/plugins/linear/tasks/workflow_state_collector.go create mode 100644 backend/plugins/linear/tasks/workflow_state_extractor.go diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv new file mode 100644 index 00000000000..72c0ddc877b --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv @@ -0,0 +1,6 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog"",""Color"":""#bec2c8"",""Position"":0}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted"",""Color"":""#e2e2e2"",""Position"":1}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started"",""Color"":""#f2c94c"",""Position"":2}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +4,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed"",""Color"":""#5e6ad2"",""Position"":3}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +5,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-canceled"",""Name"":""Canceled"",""Type"":""canceled"",""Color"":""#95a2b3"",""Position"":4}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv new file mode 100644 index 00000000000..bd1d3443ae9 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv @@ -0,0 +1,6 @@ +connection_id,id,team_id,name,type,color,position +1,state-backlog,team-1,Backlog,backlog,#bec2c8,0 +1,state-canceled,team-1,Canceled,canceled,#95a2b3,4 +1,state-done,team-1,Done,completed,#5e6ad2,3 +1,state-inprogress,team-1,In Progress,started,#f2c94c,2 +1,state-todo,team-1,Todo,unstarted,#e2e2e2,1 diff --git a/backend/plugins/linear/e2e/workflow_state_test.go b/backend/plugins/linear/e2e/workflow_state_test.go new file mode 100644 index 00000000000..5af5c4a66b9 --- /dev/null +++ b/backend/plugins/linear/e2e/workflow_state_test.go @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearWorkflowStateDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_workflow_states.csv", "_raw_linear_workflow_states") + dataflowTester.FlushTabler(&models.LinearWorkflowState{}) + dataflowTester.Subtask(tasks.ExtractWorkflowStatesMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearWorkflowState{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_workflow_states.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/impl/impl.go b/backend/plugins/linear/impl/impl.go index 2f0af31172c..5a87d0c09c0 100644 --- a/backend/plugins/linear/impl/impl.go +++ b/backend/plugins/linear/impl/impl.go @@ -99,6 +99,8 @@ func (p Linear) SubTaskMetas() []plugin.SubTaskMeta { return []plugin.SubTaskMeta{ tasks.CollectAccountsMeta, tasks.ExtractAccountsMeta, + tasks.CollectWorkflowStatesMeta, + tasks.ExtractWorkflowStatesMeta, tasks.ConvertAccountsMeta, } } diff --git a/backend/plugins/linear/tasks/workflow_state_collector.go b/backend/plugins/linear/tasks/workflow_state_collector.go new file mode 100644 index 00000000000..94c22d1d3a5 --- /dev/null +++ b/backend/plugins/linear/tasks/workflow_state_collector.go @@ -0,0 +1,100 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_WORKFLOW_STATES_TABLE = "linear_workflow_states" + +// GraphqlQueryWorkflowStateWrapper is the team-scoped paginated `states` query. +type GraphqlQueryWorkflowStateWrapper struct { + Team struct { + States struct { + Nodes []GraphqlQueryWorkflowState + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"states(first: $pageSize, after: $skipCursor)"` + } `graphql:"team(id: $teamId)"` +} + +type GraphqlQueryWorkflowState struct { + Id string + Name string + Type string + Color string + Position float64 +} + +var CollectWorkflowStatesMeta = plugin.SubTaskMeta{ + Name: "Collect Workflow States", + EntryPoint: CollectWorkflowStates, + EnabledByDefault: true, + Description: "Collect workflow states for a Linear team", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectWorkflowStates + +func CollectWorkflowStates(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_WORKFLOW_STATES_TABLE, + }, + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryWorkflowStateWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "teamId": graphql.String(data.Options.TeamId), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryWorkflowStateWrapper) + return query.Team.States.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryWorkflowStateWrapper) + for _, state := range query.Team.States.Nodes { + messages = append(messages, errors.Must1(json.Marshal(state))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/workflow_state_extractor.go b/backend/plugins/linear/tasks/workflow_state_extractor.go new file mode 100644 index 00000000000..75a2099aeaf --- /dev/null +++ b/backend/plugins/linear/tasks/workflow_state_extractor.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractWorkflowStatesMeta = plugin.SubTaskMeta{ + Name: "Extract Workflow States", + EntryPoint: ExtractWorkflowStates, + EnabledByDefault: true, + Description: "Extract raw workflow state data into tool layer table _tool_linear_workflow_states", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractWorkflowStates + +func ExtractWorkflowStates(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_WORKFLOW_STATES_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiState := &GraphqlQueryWorkflowState{} + if err := errors.Convert(json.Unmarshal(row.Data, apiState)); err != nil { + return nil, err + } + state := &models.LinearWorkflowState{ + ConnectionId: data.Options.ConnectionId, + Id: apiState.Id, + TeamId: data.Options.TeamId, + Name: apiState.Name, + Type: apiState.Type, + Color: apiState.Color, + Position: apiState.Position, + } + return []interface{}{state}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} From 3f23db20bcce4a6d8d8b4d1e83d406d3076f4a32 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 12:13:37 +1000 Subject: [PATCH 05/28] feat(linear): collect, extract and convert issues Add the team-scoped issues GraphQL collector (incremental via updatedAt ordering, inline labels), extractor to _tool_linear_issues and _tool_linear_issue_labels, and convertor to domain ticket.Issue and ticket.BoardIssue. Status maps deterministically from Linear's WorkflowState.type (backlog/unstarted->TODO, started->IN_PROGRESS, completed/canceled->DONE); priority maps to its label; lead time falls back to resolution minus creation. Includes an e2e test spanning all state types, unassigned issues, issues without a cycle, and multi-label issues. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- backend/plugins/linear/e2e/issue_test.go | 68 +++++++++ .../e2e/raw_tables/_raw_linear_issues.csv | 6 + .../_tool_linear_issue_labels.csv | 4 + .../snapshot_tables/_tool_linear_issues.csv | 6 + .../e2e/snapshot_tables/board_issues.csv | 6 + .../linear/e2e/snapshot_tables/issues.csv | 6 + backend/plugins/linear/impl/impl.go | 3 + .../plugins/linear/tasks/issue_collector.go | 138 ++++++++++++++++++ .../plugins/linear/tasks/issue_convertor.go | 124 ++++++++++++++++ .../plugins/linear/tasks/issue_extractor.go | 108 ++++++++++++++ backend/plugins/linear/tasks/shared.go | 24 +++ 11 files changed, 493 insertions(+) create mode 100644 backend/plugins/linear/e2e/issue_test.go create mode 100644 backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/board_issues.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/issues.csv create mode 100644 backend/plugins/linear/tasks/issue_collector.go create mode 100644 backend/plugins/linear/tasks/issue_convertor.go create mode 100644 backend/plugins/linear/tasks/issue_extractor.go diff --git a/backend/plugins/linear/e2e/issue_test.go b/backend/plugins/linear/e2e/issue_test.go new file mode 100644 index 00000000000..23956cb3985 --- /dev/null +++ b/backend/plugins/linear/e2e/issue_test.go @@ -0,0 +1,68 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearIssueDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // verify extraction: raw -> tool layer (issues + inline labels) + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(models.LinearIssueLabel{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_issue_labels.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: tool layer -> domain layer (issues + board_issues) + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.BoardIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/board_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv new file mode 100644 index 00000000000..f49f41c73b6 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv @@ -0,0 +1,6 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-1"",""Identifier"":""ENG-1"",""Number"":1,""Title"":""Fix login bug"",""Description"":""Users cannot log in"",""Url"":""https://linear.app/eng/issue/ENG-1"",""Priority"":1,""Estimate"":3,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-03T00:00:00Z"",""StartedAt"":""2026-05-02T00:00:00Z"",""CompletedAt"":""2026-05-03T00:00:00Z"",""CanceledAt"":null,""State"":{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed""},""Assignee"":{""Id"":""user-1""},""Creator"":{""Id"":""user-2""},""Cycle"":{""Id"":""cycle-1""},""Parent"":null,""Labels"":{""Nodes"":[{""Id"":""l1"",""Name"":""Bug""},{""Id"":""l2"",""Name"":""P1""}]}}",https://api.linear.app/graphql,null,2026-05-03 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-2"",""Identifier"":""ENG-2"",""Number"":2,""Title"":""Add dark mode"",""Description"":""Theme support"",""Url"":""https://linear.app/eng/issue/ENG-2"",""Priority"":2,""Estimate"":5,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-02T00:00:00Z"",""StartedAt"":""2026-05-02T00:00:00Z"",""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""},""Assignee"":null,""Creator"":{""Id"":""user-1""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[{""Id"":""l3"",""Name"":""Feature""}]}}",https://api.linear.app/graphql,null,2026-05-02 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-3"",""Identifier"":""ENG-3"",""Number"":3,""Title"":""Investigate flakiness"",""Description"":"""",""Url"":""https://linear.app/eng/issue/ENG-3"",""Priority"":0,""Estimate"":null,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-01T00:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""Assignee"":null,""Creator"":{""Id"":""user-1""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +4,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-4"",""Identifier"":""ENG-4"",""Number"":4,""Title"":""Deprecated feature"",""Description"":""No longer needed"",""Url"":""https://linear.app/eng/issue/ENG-4"",""Priority"":3,""Estimate"":2,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-02T12:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":""2026-05-02T00:00:00Z"",""State"":{""Id"":""state-canceled"",""Name"":""Canceled"",""Type"":""canceled""},""Assignee"":{""Id"":""user-3""},""Creator"":{""Id"":""user-1""},""Cycle"":{""Id"":""cycle-1""},""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-02 12:00:00.000 +5,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-5"",""Identifier"":""ENG-5"",""Number"":5,""Title"":""Write docs"",""Description"":""User guide"",""Url"":""https://linear.app/eng/issue/ENG-5"",""Priority"":4,""Estimate"":1,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-01T06:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""},""Assignee"":null,""Creator"":{""Id"":""user-2""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-01 06:00:00.000 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv new file mode 100644 index 00000000000..20da8a076b0 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv @@ -0,0 +1,4 @@ +connection_id,issue_id,label_name +1,issue-1,Bug +1,issue-1,P1 +1,issue-2,Feature diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv new file mode 100644 index 00000000000..cf9462cb4b7 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv @@ -0,0 +1,6 @@ +connection_id,id,team_id,identifier,number,title,description,url,priority,priority_label,estimate,state_id,state_name,state_type,creator_id,assignee_id,cycle_id,parent_id,lead_time_minutes,started_at,completed_at,canceled_at +1,issue-1,team-1,ENG-1,1,Fix login bug,Users cannot log in,https://linear.app/eng/issue/ENG-1,1,Urgent,3,state-done,Done,completed,user-2,user-1,cycle-1,,,2026-05-02T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00, +1,issue-2,team-1,ENG-2,2,Add dark mode,Theme support,https://linear.app/eng/issue/ENG-2,2,High,5,state-inprogress,In Progress,started,user-1,,,,,2026-05-02T00:00:00.000+00:00,, +1,issue-3,team-1,ENG-3,3,Investigate flakiness,,https://linear.app/eng/issue/ENG-3,0,No priority,,state-backlog,Backlog,backlog,user-1,,,,,,, +1,issue-4,team-1,ENG-4,4,Deprecated feature,No longer needed,https://linear.app/eng/issue/ENG-4,3,Medium,2,state-canceled,Canceled,canceled,user-1,user-3,cycle-1,,,,,2026-05-02T00:00:00.000+00:00 +1,issue-5,team-1,ENG-5,5,Write docs,User guide,https://linear.app/eng/issue/ENG-5,4,Low,1,state-todo,Todo,unstarted,user-2,,,,,,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/board_issues.csv b/backend/plugins/linear/e2e/snapshot_tables/board_issues.csv new file mode 100644 index 00000000000..9a7d32f9477 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/board_issues.csv @@ -0,0 +1,6 @@ +board_id,issue_id +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-1 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-2 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-3 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-4 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-5 diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues.csv b/backend/plugins/linear/e2e/snapshot_tables/issues.csv new file mode 100644 index 00000000000..da87aeffc4e --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues.csv @@ -0,0 +1,6 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-1,https://linear.app/eng/issue/ENG-1,,ENG-1,Fix login bug,Users cannot log in,,REQUIREMENT,,DONE,Done,3,2026-05-03T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00,2880,,,,linear:LinearAccount:1:user-2,,linear:LinearAccount:1:user-1,,,Urgent,,,,,0,, +linear:LinearIssue:1:issue-2,https://linear.app/eng/issue/ENG-2,,ENG-2,Add dark mode,Theme support,,REQUIREMENT,,IN_PROGRESS,In Progress,5,,2026-05-01T00:00:00.000+00:00,2026-05-02T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,High,,,,,0,, +linear:LinearIssue:1:issue-3,https://linear.app/eng/issue/ENG-3,,ENG-3,Investigate flakiness,,,REQUIREMENT,,TODO,Backlog,,,2026-05-01T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,No priority,,,,,0,, +linear:LinearIssue:1:issue-4,https://linear.app/eng/issue/ENG-4,,ENG-4,Deprecated feature,No longer needed,,REQUIREMENT,,DONE,Canceled,2,2026-05-02T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-02T12:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-1,,linear:LinearAccount:1:user-3,,,Medium,,,,,0,, +linear:LinearIssue:1:issue-5,https://linear.app/eng/issue/ENG-5,,ENG-5,Write docs,User guide,,REQUIREMENT,,TODO,Todo,1,,2026-05-01T00:00:00.000+00:00,2026-05-01T06:00:00.000+00:00,,,,,linear:LinearAccount:1:user-2,,,,,Low,,,,,0,, diff --git a/backend/plugins/linear/impl/impl.go b/backend/plugins/linear/impl/impl.go index 5a87d0c09c0..6dfb9ce5c47 100644 --- a/backend/plugins/linear/impl/impl.go +++ b/backend/plugins/linear/impl/impl.go @@ -101,7 +101,10 @@ func (p Linear) SubTaskMetas() []plugin.SubTaskMeta { tasks.ExtractAccountsMeta, tasks.CollectWorkflowStatesMeta, tasks.ExtractWorkflowStatesMeta, + tasks.CollectIssuesMeta, + tasks.ExtractIssuesMeta, tasks.ConvertAccountsMeta, + tasks.ConvertIssuesMeta, } } diff --git a/backend/plugins/linear/tasks/issue_collector.go b/backend/plugins/linear/tasks/issue_collector.go new file mode 100644 index 00000000000..bacc3df93bb --- /dev/null +++ b/backend/plugins/linear/tasks/issue_collector.go @@ -0,0 +1,138 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_ISSUES_TABLE = "linear_issues" + +// GraphqlQueryIssueWrapper is the team-scoped, paginated `issues` query. +// Issues are ordered by updatedAt (descending) so incremental runs can stop +// once they reach data older than the previous collection. +type GraphqlQueryIssueWrapper struct { + Team struct { + Issues struct { + Nodes []GraphqlQueryIssue `graphql:"nodes"` + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"issues(first: $pageSize, after: $skipCursor, orderBy: updatedAt)"` + } `graphql:"team(id: $teamId)"` +} + +type GraphqlQueryIssue struct { + Id string + Identifier string + Number int + Title string + Description string + Url string + Priority int + Estimate *float64 + CreatedAt time.Time + UpdatedAt time.Time + StartedAt *time.Time + CompletedAt *time.Time + CanceledAt *time.Time + State *struct { + Id string + Name string + Type string + } + Assignee *struct{ Id string } + Creator *struct{ Id string } + Cycle *struct{ Id string } + Parent *struct{ Id string } + Labels struct { + Nodes []struct { + Id string + Name string + } + } `graphql:"labels(first: 50)"` +} + +var CollectIssuesMeta = plugin.SubTaskMeta{ + Name: "Collect Issues", + EntryPoint: CollectIssues, + EnabledByDefault: true, + Description: "Collect issues for a Linear team, supports incremental collection", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectIssues + +func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }) + if err != nil { + return err + } + + since := apiCollector.GetSince() + err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ + GraphqlClient: data.GraphqlClient, + PageSize: 50, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryIssueWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "teamId": graphql.String(data.Options.TeamId), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryIssueWrapper) + return query.Team.Issues.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryIssueWrapper) + for _, issue := range query.Team.Issues.Nodes { + issue.CompletedAt = utils.NilIfZeroTime(issue.CompletedAt) + issue.CanceledAt = utils.NilIfZeroTime(issue.CanceledAt) + issue.StartedAt = utils.NilIfZeroTime(issue.StartedAt) + if since != nil && since.After(issue.UpdatedAt) { + return messages, helper.ErrFinishCollect + } + messages = append(messages, errors.Must1(json.Marshal(issue))) + } + return + }, + }) + if err != nil { + return err + } + return apiCollector.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_convertor.go b/backend/plugins/linear/tasks/issue_convertor.go new file mode 100644 index 00000000000..06e18272614 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_convertor.go @@ -0,0 +1,124 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertIssuesMeta = plugin.SubTaskMeta{ + Name: "Convert Issues", + EntryPoint: ConvertIssues, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_issues into domain layer tables issues and board_issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssue{}.TableName(), RAW_ISSUES_TABLE}, + ProductTables: []string{ticket.Issue{}.TableName(), ticket.BoardIssue{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertIssues + +func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + boardId := boardIdGen.Generate(connectionId, data.Options.TeamId) + + cursor, err := db.Cursor( + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssue{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + issue := inputRow.(*models.LinearIssue) + domainIssue := &ticket.Issue{ + DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(connectionId, issue.Id)}, + IssueKey: issue.Identifier, + Title: issue.Title, + Description: issue.Description, + Url: issue.Url, + Type: ticket.REQUIREMENT, + Status: StatusFromStateType(issue.StateType), + OriginalStatus: issue.StateName, + StoryPoint: issue.Estimate, + Priority: issue.PriorityLabel, + LeadTimeMinutes: issue.LeadTimeMinutes, + CreatedDate: &issue.CreatedAt, + UpdatedDate: &issue.UpdatedAt, + } + if issue.CreatorId != "" { + domainIssue.CreatorId = accountIdGen.Generate(connectionId, issue.CreatorId) + } + if issue.AssigneeId != "" { + domainIssue.AssigneeId = accountIdGen.Generate(connectionId, issue.AssigneeId) + } + if issue.ParentId != "" { + domainIssue.ParentIssueId = issueIdGen.Generate(connectionId, issue.ParentId) + domainIssue.IsSubtask = true + } + // Resolution date: completedAt, falling back to canceledAt. + if issue.CompletedAt != nil { + domainIssue.ResolutionDate = issue.CompletedAt + } else if issue.CanceledAt != nil { + domainIssue.ResolutionDate = issue.CanceledAt + } + // Fallback lead time when no history-derived value is present. + if domainIssue.LeadTimeMinutes == nil && domainIssue.ResolutionDate != nil { + minutes := uint(domainIssue.ResolutionDate.Sub(issue.CreatedAt).Minutes()) + domainIssue.LeadTimeMinutes = &minutes + } + boardIssue := &ticket.BoardIssue{ + BoardId: boardId, + IssueId: domainIssue.Id, + } + return []interface{}{domainIssue, boardIssue}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_extractor.go b/backend/plugins/linear/tasks/issue_extractor.go new file mode 100644 index 00000000000..e423076612d --- /dev/null +++ b/backend/plugins/linear/tasks/issue_extractor.go @@ -0,0 +1,108 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractIssuesMeta = plugin.SubTaskMeta{ + Name: "Extract Issues", + EntryPoint: ExtractIssues, + EnabledByDefault: true, + Description: "Extract raw issue data into tool layer tables _tool_linear_issues and _tool_linear_issue_labels", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractIssues + +func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiIssue := &GraphqlQueryIssue{} + if err := errors.Convert(json.Unmarshal(row.Data, apiIssue)); err != nil { + return nil, err + } + connectionId := data.Options.ConnectionId + issue := &models.LinearIssue{ + ConnectionId: connectionId, + Id: apiIssue.Id, + TeamId: data.Options.TeamId, + Identifier: apiIssue.Identifier, + Number: apiIssue.Number, + Title: apiIssue.Title, + Description: apiIssue.Description, + Url: apiIssue.Url, + Priority: apiIssue.Priority, + PriorityLabel: PriorityLabel(apiIssue.Priority), + Estimate: apiIssue.Estimate, + CreatedAt: apiIssue.CreatedAt, + UpdatedAt: apiIssue.UpdatedAt, + StartedAt: apiIssue.StartedAt, + CompletedAt: apiIssue.CompletedAt, + CanceledAt: apiIssue.CanceledAt, + } + if apiIssue.State != nil { + issue.StateId = apiIssue.State.Id + issue.StateName = apiIssue.State.Name + issue.StateType = apiIssue.State.Type + } + if apiIssue.Assignee != nil { + issue.AssigneeId = apiIssue.Assignee.Id + } + if apiIssue.Creator != nil { + issue.CreatorId = apiIssue.Creator.Id + } + if apiIssue.Cycle != nil { + issue.CycleId = apiIssue.Cycle.Id + } + if apiIssue.Parent != nil { + issue.ParentId = apiIssue.Parent.Id + } + + results := make([]interface{}, 0, len(apiIssue.Labels.Nodes)+1) + results = append(results, issue) + for _, label := range apiIssue.Labels.Nodes { + results = append(results, &models.LinearIssueLabel{ + ConnectionId: connectionId, + IssueId: apiIssue.Id, + LabelName: label.Name, + }) + } + return results, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/shared.go b/backend/plugins/linear/tasks/shared.go index 9ba5a0ec1d4..eedeff90e8f 100644 --- a/backend/plugins/linear/tasks/shared.go +++ b/backend/plugins/linear/tasks/shared.go @@ -17,6 +17,10 @@ limitations under the License. package tasks +import ( + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" +) + // GraphqlInlineAccount is the shared shape used to collect a Linear user that // is referenced inline on another entity (issue creator/assignee, comment // author, history actor). @@ -45,3 +49,23 @@ func PriorityLabel(priority int) string { } return "No priority" } + +// StatusFromStateType maps a Linear WorkflowState.type to a DevLake standard +// issue status. Linear's state types are standardized, so no user-supplied +// mapping is required: +// +// backlog, unstarted -> TODO +// started -> IN_PROGRESS +// completed, canceled -> DONE +func StatusFromStateType(stateType string) string { + switch stateType { + case "backlog", "unstarted": + return ticket.TODO + case "started": + return ticket.IN_PROGRESS + case "completed", "canceled": + return ticket.DONE + default: + return ticket.OTHER + } +} From ce707df6076a15d4ad31581c84b965bbe7b867d5 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 12:25:32 +1000 Subject: [PATCH 06/28] feat(linear): collect, extract and convert issue comments Add a per-issue comments GraphQL collector (driven by an input iterator over collected issues, with pagination), an extractor that recovers the owning issue id from the raw input column, and a convertor to domain ticket.IssueComment. Includes an e2e dataflow test. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- backend/plugins/linear/e2e/comment_test.go | 63 +++++++++ .../e2e/raw_tables/_raw_linear_comments.csv | 4 + .../snapshot_tables/_tool_linear_comments.csv | 4 + .../e2e/snapshot_tables/issue_comments.csv | 4 + backend/plugins/linear/impl/impl.go | 3 + .../plugins/linear/tasks/comment_collector.go | 131 ++++++++++++++++++ .../plugins/linear/tasks/comment_convertor.go | 97 +++++++++++++ .../plugins/linear/tasks/comment_extractor.go | 78 +++++++++++ 8 files changed, 384 insertions(+) create mode 100644 backend/plugins/linear/e2e/comment_test.go create mode 100644 backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv create mode 100644 backend/plugins/linear/tasks/comment_collector.go create mode 100644 backend/plugins/linear/tasks/comment_convertor.go create mode 100644 backend/plugins/linear/tasks/comment_extractor.go diff --git a/backend/plugins/linear/e2e/comment_test.go b/backend/plugins/linear/e2e/comment_test.go new file mode 100644 index 00000000000..f8178401f13 --- /dev/null +++ b/backend/plugins/linear/e2e/comment_test.go @@ -0,0 +1,63 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearCommentDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // the comment convertor joins to issues, so populate _tool_linear_issues first + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_comments.csv", "_raw_linear_comments") + dataflowTester.FlushTabler(&models.LinearComment{}) + dataflowTester.Subtask(tasks.ExtractCommentsMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearComment{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_comments.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.IssueComment{}) + dataflowTester.Subtask(tasks.ConvertCommentsMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.IssueComment{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_comments.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv new file mode 100644 index 00000000000..bb8552054c7 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv @@ -0,0 +1,4 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-1"",""Body"":""Looking into this"",""CreatedAt"":""2026-05-02T10:00:00Z"",""UpdatedAt"":""2026-05-02T10:00:00Z"",""User"":{""Id"":""user-2""}}",https://api.linear.app/graphql,"{""Id"":""issue-1""}",2026-05-02 10:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-2"",""Body"":""Fixed in PR 42"",""CreatedAt"":""2026-05-03T09:00:00Z"",""UpdatedAt"":""2026-05-03T09:30:00Z"",""User"":{""Id"":""user-1""}}",https://api.linear.app/graphql,"{""Id"":""issue-1""}",2026-05-03 09:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-3"",""Body"":""Any update?"",""CreatedAt"":""2026-05-02T11:00:00Z"",""UpdatedAt"":""2026-05-02T11:00:00Z"",""User"":{""Id"":""user-1""}}",https://api.linear.app/graphql,"{""Id"":""issue-2""}",2026-05-02 11:00:00.000 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv new file mode 100644 index 00000000000..aec4878ecb1 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv @@ -0,0 +1,4 @@ +connection_id,id,issue_id,body,author_id +1,comment-1,issue-1,Looking into this,user-2 +1,comment-2,issue-1,Fixed in PR 42,user-1 +1,comment-3,issue-2,Any update?,user-1 diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv new file mode 100644 index 00000000000..925e84e5648 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv @@ -0,0 +1,4 @@ +id,issue_id,body,account_id,created_date,updated_date +linear:LinearComment:1:comment-1,linear:LinearIssue:1:issue-1,Looking into this,linear:LinearAccount:1:user-2,2026-05-02T10:00:00.000+00:00,2026-05-02T10:00:00.000+00:00 +linear:LinearComment:1:comment-2,linear:LinearIssue:1:issue-1,Fixed in PR 42,linear:LinearAccount:1:user-1,2026-05-03T09:00:00.000+00:00,2026-05-03T09:30:00.000+00:00 +linear:LinearComment:1:comment-3,linear:LinearIssue:1:issue-2,Any update?,linear:LinearAccount:1:user-1,2026-05-02T11:00:00.000+00:00,2026-05-02T11:00:00.000+00:00 diff --git a/backend/plugins/linear/impl/impl.go b/backend/plugins/linear/impl/impl.go index 6dfb9ce5c47..2d577f31895 100644 --- a/backend/plugins/linear/impl/impl.go +++ b/backend/plugins/linear/impl/impl.go @@ -103,8 +103,11 @@ func (p Linear) SubTaskMetas() []plugin.SubTaskMeta { tasks.ExtractWorkflowStatesMeta, tasks.CollectIssuesMeta, tasks.ExtractIssuesMeta, + tasks.CollectCommentsMeta, + tasks.ExtractCommentsMeta, tasks.ConvertAccountsMeta, tasks.ConvertIssuesMeta, + tasks.ConvertCommentsMeta, } } diff --git a/backend/plugins/linear/tasks/comment_collector.go b/backend/plugins/linear/tasks/comment_collector.go new file mode 100644 index 00000000000..b7487ba5afa --- /dev/null +++ b/backend/plugins/linear/tasks/comment_collector.go @@ -0,0 +1,131 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "reflect" + + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/merico-ai/graphql" +) + +const RAW_COMMENTS_TABLE = "linear_comments" + +// SimpleLinearIssue is the iterator element used to drive per-issue collection +// of child resources (comments, history). Its JSON form is stored in the raw +// row's `input` column so extractors can recover the owning issue id. +type SimpleLinearIssue struct { + Id string +} + +// GraphqlQueryCommentWrapper is the per-issue, paginated `comments` query. +type GraphqlQueryCommentWrapper struct { + Issue struct { + Comments struct { + Nodes []GraphqlQueryComment + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"comments(first: $pageSize, after: $skipCursor)"` + } `graphql:"issue(id: $issueId)"` +} + +type GraphqlQueryComment struct { + Id string + Body string + CreatedAt time.Time + UpdatedAt time.Time + User *struct{ Id string } +} + +var CollectCommentsMeta = plugin.SubTaskMeta{ + Name: "Collect Comments", + EntryPoint: CollectComments, + EnabledByDefault: true, + Description: "Collect comments for each collected Linear issue", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ExtractIssuesMeta}, +} + +var _ plugin.SubTaskEntryPoint = CollectComments + +func CollectComments(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + + cursor, err := db.Cursor( + dal.Select("id"), + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ?", data.Options.ConnectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleLinearIssue{})) + if err != nil { + return err + } + + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_COMMENTS_TABLE, + }, + GraphqlClient: data.GraphqlClient, + Input: iterator, + InputStep: 1, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryCommentWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + issue := reqData.Input.(*SimpleLinearIssue) + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "issueId": graphql.String(issue.Id), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryCommentWrapper) + return query.Issue.Comments.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryCommentWrapper) + for _, comment := range query.Issue.Comments.Nodes { + messages = append(messages, errors.Must1(json.Marshal(comment))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/comment_convertor.go b/backend/plugins/linear/tasks/comment_convertor.go new file mode 100644 index 00000000000..8c4729f0e5c --- /dev/null +++ b/backend/plugins/linear/tasks/comment_convertor.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertCommentsMeta = plugin.SubTaskMeta{ + Name: "Convert Comments", + EntryPoint: ConvertComments, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_comments into domain layer table issue_comments", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearComment{}.TableName(), RAW_COMMENTS_TABLE}, + ProductTables: []string{ticket.IssueComment{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertComments + +func ConvertComments(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + commentIdGen := didgen.NewDomainIdGenerator(&models.LinearComment{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + + cursor, err := db.Cursor( + dal.Select("c.*"), + dal.From("_tool_linear_comments c"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = c.connection_id AND i.id = c.issue_id)"), + dal.Where("c.connection_id = ? AND i.team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_COMMENTS_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearComment{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + comment := inputRow.(*models.LinearComment) + domainComment := &ticket.IssueComment{ + DomainEntity: domainlayer.DomainEntity{Id: commentIdGen.Generate(connectionId, comment.Id)}, + IssueId: issueIdGen.Generate(connectionId, comment.IssueId), + Body: comment.Body, + CreatedDate: comment.CreatedAt, + } + if comment.AuthorId != "" { + domainComment.AccountId = accountIdGen.Generate(connectionId, comment.AuthorId) + } + if !comment.UpdatedAt.IsZero() { + domainComment.UpdatedDate = &comment.UpdatedAt + } + return []interface{}{domainComment}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/comment_extractor.go b/backend/plugins/linear/tasks/comment_extractor.go new file mode 100644 index 00000000000..de0d6a9ae98 --- /dev/null +++ b/backend/plugins/linear/tasks/comment_extractor.go @@ -0,0 +1,78 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractCommentsMeta = plugin.SubTaskMeta{ + Name: "Extract Comments", + EntryPoint: ExtractComments, + EnabledByDefault: true, + Description: "Extract raw comment data into tool layer table _tool_linear_comments", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractComments + +func ExtractComments(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_COMMENTS_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiComment := &GraphqlQueryComment{} + if err := errors.Convert(json.Unmarshal(row.Data, apiComment)); err != nil { + return nil, err + } + // The owning issue id is carried in the raw row's input column. + issueRef := &SimpleLinearIssue{} + if err := errors.Convert(json.Unmarshal(row.Input, issueRef)); err != nil { + return nil, err + } + comment := &models.LinearComment{ + ConnectionId: data.Options.ConnectionId, + Id: apiComment.Id, + IssueId: issueRef.Id, + Body: apiComment.Body, + CreatedAt: apiComment.CreatedAt, + UpdatedAt: apiComment.UpdatedAt, + } + if apiComment.User != nil { + comment.AuthorId = apiComment.User.Id + } + return []interface{}{comment}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} From ea8d79f4f10c50756f037fbec39f645912ffb09d Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 12:28:16 +1000 Subject: [PATCH 07/28] feat(linear): convert issue labels to domain layer Add the convertor from _tool_linear_issue_labels (populated inline by the issue extractor) into the domain ticket.IssueLabel table. Includes an e2e test covering issues with multiple labels and with none. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- backend/plugins/linear/e2e/label_test.go | 55 ++++++++++++ .../e2e/snapshot_tables/issue_labels.csv | 4 + backend/plugins/linear/impl/impl.go | 1 + .../plugins/linear/tasks/label_convertor.go | 85 +++++++++++++++++++ 4 files changed, 145 insertions(+) create mode 100644 backend/plugins/linear/e2e/label_test.go create mode 100644 backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv create mode 100644 backend/plugins/linear/tasks/label_convertor.go diff --git a/backend/plugins/linear/e2e/label_test.go b/backend/plugins/linear/e2e/label_test.go new file mode 100644 index 00000000000..4861f6197da --- /dev/null +++ b/backend/plugins/linear/e2e/label_test.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearLabelDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // labels are produced inline by the issue extractor + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.IssueLabel{}) + dataflowTester.Subtask(tasks.ConvertIssueLabelsMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.IssueLabel{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_labels.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv new file mode 100644 index 00000000000..e7c911e0005 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv @@ -0,0 +1,4 @@ +issue_id,label_name +linear:LinearIssue:1:issue-1,Bug +linear:LinearIssue:1:issue-1,P1 +linear:LinearIssue:1:issue-2,Feature diff --git a/backend/plugins/linear/impl/impl.go b/backend/plugins/linear/impl/impl.go index 2d577f31895..f09efc20cfc 100644 --- a/backend/plugins/linear/impl/impl.go +++ b/backend/plugins/linear/impl/impl.go @@ -107,6 +107,7 @@ func (p Linear) SubTaskMetas() []plugin.SubTaskMeta { tasks.ExtractCommentsMeta, tasks.ConvertAccountsMeta, tasks.ConvertIssuesMeta, + tasks.ConvertIssueLabelsMeta, tasks.ConvertCommentsMeta, } } diff --git a/backend/plugins/linear/tasks/label_convertor.go b/backend/plugins/linear/tasks/label_convertor.go new file mode 100644 index 00000000000..e94826d4775 --- /dev/null +++ b/backend/plugins/linear/tasks/label_convertor.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertIssueLabelsMeta = plugin.SubTaskMeta{ + Name: "Convert Issue Labels", + EntryPoint: ConvertIssueLabels, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_issue_labels into domain layer table issue_labels", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssueLabel{}.TableName(), models.LinearIssue{}.TableName(), RAW_ISSUES_TABLE}, + ProductTables: []string{ticket.IssueLabel{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertIssueLabels + +func ConvertIssueLabels(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + + cursor, err := db.Cursor( + dal.Select("l.*"), + dal.From("_tool_linear_issue_labels l"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = l.connection_id AND i.id = l.issue_id)"), + dal.Where("l.connection_id = ? AND i.team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssueLabel{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + label := inputRow.(*models.LinearIssueLabel) + domainLabel := &ticket.IssueLabel{ + IssueId: issueIdGen.Generate(connectionId, label.IssueId), + LabelName: label.LabelName, + } + return []interface{}{domainLabel}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} From 4789ba0644dafa7968ded57d7b8f4682342d6a86 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 12:32:56 +1000 Subject: [PATCH 08/28] feat(linear): collect cycles and convert to sprints Add the team-scoped cycles GraphQL collector and extractor, plus convertors producing domain ticket.Sprint and ticket.BoardSprint (status derived from completedAt), and ticket.SprintIssue linking issues to their cycle. Includes an e2e dataflow test covering closed/active cycles and issues with/without a cycle. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- backend/plugins/linear/e2e/cycle_test.go | 76 +++++++++++++ .../e2e/raw_tables/_raw_linear_cycles.csv | 3 + .../snapshot_tables/_tool_linear_cycles.csv | 3 + .../e2e/snapshot_tables/board_sprints.csv | 3 + .../e2e/snapshot_tables/sprint_issues.csv | 3 + .../linear/e2e/snapshot_tables/sprints.csv | 3 + backend/plugins/linear/impl/impl.go | 4 + .../plugins/linear/tasks/cycle_collector.go | 102 +++++++++++++++++ .../plugins/linear/tasks/cycle_convertor.go | 106 ++++++++++++++++++ .../plugins/linear/tasks/cycle_extractor.go | 72 ++++++++++++ .../linear/tasks/sprint_issue_convertor.go | 85 ++++++++++++++ 11 files changed, 460 insertions(+) create mode 100644 backend/plugins/linear/e2e/cycle_test.go create mode 100644 backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/sprints.csv create mode 100644 backend/plugins/linear/tasks/cycle_collector.go create mode 100644 backend/plugins/linear/tasks/cycle_convertor.go create mode 100644 backend/plugins/linear/tasks/cycle_extractor.go create mode 100644 backend/plugins/linear/tasks/sprint_issue_convertor.go diff --git a/backend/plugins/linear/e2e/cycle_test.go b/backend/plugins/linear/e2e/cycle_test.go new file mode 100644 index 00000000000..0a50789fa7d --- /dev/null +++ b/backend/plugins/linear/e2e/cycle_test.go @@ -0,0 +1,76 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearCycleDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // issues drive sprint_issues, so populate _tool_linear_issues first + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_cycles.csv", "_raw_linear_cycles") + dataflowTester.FlushTabler(&models.LinearCycle{}) + dataflowTester.Subtask(tasks.ExtractCyclesMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearCycle{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_cycles.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: cycles -> sprints + board_sprints + dataflowTester.FlushTabler(&ticket.Sprint{}) + dataflowTester.FlushTabler(&ticket.BoardSprint{}) + dataflowTester.Subtask(tasks.ConvertCyclesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Sprint{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/sprints.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.BoardSprint{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/board_sprints.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: issues -> sprint_issues + dataflowTester.FlushTabler(&ticket.SprintIssue{}) + dataflowTester.Subtask(tasks.ConvertSprintIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.SprintIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/sprint_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv new file mode 100644 index 00000000000..8eb59ddb145 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv @@ -0,0 +1,3 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""cycle-1"",""Number"":1,""Name"":"""",""StartsAt"":""2026-04-20T00:00:00Z"",""EndsAt"":""2026-05-04T00:00:00Z"",""CompletedAt"":""2026-05-04T00:00:00Z""}",https://api.linear.app/graphql,null,2026-05-04 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""cycle-2"",""Number"":2,""Name"":""Sprint 2"",""StartsAt"":""2026-05-04T00:00:00Z"",""EndsAt"":""2026-05-18T00:00:00Z"",""CompletedAt"":null}",https://api.linear.app/graphql,null,2026-05-04 00:00:00.000 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv new file mode 100644 index 00000000000..9e95c44d986 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv @@ -0,0 +1,3 @@ +connection_id,id,team_id,number,name,starts_at,ends_at,completed_at +1,cycle-1,team-1,1,,2026-04-20T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00 +1,cycle-2,team-1,2,Sprint 2,2026-05-04T00:00:00.000+00:00,2026-05-18T00:00:00.000+00:00, diff --git a/backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv b/backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv new file mode 100644 index 00000000000..ff0cfe33dc5 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv @@ -0,0 +1,3 @@ +board_id,sprint_id +linear:LinearTeam:1:team-1,linear:LinearCycle:1:cycle-1 +linear:LinearTeam:1:team-1,linear:LinearCycle:1:cycle-2 diff --git a/backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv new file mode 100644 index 00000000000..b13b4f85210 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv @@ -0,0 +1,3 @@ +sprint_id,issue_id +linear:LinearCycle:1:cycle-1,linear:LinearIssue:1:issue-1 +linear:LinearCycle:1:cycle-1,linear:LinearIssue:1:issue-4 diff --git a/backend/plugins/linear/e2e/snapshot_tables/sprints.csv b/backend/plugins/linear/e2e/snapshot_tables/sprints.csv new file mode 100644 index 00000000000..a09ab2ffe70 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/sprints.csv @@ -0,0 +1,3 @@ +id,name,url,status,started_date,ended_date,completed_date,original_board_id +linear:LinearCycle:1:cycle-1,Cycle 1,,CLOSED,2026-04-20T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,linear:LinearTeam:1:team-1 +linear:LinearCycle:1:cycle-2,Sprint 2,,ACTIVE,2026-05-04T00:00:00.000+00:00,2026-05-18T00:00:00.000+00:00,,linear:LinearTeam:1:team-1 diff --git a/backend/plugins/linear/impl/impl.go b/backend/plugins/linear/impl/impl.go index f09efc20cfc..3e1aecd6365 100644 --- a/backend/plugins/linear/impl/impl.go +++ b/backend/plugins/linear/impl/impl.go @@ -105,10 +105,14 @@ func (p Linear) SubTaskMetas() []plugin.SubTaskMeta { tasks.ExtractIssuesMeta, tasks.CollectCommentsMeta, tasks.ExtractCommentsMeta, + tasks.CollectCyclesMeta, + tasks.ExtractCyclesMeta, tasks.ConvertAccountsMeta, tasks.ConvertIssuesMeta, tasks.ConvertIssueLabelsMeta, tasks.ConvertCommentsMeta, + tasks.ConvertCyclesMeta, + tasks.ConvertSprintIssuesMeta, } } diff --git a/backend/plugins/linear/tasks/cycle_collector.go b/backend/plugins/linear/tasks/cycle_collector.go new file mode 100644 index 00000000000..88572650fcd --- /dev/null +++ b/backend/plugins/linear/tasks/cycle_collector.go @@ -0,0 +1,102 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_CYCLES_TABLE = "linear_cycles" + +// GraphqlQueryCycleWrapper is the team-scoped, paginated `cycles` query. +type GraphqlQueryCycleWrapper struct { + Team struct { + Cycles struct { + Nodes []GraphqlQueryCycle + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"cycles(first: $pageSize, after: $skipCursor)"` + } `graphql:"team(id: $teamId)"` +} + +type GraphqlQueryCycle struct { + Id string + Number int + Name string + StartsAt *time.Time + EndsAt *time.Time + CompletedAt *time.Time +} + +var CollectCyclesMeta = plugin.SubTaskMeta{ + Name: "Collect Cycles", + EntryPoint: CollectCycles, + EnabledByDefault: true, + Description: "Collect cycles (sprints) for a Linear team", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectCycles + +func CollectCycles(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_CYCLES_TABLE, + }, + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryCycleWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "teamId": graphql.String(data.Options.TeamId), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryCycleWrapper) + return query.Team.Cycles.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryCycleWrapper) + for _, cycle := range query.Team.Cycles.Nodes { + messages = append(messages, errors.Must1(json.Marshal(cycle))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/cycle_convertor.go b/backend/plugins/linear/tasks/cycle_convertor.go new file mode 100644 index 00000000000..ae4a0dd5a84 --- /dev/null +++ b/backend/plugins/linear/tasks/cycle_convertor.go @@ -0,0 +1,106 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertCyclesMeta = plugin.SubTaskMeta{ + Name: "Convert Cycles", + EntryPoint: ConvertCycles, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_cycles into domain layer tables sprints and board_sprints", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearCycle{}.TableName(), RAW_CYCLES_TABLE}, + ProductTables: []string{ticket.Sprint{}.TableName(), ticket.BoardSprint{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertCycles + +func ConvertCycles(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + cycleIdGen := didgen.NewDomainIdGenerator(&models.LinearCycle{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + boardId := boardIdGen.Generate(connectionId, data.Options.TeamId) + + cursor, err := db.Cursor( + dal.From(&models.LinearCycle{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_CYCLES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearCycle{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + cycle := inputRow.(*models.LinearCycle) + sprintId := cycleIdGen.Generate(connectionId, cycle.Id) + name := cycle.Name + if name == "" { + name = fmt.Sprintf("Cycle %d", cycle.Number) + } + status := "ACTIVE" + if cycle.CompletedAt != nil { + status = "CLOSED" + } + sprint := &ticket.Sprint{ + DomainEntity: domainlayer.DomainEntity{Id: sprintId}, + Name: name, + Status: status, + StartedDate: cycle.StartsAt, + EndedDate: cycle.EndsAt, + CompletedDate: cycle.CompletedAt, + OriginalBoardID: boardId, + } + boardSprint := &ticket.BoardSprint{ + BoardId: boardId, + SprintId: sprintId, + } + return []interface{}{sprint, boardSprint}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/cycle_extractor.go b/backend/plugins/linear/tasks/cycle_extractor.go new file mode 100644 index 00000000000..43a4233aa88 --- /dev/null +++ b/backend/plugins/linear/tasks/cycle_extractor.go @@ -0,0 +1,72 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractCyclesMeta = plugin.SubTaskMeta{ + Name: "Extract Cycles", + EntryPoint: ExtractCycles, + EnabledByDefault: true, + Description: "Extract raw cycle data into tool layer table _tool_linear_cycles", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractCycles + +func ExtractCycles(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_CYCLES_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiCycle := &GraphqlQueryCycle{} + if err := errors.Convert(json.Unmarshal(row.Data, apiCycle)); err != nil { + return nil, err + } + cycle := &models.LinearCycle{ + ConnectionId: data.Options.ConnectionId, + Id: apiCycle.Id, + TeamId: data.Options.TeamId, + Number: apiCycle.Number, + Name: apiCycle.Name, + StartsAt: apiCycle.StartsAt, + EndsAt: apiCycle.EndsAt, + CompletedAt: apiCycle.CompletedAt, + } + return []interface{}{cycle}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/sprint_issue_convertor.go b/backend/plugins/linear/tasks/sprint_issue_convertor.go new file mode 100644 index 00000000000..33e771ef29d --- /dev/null +++ b/backend/plugins/linear/tasks/sprint_issue_convertor.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertSprintIssuesMeta = plugin.SubTaskMeta{ + Name: "Convert Sprint Issues", + EntryPoint: ConvertSprintIssues, + EnabledByDefault: true, + Description: "Link issues to their cycle (sprint) in the domain layer table sprint_issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssue{}.TableName(), RAW_ISSUES_TABLE}, + ProductTables: []string{ticket.SprintIssue{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertSprintIssues + +func ConvertSprintIssues(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + cycleIdGen := didgen.NewDomainIdGenerator(&models.LinearCycle{}) + + cursor, err := db.Cursor( + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ? AND cycle_id != ''", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssue{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + issue := inputRow.(*models.LinearIssue) + sprintIssue := &ticket.SprintIssue{ + SprintId: cycleIdGen.Generate(connectionId, issue.CycleId), + IssueId: issueIdGen.Generate(connectionId, issue.Id), + } + return []interface{}{sprintIssue}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} From 78df69fc2266dd3e080e7c34da5cdcf0aaf3c67a Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 12:44:50 +1000 Subject: [PATCH 09/28] feat(linear): collect issue history and convert to changelogs Add a per-issue history GraphQL collector (input iterator over issues, with pagination), an extractor capturing state transitions including state types, and a convertor to domain ticket.IssueChangelogs with mapped from/to status values. Lead time is already derived from the issue's native startedAt/completedAt. Includes an e2e test of a full backlog->started->completed lifecycle. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- .../plugins/linear/e2e/issue_history_test.go | 63 ++++++++ .../raw_tables/_raw_linear_issue_history.csv | 4 + .../_tool_linear_issue_history.csv | 4 + .../e2e/snapshot_tables/issue_changelogs.csv | 4 + backend/plugins/linear/impl/impl.go | 3 + .../plugins/linear/models/issue_history.go | 2 + .../migrationscripts/archived/models.go | 2 + .../linear/tasks/issue_history_collector.go | 135 ++++++++++++++++++ .../linear/tasks/issue_history_convertor.go | 103 +++++++++++++ .../linear/tasks/issue_history_extractor.go | 85 +++++++++++ 10 files changed, 405 insertions(+) create mode 100644 backend/plugins/linear/e2e/issue_history_test.go create mode 100644 backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv create mode 100644 backend/plugins/linear/tasks/issue_history_collector.go create mode 100644 backend/plugins/linear/tasks/issue_history_convertor.go create mode 100644 backend/plugins/linear/tasks/issue_history_extractor.go diff --git a/backend/plugins/linear/e2e/issue_history_test.go b/backend/plugins/linear/e2e/issue_history_test.go new file mode 100644 index 00000000000..c8942141b2b --- /dev/null +++ b/backend/plugins/linear/e2e/issue_history_test.go @@ -0,0 +1,63 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearIssueHistoryDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // the history convertor joins to issues, so populate _tool_linear_issues first + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issue_history.csv", "_raw_linear_issue_history") + dataflowTester.FlushTabler(&models.LinearIssueHistory{}) + dataflowTester.Subtask(tasks.ExtractIssueHistoryMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearIssueHistory{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_issue_history.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.IssueChangelogs{}) + dataflowTester.Subtask(tasks.ConvertIssueHistoryMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.IssueChangelogs{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_changelogs.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv new file mode 100644 index 00000000000..6862f10e67a --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv @@ -0,0 +1,4 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-1"",""CreatedAt"":""2026-05-01T08:00:00Z"",""Actor"":{""Id"":""user-2""},""FromState"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""ToState"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""}}",https://api.linear.app/graphql,"{""Id"":""issue-1""}",2026-05-01 08:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-2"",""CreatedAt"":""2026-05-02T00:00:00Z"",""Actor"":{""Id"":""user-1""},""FromState"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""},""ToState"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""}}",https://api.linear.app/graphql,"{""Id"":""issue-1""}",2026-05-02 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-3"",""CreatedAt"":""2026-05-03T00:00:00Z"",""Actor"":{""Id"":""user-1""},""FromState"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""},""ToState"":{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed""}}",https://api.linear.app/graphql,"{""Id"":""issue-1""}",2026-05-03 00:00:00.000 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv new file mode 100644 index 00000000000..bc8fe9ec7ba --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv @@ -0,0 +1,4 @@ +connection_id,id,issue_id,actor_id,from_state_id,from_state_name,from_state_type,to_state_id,to_state_name,to_state_type +1,hist-1,issue-1,user-2,state-backlog,Backlog,backlog,state-todo,Todo,unstarted +1,hist-2,issue-1,user-1,state-todo,Todo,unstarted,state-inprogress,In Progress,started +1,hist-3,issue-1,user-1,state-inprogress,In Progress,started,state-done,Done,completed diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv new file mode 100644 index 00000000000..9d53280010f --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv @@ -0,0 +1,4 @@ +id,issue_id,author_id,author_name,field_id,field_name,original_from_value,original_to_value,from_value,to_value,created_date +linear:LinearIssueHistory:1:hist-1,linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-2,,state,status,Backlog,Todo,TODO,TODO,2026-05-01T08:00:00.000+00:00 +linear:LinearIssueHistory:1:hist-2,linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-1,,state,status,Todo,In Progress,TODO,IN_PROGRESS,2026-05-02T00:00:00.000+00:00 +linear:LinearIssueHistory:1:hist-3,linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-1,,state,status,In Progress,Done,IN_PROGRESS,DONE,2026-05-03T00:00:00.000+00:00 diff --git a/backend/plugins/linear/impl/impl.go b/backend/plugins/linear/impl/impl.go index 3e1aecd6365..643138de0c2 100644 --- a/backend/plugins/linear/impl/impl.go +++ b/backend/plugins/linear/impl/impl.go @@ -107,12 +107,15 @@ func (p Linear) SubTaskMetas() []plugin.SubTaskMeta { tasks.ExtractCommentsMeta, tasks.CollectCyclesMeta, tasks.ExtractCyclesMeta, + tasks.CollectIssueHistoryMeta, + tasks.ExtractIssueHistoryMeta, tasks.ConvertAccountsMeta, tasks.ConvertIssuesMeta, tasks.ConvertIssueLabelsMeta, tasks.ConvertCommentsMeta, tasks.ConvertCyclesMeta, tasks.ConvertSprintIssuesMeta, + tasks.ConvertIssueHistoryMeta, } } diff --git a/backend/plugins/linear/models/issue_history.go b/backend/plugins/linear/models/issue_history.go index d4b6cc818bf..027f6cb4a07 100644 --- a/backend/plugins/linear/models/issue_history.go +++ b/backend/plugins/linear/models/issue_history.go @@ -32,8 +32,10 @@ type LinearIssueHistory struct { ActorId string `gorm:"type:varchar(255)" json:"actorId"` FromStateId string `gorm:"type:varchar(255)" json:"fromStateId"` FromStateName string `gorm:"type:varchar(255)" json:"fromStateName"` + FromStateType string `gorm:"type:varchar(100)" json:"fromStateType"` ToStateId string `gorm:"type:varchar(255)" json:"toStateId"` ToStateName string `gorm:"type:varchar(255)" json:"toStateName"` + ToStateType string `gorm:"type:varchar(100)" json:"toStateType"` CreatedAt time.Time `gorm:"index" json:"createdAt"` common.NoPKModel } diff --git a/backend/plugins/linear/models/migrationscripts/archived/models.go b/backend/plugins/linear/models/migrationscripts/archived/models.go index a544b2afb5e..21514765d4b 100644 --- a/backend/plugins/linear/models/migrationscripts/archived/models.go +++ b/backend/plugins/linear/models/migrationscripts/archived/models.go @@ -156,8 +156,10 @@ type LinearIssueHistory struct { ActorId string `gorm:"type:varchar(255)"` FromStateId string `gorm:"type:varchar(255)"` FromStateName string `gorm:"type:varchar(255)"` + FromStateType string `gorm:"type:varchar(100)"` ToStateId string `gorm:"type:varchar(255)"` ToStateName string `gorm:"type:varchar(255)"` + ToStateType string `gorm:"type:varchar(100)"` CreatedAt time.Time `gorm:"index"` archived.NoPKModel } diff --git a/backend/plugins/linear/tasks/issue_history_collector.go b/backend/plugins/linear/tasks/issue_history_collector.go new file mode 100644 index 00000000000..995b323c77e --- /dev/null +++ b/backend/plugins/linear/tasks/issue_history_collector.go @@ -0,0 +1,135 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/merico-ai/graphql" +) + +const RAW_ISSUE_HISTORY_TABLE = "linear_issue_history" + +// GraphqlQueryHistoryWrapper is the per-issue, paginated `history` query. +type GraphqlQueryHistoryWrapper struct { + Issue struct { + History struct { + Nodes []GraphqlQueryHistory + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"history(first: $pageSize, after: $skipCursor)"` + } `graphql:"issue(id: $issueId)"` +} + +type GraphqlQueryHistory struct { + Id string + CreatedAt time.Time + Actor *struct{ Id string } + FromState *struct { + Id string + Name string + Type string + } + ToState *struct { + Id string + Name string + Type string + } +} + +var CollectIssueHistoryMeta = plugin.SubTaskMeta{ + Name: "Collect Issue History", + EntryPoint: CollectIssueHistory, + EnabledByDefault: true, + Description: "Collect history events for each collected Linear issue", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ExtractIssuesMeta}, +} + +var _ plugin.SubTaskEntryPoint = CollectIssueHistory + +func CollectIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + + cursor, err := db.Cursor( + dal.Select("id"), + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ?", data.Options.ConnectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleLinearIssue{})) + if err != nil { + return err + } + + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUE_HISTORY_TABLE, + }, + GraphqlClient: data.GraphqlClient, + Input: iterator, + InputStep: 1, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryHistoryWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + issue := reqData.Input.(*SimpleLinearIssue) + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "issueId": graphql.String(issue.Id), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryHistoryWrapper) + return query.Issue.History.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryHistoryWrapper) + for _, event := range query.Issue.History.Nodes { + // Only state transitions are relevant to the status changelog. + if event.FromState == nil && event.ToState == nil { + continue + } + messages = append(messages, errors.Must1(json.Marshal(event))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_history_convertor.go b/backend/plugins/linear/tasks/issue_history_convertor.go new file mode 100644 index 00000000000..6ae68d92382 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_history_convertor.go @@ -0,0 +1,103 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertIssueHistoryMeta = plugin.SubTaskMeta{ + Name: "Convert Issue History", + EntryPoint: ConvertIssueHistory, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_issue_history into domain layer table issue_changelogs", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssueHistory{}.TableName(), models.LinearIssue{}.TableName(), RAW_ISSUE_HISTORY_TABLE}, + ProductTables: []string{ticket.IssueChangelogs{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertIssueHistory + +func ConvertIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + historyIdGen := didgen.NewDomainIdGenerator(&models.LinearIssueHistory{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + + cursor, err := db.Cursor( + dal.Select("h.*"), + dal.From("_tool_linear_issue_history h"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = h.connection_id AND i.id = h.issue_id)"), + dal.Where("h.connection_id = ? AND i.team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUE_HISTORY_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssueHistory{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + event := inputRow.(*models.LinearIssueHistory) + changelog := &ticket.IssueChangelogs{ + DomainEntity: domainlayer.DomainEntity{Id: historyIdGen.Generate(connectionId, event.Id)}, + IssueId: issueIdGen.Generate(connectionId, event.IssueId), + FieldId: "state", + FieldName: "status", + OriginalFromValue: event.FromStateName, + OriginalToValue: event.ToStateName, + CreatedDate: event.CreatedAt, + } + if event.FromStateType != "" { + changelog.FromValue = StatusFromStateType(event.FromStateType) + } + if event.ToStateType != "" { + changelog.ToValue = StatusFromStateType(event.ToStateType) + } + if event.ActorId != "" { + changelog.AuthorId = accountIdGen.Generate(connectionId, event.ActorId) + } + return []interface{}{changelog}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_history_extractor.go b/backend/plugins/linear/tasks/issue_history_extractor.go new file mode 100644 index 00000000000..243c7ea2379 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_history_extractor.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractIssueHistoryMeta = plugin.SubTaskMeta{ + Name: "Extract Issue History", + EntryPoint: ExtractIssueHistory, + EnabledByDefault: true, + Description: "Extract raw issue history into tool layer table _tool_linear_issue_history", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractIssueHistory + +func ExtractIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUE_HISTORY_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiEvent := &GraphqlQueryHistory{} + if err := errors.Convert(json.Unmarshal(row.Data, apiEvent)); err != nil { + return nil, err + } + issueRef := &SimpleLinearIssue{} + if err := errors.Convert(json.Unmarshal(row.Input, issueRef)); err != nil { + return nil, err + } + event := &models.LinearIssueHistory{ + ConnectionId: data.Options.ConnectionId, + Id: apiEvent.Id, + IssueId: issueRef.Id, + CreatedAt: apiEvent.CreatedAt, + } + if apiEvent.Actor != nil { + event.ActorId = apiEvent.Actor.Id + } + if apiEvent.FromState != nil { + event.FromStateId = apiEvent.FromState.Id + event.FromStateName = apiEvent.FromState.Name + event.FromStateType = apiEvent.FromState.Type + } + if apiEvent.ToState != nil { + event.ToStateId = apiEvent.ToState.Id + event.ToStateName = apiEvent.ToState.Name + event.ToStateType = apiEvent.ToState.Type + } + return []interface{}{event}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} From c8ab70ded69f22221893a6dcc5840b2c82c24e80 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 13:36:21 +1000 Subject: [PATCH 10/28] test(linear): add blueprint v200 scope generation tests Cover makeScopesV200: a team scope with the ticket entity produces the expected domain board scope id, and a scope without the ticket entity produces none. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- .../plugins/linear/api/blueprint_v200_test.go | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 backend/plugins/linear/api/blueprint_v200_test.go diff --git a/backend/plugins/linear/api/blueprint_v200_test.go b/backend/plugins/linear/api/blueprint_v200_test.go new file mode 100644 index 00000000000..c77d67fe878 --- /dev/null +++ b/backend/plugins/linear/api/blueprint_v200_test.go @@ -0,0 +1,90 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/stretchr/testify/assert" +) + +func mockLinearPlugin(t *testing.T) { + mockMeta := mockplugin.NewPluginMeta(t) + mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/linear") + mockMeta.On("Name").Return("linear").Maybe() + _ = plugin.RegisterPlugin("linear", mockMeta) +} + +func TestMakeScopesV200(t *testing.T) { + mockLinearPlugin(t) + + const connectionId uint64 = 1 + const teamId = "team-1" + const expectDomainScopeId = "linear:LinearTeam:1:team-1" + + scopes, err := makeScopesV200( + []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig]{ + { + Scope: models.LinearTeam{ + Scope: common.Scope{ConnectionId: connectionId}, + TeamId: teamId, + Name: "Engineering", + }, + ScopeConfig: &models.LinearScopeConfig{ + ScopeConfig: common.ScopeConfig{Entities: []string{plugin.DOMAIN_TYPE_TICKET}}, + }, + }, + }, + &models.LinearConnection{ + BaseConnection: helper.BaseConnection{Model: common.Model{ID: connectionId}}, + }, + ) + assert.Nil(t, err) + assert.Equal(t, 1, len(scopes)) + assert.Equal(t, expectDomainScopeId, scopes[0].ScopeId()) +} + +func TestMakeScopesV200WithoutTicketEntity(t *testing.T) { + mockLinearPlugin(t) + + scopes, err := makeScopesV200( + []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig]{ + { + Scope: models.LinearTeam{ + Scope: common.Scope{ConnectionId: 1}, + TeamId: "team-1", + }, + ScopeConfig: &models.LinearScopeConfig{ + ScopeConfig: common.ScopeConfig{Entities: []string{}}, + }, + }, + }, + &models.LinearConnection{ + BaseConnection: helper.BaseConnection{Model: common.Model{ID: 1}}, + }, + ) + assert.Nil(t, err) + // no ticket entity selected => no domain board scope produced + assert.Equal(t, 0, len(scopes)) +} From e59037fd9b4d50495bb90445c2e7255b10e3b7da Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 13:48:57 +1000 Subject: [PATCH 11/28] docs(linear): add plugin README Document the Linear plugin: supported entities, tool/domain mapping tables, deterministic status mapping, priority/type/lead-time handling, API-key auth, connection/scope/pipeline setup examples, rate limiting, and the roadmap (OAuth, label-based type mapping, config-ui integration). Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- backend/plugins/linear/README.md | 115 +++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 backend/plugins/linear/README.md diff --git a/backend/plugins/linear/README.md b/backend/plugins/linear/README.md new file mode 100644 index 00000000000..e52b4527176 --- /dev/null +++ b/backend/plugins/linear/README.md @@ -0,0 +1,115 @@ + + +# Linear + +## Summary + +This plugin collects data from [Linear](https://linear.app) through its +[GraphQL API](https://linear.app/developers/graphql) and maps it into DevLake's +standardized `ticket` domain, so Linear issues appear in DevLake dashboards +(throughput, lead/cycle time, sprint burndown, etc.). + +The selectable **scope** is a Linear **Team**, which maps to a domain `Board`. + +## Supported data + +| Linear entity | Tool-layer table | Domain-layer table | +|-----------------|-----------------------------------|--------------------------------------------| +| Team | `_tool_linear_teams` (scope) | `boards` | +| User | `_tool_linear_accounts` | `accounts` | +| Workflow state | `_tool_linear_workflow_states` | (drives issue status mapping) | +| Issue | `_tool_linear_issues` | `issues`, `board_issues` | +| Label | `_tool_linear_issue_labels` | `issue_labels` | +| Comment | `_tool_linear_comments` | `issue_comments` | +| Cycle | `_tool_linear_cycles` | `sprints`, `board_sprints`, `sprint_issues`| +| Issue history | `_tool_linear_issue_history` | `issue_changelogs` | + +### Field mapping highlights + +- **Status** — derived deterministically from Linear's `WorkflowState.type` + (no manual mapping needed, unlike Jira): + - `backlog`, `unstarted` → `TODO` + - `started` → `IN_PROGRESS` + - `completed`, `canceled` → `DONE` +- **Priority** — Linear's integer priority maps to a label: `0` No priority, + `1` Urgent, `2` High, `3` Medium, `4` Low. +- **Type** — Linear has no native issue type, so issues default to `REQUIREMENT`. +- **Lead time** — `completedAt − createdAt` (Linear provides `startedAt`/`completedAt` + natively; the history changelog captures every status transition). +- **Story points** — Linear's `estimate`. + +## Authentication + +The plugin uses a Linear **personal API key**, passed verbatim in the +`Authorization` header (no `Bearer` prefix). Create one under +**Settings → Security & access → Personal API keys** in Linear. + +## Configuration + +Create a connection: + +``` +curl 'http://localhost:8080/api/plugins/linear/connections' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "name": "linear", + "endpoint": "https://api.linear.app/graphql", + "token": "", + "rateLimitPerHour": 1500 +}' +``` + +Add a team scope (the team id is the Linear team UUID): + +``` +curl 'http://localhost:8080/api/plugins/linear/connections//scopes' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "data": [{ "connectionId": , "teamId": "", "name": "Engineering" }] +}' +``` + +## Collecting data + +``` +curl 'http://localhost:8080/api/pipelines' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "name": "linear pipeline", + "plan": [[{ + "plugin": "linear", + "options": { "connectionId": , "teamId": "" } + }]] +}' +``` + +## Rate limiting + +Linear enforces a per-API-key request budget (1,500 requests/hour) plus a +complexity budget. The collector paces requests against the configured +`rateLimitPerHour` (default 1500). Issues are collected incrementally using +`updatedAt` ordering so re-runs only fetch changes. + +## Limitations / roadmap + +- Authentication is personal API key only; OAuth2 is a planned follow-up. +- Issue type defaults to `REQUIREMENT`; label-based type mapping via the scope + config is a planned follow-up. +- config-ui integration (connection form + team picker) and the website + documentation page are planned follow-ups; for now connections and scopes are + managed via the API calls shown above. From 42440d87079f9f2be6ef11bca36b979875268b6d Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 14:38:34 +1000 Subject: [PATCH 12/28] fix(linear): guard lead-time fallback against resolution before creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A resolution timestamp (completedAt/canceledAt) earlier than createdAt — from clock skew or migrated/imported issues — produced a negative duration that, cast to uint, yields platform-dependent garbage (0 on arm64, ~1.8e19 on amd64). Skip the fallback unless the resolution is after creation so lead time stays unset instead. Adds an isolated e2e dataflow test with a fixture whose canceledAt precedes createdAt, asserting lead_time_minutes is empty. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- .../plugins/linear/e2e/issue_leadtime_test.go | 61 +++++++++++++++++++ .../_raw_linear_issues_negative_leadtime.csv | 2 + .../issues_negative_leadtime.csv | 2 + .../plugins/linear/tasks/issue_convertor.go | 6 +- 4 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 backend/plugins/linear/e2e/issue_leadtime_test.go create mode 100644 backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv diff --git a/backend/plugins/linear/e2e/issue_leadtime_test.go b/backend/plugins/linear/e2e/issue_leadtime_test.go new file mode 100644 index 00000000000..d7df54e835c --- /dev/null +++ b/backend/plugins/linear/e2e/issue_leadtime_test.go @@ -0,0 +1,61 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearIssueNegativeLeadTime guards the lead-time fallback against +// resolution timestamps that precede the creation timestamp (clock skew or +// migrated/imported issues). A negative duration must NOT be cast to uint -- +// doing so wraps to a huge bogus value. The expected behaviour is that no lead +// time is derived (lead_time_minutes stays empty). +func TestLinearIssueNegativeLeadTime(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // extraction: raw -> tool layer + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues_negative_leadtime.csv", "_raw_linear_issues") + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_negative_leadtime.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv new file mode 100644 index 00000000000..f63750db475 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-neg"",""Identifier"":""ENG-NEG"",""Number"":99,""Title"":""Imported issue with skewed timestamps"",""Description"":""canceledAt precedes createdAt"",""Url"":""https://linear.app/eng/issue/ENG-NEG"",""Priority"":0,""Estimate"":null,""CreatedAt"":""2026-05-10T00:00:00Z"",""UpdatedAt"":""2026-05-10T00:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":""2026-05-09T00:00:00Z"",""State"":{""Id"":""state-canceled"",""Name"":""Canceled"",""Type"":""canceled""},""Assignee"":null,""Creator"":{""Id"":""user-1""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-10 00:00:00.000 diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv b/backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv new file mode 100644 index 00000000000..4375d47901b --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv @@ -0,0 +1,2 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-neg,https://linear.app/eng/issue/ENG-NEG,,ENG-NEG,Imported issue with skewed timestamps,canceledAt precedes createdAt,,REQUIREMENT,,DONE,Canceled,,2026-05-09T00:00:00.000+00:00,2026-05-10T00:00:00.000+00:00,2026-05-10T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,No priority,,,,,0,, diff --git a/backend/plugins/linear/tasks/issue_convertor.go b/backend/plugins/linear/tasks/issue_convertor.go index 06e18272614..3c01437d2b2 100644 --- a/backend/plugins/linear/tasks/issue_convertor.go +++ b/backend/plugins/linear/tasks/issue_convertor.go @@ -106,7 +106,11 @@ func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error { domainIssue.ResolutionDate = issue.CanceledAt } // Fallback lead time when no history-derived value is present. - if domainIssue.LeadTimeMinutes == nil && domainIssue.ResolutionDate != nil { + // Guard against a resolution that precedes creation (clock skew or + // migrated/imported issues): a negative duration cast to uint yields + // platform-dependent garbage, so leave lead time unset instead. + if domainIssue.LeadTimeMinutes == nil && domainIssue.ResolutionDate != nil && + domainIssue.ResolutionDate.After(issue.CreatedAt) { minutes := uint(domainIssue.ResolutionDate.Sub(issue.CreatedAt).Minutes()) domainIssue.LeadTimeMinutes = &minutes } From 8922686bc5694590b178373c6973c9990afd3295 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 14:38:43 +1000 Subject: [PATCH 13/28] fix(linear): map Linear triage state type to TODO The WorkflowState.type 'triage' (the inbox state issues land in before being accepted) previously fell through to OTHER, contradicting the documented total mapping and silently mislabeling triage issues. Map it to TODO; keep OTHER as the fallback for genuinely unrecognized types so unexpected API values surface. Adds a unit test covering every documented state type plus triage and an unknown value. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- backend/plugins/linear/tasks/shared.go | 13 ++++-- backend/plugins/linear/tasks/shared_test.go | 45 +++++++++++++++++++++ 2 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 backend/plugins/linear/tasks/shared_test.go diff --git a/backend/plugins/linear/tasks/shared.go b/backend/plugins/linear/tasks/shared.go index eedeff90e8f..78249d76124 100644 --- a/backend/plugins/linear/tasks/shared.go +++ b/backend/plugins/linear/tasks/shared.go @@ -54,12 +54,17 @@ func PriorityLabel(priority int) string { // issue status. Linear's state types are standardized, so no user-supplied // mapping is required: // -// backlog, unstarted -> TODO -// started -> IN_PROGRESS -// completed, canceled -> DONE +// triage, backlog, unstarted -> TODO +// started -> IN_PROGRESS +// completed, canceled -> DONE +// +// "triage" is the inbox state issues land in before they are accepted into a +// workflow; it is treated as not-yet-started (TODO). Any unrecognized type +// falls back to OTHER so unexpected API values surface rather than silently +// masquerading as a known status. func StatusFromStateType(stateType string) string { switch stateType { - case "backlog", "unstarted": + case "triage", "backlog", "unstarted": return ticket.TODO case "started": return ticket.IN_PROGRESS diff --git a/backend/plugins/linear/tasks/shared_test.go b/backend/plugins/linear/tasks/shared_test.go new file mode 100644 index 00000000000..eaf737c9022 --- /dev/null +++ b/backend/plugins/linear/tasks/shared_test.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/stretchr/testify/assert" +) + +// TestStatusFromStateType pins the mapping for every Linear WorkflowState.type +// value. Linear's state types are standardized; "triage" is the inbox state +// issues land in before they are accepted, so it maps to TODO. Any genuinely +// unknown type falls back to OTHER. +func TestStatusFromStateType(t *testing.T) { + cases := map[string]string{ + "backlog": ticket.TODO, + "unstarted": ticket.TODO, + "triage": ticket.TODO, + "started": ticket.IN_PROGRESS, + "completed": ticket.DONE, + "canceled": ticket.DONE, + "": ticket.OTHER, + "something": ticket.OTHER, + } + for stateType, want := range cases { + assert.Equal(t, want, StatusFromStateType(stateType), "state type %q", stateType) + } +} From b9f286131b1f1ab9a170ca0b235bdbdbb31e1c2f Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 14:54:24 +1000 Subject: [PATCH 14/28] refactor(linear): remove unused GraphqlInlineAccount struct The struct was documented as the shared inline-user shape but was never referenced; each collector declares its own inline user struct. Removing it avoids misleading a maintainer into editing a type nothing reads. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- backend/plugins/linear/tasks/shared.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/backend/plugins/linear/tasks/shared.go b/backend/plugins/linear/tasks/shared.go index 78249d76124..8cbdd96e571 100644 --- a/backend/plugins/linear/tasks/shared.go +++ b/backend/plugins/linear/tasks/shared.go @@ -21,17 +21,6 @@ import ( "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" ) -// GraphqlInlineAccount is the shared shape used to collect a Linear user that -// is referenced inline on another entity (issue creator/assignee, comment -// author, history actor). -type GraphqlInlineAccount struct { - Id string - Name string - DisplayName string - Email string - AvatarUrl string -} - // priorityLabels maps Linear's integer priority to its human-readable label. // Linear: 0 = No priority, 1 = Urgent, 2 = High, 3 = Medium, 4 = Low. var priorityLabels = map[int]string{ From 2038be50a0db301766c80a2b7c81dafd13d57dab Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 14:54:24 +1000 Subject: [PATCH 15/28] perf(linear): raise issue collector page size to 100 Every other Linear collector uses a page size of 100; issues used 50, which doubled the number of issue-page round-trips and the iterator size that drives the per-issue comment/history collectors. Linear permits first: 250. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- backend/plugins/linear/tasks/issue_collector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/plugins/linear/tasks/issue_collector.go b/backend/plugins/linear/tasks/issue_collector.go index bacc3df93bb..484e45b98ed 100644 --- a/backend/plugins/linear/tasks/issue_collector.go +++ b/backend/plugins/linear/tasks/issue_collector.go @@ -100,7 +100,7 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { since := apiCollector.GetSince() err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ GraphqlClient: data.GraphqlClient, - PageSize: 50, + PageSize: 100, BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { query := &GraphqlQueryIssueWrapper{} if reqData == nil { From 28f23e55e5d4bb2a94b11503dc0c565ed77a47f2 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 15:06:20 +1000 Subject: [PATCH 16/28] feat(linear): populate issue assignee/creator names and issue_assignees The issue convertor set only assignee_id/creator_id, leaving the denormalized assignee_name/creator_name columns blank and writing no issue_assignees rows, so dashboards reading those columns or joining through issue_assignees showed blank names. Preload account display names (matching the account convertor's displayName-then-name rule) and emit an IssueAssignee per assigned issue. The issue dataflow test now loads accounts before conversion and asserts the names plus issue_assignees; the lead-time test flushes accounts to stay order-independent on the shared test DB. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- .../plugins/linear/e2e/issue_leadtime_test.go | 3 ++ backend/plugins/linear/e2e/issue_test.go | 11 ++++++- .../e2e/snapshot_tables/issue_assignees.csv | 3 ++ .../linear/e2e/snapshot_tables/issues.csv | 10 +++---- .../plugins/linear/tasks/issue_convertor.go | 30 +++++++++++++++++-- 5 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv diff --git a/backend/plugins/linear/e2e/issue_leadtime_test.go b/backend/plugins/linear/e2e/issue_leadtime_test.go index d7df54e835c..783820f696d 100644 --- a/backend/plugins/linear/e2e/issue_leadtime_test.go +++ b/backend/plugins/linear/e2e/issue_leadtime_test.go @@ -45,6 +45,9 @@ func TestLinearIssueNegativeLeadTime(t *testing.T) { } // extraction: raw -> tool layer + // Flush accounts so this lead-time-focused test is independent of any + // account rows left behind by other tests sharing the DB. + dataflowTester.FlushTabler(&models.LinearAccount{}) dataflowTester.FlushTabler(&models.LinearIssue{}) dataflowTester.FlushTabler(&models.LinearIssueLabel{}) dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues_negative_leadtime.csv", "_raw_linear_issues") diff --git a/backend/plugins/linear/e2e/issue_test.go b/backend/plugins/linear/e2e/issue_test.go index 23956cb3985..5f98aad8a61 100644 --- a/backend/plugins/linear/e2e/issue_test.go +++ b/backend/plugins/linear/e2e/issue_test.go @@ -53,9 +53,14 @@ func TestLinearIssueDataFlow(t *testing.T) { IgnoreTypes: []interface{}{common.NoPKModel{}}, }) - // verify conversion: tool layer -> domain layer (issues + board_issues) + // accounts must be present so the convertor can resolve assignee/creator + // display names and emit issue_assignees rows. + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_linear_accounts.csv", &models.LinearAccount{}) + + // verify conversion: tool layer -> domain layer (issues + board_issues + issue_assignees) dataflowTester.FlushTabler(&ticket.Issue{}) dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.FlushTabler(&ticket.IssueAssignee{}) dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ CSVRelPath: "./snapshot_tables/issues.csv", @@ -65,4 +70,8 @@ func TestLinearIssueDataFlow(t *testing.T) { CSVRelPath: "./snapshot_tables/board_issues.csv", IgnoreTypes: []interface{}{common.NoPKModel{}}, }) + dataflowTester.VerifyTableWithOptions(ticket.IssueAssignee{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_assignees.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) } diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv new file mode 100644 index 00000000000..38f569f7094 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv @@ -0,0 +1,3 @@ +issue_id,assignee_id,assignee_name +linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-1,Alice Anderson +linear:LinearIssue:1:issue-4,linear:LinearAccount:1:user-3,Carol Clark diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues.csv b/backend/plugins/linear/e2e/snapshot_tables/issues.csv index da87aeffc4e..f55c39e0eb8 100644 --- a/backend/plugins/linear/e2e/snapshot_tables/issues.csv +++ b/backend/plugins/linear/e2e/snapshot_tables/issues.csv @@ -1,6 +1,6 @@ id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions -linear:LinearIssue:1:issue-1,https://linear.app/eng/issue/ENG-1,,ENG-1,Fix login bug,Users cannot log in,,REQUIREMENT,,DONE,Done,3,2026-05-03T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00,2880,,,,linear:LinearAccount:1:user-2,,linear:LinearAccount:1:user-1,,,Urgent,,,,,0,, -linear:LinearIssue:1:issue-2,https://linear.app/eng/issue/ENG-2,,ENG-2,Add dark mode,Theme support,,REQUIREMENT,,IN_PROGRESS,In Progress,5,,2026-05-01T00:00:00.000+00:00,2026-05-02T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,High,,,,,0,, -linear:LinearIssue:1:issue-3,https://linear.app/eng/issue/ENG-3,,ENG-3,Investigate flakiness,,,REQUIREMENT,,TODO,Backlog,,,2026-05-01T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,No priority,,,,,0,, -linear:LinearIssue:1:issue-4,https://linear.app/eng/issue/ENG-4,,ENG-4,Deprecated feature,No longer needed,,REQUIREMENT,,DONE,Canceled,2,2026-05-02T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-02T12:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-1,,linear:LinearAccount:1:user-3,,,Medium,,,,,0,, -linear:LinearIssue:1:issue-5,https://linear.app/eng/issue/ENG-5,,ENG-5,Write docs,User guide,,REQUIREMENT,,TODO,Todo,1,,2026-05-01T00:00:00.000+00:00,2026-05-01T06:00:00.000+00:00,,,,,linear:LinearAccount:1:user-2,,,,,Low,,,,,0,, +linear:LinearIssue:1:issue-1,https://linear.app/eng/issue/ENG-1,,ENG-1,Fix login bug,Users cannot log in,,REQUIREMENT,,DONE,Done,3,2026-05-03T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00,2880,,,,linear:LinearAccount:1:user-2,Bob Brown,linear:LinearAccount:1:user-1,Alice Anderson,,Urgent,,,,,0,, +linear:LinearIssue:1:issue-2,https://linear.app/eng/issue/ENG-2,,ENG-2,Add dark mode,Theme support,,REQUIREMENT,,IN_PROGRESS,In Progress,5,,2026-05-01T00:00:00.000+00:00,2026-05-02T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,Alice Anderson,,,,High,,,,,0,, +linear:LinearIssue:1:issue-3,https://linear.app/eng/issue/ENG-3,,ENG-3,Investigate flakiness,,,REQUIREMENT,,TODO,Backlog,,,2026-05-01T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,Alice Anderson,,,,No priority,,,,,0,, +linear:LinearIssue:1:issue-4,https://linear.app/eng/issue/ENG-4,,ENG-4,Deprecated feature,No longer needed,,REQUIREMENT,,DONE,Canceled,2,2026-05-02T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-02T12:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-1,Alice Anderson,linear:LinearAccount:1:user-3,Carol Clark,,Medium,,,,,0,, +linear:LinearIssue:1:issue-5,https://linear.app/eng/issue/ENG-5,,ENG-5,Write docs,User guide,,REQUIREMENT,,TODO,Todo,1,,2026-05-01T00:00:00.000+00:00,2026-05-01T06:00:00.000+00:00,,,,,linear:LinearAccount:1:user-2,Bob Brown,,,,Low,,,,,0,, diff --git a/backend/plugins/linear/tasks/issue_convertor.go b/backend/plugins/linear/tasks/issue_convertor.go index 3c01437d2b2..46a779b80af 100644 --- a/backend/plugins/linear/tasks/issue_convertor.go +++ b/backend/plugins/linear/tasks/issue_convertor.go @@ -37,7 +37,7 @@ var ConvertIssuesMeta = plugin.SubTaskMeta{ Description: "Convert tool layer table _tool_linear_issues into domain layer tables issues and board_issues", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, DependencyTables: []string{models.LinearIssue{}.TableName(), RAW_ISSUES_TABLE}, - ProductTables: []string{ticket.Issue{}.TableName(), ticket.BoardIssue{}.TableName()}, + ProductTables: []string{ticket.Issue{}.TableName(), ticket.BoardIssue{}.TableName(), ticket.IssueAssignee{}.TableName()}, } var _ plugin.SubTaskEntryPoint = ConvertIssues @@ -52,6 +52,22 @@ func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error { boardIdGen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) boardId := boardIdGen.Generate(connectionId, data.Options.TeamId) + // Preload account display names so issues can carry assignee/creator names + // and emit issue_assignees rows, mirroring how the account convertor derives + // the domain account's full name (displayName, falling back to name). + var accounts []models.LinearAccount + if err := db.All(&accounts, dal.Where("connection_id = ?", connectionId)); err != nil { + return err + } + accountNames := make(map[string]string, len(accounts)) + for _, account := range accounts { + name := account.Name + if account.DisplayName != "" { + name = account.DisplayName + } + accountNames[account.Id] = name + } + cursor, err := db.Cursor( dal.From(&models.LinearIssue{}), dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), @@ -91,9 +107,11 @@ func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error { } if issue.CreatorId != "" { domainIssue.CreatorId = accountIdGen.Generate(connectionId, issue.CreatorId) + domainIssue.CreatorName = accountNames[issue.CreatorId] } if issue.AssigneeId != "" { domainIssue.AssigneeId = accountIdGen.Generate(connectionId, issue.AssigneeId) + domainIssue.AssigneeName = accountNames[issue.AssigneeId] } if issue.ParentId != "" { domainIssue.ParentIssueId = issueIdGen.Generate(connectionId, issue.ParentId) @@ -118,7 +136,15 @@ func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error { BoardId: boardId, IssueId: domainIssue.Id, } - return []interface{}{domainIssue, boardIssue}, nil + results := []interface{}{domainIssue, boardIssue} + if domainIssue.AssigneeId != "" { + results = append(results, &ticket.IssueAssignee{ + IssueId: domainIssue.Id, + AssigneeId: domainIssue.AssigneeId, + AssigneeName: domainIssue.AssigneeName, + }) + } + return results, nil }, }) if err != nil { From 33fa318c6cca96b24611098605b2fb862b5d2728 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 15:24:26 +1000 Subject: [PATCH 17/28] fix(linear): clear stale sprint_issues when issues leave their cycle Sprint membership is derived from each issue's cycle_id, and the batch divider only deletes outdated rows when it produces at least one row of the type. When every issue is moved out of its cycle the convertor emits nothing, so the divider never fires and prior sprint_issues rows linger, leaving issues shown in sprints they no longer belong to. Delete the team's sprint_issues up front so the result is correct regardless of how many issues remain in a cycle. Adds a two-run e2e test that empties every issue's cycle and asserts sprint_issues is empty afterward. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- .../sprint_issues_after_leaving_cycle.csv | 1 + .../plugins/linear/e2e/sprint_issue_test.go | 71 +++++++++++++++++++ .../linear/tasks/sprint_issue_convertor.go | 22 ++++++ 3 files changed, 94 insertions(+) create mode 100644 backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv create mode 100644 backend/plugins/linear/e2e/sprint_issue_test.go diff --git a/backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv new file mode 100644 index 00000000000..1074725b86c --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv @@ -0,0 +1 @@ +sprint_id,issue_id diff --git a/backend/plugins/linear/e2e/sprint_issue_test.go b/backend/plugins/linear/e2e/sprint_issue_test.go new file mode 100644 index 00000000000..81e4508e43f --- /dev/null +++ b/backend/plugins/linear/e2e/sprint_issue_test.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearSprintIssueStaleCycle guards against stale sprint_issues rows when +// an issue is moved out of a cycle. Sprint membership is derived from the +// issue's cycle_id; once that empties on re-collection, the issue must no +// longer appear in sprint_issues from a previous run. +func TestLinearSprintIssueStaleCycle(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // seed issues: issue-1 and issue-4 both belong to cycle-1 + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // first conversion: both issues land in the sprint + dataflowTester.FlushTabler(&ticket.SprintIssue{}) + dataflowTester.Subtask(tasks.ConvertSprintIssuesMeta, taskData) + + // every issue is moved out of its cycle; on re-collection cycle_id empties. + // The second conversion then produces zero sprint issues, which is the case + // the batch divider's lazy delete fails to cover. + if err := dataflowTester.Dal.Exec( + "UPDATE _tool_linear_issues SET cycle_id = '' WHERE connection_id = ? AND team_id = ?", 1, "team-1", + ); err != nil { + t.Fatal(err) + } + + // second conversion (no flush) must drop ALL stale sprint_issues rows + dataflowTester.Subtask(tasks.ConvertSprintIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.SprintIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/sprint_issues_after_leaving_cycle.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/tasks/sprint_issue_convertor.go b/backend/plugins/linear/tasks/sprint_issue_convertor.go index 33e771ef29d..7ebe467a908 100644 --- a/backend/plugins/linear/tasks/sprint_issue_convertor.go +++ b/backend/plugins/linear/tasks/sprint_issue_convertor.go @@ -49,6 +49,28 @@ func ConvertSprintIssues(taskCtx plugin.SubTaskContext) errors.Error { issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) cycleIdGen := didgen.NewDomainIdGenerator(&models.LinearCycle{}) + // Sprint membership is derived from each issue's cycle_id. Clear this team's + // existing sprint_issues up front so issues that have since left their cycle + // leave no stale rows: the batch divider only deletes outdated records when + // it produces at least one row of the type, which misses the case where + // every issue has been removed from its cycle. + var teamIssues []models.LinearIssue + if err := db.All(&teamIssues, + dal.Select("id"), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ); err != nil { + return err + } + if len(teamIssues) > 0 { + issueIds := make([]string, len(teamIssues)) + for i, issue := range teamIssues { + issueIds[i] = issueIdGen.Generate(connectionId, issue.Id) + } + if err := db.Delete(&ticket.SprintIssue{}, dal.Where("issue_id IN ?", issueIds)); err != nil { + return err + } + } + cursor, err := db.Cursor( dal.From(&models.LinearIssue{}), dal.Where("connection_id = ? AND team_id = ? AND cycle_id != ''", connectionId, data.Options.TeamId), From 698a0361fc98778270e304e239fc17128f481a77 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 16:07:59 +1000 Subject: [PATCH 18/28] feat(linear): derive issue lead time from state-transition history The LinearIssue.LeadTimeMinutes field was never populated, so lead time always fell back to the coarse createdAt -> resolutionDate span. Derive it instead from the recorded history: the span from an issue's first transition into an in-progress state to its first transition into a done state thereafter (active cycle time), which is the value that genuinely requires history. ConvertIssues still seeds the fallback; ConvertIssueHistory now overrides it when the transitions exist, and issues lacking them keep the fallback. Adds an e2e test asserting issue-1 (started 05-02, completed 05-03) resolves to 1440 minutes from history rather than its 2880-minute created->resolved span. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- .../linear/e2e/issue_history_leadtime_test.go | 77 +++++++++++++++++++ .../issues_history_leadtime.csv | 6 ++ .../linear/tasks/issue_history_convertor.go | 67 +++++++++++++++- 3 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 backend/plugins/linear/e2e/issue_history_leadtime_test.go create mode 100644 backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv diff --git a/backend/plugins/linear/e2e/issue_history_leadtime_test.go b/backend/plugins/linear/e2e/issue_history_leadtime_test.go new file mode 100644 index 00000000000..7d4a4c9798a --- /dev/null +++ b/backend/plugins/linear/e2e/issue_history_leadtime_test.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearIssueHistoryLeadTime proves lead time is derived from the recorded +// state transitions. issue-1 was started on 2026-05-02 and completed on +// 2026-05-03 (1440 min of active cycle time), even though its createdAt -> +// resolutionDate span is 2880 min. ConvertIssues sets the coarse 2880 fallback; +// ConvertIssueHistory must then refine it to 1440 from the history. Issues +// without the required transitions keep the fallback (issue-4 = 1440). +func TestLinearIssueHistoryLeadTime(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // flush accounts so assignee/creator names stay empty in this lead-time- + // focused test regardless of other tests sharing the DB. + dataflowTester.FlushTabler(&models.LinearAccount{}) + + // seed issues + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // seed history + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issue_history.csv", "_raw_linear_issue_history") + dataflowTester.FlushTabler(&models.LinearIssueHistory{}) + dataflowTester.Subtask(tasks.ExtractIssueHistoryMeta, taskData) + + // ConvertIssues sets the createdAt -> resolutionDate fallback ... + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.FlushTabler(&ticket.IssueAssignee{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + + // ... then ConvertIssueHistory refines lead time from the transitions. + dataflowTester.FlushTabler(&ticket.IssueChangelogs{}) + dataflowTester.Subtask(tasks.ConvertIssueHistoryMeta, taskData) + + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_history_leadtime.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv b/backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv new file mode 100644 index 00000000000..d11625e3c3d --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv @@ -0,0 +1,6 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-1,https://linear.app/eng/issue/ENG-1,,ENG-1,Fix login bug,Users cannot log in,,REQUIREMENT,,DONE,Done,3,2026-05-03T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-2,,linear:LinearAccount:1:user-1,,,Urgent,,,,,0,, +linear:LinearIssue:1:issue-2,https://linear.app/eng/issue/ENG-2,,ENG-2,Add dark mode,Theme support,,REQUIREMENT,,IN_PROGRESS,In Progress,5,,2026-05-01T00:00:00.000+00:00,2026-05-02T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,High,,,,,0,, +linear:LinearIssue:1:issue-3,https://linear.app/eng/issue/ENG-3,,ENG-3,Investigate flakiness,,,REQUIREMENT,,TODO,Backlog,,,2026-05-01T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,No priority,,,,,0,, +linear:LinearIssue:1:issue-4,https://linear.app/eng/issue/ENG-4,,ENG-4,Deprecated feature,No longer needed,,REQUIREMENT,,DONE,Canceled,2,2026-05-02T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-02T12:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-1,,linear:LinearAccount:1:user-3,,,Medium,,,,,0,, +linear:LinearIssue:1:issue-5,https://linear.app/eng/issue/ENG-5,,ENG-5,Write docs,User guide,,REQUIREMENT,,TODO,Todo,1,,2026-05-01T00:00:00.000+00:00,2026-05-01T06:00:00.000+00:00,,,,,linear:LinearAccount:1:user-2,,,,,Low,,,,,0,, diff --git a/backend/plugins/linear/tasks/issue_history_convertor.go b/backend/plugins/linear/tasks/issue_history_convertor.go index 6ae68d92382..f1fa99d0519 100644 --- a/backend/plugins/linear/tasks/issue_history_convertor.go +++ b/backend/plugins/linear/tasks/issue_history_convertor.go @@ -19,6 +19,7 @@ package tasks import ( "reflect" + "time" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" @@ -99,5 +100,69 @@ func ConvertIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { return err } - return converter.Execute() + if err := converter.Execute(); err != nil { + return err + } + + return deriveLeadTimeFromHistory(db, connectionId, data.Options.TeamId, issueIdGen) +} + +// deriveLeadTimeFromHistory refines each issue's lead time from its recorded +// state transitions: the span from the issue's first transition into an +// in-progress state to its first transition into a done state thereafter (the +// active cycle time). This is the value that genuinely requires history and is +// more accurate than the coarse createdAt -> resolutionDate fallback set by +// ConvertIssues, so it overrides that fallback when the transitions exist. +// Issues whose history lacks an in-progress -> done sequence keep the fallback. +func deriveLeadTimeFromHistory(db dal.Dal, connectionId uint64, teamId string, issueIdGen *didgen.DomainIdGenerator) errors.Error { + var events []models.LinearIssueHistory + if err := db.All(&events, + dal.Select("h.*"), + dal.From("_tool_linear_issue_history h"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = h.connection_id AND i.id = h.issue_id)"), + dal.Where("h.connection_id = ? AND i.team_id = ?", connectionId, teamId), + dal.Orderby("h.issue_id, h.created_at"), + ); err != nil { + return err + } + + type leadWindow struct { + startedAt *time.Time + doneAt *time.Time + } + windows := map[string]*leadWindow{} + for i := range events { + event := events[i] + window := windows[event.IssueId] + if window == nil { + window = &leadWindow{} + windows[event.IssueId] = window + } + switch StatusFromStateType(event.ToStateType) { + case ticket.IN_PROGRESS: + if window.startedAt == nil { + createdAt := event.CreatedAt + window.startedAt = &createdAt + } + case ticket.DONE: + if window.startedAt != nil && window.doneAt == nil { + createdAt := event.CreatedAt + window.doneAt = &createdAt + } + } + } + + for issueId, window := range windows { + if window.startedAt == nil || window.doneAt == nil || !window.doneAt.After(*window.startedAt) { + continue + } + minutes := uint(window.doneAt.Sub(*window.startedAt).Minutes()) + if err := db.UpdateColumn( + &ticket.Issue{}, "lead_time_minutes", minutes, + dal.Where("id = ?", issueIdGen.Generate(connectionId, issueId)), + ); err != nil { + return err + } + } + return nil } From 6fd5a686d391b2de6b45ed1a48c15c68d822310b Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 16:26:29 +1000 Subject: [PATCH 19/28] feat(linear): add remote-scopes endpoints to enumerate teams Other first-party ticket plugins (jira, asana, github) expose connections/:connectionId/remote-scopes so the config UI can browse and select scopes from the API. Linear had none, forcing users to hand-craft a PUT /scopes with raw team UUIDs they had no in-product way to discover. Wire the standard DsRemoteApiProxyHelper + DsRemoteApiScopeListHelper and a lister that queries the GraphQL teams connection (flat list, cursor-paginated) through the connection's authenticated client. Adds unit tests for the response->scope-entry mapping, the pagination cursor, and the route registration. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- backend/plugins/linear/api/init.go | 4 + backend/plugins/linear/api/remote_api.go | 137 ++++++++++++++++++ backend/plugins/linear/api/remote_api_test.go | 66 +++++++++ backend/plugins/linear/impl/impl.go | 6 + backend/plugins/linear/impl/impl_test.go | 38 +++++ 5 files changed, 251 insertions(+) create mode 100644 backend/plugins/linear/api/remote_api.go create mode 100644 backend/plugins/linear/api/remote_api_test.go create mode 100644 backend/plugins/linear/impl/impl_test.go diff --git a/backend/plugins/linear/api/init.go b/backend/plugins/linear/api/init.go index 2dc7da5943a..850acf54825 100644 --- a/backend/plugins/linear/api/init.go +++ b/backend/plugins/linear/api/init.go @@ -28,6 +28,8 @@ import ( var vld *validator.Validate var basicRes context.BasicRes var dsHelper *api.DsHelper[models.LinearConnection, models.LinearTeam, models.LinearScopeConfig] +var raProxy *api.DsRemoteApiProxyHelper[models.LinearConnection] +var raScopeList *api.DsRemoteApiScopeListHelper[models.LinearConnection, models.LinearTeam, LinearRemotePagination] func Init(br context.BasicRes, p plugin.PluginMeta) { basicRes = br @@ -44,4 +46,6 @@ func Init(br context.BasicRes, p plugin.PluginMeta) { nil, nil, ) + raProxy = api.NewDsRemoteApiProxyHelper[models.LinearConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = api.NewDsRemoteApiScopeListHelper[models.LinearConnection, models.LinearTeam, LinearRemotePagination](raProxy, listLinearRemoteScopes) } diff --git a/backend/plugins/linear/api/remote_api.go b/backend/plugins/linear/api/remote_api.go new file mode 100644 index 00000000000..094c785016d --- /dev/null +++ b/backend/plugins/linear/api/remote_api.go @@ -0,0 +1,137 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// LinearRemotePagination drives cursor-based pagination through the GraphQL +// `teams` connection when listing remote scopes for the config UI. +type LinearRemotePagination struct { + Cursor string `json:"cursor"` +} + +// linearTeamsGraphqlResponse mirrors the shape of the `teams` query response. +type linearTeamsGraphqlResponse struct { + Data struct { + Teams struct { + Nodes []struct { + Id string `json:"id"` + Name string `json:"name"` + Key string `json:"key"` + Description string `json:"description"` + } `json:"nodes"` + PageInfo struct { + HasNextPage bool `json:"hasNextPage"` + EndCursor string `json:"endCursor"` + } `json:"pageInfo"` + } `json:"teams"` + } `json:"data"` + Errors []struct { + Message string `json:"message"` + } `json:"errors"` +} + +const remoteScopesPageSize = 100 + +// listLinearRemoteScopes lists Linear teams as selectable scopes. Linear teams +// are a flat list, so there are no intermediate groups. +func listLinearRemoteScopes( + _ *models.LinearConnection, + apiClient plugin.ApiClient, + _ string, + page LinearRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam], + nextPage *LinearRemotePagination, + err errors.Error, +) { + after := "" + if page.Cursor != "" { + after = fmt.Sprintf(", after: %q", page.Cursor) + } + query := fmt.Sprintf( + "query { teams(first: %d%s) { nodes { id name key description } pageInfo { hasNextPage endCursor } } }", + remoteScopesPageSize, after, + ) + + res, err := apiClient.Post("", nil, map[string]interface{}{"query": query}, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to query Linear teams") + } + var response linearTeamsGraphqlResponse + if err := api.UnmarshalResponse(res, &response); err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Linear teams response") + } + if len(response.Errors) > 0 { + return nil, nil, errors.Default.New("linear graphql teams query failed: " + response.Errors[0].Message) + } + + return mapLinearTeamsToScopeEntries(response), nextPageFrom(response), nil +} + +// mapLinearTeamsToScopeEntries converts a teams response into scope-list +// entries. Each team is a selectable (leaf) scope. +func mapLinearTeamsToScopeEntries(response linearTeamsGraphqlResponse) []dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam] { + children := make([]dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam], 0, len(response.Data.Teams.Nodes)) + for _, team := range response.Data.Teams.Nodes { + team := team + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: nil, + Id: team.Id, + Name: team.Name, + FullName: team.Name, + Data: &models.LinearTeam{ + TeamId: team.Id, + Name: team.Name, + Key: team.Key, + Description: team.Description, + }, + }) + } + return children +} + +// nextPageFrom returns the cursor for the following page, or nil when the +// teams connection has been fully traversed. +func nextPageFrom(response linearTeamsGraphqlResponse) *LinearRemotePagination { + pageInfo := response.Data.Teams.PageInfo + if pageInfo.HasNextPage && pageInfo.EndCursor != "" { + return &LinearRemotePagination{Cursor: pageInfo.EndCursor} + } + return nil +} + +// RemoteScopes lists the Linear teams available on the connection so the +// config UI can enumerate selectable scopes. +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +// Proxy forwards arbitrary requests to the Linear API through the connection. +func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raProxy.Proxy(input) +} diff --git a/backend/plugins/linear/api/remote_api_test.go b/backend/plugins/linear/api/remote_api_test.go new file mode 100644 index 00000000000..801e0d5029f --- /dev/null +++ b/backend/plugins/linear/api/remote_api_test.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "encoding/json" + "testing" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/stretchr/testify/assert" +) + +func TestMapLinearTeamsToScopeEntries(t *testing.T) { + var response linearTeamsGraphqlResponse + body := `{"data":{"teams":{"nodes":[` + + `{"id":"team-uuid-1","name":"Engineering","key":"ENG","description":"core eng"},` + + `{"id":"team-uuid-2","name":"Design","key":"DSG","description":""}` + + `],"pageInfo":{"hasNextPage":false,"endCursor":""}}}}` + assert.NoError(t, json.Unmarshal([]byte(body), &response)) + + entries := mapLinearTeamsToScopeEntries(response) + assert.Len(t, entries, 2) + + assert.Equal(t, api.RAS_ENTRY_TYPE_SCOPE, entries[0].Type) + assert.Nil(t, entries[0].ParentId) + assert.Equal(t, "team-uuid-1", entries[0].Id) + assert.Equal(t, "Engineering", entries[0].Name) + assert.Equal(t, "Engineering", entries[0].FullName) + // the scope payload must carry the team id used as the scope's primary key + assert.NotNil(t, entries[0].Data) + assert.Equal(t, "team-uuid-1", entries[0].Data.TeamId) + assert.Equal(t, "ENG", entries[0].Data.Key) + + assert.Equal(t, "team-uuid-2", entries[1].Id) + assert.Equal(t, "Design", entries[1].Name) +} + +func TestNextPageFrom(t *testing.T) { + var more linearTeamsGraphqlResponse + more.Data.Teams.PageInfo.HasNextPage = true + more.Data.Teams.PageInfo.EndCursor = "cursor-abc" + next := nextPageFrom(more) + assert.NotNil(t, next) + assert.Equal(t, "cursor-abc", next.Cursor) + + // no further pages -> nil, so the helper stops paginating + var last linearTeamsGraphqlResponse + last.Data.Teams.PageInfo.HasNextPage = false + last.Data.Teams.PageInfo.EndCursor = "cursor-xyz" + assert.Nil(t, nextPageFrom(last)) +} diff --git a/backend/plugins/linear/impl/impl.go b/backend/plugins/linear/impl/impl.go index 643138de0c2..7b85c388620 100644 --- a/backend/plugins/linear/impl/impl.go +++ b/backend/plugins/linear/impl/impl.go @@ -173,6 +173,12 @@ func (p Linear) ApiResources() map[string]map[string]plugin.ApiResourceHandler { "connections/:connectionId/test": { "POST": api.TestExistingConnection, }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/proxy/rest/*path": { + "GET": api.Proxy, + }, "connections/:connectionId/scope-configs": { "POST": api.PostScopeConfig, "GET": api.GetScopeConfigList, diff --git a/backend/plugins/linear/impl/impl_test.go b/backend/plugins/linear/impl/impl_test.go new file mode 100644 index 00000000000..d11b3ec5ae1 --- /dev/null +++ b/backend/plugins/linear/impl/impl_test.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestApiResourcesRegistersRemoteScopes guards the remote-scopes endpoints used +// by the config UI to enumerate Linear teams. +func TestApiResourcesRegistersRemoteScopes(t *testing.T) { + resources := Linear{}.ApiResources() + + remoteScopes, ok := resources["connections/:connectionId/remote-scopes"] + assert.True(t, ok, "remote-scopes route must be registered") + assert.NotNil(t, remoteScopes["GET"], "remote-scopes must handle GET") + + proxy, ok := resources["connections/:connectionId/proxy/rest/*path"] + assert.True(t, ok, "proxy route must be registered") + assert.NotNil(t, proxy["GET"], "proxy must handle GET") +} From 46db27adb5064714e20200adc9516c343fefe42f Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 16:33:12 +1000 Subject: [PATCH 20/28] perf(linear): make comment and history collection incremental Both child collectors used a plain GraphqlCollector and swept every issue in the team on every run, issuing one request per issue with no since filter - tens of thousands of requests per run on a large team against Linear's ~1500 req/hour budget. Switch them to a stateful collector and restrict the driving cursor to issues updated since the last successful collection, so steady-state runs scale with the change delta rather than the whole backlog. A full sync (since == nil) still sweeps every issue. Adds a unit test for the incremental cursor-clause builder. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- .../plugins/linear/tasks/comment_collector.go | 37 ++++++++--------- .../linear/tasks/issue_history_collector.go | 36 +++++++++-------- backend/plugins/linear/tasks/shared.go | 21 ++++++++++ .../linear/tasks/shared_clauses_test.go | 40 +++++++++++++++++++ 4 files changed, 99 insertions(+), 35 deletions(-) create mode 100644 backend/plugins/linear/tasks/shared_clauses_test.go diff --git a/backend/plugins/linear/tasks/comment_collector.go b/backend/plugins/linear/tasks/comment_collector.go index b7487ba5afa..c23520d9b7f 100644 --- a/backend/plugins/linear/tasks/comment_collector.go +++ b/backend/plugins/linear/tasks/comment_collector.go @@ -20,14 +20,11 @@ package tasks import ( "encoding/json" "reflect" - "time" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/linear/models" "github.com/merico-ai/graphql" ) @@ -73,11 +70,23 @@ func CollectComments(taskCtx plugin.SubTaskContext) errors.Error { db := taskCtx.GetDal() data := taskCtx.GetData().(*LinearTaskData) - cursor, err := db.Cursor( - dal.Select("id"), - dal.From(&models.LinearIssue{}), - dal.Where("connection_id = ? AND team_id = ?", data.Options.ConnectionId, data.Options.TeamId), - ) + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_COMMENTS_TABLE, + }) + if err != nil { + return err + } + + // Only sweep issues updated since the last successful collection: an + // unchanged issue's comments cannot have changed, so re-fetching every + // issue each run wastes a request per issue against Linear's hourly budget. + since := apiCollector.GetSince() + cursor, err := db.Cursor(issuesToCollectChildrenClauses(data.Options.ConnectionId, data.Options.TeamId, since)...) if err != nil { return err } @@ -86,15 +95,7 @@ func CollectComments(taskCtx plugin.SubTaskContext) errors.Error { return err } - collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ - RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ - Ctx: taskCtx, - Params: LinearApiParams{ - ConnectionId: data.Options.ConnectionId, - TeamId: data.Options.TeamId, - }, - Table: RAW_COMMENTS_TABLE, - }, + err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ GraphqlClient: data.GraphqlClient, Input: iterator, InputStep: 1, @@ -127,5 +128,5 @@ func CollectComments(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { return err } - return collector.Execute() + return apiCollector.Execute() } diff --git a/backend/plugins/linear/tasks/issue_history_collector.go b/backend/plugins/linear/tasks/issue_history_collector.go index 995b323c77e..11de7c218c7 100644 --- a/backend/plugins/linear/tasks/issue_history_collector.go +++ b/backend/plugins/linear/tasks/issue_history_collector.go @@ -22,11 +22,9 @@ import ( "reflect" "time" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/linear/models" "github.com/merico-ai/graphql" ) @@ -73,11 +71,23 @@ func CollectIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { db := taskCtx.GetDal() data := taskCtx.GetData().(*LinearTaskData) - cursor, err := db.Cursor( - dal.Select("id"), - dal.From(&models.LinearIssue{}), - dal.Where("connection_id = ? AND team_id = ?", data.Options.ConnectionId, data.Options.TeamId), - ) + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUE_HISTORY_TABLE, + }) + if err != nil { + return err + } + + // Only sweep issues updated since the last successful collection: an + // unchanged issue's history cannot have changed, so re-fetching it every + // run wastes a request per issue against Linear's hourly budget. + since := apiCollector.GetSince() + cursor, err := db.Cursor(issuesToCollectChildrenClauses(data.Options.ConnectionId, data.Options.TeamId, since)...) if err != nil { return err } @@ -86,15 +96,7 @@ func CollectIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { return err } - collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ - RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ - Ctx: taskCtx, - Params: LinearApiParams{ - ConnectionId: data.Options.ConnectionId, - TeamId: data.Options.TeamId, - }, - Table: RAW_ISSUE_HISTORY_TABLE, - }, + err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ GraphqlClient: data.GraphqlClient, Input: iterator, InputStep: 1, @@ -131,5 +133,5 @@ func CollectIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { return err } - return collector.Execute() + return apiCollector.Execute() } diff --git a/backend/plugins/linear/tasks/shared.go b/backend/plugins/linear/tasks/shared.go index 8cbdd96e571..0a8cfe9da7f 100644 --- a/backend/plugins/linear/tasks/shared.go +++ b/backend/plugins/linear/tasks/shared.go @@ -18,9 +18,30 @@ limitations under the License. package tasks import ( + "time" + + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/plugins/linear/models" ) +// issuesToCollectChildrenClauses builds the cursor clauses that drive per-issue +// child collection (comments, history). When `since` is non-nil (an incremental +// run), it restricts the sweep to issues updated since the last successful +// collection, so unchanged issues no longer trigger a request every run. On a +// full sync `since` is nil and all of the team's issues are swept. +func issuesToCollectChildrenClauses(connectionId uint64, teamId string, since *time.Time) []dal.Clause { + clauses := []dal.Clause{ + dal.Select("id"), + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, teamId), + } + if since != nil { + clauses = append(clauses, dal.Where("updated_at > ?", *since)) + } + return clauses +} + // priorityLabels maps Linear's integer priority to its human-readable label. // Linear: 0 = No priority, 1 = Urgent, 2 = High, 3 = Medium, 4 = Low. var priorityLabels = map[int]string{ diff --git a/backend/plugins/linear/tasks/shared_clauses_test.go b/backend/plugins/linear/tasks/shared_clauses_test.go new file mode 100644 index 00000000000..ad2fc7bc6d0 --- /dev/null +++ b/backend/plugins/linear/tasks/shared_clauses_test.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestIssuesToCollectChildrenClauses pins the incremental behaviour of the +// per-issue child collectors (comments, history): a full sync sweeps every +// issue, while an incremental run adds an updated_at filter so unchanged issues +// are skipped instead of triggering a request each run. +func TestIssuesToCollectChildrenClauses(t *testing.T) { + // full sync: no `since` -> select/from/where(connection,team) only + full := issuesToCollectChildrenClauses(1, "team-1", nil) + assert.Len(t, full, 3) + + // incremental: a `since` adds the updated_at filter clause + since := time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC) + incremental := issuesToCollectChildrenClauses(1, "team-1", &since) + assert.Len(t, incremental, 4) +} From 98fd01052ea7b12ccf2e6513021c451a63d8a175 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 16:41:55 +1000 Subject: [PATCH 21/28] fix(linear): filter issues server-side by updatedAt for incremental sync Incremental collection relied on the issues query returning newest-first and a client-side early-stop, but the query pinned no sort direction (Linear's orderBy is a scalar enum with no direction operand). If the server default were ascending, the early-stop would fire on the first (oldest) row and collect almost nothing. Pass a server-side IssueFilter { updatedAt: { gt: since } } instead and drop the early-stop, so correctness no longer depends on an undocumented default ordering. A full sync passes an empty filter (match all). Adds a unit test pinning the filter's JSON shape to Linear's IssueFilter input. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- .../plugins/linear/tasks/issue_collector.go | 38 ++++++++++++++--- .../linear/tasks/issue_collector_test.go | 41 +++++++++++++++++++ 2 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 backend/plugins/linear/tasks/issue_collector_test.go diff --git a/backend/plugins/linear/tasks/issue_collector.go b/backend/plugins/linear/tasks/issue_collector.go index 484e45b98ed..9f5a0aedd1d 100644 --- a/backend/plugins/linear/tasks/issue_collector.go +++ b/backend/plugins/linear/tasks/issue_collector.go @@ -31,17 +31,42 @@ import ( const RAW_ISSUES_TABLE = "linear_issues" // GraphqlQueryIssueWrapper is the team-scoped, paginated `issues` query. -// Issues are ordered by updatedAt (descending) so incremental runs can stop -// once they reach data older than the previous collection. +// Incremental runs filter server-side on updatedAt ($filter) rather than +// relying on result ordering, so collection no longer depends on an undocumented +// default sort direction. type GraphqlQueryIssueWrapper struct { Team struct { Issues struct { Nodes []GraphqlQueryIssue `graphql:"nodes"` PageInfo *helper.GraphqlQueryPageInfo - } `graphql:"issues(first: $pageSize, after: $skipCursor, orderBy: updatedAt)"` + } `graphql:"issues(first: $pageSize, after: $skipCursor, orderBy: updatedAt, filter: $filter)"` } `graphql:"team(id: $teamId)"` } +// IssueFilter mirrors the subset of Linear's GraphQL IssueFilter input used to +// restrict collection to issues updated after a point in time. The Go type +// name is significant: the GraphQL client emits it as the variable's type +// ($filter:IssueFilter!). +type IssueFilter struct { + UpdatedAt *DateComparator `json:"updatedAt,omitempty"` +} + +// DateComparator mirrors Linear's DateComparator input (only the `gt` operator +// is needed here). +type DateComparator struct { + Gt *time.Time `json:"gt,omitempty"` +} + +// buildIssueFilter returns an IssueFilter restricting to issues updated after +// `since`. When `since` is nil (a full sync) it returns the empty filter, which +// Linear treats as "match all". +func buildIssueFilter(since *time.Time) IssueFilter { + if since == nil { + return IssueFilter{} + } + return IssueFilter{UpdatedAt: &DateComparator{Gt: since}} +} + type GraphqlQueryIssue struct { Id string Identifier string @@ -110,6 +135,7 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { "pageSize": graphql.Int(reqData.Pager.Size), "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), "teamId": graphql.String(data.Options.TeamId), + "filter": buildIssueFilter(since), } return query, variables, nil }, @@ -119,13 +145,13 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { }, ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { query := queryWrapper.(*GraphqlQueryIssueWrapper) + // The server-side $filter already restricts to issues updated after + // `since`, so every returned issue is in scope -- no client-side + // early-stop (and thus no dependency on sort direction) is needed. for _, issue := range query.Team.Issues.Nodes { issue.CompletedAt = utils.NilIfZeroTime(issue.CompletedAt) issue.CanceledAt = utils.NilIfZeroTime(issue.CanceledAt) issue.StartedAt = utils.NilIfZeroTime(issue.StartedAt) - if since != nil && since.After(issue.UpdatedAt) { - return messages, helper.ErrFinishCollect - } messages = append(messages, errors.Must1(json.Marshal(issue))) } return diff --git a/backend/plugins/linear/tasks/issue_collector_test.go b/backend/plugins/linear/tasks/issue_collector_test.go new file mode 100644 index 00000000000..2dbbeecf586 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_collector_test.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestBuildIssueFilter pins the server-side incremental filter that replaces +// the previous reliance on result ordering plus a client-side early-stop. A +// full sync must produce an empty filter (match all); an incremental run must +// produce Linear's IssueFilter shape `{ updatedAt: { gt: } }`. +func TestBuildIssueFilter(t *testing.T) { + full, err := json.Marshal(buildIssueFilter(nil)) + assert.NoError(t, err) + assert.Equal(t, "{}", string(full)) + + since := time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC) + incremental, err := json.Marshal(buildIssueFilter(&since)) + assert.NoError(t, err) + assert.JSONEq(t, `{"updatedAt":{"gt":"2026-05-01T00:00:00Z"}}`, string(incremental)) +} From f76d398efa56101214e61e13c00edef570332d56 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 29 May 2026 17:08:31 +1000 Subject: [PATCH 22/28] refactor(linear): drop dead LeadTimeMinutes tool-layer field The _tool_linear_issues.lead_time_minutes column was never populated (the collector never requested it and no extractor set it). Now that lead time is derived into the domain ticket.Issue directly -- from state-transition history when available, otherwise the createdAt->resolutionDate fallback in the issue convertor -- the tool-layer field is pure dead weight. Remove it from the model, the init migration's archived model, the convertor, and the extractor snapshot. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- .../snapshot_tables/_tool_linear_issues.csv | 12 ++--- backend/plugins/linear/models/issue.go | 47 +++++++++---------- .../migrationscripts/archived/models.go | 47 +++++++++---------- .../plugins/linear/tasks/issue_convertor.go | 25 +++++----- 4 files changed, 64 insertions(+), 67 deletions(-) diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv index cf9462cb4b7..7c35d53d39e 100644 --- a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv @@ -1,6 +1,6 @@ -connection_id,id,team_id,identifier,number,title,description,url,priority,priority_label,estimate,state_id,state_name,state_type,creator_id,assignee_id,cycle_id,parent_id,lead_time_minutes,started_at,completed_at,canceled_at -1,issue-1,team-1,ENG-1,1,Fix login bug,Users cannot log in,https://linear.app/eng/issue/ENG-1,1,Urgent,3,state-done,Done,completed,user-2,user-1,cycle-1,,,2026-05-02T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00, -1,issue-2,team-1,ENG-2,2,Add dark mode,Theme support,https://linear.app/eng/issue/ENG-2,2,High,5,state-inprogress,In Progress,started,user-1,,,,,2026-05-02T00:00:00.000+00:00,, -1,issue-3,team-1,ENG-3,3,Investigate flakiness,,https://linear.app/eng/issue/ENG-3,0,No priority,,state-backlog,Backlog,backlog,user-1,,,,,,, -1,issue-4,team-1,ENG-4,4,Deprecated feature,No longer needed,https://linear.app/eng/issue/ENG-4,3,Medium,2,state-canceled,Canceled,canceled,user-1,user-3,cycle-1,,,,,2026-05-02T00:00:00.000+00:00 -1,issue-5,team-1,ENG-5,5,Write docs,User guide,https://linear.app/eng/issue/ENG-5,4,Low,1,state-todo,Todo,unstarted,user-2,,,,,,, +connection_id,id,team_id,identifier,number,title,description,url,priority,priority_label,estimate,state_id,state_name,state_type,creator_id,assignee_id,cycle_id,parent_id,started_at,completed_at,canceled_at +1,issue-1,team-1,ENG-1,1,Fix login bug,Users cannot log in,https://linear.app/eng/issue/ENG-1,1,Urgent,3,state-done,Done,completed,user-2,user-1,cycle-1,,2026-05-02T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00, +1,issue-2,team-1,ENG-2,2,Add dark mode,Theme support,https://linear.app/eng/issue/ENG-2,2,High,5,state-inprogress,In Progress,started,user-1,,,,2026-05-02T00:00:00.000+00:00,, +1,issue-3,team-1,ENG-3,3,Investigate flakiness,,https://linear.app/eng/issue/ENG-3,0,No priority,,state-backlog,Backlog,backlog,user-1,,,,,, +1,issue-4,team-1,ENG-4,4,Deprecated feature,No longer needed,https://linear.app/eng/issue/ENG-4,3,Medium,2,state-canceled,Canceled,canceled,user-1,user-3,cycle-1,,,,2026-05-02T00:00:00.000+00:00 +1,issue-5,team-1,ENG-5,5,Write docs,User guide,https://linear.app/eng/issue/ENG-5,4,Low,1,state-todo,Todo,unstarted,user-2,,,,,, diff --git a/backend/plugins/linear/models/issue.go b/backend/plugins/linear/models/issue.go index b3f8a15206e..eb6ef82aa86 100644 --- a/backend/plugins/linear/models/issue.go +++ b/backend/plugins/linear/models/issue.go @@ -25,30 +25,29 @@ import ( // LinearIssue is the tool-layer representation of a Linear issue. type LinearIssue struct { - ConnectionId uint64 `gorm:"primaryKey"` - Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` - TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` - Identifier string `gorm:"type:varchar(255)" json:"identifier"` - Number int `json:"number"` - Title string `gorm:"type:varchar(255)" json:"title"` - Description string `json:"description"` - Url string `gorm:"type:varchar(255)" json:"url"` - Priority int `json:"priority"` - PriorityLabel string `gorm:"type:varchar(100)" json:"priorityLabel"` - Estimate *float64 `json:"estimate"` - StateId string `gorm:"index;type:varchar(255)" json:"stateId"` - StateName string `gorm:"type:varchar(255)" json:"stateName"` - StateType string `gorm:"type:varchar(100)" json:"stateType"` - CreatorId string `gorm:"type:varchar(255)" json:"creatorId"` - AssigneeId string `gorm:"type:varchar(255)" json:"assigneeId"` - CycleId string `gorm:"index;type:varchar(255)" json:"cycleId"` - ParentId string `gorm:"type:varchar(255)" json:"parentId"` - LeadTimeMinutes *uint `json:"leadTimeMinutes"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `gorm:"index" json:"updatedAt"` - StartedAt *time.Time `json:"startedAt"` - CompletedAt *time.Time `json:"completedAt"` - CanceledAt *time.Time `json:"canceledAt"` + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` + Identifier string `gorm:"type:varchar(255)" json:"identifier"` + Number int `json:"number"` + Title string `gorm:"type:varchar(255)" json:"title"` + Description string `json:"description"` + Url string `gorm:"type:varchar(255)" json:"url"` + Priority int `json:"priority"` + PriorityLabel string `gorm:"type:varchar(100)" json:"priorityLabel"` + Estimate *float64 `json:"estimate"` + StateId string `gorm:"index;type:varchar(255)" json:"stateId"` + StateName string `gorm:"type:varchar(255)" json:"stateName"` + StateType string `gorm:"type:varchar(100)" json:"stateType"` + CreatorId string `gorm:"type:varchar(255)" json:"creatorId"` + AssigneeId string `gorm:"type:varchar(255)" json:"assigneeId"` + CycleId string `gorm:"index;type:varchar(255)" json:"cycleId"` + ParentId string `gorm:"type:varchar(255)" json:"parentId"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `gorm:"index" json:"updatedAt"` + StartedAt *time.Time `json:"startedAt"` + CompletedAt *time.Time `json:"completedAt"` + CanceledAt *time.Time `json:"canceledAt"` common.NoPKModel } diff --git a/backend/plugins/linear/models/migrationscripts/archived/models.go b/backend/plugins/linear/models/migrationscripts/archived/models.go index 21514765d4b..239db7123e8 100644 --- a/backend/plugins/linear/models/migrationscripts/archived/models.go +++ b/backend/plugins/linear/models/migrationscripts/archived/models.go @@ -71,30 +71,29 @@ type LinearAccount struct { func (LinearAccount) TableName() string { return "_tool_linear_accounts" } type LinearIssue struct { - ConnectionId uint64 `gorm:"primaryKey"` - Id string `gorm:"primaryKey;type:varchar(255)"` - TeamId string `gorm:"index;type:varchar(255)"` - Identifier string `gorm:"type:varchar(255)"` - Number int - Title string `gorm:"type:varchar(255)"` - Description string - Url string `gorm:"type:varchar(255)"` - Priority int - PriorityLabel string `gorm:"type:varchar(100)"` - Estimate *float64 - StateId string `gorm:"index;type:varchar(255)"` - StateName string `gorm:"type:varchar(255)"` - StateType string `gorm:"type:varchar(100)"` - CreatorId string `gorm:"type:varchar(255)"` - AssigneeId string `gorm:"type:varchar(255)"` - CycleId string `gorm:"index;type:varchar(255)"` - ParentId string `gorm:"type:varchar(255)"` - LeadTimeMinutes *uint - CreatedAt time.Time - UpdatedAt time.Time `gorm:"index"` - StartedAt *time.Time - CompletedAt *time.Time - CanceledAt *time.Time + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"index;type:varchar(255)"` + Identifier string `gorm:"type:varchar(255)"` + Number int + Title string `gorm:"type:varchar(255)"` + Description string + Url string `gorm:"type:varchar(255)"` + Priority int + PriorityLabel string `gorm:"type:varchar(100)"` + Estimate *float64 + StateId string `gorm:"index;type:varchar(255)"` + StateName string `gorm:"type:varchar(255)"` + StateType string `gorm:"type:varchar(100)"` + CreatorId string `gorm:"type:varchar(255)"` + AssigneeId string `gorm:"type:varchar(255)"` + CycleId string `gorm:"index;type:varchar(255)"` + ParentId string `gorm:"type:varchar(255)"` + CreatedAt time.Time + UpdatedAt time.Time `gorm:"index"` + StartedAt *time.Time + CompletedAt *time.Time + CanceledAt *time.Time archived.NoPKModel } diff --git a/backend/plugins/linear/tasks/issue_convertor.go b/backend/plugins/linear/tasks/issue_convertor.go index 46a779b80af..2e947700d6a 100644 --- a/backend/plugins/linear/tasks/issue_convertor.go +++ b/backend/plugins/linear/tasks/issue_convertor.go @@ -91,19 +91,18 @@ func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error { Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { issue := inputRow.(*models.LinearIssue) domainIssue := &ticket.Issue{ - DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(connectionId, issue.Id)}, - IssueKey: issue.Identifier, - Title: issue.Title, - Description: issue.Description, - Url: issue.Url, - Type: ticket.REQUIREMENT, - Status: StatusFromStateType(issue.StateType), - OriginalStatus: issue.StateName, - StoryPoint: issue.Estimate, - Priority: issue.PriorityLabel, - LeadTimeMinutes: issue.LeadTimeMinutes, - CreatedDate: &issue.CreatedAt, - UpdatedDate: &issue.UpdatedAt, + DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(connectionId, issue.Id)}, + IssueKey: issue.Identifier, + Title: issue.Title, + Description: issue.Description, + Url: issue.Url, + Type: ticket.REQUIREMENT, + Status: StatusFromStateType(issue.StateType), + OriginalStatus: issue.StateName, + StoryPoint: issue.Estimate, + Priority: issue.PriorityLabel, + CreatedDate: &issue.CreatedAt, + UpdatedDate: &issue.UpdatedAt, } if issue.CreatorId != "" { domainIssue.CreatorId = accountIdGen.Generate(connectionId, issue.CreatorId) From ad14037c3fa40d4aca60774d6aebe42a836a25de Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Mon, 1 Jun 2026 11:14:54 +1000 Subject: [PATCH 23/28] feat(config-ui): register Linear plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the Linear data source to config-ui so it appears in the connection picker: connection form (endpoint + personal API key + proxy + rate limit), a flat Teams data-scope backed by the plugin's remote-scopes endpoint, and the Linear logo. No scope-config transformation — Linear's status mapping is deterministic. Wired into the plugin registry. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- config-ui/src/plugins/register/index.ts | 2 + .../plugins/register/linear/assets/icon.svg | 19 +++++++ .../src/plugins/register/linear/config.tsx | 57 +++++++++++++++++++ .../src/plugins/register/linear/index.ts | 19 +++++++ 4 files changed, 97 insertions(+) create mode 100644 config-ui/src/plugins/register/linear/assets/icon.svg create mode 100644 config-ui/src/plugins/register/linear/config.tsx create mode 100644 config-ui/src/plugins/register/linear/index.ts diff --git a/config-ui/src/plugins/register/index.ts b/config-ui/src/plugins/register/index.ts index fdc82a8778c..4cbc1b52d46 100644 --- a/config-ui/src/plugins/register/index.ts +++ b/config-ui/src/plugins/register/index.ts @@ -30,6 +30,7 @@ import { GhCopilotConfig } from './gh-copilot'; import { GitLabConfig } from './gitlab'; import { JenkinsConfig } from './jenkins'; import { JiraConfig } from './jira'; +import { LinearConfig } from './linear'; import { PagerDutyConfig } from './pagerduty'; import { RootlyConfig } from './rootly'; import { SonarQubeConfig } from './sonarqube'; @@ -56,6 +57,7 @@ export const pluginConfigs: IPluginConfig[] = [ GitLabConfig, JenkinsConfig, JiraConfig, + LinearConfig, PagerDutyConfig, RootlyConfig, SlackConfig, diff --git a/config-ui/src/plugins/register/linear/assets/icon.svg b/config-ui/src/plugins/register/linear/assets/icon.svg new file mode 100644 index 00000000000..64cddfdea54 --- /dev/null +++ b/config-ui/src/plugins/register/linear/assets/icon.svg @@ -0,0 +1,19 @@ + + + + diff --git a/config-ui/src/plugins/register/linear/config.tsx b/config-ui/src/plugins/register/linear/config.tsx new file mode 100644 index 00000000000..28cf9f98a95 --- /dev/null +++ b/config-ui/src/plugins/register/linear/config.tsx @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.svg?react'; + +export const LinearConfig: IPluginConfig = { + plugin: 'linear', + name: 'Linear', + icon: ({ color }) => , + sort: 13, + connection: { + docLink: 'https://developers.linear.app/docs', + initialValues: { + endpoint: 'https://api.linear.app/graphql', + }, + fields: [ + 'name', + { + key: 'endpoint', + label: 'Endpoint', + subLabel: 'Linear GraphQL API base URL.', + }, + { + key: 'token', + label: 'API Key', + subLabel: 'Your Linear personal API key (Settings → Security & access → Personal API keys).', + }, + 'proxy', + { + key: 'rateLimitPerHour', + subLabel: 'Maximum number of API requests per hour. Leave blank for the default (1500).', + defaultValue: 1500, + }, + ], + }, + dataScope: { + title: 'Teams', + searchPlaceholder: 'Search teams...', + }, +}; diff --git a/config-ui/src/plugins/register/linear/index.ts b/config-ui/src/plugins/register/linear/index.ts new file mode 100644 index 00000000000..de415db39ab --- /dev/null +++ b/config-ui/src/plugins/register/linear/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; From 64fbd189c35a85e5021135a0853b354aaf9689b6 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:18:37 +1000 Subject: [PATCH 24/28] fix(config-ui): map Linear scope id to teamId MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getPluginScopeId fell through to the default (scope.id) for Linear, but a LinearTeam scope is keyed by teamId and has no id field — so the blueprint referenced an undefined scopeId and patching failed with 'LinearTeam not found'. Add a linear case returning scope.teamId. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- config-ui/src/plugins/utils.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config-ui/src/plugins/utils.ts b/config-ui/src/plugins/utils.ts index 88b11b55576..571b536fe9a 100644 --- a/config-ui/src/plugins/utils.ts +++ b/config-ui/src/plugins/utils.ts @@ -45,6 +45,8 @@ export const getPluginScopeId = (plugin: string, scope: any) => { return `${scope.name}`; case 'asana': return `${scope.gid}`; + case 'linear': + return `${scope.teamId}`; default: return `${scope.id}`; } From 4095d87277078975eae8f7e1c253f24c017037e7 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:29:46 +1000 Subject: [PATCH 25/28] feat(linear): add Grafana dashboard Adds grafana/dashboards/Linear.json (cloned from the Asana ticket-dashboard template) so Linear ships a per-tool dashboard like every other ticket plugin. Its board picker is scoped to Linear (boards id like 'linear%'); the 13 panels (throughput, lead/cycle time, status distribution, delivery rate, sprints) read the shared domain tables. Auto-loaded via Grafana file provisioning. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- grafana/dashboards/Linear.json | 1192 ++++++++++++++++++++++++++++++++ 1 file changed, 1192 insertions(+) create mode 100644 grafana/dashboards/Linear.json diff --git a/grafana/dashboards/Linear.json b/grafana/dashboards/Linear.json new file mode 100644 index 00000000000..1e748264206 --- /dev/null +++ b/grafana/dashboards/Linear.json @@ -0,0 +1,1192 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "Linear", + "url": "https://devlake.apache.org/docs/Configuration/Linear" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic project management metrics from Linear.\n- Data Source Required: Linear", + "mode": "markdown" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": "mysql", + "description": "Total number of issues created in the selected time range and board.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Delivered Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 120, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n count(distinct case when status != 'DONE' then i.id else null end) as \"Number of Open Issues\",\r\n count(distinct case when status = 'DONE' then i.id else null end) as \"Number of Delivered Issues\"\r\nFROM issues i\r\n\tjoin board_issues bi on i.id = bi.issue_id\r\n\tjoin boards b on bi.board_id = b.id\r\nwhere \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\ngroup by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n count(distinct i.id) as total_count,\r\n count(distinct case when i.status = 'DONE' then i.id else null end) as delivered_count\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect \r\n now() as time,\r\n 1.0 * delivered_count/total_count as requirement_delivery_rate\r\nfrom _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n 1.0 * count(distinct case when i.status = 'DONE' then i.id else null end)/count(distinct i.id) as delivered_rate\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect\r\n time,\r\n delivered_rate\r\nfrom _requirements\r\norder by time", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 14 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select \r\n avg(lead_time_minutes/1440) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time in Days [Issues Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 21 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n i.lead_time_minutes,\r\n percent_rank() over (order by lead_time_minutes asc) as ranks\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect\r\n max(lead_time_minutes/1440) as value\r\nfrom _ranks\r\nwhere \r\n ranks <= 0.8", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Issues' Lead Time are less than # days [Issues Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time(days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select \r\n DATE_ADD(date(i.resolution_date), INTERVAL -DAYOFMONTH(date(i.resolution_date))+1 DAY) as time,\r\n avg(lead_time_minutes/1440) as mean_lead_time\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect \r\n date_format(time,'%M %Y') as month,\r\n mean_lead_time\r\nfrom _requirements\r\norder by time asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Selected Time Range]", + "type": "barchart" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "mysql", + "description": "The cumulative distribution of issue lead time. Each point refers to the percent rank of a lead time.", + "fill": 0, + "fillGradient": 4, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 8, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "9.5.15", + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n round(i.lead_time_minutes/1440) as lead_time_day\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n order by lead_time_day asc\r\n)\r\n\r\nselect \r\n now() as time,\r\n lpad(concat(lead_time_day,'d'), 4, ' ') as metric,\r\n percent_rank() over (order by lead_time_day asc) as value\r\nfrom _ranks\r\norder by lead_time_day asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [ + { + "colorMode": "ok", + "fill": true, + "line": true, + "op": "lt", + "value": 0.8, + "yaxis": "right" + } + ], + "timeRegions": [], + "title": "Cumulative Distribution of Issue Lead Time [Issues Resolved in Selected Time Range]", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "series", + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "Percent Rank (%)", + "logBase": 1, + "max": "1.2", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "mysql", + "definition": "select concat(name, '--', id) from boards where id like 'linear%'", + "hide": 0, + "includeAll": true, + "label": "Choose Board", + "multi": true, + "name": "board_id", + "options": [], + "query": "select concat(name, '--', id) from boards where id like 'linear%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "mysql", + "definition": "select distinct type from issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "select distinct type from issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Linear", + "uid": "linear-dashboard", + "version": 1, + "weekStart": "" +} From adfbd25b8d16c88c716395ff54169558e749841f Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:39:04 +1000 Subject: [PATCH 26/28] fix(linear): convert team scope to a domain board board_issues and sprint_issues referenced a board_id (boardIdGen over LinearTeam), but nothing ever created the ticket.Board row itself, so the domain boards table stayed empty. Board-scoped dashboards (whose board picker is 'boards where id like linear%') and any board join therefore returned no data. Add a ConvertTeams subtask that converts the team scope in _tool_linear_teams into a ticket.Board keyed identically to those references. Adds an e2e test asserting the board is produced with the matching id. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- backend/plugins/linear/e2e/board_test.go | 56 +++++++++++ .../snapshot_tables/_tool_linear_teams.csv | 2 + .../linear/e2e/snapshot_tables/boards.csv | 2 + backend/plugins/linear/impl/impl.go | 1 + .../plugins/linear/tasks/board_convertor.go | 93 +++++++++++++++++++ 5 files changed, 154 insertions(+) create mode 100644 backend/plugins/linear/e2e/board_test.go create mode 100644 backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/boards.csv create mode 100644 backend/plugins/linear/tasks/board_convertor.go diff --git a/backend/plugins/linear/e2e/board_test.go b/backend/plugins/linear/e2e/board_test.go new file mode 100644 index 00000000000..eb4e0487323 --- /dev/null +++ b/backend/plugins/linear/e2e/board_test.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearBoardDataFlow verifies that a Linear team scope is converted into a +// domain ticket.Board, keyed identically to the board_id that board_issues +// already reference (boardIdGen over LinearTeam). Without this, the boards table +// is empty and board-scoped dashboards return no data. +func TestLinearBoardDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // the team scope lives in _tool_linear_teams (populated via the scope API) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_linear_teams.csv", &models.LinearTeam{}) + + // convert: team scope -> domain board + dataflowTester.FlushTabler(&ticket.Board{}) + dataflowTester.Subtask(tasks.ConvertTeamsMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Board{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/boards.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv new file mode 100644 index 00000000000..1541f9c95e4 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv @@ -0,0 +1,2 @@ +connection_id,scope_config_id,team_id,name,key,description +1,0,team-1,Engineering,ENG,Core engineering team diff --git a/backend/plugins/linear/e2e/snapshot_tables/boards.csv b/backend/plugins/linear/e2e/snapshot_tables/boards.csv new file mode 100644 index 00000000000..4548d25c385 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/boards.csv @@ -0,0 +1,2 @@ +id,name,description,url,created_date,type +linear:LinearTeam:1:team-1,Engineering,Core engineering team,,,linear diff --git a/backend/plugins/linear/impl/impl.go b/backend/plugins/linear/impl/impl.go index 7b85c388620..bc64218ff61 100644 --- a/backend/plugins/linear/impl/impl.go +++ b/backend/plugins/linear/impl/impl.go @@ -109,6 +109,7 @@ func (p Linear) SubTaskMetas() []plugin.SubTaskMeta { tasks.ExtractCyclesMeta, tasks.CollectIssueHistoryMeta, tasks.ExtractIssueHistoryMeta, + tasks.ConvertTeamsMeta, tasks.ConvertAccountsMeta, tasks.ConvertIssuesMeta, tasks.ConvertIssueLabelsMeta, diff --git a/backend/plugins/linear/tasks/board_convertor.go b/backend/plugins/linear/tasks/board_convertor.go new file mode 100644 index 00000000000..9366efed2ab --- /dev/null +++ b/backend/plugins/linear/tasks/board_convertor.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// RAW_TEAMS_TABLE labels the raw-data lineage for the team-scope-derived board. +// Teams are added as scopes (no collector), so this is a logical tag only. +const RAW_TEAMS_TABLE = "linear_teams" + +var ConvertTeamsMeta = plugin.SubTaskMeta{ + Name: "Convert Teams", + EntryPoint: ConvertTeams, + EnabledByDefault: true, + Description: "Convert the Linear team scope (_tool_linear_teams) into the domain layer table boards", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearTeam{}.TableName()}, + ProductTables: []string{ticket.Board{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertTeams + +func ConvertTeams(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + // boardId must be generated identically to the issue/sprint convertors so the + // board joins to the board_issues/sprint_issues that reference it. + boardIdGen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + + cursor, err := db.Cursor( + dal.From(&models.LinearTeam{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_TEAMS_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearTeam{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + team := inputRow.(*models.LinearTeam) + board := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{Id: boardIdGen.Generate(connectionId, team.TeamId)}, + Name: team.Name, + Description: team.Description, + Type: "linear", + } + return []interface{}{board}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} From da340e4d5bbc02f02bf56443b4a40134ed0566ae Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:57:49 +1000 Subject: [PATCH 27/28] fix(linear): widen issue title/url columns to avoid truncation _tool_linear_issues.title and .url were varchar(255), but Linear titles can exceed 255 chars (and the issue URL embeds a title slug), so extraction failed with 'Error 1406: Data too long for column title'. Drop the varchar limit so both are longtext, matching the domain issues.title and jira's tool summary. Adds an e2e test extracting a 300-char title without truncation. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- .../linear/e2e/issue_longtitle_test.go | 55 +++++++++++++++++++ .../_raw_linear_issues_long_title.csv | 2 + backend/plugins/linear/models/issue.go | 4 +- .../migrationscripts/archived/models.go | 4 +- 4 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 backend/plugins/linear/e2e/issue_longtitle_test.go create mode 100644 backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv diff --git a/backend/plugins/linear/e2e/issue_longtitle_test.go b/backend/plugins/linear/e2e/issue_longtitle_test.go new file mode 100644 index 00000000000..f85062cdb8b --- /dev/null +++ b/backend/plugins/linear/e2e/issue_longtitle_test.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" + "github.com/stretchr/testify/assert" +) + +// TestLinearIssueLongTitle guards against truncation/insert failure for long +// issue titles and URLs. Linear titles can exceed 255 chars (and the issue URL +// embeds a title slug), which overflowed the old varchar(255) columns and +// failed extraction with "Data too long for column 'title'". The columns are +// now untyped (longtext), matching the domain issues.title and jira's tool +// summary. +func TestLinearIssueLongTitle(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ConnectionId: 1, TeamId: "team-1"}, + } + + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues_long_title.csv", "_raw_linear_issues") + // must not error with "Data too long for column 'title'" + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + var issue models.LinearIssue + err := dataflowTester.Dal.First(&issue, dal.Where("connection_id = ? AND id = ?", 1, "issue-longtitle")) + assert.NoError(t, err) + assert.Len(t, issue.Title, 300, "full 300-char title must be stored untruncated") +} diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv new file mode 100644 index 00000000000..4842c33a61c --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-longtitle"",""Identifier"":""ENG-LONG"",""Number"":900,""Title"":""TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT"",""Description"":""long title row"",""Url"":""https://linear.app/eng/issue/ENG-LONG/tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt"",""Priority"":0,""Estimate"":null,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-01T00:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""Assignee"":null,""Creator"":null,""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 diff --git a/backend/plugins/linear/models/issue.go b/backend/plugins/linear/models/issue.go index eb6ef82aa86..991585de956 100644 --- a/backend/plugins/linear/models/issue.go +++ b/backend/plugins/linear/models/issue.go @@ -30,9 +30,9 @@ type LinearIssue struct { TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` Identifier string `gorm:"type:varchar(255)" json:"identifier"` Number int `json:"number"` - Title string `gorm:"type:varchar(255)" json:"title"` + Title string `json:"title"` Description string `json:"description"` - Url string `gorm:"type:varchar(255)" json:"url"` + Url string `json:"url"` Priority int `json:"priority"` PriorityLabel string `gorm:"type:varchar(100)" json:"priorityLabel"` Estimate *float64 `json:"estimate"` diff --git a/backend/plugins/linear/models/migrationscripts/archived/models.go b/backend/plugins/linear/models/migrationscripts/archived/models.go index 239db7123e8..1c28035387a 100644 --- a/backend/plugins/linear/models/migrationscripts/archived/models.go +++ b/backend/plugins/linear/models/migrationscripts/archived/models.go @@ -76,9 +76,9 @@ type LinearIssue struct { TeamId string `gorm:"index;type:varchar(255)"` Identifier string `gorm:"type:varchar(255)"` Number int - Title string `gorm:"type:varchar(255)"` + Title string Description string - Url string `gorm:"type:varchar(255)"` + Url string Priority int PriorityLabel string `gorm:"type:varchar(100)"` Estimate *float64 From 55dbfefb4a43c1bd7afcd2ecc29cb29064bb5aa5 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Wed, 3 Jun 2026 14:52:47 +1000 Subject: [PATCH 28/28] fix(linear): recover owning issue id for comments and history The GraphQL collector stores the query variables (which carry issueId) in the raw row's input column, but the comment and history extractors parsed it as {"Id":...} (SimpleLinearIssue.Id), so the owning issue id came out empty and the convertor joins produced zero domain comments/changelogs on real data. The e2e fixtures hand-wrote {"Id":...}, masking it. Parse issueId (with an Id fallback) and update the fixtures to the real collector shape. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --- .../e2e/raw_tables/_raw_linear_comments.csv | 6 +++--- .../raw_tables/_raw_linear_issue_history.csv | 6 +++--- .../plugins/linear/tasks/comment_collector.go | 18 +++++++++++++++++- .../plugins/linear/tasks/comment_extractor.go | 2 +- .../linear/tasks/issue_history_extractor.go | 2 +- 5 files changed, 25 insertions(+), 9 deletions(-) diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv index bb8552054c7..6d6fc21e6a8 100644 --- a/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv @@ -1,4 +1,4 @@ id,params,data,url,input,created_at -1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-1"",""Body"":""Looking into this"",""CreatedAt"":""2026-05-02T10:00:00Z"",""UpdatedAt"":""2026-05-02T10:00:00Z"",""User"":{""Id"":""user-2""}}",https://api.linear.app/graphql,"{""Id"":""issue-1""}",2026-05-02 10:00:00.000 -2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-2"",""Body"":""Fixed in PR 42"",""CreatedAt"":""2026-05-03T09:00:00Z"",""UpdatedAt"":""2026-05-03T09:30:00Z"",""User"":{""Id"":""user-1""}}",https://api.linear.app/graphql,"{""Id"":""issue-1""}",2026-05-03 09:00:00.000 -3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-3"",""Body"":""Any update?"",""CreatedAt"":""2026-05-02T11:00:00Z"",""UpdatedAt"":""2026-05-02T11:00:00Z"",""User"":{""Id"":""user-1""}}",https://api.linear.app/graphql,"{""Id"":""issue-2""}",2026-05-02 11:00:00.000 +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-1"",""Body"":""Looking into this"",""CreatedAt"":""2026-05-02T10:00:00Z"",""UpdatedAt"":""2026-05-02T10:00:00Z"",""User"":{""Id"":""user-2""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-02 10:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-2"",""Body"":""Fixed in PR 42"",""CreatedAt"":""2026-05-03T09:00:00Z"",""UpdatedAt"":""2026-05-03T09:30:00Z"",""User"":{""Id"":""user-1""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-03 09:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-3"",""Body"":""Any update?"",""CreatedAt"":""2026-05-02T11:00:00Z"",""UpdatedAt"":""2026-05-02T11:00:00Z"",""User"":{""Id"":""user-1""}}",https://api.linear.app/graphql,"{""issueId"":""issue-2""}",2026-05-02 11:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv index 6862f10e67a..d1cef3c37ab 100644 --- a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv @@ -1,4 +1,4 @@ id,params,data,url,input,created_at -1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-1"",""CreatedAt"":""2026-05-01T08:00:00Z"",""Actor"":{""Id"":""user-2""},""FromState"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""ToState"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""}}",https://api.linear.app/graphql,"{""Id"":""issue-1""}",2026-05-01 08:00:00.000 -2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-2"",""CreatedAt"":""2026-05-02T00:00:00Z"",""Actor"":{""Id"":""user-1""},""FromState"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""},""ToState"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""}}",https://api.linear.app/graphql,"{""Id"":""issue-1""}",2026-05-02 00:00:00.000 -3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-3"",""CreatedAt"":""2026-05-03T00:00:00Z"",""Actor"":{""Id"":""user-1""},""FromState"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""},""ToState"":{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed""}}",https://api.linear.app/graphql,"{""Id"":""issue-1""}",2026-05-03 00:00:00.000 +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-1"",""CreatedAt"":""2026-05-01T08:00:00Z"",""Actor"":{""Id"":""user-2""},""FromState"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""ToState"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-01 08:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-2"",""CreatedAt"":""2026-05-02T00:00:00Z"",""Actor"":{""Id"":""user-1""},""FromState"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""},""ToState"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-02 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-3"",""CreatedAt"":""2026-05-03T00:00:00Z"",""Actor"":{""Id"":""user-1""},""FromState"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""},""ToState"":{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-03 00:00:00.000 diff --git a/backend/plugins/linear/tasks/comment_collector.go b/backend/plugins/linear/tasks/comment_collector.go index c23520d9b7f..14e801ff9e3 100644 --- a/backend/plugins/linear/tasks/comment_collector.go +++ b/backend/plugins/linear/tasks/comment_collector.go @@ -34,7 +34,23 @@ const RAW_COMMENTS_TABLE = "linear_comments" // of child resources (comments, history). Its JSON form is stored in the raw // row's `input` column so extractors can recover the owning issue id. type SimpleLinearIssue struct { - Id string + // Id is populated by the DalCursorIterator (the _tool_linear_issues.id column) + // when driving per-issue child collection. + Id string `json:"Id"` + // IssueId is populated when parsing a raw row's `input` column: the GraphQL + // collector stores the query variables there (which carry `issueId`), not the + // iterator element. OwningIssueId resolves whichever is present. + IssueId string `json:"issueId" gorm:"-"` +} + +// OwningIssueId returns the issue id this child row belongs to, tolerating both +// the iterator element shape ({"Id":...}) and the collector's stored variables +// shape ({"issueId":...}). +func (s SimpleLinearIssue) OwningIssueId() string { + if s.IssueId != "" { + return s.IssueId + } + return s.Id } // GraphqlQueryCommentWrapper is the per-issue, paginated `comments` query. diff --git a/backend/plugins/linear/tasks/comment_extractor.go b/backend/plugins/linear/tasks/comment_extractor.go index de0d6a9ae98..15b28a3b969 100644 --- a/backend/plugins/linear/tasks/comment_extractor.go +++ b/backend/plugins/linear/tasks/comment_extractor.go @@ -60,7 +60,7 @@ func ExtractComments(taskCtx plugin.SubTaskContext) errors.Error { comment := &models.LinearComment{ ConnectionId: data.Options.ConnectionId, Id: apiComment.Id, - IssueId: issueRef.Id, + IssueId: issueRef.OwningIssueId(), Body: apiComment.Body, CreatedAt: apiComment.CreatedAt, UpdatedAt: apiComment.UpdatedAt, diff --git a/backend/plugins/linear/tasks/issue_history_extractor.go b/backend/plugins/linear/tasks/issue_history_extractor.go index 243c7ea2379..006b92f9701 100644 --- a/backend/plugins/linear/tasks/issue_history_extractor.go +++ b/backend/plugins/linear/tasks/issue_history_extractor.go @@ -59,7 +59,7 @@ func ExtractIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { event := &models.LinearIssueHistory{ ConnectionId: data.Options.ConnectionId, Id: apiEvent.Id, - IssueId: issueRef.Id, + IssueId: issueRef.OwningIssueId(), CreatedAt: apiEvent.CreatedAt, } if apiEvent.Actor != nil {