From 2f94a247223a3259a9e25d6f38eb544ffd56bf3c Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Wed, 12 Aug 2026 16:37:56 +0100 Subject: [PATCH 01/14] fix: expand Gerrit channel URLs in health score v2 maintainer and development pipes (IN-1229) Signed-off-by: Joana Maia --- .../pipes/health_score_v2_development.pipe | 71 +++++++--- .../pipes/health_score_v2_maintainer.pipe | 124 ++++++++++++------ 2 files changed, 135 insertions(+), 60 deletions(-) diff --git a/services/libs/tinybird/pipes/health_score_v2_development.pipe b/services/libs/tinybird/pipes/health_score_v2_development.pipe index 3f79e90194..28520ae2d1 100644 --- a/services/libs/tinybird/pipes/health_score_v2_development.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_development.pipe @@ -10,6 +10,38 @@ DESCRIPTION > (most repos) having lastCommitAt from GitHub enrichment — the fallback covers the vast majority. - Split out of health_score_v2.pipe into its own copy pipe (2026-07-22) — see health_score_v2_maintainer.pipe description for why. + - Gerrit channel fix (IN-1229): activity rows for Gerrit repos ingested via opendev.org (and + other Gerrit hosts) are stored under the /q/project: channel variant, not the plain repo URL. + Added a `channels` node that expands each repo URL to all its possible channel forms — matching + the logic in repos_to_channels.pipe — so joins against activityRelations, pull_requests_analyzed, + and issues_analyzed correctly match Gerrit activity. + +NODE channels +DESCRIPTION > + Expand each repo URL to all possible activity channel forms. Mirrors repos_to_channels.pipe + expansion logic. Non-Gerrit repos: one row (channel = url). Gerrit repos: two rows — plain url + plus the /q/project: variant used by Gerrit activity ingestion. +SQL > + SELECT r.url AS repoUrl, r.url AS channel + FROM repositories r + WHERE r.deletedAt IS NULL + UNION ALL + SELECT + r.url AS repoUrl, + CASE + WHEN position(r.url, '/r/') > 0 + THEN replaceOne(r.url, '/r/', '/r/q/project:') + WHEN position(r.url, '/gerrit/') > 0 + THEN replaceOne(r.url, '/gerrit/', '/gerrit/q/project:') + ELSE + concat( + protocol(r.url), '://', domain(r.url), '/q/project:', + if(path(r.url) = '/', '', substring(path(r.url), 2)) + ) + END AS channel + FROM repositories r + INNER JOIN integrations i ON r.sourceIntegrationId = i.id + WHERE r.deletedAt IS NULL AND i.platform = 'gerrit' NODE health_score_v2_development_calc SQL > @@ -192,37 +224,40 @@ SQL > LEFT JOIN ( SELECT - channel AS repoUrl, - countIf(timestamp > now() - INTERVAL 6 MONTH) AS commitsLast6m - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE type = 'authored-commit' - GROUP BY channel + ch.repoUrl AS repoUrl, + countIf(ar.timestamp > now() - INTERVAL 6 MONTH) AS commitsLast6m + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN channels ch ON ar.channel = ch.channel + WHERE ar.type = 'authored-commit' + GROUP BY ch.repoUrl ) AS c ON c.repoUrl = rp.repoUrl LEFT JOIN ( SELECT - channel AS repoUrl, - countIf(closedAt IS NOT NULL) AS closed12m, + ch.repoUrl AS repoUrl, + countIf(ia.closedAt IS NOT NULL) AS closed12m, count() AS opened12m, quantileIf(0.5) - (closedInSeconds, closedAt > now() - INTERVAL 12 MONTH) AS medianCloseS - FROM issues_analyzed - WHERE openedAt > now() - INTERVAL 12 MONTH - GROUP BY channel + (ia.closedInSeconds, ia.closedAt > now() - INTERVAL 12 MONTH) AS medianCloseS + FROM issues_analyzed ia + INNER JOIN channels ch ON ia.channel = ch.channel + WHERE ia.openedAt > now() - INTERVAL 12 MONTH + GROUP BY ch.repoUrl ) AS ist ON ist.repoUrl = rp.repoUrl LEFT JOIN ( SELECT - channel AS repoUrl, - countIf(mergedAt IS NOT NULL) AS merged12m, - countIf(closedAt IS NOT NULL AND mergedAt IS NULL) AS closedUnmerged12m, + ch.repoUrl AS repoUrl, + countIf(pra.mergedAt IS NOT NULL) AS merged12m, + countIf(pra.closedAt IS NOT NULL AND pra.mergedAt IS NULL) AS closedUnmerged12m, quantileIf(0.5) - (mergedInSeconds, mergedAt > now() - INTERVAL 12 MONTH) AS medianMergeS - FROM pull_requests_analyzed - WHERE openedAt > now() - INTERVAL 12 MONTH - GROUP BY channel + (pra.mergedInSeconds, pra.mergedAt > now() - INTERVAL 12 MONTH) AS medianMergeS + FROM pull_requests_analyzed pra + INNER JOIN channels ch ON pra.channel = ch.channel + WHERE pra.openedAt > now() - INTERVAL 12 MONTH + GROUP BY ch.repoUrl ) AS prs ON prs.repoUrl = rp.repoUrl ) diff --git a/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe b/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe index 73dfc398cb..dca2e8d832 100644 --- a/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe @@ -6,9 +6,9 @@ DESCRIPTION > Responsiveness follows Joana's 2026-07-23 review (per-platform, not a blanket rule): - GitHub/GitLab (and any other non-Gerrit host): scores 0/15 when there are no PRs or issues in the window — that's real signal (unresponsive), not a data gap, so it's never `blocked`. - - Gerrit (review.opendev.org, gerrit.*): Gerrit has no issue tracker, so only PR/changeset - response time counts. Scores 0/15 when there's no changeset data; issue-response is excluded - from the calculation entirely rather than penalizing repos for a concept that doesn't exist there. + - Gerrit: Gerrit has no issue tracker, so only PR/changeset response time counts. Scores 0/15 + when there's no changeset data; issue-response is excluded from the calculation entirely rather + than penalizing repos for a concept that doesn't exist there. - Repos marked `excluded` in the repositories table (e.g. .github meta repos where no PRs or issues are ever expected) are `blocked` for responsiveness regardless of platform — never scored 0 for an absence that was never expected to be filled. @@ -32,6 +32,48 @@ DESCRIPTION > trailing 12-month window. Final busFactorCount = max(curated_maintainers_count, observed_actors_count). busFactorAvailable = 1 if either source has data. This ensures teams relying on informal review duties aren't penalized for lack of formal role curation. + - Gerrit channel fix (IN-1229): activity rows for Gerrit repos ingested via opendev.org (and + other Gerrit hosts) are stored under the /q/project: channel variant, not the plain repo URL. + Added a `channels` node that expands each repo URL to all its possible channel forms — matching + the logic in repos_to_channels.pipe — so joins against activityRelations, pull_requests_analyzed, + and issues_analyzed correctly match Gerrit activity. isGerrit now derived from + integrations.platform='gerrit' rather than domain regex, covering git.opendaylight.org and any + custom-domain Gerrit instance. changeset-merged added to observedActorsCount so admin/cherry-pick + merges count toward bus factor. + +NODE channels +DESCRIPTION > + Expand each repo URL to all possible activity channel forms. Mirrors repos_to_channels.pipe + expansion logic. Non-Gerrit repos: one row (channel = url). Gerrit repos: two rows — plain url + plus the /q/project: variant used by Gerrit activity ingestion. +SQL > + SELECT + r.url AS repoUrl, + r.url AS channel, + (i.platform = 'gerrit') AS isGerrit, + r.excluded AS isExcluded + FROM repositories r + LEFT JOIN integrations i ON r.sourceIntegrationId = i.id + WHERE r.deletedAt IS NULL + UNION ALL + SELECT + r.url AS repoUrl, + CASE + WHEN position(r.url, '/r/') > 0 + THEN replaceOne(r.url, '/r/', '/r/q/project:') + WHEN position(r.url, '/gerrit/') > 0 + THEN replaceOne(r.url, '/gerrit/', '/gerrit/q/project:') + ELSE + concat( + protocol(r.url), '://', domain(r.url), '/q/project:', + if(path(r.url) = '/', '', substring(path(r.url), 2)) + ) + END AS channel, + 1 AS isGerrit, + r.excluded AS isExcluded + FROM repositories r + INNER JOIN integrations i ON r.sourceIntegrationId = i.id + WHERE r.deletedAt IS NULL AND i.platform = 'gerrit' NODE health_score_v2_maintainer_calc SQL > @@ -177,14 +219,8 @@ SQL > ) AS responsivenessScore FROM ( - SELECT DISTINCT - url AS repoUrl, - ( - domain(url) = 'review.opendev.org' OR domain(url) LIKE 'gerrit.%' - ) AS isGerrit, - excluded AS isExcluded - FROM repositories - WHERE deletedAt IS NULL + SELECT DISTINCT repoUrl, isGerrit, isExcluded + FROM channels ) AS allRepos LEFT JOIN ( @@ -194,9 +230,10 @@ SQL > FROM maintainers_roles_copy_ds mr INNER JOIN ( - SELECT DISTINCT memberId, channel AS repoUrl - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE timestamp > now() - INTERVAL 12 MONTH + SELECT DISTINCT ch.repoUrl AS repoUrl, ar.memberId AS memberId + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN channels ch ON ar.channel = ch.channel + WHERE ar.timestamp > now() - INTERVAL 12 MONTH ) AS recentActivity ON recentActivity.memberId = mr.memberId AND recentActivity.repoUrl = mr.repoUrl @@ -208,52 +245,55 @@ SQL > ON bf.repoUrl = allRepos.repoUrl LEFT JOIN ( - SELECT channel AS repoUrl, count(DISTINCT memberId) AS observedActorsCount - FROM activityRelations_deduplicated_cleaned_bucket_union + SELECT ch.repoUrl AS repoUrl, count(DISTINCT ar.memberId) AS observedActorsCount + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN channels ch ON ar.channel = ch.channel WHERE - timestamp > now() - INTERVAL 12 MONTH - AND memberId != '' + ar.timestamp > now() - INTERVAL 12 MONTH + AND ar.memberId != '' AND ( - type = 'pull_request-reviewed' - OR type = 'pull_request-merged' - OR type = 'merge_request-review-approved' - OR type = 'merge_request-merged' - OR type = 'patchset_approval-created' - OR type = 'merge_request-review-unapproved' - OR type = 'merge_request-review-commented' + ar.type = 'pull_request-reviewed' + OR ar.type = 'pull_request-merged' + OR ar.type = 'merge_request-review-approved' + OR ar.type = 'merge_request-merged' + OR ar.type = 'patchset_approval-created' + OR ar.type = 'merge_request-review-unapproved' + OR ar.type = 'merge_request-review-commented' + OR ar.type = 'changeset-merged' ) - GROUP BY channel + GROUP BY ch.repoUrl ) AS obs ON obs.repoUrl = allRepos.repoUrl LEFT JOIN ( - SELECT channel AS repoUrl, count(DISTINCT organizationId) AS orgCount - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE timestamp > now() - INTERVAL 12 MONTH AND organizationId != '' - GROUP BY channel + SELECT ch.repoUrl AS repoUrl, count(DISTINCT ar.organizationId) AS orgCount + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN channels ch ON ar.channel = ch.channel + WHERE ar.timestamp > now() - INTERVAL 12 MONTH AND ar.organizationId != '' + GROUP BY ch.repoUrl ) AS od ON od.repoUrl = allRepos.repoUrl LEFT JOIN ( SELECT - channel AS repoUrl, - quantile(0.5) - (reviewedInSeconds) AS medianPrResponseS - FROM pull_requests_analyzed - WHERE openedAt > now() - INTERVAL 12 MONTH AND reviewedInSeconds IS NOT NULL - GROUP BY channel + ch.repoUrl AS repoUrl, + quantile(0.5)(pra.reviewedInSeconds) AS medianPrResponseS + FROM pull_requests_analyzed pra + INNER JOIN channels ch ON pra.channel = ch.channel + WHERE pra.openedAt > now() - INTERVAL 12 MONTH AND pra.reviewedInSeconds IS NOT NULL + GROUP BY ch.repoUrl ) AS r ON r.repoUrl = allRepos.repoUrl LEFT JOIN ( SELECT - channel AS repoUrl, - quantile(0.5) - (respondedInSeconds) AS medianIssueResponseS - FROM issues_analyzed + ch.repoUrl AS repoUrl, + quantile(0.5)(ia.respondedInSeconds) AS medianIssueResponseS + FROM issues_analyzed ia + INNER JOIN channels ch ON ia.channel = ch.channel WHERE - openedAt > now() - INTERVAL 12 MONTH AND respondedInSeconds IS NOT NULL - GROUP BY channel + ia.openedAt > now() - INTERVAL 12 MONTH AND ia.respondedInSeconds IS NOT NULL + GROUP BY ch.repoUrl ) AS ir ON ir.repoUrl = allRepos.repoUrl ) From b92adea6733d7b938dec0899a2700c813a651177 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Wed, 12 Aug 2026 16:45:46 +0100 Subject: [PATCH 02/14] fix: add FINAL to repositories and integrations in channels nodes (IN-1229) Signed-off-by: Joana Maia --- .../libs/tinybird/pipes/health_score_v2_development.pipe | 6 +++--- .../libs/tinybird/pipes/health_score_v2_maintainer.pipe | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/services/libs/tinybird/pipes/health_score_v2_development.pipe b/services/libs/tinybird/pipes/health_score_v2_development.pipe index 28520ae2d1..9ab7a4ca58 100644 --- a/services/libs/tinybird/pipes/health_score_v2_development.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_development.pipe @@ -23,7 +23,7 @@ DESCRIPTION > plus the /q/project: variant used by Gerrit activity ingestion. SQL > SELECT r.url AS repoUrl, r.url AS channel - FROM repositories r + FROM repositories r FINAL WHERE r.deletedAt IS NULL UNION ALL SELECT @@ -39,8 +39,8 @@ SQL > if(path(r.url) = '/', '', substring(path(r.url), 2)) ) END AS channel - FROM repositories r - INNER JOIN integrations i ON r.sourceIntegrationId = i.id + FROM repositories r FINAL + INNER JOIN integrations i FINAL ON r.sourceIntegrationId = i.id WHERE r.deletedAt IS NULL AND i.platform = 'gerrit' NODE health_score_v2_development_calc diff --git a/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe b/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe index dca2e8d832..9349635150 100644 --- a/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe @@ -52,8 +52,8 @@ SQL > r.url AS channel, (i.platform = 'gerrit') AS isGerrit, r.excluded AS isExcluded - FROM repositories r - LEFT JOIN integrations i ON r.sourceIntegrationId = i.id + FROM repositories r FINAL + LEFT JOIN integrations i FINAL ON r.sourceIntegrationId = i.id WHERE r.deletedAt IS NULL UNION ALL SELECT @@ -71,8 +71,8 @@ SQL > END AS channel, 1 AS isGerrit, r.excluded AS isExcluded - FROM repositories r - INNER JOIN integrations i ON r.sourceIntegrationId = i.id + FROM repositories r FINAL + INNER JOIN integrations i FINAL ON r.sourceIntegrationId = i.id WHERE r.deletedAt IS NULL AND i.platform = 'gerrit' NODE health_score_v2_maintainer_calc From f6a5f72028a2d0e6f35c89b6148f8c65759c1d15 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Wed, 12 Aug 2026 16:59:00 +0100 Subject: [PATCH 03/14] fix: apply Gerrit channel expansion to raw inputs snapshot pipe (IN-1229) Signed-off-by: Joana Maia --- .../health_score_v2_raw_inputs_snapshot.pipe | 164 +++++++++++------- 1 file changed, 103 insertions(+), 61 deletions(-) diff --git a/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe b/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe index 54a3d0cbfd..a80fb0c1fe 100644 --- a/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe @@ -21,9 +21,40 @@ DESCRIPTION > per-column (via a `FILTER`/inline date check on `closedAt`/`mergedAt` instead of `openedAt`), which measured a different, unbounded-by-openedAt population than the category pipe — the two could never be reconciled. + - Gerrit channel fix (IN-1229): mirrors the same fix applied to health_score_v2_maintainer.pipe + and health_score_v2_development.pipe — all activityRelations, pull_requests_analyzed, and + issues_analyzed joins now route through a `channels` node that expands each Gerrit repo URL to + its /q/project: variant so Gerrit activity is correctly matched. TAGS "Validation", "Health Score v2" +NODE channels +DESCRIPTION > + Expand each repo URL to all possible activity channel forms. Mirrors repos_to_channels.pipe + expansion logic. Non-Gerrit repos: one row (channel = url). Gerrit repos: two rows — plain url + plus the /q/project: variant used by Gerrit activity ingestion. +SQL > + SELECT r.url AS repoUrl, r.url AS channel + FROM repositories r FINAL + WHERE r.deletedAt IS NULL + UNION ALL + SELECT + r.url AS repoUrl, + CASE + WHEN position(r.url, '/r/') > 0 + THEN replaceOne(r.url, '/r/', '/r/q/project:') + WHEN position(r.url, '/gerrit/') > 0 + THEN replaceOne(r.url, '/gerrit/', '/gerrit/q/project:') + ELSE + concat( + protocol(r.url), '://', domain(r.url), '/q/project:', + if(path(r.url) = '/', '', substring(path(r.url), 2)) + ) + END AS channel + FROM repositories r FINAL + INNER JOIN integrations i FINAL ON r.sourceIntegrationId = i.id + WHERE r.deletedAt IS NULL AND i.platform = 'gerrit' + NODE health_score_v2_raw_inputs_snapshot_calc DESCRIPTION > Combines raw input signals from all three categories (maintainer, security, development) @@ -80,9 +111,10 @@ SQL > FROM maintainers_roles_copy_ds mr INNER JOIN ( - SELECT DISTINCT memberId, channel AS repoUrl - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE timestamp > now() - INTERVAL 12 MONTH + SELECT DISTINCT ch.repoUrl AS repoUrl, ar.memberId AS memberId + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN channels ch ON ar.channel = ch.channel + WHERE ar.timestamp > now() - INTERVAL 12 MONTH ) AS recentActivity ON recentActivity.memberId = mr.memberId AND recentActivity.repoUrl = mr.repoUrl @@ -92,45 +124,50 @@ SQL > ON bf.repoUrl = allRepos.repoUrl LEFT JOIN ( - SELECT channel AS repoUrl, count(DISTINCT memberId) AS observedActorsCount - FROM activityRelations_deduplicated_cleaned_bucket_union + SELECT ch.repoUrl AS repoUrl, count(DISTINCT ar.memberId) AS observedActorsCount + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN channels ch ON ar.channel = ch.channel WHERE - timestamp > now() - INTERVAL 12 MONTH - AND memberId != '' + ar.timestamp > now() - INTERVAL 12 MONTH + AND ar.memberId != '' AND ( - type = 'pull_request-reviewed' - OR type = 'pull_request-merged' - OR type = 'merge_request-review-approved' - OR type = 'merge_request-merged' - OR type = 'patchset_approval-created' - OR type = 'merge_request-review-unapproved' - OR type = 'merge_request-review-commented' + ar.type = 'pull_request-reviewed' + OR ar.type = 'pull_request-merged' + OR ar.type = 'merge_request-review-approved' + OR ar.type = 'merge_request-merged' + OR ar.type = 'patchset_approval-created' + OR ar.type = 'merge_request-review-unapproved' + OR ar.type = 'merge_request-review-commented' + OR ar.type = 'changeset-merged' ) - GROUP BY channel + GROUP BY ch.repoUrl ) AS obs ON obs.repoUrl = allRepos.repoUrl LEFT JOIN ( - SELECT channel AS repoUrl, count(DISTINCT organizationId) AS orgCount - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE timestamp > now() - INTERVAL 12 MONTH AND organizationId != '' - GROUP BY channel + SELECT ch.repoUrl AS repoUrl, count(DISTINCT ar.organizationId) AS orgCount + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN channels ch ON ar.channel = ch.channel + WHERE ar.timestamp > now() - INTERVAL 12 MONTH AND ar.organizationId != '' + GROUP BY ch.repoUrl ) AS od ON od.repoUrl = allRepos.repoUrl LEFT JOIN ( - SELECT channel AS repoUrl, quantile(0.5)(reviewedInSeconds) AS medianPrResponseS - FROM pull_requests_analyzed - WHERE openedAt > now() - INTERVAL 12 MONTH AND reviewedInSeconds IS NOT NULL - GROUP BY channel + SELECT ch.repoUrl AS repoUrl, quantile(0.5)(pra.reviewedInSeconds) AS medianPrResponseS + FROM pull_requests_analyzed pra + INNER JOIN channels ch ON pra.channel = ch.channel + WHERE pra.openedAt > now() - INTERVAL 12 MONTH AND pra.reviewedInSeconds IS NOT NULL + GROUP BY ch.repoUrl ) AS r ON r.repoUrl = allRepos.repoUrl LEFT JOIN ( - SELECT channel AS repoUrl, quantile(0.5)(respondedInSeconds) AS medianIssueResponseS - FROM issues_analyzed - WHERE openedAt > now() - INTERVAL 12 MONTH AND respondedInSeconds IS NOT NULL - GROUP BY channel + SELECT ch.repoUrl AS repoUrl, quantile(0.5)(ia.respondedInSeconds) AS medianIssueResponseS + FROM issues_analyzed ia + INNER JOIN channels ch ON ia.channel = ch.channel + WHERE ia.openedAt > now() - INTERVAL 12 MONTH AND ia.respondedInSeconds IS NOT NULL + GROUP BY ch.repoUrl ) AS ir ON ir.repoUrl = allRepos.repoUrl LEFT JOIN @@ -205,14 +242,15 @@ SQL > LEFT JOIN ( SELECT - channel AS repoUrl, - countIf(timestamp > now() - INTERVAL 6 MONTH) AS commitsLast6m, + ch.repoUrl AS repoUrl, + countIf(ar.timestamp > now() - INTERVAL 6 MONTH) AS commitsLast6m, countIf( - timestamp <= now() - INTERVAL 6 MONTH AND timestamp > now() - INTERVAL 12 MONTH + ar.timestamp <= now() - INTERVAL 6 MONTH AND ar.timestamp > now() - INTERVAL 12 MONTH ) AS commitsPrior6m - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE type = 'authored-commit' - GROUP BY channel + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN channels ch ON ar.channel = ch.channel + WHERE ar.type = 'authored-commit' + GROUP BY ch.repoUrl ) AS c ON c.repoUrl = allRepos.repoUrl LEFT JOIN @@ -225,58 +263,62 @@ SQL > LEFT JOIN ( SELECT - channel AS repoUrl, - countIf(closedAt IS NOT NULL) AS closed12m, + ch.repoUrl AS repoUrl, + countIf(ia.closedAt IS NOT NULL) AS closed12m, count() AS opened12m, quantileIf(0.5) - (closedInSeconds, closedAt > now() - INTERVAL 12 MONTH) AS medianCloseS - FROM issues_analyzed - WHERE openedAt > now() - INTERVAL 12 MONTH - GROUP BY channel + (ia.closedInSeconds, ia.closedAt > now() - INTERVAL 12 MONTH) AS medianCloseS + FROM issues_analyzed ia + INNER JOIN channels ch ON ia.channel = ch.channel + WHERE ia.openedAt > now() - INTERVAL 12 MONTH + GROUP BY ch.repoUrl ) AS w ON w.repoUrl = allRepos.repoUrl LEFT JOIN ( SELECT - channel AS repoUrl, - countIf(mergedAt IS NOT NULL) AS merged12m, - countIf(closedAt IS NOT NULL AND mergedAt IS NULL) AS closedUnmerged12m, + ch.repoUrl AS repoUrl, + countIf(pra.mergedAt IS NOT NULL) AS merged12m, + countIf(pra.closedAt IS NOT NULL AND pra.mergedAt IS NULL) AS closedUnmerged12m, quantileIf(0.5) - (mergedInSeconds, mergedAt > now() - INTERVAL 12 MONTH) AS medianMergeS - FROM pull_requests_analyzed - WHERE openedAt > now() - INTERVAL 12 MONTH - GROUP BY channel + (pra.mergedInSeconds, pra.mergedAt > now() - INTERVAL 12 MONTH) AS medianMergeS + FROM pull_requests_analyzed pra + INNER JOIN channels ch ON pra.channel = ch.channel + WHERE pra.openedAt > now() - INTERVAL 12 MONTH + GROUP BY ch.repoUrl ) AS p ON p.repoUrl = allRepos.repoUrl LEFT JOIN ( SELECT - channel AS repoUrl, + ch.repoUrl AS repoUrl, countIf( - respondedInSeconds IS NULL - AND closedAt IS NULL - AND openedAt <= now() - INTERVAL 90 DAY + ia.respondedInSeconds IS NULL + AND ia.closedAt IS NULL + AND ia.openedAt <= now() - INTERVAL 90 DAY ) AS unansweredAged90d, count() AS openedLast18m - FROM issues_analyzed - WHERE openedAt > now() - INTERVAL 18 MONTH - GROUP BY channel + FROM issues_analyzed ia + INNER JOIN channels ch ON ia.channel = ch.channel + WHERE ia.openedAt > now() - INTERVAL 18 MONTH + GROUP BY ch.repoUrl ) AS uw ON uw.repoUrl = allRepos.repoUrl LEFT JOIN ( SELECT - channel AS repoUrl, + ch.repoUrl AS repoUrl, countIf( - reviewedAt IS NULL - AND approvedAt IS NULL - AND closedAt IS NULL - AND openedAt <= now() - INTERVAL 90 DAY + pra.reviewedAt IS NULL + AND pra.approvedAt IS NULL + AND pra.closedAt IS NULL + AND pra.openedAt <= now() - INTERVAL 90 DAY ) AS unansweredAged90d, count() AS openedLast18m - FROM pull_requests_analyzed - WHERE openedAt > now() - INTERVAL 18 MONTH - GROUP BY channel + FROM pull_requests_analyzed pra + INNER JOIN channels ch ON pra.channel = ch.channel + WHERE pra.openedAt > now() - INTERVAL 18 MONTH + GROUP BY ch.repoUrl ) AS upw ON upw.repoUrl = allRepos.repoUrl LEFT JOIN From 4332ec7eec54008f75cd69e442b49ef79c1b5ac8 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Wed, 12 Aug 2026 17:06:18 +0100 Subject: [PATCH 04/14] style: apply tb fmt formatting to health score v2 pipes (IN-1229) Signed-off-by: Joana Maia --- .../pipes/health_score_v2_development.pipe | 22 +++++++++---- .../pipes/health_score_v2_maintainer.pipe | 33 +++++++++++-------- .../health_score_v2_raw_inputs_snapshot.pipe | 13 +++++--- 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/services/libs/tinybird/pipes/health_score_v2_development.pipe b/services/libs/tinybird/pipes/health_score_v2_development.pipe index 9ab7a4ca58..ba7355ed85 100644 --- a/services/libs/tinybird/pipes/health_score_v2_development.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_development.pipe @@ -21,6 +21,7 @@ DESCRIPTION > Expand each repo URL to all possible activity channel forms. Mirrors repos_to_channels.pipe expansion logic. Non-Gerrit repos: one row (channel = url). Gerrit repos: two rows — plain url plus the /q/project: variant used by Gerrit activity ingestion. + SQL > SELECT r.url AS repoUrl, r.url AS channel FROM repositories r FINAL @@ -30,12 +31,15 @@ SQL > r.url AS repoUrl, CASE WHEN position(r.url, '/r/') > 0 - THEN replaceOne(r.url, '/r/', '/r/q/project:') + THEN replaceOne(r.url, '/r/', '/r/q/project:') WHEN position(r.url, '/gerrit/') > 0 - THEN replaceOne(r.url, '/gerrit/', '/gerrit/q/project:') + THEN replaceOne(r.url, '/gerrit/', '/gerrit/q/project:') ELSE concat( - protocol(r.url), '://', domain(r.url), '/q/project:', + protocol(r.url), + '://', + domain(r.url), + '/q/project:', if(path(r.url) = '/', '', substring(path(r.url), 2)) ) END AS channel @@ -239,7 +243,9 @@ SQL > countIf(ia.closedAt IS NOT NULL) AS closed12m, count() AS opened12m, quantileIf(0.5) - (ia.closedInSeconds, ia.closedAt > now() - INTERVAL 12 MONTH) AS medianCloseS + ( + ia.closedInSeconds, ia.closedAt > now() - INTERVAL 12 MONTH + ) AS medianCloseS FROM issues_analyzed ia INNER JOIN channels ch ON ia.channel = ch.channel WHERE ia.openedAt > now() - INTERVAL 12 MONTH @@ -251,9 +257,13 @@ SQL > SELECT ch.repoUrl AS repoUrl, countIf(pra.mergedAt IS NOT NULL) AS merged12m, - countIf(pra.closedAt IS NOT NULL AND pra.mergedAt IS NULL) AS closedUnmerged12m, + countIf( + pra.closedAt IS NOT NULL AND pra.mergedAt IS NULL + ) AS closedUnmerged12m, quantileIf(0.5) - (pra.mergedInSeconds, pra.mergedAt > now() - INTERVAL 12 MONTH) AS medianMergeS + ( + pra.mergedInSeconds, pra.mergedAt > now() - INTERVAL 12 MONTH + ) AS medianMergeS FROM pull_requests_analyzed pra INNER JOIN channels ch ON pra.channel = ch.channel WHERE pra.openedAt > now() - INTERVAL 12 MONTH diff --git a/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe b/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe index 9349635150..22edee359d 100644 --- a/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe @@ -46,6 +46,7 @@ DESCRIPTION > Expand each repo URL to all possible activity channel forms. Mirrors repos_to_channels.pipe expansion logic. Non-Gerrit repos: one row (channel = url). Gerrit repos: two rows — plain url plus the /q/project: variant used by Gerrit activity ingestion. + SQL > SELECT r.url AS repoUrl, @@ -60,12 +61,15 @@ SQL > r.url AS repoUrl, CASE WHEN position(r.url, '/r/') > 0 - THEN replaceOne(r.url, '/r/', '/r/q/project:') + THEN replaceOne(r.url, '/r/', '/r/q/project:') WHEN position(r.url, '/gerrit/') > 0 - THEN replaceOne(r.url, '/gerrit/', '/gerrit/q/project:') + THEN replaceOne(r.url, '/gerrit/', '/gerrit/q/project:') ELSE concat( - protocol(r.url), '://', domain(r.url), '/q/project:', + protocol(r.url), + '://', + domain(r.url), + '/q/project:', if(path(r.url) = '/', '', substring(path(r.url), 2)) ) END AS channel, @@ -217,11 +221,7 @@ SQL > 0 ) ) AS responsivenessScore - FROM - ( - SELECT DISTINCT repoUrl, isGerrit, isExcluded - FROM channels - ) AS allRepos + FROM (SELECT DISTINCT repoUrl, isGerrit, isExcluded FROM channels) AS allRepos LEFT JOIN ( SELECT @@ -245,7 +245,9 @@ SQL > ON bf.repoUrl = allRepos.repoUrl LEFT JOIN ( - SELECT ch.repoUrl AS repoUrl, count(DISTINCT ar.memberId) AS observedActorsCount + SELECT + ch.repoUrl AS repoUrl, + count(DISTINCT ar.memberId) AS observedActorsCount FROM activityRelations_deduplicated_cleaned_bucket_union ar INNER JOIN channels ch ON ar.channel = ch.channel WHERE @@ -277,10 +279,13 @@ SQL > ( SELECT ch.repoUrl AS repoUrl, - quantile(0.5)(pra.reviewedInSeconds) AS medianPrResponseS + quantile(0.5) + (pra.reviewedInSeconds) AS medianPrResponseS FROM pull_requests_analyzed pra INNER JOIN channels ch ON pra.channel = ch.channel - WHERE pra.openedAt > now() - INTERVAL 12 MONTH AND pra.reviewedInSeconds IS NOT NULL + WHERE + pra.openedAt > now() - INTERVAL 12 MONTH + AND pra.reviewedInSeconds IS NOT NULL GROUP BY ch.repoUrl ) AS r ON r.repoUrl = allRepos.repoUrl @@ -288,11 +293,13 @@ SQL > ( SELECT ch.repoUrl AS repoUrl, - quantile(0.5)(ia.respondedInSeconds) AS medianIssueResponseS + quantile(0.5) + (ia.respondedInSeconds) AS medianIssueResponseS FROM issues_analyzed ia INNER JOIN channels ch ON ia.channel = ch.channel WHERE - ia.openedAt > now() - INTERVAL 12 MONTH AND ia.respondedInSeconds IS NOT NULL + ia.openedAt > now() - INTERVAL 12 MONTH + AND ia.respondedInSeconds IS NOT NULL GROUP BY ch.repoUrl ) AS ir ON ir.repoUrl = allRepos.repoUrl diff --git a/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe b/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe index a80fb0c1fe..cc3f9a3245 100644 --- a/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe @@ -33,6 +33,7 @@ DESCRIPTION > Expand each repo URL to all possible activity channel forms. Mirrors repos_to_channels.pipe expansion logic. Non-Gerrit repos: one row (channel = url). Gerrit repos: two rows — plain url plus the /q/project: variant used by Gerrit activity ingestion. + SQL > SELECT r.url AS repoUrl, r.url AS channel FROM repositories r FINAL @@ -42,12 +43,15 @@ SQL > r.url AS repoUrl, CASE WHEN position(r.url, '/r/') > 0 - THEN replaceOne(r.url, '/r/', '/r/q/project:') + THEN replaceOne(r.url, '/r/', '/r/q/project:') WHEN position(r.url, '/gerrit/') > 0 - THEN replaceOne(r.url, '/gerrit/', '/gerrit/q/project:') + THEN replaceOne(r.url, '/gerrit/', '/gerrit/q/project:') ELSE concat( - protocol(r.url), '://', domain(r.url), '/q/project:', + protocol(r.url), + '://', + domain(r.url), + '/q/project:', if(path(r.url) = '/', '', substring(path(r.url), 2)) ) END AS channel @@ -245,7 +249,8 @@ SQL > ch.repoUrl AS repoUrl, countIf(ar.timestamp > now() - INTERVAL 6 MONTH) AS commitsLast6m, countIf( - ar.timestamp <= now() - INTERVAL 6 MONTH AND ar.timestamp > now() - INTERVAL 12 MONTH + ar.timestamp <= now() - INTERVAL 6 MONTH + AND ar.timestamp > now() - INTERVAL 12 MONTH ) AS commitsPrior6m FROM activityRelations_deduplicated_cleaned_bucket_union ar INNER JOIN channels ch ON ar.channel = ch.channel From 9a83f6d2def42f3e4de09079279a6991b33ae083 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Wed, 12 Aug 2026 17:26:55 +0100 Subject: [PATCH 05/14] refactor: centralize Gerrit channel expansion in repos_channels_ds (IN-1229) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce repos_channels_ds datasource and repos_channels_copy.pipe to pre-materialize the repo URL→channel expansion (plain + /q/project: variants for Gerrit). Health score v2 pipes now join repos_channels_ds instead of inlining the UNION ALL expansion in each pipe — eliminates duplication and avoids re-scanning repositories+integrations with FINAL once per join. Signed-off-by: Joana Maia --- .../datasources/repos_channels_ds.datasource | 18 ++++++ .../pipes/health_score_v2_development.pipe | 43 ++----------- .../pipes/health_score_v2_maintainer.pipe | 64 ++++--------------- .../health_score_v2_raw_inputs_snapshot.pipe | 59 ++++------------- .../tinybird/pipes/repos_channels_copy.pipe | 48 ++++++++++++++ 5 files changed, 100 insertions(+), 132 deletions(-) create mode 100644 services/libs/tinybird/datasources/repos_channels_ds.datasource create mode 100644 services/libs/tinybird/pipes/repos_channels_copy.pipe diff --git a/services/libs/tinybird/datasources/repos_channels_ds.datasource b/services/libs/tinybird/datasources/repos_channels_ds.datasource new file mode 100644 index 0000000000..a742f3d10b --- /dev/null +++ b/services/libs/tinybird/datasources/repos_channels_ds.datasource @@ -0,0 +1,18 @@ +DESCRIPTION > + - Materialized channel expansion for all repositories. Populated by `repos_channels_copy.pipe` + (daily, before health score pipes run). + - Each non-Gerrit repo produces one row (channel = url). Each Gerrit repo produces two rows: + the plain URL and the /q/project: variant used by Gerrit activity ingestion. + - `isGerrit` and `isExcluded` are denormalized from `repositories`/`integrations` so health + score pipes can classify repos without re-joining those tables. + - Sorting key is `channel` because all consumers join on `channel`; `repoUrl` lookups are + full-scans but the table is small enough that this is fine. + +SCHEMA > + `repoUrl` String, + `channel` String, + `isGerrit` UInt8, + `isExcluded` UInt8 + +ENGINE MergeTree +ENGINE_SORTING_KEY channel diff --git a/services/libs/tinybird/pipes/health_score_v2_development.pipe b/services/libs/tinybird/pipes/health_score_v2_development.pipe index ba7355ed85..932b7e4782 100644 --- a/services/libs/tinybird/pipes/health_score_v2_development.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_development.pipe @@ -12,40 +12,9 @@ DESCRIPTION > health_score_v2_maintainer.pipe description for why. - Gerrit channel fix (IN-1229): activity rows for Gerrit repos ingested via opendev.org (and other Gerrit hosts) are stored under the /q/project: channel variant, not the plain repo URL. - Added a `channels` node that expands each repo URL to all its possible channel forms — matching - the logic in repos_to_channels.pipe — so joins against activityRelations, pull_requests_analyzed, - and issues_analyzed correctly match Gerrit activity. - -NODE channels -DESCRIPTION > - Expand each repo URL to all possible activity channel forms. Mirrors repos_to_channels.pipe - expansion logic. Non-Gerrit repos: one row (channel = url). Gerrit repos: two rows — plain url - plus the /q/project: variant used by Gerrit activity ingestion. - -SQL > - SELECT r.url AS repoUrl, r.url AS channel - FROM repositories r FINAL - WHERE r.deletedAt IS NULL - UNION ALL - SELECT - r.url AS repoUrl, - CASE - WHEN position(r.url, '/r/') > 0 - THEN replaceOne(r.url, '/r/', '/r/q/project:') - WHEN position(r.url, '/gerrit/') > 0 - THEN replaceOne(r.url, '/gerrit/', '/gerrit/q/project:') - ELSE - concat( - protocol(r.url), - '://', - domain(r.url), - '/q/project:', - if(path(r.url) = '/', '', substring(path(r.url), 2)) - ) - END AS channel - FROM repositories r FINAL - INNER JOIN integrations i FINAL ON r.sourceIntegrationId = i.id - WHERE r.deletedAt IS NULL AND i.platform = 'gerrit' + All joins against activityRelations, pull_requests_analyzed, and issues_analyzed now route + through `repos_channels_ds` (populated by repos_channels_copy.pipe) which expands each repo URL + to all its possible channel forms. NODE health_score_v2_development_calc SQL > @@ -231,7 +200,7 @@ SQL > ch.repoUrl AS repoUrl, countIf(ar.timestamp > now() - INTERVAL 6 MONTH) AS commitsLast6m FROM activityRelations_deduplicated_cleaned_bucket_union ar - INNER JOIN channels ch ON ar.channel = ch.channel + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel WHERE ar.type = 'authored-commit' GROUP BY ch.repoUrl ) AS c @@ -247,7 +216,7 @@ SQL > ia.closedInSeconds, ia.closedAt > now() - INTERVAL 12 MONTH ) AS medianCloseS FROM issues_analyzed ia - INNER JOIN channels ch ON ia.channel = ch.channel + INNER JOIN repos_channels_ds ch ON ia.channel = ch.channel WHERE ia.openedAt > now() - INTERVAL 12 MONTH GROUP BY ch.repoUrl ) AS ist @@ -265,7 +234,7 @@ SQL > pra.mergedInSeconds, pra.mergedAt > now() - INTERVAL 12 MONTH ) AS medianMergeS FROM pull_requests_analyzed pra - INNER JOIN channels ch ON pra.channel = ch.channel + INNER JOIN repos_channels_ds ch ON pra.channel = ch.channel WHERE pra.openedAt > now() - INTERVAL 12 MONTH GROUP BY ch.repoUrl ) AS prs diff --git a/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe b/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe index 22edee359d..e93baf9c84 100644 --- a/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe @@ -34,50 +34,11 @@ DESCRIPTION > duties aren't penalized for lack of formal role curation. - Gerrit channel fix (IN-1229): activity rows for Gerrit repos ingested via opendev.org (and other Gerrit hosts) are stored under the /q/project: channel variant, not the plain repo URL. - Added a `channels` node that expands each repo URL to all its possible channel forms — matching - the logic in repos_to_channels.pipe — so joins against activityRelations, pull_requests_analyzed, - and issues_analyzed correctly match Gerrit activity. isGerrit now derived from - integrations.platform='gerrit' rather than domain regex, covering git.opendaylight.org and any - custom-domain Gerrit instance. changeset-merged added to observedActorsCount so admin/cherry-pick - merges count toward bus factor. - -NODE channels -DESCRIPTION > - Expand each repo URL to all possible activity channel forms. Mirrors repos_to_channels.pipe - expansion logic. Non-Gerrit repos: one row (channel = url). Gerrit repos: two rows — plain url - plus the /q/project: variant used by Gerrit activity ingestion. - -SQL > - SELECT - r.url AS repoUrl, - r.url AS channel, - (i.platform = 'gerrit') AS isGerrit, - r.excluded AS isExcluded - FROM repositories r FINAL - LEFT JOIN integrations i FINAL ON r.sourceIntegrationId = i.id - WHERE r.deletedAt IS NULL - UNION ALL - SELECT - r.url AS repoUrl, - CASE - WHEN position(r.url, '/r/') > 0 - THEN replaceOne(r.url, '/r/', '/r/q/project:') - WHEN position(r.url, '/gerrit/') > 0 - THEN replaceOne(r.url, '/gerrit/', '/gerrit/q/project:') - ELSE - concat( - protocol(r.url), - '://', - domain(r.url), - '/q/project:', - if(path(r.url) = '/', '', substring(path(r.url), 2)) - ) - END AS channel, - 1 AS isGerrit, - r.excluded AS isExcluded - FROM repositories r FINAL - INNER JOIN integrations i FINAL ON r.sourceIntegrationId = i.id - WHERE r.deletedAt IS NULL AND i.platform = 'gerrit' + All joins against activityRelations, pull_requests_analyzed, and issues_analyzed now route + through `repos_channels_ds` (populated by repos_channels_copy.pipe) which expands each repo URL + to all its possible channel forms. isGerrit now derived from integrations.platform='gerrit' + rather than domain regex, covering git.opendaylight.org and any custom-domain Gerrit instance. + changeset-merged added to observedActorsCount so admin/cherry-pick merges count toward bus factor. NODE health_score_v2_maintainer_calc SQL > @@ -221,7 +182,10 @@ SQL > 0 ) ) AS responsivenessScore - FROM (SELECT DISTINCT repoUrl, isGerrit, isExcluded FROM channels) AS allRepos + FROM + ( + SELECT DISTINCT repoUrl, isGerrit, isExcluded FROM repos_channels_ds + ) AS allRepos LEFT JOIN ( SELECT @@ -232,7 +196,7 @@ SQL > ( SELECT DISTINCT ch.repoUrl AS repoUrl, ar.memberId AS memberId FROM activityRelations_deduplicated_cleaned_bucket_union ar - INNER JOIN channels ch ON ar.channel = ch.channel + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel WHERE ar.timestamp > now() - INTERVAL 12 MONTH ) AS recentActivity ON recentActivity.memberId = mr.memberId @@ -249,7 +213,7 @@ SQL > ch.repoUrl AS repoUrl, count(DISTINCT ar.memberId) AS observedActorsCount FROM activityRelations_deduplicated_cleaned_bucket_union ar - INNER JOIN channels ch ON ar.channel = ch.channel + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel WHERE ar.timestamp > now() - INTERVAL 12 MONTH AND ar.memberId != '' @@ -270,7 +234,7 @@ SQL > ( SELECT ch.repoUrl AS repoUrl, count(DISTINCT ar.organizationId) AS orgCount FROM activityRelations_deduplicated_cleaned_bucket_union ar - INNER JOIN channels ch ON ar.channel = ch.channel + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel WHERE ar.timestamp > now() - INTERVAL 12 MONTH AND ar.organizationId != '' GROUP BY ch.repoUrl ) AS od @@ -282,7 +246,7 @@ SQL > quantile(0.5) (pra.reviewedInSeconds) AS medianPrResponseS FROM pull_requests_analyzed pra - INNER JOIN channels ch ON pra.channel = ch.channel + INNER JOIN repos_channels_ds ch ON pra.channel = ch.channel WHERE pra.openedAt > now() - INTERVAL 12 MONTH AND pra.reviewedInSeconds IS NOT NULL @@ -296,7 +260,7 @@ SQL > quantile(0.5) (ia.respondedInSeconds) AS medianIssueResponseS FROM issues_analyzed ia - INNER JOIN channels ch ON ia.channel = ch.channel + INNER JOIN repos_channels_ds ch ON ia.channel = ch.channel WHERE ia.openedAt > now() - INTERVAL 12 MONTH AND ia.respondedInSeconds IS NOT NULL diff --git a/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe b/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe index cc3f9a3245..f5e2ae082e 100644 --- a/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe @@ -21,44 +21,13 @@ DESCRIPTION > per-column (via a `FILTER`/inline date check on `closedAt`/`mergedAt` instead of `openedAt`), which measured a different, unbounded-by-openedAt population than the category pipe — the two could never be reconciled. - - Gerrit channel fix (IN-1229): mirrors the same fix applied to health_score_v2_maintainer.pipe - and health_score_v2_development.pipe — all activityRelations, pull_requests_analyzed, and - issues_analyzed joins now route through a `channels` node that expands each Gerrit repo URL to - its /q/project: variant so Gerrit activity is correctly matched. + - Gerrit channel fix (IN-1229): all activityRelations, pull_requests_analyzed, and + issues_analyzed joins now route through `repos_channels_ds` (populated by repos_channels_copy.pipe) + which expands each Gerrit repo URL to its /q/project: variant so Gerrit activity is correctly + matched. TAGS "Validation", "Health Score v2" -NODE channels -DESCRIPTION > - Expand each repo URL to all possible activity channel forms. Mirrors repos_to_channels.pipe - expansion logic. Non-Gerrit repos: one row (channel = url). Gerrit repos: two rows — plain url - plus the /q/project: variant used by Gerrit activity ingestion. - -SQL > - SELECT r.url AS repoUrl, r.url AS channel - FROM repositories r FINAL - WHERE r.deletedAt IS NULL - UNION ALL - SELECT - r.url AS repoUrl, - CASE - WHEN position(r.url, '/r/') > 0 - THEN replaceOne(r.url, '/r/', '/r/q/project:') - WHEN position(r.url, '/gerrit/') > 0 - THEN replaceOne(r.url, '/gerrit/', '/gerrit/q/project:') - ELSE - concat( - protocol(r.url), - '://', - domain(r.url), - '/q/project:', - if(path(r.url) = '/', '', substring(path(r.url), 2)) - ) - END AS channel - FROM repositories r FINAL - INNER JOIN integrations i FINAL ON r.sourceIntegrationId = i.id - WHERE r.deletedAt IS NULL AND i.platform = 'gerrit' - NODE health_score_v2_raw_inputs_snapshot_calc DESCRIPTION > Combines raw input signals from all three categories (maintainer, security, development) @@ -117,7 +86,7 @@ SQL > ( SELECT DISTINCT ch.repoUrl AS repoUrl, ar.memberId AS memberId FROM activityRelations_deduplicated_cleaned_bucket_union ar - INNER JOIN channels ch ON ar.channel = ch.channel + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel WHERE ar.timestamp > now() - INTERVAL 12 MONTH ) AS recentActivity ON recentActivity.memberId = mr.memberId @@ -130,7 +99,7 @@ SQL > ( SELECT ch.repoUrl AS repoUrl, count(DISTINCT ar.memberId) AS observedActorsCount FROM activityRelations_deduplicated_cleaned_bucket_union ar - INNER JOIN channels ch ON ar.channel = ch.channel + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel WHERE ar.timestamp > now() - INTERVAL 12 MONTH AND ar.memberId != '' @@ -151,7 +120,7 @@ SQL > ( SELECT ch.repoUrl AS repoUrl, count(DISTINCT ar.organizationId) AS orgCount FROM activityRelations_deduplicated_cleaned_bucket_union ar - INNER JOIN channels ch ON ar.channel = ch.channel + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel WHERE ar.timestamp > now() - INTERVAL 12 MONTH AND ar.organizationId != '' GROUP BY ch.repoUrl ) AS od @@ -160,7 +129,7 @@ SQL > ( SELECT ch.repoUrl AS repoUrl, quantile(0.5)(pra.reviewedInSeconds) AS medianPrResponseS FROM pull_requests_analyzed pra - INNER JOIN channels ch ON pra.channel = ch.channel + INNER JOIN repos_channels_ds ch ON pra.channel = ch.channel WHERE pra.openedAt > now() - INTERVAL 12 MONTH AND pra.reviewedInSeconds IS NOT NULL GROUP BY ch.repoUrl ) AS r @@ -169,7 +138,7 @@ SQL > ( SELECT ch.repoUrl AS repoUrl, quantile(0.5)(ia.respondedInSeconds) AS medianIssueResponseS FROM issues_analyzed ia - INNER JOIN channels ch ON ia.channel = ch.channel + INNER JOIN repos_channels_ds ch ON ia.channel = ch.channel WHERE ia.openedAt > now() - INTERVAL 12 MONTH AND ia.respondedInSeconds IS NOT NULL GROUP BY ch.repoUrl ) AS ir @@ -253,7 +222,7 @@ SQL > AND ar.timestamp > now() - INTERVAL 12 MONTH ) AS commitsPrior6m FROM activityRelations_deduplicated_cleaned_bucket_union ar - INNER JOIN channels ch ON ar.channel = ch.channel + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel WHERE ar.type = 'authored-commit' GROUP BY ch.repoUrl ) AS c @@ -274,7 +243,7 @@ SQL > quantileIf(0.5) (ia.closedInSeconds, ia.closedAt > now() - INTERVAL 12 MONTH) AS medianCloseS FROM issues_analyzed ia - INNER JOIN channels ch ON ia.channel = ch.channel + INNER JOIN repos_channels_ds ch ON ia.channel = ch.channel WHERE ia.openedAt > now() - INTERVAL 12 MONTH GROUP BY ch.repoUrl ) AS w @@ -288,7 +257,7 @@ SQL > quantileIf(0.5) (pra.mergedInSeconds, pra.mergedAt > now() - INTERVAL 12 MONTH) AS medianMergeS FROM pull_requests_analyzed pra - INNER JOIN channels ch ON pra.channel = ch.channel + INNER JOIN repos_channels_ds ch ON pra.channel = ch.channel WHERE pra.openedAt > now() - INTERVAL 12 MONTH GROUP BY ch.repoUrl ) AS p @@ -304,7 +273,7 @@ SQL > ) AS unansweredAged90d, count() AS openedLast18m FROM issues_analyzed ia - INNER JOIN channels ch ON ia.channel = ch.channel + INNER JOIN repos_channels_ds ch ON ia.channel = ch.channel WHERE ia.openedAt > now() - INTERVAL 18 MONTH GROUP BY ch.repoUrl ) AS uw @@ -321,7 +290,7 @@ SQL > ) AS unansweredAged90d, count() AS openedLast18m FROM pull_requests_analyzed pra - INNER JOIN channels ch ON pra.channel = ch.channel + INNER JOIN repos_channels_ds ch ON pra.channel = ch.channel WHERE pra.openedAt > now() - INTERVAL 18 MONTH GROUP BY ch.repoUrl ) AS upw diff --git a/services/libs/tinybird/pipes/repos_channels_copy.pipe b/services/libs/tinybird/pipes/repos_channels_copy.pipe new file mode 100644 index 0000000000..d4fea7fd46 --- /dev/null +++ b/services/libs/tinybird/pipes/repos_channels_copy.pipe @@ -0,0 +1,48 @@ +DESCRIPTION > + - Populates `repos_channels_ds` with the expanded channel set for every repository. + - Non-Gerrit repos: one row (channel = url). Gerrit repos: two rows — plain url plus the + /q/project: variant used by Gerrit activity ingestion. + - Mirrors the expansion logic in repos_to_channels.pipe but as a COPY pipe so the result can + be joined by other COPY pipes (which cannot use template-based API pipes). + - Runs daily at 01:00 UTC, before health score pipes (02:00 UTC), so channel mappings are + always fresh when scores are computed. + +TAGS "Repository URLs", "Gerrit", "Health Score v2" + +NODE repos_channels_copy_calc +SQL > + SELECT + r.url AS repoUrl, + r.url AS channel, + (i.platform = 'gerrit') AS isGerrit, + r.excluded AS isExcluded + FROM repositories r FINAL + LEFT JOIN integrations i FINAL ON r.sourceIntegrationId = i.id + WHERE r.deletedAt IS NULL + UNION ALL + SELECT + r.url AS repoUrl, + CASE + WHEN position(r.url, '/r/') > 0 + THEN replaceOne(r.url, '/r/', '/r/q/project:') + WHEN position(r.url, '/gerrit/') > 0 + THEN replaceOne(r.url, '/gerrit/', '/gerrit/q/project:') + ELSE + concat( + protocol(r.url), + '://', + domain(r.url), + '/q/project:', + if(path(r.url) = '/', '', substring(path(r.url), 2)) + ) + END AS channel, + 1 AS isGerrit, + r.excluded AS isExcluded + FROM repositories r FINAL + INNER JOIN integrations i FINAL ON r.sourceIntegrationId = i.id + WHERE r.deletedAt IS NULL AND i.platform = 'gerrit' + +TYPE COPY +TARGET_DATASOURCE repos_channels_ds +COPY_MODE replace +COPY_SCHEDULE 0 1 * * * From 96743f9a64e845fde2012019dbdd4800783f1f1b Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Wed, 12 Aug 2026 17:38:15 +0100 Subject: [PATCH 06/14] fix: remove changelog-style descriptions and fix channels schedule (IN-1229) - Drop 'Gerrit channel fix (IN-1229):' bullets from pipe DESCRIPTION blocks; history belongs in the PR, not inline documentation - Move repos_channels_copy.pipe schedule from 01:00 to 00:00 UTC so the channel mapping is fresh before the monthly snapshot at 00:30 on the 1st Signed-off-by: Joana Maia --- .../tinybird/pipes/health_score_v2_development.pipe | 8 +++----- .../tinybird/pipes/health_score_v2_maintainer.pipe | 12 +++++------- .../pipes/health_score_v2_raw_inputs_snapshot.pipe | 7 +++---- .../libs/tinybird/pipes/repos_channels_copy.pipe | 6 +++--- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/services/libs/tinybird/pipes/health_score_v2_development.pipe b/services/libs/tinybird/pipes/health_score_v2_development.pipe index 932b7e4782..a13aa54640 100644 --- a/services/libs/tinybird/pipes/health_score_v2_development.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_development.pipe @@ -10,11 +10,9 @@ DESCRIPTION > (most repos) having lastCommitAt from GitHub enrichment — the fallback covers the vast majority. - Split out of health_score_v2.pipe into its own copy pipe (2026-07-22) — see health_score_v2_maintainer.pipe description for why. - - Gerrit channel fix (IN-1229): activity rows for Gerrit repos ingested via opendev.org (and - other Gerrit hosts) are stored under the /q/project: channel variant, not the plain repo URL. - All joins against activityRelations, pull_requests_analyzed, and issues_analyzed now route - through `repos_channels_ds` (populated by repos_channels_copy.pipe) which expands each repo URL - to all its possible channel forms. + - All joins against activityRelations, pull_requests_analyzed, and issues_analyzed route through + `repos_channels_ds` (populated by repos_channels_copy.pipe), which expands each Gerrit repo URL + to both its plain URL and /q/project: channel variant. NODE health_score_v2_development_calc SQL > diff --git a/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe b/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe index e93baf9c84..4f3aa869c8 100644 --- a/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe @@ -32,13 +32,11 @@ DESCRIPTION > trailing 12-month window. Final busFactorCount = max(curated_maintainers_count, observed_actors_count). busFactorAvailable = 1 if either source has data. This ensures teams relying on informal review duties aren't penalized for lack of formal role curation. - - Gerrit channel fix (IN-1229): activity rows for Gerrit repos ingested via opendev.org (and - other Gerrit hosts) are stored under the /q/project: channel variant, not the plain repo URL. - All joins against activityRelations, pull_requests_analyzed, and issues_analyzed now route - through `repos_channels_ds` (populated by repos_channels_copy.pipe) which expands each repo URL - to all its possible channel forms. isGerrit now derived from integrations.platform='gerrit' - rather than domain regex, covering git.opendaylight.org and any custom-domain Gerrit instance. - changeset-merged added to observedActorsCount so admin/cherry-pick merges count toward bus factor. + - All joins against activityRelations, pull_requests_analyzed, and issues_analyzed route through + `repos_channels_ds` (populated by repos_channels_copy.pipe), which expands each Gerrit repo URL + to both its plain URL and /q/project: channel variant. isGerrit is derived from + integrations.platform='gerrit', covering all Gerrit hosts regardless of domain. + changeset-merged is counted in observedActorsCount so admin/cherry-pick merges count toward bus factor. NODE health_score_v2_maintainer_calc SQL > diff --git a/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe b/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe index f5e2ae082e..d8fbb44a7f 100644 --- a/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe @@ -21,10 +21,9 @@ DESCRIPTION > per-column (via a `FILTER`/inline date check on `closedAt`/`mergedAt` instead of `openedAt`), which measured a different, unbounded-by-openedAt population than the category pipe — the two could never be reconciled. - - Gerrit channel fix (IN-1229): all activityRelations, pull_requests_analyzed, and - issues_analyzed joins now route through `repos_channels_ds` (populated by repos_channels_copy.pipe) - which expands each Gerrit repo URL to its /q/project: variant so Gerrit activity is correctly - matched. + - All joins against activityRelations, pull_requests_analyzed, and issues_analyzed route through + `repos_channels_ds` (populated by repos_channels_copy.pipe), which expands each Gerrit repo URL + to both its plain URL and /q/project: channel variant. TAGS "Validation", "Health Score v2" diff --git a/services/libs/tinybird/pipes/repos_channels_copy.pipe b/services/libs/tinybird/pipes/repos_channels_copy.pipe index d4fea7fd46..5e68835f4f 100644 --- a/services/libs/tinybird/pipes/repos_channels_copy.pipe +++ b/services/libs/tinybird/pipes/repos_channels_copy.pipe @@ -4,8 +4,8 @@ DESCRIPTION > /q/project: variant used by Gerrit activity ingestion. - Mirrors the expansion logic in repos_to_channels.pipe but as a COPY pipe so the result can be joined by other COPY pipes (which cannot use template-based API pipes). - - Runs daily at 01:00 UTC, before health score pipes (02:00 UTC), so channel mappings are - always fresh when scores are computed. + - Runs daily at 00:00 UTC, before health score pipes (02:00 UTC) and the monthly raw inputs + snapshot (00:30 on the 1st), so channel mappings are always current when scores are computed. TAGS "Repository URLs", "Gerrit", "Health Score v2" @@ -45,4 +45,4 @@ SQL > TYPE COPY TARGET_DATASOURCE repos_channels_ds COPY_MODE replace -COPY_SCHEDULE 0 1 * * * +COPY_SCHEDULE 0 0 * * * From 92b810782ac5aa6d01319ff1a84c8462a6673091 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Wed, 12 Aug 2026 17:41:22 +0100 Subject: [PATCH 07/14] fix: source allRepos from repositories directly in maintainer pipe (IN-1229) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit repos_channels_ds is a secondary derived table — if empty on first deploy or after a failed copy job, building allRepos from it would wipe health_score_v2_maintainer_ds entirely. Source allRepos from repositories FINAL + integrations FINAL directly; use repos_channels_ds only for the activity joins where it's needed for Gerrit channel expansion. Signed-off-by: Joana Maia --- .../libs/tinybird/pipes/health_score_v2_maintainer.pipe | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe b/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe index 4f3aa869c8..fcc095633e 100644 --- a/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe @@ -182,7 +182,13 @@ SQL > ) AS responsivenessScore FROM ( - SELECT DISTINCT repoUrl, isGerrit, isExcluded FROM repos_channels_ds + SELECT + r.url AS repoUrl, + (i.platform = 'gerrit') AS isGerrit, + r.excluded AS isExcluded + FROM repositories r FINAL + LEFT JOIN integrations i FINAL ON r.sourceIntegrationId = i.id + WHERE r.deletedAt IS NULL ) AS allRepos LEFT JOIN ( From 26a4bf27d48f87b9dab0b0d6f5cf6a64dc53fba0 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Thu, 13 Aug 2026 10:18:06 +0100 Subject: [PATCH 08/14] fix: route lifecycle pipe joins through repos_channels_ds (IN-1229) Signed-off-by: Joana Maia --- .../pipes/health_score_v2_lifecycle.pipe | 68 +++++++++++-------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe b/services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe index f99eecc1b3..08dea6c911 100644 --- a/services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe @@ -115,62 +115,70 @@ SQL > LEFT JOIN ( SELECT - channel AS repoUrl, - countIf(timestamp > now() - INTERVAL 6 MONTH) AS commitsLast6m, + ch.repoUrl AS repoUrl, + countIf(ar.timestamp > now() - INTERVAL 6 MONTH) AS commitsLast6m, countIf( - timestamp <= now() - INTERVAL 6 MONTH AND timestamp > now() - INTERVAL 12 MONTH + ar.timestamp <= now() - INTERVAL 6 MONTH + AND ar.timestamp > now() - INTERVAL 12 MONTH ) AS commitsPrior6m - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE type = 'authored-commit' - GROUP BY channel + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel + WHERE ar.type = 'authored-commit' + GROUP BY ch.repoUrl ) AS c ON c.repoUrl = r.url LEFT JOIN ( - SELECT channel AS repoUrl, countIf(closedAt IS NULL) AS issuesOpenNow - FROM issues_analyzed - GROUP BY channel + SELECT ch.repoUrl AS repoUrl, countIf(ia.closedAt IS NULL) AS issuesOpenNow + FROM issues_analyzed ia + INNER JOIN repos_channels_ds ch ON ia.channel = ch.channel + GROUP BY ch.repoUrl ) AS n ON n.repoUrl = r.url LEFT JOIN ( SELECT - channel AS repoUrl, - countIf(openedAt > now() - INTERVAL 6 MONTH) AS issuesOpenedLast6m, + ch.repoUrl AS repoUrl, + countIf(ia.openedAt > now() - INTERVAL 6 MONTH) AS issuesOpenedLast6m, countIf( - openedAt <= now() - INTERVAL 6 MONTH AND openedAt > now() - INTERVAL 12 MONTH + ia.openedAt <= now() - INTERVAL 6 MONTH AND ia.openedAt > now() - INTERVAL 12 MONTH ) AS issuesOpenedPrior6m, - countIf(openedAt > now() - INTERVAL 18 MONTH) AS issuesInWindow18m, + countIf(ia.openedAt > now() - INTERVAL 18 MONTH) AS issuesInWindow18m, countIf( - openedAt > now() - INTERVAL 18 MONTH - AND respondedInSeconds IS NULL - AND closedAt IS NULL - AND openedAt <= now() - INTERVAL 90 DAY + ia.openedAt > now() - INTERVAL 18 MONTH + AND ia.respondedInSeconds IS NULL + AND ia.closedAt IS NULL + AND ia.openedAt <= now() - INTERVAL 90 DAY ) AS unansweredAged90d - FROM issues_analyzed - GROUP BY channel + FROM issues_analyzed ia + INNER JOIN repos_channels_ds ch ON ia.channel = ch.channel + GROUP BY ch.repoUrl ) AS w ON w.repoUrl = r.url LEFT JOIN ( - SELECT channel AS repoUrl, countIf(openedAt > now() - INTERVAL 18 MONTH) AS prsInWindow18m - FROM pull_requests_analyzed - GROUP BY channel + SELECT + ch.repoUrl AS repoUrl, + countIf(pra.openedAt > now() - INTERVAL 18 MONTH) AS prsInWindow18m + FROM pull_requests_analyzed pra + INNER JOIN repos_channels_ds ch ON pra.channel = ch.channel + GROUP BY ch.repoUrl ) AS p ON p.repoUrl = r.url LEFT JOIN ( SELECT - channel AS repoUrl, + ch.repoUrl AS repoUrl, countIf( - openedAt > now() - INTERVAL 18 MONTH - AND reviewedAt IS NULL - AND approvedAt IS NULL - AND closedAt IS NULL - AND openedAt <= now() - INTERVAL 90 DAY + pra.openedAt > now() - INTERVAL 18 MONTH + AND pra.reviewedAt IS NULL + AND pra.approvedAt IS NULL + AND pra.closedAt IS NULL + AND pra.openedAt <= now() - INTERVAL 90 DAY ) AS unansweredAged90d - FROM pull_requests_analyzed - GROUP BY channel + FROM pull_requests_analyzed pra + INNER JOIN repos_channels_ds ch ON pra.channel = ch.channel + GROUP BY ch.repoUrl ) AS pw ON pw.repoUrl = r.url LEFT JOIN From 18019976f1f22a4ea17dd0efd754d761756a2468 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Thu, 13 Aug 2026 10:31:01 +0100 Subject: [PATCH 09/14] fix: exclude archived (v2) and excluded repos across health score pipes (IN-1229) Signed-off-by: Joana Maia --- .../libs/tinybird/pipes/health_score_v2_development.pipe | 7 ++++++- services/libs/tinybird/pipes/health_score_v2_impact.pipe | 2 +- .../libs/tinybird/pipes/health_score_v2_lifecycle.pipe | 7 ++++++- .../libs/tinybird/pipes/health_score_v2_maintainer.pipe | 2 +- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/services/libs/tinybird/pipes/health_score_v2_development.pipe b/services/libs/tinybird/pipes/health_score_v2_development.pipe index a13aa54640..57b7450f03 100644 --- a/services/libs/tinybird/pipes/health_score_v2_development.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_development.pipe @@ -150,7 +150,12 @@ SQL > FROM ( SELECT base.url AS repoUrl, rc.lastCommitAt AS lastCommitAt - FROM (SELECT DISTINCT url FROM repositories WHERE deletedAt IS NULL) AS base + FROM + ( + SELECT DISTINCT url + FROM repositories + WHERE deletedAt IS NULL AND archived = false AND excluded = false + ) AS base LEFT JOIN ( SELECT url, argMax(lastCommitAt, updatedAt) AS lastCommitAt diff --git a/services/libs/tinybird/pipes/health_score_v2_impact.pipe b/services/libs/tinybird/pipes/health_score_v2_impact.pipe index 75b968c9d7..33c1f3ed21 100644 --- a/services/libs/tinybird/pipes/health_score_v2_impact.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_impact.pipe @@ -11,7 +11,7 @@ DESCRIPTION > NODE health_score_v2_impact_calc SQL > SELECT base.url AS repoUrl, max(toFloat64OrNull(pk.impact)) * 100 AS impactScoreRaw - FROM (SELECT DISTINCT url FROM repositories WHERE deletedAt IS NULL) AS base + FROM (SELECT DISTINCT url FROM repositories WHERE deletedAt IS NULL AND excluded = false) AS base INNER JOIN repos r ON r.url = base.url INNER JOIN packageRepos pr ON pr.repoId = r.id INNER JOIN ossPackages_enriched_ds pk ON pk.id = pr.packageId diff --git a/services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe b/services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe index 08dea6c911..44c9edb99b 100644 --- a/services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe @@ -105,7 +105,12 @@ SQL > FROM ( SELECT base.url AS url, base.archived AS archived, rc.lastCommitAt AS lastCommitAt - FROM (SELECT DISTINCT url, archived FROM repositories WHERE deletedAt IS NULL) AS base + FROM + ( + SELECT DISTINCT url, archived + FROM repositories + WHERE deletedAt IS NULL AND excluded = false + ) AS base LEFT JOIN ( SELECT url, argMax(lastCommitAt, updatedAt) AS lastCommitAt FROM repos GROUP BY url diff --git a/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe b/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe index fcc095633e..cd06bd7cce 100644 --- a/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe @@ -188,7 +188,7 @@ SQL > r.excluded AS isExcluded FROM repositories r FINAL LEFT JOIN integrations i FINAL ON r.sourceIntegrationId = i.id - WHERE r.deletedAt IS NULL + WHERE r.deletedAt IS NULL AND r.archived = false AND r.excluded = false ) AS allRepos LEFT JOIN ( From 72ad628c4a374e4bffe69da0f1423663bbb6d29c Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Thu, 13 Aug 2026 10:35:05 +0100 Subject: [PATCH 10/14] fix: exclude archived + excluded repos in v2 security, aggregator, and signal detail pipes (IN-1229) Signed-off-by: Joana Maia --- services/libs/tinybird/pipes/health_score_v2.pipe | 7 ++++++- services/libs/tinybird/pipes/health_score_v2_security.pipe | 7 ++++++- .../libs/tinybird/pipes/health_score_v2_signal_detail.pipe | 7 ++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/services/libs/tinybird/pipes/health_score_v2.pipe b/services/libs/tinybird/pipes/health_score_v2.pipe index c2ce47bd11..2a0fc69e72 100644 --- a/services/libs/tinybird/pipes/health_score_v2.pipe +++ b/services/libs/tinybird/pipes/health_score_v2.pipe @@ -61,7 +61,12 @@ SQL > ) AS coveredCategoryWeight, l.lifecycleLabelV2 AS lifecycleLabelV2, i.impactScoreRaw AS impactScoreRaw - FROM (SELECT DISTINCT url AS repoUrl FROM repositories WHERE deletedAt IS NULL) AS base + FROM + ( + SELECT DISTINCT url AS repoUrl + FROM repositories + WHERE deletedAt IS NULL AND archived = false AND excluded = false + ) AS base LEFT JOIN health_score_v2_maintainer_ds AS m ON m.repoUrl = base.repoUrl LEFT JOIN health_score_v2_security_ds AS s ON s.repoUrl = base.repoUrl LEFT JOIN health_score_v2_development_ds AS d ON d.repoUrl = base.repoUrl diff --git a/services/libs/tinybird/pipes/health_score_v2_security.pipe b/services/libs/tinybird/pipes/health_score_v2_security.pipe index 1521e560b8..d4aa4ce1cd 100644 --- a/services/libs/tinybird/pipes/health_score_v2_security.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_security.pipe @@ -117,7 +117,12 @@ SQL > ) AS dependencyHealthScore, (pkgs.repoUrl != '') AS dependencyHealthAvailable, dh.vulnerableDeps AS vulnerableDeps - FROM (SELECT DISTINCT url FROM repositories WHERE deletedAt IS NULL) AS allRepos + FROM + ( + SELECT DISTINCT url + FROM repositories + WHERE deletedAt IS NULL AND archived = false AND excluded = false + ) AS allRepos LEFT JOIN ( SELECT diff --git a/services/libs/tinybird/pipes/health_score_v2_signal_detail.pipe b/services/libs/tinybird/pipes/health_score_v2_signal_detail.pipe index ff51cf9cfc..621d9bdf64 100644 --- a/services/libs/tinybird/pipes/health_score_v2_signal_detail.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_signal_detail.pipe @@ -69,7 +69,12 @@ SQL > d.closedUnmerged12m AS closedUnmerged12m, d.medianMergeS AS medianMergeS, coalesce(m.methodologyVersion, '2.0.0') AS methodologyVersion - FROM (SELECT DISTINCT url AS repoUrl FROM repositories FINAL WHERE isNull (deletedAt)) AS base + FROM + ( + SELECT DISTINCT url AS repoUrl + FROM repositories FINAL + WHERE deletedAt IS NULL AND archived = false AND excluded = false + ) AS base LEFT JOIN health_score_v2_maintainer_ds AS m ON m.repoUrl = base.repoUrl LEFT JOIN health_score_v2_security_ds AS s ON s.repoUrl = base.repoUrl LEFT JOIN health_score_v2_development_ds AS d ON d.repoUrl = base.repoUrl From fc527787855ca493404ad9ee5fa31b6340d929a5 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Thu, 13 Aug 2026 11:10:12 +0100 Subject: [PATCH 11/14] fix: add FINAL to repositories and vulnerabilities reads in lifecycle and security pipes (IN-1229) Signed-off-by: Joana Maia --- services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe | 4 ++-- services/libs/tinybird/pipes/health_score_v2_security.pipe | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe b/services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe index 44c9edb99b..414b3058de 100644 --- a/services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe @@ -108,7 +108,7 @@ SQL > FROM ( SELECT DISTINCT url, archived - FROM repositories + FROM repositories FINAL WHERE deletedAt IS NULL AND excluded = false ) AS base LEFT JOIN @@ -189,7 +189,7 @@ SQL > LEFT JOIN ( SELECT repoUrl, countIf(status = 'OPEN' AND severity = 'CRITICAL') AS openCriticals - FROM vulnerabilities + FROM vulnerabilities FINAL GROUP BY repoUrl ) AS v ON v.repoUrl = r.url diff --git a/services/libs/tinybird/pipes/health_score_v2_security.pipe b/services/libs/tinybird/pipes/health_score_v2_security.pipe index d4aa4ce1cd..a9e0f80f19 100644 --- a/services/libs/tinybird/pipes/health_score_v2_security.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_security.pipe @@ -120,7 +120,7 @@ SQL > FROM ( SELECT DISTINCT url - FROM repositories + FROM repositories FINAL WHERE deletedAt IS NULL AND archived = false AND excluded = false ) AS allRepos LEFT JOIN @@ -150,7 +150,7 @@ SQL > countIf(status = 'OPEN' AND severity = 'CRITICAL') AS openCriticals, countIf(status = 'OPEN' AND severity = 'HIGH') AS openHighs, countIf(status = 'OPEN' AND severity = 'MEDIUM') AS openModerates - FROM vulnerabilities + FROM vulnerabilities FINAL GROUP BY repoUrl ) AS vc ON vc.repoUrl = allRepos.url From 14eb2e57bd4fc2713b1fa3be6dd426e71a9e636d Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Thu, 13 Aug 2026 13:50:52 +0100 Subject: [PATCH 12/14] fix: include archived repos in aggregator and signal detail pipes (IN-1229) Signed-off-by: Joana Maia --- services/libs/tinybird/pipes/health_score_v2.pipe | 4 ++-- .../libs/tinybird/pipes/health_score_v2_signal_detail.pipe | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/libs/tinybird/pipes/health_score_v2.pipe b/services/libs/tinybird/pipes/health_score_v2.pipe index 2a0fc69e72..96e6218bff 100644 --- a/services/libs/tinybird/pipes/health_score_v2.pipe +++ b/services/libs/tinybird/pipes/health_score_v2.pipe @@ -64,8 +64,8 @@ SQL > FROM ( SELECT DISTINCT url AS repoUrl - FROM repositories - WHERE deletedAt IS NULL AND archived = false AND excluded = false + FROM repositories FINAL + WHERE deletedAt IS NULL AND excluded = false ) AS base LEFT JOIN health_score_v2_maintainer_ds AS m ON m.repoUrl = base.repoUrl LEFT JOIN health_score_v2_security_ds AS s ON s.repoUrl = base.repoUrl diff --git a/services/libs/tinybird/pipes/health_score_v2_signal_detail.pipe b/services/libs/tinybird/pipes/health_score_v2_signal_detail.pipe index 621d9bdf64..9ae0a55759 100644 --- a/services/libs/tinybird/pipes/health_score_v2_signal_detail.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_signal_detail.pipe @@ -73,7 +73,7 @@ SQL > ( SELECT DISTINCT url AS repoUrl FROM repositories FINAL - WHERE deletedAt IS NULL AND archived = false AND excluded = false + WHERE deletedAt IS NULL AND excluded = false ) AS base LEFT JOIN health_score_v2_maintainer_ds AS m ON m.repoUrl = base.repoUrl LEFT JOIN health_score_v2_security_ds AS s ON s.repoUrl = base.repoUrl From 849feff4bd767b513c4357af54f3841c11e5975f Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Thu, 13 Aug 2026 16:26:18 +0100 Subject: [PATCH 13/14] fix: add FINAL to repositories reads in development and impact pipes (IN-1229) Signed-off-by: Joana Maia --- .../libs/tinybird/pipes/health_score_v2_development.pipe | 2 +- services/libs/tinybird/pipes/health_score_v2_impact.pipe | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/services/libs/tinybird/pipes/health_score_v2_development.pipe b/services/libs/tinybird/pipes/health_score_v2_development.pipe index 57b7450f03..439e77e53a 100644 --- a/services/libs/tinybird/pipes/health_score_v2_development.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_development.pipe @@ -153,7 +153,7 @@ SQL > FROM ( SELECT DISTINCT url - FROM repositories + FROM repositories FINAL WHERE deletedAt IS NULL AND archived = false AND excluded = false ) AS base LEFT JOIN diff --git a/services/libs/tinybird/pipes/health_score_v2_impact.pipe b/services/libs/tinybird/pipes/health_score_v2_impact.pipe index 33c1f3ed21..a90e06869d 100644 --- a/services/libs/tinybird/pipes/health_score_v2_impact.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_impact.pipe @@ -11,7 +11,12 @@ DESCRIPTION > NODE health_score_v2_impact_calc SQL > SELECT base.url AS repoUrl, max(toFloat64OrNull(pk.impact)) * 100 AS impactScoreRaw - FROM (SELECT DISTINCT url FROM repositories WHERE deletedAt IS NULL AND excluded = false) AS base + FROM + ( + SELECT DISTINCT url + FROM repositories FINAL + WHERE deletedAt IS NULL AND excluded = false + ) AS base INNER JOIN repos r ON r.url = base.url INNER JOIN packageRepos pr ON pr.repoId = r.id INNER JOIN ossPackages_enriched_ds pk ON pk.id = pr.packageId From 2e4f977d63675fb4b232324c627398c7d69a028d Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Thu, 13 Aug 2026 17:02:38 +0100 Subject: [PATCH 14/14] fix: formatting Signed-off-by: Joana Maia --- services/libs/tinybird/pipes/health_score_v2_impact.pipe | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/libs/tinybird/pipes/health_score_v2_impact.pipe b/services/libs/tinybird/pipes/health_score_v2_impact.pipe index a90e06869d..8d769a15bf 100644 --- a/services/libs/tinybird/pipes/health_score_v2_impact.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_impact.pipe @@ -13,9 +13,7 @@ SQL > SELECT base.url AS repoUrl, max(toFloat64OrNull(pk.impact)) * 100 AS impactScoreRaw FROM ( - SELECT DISTINCT url - FROM repositories FINAL - WHERE deletedAt IS NULL AND excluded = false + SELECT DISTINCT url FROM repositories FINAL WHERE deletedAt IS NULL AND excluded = false ) AS base INNER JOIN repos r ON r.url = base.url INNER JOIN packageRepos pr ON pr.repoId = r.id