diff --git a/services/libs/tinybird/datasources/repos_channels_ds.datasource b/services/libs/tinybird/datasources/repos_channels_ds.datasource new file mode 100644 index 0000000000..a742f3d10b --- /dev/null +++ b/services/libs/tinybird/datasources/repos_channels_ds.datasource @@ -0,0 +1,18 @@ +DESCRIPTION > + - Materialized channel expansion for all repositories. Populated by `repos_channels_copy.pipe` + (daily, before health score pipes run). + - Each non-Gerrit repo produces one row (channel = url). Each Gerrit repo produces two rows: + the plain URL and the /q/project: variant used by Gerrit activity ingestion. + - `isGerrit` and `isExcluded` are denormalized from `repositories`/`integrations` so health + score pipes can classify repos without re-joining those tables. + - Sorting key is `channel` because all consumers join on `channel`; `repoUrl` lookups are + full-scans but the table is small enough that this is fine. + +SCHEMA > + `repoUrl` String, + `channel` String, + `isGerrit` UInt8, + `isExcluded` UInt8 + +ENGINE MergeTree +ENGINE_SORTING_KEY channel diff --git a/services/libs/tinybird/pipes/health_score_v2.pipe b/services/libs/tinybird/pipes/health_score_v2.pipe index c2ce47bd11..96e6218bff 100644 --- a/services/libs/tinybird/pipes/health_score_v2.pipe +++ b/services/libs/tinybird/pipes/health_score_v2.pipe @@ -61,7 +61,12 @@ SQL > ) AS coveredCategoryWeight, l.lifecycleLabelV2 AS lifecycleLabelV2, i.impactScoreRaw AS impactScoreRaw - FROM (SELECT DISTINCT url AS repoUrl FROM repositories WHERE deletedAt IS NULL) AS base + FROM + ( + SELECT DISTINCT url AS repoUrl + FROM repositories FINAL + WHERE deletedAt IS NULL AND excluded = false + ) AS base LEFT JOIN health_score_v2_maintainer_ds AS m ON m.repoUrl = base.repoUrl LEFT JOIN health_score_v2_security_ds AS s ON s.repoUrl = base.repoUrl LEFT JOIN health_score_v2_development_ds AS d ON d.repoUrl = base.repoUrl diff --git a/services/libs/tinybird/pipes/health_score_v2_development.pipe b/services/libs/tinybird/pipes/health_score_v2_development.pipe index 3f79e90194..439e77e53a 100644 --- a/services/libs/tinybird/pipes/health_score_v2_development.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_development.pipe @@ -10,6 +10,9 @@ DESCRIPTION > (most repos) having lastCommitAt from GitHub enrichment — the fallback covers the vast majority. - Split out of health_score_v2.pipe into its own copy pipe (2026-07-22) — see health_score_v2_maintainer.pipe description for why. + - All joins against activityRelations, pull_requests_analyzed, and issues_analyzed route through + `repos_channels_ds` (populated by repos_channels_copy.pipe), which expands each Gerrit repo URL + to both its plain URL and /q/project: channel variant. NODE health_score_v2_development_calc SQL > @@ -147,7 +150,12 @@ SQL > FROM ( SELECT base.url AS repoUrl, rc.lastCommitAt AS lastCommitAt - FROM (SELECT DISTINCT url FROM repositories WHERE deletedAt IS NULL) AS base + FROM + ( + SELECT DISTINCT url + FROM repositories FINAL + WHERE deletedAt IS NULL AND archived = false AND excluded = false + ) AS base LEFT JOIN ( SELECT url, argMax(lastCommitAt, updatedAt) AS lastCommitAt @@ -192,37 +200,46 @@ SQL > LEFT JOIN ( SELECT - channel AS repoUrl, - countIf(timestamp > now() - INTERVAL 6 MONTH) AS commitsLast6m - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE type = 'authored-commit' - GROUP BY channel + ch.repoUrl AS repoUrl, + countIf(ar.timestamp > now() - INTERVAL 6 MONTH) AS commitsLast6m + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel + WHERE ar.type = 'authored-commit' + GROUP BY ch.repoUrl ) AS c ON c.repoUrl = rp.repoUrl LEFT JOIN ( SELECT - channel AS repoUrl, - countIf(closedAt IS NOT NULL) AS closed12m, + ch.repoUrl AS repoUrl, + countIf(ia.closedAt IS NOT NULL) AS closed12m, count() AS opened12m, quantileIf(0.5) - (closedInSeconds, closedAt > now() - INTERVAL 12 MONTH) AS medianCloseS - FROM issues_analyzed - WHERE openedAt > now() - INTERVAL 12 MONTH - GROUP BY channel + ( + ia.closedInSeconds, ia.closedAt > now() - INTERVAL 12 MONTH + ) AS medianCloseS + FROM issues_analyzed ia + INNER JOIN repos_channels_ds ch ON ia.channel = ch.channel + WHERE ia.openedAt > now() - INTERVAL 12 MONTH + GROUP BY ch.repoUrl ) AS ist ON ist.repoUrl = rp.repoUrl LEFT JOIN ( SELECT - channel AS repoUrl, - countIf(mergedAt IS NOT NULL) AS merged12m, - countIf(closedAt IS NOT NULL AND mergedAt IS NULL) AS closedUnmerged12m, + ch.repoUrl AS repoUrl, + countIf(pra.mergedAt IS NOT NULL) AS merged12m, + countIf( + pra.closedAt IS NOT NULL AND pra.mergedAt IS NULL + ) AS closedUnmerged12m, quantileIf(0.5) - (mergedInSeconds, mergedAt > now() - INTERVAL 12 MONTH) AS medianMergeS - FROM pull_requests_analyzed - WHERE openedAt > now() - INTERVAL 12 MONTH - GROUP BY channel + ( + pra.mergedInSeconds, pra.mergedAt > now() - INTERVAL 12 MONTH + ) AS medianMergeS + FROM pull_requests_analyzed pra + INNER JOIN repos_channels_ds ch ON pra.channel = ch.channel + WHERE pra.openedAt > now() - INTERVAL 12 MONTH + GROUP BY ch.repoUrl ) AS prs ON prs.repoUrl = rp.repoUrl ) diff --git a/services/libs/tinybird/pipes/health_score_v2_impact.pipe b/services/libs/tinybird/pipes/health_score_v2_impact.pipe index 75b968c9d7..8d769a15bf 100644 --- a/services/libs/tinybird/pipes/health_score_v2_impact.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_impact.pipe @@ -11,7 +11,10 @@ DESCRIPTION > NODE health_score_v2_impact_calc SQL > SELECT base.url AS repoUrl, max(toFloat64OrNull(pk.impact)) * 100 AS impactScoreRaw - FROM (SELECT DISTINCT url FROM repositories WHERE deletedAt IS NULL) AS base + FROM + ( + SELECT DISTINCT url FROM repositories FINAL WHERE deletedAt IS NULL AND excluded = false + ) AS base INNER JOIN repos r ON r.url = base.url INNER JOIN packageRepos pr ON pr.repoId = r.id INNER JOIN ossPackages_enriched_ds pk ON pk.id = pr.packageId diff --git a/services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe b/services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe index f99eecc1b3..414b3058de 100644 --- a/services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_lifecycle.pipe @@ -105,7 +105,12 @@ SQL > FROM ( SELECT base.url AS url, base.archived AS archived, rc.lastCommitAt AS lastCommitAt - FROM (SELECT DISTINCT url, archived FROM repositories WHERE deletedAt IS NULL) AS base + FROM + ( + SELECT DISTINCT url, archived + FROM repositories FINAL + WHERE deletedAt IS NULL AND excluded = false + ) AS base LEFT JOIN ( SELECT url, argMax(lastCommitAt, updatedAt) AS lastCommitAt FROM repos GROUP BY url @@ -115,68 +120,76 @@ SQL > LEFT JOIN ( SELECT - channel AS repoUrl, - countIf(timestamp > now() - INTERVAL 6 MONTH) AS commitsLast6m, + ch.repoUrl AS repoUrl, + countIf(ar.timestamp > now() - INTERVAL 6 MONTH) AS commitsLast6m, countIf( - timestamp <= now() - INTERVAL 6 MONTH AND timestamp > now() - INTERVAL 12 MONTH + ar.timestamp <= now() - INTERVAL 6 MONTH + AND ar.timestamp > now() - INTERVAL 12 MONTH ) AS commitsPrior6m - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE type = 'authored-commit' - GROUP BY channel + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel + WHERE ar.type = 'authored-commit' + GROUP BY ch.repoUrl ) AS c ON c.repoUrl = r.url LEFT JOIN ( - SELECT channel AS repoUrl, countIf(closedAt IS NULL) AS issuesOpenNow - FROM issues_analyzed - GROUP BY channel + SELECT ch.repoUrl AS repoUrl, countIf(ia.closedAt IS NULL) AS issuesOpenNow + FROM issues_analyzed ia + INNER JOIN repos_channels_ds ch ON ia.channel = ch.channel + GROUP BY ch.repoUrl ) AS n ON n.repoUrl = r.url LEFT JOIN ( SELECT - channel AS repoUrl, - countIf(openedAt > now() - INTERVAL 6 MONTH) AS issuesOpenedLast6m, + ch.repoUrl AS repoUrl, + countIf(ia.openedAt > now() - INTERVAL 6 MONTH) AS issuesOpenedLast6m, countIf( - openedAt <= now() - INTERVAL 6 MONTH AND openedAt > now() - INTERVAL 12 MONTH + ia.openedAt <= now() - INTERVAL 6 MONTH AND ia.openedAt > now() - INTERVAL 12 MONTH ) AS issuesOpenedPrior6m, - countIf(openedAt > now() - INTERVAL 18 MONTH) AS issuesInWindow18m, + countIf(ia.openedAt > now() - INTERVAL 18 MONTH) AS issuesInWindow18m, countIf( - openedAt > now() - INTERVAL 18 MONTH - AND respondedInSeconds IS NULL - AND closedAt IS NULL - AND openedAt <= now() - INTERVAL 90 DAY + ia.openedAt > now() - INTERVAL 18 MONTH + AND ia.respondedInSeconds IS NULL + AND ia.closedAt IS NULL + AND ia.openedAt <= now() - INTERVAL 90 DAY ) AS unansweredAged90d - FROM issues_analyzed - GROUP BY channel + FROM issues_analyzed ia + INNER JOIN repos_channels_ds ch ON ia.channel = ch.channel + GROUP BY ch.repoUrl ) AS w ON w.repoUrl = r.url LEFT JOIN ( - SELECT channel AS repoUrl, countIf(openedAt > now() - INTERVAL 18 MONTH) AS prsInWindow18m - FROM pull_requests_analyzed - GROUP BY channel + SELECT + ch.repoUrl AS repoUrl, + countIf(pra.openedAt > now() - INTERVAL 18 MONTH) AS prsInWindow18m + FROM pull_requests_analyzed pra + INNER JOIN repos_channels_ds ch ON pra.channel = ch.channel + GROUP BY ch.repoUrl ) AS p ON p.repoUrl = r.url LEFT JOIN ( SELECT - channel AS repoUrl, + ch.repoUrl AS repoUrl, countIf( - openedAt > now() - INTERVAL 18 MONTH - AND reviewedAt IS NULL - AND approvedAt IS NULL - AND closedAt IS NULL - AND openedAt <= now() - INTERVAL 90 DAY + pra.openedAt > now() - INTERVAL 18 MONTH + AND pra.reviewedAt IS NULL + AND pra.approvedAt IS NULL + AND pra.closedAt IS NULL + AND pra.openedAt <= now() - INTERVAL 90 DAY ) AS unansweredAged90d - FROM pull_requests_analyzed - GROUP BY channel + FROM pull_requests_analyzed pra + INNER JOIN repos_channels_ds ch ON pra.channel = ch.channel + GROUP BY ch.repoUrl ) AS pw ON pw.repoUrl = r.url LEFT JOIN ( SELECT repoUrl, countIf(status = 'OPEN' AND severity = 'CRITICAL') AS openCriticals - FROM vulnerabilities + FROM vulnerabilities FINAL GROUP BY repoUrl ) AS v ON v.repoUrl = r.url diff --git a/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe b/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe index 73dfc398cb..cd06bd7cce 100644 --- a/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_maintainer.pipe @@ -6,9 +6,9 @@ DESCRIPTION > Responsiveness follows Joana's 2026-07-23 review (per-platform, not a blanket rule): - GitHub/GitLab (and any other non-Gerrit host): scores 0/15 when there are no PRs or issues in the window — that's real signal (unresponsive), not a data gap, so it's never `blocked`. - - Gerrit (review.opendev.org, gerrit.*): Gerrit has no issue tracker, so only PR/changeset - response time counts. Scores 0/15 when there's no changeset data; issue-response is excluded - from the calculation entirely rather than penalizing repos for a concept that doesn't exist there. + - Gerrit: Gerrit has no issue tracker, so only PR/changeset response time counts. Scores 0/15 + when there's no changeset data; issue-response is excluded from the calculation entirely rather + than penalizing repos for a concept that doesn't exist there. - Repos marked `excluded` in the repositories table (e.g. .github meta repos where no PRs or issues are ever expected) are `blocked` for responsiveness regardless of platform — never scored 0 for an absence that was never expected to be filled. @@ -32,6 +32,11 @@ DESCRIPTION > trailing 12-month window. Final busFactorCount = max(curated_maintainers_count, observed_actors_count). busFactorAvailable = 1 if either source has data. This ensures teams relying on informal review duties aren't penalized for lack of formal role curation. + - All joins against activityRelations, pull_requests_analyzed, and issues_analyzed route through + `repos_channels_ds` (populated by repos_channels_copy.pipe), which expands each Gerrit repo URL + to both its plain URL and /q/project: channel variant. isGerrit is derived from + integrations.platform='gerrit', covering all Gerrit hosts regardless of domain. + changeset-merged is counted in observedActorsCount so admin/cherry-pick merges count toward bus factor. NODE health_score_v2_maintainer_calc SQL > @@ -177,14 +182,13 @@ SQL > ) AS responsivenessScore FROM ( - SELECT DISTINCT - url AS repoUrl, - ( - domain(url) = 'review.opendev.org' OR domain(url) LIKE 'gerrit.%' - ) AS isGerrit, - excluded AS isExcluded - FROM repositories - WHERE deletedAt IS NULL + SELECT + r.url AS repoUrl, + (i.platform = 'gerrit') AS isGerrit, + r.excluded AS isExcluded + FROM repositories r FINAL + LEFT JOIN integrations i FINAL ON r.sourceIntegrationId = i.id + WHERE r.deletedAt IS NULL AND r.archived = false AND r.excluded = false ) AS allRepos LEFT JOIN ( @@ -194,9 +198,10 @@ SQL > FROM maintainers_roles_copy_ds mr INNER JOIN ( - SELECT DISTINCT memberId, channel AS repoUrl - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE timestamp > now() - INTERVAL 12 MONTH + SELECT DISTINCT ch.repoUrl AS repoUrl, ar.memberId AS memberId + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel + WHERE ar.timestamp > now() - INTERVAL 12 MONTH ) AS recentActivity ON recentActivity.memberId = mr.memberId AND recentActivity.repoUrl = mr.repoUrl @@ -208,52 +213,62 @@ SQL > ON bf.repoUrl = allRepos.repoUrl LEFT JOIN ( - SELECT channel AS repoUrl, count(DISTINCT memberId) AS observedActorsCount - FROM activityRelations_deduplicated_cleaned_bucket_union + SELECT + ch.repoUrl AS repoUrl, + count(DISTINCT ar.memberId) AS observedActorsCount + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel WHERE - timestamp > now() - INTERVAL 12 MONTH - AND memberId != '' + ar.timestamp > now() - INTERVAL 12 MONTH + AND ar.memberId != '' AND ( - type = 'pull_request-reviewed' - OR type = 'pull_request-merged' - OR type = 'merge_request-review-approved' - OR type = 'merge_request-merged' - OR type = 'patchset_approval-created' - OR type = 'merge_request-review-unapproved' - OR type = 'merge_request-review-commented' + ar.type = 'pull_request-reviewed' + OR ar.type = 'pull_request-merged' + OR ar.type = 'merge_request-review-approved' + OR ar.type = 'merge_request-merged' + OR ar.type = 'patchset_approval-created' + OR ar.type = 'merge_request-review-unapproved' + OR ar.type = 'merge_request-review-commented' + OR ar.type = 'changeset-merged' ) - GROUP BY channel + GROUP BY ch.repoUrl ) AS obs ON obs.repoUrl = allRepos.repoUrl LEFT JOIN ( - SELECT channel AS repoUrl, count(DISTINCT organizationId) AS orgCount - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE timestamp > now() - INTERVAL 12 MONTH AND organizationId != '' - GROUP BY channel + SELECT ch.repoUrl AS repoUrl, count(DISTINCT ar.organizationId) AS orgCount + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel + WHERE ar.timestamp > now() - INTERVAL 12 MONTH AND ar.organizationId != '' + GROUP BY ch.repoUrl ) AS od ON od.repoUrl = allRepos.repoUrl LEFT JOIN ( SELECT - channel AS repoUrl, + ch.repoUrl AS repoUrl, quantile(0.5) - (reviewedInSeconds) AS medianPrResponseS - FROM pull_requests_analyzed - WHERE openedAt > now() - INTERVAL 12 MONTH AND reviewedInSeconds IS NOT NULL - GROUP BY channel + (pra.reviewedInSeconds) AS medianPrResponseS + FROM pull_requests_analyzed pra + INNER JOIN repos_channels_ds ch ON pra.channel = ch.channel + WHERE + pra.openedAt > now() - INTERVAL 12 MONTH + AND pra.reviewedInSeconds IS NOT NULL + GROUP BY ch.repoUrl ) AS r ON r.repoUrl = allRepos.repoUrl LEFT JOIN ( SELECT - channel AS repoUrl, + ch.repoUrl AS repoUrl, quantile(0.5) - (respondedInSeconds) AS medianIssueResponseS - FROM issues_analyzed + (ia.respondedInSeconds) AS medianIssueResponseS + FROM issues_analyzed ia + INNER JOIN repos_channels_ds ch ON ia.channel = ch.channel WHERE - openedAt > now() - INTERVAL 12 MONTH AND respondedInSeconds IS NOT NULL - GROUP BY channel + ia.openedAt > now() - INTERVAL 12 MONTH + AND ia.respondedInSeconds IS NOT NULL + GROUP BY ch.repoUrl ) AS ir ON ir.repoUrl = allRepos.repoUrl ) diff --git a/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe b/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe index 54a3d0cbfd..d8fbb44a7f 100644 --- a/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_raw_inputs_snapshot.pipe @@ -21,6 +21,9 @@ DESCRIPTION > per-column (via a `FILTER`/inline date check on `closedAt`/`mergedAt` instead of `openedAt`), which measured a different, unbounded-by-openedAt population than the category pipe — the two could never be reconciled. + - All joins against activityRelations, pull_requests_analyzed, and issues_analyzed route through + `repos_channels_ds` (populated by repos_channels_copy.pipe), which expands each Gerrit repo URL + to both its plain URL and /q/project: channel variant. TAGS "Validation", "Health Score v2" @@ -80,9 +83,10 @@ SQL > FROM maintainers_roles_copy_ds mr INNER JOIN ( - SELECT DISTINCT memberId, channel AS repoUrl - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE timestamp > now() - INTERVAL 12 MONTH + SELECT DISTINCT ch.repoUrl AS repoUrl, ar.memberId AS memberId + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel + WHERE ar.timestamp > now() - INTERVAL 12 MONTH ) AS recentActivity ON recentActivity.memberId = mr.memberId AND recentActivity.repoUrl = mr.repoUrl @@ -92,45 +96,50 @@ SQL > ON bf.repoUrl = allRepos.repoUrl LEFT JOIN ( - SELECT channel AS repoUrl, count(DISTINCT memberId) AS observedActorsCount - FROM activityRelations_deduplicated_cleaned_bucket_union + SELECT ch.repoUrl AS repoUrl, count(DISTINCT ar.memberId) AS observedActorsCount + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel WHERE - timestamp > now() - INTERVAL 12 MONTH - AND memberId != '' + ar.timestamp > now() - INTERVAL 12 MONTH + AND ar.memberId != '' AND ( - type = 'pull_request-reviewed' - OR type = 'pull_request-merged' - OR type = 'merge_request-review-approved' - OR type = 'merge_request-merged' - OR type = 'patchset_approval-created' - OR type = 'merge_request-review-unapproved' - OR type = 'merge_request-review-commented' + ar.type = 'pull_request-reviewed' + OR ar.type = 'pull_request-merged' + OR ar.type = 'merge_request-review-approved' + OR ar.type = 'merge_request-merged' + OR ar.type = 'patchset_approval-created' + OR ar.type = 'merge_request-review-unapproved' + OR ar.type = 'merge_request-review-commented' + OR ar.type = 'changeset-merged' ) - GROUP BY channel + GROUP BY ch.repoUrl ) AS obs ON obs.repoUrl = allRepos.repoUrl LEFT JOIN ( - SELECT channel AS repoUrl, count(DISTINCT organizationId) AS orgCount - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE timestamp > now() - INTERVAL 12 MONTH AND organizationId != '' - GROUP BY channel + SELECT ch.repoUrl AS repoUrl, count(DISTINCT ar.organizationId) AS orgCount + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel + WHERE ar.timestamp > now() - INTERVAL 12 MONTH AND ar.organizationId != '' + GROUP BY ch.repoUrl ) AS od ON od.repoUrl = allRepos.repoUrl LEFT JOIN ( - SELECT channel AS repoUrl, quantile(0.5)(reviewedInSeconds) AS medianPrResponseS - FROM pull_requests_analyzed - WHERE openedAt > now() - INTERVAL 12 MONTH AND reviewedInSeconds IS NOT NULL - GROUP BY channel + SELECT ch.repoUrl AS repoUrl, quantile(0.5)(pra.reviewedInSeconds) AS medianPrResponseS + FROM pull_requests_analyzed pra + INNER JOIN repos_channels_ds ch ON pra.channel = ch.channel + WHERE pra.openedAt > now() - INTERVAL 12 MONTH AND pra.reviewedInSeconds IS NOT NULL + GROUP BY ch.repoUrl ) AS r ON r.repoUrl = allRepos.repoUrl LEFT JOIN ( - SELECT channel AS repoUrl, quantile(0.5)(respondedInSeconds) AS medianIssueResponseS - FROM issues_analyzed - WHERE openedAt > now() - INTERVAL 12 MONTH AND respondedInSeconds IS NOT NULL - GROUP BY channel + SELECT ch.repoUrl AS repoUrl, quantile(0.5)(ia.respondedInSeconds) AS medianIssueResponseS + FROM issues_analyzed ia + INNER JOIN repos_channels_ds ch ON ia.channel = ch.channel + WHERE ia.openedAt > now() - INTERVAL 12 MONTH AND ia.respondedInSeconds IS NOT NULL + GROUP BY ch.repoUrl ) AS ir ON ir.repoUrl = allRepos.repoUrl LEFT JOIN @@ -205,14 +214,16 @@ SQL > LEFT JOIN ( SELECT - channel AS repoUrl, - countIf(timestamp > now() - INTERVAL 6 MONTH) AS commitsLast6m, + ch.repoUrl AS repoUrl, + countIf(ar.timestamp > now() - INTERVAL 6 MONTH) AS commitsLast6m, countIf( - timestamp <= now() - INTERVAL 6 MONTH AND timestamp > now() - INTERVAL 12 MONTH + ar.timestamp <= now() - INTERVAL 6 MONTH + AND ar.timestamp > now() - INTERVAL 12 MONTH ) AS commitsPrior6m - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE type = 'authored-commit' - GROUP BY channel + FROM activityRelations_deduplicated_cleaned_bucket_union ar + INNER JOIN repos_channels_ds ch ON ar.channel = ch.channel + WHERE ar.type = 'authored-commit' + GROUP BY ch.repoUrl ) AS c ON c.repoUrl = allRepos.repoUrl LEFT JOIN @@ -225,58 +236,62 @@ SQL > LEFT JOIN ( SELECT - channel AS repoUrl, - countIf(closedAt IS NOT NULL) AS closed12m, + ch.repoUrl AS repoUrl, + countIf(ia.closedAt IS NOT NULL) AS closed12m, count() AS opened12m, quantileIf(0.5) - (closedInSeconds, closedAt > now() - INTERVAL 12 MONTH) AS medianCloseS - FROM issues_analyzed - WHERE openedAt > now() - INTERVAL 12 MONTH - GROUP BY channel + (ia.closedInSeconds, ia.closedAt > now() - INTERVAL 12 MONTH) AS medianCloseS + FROM issues_analyzed ia + INNER JOIN repos_channels_ds ch ON ia.channel = ch.channel + WHERE ia.openedAt > now() - INTERVAL 12 MONTH + GROUP BY ch.repoUrl ) AS w ON w.repoUrl = allRepos.repoUrl LEFT JOIN ( SELECT - channel AS repoUrl, - countIf(mergedAt IS NOT NULL) AS merged12m, - countIf(closedAt IS NOT NULL AND mergedAt IS NULL) AS closedUnmerged12m, + ch.repoUrl AS repoUrl, + countIf(pra.mergedAt IS NOT NULL) AS merged12m, + countIf(pra.closedAt IS NOT NULL AND pra.mergedAt IS NULL) AS closedUnmerged12m, quantileIf(0.5) - (mergedInSeconds, mergedAt > now() - INTERVAL 12 MONTH) AS medianMergeS - FROM pull_requests_analyzed - WHERE openedAt > now() - INTERVAL 12 MONTH - GROUP BY channel + (pra.mergedInSeconds, pra.mergedAt > now() - INTERVAL 12 MONTH) AS medianMergeS + FROM pull_requests_analyzed pra + INNER JOIN repos_channels_ds ch ON pra.channel = ch.channel + WHERE pra.openedAt > now() - INTERVAL 12 MONTH + GROUP BY ch.repoUrl ) AS p ON p.repoUrl = allRepos.repoUrl LEFT JOIN ( SELECT - channel AS repoUrl, + ch.repoUrl AS repoUrl, countIf( - respondedInSeconds IS NULL - AND closedAt IS NULL - AND openedAt <= now() - INTERVAL 90 DAY + ia.respondedInSeconds IS NULL + AND ia.closedAt IS NULL + AND ia.openedAt <= now() - INTERVAL 90 DAY ) AS unansweredAged90d, count() AS openedLast18m - FROM issues_analyzed - WHERE openedAt > now() - INTERVAL 18 MONTH - GROUP BY channel + FROM issues_analyzed ia + INNER JOIN repos_channels_ds ch ON ia.channel = ch.channel + WHERE ia.openedAt > now() - INTERVAL 18 MONTH + GROUP BY ch.repoUrl ) AS uw ON uw.repoUrl = allRepos.repoUrl LEFT JOIN ( SELECT - channel AS repoUrl, + ch.repoUrl AS repoUrl, countIf( - reviewedAt IS NULL - AND approvedAt IS NULL - AND closedAt IS NULL - AND openedAt <= now() - INTERVAL 90 DAY + pra.reviewedAt IS NULL + AND pra.approvedAt IS NULL + AND pra.closedAt IS NULL + AND pra.openedAt <= now() - INTERVAL 90 DAY ) AS unansweredAged90d, count() AS openedLast18m - FROM pull_requests_analyzed - WHERE openedAt > now() - INTERVAL 18 MONTH - GROUP BY channel + FROM pull_requests_analyzed pra + INNER JOIN repos_channels_ds ch ON pra.channel = ch.channel + WHERE pra.openedAt > now() - INTERVAL 18 MONTH + GROUP BY ch.repoUrl ) AS upw ON upw.repoUrl = allRepos.repoUrl LEFT JOIN diff --git a/services/libs/tinybird/pipes/health_score_v2_security.pipe b/services/libs/tinybird/pipes/health_score_v2_security.pipe index 1521e560b8..a9e0f80f19 100644 --- a/services/libs/tinybird/pipes/health_score_v2_security.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_security.pipe @@ -117,7 +117,12 @@ SQL > ) AS dependencyHealthScore, (pkgs.repoUrl != '') AS dependencyHealthAvailable, dh.vulnerableDeps AS vulnerableDeps - FROM (SELECT DISTINCT url FROM repositories WHERE deletedAt IS NULL) AS allRepos + FROM + ( + SELECT DISTINCT url + FROM repositories FINAL + WHERE deletedAt IS NULL AND archived = false AND excluded = false + ) AS allRepos LEFT JOIN ( SELECT @@ -145,7 +150,7 @@ SQL > countIf(status = 'OPEN' AND severity = 'CRITICAL') AS openCriticals, countIf(status = 'OPEN' AND severity = 'HIGH') AS openHighs, countIf(status = 'OPEN' AND severity = 'MEDIUM') AS openModerates - FROM vulnerabilities + FROM vulnerabilities FINAL GROUP BY repoUrl ) AS vc ON vc.repoUrl = allRepos.url diff --git a/services/libs/tinybird/pipes/health_score_v2_signal_detail.pipe b/services/libs/tinybird/pipes/health_score_v2_signal_detail.pipe index ff51cf9cfc..9ae0a55759 100644 --- a/services/libs/tinybird/pipes/health_score_v2_signal_detail.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_signal_detail.pipe @@ -69,7 +69,12 @@ SQL > d.closedUnmerged12m AS closedUnmerged12m, d.medianMergeS AS medianMergeS, coalesce(m.methodologyVersion, '2.0.0') AS methodologyVersion - FROM (SELECT DISTINCT url AS repoUrl FROM repositories FINAL WHERE isNull (deletedAt)) AS base + FROM + ( + SELECT DISTINCT url AS repoUrl + FROM repositories FINAL + WHERE deletedAt IS NULL AND excluded = false + ) AS base LEFT JOIN health_score_v2_maintainer_ds AS m ON m.repoUrl = base.repoUrl LEFT JOIN health_score_v2_security_ds AS s ON s.repoUrl = base.repoUrl LEFT JOIN health_score_v2_development_ds AS d ON d.repoUrl = base.repoUrl diff --git a/services/libs/tinybird/pipes/repos_channels_copy.pipe b/services/libs/tinybird/pipes/repos_channels_copy.pipe new file mode 100644 index 0000000000..5e68835f4f --- /dev/null +++ b/services/libs/tinybird/pipes/repos_channels_copy.pipe @@ -0,0 +1,48 @@ +DESCRIPTION > + - Populates `repos_channels_ds` with the expanded channel set for every repository. + - Non-Gerrit repos: one row (channel = url). Gerrit repos: two rows — plain url plus the + /q/project: variant used by Gerrit activity ingestion. + - Mirrors the expansion logic in repos_to_channels.pipe but as a COPY pipe so the result can + be joined by other COPY pipes (which cannot use template-based API pipes). + - Runs daily at 00:00 UTC, before health score pipes (02:00 UTC) and the monthly raw inputs + snapshot (00:30 on the 1st), so channel mappings are always current when scores are computed. + +TAGS "Repository URLs", "Gerrit", "Health Score v2" + +NODE repos_channels_copy_calc +SQL > + SELECT + r.url AS repoUrl, + r.url AS channel, + (i.platform = 'gerrit') AS isGerrit, + r.excluded AS isExcluded + FROM repositories r FINAL + LEFT JOIN integrations i FINAL ON r.sourceIntegrationId = i.id + WHERE r.deletedAt IS NULL + UNION ALL + SELECT + r.url AS repoUrl, + CASE + WHEN position(r.url, '/r/') > 0 + THEN replaceOne(r.url, '/r/', '/r/q/project:') + WHEN position(r.url, '/gerrit/') > 0 + THEN replaceOne(r.url, '/gerrit/', '/gerrit/q/project:') + ELSE + concat( + protocol(r.url), + '://', + domain(r.url), + '/q/project:', + if(path(r.url) = '/', '', substring(path(r.url), 2)) + ) + END AS channel, + 1 AS isGerrit, + r.excluded AS isExcluded + FROM repositories r FINAL + INNER JOIN integrations i FINAL ON r.sourceIntegrationId = i.id + WHERE r.deletedAt IS NULL AND i.platform = 'gerrit' + +TYPE COPY +TARGET_DATASOURCE repos_channels_ds +COPY_MODE replace +COPY_SCHEDULE 0 0 * * *