From 5b085d3bf8e09a59f5aaf83eeda1136eb41cca27 Mon Sep 17 00:00:00 2001 From: Haoqian Zhang Date: Wed, 26 Aug 2026 16:56:46 -0400 Subject: [PATCH 1/2] fix: run Analytics on native PostgreSQL --- .github/workflows/ci.yml | 3 + packages/backend/package.json | 1 + .../backend/src/services/ANALYTICS_README.md | 3 +- .../src/services/ANALYTICS_v2_README.md | 6 +- .../src/services/analytics-postgres-check.ts | 106 ++++++++++++++++++ .../backend/src/services/analytics.service.ts | 16 ++- scripts/test-analytics-postgres.sh | 42 +++++++ 7 files changed, 168 insertions(+), 9 deletions(-) create mode 100644 packages/backend/src/services/analytics-postgres-check.ts create mode 100755 scripts/test-analytics-postgres.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3851552e..c7f6dac3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,6 +176,9 @@ jobs: - name: Run runnable tests run: pnpm test:runnable + - name: Verify Analytics on PostgreSQL 15 + run: pnpm --filter @humanly/backend test:analytics-postgres + - name: Audit discovered tests run: pnpm test:audit diff --git a/packages/backend/package.json b/packages/backend/package.json index 672846ca..d8ad37a7 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -11,6 +11,7 @@ "migrate": "COMPOSE_FILE=../../docker-compose.quickstart.yml POSTGRES_DB=humanly_dev POSTGRES_USER=humanly_user MIGRATIONS_DIR=src/db/migrations bash ../../scripts/run-migrations.sh", "cleanup:orphan-storage": "tsx src/jobs/cleanup-orphan-storage.ts", "test:edition": "RATE_LIMIT_ENABLED=false DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/app.edition.test.ts", + "test:analytics-postgres": "bash ../../scripts/test-analytics-postgres.sh", "test:runnable": "pnpm test:edition && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/config/cors.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/config/env.ai-encryption.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/controllers/ai-settings.controller.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/controllers/ai.controller.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/controllers/file.controller.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/services/task-public-share-link.service.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/services/task-dashboard-list.service.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/services/writing-detector-config.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/services/certificate-detector-seal.service.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/services/file-text-index.service.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/utils/http-range.test.ts", "lint": "tsc --noEmit", "clean": "rm -rf dist" diff --git a/packages/backend/src/services/ANALYTICS_README.md b/packages/backend/src/services/ANALYTICS_README.md index fcc5a058..82ec0c82 100644 --- a/packages/backend/src/services/ANALYTICS_README.md +++ b/packages/backend/src/services/ANALYTICS_README.md @@ -35,8 +35,7 @@ Returns time-series event data with configurable grouping: - Returns array of {date, eventCount} **Optimizations:** -- Uses `events_hourly` continuous aggregate when possible -- Falls back to `time_bucket()` for custom queries +- Uses PostgreSQL `date_bin()` with a fixed UTC origin - Indexed timestamp queries #### `getEventTypeDistribution(taskId, userId, filters)` diff --git a/packages/backend/src/services/ANALYTICS_v2_README.md b/packages/backend/src/services/ANALYTICS_v2_README.md index 68f95971..b1475a80 100644 --- a/packages/backend/src/services/ANALYTICS_v2_README.md +++ b/packages/backend/src/services/ANALYTICS_v2_README.md @@ -457,9 +457,9 @@ Source timestamps: Grouping: ```text -groupBy=hour -> time_bucket('1 hour', timestamp) -groupBy=day -> time_bucket('1 day', timestamp) -groupBy=week -> time_bucket('1 week', timestamp) +groupBy=hour -> date_bin('1 hour', timestamp, UTC origin) +groupBy=day -> date_bin('1 day', timestamp, UTC origin) +groupBy=week -> date_bin('1 week', timestamp, UTC Monday origin) ``` Date formatting: diff --git a/packages/backend/src/services/analytics-postgres-check.ts b/packages/backend/src/services/analytics-postgres-check.ts new file mode 100644 index 00000000..618c8421 --- /dev/null +++ b/packages/backend/src/services/analytics-postgres-check.ts @@ -0,0 +1,106 @@ +import assert from 'node:assert/strict'; +import { pool } from '../config/database'; +import { TaskModel } from '../models/task.model'; +import { AnalyticsService } from './analytics.service'; + +const taskId = '00000000-0000-4000-8000-000000000101'; +const userId = '00000000-0000-4000-8000-000000000102'; +const documentId = '00000000-0000-4000-8000-000000000103'; +const sessionId = '00000000-0000-4000-8000-000000000104'; + +async function main(): Promise { + await pool.query(` + CREATE TABLE sessions ( + id uuid PRIMARY KEY, + task_id uuid NOT NULL, + external_user_id text NOT NULL + ); + CREATE TABLE events ( + session_id uuid NOT NULL, + task_id uuid NOT NULL, + event_type text NOT NULL, + "timestamp" timestamp without time zone NOT NULL + ); + CREATE TABLE users ( + id uuid PRIMARY KEY, + email text NOT NULL + ); + CREATE TABLE task_enrollments ( + task_id uuid NOT NULL, + submission_document_id uuid + ); + CREATE TABLE document_events ( + document_id uuid NOT NULL, + user_id uuid NOT NULL, + event_type text NOT NULL, + "timestamp" timestamp without time zone NOT NULL + ); + + INSERT INTO users (id, email) + VALUES ('${userId}', 'analytics-postgres@writehumanly.test'); + INSERT INTO sessions (id, task_id, external_user_id) + VALUES ('${sessionId}', '${taskId}', 'analytics-postgres@writehumanly.test'); + INSERT INTO task_enrollments (task_id, submission_document_id) + VALUES ('${taskId}', '${documentId}'); + + INSERT INTO events (session_id, task_id, event_type, "timestamp") VALUES + ('${sessionId}', '${taskId}', 'input', TIMESTAMP '2020-12-27 23:30:00'), + ('${sessionId}', '${taskId}', 'keydown', TIMESTAMP '2020-12-28 00:15:00'), + ('${sessionId}', '${taskId}', 'input', TIMESTAMP '2020-12-31 23:55:00'), + ('${sessionId}', '${taskId}', 'keydown', TIMESTAMP '2021-01-01 00:05:00'); + INSERT INTO document_events (document_id, user_id, event_type, "timestamp") VALUES + ('${documentId}', '${userId}', 'input', TIMESTAMP '2021-01-04 00:01:00'), + ('${documentId}', '${userId}', 'keydown', TIMESTAMP '2021-01-04 00:15:00'), + ('${documentId}', '${userId}', 'input', TIMESTAMP '2021-01-04 00:45:00'); + `); + + const originalVerifyOwnership = TaskModel.verifyOwnership; + TaskModel.verifyOwnership = async () => true; + try { + assert.deepEqual( + await AnalyticsService.getEventsTimeline(taskId, userId, 'hour'), + [ + { date: '2020-12-27 23:00:00', eventCount: 1 }, + { date: '2020-12-28 00:00:00', eventCount: 1 }, + { date: '2020-12-31 23:00:00', eventCount: 1 }, + { date: '2021-01-01 00:00:00', eventCount: 1 }, + { date: '2021-01-04 00:00:00', eventCount: 3 }, + ], + ); + assert.deepEqual( + await AnalyticsService.getEventsTimeline(taskId, userId, 'day'), + [ + { date: '2020-12-27', eventCount: 1 }, + { date: '2020-12-28', eventCount: 1 }, + { date: '2020-12-31', eventCount: 1 }, + { date: '2021-01-01', eventCount: 1 }, + { date: '2021-01-04', eventCount: 3 }, + ], + ); + assert.deepEqual( + await AnalyticsService.getEventsTimeline(taskId, userId, 'week'), + [ + { date: '2020-52', eventCount: 1 }, + { date: '2020-53', eventCount: 3 }, + { date: '2021-01', eventCount: 3 }, + ], + ); + assert.deepEqual( + await AnalyticsService.getEventsTimeline( + '00000000-0000-4000-8000-000000000999', + userId, + 'week', + ), + [], + ); + } finally { + TaskModel.verifyOwnership = originalVerifyOwnership; + } +} + +main() + .finally(() => pool.end()) + .catch((error) => { + console.error(error); + process.exitCode = 1; + }); diff --git a/packages/backend/src/services/analytics.service.ts b/packages/backend/src/services/analytics.service.ts index ef132f6b..c2a7abac 100644 --- a/packages/backend/src/services/analytics.service.ts +++ b/packages/backend/src/services/analytics.service.ts @@ -311,13 +311,21 @@ export class AnalyticsService { AND ($3::timestamptz IS NULL OR de.timestamp <= $3::timestamptz) AND ($4::text IS NULL OR u.email = $4::text) AND ($5::text IS NULL OR de.event_type = $5::text) + ), + bucketed_events AS ( + SELECT date_bin( + '${bucketInterval}', + timestamp AT TIME ZONE 'UTC', + TIMESTAMPTZ '2000-01-03 00:00:00+00' + ) AS bucket + FROM all_events ) SELECT - TO_CHAR(time_bucket('${bucketInterval}', timestamp), '${dateFormat}') as date, + TO_CHAR(bucket AT TIME ZONE 'UTC', '${dateFormat}') as date, COUNT(*)::integer as "eventCount" - FROM all_events - GROUP BY time_bucket('${bucketInterval}', timestamp) - ORDER BY time_bucket('${bucketInterval}', timestamp) ASC + FROM bucketed_events + GROUP BY bucket + ORDER BY bucket ASC `; const params = [ diff --git a/scripts/test-analytics-postgres.sh b/scripts/test-analytics-postgres.sh new file mode 100755 index 00000000..ec60f0b4 --- /dev/null +++ b/scripts/test-analytics-postgres.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +container_name="humanly-analytics-postgres-$$-$RANDOM" + +cleanup() { + docker rm -f "$container_name" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +docker run --detach --rm \ + --name "$container_name" \ + --publish 127.0.0.1::5432 \ + --env POSTGRES_DB=humanly_analytics_test \ + --env POSTGRES_USER=humanly_analytics_test \ + --env POSTGRES_PASSWORD=humanly_analytics_test \ + postgres:15.13-alpine3.20 >/dev/null + +for _ in $(seq 1 60); do + if docker exec "$container_name" pg_isready \ + -U humanly_analytics_test -d humanly_analytics_test >/dev/null 2>&1; then + break + fi + sleep 1 +done + +if ! docker exec "$container_name" pg_isready \ + -U humanly_analytics_test -d humanly_analytics_test >/dev/null 2>&1; then + echo "Analytics PostgreSQL fixture did not become ready." >&2 + docker logs "$container_name" >&2 || true + exit 1 +fi + +host_port="$(docker port "$container_name" 5432/tcp | sed 's/.*://')" +cd "$repo_root" +DATABASE_URL="postgres://humanly_analytics_test:humanly_analytics_test@127.0.0.1:${host_port}/humanly_analytics_test?options=-c%20timezone%3DAmerica%2FToronto" \ +REDIS_URL=redis://127.0.0.1:1 \ +JWT_SECRET=test-jwt-secret \ +EMAIL_FROM=test@example.com \ + pnpm --filter @humanly/backend exec tsx \ + src/services/analytics-postgres-check.ts From 5c0b422f4dae4e7abd99cb586ba7953b42e130b2 Mon Sep 17 00:00:00 2001 From: Haoqian Zhang Date: Wed, 26 Aug 2026 17:17:09 -0400 Subject: [PATCH 2/2] test: wait for final PostgreSQL startup --- scripts/test-analytics-postgres.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/test-analytics-postgres.sh b/scripts/test-analytics-postgres.sh index ec60f0b4..d0a225b5 100755 --- a/scripts/test-analytics-postgres.sh +++ b/scripts/test-analytics-postgres.sh @@ -18,15 +18,17 @@ docker run --detach --rm \ postgres:15.13-alpine3.20 >/dev/null for _ in $(seq 1 60); do - if docker exec "$container_name" pg_isready \ - -U humanly_analytics_test -d humanly_analytics_test >/dev/null 2>&1; then + if docker exec "$container_name" sh -c \ + 'test "$(cat /proc/1/comm)" = postgres && pg_isready -U humanly_analytics_test -d humanly_analytics_test' \ + >/dev/null 2>&1; then break fi sleep 1 done -if ! docker exec "$container_name" pg_isready \ - -U humanly_analytics_test -d humanly_analytics_test >/dev/null 2>&1; then +if ! docker exec "$container_name" sh -c \ + 'test "$(cat /proc/1/comm)" = postgres && pg_isready -U humanly_analytics_test -d humanly_analytics_test' \ + >/dev/null 2>&1; then echo "Analytics PostgreSQL fixture did not become ready." >&2 docker logs "$container_name" >&2 || true exit 1