Run Analytics on native PostgreSQL (#5)

.github/workflows/ci.yml

@@ -176,6 +176,9 @@ jobs:
       - name: Run runnable tests
         run: pnpm test:runnable
 
+      - name: Verify Analytics on PostgreSQL 15
+        run: pnpm --filter @humanly/backend test:analytics-postgres
+
       - name: Audit discovered tests
         run: pnpm test:audit
 

packages/backend/package.json

@@ -11,6 +11,7 @@
     "migrate": "COMPOSE_FILE=../../docker-compose.quickstart.yml POSTGRES_DB=humanly_dev POSTGRES_USER=humanly_user MIGRATIONS_DIR=src/db/migrations bash ../../scripts/run-migrations.sh",
     "cleanup:orphan-storage": "tsx src/jobs/cleanup-orphan-storage.ts",
     "test:edition": "RATE_LIMIT_ENABLED=false DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/app.edition.test.ts",
+    "test:analytics-postgres": "bash ../../scripts/test-analytics-postgres.sh",
     "test:runnable": "pnpm test:edition && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/config/cors.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/config/env.ai-encryption.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/controllers/ai-settings.controller.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/controllers/ai.controller.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/controllers/file.controller.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/services/task-public-share-link.service.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/services/task-dashboard-list.service.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/services/writing-detector-config.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/services/certificate-detector-seal.service.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/services/file-text-index.service.test.ts && DATABASE_URL=postgres://humanly:humanly@localhost:5432/humanly_test REDIS_URL=redis://localhost:6379 JWT_SECRET=test-jwt-secret EMAIL_FROM=test@example.com tsx src/utils/http-range.test.ts",
     "lint": "tsc --noEmit",
     "clean": "rm -rf dist"

packages/backend/src/services/ANALYTICS_README.md

@@ -35,8 +35,7 @@ Returns time-series event data with configurable grouping:
 - Returns array of {date, eventCount}
 
 **Optimizations:**
-- Uses `events_hourly` continuous aggregate when possible
-- Falls back to `time_bucket()` for custom queries
+- Uses PostgreSQL `date_bin()` with a fixed UTC origin
 - Indexed timestamp queries
 
 #### `getEventTypeDistribution(taskId, userId, filters)`

packages/backend/src/services/ANALYTICS_v2_README.md

@@ -457,9 +457,9 @@ Source timestamps:
 Grouping:
 
 ```text
-groupBy=hour -> time_bucket('1 hour', timestamp)
-groupBy=day  -> time_bucket('1 day', timestamp)
-groupBy=week -> time_bucket('1 week', timestamp)
+groupBy=hour -> date_bin('1 hour', timestamp, UTC origin)
+groupBy=day  -> date_bin('1 day', timestamp, UTC origin)
+groupBy=week -> date_bin('1 week', timestamp, UTC Monday origin)
 ```
 
 Date formatting:

packages/backend/src/services/analytics-postgres-check.ts

@@ -0,0 +1,106 @@
+import assert from 'node:assert/strict';
+import { pool } from '../config/database';
+import { TaskModel } from '../models/task.model';
+import { AnalyticsService } from './analytics.service';
+
+const taskId = '00000000-0000-4000-8000-000000000101';
+const userId = '00000000-0000-4000-8000-000000000102';
+const documentId = '00000000-0000-4000-8000-000000000103';
+const sessionId = '00000000-0000-4000-8000-000000000104';
+
+async function main(): Promise<void> {
+  await pool.query(`
+    CREATE TABLE sessions (
+      id uuid PRIMARY KEY,
+      task_id uuid NOT NULL,
+      external_user_id text NOT NULL
+    );
+    CREATE TABLE events (
+      session_id uuid NOT NULL,
+      task_id uuid NOT NULL,
+      event_type text NOT NULL,
+      "timestamp" timestamp without time zone NOT NULL
+    );
+    CREATE TABLE users (
+      id uuid PRIMARY KEY,
+      email text NOT NULL
+    );
+    CREATE TABLE task_enrollments (
+      task_id uuid NOT NULL,
+      submission_document_id uuid
+    );
+    CREATE TABLE document_events (
+      document_id uuid NOT NULL,
+      user_id uuid NOT NULL,
+      event_type text NOT NULL,
+      "timestamp" timestamp without time zone NOT NULL
+    );
+
+    INSERT INTO users (id, email)
+    VALUES ('${userId}', 'analytics-postgres@writehumanly.test');
+    INSERT INTO sessions (id, task_id, external_user_id)
+    VALUES ('${sessionId}', '${taskId}', 'analytics-postgres@writehumanly.test');
+    INSERT INTO task_enrollments (task_id, submission_document_id)
+    VALUES ('${taskId}', '${documentId}');
+
+    INSERT INTO events (session_id, task_id, event_type, "timestamp") VALUES
+      ('${sessionId}', '${taskId}', 'input', TIMESTAMP '2020-12-27 23:30:00'),
+      ('${sessionId}', '${taskId}', 'keydown', TIMESTAMP '2020-12-28 00:15:00'),
+      ('${sessionId}', '${taskId}', 'input', TIMESTAMP '2020-12-31 23:55:00'),
+      ('${sessionId}', '${taskId}', 'keydown', TIMESTAMP '2021-01-01 00:05:00');
+    INSERT INTO document_events (document_id, user_id, event_type, "timestamp") VALUES
+      ('${documentId}', '${userId}', 'input', TIMESTAMP '2021-01-04 00:01:00'),
+      ('${documentId}', '${userId}', 'keydown', TIMESTAMP '2021-01-04 00:15:00'),
+      ('${documentId}', '${userId}', 'input', TIMESTAMP '2021-01-04 00:45:00');
+  `);
+
+  const originalVerifyOwnership = TaskModel.verifyOwnership;
+  TaskModel.verifyOwnership = async () => true;
+  try {
+    assert.deepEqual(
+      await AnalyticsService.getEventsTimeline(taskId, userId, 'hour'),
+      [
+        { date: '2020-12-27 23:00:00', eventCount: 1 },
+        { date: '2020-12-28 00:00:00', eventCount: 1 },
+        { date: '2020-12-31 23:00:00', eventCount: 1 },
+        { date: '2021-01-01 00:00:00', eventCount: 1 },
+        { date: '2021-01-04 00:00:00', eventCount: 3 },
+      ],
+    );
+    assert.deepEqual(
+      await AnalyticsService.getEventsTimeline(taskId, userId, 'day'),
+      [
+        { date: '2020-12-27', eventCount: 1 },
+        { date: '2020-12-28', eventCount: 1 },
+        { date: '2020-12-31', eventCount: 1 },
+        { date: '2021-01-01', eventCount: 1 },
+        { date: '2021-01-04', eventCount: 3 },
+      ],
+    );
+    assert.deepEqual(
+      await AnalyticsService.getEventsTimeline(taskId, userId, 'week'),
+      [
+        { date: '2020-52', eventCount: 1 },
+        { date: '2020-53', eventCount: 3 },
+        { date: '2021-01', eventCount: 3 },
+      ],
+    );
+    assert.deepEqual(
+      await AnalyticsService.getEventsTimeline(
+        '00000000-0000-4000-8000-000000000999',
+        userId,
+        'week',
+      ),
+      [],
+    );
+  } finally {
+    TaskModel.verifyOwnership = originalVerifyOwnership;
+  }
+}
+
+main()
+  .finally(() => pool.end())
+  .catch((error) => {
+    console.error(error);
+    process.exitCode = 1;
+  });

packages/backend/src/services/analytics.service.ts

@@ -311,13 +311,21 @@ export class AnalyticsService {
             AND ($3::timestamptz IS NULL OR de.timestamp <= $3::timestamptz)
             AND ($4::text IS NULL OR u.email = $4::text)
             AND ($5::text IS NULL OR de.event_type = $5::text)
+        ),
+        bucketed_events AS (
+          SELECT date_bin(
+            '${bucketInterval}',
+            timestamp AT TIME ZONE 'UTC',
+            TIMESTAMPTZ '2000-01-03 00:00:00+00'
+          ) AS bucket
+          FROM all_events
         )
         SELECT
-          TO_CHAR(time_bucket('${bucketInterval}', timestamp), '${dateFormat}') as date,
+          TO_CHAR(bucket AT TIME ZONE 'UTC', '${dateFormat}') as date,
           COUNT(*)::integer as "eventCount"
-        FROM all_events
-        GROUP BY time_bucket('${bucketInterval}', timestamp)
-        ORDER BY time_bucket('${bucketInterval}', timestamp) ASC
+        FROM bucketed_events
+        GROUP BY bucket
+        ORDER BY bucket ASC
       `;
 
       const params = [

scripts/test-analytics-postgres.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+container_name="humanly-analytics-postgres-$$-$RANDOM"
+
+cleanup() {
+  docker rm -f "$container_name" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+docker run --detach --rm \
+  --name "$container_name" \
+  --publish 127.0.0.1::5432 \
+  --env POSTGRES_DB=humanly_analytics_test \
+  --env POSTGRES_USER=humanly_analytics_test \
+  --env POSTGRES_PASSWORD=humanly_analytics_test \
+  postgres:15.13-alpine3.20 >/dev/null
+
+for _ in $(seq 1 60); do
+  if docker exec "$container_name" sh -c \
+    'test "$(cat /proc/1/comm)" = postgres && pg_isready -U humanly_analytics_test -d humanly_analytics_test' \
+    >/dev/null 2>&1; then
+    break
+  fi
+  sleep 1
+done
+
+if ! docker exec "$container_name" sh -c \
+  'test "$(cat /proc/1/comm)" = postgres && pg_isready -U humanly_analytics_test -d humanly_analytics_test' \
+  >/dev/null 2>&1; then
+  echo "Analytics PostgreSQL fixture did not become ready." >&2
+  docker logs "$container_name" >&2 || true
+  exit 1
+fi
+
+host_port="$(docker port "$container_name" 5432/tcp | sed 's/.*://')"
+cd "$repo_root"
+DATABASE_URL="postgres://humanly_analytics_test:humanly_analytics_test@127.0.0.1:${host_port}/humanly_analytics_test?options=-c%20timezone%3DAmerica%2FToronto" \
+REDIS_URL=redis://127.0.0.1:1 \
+JWT_SECRET=test-jwt-secret \
+EMAIL_FROM=test@example.com \
+  pnpm --filter @humanly/backend exec tsx \
+    src/services/analytics-postgres-check.ts