feat(decabill): file manager

apps/decabill/backend-billing-manager/.start-containers.env.example

@@ -57,6 +57,14 @@ BILLING_TAX_RATE_REDUCED=7
 BILLING_OSS_REGISTERED=false
 BILLING_OSS_THRESHOLD_EUR=10000
 BILLING_NON_EU_ISSUER_EU_B2B_CHARGE_VAT=false
+# Canonical file root (scopes: customer/*, supplier/*, export/datev)
+FILE_STORAGE_ROOT=/data
+FILE_STORAGE_PROVIDER=local
+FILE_STORAGE_LAYOUT_MIGRATION_ENABLED=true
+FILE_STORAGE_LAYOUT_DUAL_READ_ENABLED=true
+# Global HMAC secret for billing_stored_files signatures (required in production)
+BILLING_FILE_SIGNING_SECRET=
+# Deprecated legacy roots (migration sources only)
 BILLING_INVOICE_PDF_STORAGE_PATH=/data/invoices
 
 # SMTP / transactional email (MailHog when using docker-compose)
@@ -77,13 +85,14 @@ EMAIL_COMPANY_EMAIL=
 
 # DATEV EXTF export (monthly Buchungsstapel + optional document bundle)
 BILLING_DATEV_EXPORT_ENABLED=true
+# Deprecated: layout migrator source for previous datev-exports segment
 BILLING_DATEV_EXPORT_STORAGE_PATH=/data/datev-exports
 BILLING_DATEV_EXPORT_CRON=0 0 1 * *
 BILLING_DATEV_EXPORT_TIMEZONE=Europe/Berlin
 BILLING_DATEV_CONSULTANT_NUMBER=
 BILLING_DATEV_CLIENT_NUMBER=
 BILLING_DATEV_UNIFIED_EXPORT_ENABLED=false
-BILLING_DATEV_UNIFIED_EXPORT_ALLOWED_TENANTS=default
+TENANTS_ALLOW_GLOBAL_VIEWS=default
 
 BILLING_SKIP_FILE_CACHE=false
 BILLING_DEFAULT_PAYMENT_PROCESSOR=stripe

apps/decabill/backend-billing-manager/Dockerfile.api

@@ -69,16 +69,29 @@ RUN . "${NVM_DIR}/nvm.sh" && \
 
 COPY --chown=${APP_UID}:${APP_GID} . /app
 
-RUN mkdir -p /data/invoices /var/lib/forepath/provider-plugins && \
+RUN mkdir -p \
+      /data/customer/invoices \
+      /data/customer/offers \
+      /data/customer/timesheets \
+      /data/supplier/invoices \
+      /data/export/datev \
+      /data/invoices \
+      /var/lib/forepath/provider-plugins && \
     chown -R "${APP_UID}:${APP_GID}" /data /var/lib/forepath/provider-plugins
 
 # Expand APP_UID/APP_GID into the entrypoint at build time (runtime has no ARG).
 RUN printf '%s\n' \
     '#!/bin/bash' \
     'set -euo pipefail' \
-    'storage_path="${BILLING_INVOICE_PDF_STORAGE_PATH:-/data/invoices}"' \
-    'mkdir -p "${storage_path}"' \
-    "chown -R ${APP_UID}:${APP_GID} /data \"\${storage_path}\"" \
+    'storage_root="${FILE_STORAGE_ROOT:-/data}"' \
+    'mkdir -p \' \
+    '  "${storage_root}/customer/invoices" \' \
+    '  "${storage_root}/customer/offers" \' \
+    '  "${storage_root}/customer/timesheets" \' \
+    '  "${storage_root}/supplier/invoices" \' \
+    '  "${storage_root}/export/datev" \' \
+    '  "${storage_root}/invoices"' \
+    "chown -R ${APP_UID}:${APP_GID} \"\${storage_root}\" /var/lib/forepath/provider-plugins" \
     '. "${NVM_DIR}/nvm.sh"' \
     'if [ -n "${DYNAMIC_PROVIDER_PLUGIN_PATH:-}" ]; then' \
     '  runuser -u decabill -g decabill -- node install-provider-plugins.js' \

apps/decabill/backend-billing-manager/docker-compose.yaml

@@ -72,6 +72,9 @@ x-backend-billing-manager-environment: &backend-billing-manager-environment
   FILE_STORAGE_PROVIDER: ${FILE_STORAGE_PROVIDER:-local}
   FILE_STORAGE_ROOT: ${FILE_STORAGE_ROOT:-/data}
   FILE_STORAGE_LEGACY_MIGRATION_ENABLED: ${FILE_STORAGE_LEGACY_MIGRATION_ENABLED:-true}
+  FILE_STORAGE_LAYOUT_MIGRATION_ENABLED: ${FILE_STORAGE_LAYOUT_MIGRATION_ENABLED:-true}
+  FILE_STORAGE_LAYOUT_DUAL_READ_ENABLED: ${FILE_STORAGE_LAYOUT_DUAL_READ_ENABLED:-true}
+  BILLING_FILE_SIGNING_SECRET: ${BILLING_FILE_SIGNING_SECRET:-}
   FILE_STORAGE_S3_BUCKET: ${FILE_STORAGE_S3_BUCKET:-}
   FILE_STORAGE_S3_ACCESS_KEY_ID: ${FILE_STORAGE_S3_ACCESS_KEY_ID:-}
   FILE_STORAGE_S3_SECRET_ACCESS_KEY: ${FILE_STORAGE_S3_SECRET_ACCESS_KEY:-}
@@ -87,7 +90,7 @@ x-backend-billing-manager-environment: &backend-billing-manager-environment
   BILLING_DATEV_CONSULTANT_NUMBER: ${BILLING_DATEV_CONSULTANT_NUMBER:-}
   BILLING_DATEV_CLIENT_NUMBER: ${BILLING_DATEV_CLIENT_NUMBER:-}
   BILLING_DATEV_UNIFIED_EXPORT_ENABLED: ${BILLING_DATEV_UNIFIED_EXPORT_ENABLED:-false}
-  BILLING_DATEV_UNIFIED_EXPORT_ALLOWED_TENANTS: ${BILLING_DATEV_UNIFIED_EXPORT_ALLOWED_TENANTS:-default}
+  TENANTS_ALLOW_GLOBAL_VIEWS: ${TENANTS_ALLOW_GLOBAL_VIEWS:-default}
   BILLING_SKIP_FILE_CACHE: ${BILLING_SKIP_FILE_CACHE:-false}
   BILLING_DEFAULT_PAYMENT_PROCESSOR: ${BILLING_DEFAULT_PAYMENT_PROCESSOR:-stripe}
   STRIPE_SECRET_KEY: ${STRIPE_SECRET_KEY:-}

apps/decabill/backend-billing-manager/src/migrations/1778000000000_CreateBillingStoredFilesAndStripOfferKeys.ts

@@ -0,0 +1,130 @@
+import type { MigrationInterface, QueryRunner } from 'typeorm';
+
+/**
+ * Creates billing_stored_files registry and backfills rows from domain storage keys
+ * using the customer/supplier/export scope layout. Strips offers/ prefix from offer keys.
+ */
+export class CreateBillingStoredFilesAndStripOfferKeys1778000000000 implements MigrationInterface {
+  name = 'CreateBillingStoredFilesAndStripOfferKeys1778000000000';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS pgcrypto`);
+
+    await queryRunner.query(`
+      CREATE TABLE IF NOT EXISTS "billing_stored_files" (
+        "id" uuid NOT NULL DEFAULT gen_random_uuid(),
+        "long_sha" character varying(40) NOT NULL,
+        "tenant_id" character varying(64) NOT NULL,
+        "scope" character varying(64) NOT NULL,
+        "storage_key" character varying(512) NOT NULL,
+        "content_md5" character varying(32),
+        "content_sha1" character varying(40),
+        "content_sha256" character varying(64),
+        "content_sha512" character varying(128),
+        "byte_size" bigint,
+        "signature" character varying(64),
+        "signature_alg" character varying(32),
+        "signature_version" character varying(8),
+        "signed_at" TIMESTAMPTZ,
+        "created_at" TIMESTAMPTZ NOT NULL DEFAULT now(),
+        "updated_at" TIMESTAMPTZ NOT NULL DEFAULT now(),
+        CONSTRAINT "PK_billing_stored_files" PRIMARY KEY ("id")
+      )
+    `);
+
+    await queryRunner.query(`
+      CREATE UNIQUE INDEX IF NOT EXISTS "UQ_billing_stored_files_scope_key"
+      ON "billing_stored_files" ("scope", "storage_key")
+    `);
+
+    await queryRunner.query(`
+      CREATE INDEX IF NOT EXISTS "IDX_billing_stored_files_long_sha"
+      ON "billing_stored_files" ("long_sha")
+    `);
+
+    await queryRunner.query(`
+      UPDATE "billing_offers"
+      SET "pdf_storage_key" = substring("pdf_storage_key" from 8)
+      WHERE "pdf_storage_key" LIKE 'offers/%'
+    `);
+
+    await queryRunner.query(`
+      INSERT INTO "billing_stored_files" ("id", "long_sha", "tenant_id", "scope", "storage_key")
+      SELECT gen_random_uuid(), '', u.tenant_id, 'customerInvoices', i.pdf_storage_key
+      FROM "billing_invoices" i
+      INNER JOIN "users" u ON u.id = i.user_id
+      WHERE i.pdf_storage_key IS NOT NULL
+      ON CONFLICT ("scope", "storage_key") DO NOTHING
+    `);
+
+    await queryRunner.query(`
+      INSERT INTO "billing_stored_files" ("id", "long_sha", "tenant_id", "scope", "storage_key")
+      SELECT gen_random_uuid(), '', u.tenant_id, 'customerTimesheets', i.time_report_storage_key
+      FROM "billing_invoices" i
+      INNER JOIN "users" u ON u.id = i.user_id
+      WHERE i.time_report_storage_key IS NOT NULL
+      ON CONFLICT ("scope", "storage_key") DO NOTHING
+    `);
+
+    await queryRunner.query(`
+      INSERT INTO "billing_stored_files" ("id", "long_sha", "tenant_id", "scope", "storage_key")
+      SELECT gen_random_uuid(), '', u.tenant_id, 'customerInvoices', d.pdf_storage_key
+      FROM "billing_invoice_void_documents" d
+      INNER JOIN "billing_invoices" i ON i.id = d.invoice_id
+      INNER JOIN "users" u ON u.id = i.user_id
+      ON CONFLICT ("scope", "storage_key") DO NOTHING
+    `);
+
+    await queryRunner.query(`
+      INSERT INTO "billing_stored_files" ("id", "long_sha", "tenant_id", "scope", "storage_key")
+      SELECT gen_random_uuid(), '', u.tenant_id, 'customerInvoices', d.pdf_storage_key
+      FROM "billing_invoice_credit_documents" d
+      INNER JOIN "billing_invoices" i ON i.id = d.invoice_id
+      INNER JOIN "users" u ON u.id = i.user_id
+      ON CONFLICT ("scope", "storage_key") DO NOTHING
+    `);
+
+    await queryRunner.query(`
+      INSERT INTO "billing_stored_files" ("id", "long_sha", "tenant_id", "scope", "storage_key")
+      SELECT gen_random_uuid(), '', u.tenant_id, 'customerOffers', o.pdf_storage_key
+      FROM "billing_offers" o
+      INNER JOIN "users" u ON u.id = o.user_id
+      WHERE o.pdf_storage_key IS NOT NULL
+      ON CONFLICT ("scope", "storage_key") DO NOTHING
+    `);
+
+    await queryRunner.query(`
+      INSERT INTO "billing_stored_files" ("id", "long_sha", "tenant_id", "scope", "storage_key")
+      SELECT gen_random_uuid(), '', sp.tenant_id, 'supplierInvoices', si.document_storage_key
+      FROM "billing_supplier_invoices" si
+      INNER JOIN "billing_supplier_profiles" sp ON sp.id = si.supplier_id
+      WHERE si.document_storage_key IS NOT NULL
+      ON CONFLICT ("scope", "storage_key") DO NOTHING
+    `);
+
+    await queryRunner.query(`
+      INSERT INTO "billing_stored_files" ("id", "long_sha", "tenant_id", "scope", "storage_key")
+      SELECT gen_random_uuid(), '', de.tenant_id, 'datevExports', de.storage_key
+      FROM "billing_datev_exports" de
+      WHERE de.storage_key IS NOT NULL
+      ON CONFLICT ("scope", "storage_key") DO NOTHING
+    `);
+
+    await queryRunner.query(`
+      UPDATE "billing_stored_files"
+      SET "long_sha" = encode(digest("id"::text, 'sha1'), 'hex')
+      WHERE "long_sha" = '' OR "long_sha" IS NULL
+    `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      UPDATE "billing_offers"
+      SET "pdf_storage_key" = 'offers/' || "pdf_storage_key"
+      WHERE "pdf_storage_key" IS NOT NULL
+        AND "pdf_storage_key" NOT LIKE 'offers/%'
+    `);
+
+    await queryRunner.query(`DROP TABLE IF EXISTS "billing_stored_files"`);
+  }
+}

apps/decabill/backend-billing-manager/src/migrations/1778100000000_AddBillingStoredFilesContentSha256Index.ts

@@ -0,0 +1,16 @@
+import type { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddBillingStoredFilesContentSha256Index1778100000000 implements MigrationInterface {
+  name = 'AddBillingStoredFilesContentSha256Index1778100000000';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      CREATE INDEX IF NOT EXISTS "IDX_billing_stored_files_content_sha256"
+      ON "billing_stored_files" ("content_sha256")
+    `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP INDEX IF EXISTS "IDX_billing_stored_files_content_sha256"`);
+  }
+}

apps/decabill/backend-billing-manager/src/queue/job-registry.ts

@@ -53,6 +53,8 @@ export const BillingJobName = {
   SEARCH_REINDEX_COORDINATOR: 'search-reindex.coordinator',
   SEARCH_REINDEX_UNIT: 'search-reindex.unit',
   SEARCH_INDEX_SYNC_UNIT: 'search-index-sync.unit',
+  STORED_FILES_BACKFILL_COORDINATOR: 'stored-files.backfill.coordinator',
+  STORED_FILES_BACKFILL_UNIT: 'stored-files.backfill.unit',
   OFFER_EXPIRATION_COORDINATOR: 'offer-expiration.coordinator',
   OFFER_EXPIRATION_UNIT: 'offer-expiration.unit',
   OFFER_FULFILLMENT_COORDINATOR: 'offer-fulfillment.coordinator',
@@ -245,6 +247,12 @@ export function getBillingRepeatableJobs(): BillingRepeatableJobDefinition[] {
     everyMs: parseIntervalMs('SEARCH_REINDEX_INTERVAL', 900_000),
   });
 
+  jobs.push({
+    name: BillingJobName.STORED_FILES_BACKFILL_COORDINATOR,
+    coordinatorJobId: buildCoordinatorJobId('stored-files-backfill'),
+    everyMs: parseIntervalMs('STORED_FILES_BACKFILL_INTERVAL', 300_000),
+  });
+
   jobs.push({
     name: BillingJobName.OFFER_EXPIRATION_COORDINATOR,
     coordinatorJobId: buildCoordinatorJobId('offer-expiration'),

apps/decabill/backend-billing-manager/src/queue/processors/billing-jobs.processor.ts

@@ -16,6 +16,7 @@ import {
   SearchReindexJobHandler,
   type SearchIndexSyncUnitPayload,
   type SearchReindexUnitPayload,
+  StoredFilesBackfillJobHandler,
   OfferExpirationJobHandler,
   OfferFulfillmentJobHandler,
   SubscriptionBillingJobHandler,
@@ -89,6 +90,7 @@ export class BillingJobsProcessor extends WorkerHost {
     private readonly emailDeliveryService: EmailDeliveryService,
     private readonly updateCheckService: UpdateCheckService,
     private readonly searchReindexJobHandler: SearchReindexJobHandler,
+    private readonly storedFilesBackfillJobHandler: StoredFilesBackfillJobHandler,
     private readonly offerExpiration: OfferExpirationJobHandler,
     private readonly offerFulfillment: OfferFulfillmentJobHandler,
   ) {
@@ -157,6 +159,9 @@ export class BillingJobsProcessor extends WorkerHost {
       case BillingJobName.SEARCH_REINDEX_COORDINATOR:
         await this.runSearchReindexCoordinator();
         break;
+      case BillingJobName.STORED_FILES_BACKFILL_COORDINATOR:
+        await this.runStoredFilesBackfillCoordinator();
+        break;
       case BillingJobName.OFFER_EXPIRATION_COORDINATOR:
         await this.runOfferExpirationCoordinator();
         break;
@@ -267,6 +272,9 @@ export class BillingJobsProcessor extends WorkerHost {
             case BillingJobName.SEARCH_REINDEX_UNIT:
               await this.runSearchReindexUnit(job.data as SearchReindexUnitPayload);
               break;
+            case BillingJobName.STORED_FILES_BACKFILL_UNIT:
+              await this.runStoredFilesBackfillUnit(job.data as { offset?: number; limit?: number; tenantId?: string });
+              break;
             case BillingJobName.SEARCH_INDEX_SYNC_UNIT:
               await this.searchReindexJobHandler.processSyncUnit(job.data as SearchIndexSyncUnitPayload);
               break;
@@ -796,6 +804,46 @@ export class BillingJobsProcessor extends WorkerHost {
     await this.contributorCollect.processTenant(data.tenantId);
   }
 
+  private async runStoredFilesBackfillCoordinator(): Promise<void> {
+    if (!(await this.storedFilesBackfillJobHandler.hasPendingWork())) {
+      return;
+    }
+
+    await this.enqueueBillingUnitJob({
+      queue: this.billingQueue,
+      jobName: BillingJobName.STORED_FILES_BACKFILL_UNIT,
+      payload: { tenantId: DEFAULT_TENANT, offset: 0, limit: 25 },
+      jobIdNamespace: 'stored-files-backfill',
+      jobIdParts: ['offset', '0'],
+    });
+  }
+
+  private async runStoredFilesBackfillUnit(data: {
+    offset?: number;
+    limit?: number;
+    tenantId?: string;
+  }): Promise<void> {
+    const offset = data.offset ?? 0;
+    const limit = data.limit ?? 25;
+    const result = await this.storedFilesBackfillJobHandler.processBackfillUnit(offset, limit);
+
+    if (!result.hasMore) {
+      return;
+    }
+
+    await this.enqueueBillingUnitJob({
+      queue: this.billingQueue,
+      jobName: BillingJobName.STORED_FILES_BACKFILL_UNIT,
+      payload: {
+        tenantId: data.tenantId ?? DEFAULT_TENANT,
+        offset: result.nextOffset,
+        limit,
+      },
+      jobIdNamespace: 'stored-files-backfill',
+      jobIdParts: ['offset', String(result.nextOffset)],
+    });
+  }
+
   private async runSearchReindexCoordinator(): Promise<void> {
     await this.forEachConfiguredTenant(async (tenantId) => {
       try {

apps/decabill/backend-billing-manager/src/typeorm.config.ts

@@ -12,6 +12,7 @@ import {
   DatevDebtorAccountEntity,
   DatevCreditorAccountEntity,
   DatevExportEntity,
+  StoredFileEntity,
   SupplierProfileEntity,
   SupplierContractEntity,
   SupplierInvoiceEntity,
@@ -130,6 +131,7 @@ export const typeormConfig: DataSourceOptions = {
     DatevDebtorAccountEntity,
     DatevCreditorAccountEntity,
     DatevExportEntity,
+    StoredFileEntity,
     SupplierProfileEntity,
     SupplierContractEntity,
     SupplierInvoiceEntity,