From b2c3e98564fee148c605873b57745d2dc13000ca Mon Sep 17 00:00:00 2001 From: maruson08 Date: Mon, 24 Aug 2026 19:39:32 +0900 Subject: [PATCH 1/3] =?UTF-8?q?=E2=9C=A8[Feat]=20Add=20JPEG=20cleaning=20a?= =?UTF-8?q?nd=20verification?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/core/errors.ts | 35 ++++++- src/core/limits.ts | 13 +++ src/core/types.ts | 33 +++++- src/index.ts | 12 ++- src/inspect.ts | 36 ++----- src/jpeg/parser.ts | 12 ++- src/jpeg/types.ts | 3 + src/policy/clean.ts | 241 ++++++++++++++++++++++++++++++++++++++++++- src/verify/verify.ts | 70 ++++++++++++- 9 files changed, 414 insertions(+), 41 deletions(-) diff --git a/src/core/errors.ts b/src/core/errors.ts index e7c0d07..2c5f5d3 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -1,10 +1,15 @@ +import type { Diagnostic } from "./diagnostics.js"; + export type SecureMetadataErrorCode = | "NOT_IMPLEMENTED" | "INVALID_OFFSET" | "INVALID_LENGTH" | "OUT_OF_BOUNDS" | "INVALID_LIMIT" - | "INPUT_LIMIT_EXCEEDED"; + | "INPUT_LIMIT_EXCEEDED" + | "UNSUPPORTED_FORMAT" + | "INCOMPLETE_JPEG" + | "CLEAN_OUTPUT_SIZE_INVALID"; export class SecureMetadataError extends Error { override readonly name: string = "SecureMetadataError"; @@ -75,3 +80,31 @@ export class InputLimitExceededError extends SecureMetadataError { ); } } + +export class UnsupportedFormatError extends SecureMetadataError { + override readonly name: string = "UnsupportedFormatError"; + + constructor( + readonly operation: "cleanMetadata" | "verifyMetadata", + readonly format: "png" | "webp" | "unknown", + ) { + super( + `${operation} does not support ${format} input.`, + "UNSUPPORTED_FORMAT", + ); + } +} + +export class IncompleteJpegError extends SecureMetadataError { + override readonly name: string = "IncompleteJpegError"; + + constructor( + readonly operation: "cleanMetadata" | "verifyMetadata", + readonly diagnostics: readonly Diagnostic[], + ) { + super( + `${operation} requires a structurally complete JPEG ending at EOI.`, + "INCOMPLETE_JPEG", + ); + } +} diff --git a/src/core/limits.ts b/src/core/limits.ts index f7be90b..64ae884 100644 --- a/src/core/limits.ts +++ b/src/core/limits.ts @@ -1,3 +1,5 @@ +import { InvalidParseLimitError } from "./errors.js"; + export interface ParseLimits { readonly maxInputBytes: number; readonly maxSegments: number; @@ -21,3 +23,14 @@ export const DEFAULT_PARSE_LIMITS: Readonly = Object.freeze({ maxDecompressedBytes: 16 * 1024 * 1024, maxDiagnostics: 256, }); + +export function resolveParseLimit( + name: keyof ParseLimits, + configured: number | undefined, +): number { + const value = configured ?? DEFAULT_PARSE_LIMITS[name]; + if (!Number.isSafeInteger(value) || value < 0) { + throw new InvalidParseLimitError(name, value); + } + return value; +} diff --git a/src/core/types.ts b/src/core/types.ts index 7772761..33e1eaa 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -93,23 +93,54 @@ export interface MetadataReport { } export interface CleaningPolicy { + readonly removeExif?: boolean; + readonly removeXmp?: boolean; + readonly removeIptc?: boolean; + readonly removeComments?: boolean; + readonly preserveIcc?: boolean; + /** @deprecated Use preserveIcc. */ readonly preserveColorProfiles?: boolean; readonly limits?: Partial; } +export interface MetadataChange { + readonly namespace: MetadataNamespace; + readonly action: "removed" | "preserved"; + readonly name: string; + readonly source: MetadataSource; +} + export interface CleanResult { readonly output: Uint8Array; + readonly format: "jpeg"; readonly report: MetadataReport; - readonly removedEntryIds: readonly string[]; + readonly removed: readonly MetadataChange[]; + readonly preserved: readonly MetadataChange[]; + readonly diagnostics: readonly Diagnostic[]; } +export type VerificationExpectation = "absent" | "present" | "ignore"; + export interface VerificationPolicy { + readonly exif?: VerificationExpectation; + readonly xmp?: VerificationExpectation; + readonly iptc?: VerificationExpectation; + readonly comments?: VerificationExpectation; + readonly icc?: VerificationExpectation; readonly requireNoPrivacyRelevantMetadata?: boolean; readonly limits?: Partial; } +export interface VerificationCheck { + readonly namespace: "exif" | "xmp" | "iptc" | "jpeg-comment" | "icc"; + readonly expected: Exclude; + readonly actual: "absent" | "present"; + readonly passed: boolean; +} + export interface VerificationResult { readonly valid: boolean; + readonly checks: readonly VerificationCheck[]; readonly report: MetadataReport; readonly diagnostics: readonly Diagnostic[]; } diff --git a/src/index.ts b/src/index.ts index ff7b780..19ad72e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,13 +1,18 @@ export { inspectMetadata } from "./inspect.js"; -export { cleanMetadata } from "./policy/clean.js"; -export { verifyMetadata } from "./verify/verify.js"; +export { cleanMetadata, DEFAULT_JPEG_CLEANING_POLICY } from "./policy/clean.js"; +export { + DEFAULT_JPEG_VERIFICATION_POLICY, + verifyMetadata, +} from "./verify/verify.js"; export { BinaryBoundsError, + IncompleteJpegError, InputLimitExceededError, InvalidParseLimitError, NotImplementedError, SecureMetadataError, + UnsupportedFormatError, } from "./core/errors.js"; export { DEFAULT_PARSE_LIMITS } from "./core/limits.js"; @@ -29,6 +34,7 @@ export type { InspectionStatus, InspectOptions, MetadataCategory, + MetadataChange, MetadataContainer, MetadataEntry, MetadataNamespace, @@ -37,6 +43,8 @@ export type { MetadataValue, PrivacyRelevance, RationalValue, + VerificationCheck, + VerificationExpectation, VerificationPolicy, VerificationResult, } from "./core/types.js"; diff --git a/src/inspect.ts b/src/inspect.ts index ff86b51..ffdbdc8 100644 --- a/src/inspect.ts +++ b/src/inspect.ts @@ -1,10 +1,7 @@ import { ByteReader, toUint8Array } from "./core/binary/index.js"; import { detectFormat } from "./core/detect-format.js"; -import { - InputLimitExceededError, - InvalidParseLimitError, -} from "./core/errors.js"; -import { DEFAULT_PARSE_LIMITS } from "./core/limits.js"; +import { InputLimitExceededError } from "./core/errors.js"; +import { DEFAULT_PARSE_LIMITS, resolveParseLimit } from "./core/limits.js"; import type { BinaryInput, InspectOptions, @@ -13,29 +10,12 @@ import type { import { inspectJpegMetadata } from "./jpeg/metadata.js"; import { parseJpeg } from "./jpeg/parser.js"; -function effectiveLimit( - name: - | "maxInputBytes" - | "maxSegments" - | "maxIfdEntries" - | "maxIfdDepth" - | "maxMetadataEntries" - | "maxStringBytes", - configured: number | undefined, -): number { - const value = configured ?? DEFAULT_PARSE_LIMITS[name]; - if (!Number.isSafeInteger(value) || value < 0) { - throw new InvalidParseLimitError(name, value); - } - return value; -} - export function inspectMetadata( input: BinaryInput, options?: InspectOptions, ): MetadataReport { const bytes = toUint8Array(input); - const maxInputBytes = effectiveLimit( + const maxInputBytes = resolveParseLimit( "maxInputBytes", options?.limits?.maxInputBytes, ); @@ -47,7 +27,7 @@ export function inspectMetadata( const reader = new ByteReader(bytes); const format = detectFormat(reader); if (format === "jpeg") { - const maxSegments = effectiveLimit( + const maxSegments = resolveParseLimit( "maxSegments", options?.limits?.maxSegments, ); @@ -57,19 +37,19 @@ export function inspectMetadata( ); const tiffLimits = { maxIfdEntries: hasExif - ? effectiveLimit("maxIfdEntries", options?.limits?.maxIfdEntries) + ? resolveParseLimit("maxIfdEntries", options?.limits?.maxIfdEntries) : DEFAULT_PARSE_LIMITS.maxIfdEntries, maxIfdDepth: hasExif - ? effectiveLimit("maxIfdDepth", options?.limits?.maxIfdDepth) + ? resolveParseLimit("maxIfdDepth", options?.limits?.maxIfdDepth) : DEFAULT_PARSE_LIMITS.maxIfdDepth, maxMetadataEntries: hasExif - ? effectiveLimit( + ? resolveParseLimit( "maxMetadataEntries", options?.limits?.maxMetadataEntries, ) : DEFAULT_PARSE_LIMITS.maxMetadataEntries, maxStringBytes: hasExif - ? effectiveLimit("maxStringBytes", options?.limits?.maxStringBytes) + ? resolveParseLimit("maxStringBytes", options?.limits?.maxStringBytes) : DEFAULT_PARSE_LIMITS.maxStringBytes, }; const metadata = inspectJpegMetadata(reader, jpeg, tiffLimits); diff --git a/src/jpeg/parser.ts b/src/jpeg/parser.ts index aeb17a1..f220429 100644 --- a/src/jpeg/parser.ts +++ b/src/jpeg/parser.ts @@ -14,6 +14,7 @@ import type { JpegParseResult, JpegSegment } from "./types.js"; interface MarkerPosition { readonly marker: number; readonly markerOffset: number; + readonly rangeOffset: number; readonly afterMarker: number; } @@ -73,6 +74,7 @@ function readMarker( return { marker, markerOffset: cursor - 1, + rangeOffset: offset, afterMarker: cursor + 1, }; } @@ -159,6 +161,8 @@ function skipScanData( markerName: markerName(marker), offset: markerOffset, length: 2, + rangeOffset: fillStart, + rangeLength: cursor + 1 - fillStart, kind: "standalone", }, maxSegments, @@ -210,6 +214,8 @@ export function parseJpeg( markerName: "SOI", offset: 0, length: 2, + rangeOffset: 0, + rangeLength: 2, kind: "standalone", }, maxSegments, @@ -226,7 +232,7 @@ export function parseJpeg( return incompleteResult(state, true); } - const { marker, markerOffset, afterMarker } = markerResult; + const { marker, markerOffset, rangeOffset, afterMarker } = markerResult; if (marker === JPEG_MARKER.SOI) { state.diagnostics.push( diagnostic( @@ -248,6 +254,8 @@ export function parseJpeg( markerName: markerName(marker), offset: markerOffset, length: 2, + rangeOffset, + rangeLength: afterMarker - rangeOffset, kind: "standalone", }, maxSegments, @@ -327,6 +335,8 @@ export function parseJpeg( markerName: markerName(marker), offset: markerOffset, length: declaredLength + 2, + rangeOffset, + rangeLength: segmentEnd - rangeOffset, payloadOffset, payloadLength, kind: classifySegmentKind(marker), diff --git a/src/jpeg/types.ts b/src/jpeg/types.ts index 6e93882..8a75bc9 100644 --- a/src/jpeg/types.ts +++ b/src/jpeg/types.ts @@ -19,6 +19,9 @@ export interface JpegSegment { readonly markerName: string; readonly offset: number; readonly length: number; + /** Internal copy/remove range, including any marker fill bytes. */ + readonly rangeOffset: number; + readonly rangeLength: number; readonly payloadOffset?: number; readonly payloadLength?: number; readonly kind: JpegSegmentKind; diff --git a/src/policy/clean.ts b/src/policy/clean.ts index 450ef11..17af914 100644 --- a/src/policy/clean.ts +++ b/src/policy/clean.ts @@ -1,15 +1,248 @@ -import { NotImplementedError } from "../core/errors.js"; +import { ByteReader, toUint8Array } from "../core/binary/index.js"; +import { detectFormat } from "../core/detect-format.js"; +import { + IncompleteJpegError, + InputLimitExceededError, + SecureMetadataError, + UnsupportedFormatError, +} from "../core/errors.js"; +import { resolveParseLimit } from "../core/limits.js"; import type { BinaryInput, CleaningPolicy, CleanResult, + MetadataChange, + MetadataNamespace, } from "../core/types.js"; +import { inspectMetadata } from "../inspect.js"; +import { parseJpeg } from "../jpeg/parser.js"; +import type { JpegSegment } from "../jpeg/types.js"; + +export const DEFAULT_JPEG_CLEANING_POLICY = Object.freeze({ + removeExif: true, + removeXmp: true, + removeIptc: true, + removeComments: true, + preserveIcc: true, +}); + +interface EffectivePolicy { + readonly removeExif: boolean; + readonly removeXmp: boolean; + readonly removeIptc: boolean; + readonly removeComments: boolean; + readonly preserveIcc: boolean; +} + +function effectivePolicy(policy: CleaningPolicy | undefined): EffectivePolicy { + return { + removeExif: policy?.removeExif ?? true, + removeXmp: policy?.removeXmp ?? true, + removeIptc: policy?.removeIptc ?? true, + removeComments: policy?.removeComments ?? true, + preserveIcc: policy?.preserveIcc ?? policy?.preserveColorProfiles ?? true, + }; +} + +function shouldRemove(segment: JpegSegment, policy: EffectivePolicy): boolean { + if (segment.kind === "comment") { + return policy.removeComments; + } + + switch (segment.metadataKind) { + case "exif": + return policy.removeExif; + case "xmp": + return policy.removeXmp; + case "iptc": + return policy.removeIptc; + case "icc": + return !policy.preserveIcc; + default: + return false; + } +} + +function changeFor( + segment: JpegSegment, + action: MetadataChange["action"], +): MetadataChange { + let namespace: MetadataNamespace = "container"; + let name = segment.markerName; + + if (segment.kind === "comment") { + namespace = "jpeg-comment"; + name = "JPEG comment"; + } else { + switch (segment.metadataKind) { + case "exif": + namespace = "exif"; + name = "EXIF container"; + break; + case "xmp": + namespace = "xmp"; + name = + segment.metadataSubtype === "extended-xmp" + ? "Extended XMP container" + : "XMP container"; + break; + case "iptc": + namespace = "iptc"; + name = "Photoshop/IPTC container"; + break; + case "icc": + namespace = "icc"; + name = "ICC profile container"; + break; + case "jfif": + name = + segment.metadataSubtype === "jfxx" + ? "JFXX application segment" + : "JFIF application segment"; + break; + case "adobe": + name = "Adobe application segment"; + break; + case "unknown": + namespace = "unknown"; + name = `Unknown ${segment.markerName} application segment`; + break; + } + } + + return { + namespace, + action, + name, + source: { + format: "jpeg", + container: "jpeg-segment", + offset: segment.offset, + length: segment.length, + jpegMarker: segment.marker, + }, + }; +} + +function copyWithoutSegments( + input: Uint8Array, + removals: readonly JpegSegment[], +): Uint8Array { + const retained: Array<{ offset: number; length: number }> = []; + let inputOffset = 0; + let outputLength = 0; + + for (const segment of removals) { + const end = segment.rangeOffset + segment.rangeLength; + if ( + !Number.isSafeInteger(segment.rangeOffset) || + !Number.isSafeInteger(segment.rangeLength) || + segment.rangeLength <= 0 || + !Number.isSafeInteger(end) || + segment.rangeOffset < inputOffset || + end > input.byteLength + ) { + throw new SecureMetadataError( + "JPEG cleaner produced an invalid removal range.", + "CLEAN_OUTPUT_SIZE_INVALID", + ); + } + + const length = segment.rangeOffset - inputOffset; + retained.push({ offset: inputOffset, length }); + outputLength += length; + if ( + !Number.isSafeInteger(outputLength) || + outputLength > input.byteLength + ) { + throw new SecureMetadataError( + "JPEG cleaner output size is invalid.", + "CLEAN_OUTPUT_SIZE_INVALID", + ); + } + inputOffset = end; + } + + const tailLength = input.byteLength - inputOffset; + retained.push({ offset: inputOffset, length: tailLength }); + outputLength += tailLength; + if ( + !Number.isSafeInteger(outputLength) || + outputLength < 0 || + outputLength > input.byteLength + ) { + throw new SecureMetadataError( + "JPEG cleaner output size is invalid.", + "CLEAN_OUTPUT_SIZE_INVALID", + ); + } + + const output = new Uint8Array(outputLength); + let outputOffset = 0; + for (const range of retained) { + output.set( + input.subarray(range.offset, range.offset + range.length), + outputOffset, + ); + outputOffset += range.length; + } + return output; +} export function cleanMetadata( input: BinaryInput, policy?: CleaningPolicy, ): CleanResult { - void input; - void policy; - throw new NotImplementedError("cleanMetadata"); + const bytes = toUint8Array(input); + const maxInputBytes = resolveParseLimit( + "maxInputBytes", + policy?.limits?.maxInputBytes, + ); + if (bytes.byteLength > maxInputBytes) { + throw new InputLimitExceededError(bytes.byteLength, maxInputBytes); + } + + const reader = new ByteReader(bytes); + const format = detectFormat(reader); + if (format !== "jpeg") { + throw new UnsupportedFormatError("cleanMetadata", format); + } + + const jpeg = parseJpeg( + reader, + resolveParseLimit("maxSegments", policy?.limits?.maxSegments), + ); + if (!jpeg.complete) { + throw new IncompleteJpegError("cleanMetadata", jpeg.diagnostics); + } + + const resolved = effectivePolicy(policy); + const removals = jpeg.segments.filter((segment) => + shouldRemove(segment, resolved), + ); + const removed = removals.map((segment) => changeFor(segment, "removed")); + const preserved = jpeg.segments + .filter( + (segment) => + (segment.kind === "application" || segment.kind === "comment") && + !shouldRemove(segment, resolved), + ) + .map((segment) => changeFor(segment, "preserved")); + const output = copyWithoutSegments(bytes, removals); + const report = inspectMetadata( + output, + policy?.limits === undefined ? undefined : { limits: policy.limits }, + ); + if (report.inspectionStatus === "container-partial") { + throw new IncompleteJpegError("cleanMetadata", report.diagnostics); + } + + return { + output, + format: "jpeg", + report, + removed, + preserved, + diagnostics: report.diagnostics, + }; } diff --git a/src/verify/verify.ts b/src/verify/verify.ts index 6eef2ec..b03d69b 100644 --- a/src/verify/verify.ts +++ b/src/verify/verify.ts @@ -1,15 +1,77 @@ -import { NotImplementedError } from "../core/errors.js"; +import { IncompleteJpegError, UnsupportedFormatError } from "../core/errors.js"; import type { BinaryInput, + VerificationCheck, + VerificationExpectation, VerificationPolicy, VerificationResult, } from "../core/types.js"; +import { inspectMetadata } from "../inspect.js"; + +export const DEFAULT_JPEG_VERIFICATION_POLICY = Object.freeze({ + exif: "absent", + xmp: "absent", + iptc: "absent", + comments: "absent", + icc: "ignore", +} satisfies Record); export function verifyMetadata( input: BinaryInput, expectation?: VerificationPolicy, ): VerificationResult { - void input; - void expectation; - throw new NotImplementedError("verifyMetadata"); + const report = inspectMetadata( + input, + expectation?.limits === undefined + ? undefined + : { limits: expectation.limits }, + ); + if (report.format !== "jpeg") { + throw new UnsupportedFormatError("verifyMetadata", report.format); + } + if (report.inspectionStatus === "container-partial") { + throw new IncompleteJpegError("verifyMetadata", report.diagnostics); + } + + const privacyDefault = + expectation?.requireNoPrivacyRelevantMetadata === false + ? "ignore" + : "absent"; + const expected = { + exif: expectation?.exif ?? privacyDefault, + xmp: expectation?.xmp ?? privacyDefault, + iptc: expectation?.iptc ?? privacyDefault, + "jpeg-comment": expectation?.comments ?? privacyDefault, + icc: expectation?.icc ?? "ignore", + } satisfies Record< + "exif" | "xmp" | "iptc" | "jpeg-comment" | "icc", + VerificationExpectation + >; + + const checks: VerificationCheck[] = []; + for (const [namespace, wanted] of Object.entries(expected) as Array< + [VerificationCheck["namespace"], VerificationExpectation] + >) { + if (wanted === "ignore") { + continue; + } + + const present = report.entries.some( + (entry) => entry.namespace === namespace, + ); + const actual = present ? "present" : "absent"; + checks.push({ + namespace, + expected: wanted, + actual, + passed: actual === wanted, + }); + } + + return { + valid: checks.every(({ passed }) => passed), + checks, + report, + diagnostics: report.diagnostics, + }; } From 9ac6af4ba390214d3a1eef59730d81c2d74db0b9 Mon Sep 17 00:00:00 2001 From: maruson08 Date: Mon, 24 Aug 2026 19:39:33 +0900 Subject: [PATCH 2/3] =?UTF-8?q?=E2=9C=85[Test]=20Cover=20JPEG=20cleaner=20?= =?UTF-8?q?and=20verification=20invariants?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/unit/jpeg-cleaner-verification.test.ts | 253 +++++++++++++++++++ tests/unit/public-api.test.ts | 16 -- 2 files changed, 253 insertions(+), 16 deletions(-) create mode 100644 tests/unit/jpeg-cleaner-verification.test.ts diff --git a/tests/unit/jpeg-cleaner-verification.test.ts b/tests/unit/jpeg-cleaner-verification.test.ts new file mode 100644 index 0000000..5553590 --- /dev/null +++ b/tests/unit/jpeg-cleaner-verification.test.ts @@ -0,0 +1,253 @@ +import { describe, expect, it } from "vitest"; + +import { + cleanMetadata, + IncompleteJpegError, + inspectMetadata, + UnsupportedFormatError, + verifyMetadata, +} from "../../src/index.js"; +import { + concat, + EXIF, + EXTENDED_XMP, + ICC, + JFIF, + jpeg, + marker, + MARKER, + PHOTOSHOP, + segment, + XMP, +} from "../helpers/jpeg-builder.js"; +import { TiffBuilder } from "../helpers/tiff-builder.js"; + +function canonicalFixture() { + const jfif = segment(MARKER.APP0, concat(JFIF, Uint8Array.of(1, 2)), 2); + const exif = segment( + MARKER.APP1, + concat(EXIF, new TiffBuilder().ifd(8, []).finish(14)), + 3, + ); + const xmp = segment(MARKER.APP1, concat(XMP, Uint8Array.of(0x78))); + const extendedXmp = segment( + MARKER.APP1, + concat(EXTENDED_XMP, Uint8Array.of(0x79)), + ); + const secondExif = segment( + MARKER.APP1, + concat(EXIF, new TiffBuilder("big").ifd(8, []).finish(14)), + ); + const icc = segment(MARKER.APP2, concat(ICC, Uint8Array.of(1, 1, 0xaa))); + const unknown = segment(0xe3, Uint8Array.of(0xde, 0xad, 0xbe, 0xef)); + const comment = segment(MARKER.COM, Uint8Array.of(0x80, 0x00), 2); + const dqt = segment(MARKER.DQT, Uint8Array.of(0x01, 0x02)); + const firstSos = segment(MARKER.SOS, Uint8Array.of(0x01)); + const firstScan = Uint8Array.of(0x11, 0xff, 0x00, 0x22, 0xff, 0xd0, 0x33); + const iptc = segment(MARKER.APP13, concat(PHOTOSHOP, Uint8Array.of(0x44))); + const dht = segment(MARKER.DHT, Uint8Array.of(0x55)); + const secondSos = segment(MARKER.SOS, Uint8Array.of(0x02)); + const secondScan = Uint8Array.of(0x66, 0xff, 0x00, 0x77); + const trailing = Uint8Array.of(0xfa, 0xfb); + + return { + input: concat( + jpeg( + jfif, + exif, + xmp, + icc, + unknown, + extendedXmp, + secondExif, + comment, + dqt, + firstSos, + firstScan, + iptc, + dht, + secondSos, + secondScan, + ), + trailing, + ), + expected: concat( + jpeg( + jfif, + icc, + unknown, + dqt, + firstSos, + firstScan, + dht, + secondSos, + secondScan, + ), + trailing, + ), + }; +} + +describe("JPEG Privacy Clean", () => { + it("removes every targeted container while preserving retained bytes, scans, ordering, and trailing data", () => { + const { input, expected } = canonicalFixture(); + const before = Uint8Array.from(input); + + const first = cleanMetadata(input); + const second = cleanMetadata(input); + const idempotent = cleanMetadata(first.output); + const verification = verifyMetadata(first.output, { icc: "present" }); + + expect(first.output).toEqual(expected); + expect(first.output).not.toBe(input); + expect(first.output.byteLength).toBeLessThan(input.byteLength); + expect(input).toEqual(before); + expect(second.output).toEqual(first.output); + expect(idempotent.output).toEqual(first.output); + expect(idempotent.removed).toEqual([]); + expect(first.removed.map(({ namespace }) => namespace)).toEqual([ + "exif", + "xmp", + "xmp", + "exif", + "jpeg-comment", + "iptc", + ]); + expect(first.preserved.map(({ namespace }) => namespace)).toEqual([ + "container", + "icc", + "unknown", + ]); + expect(first.report.entries.map(({ namespace }) => namespace)).toEqual([ + "icc", + ]); + expect(verification.valid).toBe(true); + expect(verification.checks).toHaveLength(5); + }); + + it("returns a separate byte-identical output when there is nothing to remove", () => { + const input = jpeg( + segment(MARKER.APP0, JFIF), + segment(MARKER.APP2, ICC), + segment(0xe4, Uint8Array.of(1, 2, 3)), + ); + + const result = cleanMetadata(input); + + expect(result.output).toEqual(input); + expect(result.output).not.toBe(input); + expect(result.removed).toEqual([]); + expect(verifyMetadata(result.output).valid).toBe(true); + }); + + it("supports one explicit custom policy without inferring unknown removal", () => { + const xmp = segment(MARKER.APP1, XMP); + const comment = segment(MARKER.COM, Uint8Array.of(1, 2)); + const input = jpeg( + segment(MARKER.APP1, EXIF), + xmp, + segment(MARKER.APP2, ICC), + segment(0xe3, Uint8Array.of(3, 4)), + comment, + ); + + const result = cleanMetadata(input, { + removeXmp: false, + removeComments: false, + preserveIcc: false, + }); + + expect(result.output).toEqual( + jpeg(xmp, segment(0xe3, Uint8Array.of(3, 4)), comment), + ); + expect(result.removed.map(({ namespace }) => namespace)).toEqual([ + "exif", + "icc", + ]); + expect(result.preserved.map(({ namespace }) => namespace)).toEqual([ + "xmp", + "unknown", + "jpeg-comment", + ]); + }); + + it.each([ + concat(marker(MARKER.SOI), marker(MARKER.APP1)), + concat( + marker(MARKER.SOI), + segment(MARKER.SOS), + Uint8Array.of(1, 0xff, 0x00), + ), + ])("rejects structurally incomplete JPEG without partial output", (input) => { + expect(() => cleanMetadata(input)).toThrowError(IncompleteJpegError); + expect(() => cleanMetadata(input)).toThrowError( + expect.objectContaining({ code: "INCOMPLETE_JPEG" }), + ); + }); + + it("removes structurally bounded EXIF even when its TIFF payload is malformed", () => { + const input = jpeg(segment(MARKER.APP1, concat(EXIF, Uint8Array.of(0x49)))); + + expect(inspectMetadata(input).diagnostics).toContainEqual( + expect.objectContaining({ code: "TIFF_TRUNCATED_HEADER" }), + ); + + const result = cleanMetadata(input); + + expect(result.output).toEqual(jpeg()); + expect(result.removed).toEqual([ + expect.objectContaining({ namespace: "exif", action: "removed" }), + ]); + expect(verifyMetadata(result.output).valid).toBe(true); + }); + + it("honors the exact supplied Uint8Array subview", () => { + const embedded = canonicalFixture().input; + const prefix = Uint8Array.of(0xaa, 0xbb, 0xcc); + const suffix = Uint8Array.of(0xdd, 0xee); + const backing = concat(prefix, embedded, suffix); + const view = new Uint8Array( + backing.buffer, + backing.byteOffset + prefix.byteLength, + embedded.byteLength, + ); + + expect(cleanMetadata(view).output).toEqual(canonicalFixture().expected); + }); +}); + +describe("JPEG verification", () => { + it("returns a precise failed check when an expected-absent container remains", () => { + const result = verifyMetadata(jpeg(segment(MARKER.APP1, XMP))); + + expect(result.valid).toBe(false); + expect(result.checks).toContainEqual({ + namespace: "xmp", + expected: "absent", + actual: "present", + passed: false, + }); + }); + + it.each([ + [Uint8Array.of(0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a), "png"], + [ + Uint8Array.of(0x52, 0x49, 0x46, 0x46, 0, 0, 0, 0, 0x57, 0x45, 0x42, 0x50), + "webp", + ], + [new Uint8Array(), "unknown"], + ] as const)( + "rejects unsupported $format cleaning and verification", + (input, format) => { + for (const operation of [cleanMetadata, verifyMetadata]) { + expect(() => operation(input)).toThrowError(UnsupportedFormatError); + expect(() => operation(input)).toThrowError( + expect.objectContaining({ + code: "UNSUPPORTED_FORMAT", + format, + }), + ); + } + }, + ); +}); diff --git a/tests/unit/public-api.test.ts b/tests/unit/public-api.test.ts index 37ede02..9ae67e8 100644 --- a/tests/unit/public-api.test.ts +++ b/tests/unit/public-api.test.ts @@ -3,7 +3,6 @@ import { describe, expect, it } from "vitest"; import { cleanMetadata, inspectMetadata, - NotImplementedError, verifyMetadata, type BinaryInput, type MetadataEntry, @@ -16,21 +15,6 @@ describe("public API", () => { expect(verifyMetadata).toBeTypeOf("function"); }); - it.each([ - ["cleanMetadata", cleanMetadata], - ["verifyMetadata", verifyMetadata], - ] as const)( - "keeps deterministic unimplemented behavior for %s", - (_, operation) => { - expect(() => operation(new Uint8Array())).toThrowError( - NotImplementedError, - ); - expect(() => operation(new Uint8Array())).toThrowError( - expect.objectContaining({ code: "NOT_IMPLEMENTED" }), - ); - }, - ); - it("accepts Uint8Array and ArrayBuffer as public inspection inputs", () => { const inputs: readonly BinaryInput[] = [ new Uint8Array(), From 739bfceb1110a6c0e8b6bf6fc30324b5ae47320d Mon Sep 17 00:00:00 2001 From: maruson08 Date: Mon, 24 Aug 2026 19:41:34 +0900 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=93=84[Docs]=20Document=20JPEG=20clea?= =?UTF-8?q?ning=20guarantees?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 10 ++++++++++ README.md | 12 ++++++++---- docs/architecture.md | 14 ++++++++++++++ docs/cleaning-policy.md | 28 ++++++++++++++++++---------- docs/format-support.md | 9 ++++++++- docs/security-model.md | 12 ++++++++++-- 6 files changed, 68 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79938dc..2a08593 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ All notable changes will be documented here. The project intends to follow seman ### Added +- Deterministic JPEG Privacy Clean for whole EXIF, XMP, Photoshop/IPTC, and COM segments. +- Checked single-allocation JPEG reconstruction preserving ICC, unknown APP, structural, scan, and trailing bytes. +- Structured JPEG presence/absence verification and typed unsupported/incomplete-input errors. +- Compact canonical coverage for determinism, idempotency, multiple scans and metadata instances, malformed TIFF removal, and exact subviews. + - Shared bounded little- and big-endian TIFF/EXIF decoder. - Iterative IFD0, ExifIFD, GPSIFD, and next-IFD traversal. - IFD entry/depth limits and repeated-offset cycle protection. @@ -17,6 +22,11 @@ All notable changes will be documented here. The project intends to follow seman - Bounded JPEG marker traversal and EXIF/XMP/ICC/IPTC container detection. - Binary boundary, JPEG container, TIFF endian, malformed, cycle, and integration tests. +### Changed + +- JPEG parser records internal fill-aware rewrite ranges while retaining existing public source offsets. +- Parse-limit validation is shared by inspection and cleaning. + ### Foundation - Repository and TypeScript library scaffold. diff --git a/README.md b/README.md index e470e37..9b443f0 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,11 @@ Current implementation: - JPEG EXIF, XMP, ICC, Photoshop/IPTC, and comment container detection; - shared little- and big-endian TIFF/EXIF decoder; - iterative IFD0, ExifIFD, GPSIFD, and next-IFD traversal with cycle and depth protection; -- common TIFF, EXIF, and GPS field decoding with exact rational values. +- common TIFF, EXIF, and GPS field decoding with exact rational values; +- deterministic whole-segment JPEG Privacy Clean with byte-preserving reconstruction; +- structured JPEG verification for observable container presence or absence. -Not implemented: MakerNote or thumbnail decoding, XMP/IPTC/ICC payload parsing, PNG/WebP container parsing, metadata cleaning, and verification. +Not implemented: MakerNote or thumbnail decoding, XMP/IPTC/ICC payload parsing, PNG/WebP container parsing or cleaning, and PNG/WebP verification. ## Format status @@ -38,11 +40,13 @@ import { GPS rational components remain exact numerator/denominator pairs; decimal coordinates are not derived. Unknown TIFF tags and MakerNote are represented structurally without dumping or recursively parsing their payloads. -`cleanMetadata` and `verifyMetadata` still throw a typed `NotImplementedError`. +`cleanMetadata` supports JPEG. Its default policy removes complete EXIF, standard/extended XMP, Photoshop/IPTC, and COM segments while preserving ICC, JFIF/JFXX, Adobe APP14, unknown APP segments, structural data, scan bytes, and trailing bytes. It returns a separate output, container-level change evidence, and an inspection report of that output. + +`verifyMetadata` supports JPEG expectations of `absent`, `present`, or `ignore` for EXIF, XMP, IPTC, comments, and ICC. The default checks the four privacy-clean removal targets. A single-file verification can observe presence or absence; it cannot prove that bytes came from an original file. ## Security philosophy -Every byte is untrusted. All offsets are interpreted within bounded views, traversal is iterative and limited, repeated IFD offsets are rejected, and malformed entries recover without unchecked access. Unknown structures remain unknown and should be preserved by future cleaning. See the [security model](docs/security-model.md), [architecture](docs/architecture.md), and [cleaning policy](docs/cleaning-policy.md). +Every byte is untrusted. All offsets are interpreted within bounded views, traversal is iterative and limited, repeated IFD offsets are rejected, and malformed entries recover without unchecked access. Unknown JPEG APP structures remain unknown and are preserved by cleaning. See the [security model](docs/security-model.md), [architecture](docs/architecture.md), and [cleaning policy](docs/cleaning-policy.md). ## Non-goals diff --git a/docs/architecture.md b/docs/architecture.md index 8d6a414..6bf23a7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -32,3 +32,17 @@ Unknown tags retain namespace, tag number, TIFF type, count, entry offset, and s - `container-partial`: JPEG traversal stopped on corruption, truncation, or a limit. - `metadata-partial`: JPEG container traversal completed and common TIFF/EXIF decoding was attempted; XMP/IPTC/ICC and unknown fields remain incomplete. - `metadata-inspected`: reserved for future broader decoders. + +## JPEG clean and verify flow + +```text +input JPEG + → bounded JPEG parser and existing APP classification + → direct keep/remove policy + → checked retained ranges + → one output allocation and ordered byte copies + → inspectMetadata(output) + → structured verification checks +``` + +The parser remains the structural source of truth. Internal rewrite ranges include marker fill bytes associated with a removed marker while public source offsets retain their existing meaning. Cleaning does not invoke TIFF decoding on the source: a structurally bounded EXIF APP1 can be removed even if its TIFF body is malformed. The post-write inspection and verifier use the normal inspection layer. diff --git a/docs/cleaning-policy.md b/docs/cleaning-policy.md index 169d899..356f8fa 100644 --- a/docs/cleaning-policy.md +++ b/docs/cleaning-policy.md @@ -1,15 +1,23 @@ -# Cleaning Policy Direction +# Cleaning Policy -Cleaning is not implemented in Sprint 0. This document records the intended conservative policy for future work. +JPEG Privacy Clean removes complete recognized metadata containers. It never rewrites TIFF/EXIF fields, XMP XML, IPTC blocks, comments, or ICC payloads. -An initial privacy-clean mode should remove EXIF, GPS, XMP, IPTC, comments, and privacy-relevant textual metadata. It should preserve the encoded image payload, required container structures, ICC and other color profiles, rendering-critical metadata, and unknown structures unless the relevant format specification proves removal is safe. +| JPEG structure | Default action | +| -------------------------------------- | -------------- | +| EXIF APP1 | Remove | +| Standard XMP APP1 | Remove | +| Extended XMP APP1 | Remove | +| Photoshop/IPTC APP13 | Remove | +| COM | Remove | +| ICC APP2 | Preserve | +| JFIF/JFXX APP0 | Preserve | +| Adobe APP14 | Preserve | +| Unknown APP | Preserve | +| Structural markers and image/scan data | Preserve | +| Data after EOI | Preserve | -For v0.1, whole EXIF containers are preferred over selective TIFF rewriting: +Every recognized instance is handled independently and retained content keeps its original order and bytes. Unknown APP removal is intentionally unavailable in Sprint 4. Callers may override the four removal booleans and ICC preservation; `preserveColorProfiles` remains a deprecated alias for `preserveIcc`. -```text -JPEG APP1 EXIF → remove whole EXIF APP1 -PNG eXIf → remove whole eXIf chunk -WebP EXIF → remove whole EXIF chunk -``` +v0.1 removes the entire EXIF APP1, including malformed TIFF bodies whose JPEG segment boundary is valid. Selective GPS or tag rewriting and TIFF reserialization are deferred. -Selective EXIF field rewriting is postponed. This reduces offset-rewrite complexity and makes cleaner behavior easier to audit. Unaffected bytes should remain byte-for-byte identical whenever the container format permits it, and output must be re-inspected rather than trusted merely because a write completed. +`cleanMetadata` returns a new `Uint8Array`, container-level removed/preserved records, diagnostics, and an inspection report of the produced JPEG. A structurally incomplete JPEG is rejected before allocation. PNG, WebP, and unknown inputs return a typed unsupported-format error. diff --git a/docs/format-support.md b/docs/format-support.md index b1e5581..c6b2720 100644 --- a/docs/format-support.md +++ b/docs/format-support.md @@ -11,7 +11,8 @@ | MakerNote decoding | Not supported | Not supported | Not supported | | XMP payload decoding | Not yet | Not yet | Not yet | | IPTC/ICC payload decoding | Not yet | Not yet | Not yet | -| Cleaning and verification | Not yet | Not yet | Not yet | +| Whole-container cleaning | Supported | Not yet | Not yet | +| Structured verification | Supported | Not yet | Not yet | ## TIFF/EXIF subset @@ -28,3 +29,9 @@ Unknown tags remain structurally represented without speculative meaning or larg ## Remaining container support JPEG marker and scan traversal remains supported. XMP, ICC, and Photoshop/IPTC signatures are container-detected only. PNG requires its complete signature and WebP requires `RIFF....WEBP`; their chunks and metadata are not parsed yet. + +## JPEG cleaning and verification + +JPEG Privacy Clean removes recognized EXIF, standard/extended XMP, Photoshop/IPTC, and comment segments. ICC, JFIF/JFXX, Adobe APP14, unknown APP segments, structural markers, all scan data, and trailing bytes are retained. Structurally incomplete JPEGs are rejected; malformed TIFF inside a bounded removable EXIF segment does not block cleaning. + +Verification reports observable container presence or absence for EXIF, XMP, IPTC, comments, and ICC. It does not decode XMP/IPTC/ICC payloads or prove preservation from an original input. diff --git a/docs/security-model.md b/docs/security-model.md index 8459b57..e5680ea 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -10,8 +10,8 @@ Binary metadata parsing processes attacker-controlled structures, sizes, offsets 4. Core functions make no network requests and access no filesystem or DOM APIs. 5. Image pixel payloads are never decoded. 6. Unknown metadata is not deleted or assigned speculative meaning. -7. ICC and color data will be preserved by default during future cleaning. -8. Cleaner output must eventually be independently inspected and verified. +7. JPEG cleaning preserves ICC, unknown APP, and rendering/container segments by default. +8. Cleaner output is re-inspected before it is returned. 9. Metadata absence never proves an image contains no private information. 10. Steganography detection, malware scanning, visual redaction, and pixel privacy analysis are outside scope. @@ -40,3 +40,11 @@ Every traversal or decoding loop has a validated finite count or advances a boun ## Environment and dependencies Core code is local-only and side-effect-free. It has no network, analytics, telemetry, filesystem, DOM, or pixel-codec behavior. The package has zero runtime dependencies. + +## JPEG cleaning properties + +Cleaning proceeds only after bounded traversal reaches EOI. Truncated lengths, invalid marker structure, unterminated scans, and segment-limit failures produce a typed `IncompleteJpegError`; no partial output is returned. TIFF validity is not required to remove a structurally bounded EXIF APP1. + +Removal uses checked, non-overlapping parser ranges. Output length is a safe integer no larger than input length, one output buffer is allocated, and retained ranges are copied in original order. Entropy-coded bytes, restart markers, retained marker fill, structural segments, and bytes after EOI are neither decoded nor regenerated. Exact `Uint8Array` views are honored and caller input is never mutated. + +The default policy preserves every ICC and unknown APP segment. Verification proves only the requested observable container state supported by inspection. It does not prove provenance, byte preservation without an original, absence of unknown metadata, or absence of personal information in pixels or unsupported structures.