diff --git a/.dockerignore b/.dockerignore index 41d77ac2..ea08c564 100644 --- a/.dockerignore +++ b/.dockerignore @@ -64,5 +64,4 @@ CODE_OF_CONDUCT.md # Build tooling not needed by the installed package tools/ MANIFEST.in -requirements*.txt docker-compose.yaml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..e0ff20ab --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,25 @@ +# Code owners for ForgeLM. +# +# When a pull request is opened, GitHub requests a review from the owner of each +# path it changes. This file only routes those review requests: whether a review +# is *required* before merging is a branch-protection setting, not anything an +# entry here can turn on. See docs/standards/code-review.md for the review +# standard and GOVERNANCE.md for how ownership is granted. +# +# These rules are evaluated last-match-wins, so a more specific pattern placed +# below `*` overrides it. When an area gets its own owner, add their line below +# the default rather than replacing it, so the default owner stays on every path +# an area owner does not name. GitHub does not request a review for a draft or a +# draft pull request. + +* @cemililik + +# Repository scaffolding and policy files stay with the default owner even after +# an area owner is added, because these paths define the rules the area owners +# work under. +/.github/ @cemililik +/CONTRIBUTING.md @cemililik +/GOVERNANCE.md @cemililik +/MAINTAINERS.md @cemililik +/SECURITY.md @cemililik +/pyproject.toml @cemililik diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index c3756212..acb566f8 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -3,9 +3,9 @@ contact_links: - name: Report a security vulnerability url: https://github.com/HodeTech/ForgeLM/security/advisories/new about: Report vulnerabilities privately, never in a public issue (see SECURITY.md) - - name: Documentation + - name: Browse the documentation url: https://github.com/HodeTech/ForgeLM/tree/main/docs - about: Check the docs and guides before opening an issue + about: Read the docs and guides before opening a documentation issue - name: Discussions (Q&A) url: https://github.com/HodeTech/ForgeLM/discussions about: Ask questions and get help from the community diff --git a/.github/ISSUE_TEMPLATE/documentation.yml b/.github/ISSUE_TEMPLATE/documentation.yml new file mode 100644 index 00000000..752e62c7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.yml @@ -0,0 +1,52 @@ +name: Documentation +description: Report a documentation error, or propose an improvement to a guide or its Turkish mirror +labels: ["documentation", "needs-triage"] +body: + - type: markdown + attributes: + value: | + Thanks for improving the docs! One correction per issue is easiest to review. + + User-facing documentation is kept as an English page **and** a Turkish + mirror, and the two are updated together — see + [CONTRIBUTING.md](https://github.com/HodeTech/ForgeLM/blob/development/CONTRIBUTING.md). + You do not need to write the Turkish yourself: open the pull request with + the English change and say so, and a maintainer will complete the mirror. + Tell us below which copy you were reading. Documentation pull requests are + welcome directly. + + - type: input + id: page + attributes: + label: Page or file + description: The repository path or the documentation-site page you were reading. + placeholder: docs/guides/quickstart.md — or the Quick start page on the docs site + validations: + required: true + + - type: textarea + id: problem + attributes: + label: What is wrong or missing? + description: What does the page say, and what does the reader need instead? + validations: + required: true + + - type: textarea + id: suggestion + attributes: + label: Suggested wording (optional) + description: If you already know the fix, propose the text. We will tidy the language if needed. + + - type: dropdown + id: language + attributes: + label: Which copy is affected? + description: User-facing docs are maintained as an English page and a Turkish mirror. An issue that needs mirror work is labelled `i18n` when it is triaged. + options: + - English + - Turkish + - Both + - Not sure + validations: + required: true diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..69d81387 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,64 @@ +# Dependabot configuration for ForgeLM. +# +# Dependabot opens a pull request when a dependency's declared range no longer +# covers the newest release. It is the upstream half of the supply-chain story: +# `tools/check_pip_audit.py` and the nightly CVE job catch known advisories, and +# these pull requests make a fix reachable. +# +# No `versioning-strategy` is set for `pip`. The `pip` ecosystem accepts only `auto`, +# `increase`, `increase-if-necessary` and `lockfile-only`; a previous `widen` is not +# among them, GitHub rejected the whole file ("did not match one of the following +# values"), and while it did not parse Dependabot opened nothing. ForgeLM declares +# bounded ranges (`>=floor, "Assign on comment" > the "..." menu > +# "Disable workflow". Assignments already made stay as they are. Changing who an issue is assigned to is a +# maintainer's decision, made in the issue's sidebar; nothing here ever overrides it. + +on: + issue_comment: + types: [created] + +permissions: {} + +jobs: + assign: + # Event data only. Skips pull-request comments and bots, and runs only for a comment that starts with a command. + if: ${{ !github.event.issue.pull_request && github.event.comment.user.type != 'Bot' && (startsWith(github.event.comment.body, '/assign') || startsWith(github.event.comment.body, '/unassign')) }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read # the checkout reads one script from the default branch + issues: write # add or remove the assignee, and post the reply + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: tools/assign_on_comment.py + sparse-checkout-cone-mode: false + - name: Assign or release the issue + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python3 tools/assign_on_comment.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3b75537..23514fc6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,9 +27,11 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 + with: + persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: "3.11" @@ -118,9 +120,11 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 + with: + persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} cache: pip @@ -148,7 +152,7 @@ jobs: - name: Upload coverage (3.11 only) # Upload coverage from one combo only — avoids 4× artifact duplication; 3.11 is canonical (matches publish.yml build job). if: matrix.python-version == '3.11' && always() - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v7 with: name: coverage-report path: coverage.xml @@ -159,14 +163,16 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 + with: + persist-credentials: false - uses: actions/setup-node@v7 with: node-version: "24" package-manager-cache: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: "3.11" cache: pip diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ae7d4821..639cb57c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -23,9 +23,11 @@ jobs: matrix: python-version: ["3.11", "3.12"] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 + with: + persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} @@ -149,9 +151,11 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 + with: + persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: "3.11" @@ -218,9 +222,11 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 + with: + persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: "3.10" @@ -237,14 +243,19 @@ jobs: # EXPLICITLY so this job tests the true documented minimum — trl puts # no upper bound on transformers, so an unpinned resolve would silently # pull the newest 5.x and stop exercising the floor. transformers - # 5.3.0 is the CVE-2026-4372 fix floor and requires torch>=2.4 / - # huggingface_hub>=1.3 / tokenizers>=0.22; trl 1.0.0 pulls the - # accelerate>=1.4 and datasets>=4.7 floor. Keep 1:1 with the pyproject - # [project].dependencies minimums. + # 5.10.1 is the first installable release past CVE-2026-4372, + # CVE-2026-5241 and CVE-2026-9856 (5.10.0, which carries the last of + # them, was yanked); it requires huggingface_hub>=1.5 / + # tokenizers>=0.22 and imports torch.float8_e8m0fnu, which first + # exists in torch 2.7.0 (this job failed on torch 2.4.0 with + # AttributeError at `import transformers`); trl 1.0.0 + # pulls the accelerate>=1.4 and datasets>=4.7 floor. Keep 1:1 with the + # pyproject [project].dependencies minimums (tests/test_packaging.py + # compares them). pip install \ - "torch==2.4.0" \ - "transformers==5.3.0" \ - "huggingface_hub==1.3.0" \ + "torch==2.7.0" \ + "transformers==5.10.1" \ + "huggingface_hub==1.5.0" \ "peft==0.19.0" \ "accelerate==1.4.0" \ "datasets==4.7.0" \ @@ -270,7 +281,7 @@ jobs: # the moment a dependency advisory drops, even before the team # re-bumps versions: # - # - pip-audit (transitive CVEs against the OSV / GHSA databases) + # - pip-audit (transitive CVEs against PyPI's vulnerability data, its default service) # - bandit (static-analysis on forgelm/ production code only) # # Severity policy (per tools/check_pip_audit.py + tools/check_bandit.py): @@ -284,9 +295,11 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 + with: + persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: "3.11" cache: pip @@ -335,7 +348,7 @@ jobs: - name: Upload security scan artifacts if: always() - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v7 with: name: supply-chain-scans path: | @@ -352,7 +365,7 @@ jobs: issues: write # required for creating failure notification issues steps: - name: Create issue on failure - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | const title = `Nightly CI failure — ${new Date().toISOString().split('T')[0]}`; diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 070f78a7..28100f11 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,9 +18,11 @@ jobs: permissions: contents: read # checkout reads the tagged ref steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 + with: + persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: "3.11" @@ -35,7 +37,7 @@ jobs: python -m pip install twine twine check dist/* - - uses: actions/upload-artifact@v5 + - uses: actions/upload-artifact@v7 with: name: dist path: dist/ @@ -51,13 +53,15 @@ jobs: permissions: contents: read # checkout reads the tagged ref for tests/ + tools/ steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 + with: + persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python }} - - uses: actions/download-artifact@v5 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: dist path: dist @@ -105,7 +109,7 @@ jobs: shell: bash run: python tools/generate_sbom.py > sbom-${{ matrix.os }}-py${{ matrix.python }}.json - - uses: actions/upload-artifact@v5 + - uses: actions/upload-artifact@v7 with: name: sbom-${{ matrix.os }}-py${{ matrix.python }} path: sbom-${{ matrix.os }}-py${{ matrix.python }}.json @@ -120,7 +124,7 @@ jobs: contents: read # download-artifact reads workflow run state id-token: write # OIDC trusted publishing to PyPI steps: - - uses: actions/download-artifact@v5 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: dist path: dist/ @@ -133,4 +137,4 @@ jobs: # Metadata-Version 2.4 wheels (setuptools ≥77 emits 2.4 for PEP 639 # license expressions) with "Metadata is missing required fields: # Name, Version"; v1.14.0 parses 2.4 correctly. - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 diff --git a/.github/workflows/site-deploy.yml b/.github/workflows/site-deploy.yml index 3c49abd5..d2418d68 100644 --- a/.github/workflows/site-deploy.yml +++ b/.github/workflows/site-deploy.yml @@ -46,9 +46,11 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 + with: + persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: "3.11" @@ -69,10 +71,10 @@ jobs: test -f "site/js/usermanuals/${lang}.js" || { echo "missing ${lang}.js"; exit 1; } done - - uses: actions/configure-pages@v5 + - uses: actions/configure-pages@v6 - name: Upload Pages artifact - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@v5 with: path: site @@ -87,4 +89,4 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} steps: - id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@v5 diff --git a/.github/workflows/usermanuals-validate.yml b/.github/workflows/usermanuals-validate.yml index 2361d69b..3bdf31b1 100644 --- a/.github/workflows/usermanuals-validate.yml +++ b/.github/workflows/usermanuals-validate.yml @@ -34,9 +34,11 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 + with: + persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: "3.11" diff --git a/AGENTS.md b/AGENTS.md index ce249c7d..1b8fd74a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,7 +85,7 @@ ForgeLM/ │ # export, inference, results, utils, __main__, │ # _pypdf_normalise, _script_sanity, │ # _strip_pattern -├── tests/ # 130 test modules; count grows over time (run `ls tests/test_*.py | wc -l` for current) +├── tests/ # 131 test modules; count grows over time (run `ls tests/test_*.py | wc -l` for current) ├── tools/ # CI guards: check_anchor_resolution, │ # check_bilingual_parity, check_cli_help_consistency, │ # check_field_descriptions, check_no_analysis_refs, @@ -386,6 +386,7 @@ Work on ForgeLM is planned and tracked on GitHub, not in local notes: acceptance criteria) or tell the user so it can be filed. The exception is a security vulnerability — never a public issue; it goes to a private advisory as [SECURITY.md](SECURITY.md) describes. +- **Closing is not triage.** Every item is an issue; which issues appear on the front door is decided by labels and views, never by closing them. - **Name what you resolved.** For a grouped issue, list in the pull request the finding IDs your change resolves, so each can be ticked. diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c510f6d..7680d5c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,8 +13,45 @@ All notable changes to ForgeLM are documented here. and valid port; schemeless addresses are rejected instead of silently becoming `` in evidence. Use `https://monitor.example.com:8443/plan` or an `endpoint_env` name (#93). +### Added + +- `forgelm export` now writes `.sha256` next to the GGUF (`sha256sum` format), the file `forgelm verify-gguf` + and the documentation always expected. It had never been written, so a sidecar-based check had nothing to compare. + A sidecar that cannot be written fails the export (exit 2) instead of leaving a GGUF without it; for a K-quant + request it belongs to the intermediate `*.f16.gguf`. A `model_integrity.json` that cannot be updated is now logged + as a warning, not at debug level (#84). +- `CITATION.cff`, so the repository can be cited through GitHub's "Cite this + repository" panel (#964). +- A contributor can claim a `good first issue` by commenting `/assign` and give it + back with `/unassign`: the `Assign on comment` workflow (`tools/assign_on_comment.py`) assigns the commenter + when nobody holds the issue, refuses an issue that already has an assignee without changing it, allows at most 2 + open assignments per contributor (maintainers are exempt) and leaves `security`, `needs-triage`, `deferred`, + epic and Phase 16 issues to a maintainer. Reassigning stays a maintainer's decision. +- `GOVERNANCE.md` and `MAINTAINERS.md` at the repository root, recording how + decisions are made, how the maintainer roles work, and which areas the current + maintainer owns (#964). +- `.github/dependabot.yml`, so dependency updates arrive as reviewed pull requests + rather than waiting for a manual version check; `.github/CODEOWNERS` to route + review requests without gating a merge; and a Documentation issue template, since + the issue chooser previously offered only a bug and a feature form (#964). + ### Changed +- Admit newer majors of five dependencies: `huggingface_hub` 2, `pypdf` 6 and `xxhash` 4 (the `ingestion` extra), `datasketch` 2 (the + `ingestion-scale` extra) and `pytest-randomly` 4 and 5 (the `dev` extra). The ingestion, data-audit and doctor test modules pass on pypdf + 6.21.1, xxhash 4.0.1 and datasketch 2.0.0 with the same results as on the previous majors; CI installs datasketch 2.0.0 and + pytest-randomly 5.0.0 for the full suite. With `huggingface_hub>=1.5.0,<3.0.0` CI resolves `huggingface_hub` 2.2.0 (with transformers + 5.20.1) and the full suite passes on Python 3.10 to 3.13; every `huggingface_hub` symbol ForgeLM imports exists in 2.2.0. Hub 2 uses a + different HTTP stack (`httpx2`), so an environment sees new transitive packages. The tests mock the network, so real Hub calls on 2.x are + not covered, and transformers 5.10 to 5.17 still keep pip on `huggingface_hub` 1.x (5.18 and later allow 2.x). +- Raise the `transformers` floor from `>=5.3.0` to `>=5.10.1` (and `huggingface_hub` from `>=1.3.0` to `>=1.5.0`, which + transformers 5.10 requires). `5.3.0` was described as the CVE fix floor but still admitted CVE-2026-5241 (a LightGlue + `config.json` overrides `trust_remote_code: false`, fixed in 5.5.0) and CVE-2026-9856 (a `chat_template` name in + `tokenizer_config.json` is used unsanitised as a file name by `save_pretrained`, so `final_model/` saves, GGUF export + and merges could write outside the output directory; fixed in 5.10.0, which PyPI yanked, hence 5.10.1). The nightly + minimum-versions job pins the new floor, and `tests/test_packaging.py` keeps it from drifting from + `pyproject.toml` and checks that a traversal `chat_template` name cannot escape on save. Environments pinned to + transformers 5.3 to 5.9 must upgrade (#109). - Record monitoring default provenance in `defaulted_fields` and mark literal URL projection with `endpoint_redacted` (also true when the URL already contains only scheme/host/port). Literal endpoints retain scheme/host/port in JSON evidence, including IPv6 brackets; userinfo, path, @@ -44,9 +81,170 @@ All notable changes to ForgeLM are documented here. - Require history-preserving merge commits for `main` promotions and release synchronization; retain squash as the default for contributor PRs into `development`, and disable rebase-and-merge. Align the contributor, code-review, and release guidance with the repository settings. +- Point first-time contributors at a *Front door* view of the project plan (open, unassigned + `good first issue`s with their size and area) from the README and CONTRIBUTING, and record in + `CONTRIBUTING.md`, `CLAUDE.md` and `AGENTS.md` that the front door is chosen with labels and + views, never by closing issues. + +### Removed + +- Delete the stale `requirements.txt`, `requirements-eval.txt` and `requirements-linux.txt`. Nothing + referenced them, their floors were far below `pyproject.toml`'s, and Dependabot would have opened weekly + pull requests against them; installation goes through the `pyproject.toml` extras (#968). ### Fixed +- Preserve top-level `--dry-run`, `--config`, `--offline` and `--output` values when a subcommand omits the same flag (CR-A05-001, #414). +- Stop describing the human-approval gate as enforcing segregation of duties (English and Turkish): it records the + trainer and approver identities and does not compare them, and both are self-asserted. The ISO 27001 and SOC 2 + mappings, the Statement of Applicability, the change-management SOP, the deployer guide and the user manual now + say what the gate records and that keeping the two people different is the deployer's control. The guide also no + longer says `require_human_approval` is enforced for strict risk tiers: it is not (#73, #99, #90). +- Rewrite the GDPR / KVKK user-manual page (English and Turkish) to what ForgeLM does. It claimed that "by the time + data lands in JSONL, identifiable subjects are removed", that "no specific person's data is recoverable from the model" + and that the audit report "confirms this", that the audit flags memorisation, that a retention setting "auto-deletes" + originals and "records the intended TTL in the audit log", and it showed an `ingestion: pii_mask:` block and an + `international_transfers.enabled` setting that do not exist. Masking now reads as reducing the eight detected identifier + formats (names, addresses and free text are not detected), the audit report as the plain JSON it is (no source + attribution, no hash manifest), and access and erasure requests as `forgelm reverse-pii` and `forgelm purge` (#101). +- Document the one case where a failed safety gate still exits 0 with `auto_revert: true`: an evaluation that produced no + verdict (`safety.evaluation_completed: false` — classifier not loaded, probes file missing or empty, or most verdicts + unreadable) keeps the model and the run exits 0, or 4 with the approval gate. The error-handling standard, the exit-codes and + JSON-output pages, the audit-event catalogue, the safety guide, the deployer guide and the risk-treatment plan said that exit + 0 meant every gate passed, that a classifier load failure "halts" the pipeline and leaves no `final_model/`, and that + classification errors count as unsafe. The behaviour is unchanged; making it fail closed is a breaking exit-code change + left for a major version (#87). +- Say where the per-release SBOMs actually are and what they contain. The supply-chain reference, the ISO 27001 / SOC 2 deployer guide, + the change-management SOP and the supply-chain and deployer user-manual pages told an auditor to run `gh release download … --pattern 'sbom-*'` + and to "re-emit the SBOM from the git tag and diff it against the one attached to the release". Nothing attaches an SBOM to a + GitHub Release: `publish.yml` uploads the twelve files as workflow-run artifacts, which GitHub deletes after 90 days. The docs now give + the working commands (`gh run list --workflow publish.yml --branch v --status success`, then `gh run download`), say that a + release SBOM is the CI runner's `pip list` (test tooling such as `pytest` included, no dependency graph, no licence data) and so is + not reproducible from the tag, and drop "license extraction in SBOM" from the ISO A.5.32, SOC 2 CC9.2 and statement-of-applicability + rows. The nightly `pip-audit` is described as querying PyPI's vulnerability data (its default service, not OSV), and the manual's + "exit 1 means HIGH/CRITICAL" now includes UNKNOWN-severity findings. English and Turkish. Attaching the SBOMs to the release and + generating them from a clean runtime-only environment are code changes and stay open on the issue (#114). +- Stop saying that ForgeLM's audit log is the record-keeping artefact "mandated by EU AI Act Article 12" and that the approval gate + "implements" Article 14. Article 12 asks the deployed high-risk system to log events automatically over its lifetime, and Article 14's + oversight (monitor, interpret, override, stop) happens while the system is in use. `audit_log.jsonl` is development-time evidence of the + training pipeline, and `evaluation.require_human_approval` is a pre-promotion sign-off that supports design-time oversight documentation. + The audit-event catalogue, the audit-log and human-oversight user-manual pages (English and Turkish) now say that (#75). +- Correct site copy that promised more than the code does, in all six languages (`site/js/translations.js`, the HTML fallbacks and a code + comment in `site/js/wizard.js`): a failing benchmark gate exits 3 only with `evaluation.auto_revert: true` (and a safety evaluation with + no verdict keeps the model and exits 0); the approval gate records the approver, it is not "dual-controlled" and does not implement + Article 14's in-use oversight; webhook alerts are Slack-format, Teams and Discord need a relay; PII masking covers common formats, not + names or free text, and `reverse-pii` does not "satisfy" an access request; the wizard's retention "enforce" option is reserved and + deletes nothing; GGUF export writes a SHA-256 sidecar (K-quants need `llama-quantize`) rather than a "manifest"; DeepSpeed preset names + need a source checkout. A new site test keeps those phrases from coming back. English and Turkish are the maintained tiers; the German, + French, Spanish and Chinese strings were written by the assistant, not a native reviewer, and need review (#86, #89, #92, #94, #98, #99, + #101, #110, #112). The `forgelm doctor` sample on the quickstart page now uses the real renderer's format (#534). +- Make `.github/dependabot.yml` parse. It set `versioning-strategy: widen` for `pip`, which that ecosystem does not accept (GitHub's + "Dependabot" check on any pull request touching the file said so), so Dependabot opened no pull requests at all. The setting is removed, the + header comment explains the restriction. +- Stop saying that Microsoft Teams and Discord accept the webhook payload directly. The user manual (English and Turkish) and the + `webhook` field descriptions said all three ingest it; the body is a Slack-style `attachments` block with no top-level `text`, + `content` or `embeds`. Discord needs `content` or `embeds` and answers HTTP 400; Teams retired Office 365 Connectors in May 2026 and + its Workflows webhooks expect an Adaptive Card. Both need a relay. The page now shows the real payload, and the access-control SOP + no longer claims that a payload carries `FORGELM_OPERATOR` (it does not) or lists five events (there are eight). The `url_env` + description said it overrides `url`; it is read only when `url` is empty, so `url` wins. A formatter for Teams and Discord is a + separate, larger change and stays open on the issue (#112). +- Say that the DeepSpeed preset names (`zero2`, `zero3`, `zero3_offload`) work only from a source checkout. They resolve to + `configs/deepspeed/.json` beside the package or in the current directory, and neither the wheel nor the Docker image ships + that folder, so on a `pip install` the names fail with a configuration error (and the unset-`deepspeed_config` default is + `zero2`). The configuration and distributed-training references, the troubleshooting entry (its sample error is now the real + message), the enterprise guide, the user manuals, `config_template.yaml` and the field description now say so and point at a + JSON path; the template's `torchrun … forgelm` launch line is `-m forgelm`, the only form that runs the checkout. The + unverified "`zero3_offload` fits 70B on 8×24 GB" sentence is gone, and the `zero2` preset's own start-up failure is noted + (#410). Shipping the presets in the package, and a test that resolves one from an installed wheel, are code changes and stay + open on the issues (#110, #111, #115). +- Show the real output directory on the first-run, project-layout and human-oversight pages (English and Turkish). They described a + `checkpoints//artifacts/` bundle with a `manifest.json` that hashes "every file inside … for tamper-evidence", a + `config_snapshot.yaml`, a `safety_report.json` and a `data_audit_report.json`. None of those exist: a run writes `audit_log.jsonl` with its + `.manifest.json` genesis pin, a `compliance/` directory (`compliance_report.json`, `training_manifest.yaml`, `data_provenance.json`, + `data_governance_report.json`, and `annex_iv_metadata.json` only when the YAML names a provider, system or purpose), `safety/` and + `benchmark/` only when enabled, and `final_model/` with the weights, model card, `deployer_instructions.md` and `model_integrity.json`. + Only the audit log and `final_model/` are tamper-checked; nothing hashes `compliance/`, and the pages now say so. The invented training log + (a DPO pass in an SFT-only template, `Llama Guard S1-S14: clean`) is replaced by a description of the real stages, the approval-gate + sample shows the two log lines the trainer prints, and the `forgelm doctor` sample uses the real renderer's format (ASCII header and + `[+ pass]` markers, "discoverable" wording, an abridged row list) (#96, #534). +- Describe `--offline` / `model.offline` as what it does. It sets `HF_HUB_OFFLINE`, `TRANSFORMERS_OFFLINE` and `HF_DATASETS_OFFLINE`; it does not + stop webhooks, the LLM-judge and synthetic-data API clients, or W&B / MLflow. The `--offline` help text, the `model.offline` field + description, the configuration and CLI references (English and Turkish) and the enterprise deployment guide said "Air-gapped mode" and + "prevents any network calls"; the air-gap page no longer lists Comet, which `training.report_to` does not accept. Gating those clients + behind the flag is a code change and stays open (#74, #100, #893). +- `training.rope_scaling` works on transformers 5 again. The setting was passed to `from_pretrained(rope_scaling=...)`, + which no longer works: the documented `{type: linear, factor: 4}` raised `KeyError: 'rope_type'` while the model was + built, and `{rope_type: ..., factor: ...}` replaced the whole parameter dict so `rope_theta` was lost (`TypeError`). + The requested keys are now merged over the model's own `rope_parameters` (the legacy key `type` is read as + `rope_type`), applied to the text config of a multimodal checkpoint, and the finished config is loaded. A model that + keeps one entry per layer type (Gemma 3: `full_attention` and `sliding_attention`) gets the scaling on its + `full_attention` entry, and one with per-layer-type entries but no `full_attention` is refused with exit 1 instead of + being scaled nowhere. Verified on transformers 5.13.0 for `linear`, `dynamic` and `yarn` on a tiny local Llama, and + on real Gemma 3 configs (#103). +- `training.sliding_window_attention` reaches the text config and takes effect. The override was passed as a + `from_pretrained(sliding_window=...)` keyword, which transformers 5 forwards to the model constructor once `config=` + is set, and a multimodal model (Qwen2-VL) keeps `sliding_window` on its text config, not the outer one. It now travels in + the same config object as `rope_scaling`. For the Qwen2 family the config also derives a per-layer `layer_types` + list from `use_sliding_window`, `sliding_window` and `max_window_layers`, so setting the number alone left every layer + on full attention; the text config is rebuilt with the flag on so the list is derived again (a pattern the checkpoint + supplied itself is kept), and a warning is logged when the result still has no sliding-attention layer. The unsloth backend builds no transformers config, so it + applied neither `rope_scaling` nor `sliding_window_attention` and trained with the model's own settings; it now + refuses a config that sets either when the config loads (exit 1, `--dry-run` included) instead of ignoring it (#103). +- The `torch` floor is `>=2.7.0`, not `>=2.4.0`. transformers 5.10.1 imports `torch.float8_e8m0fnu` when + `transformers.modeling_utils` loads, and that dtype first exists in torch 2.7.0; on 2.6.0 the import raises + `AttributeError`. transformers lists torch only as an extra, so pip never raised the old floor. The Nightly + Compatibility job's minimum-versions leg caught it (it installs every floor with `--no-deps`); its torch pin, the + installation pages (English and Turkish) and the `forgelm doctor` samples in the guides, references and user manuals + now say 2.7.0. The Intel Mac note stands: there is no `torch>=2.7` wheel for x86_64 macOS. +- The `forgelm` CLI no longer sends Hugging Face's anonymous usage ping. TRL's trainers call `send_telemetry` when they are + built, and `huggingface_hub` sends it unless `HF_HUB_DISABLE_TELEMETRY` is true or offline mode is on, which contradicted the + privacy page ("does not send usage data anywhere"). `main()` now sets `HF_HUB_DISABLE_TELEMETRY=1` before anything imports + `huggingface_hub`, and an operator's own value wins (`HF_HUB_DISABLE_TELEMETRY=0` opts back in). Using ForgeLM as a library + does not set it (#95). +- `retention.enforce` is documented for what it is: reserved. No command reads it, so `warn_on_excess` and + `block_on_excess` never warned or blocked anything and nothing was deleted automatically; retention is only reported, + read-only, by `forgelm purge --check-policy` (always exit 0). The config description, the wizard prompt, the + configuration reference, the GDPR erasure guide and `config_template.yaml` now say so (English and Turkish), and + the CLI logs a warning when a config sets a non-default `enforce` instead of staying silent (#92, #98). +- `distributed.fsdp_strategy: no_shard` and `hybrid_shard` are rejected when the config loads (exit 1). They were documented, passed `--dry-run`, + and then failed with exit 2 at trainer construction (`fsdp=hybrid_shard is only available with FSDP1`), because transformers 5.10+ runs + FSDP2 by default. `fsdp_auto_wrap: false` and `fsdp_backward_prefetch: backward_post` are still accepted but change nothing under FSDP2; + ForgeLM now logs a warning when either is set, and the field descriptions, reference pages, user manuals and `config_template.yaml` + say so. Moving to FSDP2 settings (`fsdp=True` plus an explicit `fsdp_config`) is not part of this change; the string form is + deprecated in transformers and slated for removal in v5.20 (#104). +- `forgelm purge --row-id` without `--output-dir` now logs a warning that names the directory it resolved (the corpus's) + and the `final_model/` the memorisation check will look for. The documentation promised that warning but the code never + logged it, and a default run looks for a model next to the corpus, where a training run rarely writes one, so it finished + with a completed-erasure record and no memorisation event. The audit-event catalogue, the purge reference, the GDPR + erasure guide, the user manual and the risk-treatment plan now state the real condition: a `final_model/` directly under + `--output-dir`, not "any run that consumed the corpus" (English and Turkish). Reading `training.output_dir` from + `--config` is still open (#102). +- Correct the GGUF pages (English and Turkish): the export example shows the real output, K-quants are described as + the two-step operation they are, the invented "manifest" and `verify-gguf` output (vocabulary and tokenizer checks that + do not exist) are replaced by the real ones, a clean `verify-gguf` exit is said to prove only what ran, and the sidecar + is described as corruption detection, not tamper evidence (#83, #84, #85). +- Stop saying that `pip install 'forgelm[export]'` brings the GGUF converter. `forgelm export` runs llama.cpp's `convert_hf_to_gguf.py`; + no `llama-cpp-python` wheel puts it where ForgeLM searches (0.2.90 installs it under `bin/`, 0.3.x does not ship it), so a plain + install ends with `convert_hf_to_gguf.py not found inside the llama-cpp-python installation` (exit 2) unless `FORGELM_GGUF_CONVERTER` + points at the script in a llama.cpp checkout. The installation page, the CLI reference, the usage and quickstart guides, the + post-training notebook, the `export` extra's comment in `pyproject.toml`, the module docstring and the error message (which advised + `pip install 'llama-cpp-python>=0.2.90'`) now say so. Searching the locations real wheels use, or bundling the script, is a code + change and stays open (#113). +- Correct claims in the roadmap and strategy documents: `forgelm chat` does not screen for safety and reads no YAML + config (#91), the live risk rows no longer name a past "August 2026" deadline (#75), and + `docs/product_strategy.md` (English and Turkish) names the real `evaluation.auto_revert` field instead of a + non-existent `auto_revert_on_safety_failure`. +- Correct the `medical-qa-tr` row of the YAML templates reference (English and Turkish): the template enables no safety + evaluation and no `risk_assessment`, so it no longer says it ships with both pre-wired (#97). +- Fix the KTO bullet on the Alignment overview user-manual page (English and Turkish): it named the + fields `(prompt, response, good?)`, but the KTO loader reads `prompt`, `completion` and a boolean + `label` (#965). +- Make the site's code comments readable: `--code-comment` was `#6a6f80`, 3.8:1 on the dark code + surface, below the 4.5:1 AA floor, and it colours code comments, terminal and code-block title bars, + language tags, copy buttons and the wizard's YAML pane header and comment lines. It is now `#8f96a8`, + 6.43:1 on the code surface and at least 5.56:1 on the title-bar gradient, in both themes (the light + theme keeps a dark code surface). Measured on the rendered pages, every text colour on the code + surface reaches 4.5:1 (F-W1-SITE-06, #887). - Use the Regulation's own Article 72 terms for post-market monitoring in the German and Spanish wizard copy ("Beobachtung nach dem Inverkehrbringen", "vigilancia poscomercialización"), and restore the "metadata" the French and Spanish hints had dropped, so every locale again separates ForgeLM's @@ -54,6 +252,10 @@ All notable changes to ForgeLM are documented here. article and gender slips in the touched German, French and Spanish strings, and use one German term for the external stack. The new validation and interval keys keep their English fallback; native review of the deferred locales remains open (refs #948). +- Name the right columns for KTO in the EN/TR "Choosing a Trainer" user manual: its data-shape table + said `{prompt, response, label: bool}`, but the loader requires `{prompt, completion, label}` and + rejects the old shape with a `KeyError` (F-W1-DOCS-TRAIN-02, #885). The table's SFT row is a separate + defect, tracked in #403, and is unchanged here. - Say what Annex IV §9 asks for: a detailed description of the post-market monitoring system, its plan included. The wizard hint in every locale and the EN/TR configuration and compliance pages said it "includes the detailed plan"; German, French and Spanish now follow the Regulation's wording, and diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000..9459c30c --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,37 @@ +cff-version: 1.2.0 +message: "If you use ForgeLM in your work, please cite it using the metadata below." +title: "ForgeLM" +abstract: >- + Config-driven LLM fine-tuning: validated YAML in, a fine-tuned model and an + audit trail out. Covers SFT, DPO, SimPO, KTO, ORPO and GRPO, with optional + dataset auditing, safety evaluation, evaluation gates, human approval and + EU AI Act compliance artifacts. The same configuration runs from a terminal, + a notebook, or a CI/CD pipeline. +type: software +authors: + - name: "ForgeLM Contributors" +repository-code: "https://github.com/HodeTech/ForgeLM" +url: "https://hodetech.github.io/ForgeLM/" +license: Apache-2.0 +keywords: + - llm + - fine-tuning + - lora + - qlora + - peft + - trl + - dpo + - grpo + - reproducibility + - mlops + - eu-ai-act + - ai-governance +# `version` and `date-released` are intentionally omitted for now: the released +# version is single-sourced in `pyproject.toml` (read at runtime by +# `forgelm/_version.py` via `importlib.metadata`, which carries no literal of its +# own) and the site's displayed version is derived from `CHANGELOG.md` by +# `tools/update_site_version.py --check`. This file is not one of that tool's +# targets, so a version here would be an un-guarded copy that no check keeps in +# sync — exactly what `docs/standards/release.md` exists to prevent. Extending the +# tool to rewrite this file (and adding a Zenodo DOI for citable releases) is +# tracked separately. diff --git a/CLAUDE.md b/CLAUDE.md index 518a3ea4..b934017e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -85,7 +85,7 @@ ForgeLM/ │ # export, inference, results, utils, __main__, │ # _pypdf_normalise, _script_sanity, │ # _strip_pattern -├── tests/ # 130 test modules; count grows over time (run `ls tests/test_*.py | wc -l` for current) +├── tests/ # 131 test modules; count grows over time (run `ls tests/test_*.py | wc -l` for current) ├── tools/ # CI guards: check_anchor_resolution, │ # check_bilingual_parity, check_cli_help_consistency, │ # check_field_descriptions, check_no_analysis_refs, @@ -386,6 +386,7 @@ Work on ForgeLM is planned and tracked on GitHub, not in local notes: acceptance criteria) or tell the user so it can be filed. The exception is a security vulnerability — never a public issue; it goes to a private advisory as [SECURITY.md](SECURITY.md) describes. +- **Closing is not triage.** Every item is an issue; which issues appear on the front door is decided by labels and views, never by closing them. - **Name what you resolved.** For a grouped issue, list in the pull request the finding IDs your change resolves, so each can be ticked. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bab77ca5..ba4fa056 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ Thanks for your interest in contributing! ForgeLM is an open-source project and ## Your first pull request 1. Pick an open, unassigned [`good first issue`][good-first-issue]. - Comment to claim it and confirm the problem on `development`; see [Working on an issue](#working-on-an-issue). + Comment `/assign` to claim it and confirm the problem on `development`; see [Working on an issue](#working-on-an-issue). 2. [Fork and clone](#1-fork--clone), then [create a branch from `development`](#3-create-a-branch). Use Python **3.10, 3.11, 3.12 or 3.13** — CI tests all four versions. Create a virtual environment with that interpreter (replace `python3.13` below if you selected another supported version), @@ -20,6 +20,8 @@ Thanks for your interest in contributing! ForgeLM is an open-source project and 3. Make the change and, for a code fix, add a regression test that reproduces the issue; follow the [testing standard](docs/standards/testing.md). Documentation changes update the English page and its Turkish mirror together. + If you do not write Turkish, open the pull request with the English change and say + so in the description — a maintainer will complete the mirror for you. 4. Run fast checks before pushing (replace the example test module with the one you touched): ```bash @@ -66,11 +68,11 @@ Thanks for your interest in contributing! ForgeLM is an open-source project and Work is tracked in two places that play different roles: - **Issues are where the work happens.** Each issue is one unit of work: it describes the problem, links the code, and lists acceptance criteria. Discussion, claiming and pull-request links all happen on the issue. -- **The [ForgeLM Project Plan](https://github.com/orgs/HodeTech/projects/5) is where the work is planned.** It shows every issue with its priority, release target and iteration, in several views: *Backlog* (table), *Board* (by status), *Current iteration* and *Roadmap* (timeline), plus filtered views such as *Small tasks*, *Epics*, *Docs & TR mirror* and *Roadmap — Phase 16 & deferred*. Use it to choose what to work on; you never need to edit it — maintainers keep it current, and a card moves automatically when its issue is closed. +- **The [ForgeLM Project Plan](https://github.com/orgs/HodeTech/projects/5) is where the work is planned.** It shows every issue with its priority, release target and iteration, in several views: *Backlog* (table), *Board* (by status), *Current iteration* and *Roadmap* (timeline), plus filtered views such as *Front door*, *Small tasks*, *Epics*, *Docs & TR mirror* and *Roadmap — Phase 16 & deferred*. Use it to choose what to work on; you never need to edit it — maintainers keep it current, and a card moves automatically when its issue is closed. ### Planned work lives here too -Every piece of planned or deferred work has an issue and a project card, not only the defects: the remaining steps of [Phase 16](docs/roadmap/phase-16-trust-surface-hardening.md) are sub-issues of [#903](https://github.com/HodeTech/ForgeLM/issues/903), and the deferred items recorded in [risks-and-decisions.md](docs/roadmap/risks-and-decisions.md) are sub-issues of [#929](https://github.com/HodeTech/ForgeLM/issues/929). Each Phase 16 step, and each deferred row that has an issue, links that issue directly from the roadmap entry. New issues land in the project's *Intake — unplanned* view until a maintainer plans them. If you notice work that your pull request is not going to do, open an issue for it instead of leaving a `TODO` or a roadmap line. +Every piece of planned or deferred work has an issue and a project card, not only the defects: the remaining steps of [Phase 16](docs/roadmap/phase-16-trust-surface-hardening.md) are sub-issues of [#903](https://github.com/HodeTech/ForgeLM/issues/903), and the deferred items recorded in [risks-and-decisions.md](docs/roadmap/risks-and-decisions.md) are sub-issues of [#929](https://github.com/HodeTech/ForgeLM/issues/929). Each Phase 16 step, and each deferred row that has an issue, links that issue directly from the roadmap entry. New issues land in the project's *Intake — unplanned* view until a maintainer plans them. If you notice work that your pull request is not going to do, open an issue for it instead of leaving a `TODO` or a roadmap line. Every item is an issue; which issues appear on the front door is decided by labels and views, never by closing them. ### How the backlog is organised @@ -97,13 +99,18 @@ In the project, the **Wave**, **Severity**, **Theme**, **Kind**, **Area**, **Siz Useful starting points: +- The project's [*Front door* view](https://github.com/orgs/HodeTech/projects/5/views/16) — open, unassigned `good first issue`s with their size and area, the place to start on a first contribution - [Open, unassigned wave 0–1 issues](https://github.com/HodeTech/ForgeLM/issues?q=is%3Aissue+is%3Aopen+no%3Aassignee+label%3A%22wave%3A+0%22%2C%22wave%3A+1%22) — the highest-priority work - [Documentation issues](https://github.com/HodeTech/ForgeLM/issues?q=is%3Aissue+is%3Aopen+label%3Adocumentation+no%3Aassignee) — often a good first contribution - The project's *Backlog* view, filtered by an **Area** you know or by **Size** `XS`/`S` ### Working on an issue -1. **Claim it first.** Comment on the issue that you would like to work on it, so a maintainer can assign it to you and nobody duplicates the work. Skip issues labelled `needs-triage`, and check the recorded condition of an issue labelled `deferred` before you start. +1. **Claim it first.** Start a comment on the issue with `/assign` and, once GitHub confirms it, the issue is assigned to you, so nobody duplicates the work. If the automation cannot complete, its reply says so and a maintainer assigns the issue by hand. This works on an open issue labelled `good first issue` that nobody holds yet. A contributor can hold at most 2 open issues at a time; the maintainers listed in [MAINTAINERS.md](MAINTAINERS.md) are exempt. Comment `/unassign` to give one back. A few rules keep it predictable: + - It is first come, first served. A later `/assign` never changes an assignment that exists, whoever wrote it and however it was made. Only a maintainer can reassign an issue, in the issue's sidebar, and a maintainer's decision always stands. + - `/assign` and `/unassign` act on the person who writes them, and the command must be the first thing in the comment; they cannot name someone else. If two people claim an issue in the same second, the maintainer's claim wins, and between two contributors one claim is released again and its author is told so. + - Anything the automation does not cover stays a maintainer's call: an issue labelled `help wanted` without `good first issue` (grouped sweeps are taken one finding ID at a time, see below), an issue labelled `needs-triage`, `deferred`, `security`, `epic` or `phase: 16`, and a request for more than 2 open issues. Comment on the issue, say what you would take on and mention a maintainer ([MAINTAINERS.md](MAINTAINERS.md)). A plain comment such as "I would like to work on this" is read by a maintainer too, who assigns it by hand. + - Skip issues labelled `needs-triage`, and check the recorded condition of an issue labelled `deferred` before you start. 2. **Re-check against `development`.** Issues from the review link code at the commit they were found on; the links stay valid, but line numbers may have moved. Confirm the problem still exists on `development` before fixing it. 3. **Follow the acceptance criteria** in the issue. In particular, a code fix carries a regression test that reproduces the described failure, and a documentation fix updates the English page and its Turkish mirror (`-tr.md`, or the `tr/` tree for user manuals) in the same pull request. 4. **Reference the issue in your pull request.** Use `Fixes #123` when the PR resolves the whole issue, and `Refs #123` when it resolves only part of it. @@ -247,7 +254,7 @@ ForgeLM is a single-package layout: a mix of single-file modules and five focused sub-packages (`forgelm/cli/` post-Phase-15 split, `forgelm/data_audit/` post-Phase-14 split, `forgelm/wizard/` from Phase 22, `forgelm/safety/` from the post-v0.9.1 split and `forgelm/verify/` from -Phase 16 S1) under `forgelm/`, 130 test files +Phase 16 S1) under `forgelm/`, 131 test files under `tests/` (collected-test count grows over time — run `pytest --collect-only -q` for current), plus `configs/`, `docs/`, `tools/` (CI guards), and `notebooks/`. For the authoritative module-by-module map diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 00000000..30d303b1 --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,123 @@ +# ForgeLM Governance + +> **How decisions are made in ForgeLM, who makes them, and how you can take on +> more responsibility over time.** For how to contribute, see +> [CONTRIBUTING.md](CONTRIBUTING.md); for behaviour expectations, see +> [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md). + +## Current model + +ForgeLM is **maintainer-led**. The project has a small group of maintainers who +are accountable for the code, the standards and the releases — today, one +(see [MAINTAINERS.md](MAINTAINERS.md)). There is no foundation and no steering +committee; the goal of this document is not to describe a large organisation but +to make the path into one explicit. + +The project's product principles — reliability before features, enterprise +differentiation over feature parity, every capability config-driven and testable +— are recorded in [`docs/roadmap.md`](docs/roadmap.md) and are not restated here. +How decisions get made rests on two rules: + +1. **The standards are the rulebook.** `docs/standards/` holds the engineering + standards, `docs/roadmap/` holds what ships next, and the + [CHANGELOG.md](CHANGELOG.md) records what shipped. A decision that changes + behaviour is written into one of them; a decision that exists only in a chat + is not a decision. +2. **Public reasoning.** Non-trivial proposals, breaking changes and anything + that changes a documented contract are discussed in an **issue** before a + pull request is opened, so the reasoning is reviewable and linkable. + +## Roles + +| Role | What they do | How it is granted | +|---|---|---| +| **Contributor** | Reports bugs, opens pull requests, reviews others' changes, improves docs | Anyone; no appointment | +| **Maintainer** | Sets direction, merges to `development`, cuts releases, holds the standards | A maintainer invites them (see below) | + +Everyone contributing is bound by the +[Code of Conduct](CODE_OF_CONDUCT.md) and asked to follow the +[code-review standard](docs/standards/code-review.md). + +## How decisions are made + +- **Routine changes** — bug fixes, documentation and mirror updates, dependency + bumps — are decided in the pull request. A pull request merges once the + required status checks pass, every review conversation is resolved, and a + maintainer has reviewed it. The exact gate is a branch-protection setting, not + this document. While the project has one maintainer, that maintainer merges + their own pull requests under the same conditions, which is what + [docs/standards/code-review.md](docs/standards/code-review.md) records. +- **Non-trivial changes** — a new config field, a new module, a new optional + dependency, a change to an exit code, the audit-event vocabulary or a + documented guarantee — start as an issue. Work that has no issue may be asked + to open one first, so the design is agreed before code is written. If the + change is larger than one issue, it becomes a roadmap phase under + [docs/roadmap/](docs/roadmap/). +- **Breaking changes** to the YAML schema or the CLI contract require a + deprecation period. Removing a YAML field is a MAJOR change; the release a + removal lands in is named by a single constant, + `DEPRECATION_REMOVAL_VERSION` in `forgelm/config.py`, and the + [release standard](docs/standards/release.md) governs the sequence. +- **Disagreement** is resolved by discussion on the issue. If a consensus does + not form, the maintainer decides and records the reasoning in the issue and, + when it sets a lasting constraint, in + [docs/roadmap/risks-and-decisions.md](docs/roadmap/risks-and-decisions.md). +- **Security reports** do not follow this path. They go through a private + advisory as [SECURITY.md](SECURITY.md) describes. + +## Becoming a maintainer + +There is no exam and no vote — and no promise. A maintainer role is offered when +the project needs a second maintainer and someone has earned it, not +automatically after a number of merged pull requests. The invitation is normally +made when all of the following hold: + +- they have landed a handful of pull requests across more than one area; +- they have reviewed other people's pull requests with useful, specific + feedback, and respected the standards while doing so; +- they have shown judgement on the edge cases — an error path, a documented + contract, a supply-chain or compliance question; +- they are willing to be reachable, and to triage issues and review pull + requests on a predictable cadence rather than only when they feel like it. + +Area ownership follows the same idea: `MAINTAINERS.md` names who owns which area, +and `.github/CODEOWNERS` routes reviews there. The list is kept accurate rather +than flattering, and a maintainer who needs to step back can say so; that is not +held against them. + +## Access and continuity + +GitHub permissions are granted per role, and the smallest permission that does +the job is the default: + +| Role | Repository permission | Why | +|---|---|---| +| Contributor | None — fork and pull request | Every contribution arrives as a pull request, so no repository access is needed | +| Maintainer | *Write* | Can merge to `development`, push branches and manage issues | +| Owner | *Admin* | Also carries branch protection, the tag ruleset and the release workflow's environment approval | + +*Admin* is not the next rung of the contribution ladder — it is the ability to +change the rules the other roles work under, so it stays with the project owner +and moves only on a deliberate decision, with two-factor authentication required +on any account that holds it. + +**Continuity: a known gap, recorded rather than promised.** There is one owner +today, and nothing in the release path is asynchronous — the same single account +holds the tag ruleset and approves the release workflow's `pypi` environment. An +unplanned absence therefore stops merges, releases and security advisories at +once. The fix is a second organisation owner (or an organisation-level owner with +the release environment's approval); it is not scheduled, and this paragraph is +what says so. + +## Releases + +Releases are cut from `main` by a maintainer, following +[docs/standards/release.md](docs/standards/release.md). `development` is the +integration branch and the default branch; changes land there first and reach +`main` through a release pull request. Only maintainers merge to `main` and tag +releases. + +## Changing this document + +This document is a normal file in the repository: propose a change with a pull +request, and explain the reasoning in the pull request description. diff --git a/MAINTAINERS.md b/MAINTAINERS.md new file mode 100644 index 00000000..e9d6171d --- /dev/null +++ b/MAINTAINERS.md @@ -0,0 +1,47 @@ +# ForgeLM Maintainers + +> **Who maintains ForgeLM, what each of them owns, and how to reach them.** +> See [GOVERNANCE.md](GOVERNANCE.md) for how the roles work and how to take one +> on, and [CONTRIBUTING.md](CONTRIBUTING.md) for how to contribute. + +## Current maintainers + +| GitHub | Areas | Reachable for | +|---|---|---| +| [@cemililik](https://github.com/cemililik) | All areas — the repository's `area:` label set (cli, compliance, config, data-audit, dev-tooling, eval-gates, gdpr-approval, ingestion, integrations, model-data, onboarding, operator-cmds, packaging-ci, pipeline, post-training, site, trainer, verify) | Triage, design questions, reviews, releases | + +The project has a single maintainer today. When a co-maintainer takes +responsibility for an area, they are added here and to +[`.github/CODEOWNERS`](.github/CODEOWNERS), which routes reviews for that area to +them. The "Areas" column mirrors the repository's `area:` labels, so an issue +labelled `area: site` and this table name the same area. + +## How to reach a maintainer + +- **A bug or a feature request** — open an + [issue](https://github.com/HodeTech/ForgeLM/issues/new/choose). +- **A question** — start a + [discussion](https://github.com/HodeTech/ForgeLM/discussions); questions are not + filed on the public issue tracker. Asking in public means the answer is + searchable for the next person. +- **A security vulnerability** — never in public. Use the repository's + **Security** tab, as [SECURITY.md](SECURITY.md) describes. +- **Anything covered by the Code of Conduct** — follow the enforcement section of + the [Code of Conduct](CODE_OF_CONDUCT.md#enforcement). + +## Review cadence + +Issues are triaged and pull requests are reviewed on a best-effort basis, with an +initial-reply target of **72 hours** — the same window [SECURITY.md](SECURITY.md) +commits to for a vulnerability report, so the project has one number rather than +two. An unanswered first pull request is the most expensive thing a young project +can ship, so a **first-time contributor**'s pull request is answered first. If a +pull request has not been reviewed for a while, a polite nudge on it is welcome; +it is not rude. + +## Contributors + +This file names maintainers and area owners only; it is not the contributor list. +Everyone whose work is merged appears in the repository's commit history and on +its GitHub contributors page, and non-maintainer contributions are credited in +the release notes. diff --git a/MANIFEST.in b/MANIFEST.in index f1409d9a..1280670a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include LICENSE include README.md include CHANGELOG.md +include CITATION.cff include config_template.yaml recursive-include configs *.json *.jsonl diff --git a/README.md b/README.md index 19e1b35d..e19d295a 100644 --- a/README.md +++ b/README.md @@ -180,8 +180,9 @@ Help make fine-tuning easier to reproduce and review. You can improve a guide, translate an example, reproduce a bug, add a regression test, or work on a planned feature. Start small; you do not need a GPU to run the unit tests or documentation checks. -1. Find an [unassigned good first issue](https://github.com/HodeTech/ForgeLM/issues?q=is%3Aissue%20is%3Aopen%20label%3A%22good%20first%20issue%22%20no%3Aassignee) - or browse the [project plan](https://github.com/orgs/HodeTech/projects/5). +1. Start at the [Front door](https://github.com/orgs/HodeTech/projects/5/views/16): the open, unassigned `good first issue`s + with their size and area. You can also [search by label](https://github.com/HodeTech/ForgeLM/issues?q=is%3Aissue%20is%3Aopen%20label%3A%22good%20first%20issue%22%20no%3Aassignee). + The rest of the planned work is in the [project plan](https://github.com/orgs/HodeTech/projects/5). 2. Follow [your first pull request](https://github.com/HodeTech/ForgeLM/blob/development/CONTRIBUTING.md#your-first-pull-request) to set up the environment and run the relevant checks. 3. Open a PR against **`development`** and link its issue. Keep English/Turkish @@ -192,6 +193,11 @@ feature. Start small; you do not need a GPU to run the unit tests or documentati [Engineering standards](https://github.com/HodeTech/ForgeLM/tree/development/docs/standards). Report vulnerabilities privately using [SECURITY.md](https://github.com/HodeTech/ForgeLM/blob/main/SECURITY.md). +Who decides what, and who owns which area: [GOVERNANCE.md](https://github.com/HodeTech/ForgeLM/blob/development/GOVERNANCE.md) · +[MAINTAINERS.md](https://github.com/HodeTech/ForgeLM/blob/development/MAINTAINERS.md). +To cite ForgeLM, use GitHub's **Cite this repository** button or +[`CITATION.cff`](https://github.com/HodeTech/ForgeLM/blob/development/CITATION.cff). + ## License **Code:** [Apache 2.0](https://github.com/HodeTech/ForgeLM/blob/main/LICENSE). diff --git a/config_template.yaml b/config_template.yaml index 8aec1398..415cc2c9 100644 --- a/config_template.yaml +++ b/config_template.yaml @@ -162,12 +162,12 @@ data: # dare_seed: 42 # DARE: RNG seed for the random drop mask # Distributed training (multi-GPU). Requires: pip install forgelm[distributed] -# Launch with: torchrun --nproc_per_node=N forgelm --config my_config.yaml +# Launch with: torchrun --nproc_per_node=N -m forgelm --config my_config.yaml # distributed: # strategy: "deepspeed" # "deepspeed" or "fsdp" -# deepspeed_config: "zero2" # preset: "zero2", "zero3", "zero3_offload" or path to JSON +# deepspeed_config: "ds.json" # path to a DeepSpeed JSON (the presets "zero2", "zero3", "zero3_offload" need a source checkout) # # --- FSDP options (when strategy: "fsdp") --- -# # fsdp_strategy: "full_shard" # "full_shard", "shard_grad_op", "no_shard", "hybrid_shard" +# # fsdp_strategy: "full_shard" # "full_shard" or "shard_grad_op" (FSDP2 supports no other) # # fsdp_auto_wrap: true # # fsdp_offload: false @@ -276,7 +276,7 @@ data: # staging_ttl_days: 7 # final_model.staging./ TTL after `forgelm reject` # ephemeral_artefact_retention_days: 90 # compliance bundles + audit reports # raw_documents_retention_days: 90 # ingested raw docs (PDF/DOCX/EPUB/TXT/MD) -# enforce: "log_only" # "log_only" | "warn_on_excess" | "block_on_excess" +# enforce: "log_only" # reserved: no command acts on it yet ("log_only" | "warn_on_excess" | "block_on_excess") # Phase 14 — Multi-stage training pipeline chains (SFT → DPO → GRPO). # Uncomment the ``pipeline:`` block below to run multiple training stages diff --git a/docs/guides/air_gap_deployment-tr.md b/docs/guides/air_gap_deployment-tr.md index 18c5a29b..a583f3f8 100644 --- a/docs/guides/air_gap_deployment-tr.md +++ b/docs/guides/air_gap_deployment-tr.md @@ -106,8 +106,8 @@ forgelm doctor - environment check [+ pass] forgelm.install ForgeLM 0.9.0 running from /opt/airgap-venv/lib/python3.11/site-packages/forgelm (inside site-packages). [+ pass] python.version Python 3.11.4 (CPython). - [+ pass] torch.cuda torch 2.4.0 with CUDA 12.4. - [+ pass] numpy.torch_abi torch 2.4.0 + numpy 1.26.4 are ABI-compatible. + [+ pass] torch.cuda torch 2.7.0 with CUDA 12.6. + [+ pass] numpy.torch_abi torch 2.7.0 + numpy 1.26.4 are ABI-compatible. [+ pass] gpu.inventory 1 GPU(s) — GPU0: NVIDIA A100 (80.0 GiB). [+ pass] extras.qlora Discoverable, not import-tested (module bitsandbytes, purpose: 4-bit / 8-bit QLoRA training). [+ pass] extras.eval Discoverable, not import-tested (module lm_eval, purpose: lm-evaluation-harness benchmark scoring). diff --git a/docs/guides/air_gap_deployment.md b/docs/guides/air_gap_deployment.md index 30caa487..a25de50c 100644 --- a/docs/guides/air_gap_deployment.md +++ b/docs/guides/air_gap_deployment.md @@ -106,8 +106,8 @@ forgelm doctor - environment check [+ pass] forgelm.install ForgeLM 0.9.0 running from /opt/airgap-venv/lib/python3.11/site-packages/forgelm (inside site-packages). [+ pass] python.version Python 3.11.4 (CPython). - [+ pass] torch.cuda torch 2.4.0 with CUDA 12.4. - [+ pass] numpy.torch_abi torch 2.4.0 + numpy 1.26.4 are ABI-compatible. + [+ pass] torch.cuda torch 2.7.0 with CUDA 12.6. + [+ pass] numpy.torch_abi torch 2.7.0 + numpy 1.26.4 are ABI-compatible. [+ pass] gpu.inventory 1 GPU(s) — GPU0: NVIDIA A100 (80.0 GiB). [+ pass] extras.qlora Discoverable, not import-tested (module bitsandbytes, purpose: 4-bit / 8-bit QLoRA training). [+ pass] extras.eval Discoverable, not import-tested (module lm_eval, purpose: lm-evaluation-harness benchmark scoring). diff --git a/docs/guides/enterprise_deployment.md b/docs/guides/enterprise_deployment.md index f9c70e60..67231902 100644 --- a/docs/guides/enterprise_deployment.md +++ b/docs/guides/enterprise_deployment.md @@ -121,7 +121,7 @@ ForgeLM sets `HF_HUB_OFFLINE=1`, `TRANSFORMERS_OFFLINE=1`, `HF_DATASETS_OFFLINE= ```yaml distributed: strategy: "deepspeed" - deepspeed_config: "zero2" # or "zero3", "zero3_offload" + deepspeed_config: "/etc/forgelm/ds_zero2.json" # your own DeepSpeed JSON; the zero2/zero3/zero3_offload preset names need a source checkout ``` ```bash @@ -152,7 +152,7 @@ See the [Distributed Training Guide](../reference/distributed_training.md) for d | `trust_remote_code` | `false` | Prevents arbitrary code execution from model repos | | `auth.hf_token` | Use `HUGGINGFACE_TOKEN` env var | Don't hardcode tokens in YAML | | `webhook.url` | Always use `url_env` with environment variable | Webhook tokens excluded from model cards (v0.3.1rc1+), but avoid direct URLs for credential hygiene | -| `offline` | `true` for air-gapped | Prevents any network calls | +| `offline` | `true` for air-gapped | Blocks Hugging Face Hub, transformers and datasets network calls (it sets `HF_HUB_OFFLINE`, `TRANSFORMERS_OFFLINE`, `HF_DATASETS_OFFLINE`). It does **not** stop webhooks, judge or synthetic-data API clients, or W&B / MLflow: disable those (omit `webhook:`, use local judge and teacher models, set `report_to: none`) for a true air gap | --- diff --git a/docs/guides/gdpr_erasure-tr.md b/docs/guides/gdpr_erasure-tr.md index 64a1bb16..7fa432c1 100644 --- a/docs/guides/gdpr_erasure-tr.md +++ b/docs/guides/gdpr_erasure-tr.md @@ -19,7 +19,7 @@ **Yapmaz:** -- Modelleri yeniden eğitmez. Bir satırı corpus'tan kaldırmak onu zaten eğitilmiş ağırlıklardan unutturmaz — silinmiş satır olmadan tam yeniden eğitim tek doğru çözüm. Tool, corpus'u tüketen herhangi bir koşum için `final_model/` mevcut olduğunda `data.erasure_warning_memorisation` yayar; böylece operatör boşluğu görür. +- Modelleri yeniden eğitmez. Bir satırı corpus'tan kaldırmak onu zaten eğitilmiş ağırlıklardan unutturmaz — silinmiş satır olmadan tam yeniden eğitim tek doğru çözüm. Tool, `--output-dir` altında doğrudan bir `final_model/` dizini mevcut olduğunda `data.erasure_warning_memorisation` yayar (`--output-dir` varsayılan olarak corpus'un dizinidir; bu yüzden eğitim çıktı dizininizi verin); böylece operatör boşluğu görür. Corpus'u hangi koşumların tükettiğini ayırt edemez. - Audit log'un kendisinden entry silmez. Madde 17(3)(b) audit / muhasebe kayıtlarını bir hukuki yükümlülük savunması olarak korur; tool silmeyi geçmişi yeniden yazmak yerine yeni bir event olarak kaydeder. - Downstream tüketicilere silme bildirimi göndermez. Webhook receiver'ları, dataset mirror'ları, deploy edilmiş model endpoint'leri ve yayınlanmış checkpoint'ten türetilmiş üçüncü-taraf fine-tune'lar operatörün runtime-katmanı sorumluluğudur (Madde 17(2)). Tool yüklü config webhook block'u içerdiğinde `data.erasure_warning_external_copies` yayar; böylece audit log'u sorgulayan downstream tüketici açık hatırlatmayı görür. - Yedekleri silmez. Operatörün `` sınırı dışındaki replikalar, snapshot'lar ve storage backup'lar altyapı tarafı endişelerdir. @@ -78,10 +78,10 @@ retention: staging_ttl_days: 7 # `forgelm reject` üzerine aksiyon almak için bir iş haftası ephemeral_artefact_retention_days: 90 # üç-aylık review cadence'i raw_documents_retention_days: 90 # data audit yeniden çalıştırmadan önce ingestion penceresi - enforce: log_only # log_only / warn_on_excess / block_on_excess + enforce: log_only # ayrılmış: henüz hiçbir komut onu kullanmaz (log_only / warn_on_excess / block_on_excess) ``` -Herhangi bir horizon'u `0` olarak ayarlamak o artefact kind'ı için policy'yi devre dışı bırakır (sınırsız tut). `enforce`, trainer pre-flight gate'inin ihlallere nasıl tepki vereceğini kontrol eder. +Herhangi bir horizon'u `0` olarak ayarlamak o artefact kind'ı için policy'yi devre dışı bırakır (sınırsız tut). Horizon'lar `forgelm purge --check-policy` ile raporlanır (salt-okunur, her zaman 0 ile çıkar); hiçbir şey otomatik silinmez ya da engellenmez. `enforce` ayrılmıştır: henüz hiçbir komut onu kullanmaz, bu yüzden üç değer de `log_only` gibi davranır. ## Kaldırıldı: `evaluation.staging_ttl_days` @@ -96,7 +96,7 @@ Herhangi bir horizon'u `0` olarak ayarlamak o artefact kind'ı için policy'yi d | `data.erasure_requested` | Herhangi bir `forgelm purge --row-id` / `--run-id` çağrısının ilk adımı, herhangi bir silmeden önce (`--check-policy` salt-okunur; audit event yaymaz) | `target_kind` ∈ `{row, staging, artefacts}`, `target_id` (row mode'da hash'lenmiş), `salt_source` (row mode), `corpus_path` (row), `output_dir` (run), `justification`, `dry_run` | | `data.erasure_completed` | Başarılı silme bittiğinde | Tüm `requested` field'ları + `bytes_freed`, `files_modified`, `pre_erasure_line_number` (row mode), `match_count` (row mode) | | `data.erasure_failed` | Disk operasyonu raise etti VEYA eşleşen satır/koşum bulunamadı VEYA çoklu-satır policy belirsizliği reddetti | Tüm `requested` field'ları + `error_class`, `error_message` | -| `data.erasure_warning_memorisation` | **Yalnız row modunda.** Row erasure × corpus'u tüketen herhangi bir koşum için `final_model/` mevcut. (Run modu memorisation uyarısı emit etmez — run-scoped `staging` / `artefacts` türleri tanım gereği promote-öncesi.) | Tüm `completed` field'ları + `affected_run_ids` | +| `data.erasure_warning_memorisation` | **Yalnız row modunda.** Row erasure × `--output-dir` altında doğrudan bir `final_model/` dizini mevcut. (Run modu memorisation uyarısı emit etmez — run-scoped `staging` / `artefacts` türleri tanım gereği promote-öncesi.) | Tüm `completed` field'ları + `affected_run_ids` | | `data.erasure_warning_synthetic_data_present` | **Yalnız row modunda.** Row erasure × `output_dir`'de `synthetic_data*.jsonl` mevcut. (Run modu bu uyarıyı yüzeylemez — synthetic veri corpus-scoped'dur, run-scoped değil.) | Tüm `completed` field'ları + `synthetic_files` | | `data.erasure_warning_external_copies` | Yüklü config webhook block'u içeriyor | Tüm `completed` field'ları + `webhook_targets` | diff --git a/docs/guides/gdpr_erasure.md b/docs/guides/gdpr_erasure.md index 24941879..fd4f7e21 100644 --- a/docs/guides/gdpr_erasure.md +++ b/docs/guides/gdpr_erasure.md @@ -19,7 +19,7 @@ **Does not:** -- Re-train models. Removing a row from the corpus does not unmemorise it from already-trained weights — full retraining without the erased row is the only proper mitigation. The tool emits `data.erasure_warning_memorisation` when a `final_model/` exists for any run that consumed the corpus, so the operator sees the gap. +- Re-train models. Removing a row from the corpus does not unmemorise it from already-trained weights — full retraining without the erased row is the only proper mitigation. The tool emits `data.erasure_warning_memorisation` when a `final_model/` directory exists directly under `--output-dir` (which defaults to the corpus's directory, so pass your training output directory), so the operator sees the gap. It cannot tell which runs consumed the corpus. - Delete entries from the audit log itself. Article 17(3)(b) preserves audit / accounting records as a legal-obligation defence; the tool records the erasure as a new event rather than rewriting history. - Push erasure notices to downstream consumers. Webhook receivers, dataset mirrors, deployed model endpoints, and third-party fine-tunes derived from a published checkpoint are the operator's runtime-layer responsibility (Article 17(2)). The tool emits `data.erasure_warning_external_copies` when the loaded config has a webhook block, so a downstream consumer querying the audit log sees the explicit reminder. - Erase backups. Replicas, snapshots, and storage backups outside the operator's `` are infrastructure-side concerns. @@ -78,10 +78,10 @@ retention: staging_ttl_days: 7 # one work-week to act on a `forgelm reject` ephemeral_artefact_retention_days: 90 # quarterly review cadence raw_documents_retention_days: 90 # ingestion-window before re-running data audit - enforce: log_only # log_only / warn_on_excess / block_on_excess + enforce: log_only # reserved: no command acts on it yet (log_only / warn_on_excess / block_on_excess) ``` -Setting any horizon to `0` disables the policy for that artefact kind (retain indefinitely). `enforce` controls how the trainer pre-flight gate reacts to violations. +Setting any horizon to `0` disables the policy for that artefact kind (retain indefinitely). The horizons are reported by `forgelm purge --check-policy` (read-only, always exits 0); nothing is deleted or blocked automatically. `enforce` is reserved: no command acts on it yet, so all three values behave like `log_only`. ## Removed: `evaluation.staging_ttl_days` @@ -96,7 +96,7 @@ Six new events ship with `forgelm purge` (catalogued in `docs/reference/audit_ev | `data.erasure_requested` | First step of any `forgelm purge --row-id` / `--run-id` invocation, before any deletion (`--check-policy` is read-only and emits no audit events) | `target_kind` ∈ `{row, staging, artefacts}`, `target_id` (hashed in row mode), `salt_source` (row mode), `corpus_path` (row), `output_dir` (run), `justification`, `dry_run` | | `data.erasure_completed` | Successful deletion finishes | All `requested` fields + `bytes_freed`, `files_modified`, `pre_erasure_line_number` (row mode), `match_count` (row mode) | | `data.erasure_failed` | Disk operation raised, OR no matching row/run found, OR multi-row policy refused on ambiguity | All `requested` fields + `error_class`, `error_message` | -| `data.erasure_warning_memorisation` | **Row mode only.** Row erasure × `final_model/` exists for any run that consumed the corpus. (Run mode does not emit memorisation warnings — the run-scoped `staging` / `artefacts` kinds are by definition pre-promotion.) | All `completed` fields + `affected_run_ids` | +| `data.erasure_warning_memorisation` | **Row mode only.** Row erasure × a `final_model/` directory exists directly under `--output-dir`. (Run mode does not emit memorisation warnings — the run-scoped `staging` / `artefacts` kinds are by definition pre-promotion.) | All `completed` fields + `affected_run_ids` | | `data.erasure_warning_synthetic_data_present` | **Row mode only.** Row erasure × `synthetic_data*.jsonl` exists in `output_dir`. (Run mode does not surface this warning — synthetic data is corpus-scoped, not run-scoped.) | All `completed` fields + `synthetic_files` | | `data.erasure_warning_external_copies` | Loaded config has a webhook block (both modes). | All `completed` fields + `webhook_targets` | diff --git a/docs/guides/getting-started-tr.md b/docs/guides/getting-started-tr.md index 7d5add86..ac9be8e7 100644 --- a/docs/guides/getting-started-tr.md +++ b/docs/guides/getting-started-tr.md @@ -46,8 +46,8 @@ forgelm doctor - environment check [+ pass] forgelm.install ForgeLM 0.9.0 running from /home/me/.venv/lib/python3.11/site-packages/forgelm (inside site-packages). [+ pass] python.version Python 3.11.4 (CPython). - [+ pass] torch.cuda torch 2.4.0 with CUDA 12.4. - [+ pass] numpy.torch_abi torch 2.4.0 + numpy 1.26.4 are ABI-compatible. + [+ pass] torch.cuda torch 2.7.0 with CUDA 12.6. + [+ pass] numpy.torch_abi torch 2.7.0 + numpy 1.26.4 are ABI-compatible. [+ pass] gpu.inventory 1 GPU(s) — GPU0: NVIDIA RTX 4090 (24.0 GiB). [+ pass] extras.qlora Discoverable, not import-tested (module bitsandbytes, purpose: 4-bit / 8-bit QLoRA training). [! warn] extras.tracking Optional extra missing — install with: pip install 'forgelm[tracking]' (purpose: Weights & Biases experiment tracking). diff --git a/docs/guides/getting-started.md b/docs/guides/getting-started.md index e1f899a0..86313c27 100644 --- a/docs/guides/getting-started.md +++ b/docs/guides/getting-started.md @@ -46,8 +46,8 @@ forgelm doctor - environment check [+ pass] forgelm.install ForgeLM 0.9.0 running from /home/me/.venv/lib/python3.11/site-packages/forgelm (inside site-packages). [+ pass] python.version Python 3.11.4 (CPython). - [+ pass] torch.cuda torch 2.4.0 with CUDA 12.4. - [+ pass] numpy.torch_abi torch 2.4.0 + numpy 1.26.4 are ABI-compatible. + [+ pass] torch.cuda torch 2.7.0 with CUDA 12.6. + [+ pass] numpy.torch_abi torch 2.7.0 + numpy 1.26.4 are ABI-compatible. [+ pass] gpu.inventory 1 GPU(s) — GPU0: NVIDIA RTX 4090 (24.0 GiB). [+ pass] extras.qlora Discoverable, not import-tested (module bitsandbytes, purpose: 4-bit / 8-bit QLoRA training). [! warn] extras.tracking Optional extra missing — install with: pip install 'forgelm[tracking]' (purpose: Weights & Biases experiment tracking). diff --git a/docs/guides/iso_soc2_deployer_guide-tr.md b/docs/guides/iso_soc2_deployer_guide-tr.md index fd52e63e..196d0689 100644 --- a/docs/guides/iso_soc2_deployer_guide-tr.md +++ b/docs/guides/iso_soc2_deployer_guide-tr.md @@ -26,16 +26,20 @@ ForgeLM kanıtına sahip: 2. **Change control** — Madde 14 staging gate (`forgelm approve` / `reject`) + `human_approval.required/granted/rejected` audit olayları + koşum başına damgalanan `config_hash` (manifest sidecar - alanı). Her model promotion çift kontrollü ve forensic olarak - attribute edilmiştir. + alanı). Onay + kapısı etkinleştirildiğinde her promotion, audit log'a kaydedilen onaylayıcı kimliğine atfedilir. ForgeLM + eğitici ve onaylayıcı kimliklerini kaydeder ama karşılaştırmaz: bunların farklı kişiler olmasını kimlik + sağlayıcınızda uygularsınız. 3. **Data lineage** — `data_provenance.json` (SHA-256 fingerprint + size + mtime + `hf_revision_source` ile derecelendirilmiş dataset HF Hub commit SHA'sı); `data_governance_report.json` (collection_method, annotation_process, known_biases, personal_data_included, dpia_completed). 4. **Supply chain** — yayın etiketi başına yayın matrisinin her (OS - × Python-version) hücresi için emit edilen CycloneDX 1.5 SBOM. - Wave 4 statik + dinamik güvenlik taraması için `pip-audit` + × Python-version) hücresi için emit edilen CycloneDX 1.5 SBOM; + 90 gün boyunca workflow-run artifact olarak saklanır (GitHub + release'e eklenmez; CI runner'ının `pip list`'ini, test araçları + dahil, lisans verisi olmadan listeler). Wave 4 statik + dinamik güvenlik taraması için `pip-audit` nightly + `bandit` CI ekler. ## Denetim gözlem dönemi öncesi setup checklist'i @@ -65,7 +69,9 @@ Denetçi sormadan **önce** indirmeniz gereken on iki madde. - [ ] **`evaluation.require_human_approval: true`** her `risk_classification` ∈ `{high-risk, unacceptable}` koşumu - için. Unutursanız F-compliance-110 `ConfigError` raise eder. + için. ForgeLM bunu zorlamaz: F-compliance-110 yalnızca bu katmanlarda `evaluation.safety.enabled` + kapalıysa `ConfigError` raise eder; dolayısıyla onay kapısı olmayan, elle yazılmış bir strict-katman + YAML'ı kabul edilir. CLI sihirbazı (`forgelm --wizard`) bu katmanlarda kapıyı açar. - [ ] **Webhook lifecycle yapılandır** (`url_env`, `secret_env`, `notify_on_*`) böylece SIEM her durum geçişini içe alır. - [ ] **`auto_revert: true` etkinleştir** üretim eğitimi için — @@ -155,8 +161,9 @@ jq -r 'select(.run_id == "X" and (.event == "training.started" or .event == "hum ./outputs/audit_log.jsonl | sort -u ``` -İki farklı operator ID görev ayrılığını kanıtlar (ISO A.5.3, SOC 2 -CC1.5). +`training.started` üzerindeki operatörü `human_approval.granted` üzerindeki onaylayıcıyla karşılaştırın. İki kimlik de +kendi beyanıdır (`FORGELM_OPERATOR`, yoksa OS kullanıcısı); dolayısıyla görev ayrılığı (ISO A.5.3, SOC 2 CC1.5) ancak +kimlik sağlayıcınız bunları farklı kişiler yapıyorsa geçerlidir. ### S3: "Data lineage göster" @@ -198,19 +205,25 @@ eğittiğinin kanıtı saymayın. ### S4: "Supply chain göster" ```bash -# SBOM artefaktını GitHub release sayfasından indir. -gh release download v0.5.5 --pattern 'sbom-*' - -# Veya istek üzerine yeniden üret. +# Tag'in publish çalıştırmasından bir SBOM indir. Bunlar workflow-run +# artifact'larıdır, 90 günde silinir; GitHub release asset'i değildir. +gh run list --repo HodeTech/ForgeLM --workflow publish.yml --branch v0.11.0 --status success \ + --json databaseId,createdAt +gh run download --repo HodeTech/ForgeLM --name sbom-ubuntu-latest-py3.11 + +# Veya kendi ortamınızı tanımlayın: modeli eğiten ya da sunan ortamda, +# bir ForgeLM checkout'undan çalıştırın. python3 tools/generate_sbom.py > sbom.json # Son yayınla diff al. diff <(jq -S . sbom-prev.json) <(jq -S . sbom.json) ``` -CycloneDX 1.5 JSON her transitive bağımlılığı purl -(`pkg:pypi/...`) ve sürümle listeler. Dependency-Track CVE -korelasyonu için bunu doğal olarak içe alır. +CycloneDX 1.5 JSON kurulu her paketi purl (`pkg:pypi/...`) ve +sürümle listeler; bağımlılık grafiği ve lisans verisi yoktur. +Release SBOM'ları CI runner'ını tanımlar, bu yüzden `pytest` gibi +test araçlarını da adlandırır. Dependency-Track dosyayı CVE +korelasyonu için doğal olarak içe alır. ### S5: "Erişim kontrolleri göster — yalnız yetkili reviewer'ların model onayladığını nasıl ispatlıyorsunuz?" @@ -249,8 +262,11 @@ Referans: - `docs/qms/sop_incident_response.md` §4 (Wave 4 / Faz 23 genişletme) güvenlik-incident playbook için. -- Zincirin kendisi: `audit.classifier_load_failed` event yangın + - `pipeline.failed` propagate + koşum `final_model/` dizini ÜRETMEZ. +- Zincirin kendisi: `audit.classifier_load_failed` event'i tetiklenir + ve `safety.evaluation_completed` `evaluation_completed: false` kaydeder. + Koşu **durdurulmaz**: model `final_model/` içinde kalır ve koşu 0 ile + çıkar (onay kapısıyla 4); bu yüzden exit koduna değil, bu event'lere ya da + JSON zarfının `safety` bloğuna göre alarm kurun. - F-compliance-110 strict gate: `risk_classification ∈ {high-risk, unacceptable}` AND `evaluation.safety.enabled = false` ise config validator koşum başlamadan ÖNCE `ConfigError` raise eder. Denetçi diff --git a/docs/guides/iso_soc2_deployer_guide.md b/docs/guides/iso_soc2_deployer_guide.md index 3132ef96..9d020375 100644 --- a/docs/guides/iso_soc2_deployer_guide.md +++ b/docs/guides/iso_soc2_deployer_guide.md @@ -25,15 +25,20 @@ direct ForgeLM evidence: `forgelm verify-audit` validates the chain end-to-end. 2. **Change control** — Article 14 staging gate (`forgelm approve` / `reject`) + `human_approval.required/granted/rejected` audit - events + `config_hash` (per-run manifest sidecar field) stamped per run. Every - model promotion is dual-controlled and forensically attributed. + events + `config_hash` (per-run manifest sidecar field) stamped per run. When + the approval gate is enabled, every promotion is attributed to the approver identity recorded in the audit + log. ForgeLM records the trainer and approver identities but does not compare them: keeping them different + people is a control you enforce at your identity provider. 3. **Data lineage** — `data_provenance.json` (SHA-256 fingerprint + size + mtime + the dataset's HF Hub commit SHA, graded by `hf_revision_source`); `data_governance_report.json` (collection_method, annotation_process, known_biases, personal_data_included, dpia_completed). 4. **Supply chain** — CycloneDX 1.5 SBOM emitted per release tag for - every (OS × Python-version) cell of the publish matrix. Wave 4 + every (OS × Python-version) cell of the publish matrix, kept as a + workflow-run artifact for 90 days (not attached to the GitHub + release; it lists the CI runner's `pip list`, test tooling + included, with no licence data). Wave 4 adds `pip-audit` nightly + `bandit` CI for static + dynamic security scanning. @@ -63,7 +68,9 @@ Twelve items to land **before** the auditor asks. - [ ] **`evaluation.require_human_approval: true`** for every `risk_classification` ∈ `{high-risk, unacceptable}` run. - F-compliance-110 raises `ConfigError` if you forget. + ForgeLM does not enforce this: F-compliance-110 only raises `ConfigError` when + `evaluation.safety.enabled` is off for these tiers, so a hand-written strict-tier YAML without the + approval gate is accepted. The CLI wizard (`forgelm --wizard`) switches the gate on for these tiers. - [ ] **Configure webhook lifecycle** (`url_env`, `secret_env`, `notify_on_*`) so SIEM ingests every state transition. - [ ] **Enable `auto_revert: true`** for production training so @@ -155,8 +162,9 @@ jq -r 'select(.run_id == "X" and (.event == "training.started" or .event == "hum ./outputs/audit_log.jsonl | sort -u ``` -Two distinct operator IDs prove segregation of duties (ISO A.5.3, -SOC 2 CC1.5). +Compare the operator on `training.started` with the approver on `human_approval.granted`. Both identities are +self-asserted (`FORGELM_OPERATOR`, else the OS user), so segregation of duties (ISO A.5.3, SOC 2 CC1.5) holds only +if your identity provider makes them different people. ### Q3: "Show me the data lineage" @@ -198,19 +206,25 @@ corpus trained the model. ### Q4: "Show me the supply chain" ```bash -# Download the SBOM artefact from the GitHub release page. -gh release download v0.5.5 --pattern 'sbom-*' - -# Or regenerate on demand. +# Download one SBOM from the tag's publish run. They are workflow-run +# artifacts that expire after 90 days, not GitHub release assets. +gh run list --repo HodeTech/ForgeLM --workflow publish.yml --branch v0.11.0 --status success \ + --json databaseId,createdAt +gh run download --repo HodeTech/ForgeLM --name sbom-ubuntu-latest-py3.11 + +# Or describe your own environment: run this from a ForgeLM checkout in the +# environment that trains or serves the model. python3 tools/generate_sbom.py > sbom.json # Diff against last release. diff <(jq -S . sbom-prev.json) <(jq -S . sbom.json) ``` -The CycloneDX 1.5 JSON lists every transitive dependency with its -purl (`pkg:pypi/...`) and version. Dependency-Track ingests this -natively for CVE correlation. +The CycloneDX 1.5 JSON lists every installed package with its purl +(`pkg:pypi/...`) and version; it has no dependency graph and no licence +data. The release SBOMs describe the CI runner, so they also name test +tooling such as `pytest`. Dependency-Track ingests the file natively for +CVE correlation. ### Q5: "Show me the access controls — how do you prove only authorised reviewers approve models?" @@ -248,9 +262,11 @@ Reference: - `docs/qms/sop_incident_response.md` §4 (Wave 4 / Faz 23 expansion) for the security-incident playbook. -- The audit chain itself: `audit.classifier_load_failed` event fires - + `pipeline.failed` propagates + the run does NOT produce a - `final_model/` directory. +- The audit chain itself: the `audit.classifier_load_failed` event fires + and `safety.evaluation_completed` records `evaluation_completed: false`. + The run is **not** halted: the model stays in `final_model/` and the + run exits 0 (4 with the approval gate), so alert on those events or on + the JSON envelope's `safety` block, not on the exit code. - F-compliance-110 strict gate: if `risk_classification ∈ {high-risk, unacceptable}` AND `evaluation.safety.enabled = false`, the config validator raises `ConfigError` BEFORE the run starts. diff --git a/docs/guides/quickstart-tr.md b/docs/guides/quickstart-tr.md index 43b6ab2a..e239e28c 100644 --- a/docs/guides/quickstart-tr.md +++ b/docs/guides/quickstart-tr.md @@ -204,7 +204,8 @@ Eğitilmiş modelinizle doğrudan etkileşim kurun ve deploy edin: forgelm chat ./checkpoints/final_model # GGUF'a export (Ollama, LM Studio, llama.cpp için) -# Gerekli: pip install forgelm[export] +# Gerekli: pip install forgelm[export] VE FORGELM_GGUF_CONVERTER'ın bir llama.cpp checkout'undaki +# convert_hf_to_gguf.py'ye işaret etmesi (hiçbir llama-cpp-python wheel'i onu ForgeLM'in baktığı yere koymaz) forgelm export ./checkpoints/final_model --output model.gguf --quant q4_k_m # Deployment config'leri üret (sunucu başlatılmaz) diff --git a/docs/guides/quickstart.md b/docs/guides/quickstart.md index d91cb21d..f95362fe 100644 --- a/docs/guides/quickstart.md +++ b/docs/guides/quickstart.md @@ -191,7 +191,8 @@ Interact with and deploy your trained model directly: forgelm chat ./checkpoints/final_model # Export to GGUF (for Ollama, LM Studio, llama.cpp) -# Requires: pip install forgelm[export] +# Requires: pip install forgelm[export] AND FORGELM_GGUF_CONVERTER pointing at convert_hf_to_gguf.py +# from a llama.cpp checkout (no llama-cpp-python wheel installs it where ForgeLM looks) forgelm export ./checkpoints/final_model --output model.gguf --quant q4_k_m # Generate deployment configs (no server is started) diff --git a/docs/guides/safety_compliance-tr.md b/docs/guides/safety_compliance-tr.md index 547edf2f..a4ece7f1 100644 --- a/docs/guides/safety_compliance-tr.md +++ b/docs/guides/safety_compliance-tr.md @@ -95,8 +95,7 @@ Kapı 2: Confidence-weighted → safety_score < min_safety_score? Kapı 3: Şiddet → critical_count > 0? high_count > eşik? ``` -Üç kapı bağımsızdır. Herhangi bir kapının başarısızlığı (etkinse) -auto-revert tetikler. +Üç kapı bağımsızdır. Bir kapının başarısızlığı (etkinse) auto-revert tetikler; güvenlik değerlendirmesi hiç karar üretemediyse bu geçerli değildir (bkz. [Fail-safe davranış](#fail-safe-davranış)). ### Zarar kategorileri (Llama Guard 3) @@ -204,13 +203,12 @@ iyileştirip bozmadığını izlemesine yardım eder. ### Fail-safe davranış -- Sınıflandırma hataları **güvensiz** olarak değerlendirilir (fail-safe - prensibi) +- Sınıflandırıcının yanıtlayamadığı bir çift **skorlanmamış** sayılır, güvenli değil - Düşük-güvenli sınıflandırmalar manuel inceleme için işaretlenir (uyarı log'u) -- Güvenlik değerlendirmesi başarısız olursa ve `auto_revert: true` ise - model otomatik olarak silinir -- Hat entegrasyonu için `3` exit kodu döner +- Skorlanmış bir güvenlik değerlendirmesi başarısız olursa ve `auto_revert: true` ise + model otomatik olarak silinir ve hat entegrasyonu için `3` exit kodu döner +- **Değerlendirme karar üretemezse** (`safety.evaluation_completed: false` — sınıflandırıcı yüklenemedi, probe dosyası yok ya da boş, ya da çoğu karar okunamadı), model `auto_revert: true` olsa bile **korunur** ve koşu `0` ile çıkar (onay kapısıyla `4`); çünkü okunmamış bir karar zarar kanıtı değildir. Yalnızca exit koduna değil, JSON çıktısındaki `safety.evaluation_completed` ve `safety.passed`'e bakın --- @@ -522,6 +520,8 @@ Herhangi bir adım başarısız olursa ve `auto_revert: true` ise: Model silinir + Webhook bildirimi (başarısızlık) + Exit kodu 3 ``` +Karar üretemeyen bir güvenlik değerlendirmesi (`evaluation_completed: false`) istisnadır: model korunur ve koşu 0 ile çıkar. + ### İnsan onay kapısı (Madde 14) Tüm otomatik kontroller geçtikten sonra hattı duraklatmak için diff --git a/docs/guides/safety_compliance.md b/docs/guides/safety_compliance.md index 5569369c..6f4f585a 100644 --- a/docs/guides/safety_compliance.md +++ b/docs/guides/safety_compliance.md @@ -77,7 +77,7 @@ Gate 2: Confidence-weighted → safety_score < min_safety_score? Gate 3: Severity → critical_count > 0? high_count > threshold? ``` -All three gates are independent. Any gate failure triggers auto-revert (if enabled). +All three gates are independent. A gate failure triggers auto-revert (if enabled), unless the safety evaluation produced no verdict at all (see [Fail-Safe Behavior](#fail-safe-behavior)). ### Harm Categories (Llama Guard 3) @@ -179,10 +179,10 @@ This helps teams track whether data/config changes are improving or degrading sa ### Fail-Safe Behavior -- Classification errors are treated as **unsafe** (fail-safe principle) +- A pair the classifier cannot answer is counted as **unscored**, not as safe - Low-confidence classifications are flagged for manual review (warning log) -- If safety evaluation fails and `auto_revert: true`, the model is automatically deleted -- Exit code `3` is returned for pipeline integration +- If a scored safety evaluation fails and `auto_revert: true`, the model is automatically deleted and exit code `3` is returned for pipeline integration +- **If the evaluation produces no verdict** (`safety.evaluation_completed: false` — the classifier failed to load, the probes file is missing or empty, or most verdicts were unreadable), the model is **kept** and the run exits `0` (`4` with the approval gate) even with `auto_revert: true`, because an unread verdict is not evidence of harm. Branch on `safety.evaluation_completed` and `safety.passed` in the JSON output, not on the exit code alone --- @@ -461,12 +461,14 @@ Model saved + Model card + Compliance artifacts Webhook notification (success) ``` -If any step fails and `auto_revert: true`: +If a gate fails with a verdict and `auto_revert: true`: ``` ↓ fail Model deleted + Webhook notification (failure) + Exit code 3 ``` +A safety evaluation that produced no verdict (`evaluation_completed: false`) is the exception: the model is kept and the run exits 0. + ### Human Approval Gate (Art. 14) Add `require_human_approval: true` to pause the pipeline after all automated checks pass: diff --git a/docs/guides/troubleshooting-tr.md b/docs/guides/troubleshooting-tr.md index c4cb2408..29e6331c 100644 --- a/docs/guides/troubleshooting-tr.md +++ b/docs/guides/troubleshooting-tr.md @@ -363,11 +363,12 @@ nvidia-smi ### DeepSpeed Config bulunamadı ```text -FileNotFoundError: DeepSpeed preset 'zero2' not found +DeepSpeed preset 'zero2' not found at /configs/deepspeed/zero2.json ``` -ForgeLM proje kökünden çalıştırdığınızdan emin olun ya da mutlak yol -kullanın: +Ön ayar adları yalnızca bir kaynak checkout'unda çözülür (paketin yanındaki ya da geçerli dizindeki bir +`configs/deepspeed/` dizini); `pip install` veya Docker imajı bunları içermez. ForgeLM proje kökünden çalıştırın ya da +bir DeepSpeed JSON dosyasının yolunu kullanın: ```yaml distributed: diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index 7533eedb..9549875b 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -344,10 +344,12 @@ nvidia-smi ### DeepSpeed Config Not Found ```text -FileNotFoundError: DeepSpeed preset 'zero2' not found +DeepSpeed preset 'zero2' not found at /configs/deepspeed/zero2.json ``` -Ensure you're running from the ForgeLM project root, or use an absolute path: +The preset names resolve only from a source checkout (a `configs/deepspeed/` directory beside the package or in the +current directory); a `pip install` or the Docker image does not contain them. Run from the ForgeLM project root, or +use the path of a DeepSpeed JSON file: ```yaml distributed: diff --git a/docs/product_strategy-tr.md b/docs/product_strategy-tr.md index 97d465cd..d9010672 100644 --- a/docs/product_strategy-tr.md +++ b/docs/product_strategy-tr.md @@ -21,7 +21,7 @@ ForgeLM, LLM ince ayarını (fine-tuning) **güvenli, denetlenebilir ve tekrarla - **3 katmanlı güvenlik kapısı**: ikili geçer/kalır + güven ağırlıklı skor + 14 zarar kategorisinde (S1-S14) önem analizi - **Otomatik geri alma (auto-revert)**: güvenlik regresyonu tespit edilirse, çıktılar yazılmadan önce eğitim sonucu otomatik olarak reddedilir - **Çalıştırmalar arası trend izleme**: güvenlik skorları her çalıştırmada loglanır; zamanla oluşan kayan bozulma yakalanır -- **`auto_revert_on_safety_failure`** config alanı doğrudan YAML'a yazılır — CI'da güvenlik kapısı uygulamak için özel script gerekmez +- **`evaluation.auto_revert`** config alanı doğrudan YAML'a yazılır — bir güvenlik, benchmark ya da judge kapısı başarısız olduğunda çalışma eğitilmiş çıktıları silip 3 koduyla çıkabilir; CI'da kapıları uygulamak için özel script gerekmez ### 2. Uyumluluk Yerleşik (Compliance-Native) diff --git a/docs/product_strategy.md b/docs/product_strategy.md index 06ef3f89..12105967 100644 --- a/docs/product_strategy.md +++ b/docs/product_strategy.md @@ -21,7 +21,7 @@ Fine-tuning can degrade a model's safety properties — silently. ForgeLM treats - **3-layer safety gate**: binary pass/fail + confidence-weighted score + per-category severity analysis covering 14 harm categories (S1-S14) - **Auto-revert**: if safety regression is detected against baseline, training output is automatically rejected before artifacts are written - **Cross-run trend tracking**: safety scores are logged per-run and trended over time to catch gradual drift -- **`auto_revert_on_safety_failure`** is a first-class config field — teams can enforce safety gates in CI without any custom scripting +- **`evaluation.auto_revert`** is a first-class config field — when a safety, benchmark or judge gate fails, the run can delete the trained artefacts and exit with code 3, so teams can enforce the gates in CI without any custom scripting ### 2. Compliance-Native diff --git a/docs/qms/access_control-tr.md b/docs/qms/access_control-tr.md index 13341c3a..67f3ea60 100644 --- a/docs/qms/access_control-tr.md +++ b/docs/qms/access_control-tr.md @@ -230,20 +230,26 @@ webhook: ForgeLM webhook gövdelerini HMAC ile **imzalamaz** — `WebhookConfig` üzerinde `webhook.secret_env` alanı yoktur (bkz. -`forgelm/config.py::WebhookConfig`). Hedef-tarafı atıf (a) HTTPS + -`webhook.url_env` üzerinden URL-gizliliğine, (b) curated payload -içinde taşınan `FORGELM_OPERATOR` kimliğine, ve (c) alıcı sistemin -kendi bearer-token / imzalı-istek kontrollerine (Slack signing -secret, Teams connector token) düşer. - -Webhook wire-format olay sözlüğü (5 olay): `training.start`, -`training.success`, `training.failure`, `training.reverted`, -`approval.required` — Phase 8. In-process notifier metot adları -`notify_*` öneki kullanır (`notify_start`, `notify_success`, -`notify_failure`, `notify_reverted`, `notify_awaiting_approval`) -ve bu beş wire olayına dispatch eder. Her payload `FORGELM_OPERATOR` -kimliğini taşır böylece alıcı sistem (Slack, Teams, custom -incident-management) bildirimi attribute edebilir. +`forgelm/config.py::WebhookConfig`). Hedef-tarafı özgünlük (a) HTTPS + +`webhook.url_env` üzerinden URL-gizliliğine ve (b) alıcı tarafın kendi +uyguladığı kimlik doğrulamaya düşer. Bir Slack incoming webhook'unun gizli +URL'sinden başka bir doğrulaması yoktur (Slack'in signing secret'ı Slack'ten +*gelen* istekleri doğrular, bir webhook URL'sine gelenleri değil); bu yüzden +özel bir relay kendi kimlik doğrulamasını istemelidir, çünkü ForgeLM imza +göndermez. Webhook payload'u `FORGELM_OPERATOR` +kimliğini **taşımaz**; bir modeli kimin çalıştırdığı veya onayladığı +bildirimden değil, audit log'dan okunur. + +Webhook wire-format olay sözlüğü sekiz olaydır: beş tek-aşamalı +yaşam-döngüsü olayı `training.start`, `training.success`, +`training.failure`, `training.reverted` ve `approval.required`, ayrıca +çok-aşamalı bir koşunun bunlarla birlikte yaydığı üç `pipeline.*` +olayı (`pipeline.started`, `pipeline.completed`, +`pipeline.stage_reverted`). In-process notifier metotları `notify_*` +öneki kullanır (`notify_start`, `notify_success`, `notify_failure`, +`notify_reverted`, `notify_awaiting_approval`, +`notify_pipeline_started`, `notify_pipeline_completed`, +`notify_pipeline_reverted`) ve bu wire olaylarına dispatch eder. ## 8. Doğrulama checklist'i diff --git a/docs/qms/access_control.md b/docs/qms/access_control.md index e3cb1aed..9e5f344a 100644 --- a/docs/qms/access_control.md +++ b/docs/qms/access_control.md @@ -228,21 +228,26 @@ webhook: ForgeLM does **not** HMAC-sign webhook bodies — there is no `webhook.secret_env` field on `WebhookConfig` (see -`forgelm/config.py::WebhookConfig`). Destination-side attribution falls to (a) -HTTPS + URL-secrecy via `webhook.url_env`, (b) the -`FORGELM_OPERATOR` identity carried inside the curated payload, and -(c) the receiving system's own bearer-token / signed-request -controls (Slack signing secret, Teams connector token). - -The webhook wire-format event vocabulary (5 events) is -`training.start`, `training.success`, `training.failure`, -`training.reverted`, and `approval.required` — Phase 8. The -in-process notifier method names use the `notify_*` prefix -(`notify_start`, `notify_success`, `notify_failure`, -`notify_reverted`, `notify_awaiting_approval`) and dispatch to those -five wire events. Every payload carries the `FORGELM_OPERATOR` -identity so the receiving system (Slack, Teams, custom -incident-management) can attribute the notification. +`forgelm/config.py::WebhookConfig`). Destination-side authenticity falls to +(a) HTTPS + URL-secrecy via `webhook.url_env` and (b) authentication +the receiving side applies itself. A Slack incoming webhook has none +beyond its secret URL (Slack's signing secret verifies requests *from* +Slack, not requests to a webhook URL), so a custom relay should require +its own authentication, because ForgeLM sends no signature. A webhook +payload does **not** carry the +`FORGELM_OPERATOR` identity; attribution of who ran or approved a model +comes from the audit log, not from the notification. + +The webhook wire-format event vocabulary has eight events: the five +single-stage lifecycle events `training.start`, `training.success`, +`training.failure`, `training.reverted` and `approval.required`, plus the +three `pipeline.*` events (`pipeline.started`, `pipeline.completed`, +`pipeline.stage_reverted`) that a multi-stage run emits alongside them. The +in-process notifier methods use the `notify_*` prefix (`notify_start`, +`notify_success`, `notify_failure`, `notify_reverted`, +`notify_awaiting_approval`, `notify_pipeline_started`, +`notify_pipeline_completed`, `notify_pipeline_reverted`) and dispatch to +those wire events. ## 8. Verification checklist diff --git a/docs/qms/risk_treatment_plan-tr.md b/docs/qms/risk_treatment_plan-tr.md index 1ac0e752..073d0c25 100644 --- a/docs/qms/risk_treatment_plan-tr.md +++ b/docs/qms/risk_treatment_plan-tr.md @@ -90,7 +90,7 @@ ekler. |---|---| | Açıklama | Operatör `forgelm purge --row-id alice@example.com` corpusa karşı çalıştırır, ama haftalar önce satırı eğiten model öğrenilen ağırlıklardan onu hala üretebilir | | L × I (inherent) | High × Med = HIGH | -| Treatment | ForgeLM, silinen satırı tüketen koşum bir `final_model/` artefaktına sahipse `data.erasure_warning_memorisation` event'ı yayar — bu bir **detection** sinyalidir, mitigation değildir. Operatör residual riski kabul eder ve organizasyonel kontrolleri uygular (özne bildirimi, yüksek-stakes dağıtımlar için sıfırdan yeniden eğit, DPIA güncellemesi). | +| Treatment | ForgeLM, purge'ün `--output-dir`'i altında doğrudan bir `final_model/` varsa `data.erasure_warning_memorisation` event'ı yayar (satırı hangi koşumun tükettiğini bilmez ve varsayılan dizin eğitim çıktısı değil corpus'unkidir) — bu bir **detection** sinyalidir, mitigation değildir. Operatör residual riski kabul eder ve organizasyonel kontrolleri uygular (özne bildirimi, yüksek-stakes dağıtımlar için sıfırdan yeniden eğit, DPIA güncellemesi). | | Residual L × I | High × Med = HIGH (risk **kabul edildi**, bkz. §5) | | Sahip | Data Protection Officer (DPO) | | Review cadence | Madde 17 talebi başına | @@ -103,7 +103,7 @@ ekler. |---|---| | Açıklama | Llama Guard / yapılandırılmış safety classifier yüklenemiyor (HF Hub down, OOM, version mismatch); ForgeLM gate yangın etmeden devam eder | | L × I (inherent) | Med × High = HIGH | -| Treatment | F-compliance-110 strict gate: `risk_classification` ∈ `{high-risk, unacceptable}` + `evaluation.safety.enabled: false` `ConfigError` raise eder; `audit.classifier_load_failed` event arızayı kaydeder; operatör pipeline'ı non-zero exit'te durur | +| Treatment | F-compliance-110 strict gate: `risk_classification` ∈ `{high-risk, unacceptable}` + `evaluation.safety.enabled: false` `ConfigError` raise eder; `audit.classifier_load_failed` event arızayı kaydeder. Koşu **durdurulmaz**: karar üretemeyen bir güvenlik değerlendirmesi modeli korur ve 0 ile çıkar (onay kapısıyla 4), bu yüzden operatör pipeline'ı yalnızca exit koduna değil, JSON zarfındaki `safety.evaluation_completed` / `safety.passed`'e ya da o audit event'ine bakmalıdır | | Residual L × I | Low × Low = LOW | | Sahip | ML Lead | | Review cadence | Eğitim koşumu başına | diff --git a/docs/qms/risk_treatment_plan.md b/docs/qms/risk_treatment_plan.md index c19a9d35..5b5ef08b 100644 --- a/docs/qms/risk_treatment_plan.md +++ b/docs/qms/risk_treatment_plan.md @@ -89,7 +89,7 @@ section §4. |---|---| | Description | Operator runs `forgelm purge --row-id alice@example.com` against the corpus, but the model trained on the row weeks earlier may still reproduce it from learned weights | | L × I (inherent) | High × Med = HIGH | -| Treatment | ForgeLM emits `data.erasure_warning_memorisation` when the run that consumed the deleted row has a `final_model/` artefact — this is a **detection** signal, not a mitigation. Deployer accepts the residual risk and applies organisational controls (subject notification, retrain from scratch for high-stakes deployments, DPIA update). | +| Treatment | ForgeLM emits `data.erasure_warning_memorisation` when a `final_model/` exists directly under the purge's `--output-dir` (it does not know which run consumed the row, and the default directory is the corpus's, not the training output) — this is a **detection** signal, not a mitigation. Deployer accepts the residual risk and applies organisational controls (subject notification, retrain from scratch for high-stakes deployments, DPIA update). | | Residual L × I | High × Med = HIGH (risk **accepted**, see §5) | | Owner | Data Protection Officer (DPO) | | Review cadence | Per Article 17 request | @@ -102,7 +102,7 @@ section §4. |---|---| | Description | Llama Guard / configured safety classifier fails to load (HF Hub down, OOM, version mismatch); ForgeLM proceeds without the gate fired | | L × I (inherent) | Med × High = HIGH | -| Treatment | F-compliance-110 strict gate: `risk_classification` ∈ `{high-risk, unacceptable}` + `evaluation.safety.enabled: false` raises `ConfigError`; `audit.classifier_load_failed` event records the failure; deployer pipeline halts on non-zero exit | +| Treatment | F-compliance-110 strict gate: `risk_classification` ∈ `{high-risk, unacceptable}` + `evaluation.safety.enabled: false` raises `ConfigError`; `audit.classifier_load_failed` event records the failure. The run is **not** halted: a safety evaluation with no verdict keeps the model and exits 0 (4 with the approval gate), so the deployer pipeline must branch on the JSON envelope's `safety.evaluation_completed` / `safety.passed` or on that audit event, not on the exit code alone | | Residual L × I | Low × Low = LOW | | Owner | ML Lead | | Review cadence | Per training run | diff --git a/docs/qms/sop_change_management-tr.md b/docs/qms/sop_change_management-tr.md index c010fc36..8a3d1a4b 100644 --- a/docs/qms/sop_change_management-tr.md +++ b/docs/qms/sop_change_management-tr.md @@ -109,8 +109,9 @@ CAB'sıdır: retention süresi dolana kadar (`retention.staging_ttl_days` — config'te). -Bu, operatöre SOC 2 CC8.1 kanıtı verir: her model promotion -attribute, çift kontrollü ve forensic olarak kayıtlıdır. +Bu, operatöre SOC 2 CC8.1 kanıtı verir: onay kapısı etkinleştirildiğinde her +model promotion kaydedilen onaylayıcı kimliğine atfedilir ve forensic olarak kayıtlıdır. Eğitici ile onaylayıcıyı +ayırmak dağıtımı yapanın kontrolüdür: ForgeLM iki kimliği karşılaştırmaz. ### 4.2 Yapılandırma drift tespiti @@ -134,9 +135,12 @@ olarak atfeder. 1.5 SBOM üretir. Wave 4 bir determinism contract testi ekler (`tests/test_supply_chain_security.py::TestGenerateSbomDeterministic`). -Drift tespiti: bir release'in SBOM'u `git tag`'inden tekrarlanabilir. -Bir denetçi bunu istek üzerine yeniden emit edip GitHub release'e -ekli artefakta diff alabilir. +Drift tespiti: SBOM, release'in build edildiği ortamın kaydıdır (matris runner'ında +`pip list`); tag'in `publish.yml` çalıştırmasının workflow-run artifact'ı olarak 90 gün +saklanır, GitHub release'e eklenmez. Bağımlılıklar build anında PyPI'dan çözüldüğü için +yalnızca `git tag`'den tekrarlanamaz. İki release'in SBOM'unu `diff` ile karşılaştırın; +dağıtılmış bir ortamı kanıtlamak için orada `python3 tools/generate_sbom.py` ile yeni +bir tane üretin. ## 5. Versiyon İzleme diff --git a/docs/qms/sop_change_management.md b/docs/qms/sop_change_management.md index 79336c3d..cffc7820 100644 --- a/docs/qms/sop_change_management.md +++ b/docs/qms/sop_change_management.md @@ -104,8 +104,9 @@ in-pipeline CAB: in `final_model.staging./` for forensic review until retention expires (`retention.staging_ttl_days` in the config). -This gives a deployer SOC 2 CC8.1 evidence: every model promotion is -attributed, dual-controlled, and forensically recorded. +This gives a deployer SOC 2 CC8.1 evidence: when the approval gate is +enabled, every model promotion is attributed to the recorded approver identity and forensically recorded. +Separating trainer and approver is the deployer's control: ForgeLM does not compare the two identities. ### 4.2 Configuration drift detection @@ -129,9 +130,12 @@ CycloneDX 1.5 SBOM per (OS × Python-version) on every release tag (`publish.yml`). Wave 4 adds a determinism contract test (`tests/test_supply_chain_security.py::TestGenerateSbomDeterministic`). -Drift detection: a release's SBOM is reproducible from its `git tag`. -An auditor can re-emit it on demand and diff against the artefact -attached to the GitHub release. +Drift detection: the SBOM is a record of the environment the release was built in +(`pip list` on the matrix runner), kept as a workflow-run artifact of the tag's +`publish.yml` run for 90 days; it is not attached to the GitHub release. It is not +reproducible from the `git tag` alone, because dependencies resolve from PyPI at build +time. Compare two releases' SBOMs with `diff`; to evidence a deployed environment, emit +a fresh one there with `python3 tools/generate_sbom.py`. ## 5. Version Tracking diff --git a/docs/qms/statement_of_applicability-tr.md b/docs/qms/statement_of_applicability-tr.md index ac90f3c5..3f93b03f 100644 --- a/docs/qms/statement_of_applicability-tr.md +++ b/docs/qms/statement_of_applicability-tr.md @@ -59,7 +59,7 @@ operatör-tarafı eylem. | A.5.29 Aksaklık sırasında bilgi güvenliği | YES | `auto_revert` baseline-flip; `model.reverted` event | Base-model retention dokümante et | | A.5.30 İş sürekliliği için ICT hazırlığı | YES | DR planlama | Benimse | | A.5.31 Yasal, kanuni, düzenleyici ve sözleşmesel gerekliliklerin tanımlanması | YES | EU AI Act + GDPR mapping'ler; Annex IV bundle | Kural değişikliklerini izle | -| A.5.32 Fikri mülkiyet hakları | YES | SBOM'da lisans çıkarımı; HF model-card metadata | Per-model lisans review | +| A.5.32 Fikri mülkiyet hakları | YES | HF model-card metadata (SBOM lisans verisi taşımaz) | Per-model lisans review | | A.5.33 Kayıtların korunması | YES | Append-only + HMAC + manifest sidecar; off-site replica | Off-site backup | | A.5.34 Gizlilik ve PII korunması | YES | `forgelm reverse-pii` Md. 15; `forgelm purge` Md. 17; `forgelm audit` PII tespiti + maskeleme | DSAR workflow + DPIA | | A.5.35 Bilgi güvenliğinin bağımsız incelenmesi | YES | Yıllık external denetim | Benimse | @@ -108,7 +108,7 @@ ForgeLM-spesifik kontrol envanteriyle ilgilidir. | Kontrol | Applicable? | ForgeLM kanıtı | Operatör-tarafı eylem | |---|---|---|---| | A.8.1 Kullanıcı uç cihazları | YES | `forgelm doctor` — Python / CUDA / GPU / extras / HF auth / disk / `FORGELM_OPERATOR` checks | Endpoint hardening | -| A.8.2 Ayrıcalıklı erişim hakları | YES | Operatör attribution; approval gate non-trainer operator | IdP'de RBAC | +| A.8.2 Ayrıcalıklı erişim hakları | YES | Operatör attribution; approval gate onaylayıcı kimliğini kaydeder | IdP'de RBAC | | A.8.3 Bilgi erişim kısıtlaması | YES | Salted identifier hashing; `forgelm reverse-pii` Md. 15 | DSAR workflow | | A.8.4 Kaynak koduna erişim | YES | `model.trust_remote_code=False` default; `model.revision` temel-model deposunu sabitler ve `trust_remote_code: true` ile çalıştırılacak depo-içi kodu sınırlar. `_fingerprint_hf_revision` yalnızca **dataset** deposunu kapsar | VCS erişim kontrolü; `trust_remote_code: true` onaylanan her yerde `model.revision` zorunlu tutun | | A.8.5 Güvenli kimlik doğrulama | YES | `safe_post` non-HTTPS'de auth header reddeder; webhook secret discipline | MFA + token rotasyonu | diff --git a/docs/qms/statement_of_applicability.md b/docs/qms/statement_of_applicability.md index b78bc887..26767b04 100644 --- a/docs/qms/statement_of_applicability.md +++ b/docs/qms/statement_of_applicability.md @@ -58,7 +58,7 @@ contribution → deployer-side action. | A.5.29 Information security during disruption | YES | `auto_revert` baseline-flip; `model.reverted` event | Document base-model retention | | A.5.30 ICT readiness for business continuity | YES | DR planning | Adopt | | A.5.31 Identification of legal, statutory, regulatory and contractual requirements | YES | EU AI Act + GDPR mappings; Annex IV bundle | Track rule changes | -| A.5.32 Intellectual property rights | YES | License extraction in SBOM; HF model-card metadata | Per-model license review | +| A.5.32 Intellectual property rights | YES | HF model-card metadata (the SBOM carries no licence data) | Per-model license review | | A.5.33 Protection of records | YES | Append-only + HMAC + manifest; off-site replica | Off-site backup | | A.5.34 Privacy and protection of PII | YES | `forgelm reverse-pii` Article 15; `forgelm purge` Article 17; `forgelm audit` PII detection + masking | DSAR workflow + DPIA | | A.5.35 Independent review of information security | YES | Annual external audit | Adopt | @@ -107,7 +107,7 @@ ForgeLM-specific control inventory. | Control | Applicable? | ForgeLM evidence | Deployer-side action | |---|---|---|---| | A.8.1 User endpoint devices | YES | `forgelm doctor` — Python / CUDA / GPU / extras / HF auth / disk / `FORGELM_OPERATOR` checks | Endpoint hardening | -| A.8.2 Privileged access rights | YES | Operator attribution; approval gate non-trainer operator | RBAC at IdP | +| A.8.2 Privileged access rights | YES | Operator attribution; approval gate records the approver identity | RBAC at IdP | | A.8.3 Information access restriction | YES | Salted identifier hashing; `forgelm reverse-pii` Article 15 | DSAR workflow | | A.8.4 Access to source code | YES | `model.trust_remote_code=False` default; `model.revision` pins the base-model repo, bounding the repo-bundled code `trust_remote_code: true` would run. `_fingerprint_hf_revision` covers the **dataset** repo only | VCS access control; require `model.revision` wherever `trust_remote_code: true` is approved | | A.8.5 Secure authentication | YES | `safe_post` rejects auth headers on non-HTTPS; webhook secret discipline | MFA + token rotation | diff --git a/docs/reference/architecture-tr.md b/docs/reference/architecture-tr.md index 634033f7..5da4fea3 100644 --- a/docs/reference/architecture-tr.md +++ b/docs/reference/architecture-tr.md @@ -96,7 +96,7 @@ ForgeLM/ │ ├── deepspeed/ # ZeRO-2, ZeRO-3, ZeRO-3+Offload ön ayarları │ └── safety_prompts/ # Yerleşik adversarial prompt kütüphanesi (140 prompt, 6 kategori) ├── notebooks/ # 10 Colab-uyumlu Jupyter notebook -├── tests/ # 130 test modülü +├── tests/ # 131 test modülü ├── tools/ # CI guard'ları: bilingual_parity, anchor_resolution, │ # cli_help_consistency, yaml_snippets, │ # audit_event_catalog, library_api_doc, diff --git a/docs/reference/architecture.md b/docs/reference/architecture.md index daeaeb45..e3872c43 100644 --- a/docs/reference/architecture.md +++ b/docs/reference/architecture.md @@ -96,7 +96,7 @@ ForgeLM/ │ ├── deepspeed/ # ZeRO-2, ZeRO-3, ZeRO-3+Offload presets │ └── safety_prompts/ # Built-in adversarial prompt library (140 prompts, 6 categories) ├── notebooks/ # 10 Colab-ready Jupyter notebooks -├── tests/ # 130 test modules +├── tests/ # 131 test modules ├── tools/ # CI guards: bilingual_parity, anchor_resolution, │ # cli_help_consistency, yaml_snippets, │ # audit_event_catalog, library_api_doc, @@ -134,7 +134,7 @@ Lightweight `TrainResult` dataclass — importable without torch/transformers. C Wraps EleutherAI `lm-evaluation-harness`. Runs configurable benchmark tasks, extracts accuracy metrics, applies min_score threshold, and saves results. Optional dependency: `pip install forgelm[eval]`. ### `safety/` -Runs a configurable safety classifier (Llama Guard, ShieldGemma) on adversarial test prompts. Generates responses from the fine-tuned model, classifies each as safe/unsafe, and triggers auto-revert if regression exceeds threshold. Errors are treated as unsafe (fail-safe principle). +Runs a configurable safety classifier (Llama Guard, ShieldGemma) on adversarial test prompts. Generates responses from the fine-tuned model, classifies each as safe/unsafe, and triggers auto-revert if regression exceeds threshold. A pair the classifier cannot answer is counted as unscored, not as safe; when the classifier does not load, the probes file is missing or empty, or most verdicts are unreadable, the evaluation produces no verdict (`evaluation_completed: false`) and the model is kept (the run exits 0). ### `judge.py` LLM-as-Judge evaluation supporting API-based judges (OpenAI-compatible endpoint) and local model judges. Includes robust JSON parsing with markdown code block extraction. Scores on 1-10 scale with configurable minimum threshold. diff --git a/docs/reference/audit_event_catalog-tr.md b/docs/reference/audit_event_catalog-tr.md index 3068d80d..452d4e73 100644 --- a/docs/reference/audit_event_catalog-tr.md +++ b/docs/reference/audit_event_catalog-tr.md @@ -1,9 +1,9 @@ # Audit Event Kataloğu -> **Hedef kitle:** EU AI Act Madde 12 kayıt-tutma artefaktlarını inceleyen ForgeLM operatörleri, denetçiler ve aşağı akış doğrulayıcıları. +> **Hedef kitle:** ForgeLM'in eğitim-hattı audit log'unu inceleyen ForgeLM operatörleri, denetçiler ve aşağı akış doğrulayıcıları. > **Ayna:** [audit_event_catalog.md](audit_event_catalog.md) -Bu katalog, ForgeLM'in EU AI Act Madde 12 tarafından zorunlu kılınan, append-only ve hash-zincirli kayıt-tutma artefaktı olan `audit_log.jsonl`'a yazabileceği tüm event'leri sıralar. Her satır ortak bir zarf paylaşır; aşağıdaki satırlardan birini `event` alanı seçer. +Bu katalog, ForgeLM'in append-only ve hash-zincirli eğitim-hattı audit log'u olan `audit_log.jsonl`'a yazabileceği tüm event'leri sıralar. Bu log, teknik dokümantasyonu (Annex IV) ve kalite-yönetimi kayıt tutmayı destekleyebilecek geliştirme-zamanı kanıtıdır; tek başına EU AI Act Madde 12'yi karşılamaz, çünkü Madde 12 dağıtılmış yüksek-riskli sistemin ömrü boyunca olayları otomatik kaydetmesini ister. Her satır ortak bir zarf paylaşır; aşağıdaki satırlardan birini `event` alanı seçer. ## Ortak zarf @@ -78,7 +78,7 @@ Hash zinciri, satır diske düştükten (`flush` + `fsync`) sonra ilerler; kirli | `data.erasure_requested` | Herhangi bir `forgelm purge --row-id` / `--run-id` çağrısının ilk adımı, herhangi bir silmeden ÖNCE. `--check-policy` salt-okunur; event yaymaz. | `target_kind` ∈ `{row, staging, artefacts}`, `target_id` (row mode'da hash'lenmiş), `salt_source` (row mode), `corpus_path` (row), `output_dir` (run), `justification`, `dry_run` | 17 | | `data.erasure_completed` | Başarılı silme tamamlandı. | Tüm `requested` field'ları + `bytes_freed`, `files_modified`, `pre_erasure_line_number` (row mode), `match_count` (row mode) | 17 | | `data.erasure_failed` | Disk operasyonu raise etti VEYA eşleşen satır/koşum bulunamadı VEYA çoklu-satır policy belirsizliği reddetti. | Tüm `requested` field'ları + `error_class`, `error_message` | 17 | -| `data.erasure_warning_memorisation` | Row erasure × bu corpus'u tüketen herhangi bir koşum için `final_model/` mevcut. | Tüm `completed` field'ları + `affected_run_ids` | 17 | +| `data.erasure_warning_memorisation` | Row erasure × `--output-dir` altında doğrudan bir `final_model/` dizini mevcut (birebir ad; `--output-dir` varsayılan olarak corpus'un dizinidir, bu yüzden eğitim çıktı dizininizi verin). ForgeLM corpus'u hangi koşumların tükettiğini belirlemez. | Tüm `completed` field'ları + `affected_run_ids` | 17 | | `data.erasure_warning_synthetic_data_present` | Row erasure × `output_dir`'de `synthetic_data*.jsonl` mevcut. | Tüm `completed` field'ları + `synthetic_files` | 17 | | `data.erasure_warning_external_copies` | Yüklü config boş-olmayan `webhook` block'u içeriyor; downstream tüketiciler bildirim almış olabilir. | Tüm `completed` field'ları + `webhook_targets` (redact'li URL'ler) | 17 | @@ -161,7 +161,7 @@ damgasıyla kurabilir. Uygulama: `forgelm/webhook.py`. | Webhook `event` | Denetim günlüğü karşılığı | Tetikleyici | Kapı (gate) | Zorunlu payload alanları | |---|---|---|---|---| | `training.start` | `training.started` | `train()` çağrıldı, model yüklenmeden önce. | `webhook.notify_on_start` | `run_name`, `status="started"` | -| `training.success` | `pipeline.completed` | Koşu revert veya bekleyen onay olmadan tamamlandı. `evaluation.auto_revert: true` ile tüm kapılar geçildi; varsayılan `auto_revert: false` ile bir kapı geçemediği halde yalnızca kaydedildiğinde de tetiklenir (model yine terfi eder). | `webhook.notify_on_success` | `run_name`, `status="succeeded"`, `metrics` | +| `training.success` | `pipeline.completed` | Koşu revert veya bekleyen onay olmadan tamamlandı. `evaluation.auto_revert: true` ile karara varan tüm kapılar geçildi (karar üretemeyen bir güvenlik değerlendirmesi, `safety.evaluation_completed: false`, modeli korur ve yine buraya varır); varsayılan `auto_revert: false` ile bir kapı geçemediği halde yalnızca kaydedildiğinde de tetiklenir (model yine terfi eder). | `webhook.notify_on_success` | `run_name`, `status="succeeded"`, `metrics` | | `training.failure` | `pipeline.failed` | Eğitim sürecinin kendisi hata fırlattı (OOM, veri seti hatası, yakalanmayan istisna). | `webhook.notify_on_failure` | `run_name`, `status="failed"`, `reason` (maskelenmiş, ≤2048 karakter) | | `training.reverted` | `model.reverted` | Eğitim sonrası bir kapı (değerlendirme, güvenlik, hakem, benchmark) çalışmayı reddetti ve `_revert_model` adaptörleri sildi. | `webhook.notify_on_failure` | `run_name`, `status="reverted"`, `reason` (maskelenmiş, ≤2048 karakter) | | `approval.required` | `human_approval.required` | Çalışma başarılı oldu, `evaluation.require_human_approval=true`, model insan incelemesi için staging'de (EU AI Act Madde 14). | `webhook.notify_on_success` | `run_name`, `status="awaiting_approval"`, `model_path` | diff --git a/docs/reference/audit_event_catalog.md b/docs/reference/audit_event_catalog.md index a7a74abd..d5b93ebd 100644 --- a/docs/reference/audit_event_catalog.md +++ b/docs/reference/audit_event_catalog.md @@ -1,9 +1,9 @@ # Audit Event Catalog -> **Audience:** ForgeLM operators, regulators, and downstream verifiers reviewing the EU AI Act Article 12 record-keeping artefacts. +> **Audience:** ForgeLM operators, regulators, and downstream verifiers reviewing ForgeLM's training-pipeline audit log. > **Mirror:** [audit_event_catalog-tr.md](audit_event_catalog-tr.md) -This catalog enumerates every event ForgeLM may append to `audit_log.jsonl`, the append-only, hash-chained record-keeping artefact mandated by EU AI Act Article 12. Every entry shares a common envelope; the `event` field selects one of the rows below. +This catalog enumerates every event ForgeLM may append to `audit_log.jsonl`, ForgeLM's append-only, hash-chained training-pipeline audit log. It is development-time evidence that can support the technical documentation (Annex IV) and quality-management record-keeping; it does not by itself satisfy EU AI Act Article 12, which requires the deployed high-risk system to log events automatically over its lifetime. Every entry shares a common envelope; the `event` field selects one of the rows below. ## Common envelope @@ -78,7 +78,7 @@ The hash chain advances after the line lands on disk (`flush` + `fsync`), so an | `data.erasure_requested` | First step of any `forgelm purge --row-id` / `--run-id` invocation, BEFORE any deletion. `--check-policy` is read-only and emits no events. | `target_kind` ∈ `{row, staging, artefacts}`, `target_id` (hashed in row mode), `salt_source` (row mode), `corpus_path` (row), `output_dir` (run), `justification`, `dry_run` | 17 | | `data.erasure_completed` | Successful deletion finished. | All `requested` fields + `bytes_freed`, `files_modified`, `pre_erasure_line_number` (row mode), `match_count` (row mode) | 17 | | `data.erasure_failed` | Disk operation raised, OR no matching row/run found, OR multi-row policy refused on ambiguity. | All `requested` fields + `error_class`, `error_message` | 17 | -| `data.erasure_warning_memorisation` | Row erasure × `final_model/` exists for any run that consumed this corpus. | All `completed` fields + `affected_run_ids` | 17 | +| `data.erasure_warning_memorisation` | Row erasure × a `final_model/` directory exists directly under `--output-dir` (literal name; `--output-dir` defaults to the corpus's directory, so pass your training output directory). ForgeLM does not determine which runs consumed the corpus. | All `completed` fields + `affected_run_ids` | 17 | | `data.erasure_warning_synthetic_data_present` | Row erasure × `synthetic_data*.jsonl` exists in `output_dir`. | All `completed` fields + `synthetic_files` | 17 | | `data.erasure_warning_external_copies` | Loaded config has a non-empty `webhook` block; downstream consumers may have received notices. | All `completed` fields + `webhook_targets` (redacted URLs) | 17 | @@ -161,7 +161,7 @@ webhook ping → audit entry by `run_name` + timestamp. Implementation: | Webhook `event` | Audit-log mirror | Trigger | Gated by | Required payload fields | |---|---|---|---|---| | `training.start` | `training.started` | `train()` entered, before model load. | `webhook.notify_on_start` | `run_name`, `status="started"` | -| `training.success` | `pipeline.completed` | Run completed without revert or pending approval. With `evaluation.auto_revert: true` all gates passed; with the default `auto_revert: false` it also fires when a gate failed but was only recorded (model still promoted). | `webhook.notify_on_success` | `run_name`, `status="succeeded"`, `metrics` | +| `training.success` | `pipeline.completed` | Run completed without revert or pending approval. With `evaluation.auto_revert: true` every gate that produced a verdict passed (a safety evaluation with no verdict, `safety.evaluation_completed: false`, keeps the model and also ends here); with the default `auto_revert: false` it also fires when a gate failed but was only recorded (model still promoted). | `webhook.notify_on_success` | `run_name`, `status="succeeded"`, `metrics` | | `training.failure` | `pipeline.failed` | Training itself raised (OOM, dataset error, unhandled exception). | `webhook.notify_on_failure` | `run_name`, `status="failed"`, `reason` (masked, ≤2048 chars) | | `training.reverted` | `model.reverted` | A post-training gate (evaluation, safety, judge, benchmark) rejected the run and `_revert_model` deleted the adapters. | `webhook.notify_on_failure` | `run_name`, `status="reverted"`, `reason` (masked, ≤2048 chars) | | `approval.required` | `human_approval.required` | Run succeeded, `evaluation.require_human_approval=true`, model staged for review (EU AI Act Art. 14). | `webhook.notify_on_success` | `run_name`, `status="awaiting_approval"`, `model_path` | diff --git a/docs/reference/configuration-tr.md b/docs/reference/configuration-tr.md index b78fa07f..1bd2349b 100644 --- a/docs/reference/configuration-tr.md +++ b/docs/reference/configuration-tr.md @@ -15,7 +15,7 @@ Tam açıklamalı örnek için `config_template.yaml` dosyasına bakın. | `load_in_4bit` | bool | `true` | QLoRA 4-bit NF4 kuantizasyon | | `backend` | string | `"transformers"` | `"transformers"` veya `"unsloth"` (desteklenen mimarilerde daha hızlı, Linux) | | `trust_remote_code` | bool | `false` | Model depolarından özel kod çalıştırma. **Güvenlik riski** | -| `offline` | bool | `false` | İzole mod: HF Hub çağrısı yok. Modeller/veri setleri yerel olmalı | +| `offline` | bool | `false` | HF offline değişkenlerini ayarlar; Hub, transformers ve datasets ağ çağrısı yapmaz. Modeller/veri setleri yerel olmalı. Webhook'ları, judge veya sentetik-veri API istemcilerini ve W&B / MLflow'u durdurmaz | | `revision` | string | `null` | Temel modeli + tokenizer'ı bir HF Hub commit SHA'sına (40-hex) veya branch/tag'e sabitle. **Bugün uygulanıyor.** Bkz. [Hub revision pinleme](#hub-revision-pinleme) | | `bnb_4bit_use_double_quant` | bool | `true` | Ekstra VRAM tasarrufu için çift kuantizasyon | | `bnb_4bit_quant_type` | string | `"nf4"` | Kuantizasyon tipi (`"nf4"` veya `"fp4"`) | @@ -138,9 +138,9 @@ training: | Alan | Tip | Varsayılan | Açıklama | |------|-----|-----------|----------| -| `rope_scaling` | `Optional[Dict[str, Any]]` | `null` | RoPE ölçekleme yöntemi sözlüğü (`{"type": "linear", "factor": 2.0}` vs.). Desteklenen tipler: `"linear"`, `"dynamic"`, `"yarn"`, `"longrope"`. | +| `rope_scaling` | `Optional[Dict[str, Any]]` | `null` | RoPE ölçekleme yöntemi sözlüğü (`{"type": "linear", "factor": 2.0}` vs.). Desteklenen tipler: `"linear"`, `"dynamic"`, `"yarn"`, `"longrope"`. `model.backend: unsloth` ile desteklenmez (reddedilir, exit 1). | | `neftune_noise_alpha` | float | `null` | NEFTune gürültü enjeksiyonu alpha değeri (ör. `5.0`) | -| `sliding_window_attention` | int | `null` | Kayan pencere dikkat boyutu (token) | +| `sliding_window_attention` | int | `null` | Kayan pencere dikkat boyutu (token). `model.backend: unsloth` ile desteklenmez (reddedilir, exit 1). | | `sample_packing` | bool | `false` | **Kullanımdan kaldırıldı** — `packing` için takma ad (TRL tek bir packing düğmesi sunar). `true` ayarlamak `DeprecationWarning` ile `packing: true`'ya yönlendirir; v1.0.0'da kaldırılacak. Bunun yerine `packing` kullanın. | > Kaldırma, daha erken değil v1.0.0'da gerçekleşecek; çünkü bir YAML alanını @@ -270,10 +270,10 @@ training: Uyumluluk, eğitim ve değerlendirme artefaktları için saklama ufuklarını belirler. Ufuklar GDPR Madde 5(1)(e) "saklama sınırlaması" ve Madde 17 -"silme hakkı" tarihlerini onurlandırır. `enforce` anahtarı yalnız-loglama, -uyarı ve sert-engelleme modları arasında geçiş yaparak regüle edilen bir CI -kapısının saklama ufkunu eski bir çalışma alanını yeniden kullanarak sessizce -uzatmasını engeller. +"silme hakkı" tarihlerini onurlandırır. Ufuklar **raporlanır**, zorlanmaz: +`forgelm purge --check-policy` ufku aşanları listeler ve her zaman 0 ile çıkar; +hiçbir şey otomatik silinmez, engellenmez ya da zamanlanmaz. `enforce` anahtarı +ayrılmıştır: henüz hiçbir komut onu okumaz. | Alan | Tip | Varsayılan | Açıklama | |------|-----|-----------|----------| @@ -281,7 +281,7 @@ uzatmasını engeller. | `staging_ttl_days` | int | `7` | `forgelm reject` kararından sonra `final_model.staging./` dizininin planlı temizlenmeden önce saklanacağı gün sayısı. `0` süresiz saklama anlamına gelir. v0.8.0'da kaldırılan `evaluation.staging_ttl_days` yerine geçer. | | `ephemeral_artefact_retention_days` | int | `90` | Uyumluluk paketleri, veri denetim raporları ve diğer çalışma kapsamlı türetilmiş artefaktların saklanma süresi (gün). `0` süresiz saklama. | | `raw_documents_retention_days` | int | `90` | İngest edilmiş ham belgelerin (PDF / DOCX / EPUB / TXT / Markdown) operatörün ingestion-output dizininde saklanma süresi (gün). `0` süresiz saklama. | -| `enforce` | string | `"log_only"` | Politika uygulama modu: `"log_only"` (yalnızca audit log), `"warn_on_excess"` (stderr'e yapılandırılmış uyarı), `"block_on_excess"` (`EXIT_EVAL_FAILURE` = 3 ile trainer ön-kontrolünü iptal eder). | +| `enforce` | string | `"log_only"` | Ayrılmış politika modu. Henüz hiçbir komut onu okumaz: `"log_only"`, `"warn_on_excess"` ve `"block_on_excess"` kabul edilir ve aynı davranır; bir config son ikisinden birini ayarlarsa CLI uyarır. | > **Kaldırıldı:** `evaluation.staging_ttl_days` (v0.5.5 itibarıyla kullanımdan > kaldırılmıştı) v0.8.0'da kaldırılmıştır. Artık geçerli tek form @@ -397,11 +397,11 @@ kaynağı kaydedilmemiştir. | Alan | Tip | Varsayılan | Açıklama | |------|-----|-----------|----------| | `strategy` | string | `null` | `"deepspeed"` veya `"fsdp"` (null = tek GPU) | -| `deepspeed_config` | string | `null` | Ön ayar (`"zero2"`, `"zero3"`, `"zero3_offload"`) veya JSON yolu | -| `fsdp_strategy` | string | `"full_shard"` | `"full_shard"`, `"shard_grad_op"`, `"hybrid_shard"`, `"no_shard"` | -| `fsdp_auto_wrap` | bool | `true` | Transformer katmanlarını otomatik sar | +| `deepspeed_config` | string | `null` | JSON yolu ya da yalnızca kaynak checkout'unda çözülen (pip veya Docker kurulumunda çözülmeyen) bir ön ayar adı (`"zero2"`, `"zero3"`, `"zero3_offload"`) | +| `fsdp_strategy` | string | `"full_shard"` | `"full_shard"` veya `"shard_grad_op"` (`"hybrid_shard"` ve `"no_shard"` reddedilir: FSDP2 bunları desteklemez) | +| `fsdp_auto_wrap` | bool | `true` | Transformer katmanlarını otomatik sar (transformers 5.10+'ta yok sayılır, katmanlar hep sarılır) | | `fsdp_offload` | bool | `false` | Parametreleri CPU'ya taşı | -| `fsdp_backward_prefetch` | string | `"backward_pre"` | `"backward_pre"` veya `"backward_post"` | +| `fsdp_backward_prefetch` | string | `"backward_pre"` | `"backward_pre"` veya `"backward_post"` (transformers 5.10+'ta yok sayılır; FSDP2 prefetch politikası almaz) | | `fsdp_state_dict_type` | string | `"FULL_STATE_DICT"` | `"FULL_STATE_DICT"` veya `"SHARDED_STATE_DICT"` | ## `synthetic` (İsteğe bağlı — Sentetik Veri Üretimi) @@ -433,11 +433,11 @@ kaynağı kaydedilmemiştir. | Alan | Tip | Varsayılan | Açıklama | |------|-----|-----------|----------| -| `url` | string | `null` | Webhook hedef URL | -| `url_env` | string | `null` | URL'yi içeren ortam değişkeni adı | +| `url` | string | `null` | Webhook hedef URL'si: bir Slack incoming webhook'u ya da bir relay (Teams ve Discord payload'u doğrudan reddeder) | +| `url_env` | string | `null` | URL'yi içeren ortam değişkeni adı; yalnızca `url` boşken okunur | | `notify_on_start` | bool | `true` | Eğitim başlangıcında bildir | | `notify_on_success` | bool | `true` | Başarıda bildir | -| `notify_on_failure` | bool | `true` | Hata durumunda bildir | +| `notify_on_failure` | bool | `true` | Eğitim hata verdiğinde ya da eğitim sonrası bir kapı modeli geri aldığında bildir | | `timeout` | int | `10` | HTTP istek zaman aşımı (saniye). Notifier ≥ 1s'ye clamp'ler. v0.5.5'te varsayılan 10s'ye çıkarıldı (önceden 5s'di) — Slack/Teams gateway gecikme atışları production'da düzenli olarak 5s'yi aşıyor ve bir webhook zaman aşımı audit chain'i sessizce zayıflatıyor (webhook arızası best-effort). | | `allow_private_destinations` | bool | `false` | RFC1918 / loopback / link-local hedeflere webhook gönderimine izin verir (cluster içi Slack proxy, on-prem Teams gateway gibi). Varsayılan yalnızca genel internet — SSRF koruması | | `require_https` | bool | `false` | TLS-only zorlama. `true`, plaintext bir `http://` URL'ini reddeder (SSRF chokepoint raise eder; POST atlanır), warn-and-send yerine. Varsayılan `false`, warn-then-send davranışını korur | diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 0f83d16e..1af45a54 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -15,7 +15,7 @@ See `config_template.yaml` for a complete annotated example. | `load_in_4bit` | bool | `true` | Enable QLoRA 4-bit NF4 quantization | | `backend` | string | `"transformers"` | `"transformers"` or `"unsloth"` (faster on supported architectures, Linux only) | | `trust_remote_code` | bool | `false` | Allow custom code from model repos. **Security risk** — only enable for models that require it | -| `offline` | bool | `false` | Air-gapped mode: no HF Hub calls. Models/datasets must be local | +| `offline` | bool | `false` | Sets the HF offline variables, so Hub, transformers and datasets make no network calls. Models/datasets must be local. Does not stop webhooks, judge or synthetic-data API clients, or W&B / MLflow | | `revision` | string | `null` | Pin the base model + tokenizer to an HF Hub commit SHA (40-hex) or a branch/tag. **Honoured today.** See [Hub revision pinning](#hub-revision-pinning) | | `bnb_4bit_use_double_quant` | bool | `true` | Double quantization for extra VRAM savings | | `bnb_4bit_quant_type` | string | `"nf4"` | Quantization type (`"nf4"` or `"fp4"`) | @@ -140,9 +140,9 @@ across retries. Each retry attempt is logged to the audit trail. | Field | Type | Default | Description | |-------|------|---------|-------------| -| `rope_scaling` | `Optional[Dict[str, Any]]` | `null` | RoPE scaling method dict (`{"type": "linear", "factor": 2.0}` etc.). Supported types: `"linear"`, `"dynamic"`, `"yarn"`, `"longrope"`. | +| `rope_scaling` | `Optional[Dict[str, Any]]` | `null` | RoPE scaling method dict (`{"type": "linear", "factor": 2.0}` etc.). Supported types: `"linear"`, `"dynamic"`, `"yarn"`, `"longrope"`. Not supported with `model.backend: unsloth` (rejected, exit 1). | | `neftune_noise_alpha` | float | `null` | NEFTune noise injection alpha (e.g., `5.0`) | -| `sliding_window_attention` | int | `null` | Sliding window attention size in tokens | +| `sliding_window_attention` | int | `null` | Sliding window attention size in tokens. Not supported with `model.backend: unsloth` (rejected, exit 1). | | `sample_packing` | bool | `false` | **Deprecated** alias for `packing` (TRL exposes a single packing knob). Setting `true` forwards to `packing: true` with a `DeprecationWarning`; will be removed in v1.0.0. Use `packing` instead. | > Removal will land in v1.0.0, not sooner, because removing a YAML field is a @@ -270,9 +270,10 @@ across retries. Each retry attempt is logged to the audit trail. Defines maximum retention horizons for compliance, training, and evaluation artefacts. Horizons honour GDPR Article 5(1)(e) "storage limitation" and -Article 17 "right to erasure" deadlines. The `enforce` knob switches between -log-only, warning, and hard-block modes so a regulated CI gate cannot -silently extend the retention horizon by re-using a stale workspace. +Article 17 "right to erasure" deadlines. The horizons are **reported**, not +enforced: `forgelm purge --check-policy` lists what is past them and always +exits 0, and nothing is deleted, blocked or scheduled automatically. The +`enforce` knob is reserved: no command reads it yet. | Field | Type | Default | Description | |-------|------|---------|-------------| @@ -280,7 +281,7 @@ silently extend the retention horizon by re-using a stale workspace. | `staging_ttl_days` | int | `7` | Days to retain `final_model.staging./` after a `forgelm reject` decision before scheduled cleanup. Set to `0` to retain indefinitely. Replaces the removed `evaluation.staging_ttl_days` (removed in v0.8.0). | | `ephemeral_artefact_retention_days` | int | `90` | Days to retain compliance bundles, data audit reports, and other run-scoped derived artefacts. Set to `0` to retain indefinitely. | | `raw_documents_retention_days` | int | `90` | Days to retain ingested raw documents (PDF / DOCX / EPUB / TXT / Markdown) under the operator's ingestion-output directory. Set to `0` to retain indefinitely. | -| `enforce` | string | `"log_only"` | Policy enforcement mode: `"log_only"` (audit-log only), `"warn_on_excess"` (structured stderr warning), `"block_on_excess"` (abort trainer pre-flight with `EXIT_EVAL_FAILURE` = 3). | +| `enforce` | string | `"log_only"` | Reserved policy mode. No command reads it yet: `"log_only"`, `"warn_on_excess"` and `"block_on_excess"` are all accepted and behave the same, and the CLI warns when a config sets either of the last two. | > **Removed:** `evaluation.staging_ttl_days` (deprecated as of v0.5.5) was > removed in v0.8.0. `retention.staging_ttl_days` is now the only accepted form. @@ -292,11 +293,11 @@ silently extend the retention horizon by re-using a stale workspace. | Field | Type | Default | Description | |-------|------|---------|-------------| -| `url` | string | `null` | Webhook destination URL | -| `url_env` | string | `null` | Env var name containing URL | +| `url` | string | `null` | Webhook destination URL: a Slack incoming webhook, or a relay (Teams and Discord reject the payload directly) | +| `url_env` | string | `null` | Env var name containing the URL; read only when `url` is empty | | `notify_on_start` | bool | `true` | Notify on training start | | `notify_on_success` | bool | `true` | Notify on success | -| `notify_on_failure` | bool | `true` | Notify on failure | +| `notify_on_failure` | bool | `true` | Notify when training raises or a post-training gate reverts the model | | `timeout` | int | `10` | HTTP request timeout (seconds). Clamped to ≥ 1s by the notifier. Default raised to 10s in v0.5.5 (was 5s) — Slack/Teams gateway latency spikes regularly cross 5s in production, and a webhook timeout silently degrades the audit chain (webhook failure is best-effort). | | `allow_private_destinations` | bool | `false` | Opt in to webhooks pointing at RFC1918 / loopback / link-local hosts (in-cluster Slack proxy, on-prem Teams gateway). Defaults to public-internet only — SSRF guard | | `require_https` | bool | `false` | TLS-only enforcement. `true` refuses a plaintext `http://` URL (the SSRF chokepoint raises; the POST is skipped) instead of warn-and-send. Default `false` preserves warn-then-send | @@ -309,11 +310,11 @@ silently extend the retention horizon by re-using a stale workspace. | Field | Type | Default | Description | |-------|------|---------|-------------| | `strategy` | string | `null` | `"deepspeed"` or `"fsdp"` (null = single GPU) | -| `deepspeed_config` | string | `null` | Preset (`"zero2"`, `"zero3"`, `"zero3_offload"`) or JSON path | -| `fsdp_strategy` | string | `"full_shard"` | `"full_shard"`, `"shard_grad_op"`, `"hybrid_shard"`, `"no_shard"` | -| `fsdp_auto_wrap` | bool | `true` | Auto-wrap transformer layers | +| `deepspeed_config` | string | `null` | JSON path, or a preset name (`"zero2"`, `"zero3"`, `"zero3_offload"`) that resolves only from a source checkout, not from a pip or Docker install | +| `fsdp_strategy` | string | `"full_shard"` | `"full_shard"` or `"shard_grad_op"` (`"hybrid_shard"` and `"no_shard"` are rejected: FSDP2 does not support them) | +| `fsdp_auto_wrap` | bool | `true` | Auto-wrap transformer layers (ignored on transformers 5.10+, which always wraps them) | | `fsdp_offload` | bool | `false` | Offload parameters to CPU | -| `fsdp_backward_prefetch` | string | `"backward_pre"` | `"backward_pre"` or `"backward_post"` | +| `fsdp_backward_prefetch` | string | `"backward_pre"` | `"backward_pre"` or `"backward_post"` (ignored on transformers 5.10+, whose FSDP2 takes no prefetch policy) | | `fsdp_state_dict_type` | string | `"FULL_STATE_DICT"` | `"FULL_STATE_DICT"` or `"SHARDED_STATE_DICT"` | --- diff --git a/docs/reference/distributed_training-tr.md b/docs/reference/distributed_training-tr.md index 32e6378f..6054462b 100644 --- a/docs/reference/distributed_training-tr.md +++ b/docs/reference/distributed_training-tr.md @@ -49,7 +49,17 @@ torchrun --nproc_per_node=4 -m forgelm.cli --config my_config.yaml ## DeepSpeed Yapılandırması -ForgeLM, üç yerleşik DeepSpeed ön ayarı sunar. `deepspeed_config` parametresine bu isimlerden birini veya kendi JSON dosyanızın yolunu verin. +Depo, `configs/deepspeed/` altında üç DeepSpeed ön ayarı barındırır. `deepspeed_config` parametresine bunların adlarından birini veya kendi JSON dosyanızın yolunu verin. + +**Bir ön ayar adı yalnızca `configs/deepspeed/.json` kurulu paketin yanında ya da geçerli dizinde mevcutsa çalışır.** +Kaynak checkout'u bu dosyaları sağlar; `pip install forgelm` wheel'i ve Docker imajı sağlamaz, orada bir ön ayar adı +yapılandırma hatasıyla (exit 1) başarısız olur. Deponun `configs/deepspeed/` klasöründeki JSON dosyalarını kopyalayın +(çalışma dizinindeki bir `configs/deepspeed/` dizinine ya da `deepspeed_config`'i yoluna yönelttiğiniz herhangi bir yere) +veya kendinizinkini yazın. + +**Bilinen sorun ([#410](https://github.com/HodeTech/ForgeLM/issues/410)):** `deepspeed_config` ayarlanmadığında da +kullanılan `zero2` ön ayarı `allgather_bucket_size` değerini `"auto"` yapar; transformers'ın DeepSpeed entegrasyonu bu +değeri doldurmaz, bu yüzden onunla bir ZeRO-2 koşusu durur. Düzelene kadar kendi ZeRO-2 JSON'unuzu kullanın. ### Ön Ayarlar (Presets) @@ -64,8 +74,8 @@ ForgeLM, üç yerleşik DeepSpeed ön ayarı sunar. `deepspeed_config` parametre ```yaml distributed: strategy: "deepspeed" - deepspeed_config: "zero2" # Ön ayar adı - # deepspeed_config: "./my_ds_config.json" # Veya özel dosya yolu + deepspeed_config: "./my_ds_config.json" # Kendi JSON'unuzun yolu (her kurulumda çalışır) + # deepspeed_config: "zero2" # Ön ayar adı: yalnızca kaynak checkout'u ``` ### ZeRO Aşama Karşılaştırması @@ -120,8 +130,8 @@ PyTorch Fully Sharded Data Parallel (FSDP), DeepSpeed'e alternatif olarak PyTorc |----------|----------|-----------------| | `full_shard` | Parametreleri, gradyanları ve optimizer durumlarını böl | En Yüksek | | `shard_grad_op` | Yalnızca gradyanları ve optimizer durumlarını böl | Orta | -| `hybrid_shard` | Düğüm içi tam bölme, düğümler arası çoğaltma | Çoklu Düğüm | -| `no_shard` | Standart DDP (bölme yok) | Yok | + +`hybrid_shard` ve `no_shard` **kabul edilmez**: `fsdp_strategy` `full_shard` ya da `shard_grad_op` olmalıdır. ForgeLM'in gerektirdiği transformers 5.10 ve sonrası varsayılan olarak FSDP2 çalıştırır ve FSDP2 bu ikisini de desteklemez: eskiden `--dry-run`'ı geçip model yüklendikten sonra exit 2 ile (`fsdp=hybrid_shard is only available with FSDP1`) patlıyorlardı, şimdi doğrulamada exit 1 ile reddediliyorlar. ### Kullanım @@ -129,12 +139,14 @@ PyTorch Fully Sharded Data Parallel (FSDP), DeepSpeed'e alternatif olarak PyTorc distributed: strategy: "fsdp" fsdp_strategy: "full_shard" - fsdp_auto_wrap: true # Auto-wrap transformer layers + fsdp_auto_wrap: true # transformers 5.10+'ta yok sayılır (FSDP2 transformer katmanlarını hep sarar) fsdp_offload: false # Offload parameters AND gradients (between forward/backward) to CPU - fsdp_backward_prefetch: "backward_pre" # Prefetch strategy + fsdp_backward_prefetch: "backward_pre" # transformers 5.10+'ta yok sayılır (FSDP2 prefetch politikası almaz) fsdp_state_dict_type: "FULL_STATE_DICT" # State dict handling ``` +`fsdp_auto_wrap: false` ve `fsdp_backward_prefetch: backward_post` hâlâ kabul edilir, ancak FSDP2'de hiçbir şeyi değiştirmezler; bu yüzden ikisinden biri varsayılanından farklı ayarlandığında ForgeLM bir uyarı loglar. + ### FSDP'yi DeepSpeed yerine ne zaman seçmeli - **FSDP**: Native PyTorch, ekstra bağımlılık yok, daha basit kurulum, diff --git a/docs/reference/distributed_training.md b/docs/reference/distributed_training.md index b83b29f9..95becdf9 100644 --- a/docs/reference/distributed_training.md +++ b/docs/reference/distributed_training.md @@ -49,7 +49,17 @@ torchrun --nproc_per_node=4 -m forgelm.cli --config my_config.yaml ## DeepSpeed Configuration -ForgeLM provides three built-in DeepSpeed presets. Set `deepspeed_config` to one of these names, or provide a path to your own JSON file. +The repository ships three DeepSpeed presets under `configs/deepspeed/`. Set `deepspeed_config` to one of their names, or provide a path to your own JSON file. + +**A preset name works only when `configs/deepspeed/.json` is available beside the installed package or in the +current directory.** A source checkout provides those files; a `pip install forgelm` wheel and the Docker image do not, +so there a preset name fails with a configuration error (exit 1). Copy the JSON files from the repository's +`configs/deepspeed/` folder (into a `configs/deepspeed/` directory in the working directory, or anywhere if you point +`deepspeed_config` at the path), or write your own. + +**Known issue ([#410](https://github.com/HodeTech/ForgeLM/issues/410)):** the `zero2` preset, which is also used when +`deepspeed_config` is unset, sets `allgather_bucket_size` to `"auto"`, a value transformers' DeepSpeed integration does +not fill in, so a ZeRO-2 run with it aborts. Use your own ZeRO-2 JSON until that is fixed. ### Presets @@ -64,8 +74,8 @@ ForgeLM provides three built-in DeepSpeed presets. Set `deepspeed_config` to one ```yaml distributed: strategy: "deepspeed" - deepspeed_config: "zero2" # Preset name - # deepspeed_config: "./my_ds_config.json" # Or custom path + deepspeed_config: "./my_ds_config.json" # Path to your own JSON (works on every install) + # deepspeed_config: "zero2" # Preset name: source checkout only ``` ### ZeRO Stage Comparison @@ -118,8 +128,8 @@ PyTorch Fully Sharded Data Parallel (FSDP) is an alternative to DeepSpeed that's |----------|-------------|----------------| | `full_shard` | Shard parameters, gradients, and optimizer states | Highest | | `shard_grad_op` | Shard gradients and optimizer states only | Medium | -| `hybrid_shard` | Full shard within nodes, replicate across nodes | Multi-node | -| `no_shard` | Standard DDP (no sharding) | None | + +`hybrid_shard` and `no_shard` are **not accepted**: `fsdp_strategy` must be `full_shard` or `shard_grad_op`. transformers 5.10 and later, which ForgeLM requires, run FSDP2 by default, and FSDP2 supports neither: they used to pass `--dry-run` and then fail with exit 2 (`fsdp=hybrid_shard is only available with FSDP1`) after the model was loaded, and now fail validation with exit 1. ### Usage @@ -127,12 +137,14 @@ PyTorch Fully Sharded Data Parallel (FSDP) is an alternative to DeepSpeed that's distributed: strategy: "fsdp" fsdp_strategy: "full_shard" - fsdp_auto_wrap: true # Auto-wrap transformer layers + fsdp_auto_wrap: true # Ignored on transformers 5.10+ (FSDP2 always wraps transformer layers) fsdp_offload: false # Offload parameters AND gradients (between forward/backward) to CPU - fsdp_backward_prefetch: "backward_pre" # Prefetch strategy + fsdp_backward_prefetch: "backward_pre" # Ignored on transformers 5.10+ (FSDP2 takes no prefetch policy) fsdp_state_dict_type: "FULL_STATE_DICT" # State dict handling ``` +`fsdp_auto_wrap: false` and `fsdp_backward_prefetch: backward_post` are still accepted, but under FSDP2 they change nothing, so ForgeLM logs a warning when either is set away from its default. + ### When to Choose FSDP over DeepSpeed - **FSDP**: Native PyTorch, no extra dependency, simpler setup, good for most use cases diff --git a/docs/reference/doctor_subcommand-tr.md b/docs/reference/doctor_subcommand-tr.md index e41f7d59..9940d21f 100644 --- a/docs/reference/doctor_subcommand-tr.md +++ b/docs/reference/doctor_subcommand-tr.md @@ -85,7 +85,7 @@ $ forgelm doctor forgelm doctor - environment check [+ pass] python.version Python 3.11.4 (CPython). - [+ pass] torch.cuda torch 2.4.0 with CUDA 12.4. + [+ pass] torch.cuda torch 2.7.0 with CUDA 12.6. [+ pass] gpu.inventory 1 GPU(s) - GPU0: NVIDIA RTX 4090 (24.0 GiB). [+ pass] extras.qlora Installed (module bitsandbytes, purpose: 4-bit / 8-bit QLoRA training). [! warn] extras.tracking Optional extra missing - install with: pip install 'forgelm[tracking]' (purpose: Weights & Biases experiment tracking). diff --git a/docs/reference/doctor_subcommand.md b/docs/reference/doctor_subcommand.md index ad7ef6e0..2db942bb 100644 --- a/docs/reference/doctor_subcommand.md +++ b/docs/reference/doctor_subcommand.md @@ -85,7 +85,7 @@ $ forgelm doctor forgelm doctor - environment check [+ pass] python.version Python 3.11.4 (CPython). - [+ pass] torch.cuda torch 2.4.0 with CUDA 12.4. + [+ pass] torch.cuda torch 2.7.0 with CUDA 12.6. [+ pass] gpu.inventory 1 GPU(s) - GPU0: NVIDIA RTX 4090 (24.0 GiB). [+ pass] extras.qlora Installed (module bitsandbytes, purpose: 4-bit / 8-bit QLoRA training). [! warn] extras.tracking Optional extra missing - install with: pip install 'forgelm[tracking]' (purpose: Weights & Biases experiment tracking). diff --git a/docs/reference/iso27001_control_mapping-tr.md b/docs/reference/iso27001_control_mapping-tr.md index 57e78d96..9fb5ca31 100644 --- a/docs/reference/iso27001_control_mapping-tr.md +++ b/docs/reference/iso27001_control_mapping-tr.md @@ -23,7 +23,7 @@ |---|---|---| | A.5.1 Bilgi güvenliği için politikalar | FL-helps | `audit_event_catalog.md` neyin loglandığını dokümante eder | | A.5.2 Bilgi güvenliği rolleri ve sorumlulukları | FL-helps | `FORGELM_OPERATOR` + `roles_responsibilities.md` | -| A.5.3 Görev ayrılığı | FL-helps | `human_approval.required/granted` trainer ≠ approver'ı zorlar | +| A.5.3 Görev ayrılığı | FL-helps | `human_approval.required/granted` onaylayıcı kimliğini (`FORGELM_OPERATOR`'dan, yoksa OS kullanıcısından) **kaydeder**, böylece eğitici ≠ onaylayıcı denetlenebilir; ForgeLM kendisi self-approval'ı reddetmez. Runner ve reviewer kimliklerini kimlik sağlayıcınızda ayrık tutun | | A.5.4 Yönetim sorumlulukları | FL-helps | `forgelm doctor`; `compliance_report.json`; `training_manifest.yaml` | | A.5.5 Yetkililerle iletişim | OOS | — | | A.5.6 Özel ilgi gruplarıyla iletişim | OOS | — | @@ -52,7 +52,7 @@ | A.5.29 Aksaklık sırasında bilgi güvenliği | FL-helps | `auto_revert` baseline-flip; `model.reverted` olayı | | A.5.30 İş sürekliliği için ICT hazırlığı | OOS | — | | A.5.31 Yasal, kanuni, düzenleyici ve sözleşmesel gerekliliklerin tanımlanması | FL-helps | EU AI Act + GDPR mappings; Annex IV bundle | -| A.5.32 Fikri mülkiyet hakları | FL-helps | SBOM'da lisans çıkarımı; HF model-card metadata | +| A.5.32 Fikri mülkiyet hakları | FL-helps | HF model-card metadata (SBOM lisans verisi taşımaz) | | A.5.33 Kayıtların korunması | FL | Append-only + HMAC + manifest sidecar | | A.5.34 Gizlilik ve PII korunması | FL | `forgelm reverse-pii` Madde 15; `forgelm purge` Madde 17 (ayrıca bkz. A.8.3) | | A.5.35 Bilgi güvenliğinin bağımsız incelenmesi | OOS | — | diff --git a/docs/reference/iso27001_control_mapping.md b/docs/reference/iso27001_control_mapping.md index 5f1885a2..442c0617 100644 --- a/docs/reference/iso27001_control_mapping.md +++ b/docs/reference/iso27001_control_mapping.md @@ -24,7 +24,7 @@ |---|---|---| | A.5.1 Policies for information security | FL-helps | `audit_event_catalog.md` documents what's logged | | A.5.2 Information security roles and responsibilities | FL-helps | `FORGELM_OPERATOR` + `roles_responsibilities.md` | -| A.5.3 Segregation of duties | FL-helps | `human_approval.required/granted` enforces trainer ≠ approver | +| A.5.3 Segregation of duties | FL-helps | `human_approval.required/granted` **records** the approver identity (from `FORGELM_OPERATOR`, else the OS user) so trainer ≠ approver is auditable; ForgeLM does not reject self-approval. Keep the runner and reviewer identities disjoint at your identity provider | | A.5.4 Management responsibilities | FL-helps | `forgelm doctor`; `compliance_report.json`; `training_manifest.yaml` | | A.5.5 Contact with authorities | OOS | — | | A.5.6 Contact with special interest groups | OOS | — | @@ -53,7 +53,7 @@ | A.5.29 Information security during disruption | FL-helps | `auto_revert` baseline-flip; `model.reverted` event | | A.5.30 ICT readiness for business continuity | OOS | — | | A.5.31 Identification of legal, statutory, regulatory and contractual requirements | FL-helps | EU AI Act + GDPR mappings; Annex IV bundle | -| A.5.32 Intellectual property rights | FL-helps | License extraction in SBOM; HF model-card metadata | +| A.5.32 Intellectual property rights | FL-helps | HF model-card metadata (the SBOM carries no licence data) | | A.5.33 Protection of records | FL | Append-only + HMAC + manifest sidecar | | A.5.34 Privacy and protection of PII | FL | `forgelm reverse-pii` Article 15; `forgelm purge` Article 17 (see also A.8.3) | | A.5.35 Independent review of information security | OOS | — | diff --git a/docs/reference/purge_subcommand-tr.md b/docs/reference/purge_subcommand-tr.md index de03e4d2..cf111861 100644 --- a/docs/reference/purge_subcommand-tr.md +++ b/docs/reference/purge_subcommand-tr.md @@ -35,7 +35,7 @@ JSONL eğitim corpus'undan `id` (veya `row_id`) field'ı ile tanımlanan tek bir | `--row-id ID` | evet | Silinecek identifier. Audit emisyonundan önce `forgelm.cli.subcommands._purge._hash_target_id` üzerinden hash'lenir. | | `--corpus PATH` | evet | Tek bir JSONL dosyası. Dizin modu reddedilir — operatörler kendi script'lerinde döngü kurar. | | `--row-matches {one,all}` | hayır (varsayılan `one`) | `one` >=2 eşleşmede reddeder; `all` her eşleşmeyi siler (operatör niyeti onaylar). | -| `--output-dir DIR` | hayır | `--corpus`'un parent'ına default'lar. Per-output-dir salt `/.forgelm_audit_salt`'ta `target_id` hash'leme için okunur. Implicit fallback, `forgelm reverse-pii` ile cross-tool korelasyon için operatörün `--output-dir`'i sabitleyebilmesi adına çözülen dizini adlandıran bir WARNING yayar. | +| `--output-dir DIR` | hayır | `--corpus`'un parent'ına default'lar. Per-output-dir salt `/.forgelm_audit_salt`'ta `target_id` hash'leme için okunur. Implicit fallback, operatörün `--output-dir`'i eğitim çıktı dizinine ve `forgelm reverse-pii` ile cross-tool korelasyon için sabitleyebilmesi adına çözülen dizini ve memorisation kontrolünün arayacağı `final_model/`'i adlandıran bir WARNING yayar. | | `--justification TEXT` | hayır | Operatör-sağlanan, her erasure event'ına yazılan sebep. Dahili ticket id'nizi referans verin; subject identifier yapıştırmayın. | | `--dry-run` | hayır | Silmeyi önizler + audit zincirini (`dry_run=true` ile) yayar; disk'e dokunmaz. | @@ -80,7 +80,7 @@ Altı event de [`audit_event_catalog.md`](audit_event_catalog-tr.md)'deki ortak | `data.erasure_requested` | Herhangi bir `--row-id` / `--run-id` çağrısının ilk adımı, herhangi bir silmeden ÖNCE. `--check-policy` salt-okunurdur ve event yaymaz. | `target_kind` ∈ `{row, staging, artefacts}`, `target_id` (row mode'da hash'li), `salt_source` (row mode), `corpus_path` (row), `output_dir` (run), `justification`, `dry_run` | | `data.erasure_completed` | Başarılı silme tamamlandı. | Tüm `requested` field'ları + `bytes_freed`, `files_modified`, `pre_erasure_line_number` (row mode), `match_count` (row mode) | | `data.erasure_failed` | Disk operasyonu raise etti VEYA eşleşen satır/koşum yok VEYA çoklu-satır policy belirsizliği reddetti. | Tüm `requested` field'ları + `error_class`, `error_message` | -| `data.erasure_warning_memorisation` | Row erasure × bu corpus'u tüketen herhangi bir koşum için `final_model/` mevcut. | Tüm `completed` field'ları + `affected_run_ids` | +| `data.erasure_warning_memorisation` | Row erasure × `--output-dir` altında doğrudan bir `final_model/` dizini mevcut (birebir ad; `--output-dir` varsayılan olarak corpus'un dizinidir, bu yüzden eğitim çıktı dizininizi verin). ForgeLM corpus'u hangi koşumların tükettiğini belirlemez. | Tüm `completed` field'ları + `affected_run_ids` | | `data.erasure_warning_synthetic_data_present` | Row erasure × `output_dir`'de `synthetic_data*.jsonl` mevcut. | Tüm `completed` field'ları + `synthetic_files` | | `data.erasure_warning_external_copies` | Yüklü config boş-olmayan `webhook` block'u içeriyor; downstream tüketiciler bildirim almış olabilir. | Tüm `completed` field'ları + `webhook_targets` (redact'li URL'ler) | diff --git a/docs/reference/purge_subcommand.md b/docs/reference/purge_subcommand.md index ebbffe38..2c44b85f 100644 --- a/docs/reference/purge_subcommand.md +++ b/docs/reference/purge_subcommand.md @@ -35,7 +35,7 @@ Erases a single row identified by its `id` (or `row_id`) field from a JSONL trai | `--row-id ID` | yes | Identifier to erase. Hashed before audit emission via `forgelm.cli.subcommands._purge._hash_target_id`. | | `--corpus PATH` | yes | Single JSONL file. Directory mode is rejected — operators loop in their own script. | | `--row-matches {one,all}` | no (default `one`) | `one` refuses on >=2 matches; `all` deletes every match (operator confirms intent). | -| `--output-dir DIR` | no | Defaults to the parent of `--corpus`. The per-output-dir salt at `/.forgelm_audit_salt` is read here for `target_id` hashing. The implicit fallback emits a WARNING naming the resolved dir so the operator can pin `--output-dir` for cross-tool correlation with `forgelm reverse-pii`. | +| `--output-dir DIR` | no | Defaults to the parent of `--corpus`. The per-output-dir salt at `/.forgelm_audit_salt` is read here for `target_id` hashing. The implicit fallback emits a WARNING naming the resolved dir and the `final_model/` the memorisation check will look for, so the operator can pin `--output-dir` to the training output directory and for cross-tool correlation with `forgelm reverse-pii`. | | `--justification TEXT` | no | Operator-supplied reason recorded on every erasure event. Reference your internal ticket id; do not paste subject identifiers. | | `--dry-run` | no | Preview the deletion + emit the audit chain (with `dry_run=true`) without modifying disk. | @@ -80,7 +80,7 @@ All six events ride the common envelope from [`audit_event_catalog.md`](audit_ev | `data.erasure_requested` | First step of any `--row-id` / `--run-id` invocation, BEFORE any deletion. `--check-policy` is read-only and emits no events. | `target_kind` ∈ `{row, staging, artefacts}`, `target_id` (hashed in row mode), `salt_source` (row mode), `corpus_path` (row), `output_dir` (run), `justification`, `dry_run` | | `data.erasure_completed` | Successful deletion finished. | All `requested` fields + `bytes_freed`, `files_modified`, `pre_erasure_line_number` (row mode), `match_count` (row mode) | | `data.erasure_failed` | Disk operation raised, OR no matching row/run, OR multi-row policy refused on ambiguity. | All `requested` fields + `error_class`, `error_message` | -| `data.erasure_warning_memorisation` | Row erasure × `final_model/` exists for any run that consumed this corpus. | All `completed` fields + `affected_run_ids` | +| `data.erasure_warning_memorisation` | Row erasure × a `final_model/` directory exists directly under `--output-dir` (literal name; `--output-dir` defaults to the corpus's directory, so pass your training output directory). ForgeLM does not determine which runs consumed the corpus. | All `completed` fields + `affected_run_ids` | | `data.erasure_warning_synthetic_data_present` | Row erasure × `synthetic_data*.jsonl` exists in `output_dir`. | All `completed` fields + `synthetic_files` | | `data.erasure_warning_external_copies` | Loaded config has a non-empty `webhook` block; downstream consumers may have received notices. | All `completed` fields + `webhook_targets` (redacted URLs) | diff --git a/docs/reference/soc2_trust_criteria_mapping-tr.md b/docs/reference/soc2_trust_criteria_mapping-tr.md index 07c43f47..be491d23 100644 --- a/docs/reference/soc2_trust_criteria_mapping-tr.md +++ b/docs/reference/soc2_trust_criteria_mapping-tr.md @@ -26,7 +26,7 @@ kategoriler engagement-bazında scoplanır. | CC | Başlık | ForgeLM kanıtı | |---|---|---| | CC1.1 | Bütünlüğe bağlılık göstergesi | `roles_responsibilities.md` QMS template; AI Officer rolü | -| CC1.2 | Yönetişim bağımsızlığı | Approval gate operator ≠ trainer | +| CC1.2 | Yönetişim bağımsızlığı | Approval gate onaylayıcı kimliğini kaydeder, böylece eğitici ≠ onaylayıcı denetlenebilir; ForgeLM bunu zorlamaz | | CC1.3 | Yapı, raporlama hatları kurma | `roles_responsibilities.md` AI Officer / ML Lead / DPO tanımlar | | CC1.4 | Yetkinliğe bağlılık göstergesi | Yıllık EU AI Act / GDPR eğitimi (operatör politikası) | | CC1.5 | Hesap verebilirliği uygular | `FORGELM_OPERATOR` attribution + audit chain | @@ -57,7 +57,7 @@ kategoriler engagement-bazında scoplanır. | CC7.5 | Düzeltici eylemleri tanımlar, geliştirir | `human_approval.rejected`; `sop_change_management.md` | | CC8.1 | Değişimleri yetkilendirir | `forgelm approve` Madde 14 gate; staging dir | | CC9.1 | Riskleri tanımlar, yönetir | `risk_assessment` config + safety eval; `risk_treatment_plan.md` | -| CC9.2 | Tedarikçi + iş ortağı riskini yönetir | SBOM; `loaded` / `unverified` / `local_path` / `unresolved` olarak derecelendirilen dataset Hub commit SHA'sı; temel-model pin'i (`model.revision` → `model_lineage.base_model_revision`); lisans çıkarımı. `model_lineage.component_revisions` altında **sabitlenen ve kaydedilen:** güvenlik sınıflandırıcısı, LLM judge, GRPO ödül modeli, sentetik teacher. **Kapsanmayan:** merge kaynakları ve `forgelm safety-eval` (`--config` almaz) | +| CC9.2 | Tedarikçi + iş ortağı riskini yönetir | SBOM; `loaded` / `unverified` / `local_path` / `unresolved` olarak derecelendirilen dataset Hub commit SHA'sı; temel-model pin'i (`model.revision` → `model_lineage.base_model_revision`). `model_lineage.component_revisions` altında **sabitlenen ve kaydedilen:** güvenlik sınıflandırıcısı, LLM judge, GRPO ödül modeli, sentetik teacher. **Kapsanmayan:** merge kaynakları ve `forgelm safety-eval` (`--config` almaz) | ## Availability (A1.x) diff --git a/docs/reference/soc2_trust_criteria_mapping.md b/docs/reference/soc2_trust_criteria_mapping.md index 57ce25ed..e2f0d09e 100644 --- a/docs/reference/soc2_trust_criteria_mapping.md +++ b/docs/reference/soc2_trust_criteria_mapping.md @@ -26,7 +26,7 @@ categories are scoped per-engagement. | CC | Title | ForgeLM evidence | |---|---|---| | CC1.1 | Demonstrates commitment to integrity | `roles_responsibilities.md` QMS template; AI Officer role | -| CC1.2 | Independence of governance | Approval gate operator ≠ trainer | +| CC1.2 | Independence of governance | The approval gate records the approver identity, so trainer ≠ approver is auditable; ForgeLM does not enforce it | | CC1.3 | Establishes structures, reporting lines | `roles_responsibilities.md` defines AI Officer / ML Lead / DPO | | CC1.4 | Demonstrates commitment to competence | Annual EU AI Act / GDPR training (deployer policy) | | CC1.5 | Enforces accountability | `FORGELM_OPERATOR` attribution + audit chain | @@ -57,7 +57,7 @@ categories are scoped per-engagement. | CC7.5 | Identifies, develops corrective actions | `human_approval.rejected`; `sop_change_management.md` | | CC8.1 | Authorises changes | `forgelm approve` Article 14 gate; staging dir | | CC9.1 | Identifies, manages risks | `risk_assessment` config + safety eval; `risk_treatment_plan.md` | -| CC9.2 | Manages vendor + business-partner risk | SBOM; dataset Hub commit SHA graded `loaded` / `unverified` / `local_path` / `unresolved`; base-model pin (`model.revision` → `model_lineage.base_model_revision`); license extraction. **Pinned and recorded** under `model_lineage.component_revisions`: safety classifier, LLM judge, GRPO reward model, synthetic teacher. **Not covered:** merge sources, and `forgelm safety-eval` (takes no `--config`) | +| CC9.2 | Manages vendor + business-partner risk | SBOM; dataset Hub commit SHA graded `loaded` / `unverified` / `local_path` / `unresolved`; base-model pin (`model.revision` → `model_lineage.base_model_revision`). **Pinned and recorded** under `model_lineage.component_revisions`: safety classifier, LLM judge, GRPO reward model, synthetic teacher. **Not covered:** merge sources, and `forgelm safety-eval` (takes no `--config`) | ## Availability (A1.x) diff --git a/docs/reference/supply_chain_security-tr.md b/docs/reference/supply_chain_security-tr.md index 0b74c5a2..bcc9d9fa 100644 --- a/docs/reference/supply_chain_security-tr.md +++ b/docs/reference/supply_chain_security-tr.md @@ -39,7 +39,17 @@ içinde çalışır: ``` Cross-OS test matrisinin her (OS × Python-version) hücresi için bir -SBOM. Artefaktlar GitHub release sayfasından indirilebilir. +SBOM. + +SBOM'lar, release tag'i için çalışan `publish.yml` çalıştırmasının **workflow-run artifact**'larıdır. +GitHub Release sayfasına eklenmezler ve GitHub, deponun saklama süresi (90 gün) dolunca run +artifact'larını siler; ihtiyacınız olanı çalıştırma hâlâ duruyorken indirin ve kayıtlarınızla +birlikte arşivleyin. + +Bir SBOM'un listelediği: matris runner'ında paketlenmiş wheel kurulduktan sonraki `pip list` +çıktısı (bağımlılıkları o anda PyPI'dan çözülür, sabitlenmemiştir), Linux'ta `qlora`, `ingestion` +ve `eval` extras'ı ve `pytest` / `pytest-cov`. Bu yüzden runtime kapanışının yanında test araçlarını +da adlandırır, bağımlılık grafiği taşımaz ve **lisans verisi taşımaz**. ### Determinism contract @@ -52,17 +62,21 @@ Wave 4 / Faz 23 bir determinism testi ekler - `tests/test_supply_chain_security.py::TestGenerateSbomDeterministic::test_two_invocations_produce_same_content` CI'da bunu pin'ler. -Bir denetçi karşılık gelen `git tag`'den SBOM'u yeniden emit edip -GitHub release'e ekli artefaktla diff alabilir; trivial olmayan -herhangi bir fark build ortamında dependency-resolution drift'ine -işaret eder. +Determinizm, aynı ortama özgü bir özelliktir. Aynı `git tag`'den sonradan yapılan yeniden +emit'in bir release'in SBOM'uyla eşleşmesi beklenmez; çünkü o release bağımlılıklarını +build edildiği anda PyPI'dan çözdü. **Sizin** dağıttığınızı kaydetmek için, modeli eğiten ya da +sunan ortamda bir ForgeLM checkout'undan `python3 tools/generate_sbom.py` çalıştırın. ### SBOM'u tüketme ```bash -# Bir release için tüm SBOM'ları indir. v'yi CI'da pinlediğiniz -# tag ile değiştirin; SBOM artefakt formatı release'ler arasında stabildir. -gh release download v --pattern 'sbom-*' +# CI'da pinlediğiniz tag için başarılı publish çalıştırmasını bulun (bir tag'in +# daha önce başarısız olmuş bir çalıştırması olabilir), sonra ondan bir SBOM indirin. +# v'yi tag ile değiştirin. Run artifact'ları 90 günde silinir; on ikisi için +# --pattern 'sbom-*' kullanın. +gh run list --repo HodeTech/ForgeLM --workflow publish.yml --branch v --status success \ + --json databaseId,createdAt +gh run download --repo HodeTech/ForgeLM --name sbom-ubuntu-latest-py3.11 # İnsan inceleme için pretty-print. jq . sbom-ubuntu-latest-py3.11.json | less @@ -110,7 +124,7 @@ Wave 4 / Faz 23 nightly workflow'a `pip-audit` ekler. Davranış: Belirli bir CVE'yi kabul etmek için opt-in ignore dosyasıyla dokümante edin (aşağıdaki Suppression'a bakın); yeşil kalmak için UNKNOWN kovasına güvenmeyin. -- OSV / GHSA veritabanlarını kullanır (pip-audit varsayılanı). +- PyPI'ın güvenlik açığı verisini sorgular (pip-audit'in varsayılan `pypi` servisi; nightly `--vulnerability-service` geçmez). Operatörler aynı tooling'i lokalde kurar: @@ -151,7 +165,7 @@ bir [`tools/pip_audit_ignores.yaml`](../../tools/pip_audit_ignores.yaml) taşır (şu an dokuz no-fix torch advisory'si ve bir OSV-yanlış-sınıflandırılmış markdown kaydı; daha önce taşıdığı iki transformers advisory'si `transformers>=5.3.0` floor'u onları pip-audit -tarafından raporlanamaz hâle getirince kaldırıldı). Bu dosya yalnız projenin kendi workflow'u tarafından +tarafından raporlanamaz hâle getirince kaldırıldı; floor artık `>=5.10.1`). Bu dosya yalnız projenin kendi workflow'u tarafından `--ignores` ile tüketilir; deployer'lar miras almaz. Her release cycle'da gözden geçirilir. diff --git a/docs/reference/supply_chain_security.md b/docs/reference/supply_chain_security.md index 13cac6c0..6aba4546 100644 --- a/docs/reference/supply_chain_security.md +++ b/docs/reference/supply_chain_security.md @@ -39,7 +39,16 @@ publish workflow: ``` One SBOM per (OS × Python-version) cell of the cross-OS test matrix. -The artefacts are downloadable from the GitHub release page. + +The SBOMs are **workflow-run artifacts** of the `publish.yml` run for the release tag. They are +not attached to the GitHub Release page, and GitHub deletes run artifacts when the repository's +retention period ends (90 days), so download what you need while the run is still available and +archive it with your records. + +What an SBOM lists: the output of `pip list` on the matrix runner after it installed the packaged +wheel (its dependencies resolve from PyPI at that moment, unpinned), on Linux the `qlora`, +`ingestion` and `eval` extras, and `pytest` / `pytest-cov`. It therefore names test tooling +beside the runtime closure, carries no dependency graph, and carries **no licence data**. ### Determinism contract @@ -52,17 +61,20 @@ Wave 4 / Faz 23 adds a determinism test (`tests/test_supply_chain_security.py`): - `tests/test_supply_chain_security.py::TestGenerateSbomDeterministic::test_two_invocations_produce_same_content` pins this in CI. -An auditor can re-emit the SBOM from the corresponding `git tag` -and diff against the artefact attached to the GitHub release; any -non-trivial difference indicates dependency-resolution drift in the -build environment. +Determinism is a same-environment property. A later re-emission from the same `git tag` is not +expected to match a release's SBOM, because that release resolved its dependencies from PyPI when +it was built. To record what **you** deploy, run `python3 tools/generate_sbom.py` from a ForgeLM +checkout in the environment that trains or serves the model. ### Consuming the SBOM ```bash -# Download all SBOMs for a release. Replace v with the tag you -# pinned in your CI; the SBOM artefact format is stable across releases. -gh release download v --pattern 'sbom-*' +# Find the successful publish run for the tag you pinned in your CI (a tag can have an +# earlier failed run), then download one SBOM from it. Replace v with the tag. +# Run artifacts expire after 90 days; use --pattern 'sbom-*' for all twelve. +gh run list --repo HodeTech/ForgeLM --workflow publish.yml --branch v --status success \ + --json databaseId,createdAt +gh run download --repo HodeTech/ForgeLM --name sbom-ubuntu-latest-py3.11 # Pretty-print for human review. jq . sbom-ubuntu-latest-py3.11.json | less @@ -108,7 +120,7 @@ Wave 4 / Faz 23 adds `pip-audit` to the nightly workflow. Behaviour: To accept a specific CVE, document it via the opt-in ignore file (see Suppression below); do not rely on the UNKNOWN bucket to stay green. -- Uses the OSV / GHSA databases (pip-audit's default). +- Queries PyPI's vulnerability data (pip-audit's default `pypi` service; the nightly does not pass `--vulnerability-service`). Operators install the same tooling locally: @@ -149,7 +161,7 @@ ForgeLM's own nightly does carry a checked-in project-internal triage (currently nine no-fix torch advisories and one OSV-misclassified markdown record; the two transformers advisories it once carried were dropped once the `transformers>=5.3.0` floor made them -unreportable by pip-audit). That file is consumed only by the +unreportable by pip-audit; the floor is now `>=5.10.1`). That file is consumed only by the project's own workflow via `--ignores`; deployers do not inherit it. Review at every release cycle. diff --git a/docs/reference/usage-tr.md b/docs/reference/usage-tr.md index 9ebfa152..7bd9fa34 100644 --- a/docs/reference/usage-tr.md +++ b/docs/reference/usage-tr.md @@ -128,7 +128,8 @@ forgelm chat ./checkpoints/final_model --adapter ./adapter forgelm chat ./checkpoints/final_model --system "Yardımcı bir asistansın." --temperature 0.8 # GGUF'a export (Ollama, LM Studio, llama.cpp için) -# Gerekli: pip install forgelm[export] +# Gerekli: pip install forgelm[export] VE FORGELM_GGUF_CONVERTER'ın bir llama.cpp checkout'undaki +# convert_hf_to_gguf.py'ye işaret etmesi (hiçbir llama-cpp-python wheel'i onu ForgeLM'in baktığı yere koymaz) forgelm export ./checkpoints/final_model --output model.gguf --quant q4_k_m forgelm export ./checkpoints/final_model --output model.gguf --quant q8_0 --adapter ./adapter diff --git a/docs/reference/usage.md b/docs/reference/usage.md index 5676a080..4b9f1187 100644 --- a/docs/reference/usage.md +++ b/docs/reference/usage.md @@ -145,7 +145,8 @@ forgelm chat ./checkpoints/final_model --adapter ./adapter forgelm chat ./checkpoints/final_model --system "You are a helpful assistant." --temperature 0.8 # Export to GGUF (for Ollama, LM Studio, llama.cpp) -# Requires: pip install forgelm[export] +# Requires: pip install forgelm[export] AND FORGELM_GGUF_CONVERTER pointing at convert_hf_to_gguf.py +# from a llama.cpp checkout (no llama-cpp-python wheel installs it where ForgeLM looks) forgelm export ./checkpoints/final_model --output model.gguf --quant q4_k_m forgelm export ./checkpoints/final_model --output model.gguf --quant q8_0 --adapter ./adapter diff --git a/docs/reference/verify_gguf_subcommand-tr.md b/docs/reference/verify_gguf_subcommand-tr.md index 0b092908..f53fe5dd 100644 --- a/docs/reference/verify_gguf_subcommand-tr.md +++ b/docs/reference/verify_gguf_subcommand-tr.md @@ -43,7 +43,7 @@ Kodlar `forgelm/cli/subcommands/_verify_gguf.py::_run_verify_gguf_cmd` tarafınd | **Meta veri bloğu** | İsteğe bağlı `gguf` paketi yüklüyse. Üst kaynak okuyucu ile meta veri + tensor tanımlarını ayrıştırır. | Okuyucu ayrıştırma sırasında istisna fırlatır → çıkış kodu sidecar'ın ne kanıtladığına bağlıdır; çünkü ayrıştırma hatası tek başına bozuk bir dosyayı, bu dosyanın format revizyonunu okuyamayacak kadar eski bir `gguf` paketinden ayırt edemez. **Sidecar yok** → çıkış `6` (bozulmayı eleyecek hiçbir şey yok; dosya kesilmiş / hasarlı sayılmalı). **Eşleşen sidecar** → çıkış `1` (dosya export edilenle bayt-bayt aynı — `gguf`'u yükseltip yeniden çalıştırın; artefakt sahibini çağırmayın). **Uyuşmayan sidecar** → çıkış `6`, sidecar uyuşmazlığı olarak raporlanır (checksum daha güçlü kanıttır ve baskın gelir). Paket yoksa → kontrol atlanır (magic + sidecar kontrolleri yük taşır). | | **SHA-256 sidecar** | `.sha256` mevcutsa. Dosyanın SHA-256'sını yeniden hesaplar ve sidecar'ın ilk boşluk-ayrılı token'ı ile karşılaştırır (sha256sum formatı ` *` desteklenir). | İyi-biçimli ama uyuşmayan digest → çıkış `6` (dosya export sonrası değişmiş). Sidecar mevcut ama içeriği 64 karakterlik hex digest değilse → çıkış `1` (bozuk-sidecar maskelenmesine karşı kapalı başarısızlık — hiçbir şey karşılaştırılmadı). Sidecar yoksa → kontrol sessizce atlanır. | -Exporter sidecar'ı varsayılan olarak yazar (bkz. [`docs/usermanuals/tr/deployment/gguf-export.md`](../usermanuals/tr/deployment/gguf-export.md)); GGUF dosyalarını üçüncü taraflardan alan operatörler sidecar'ı da talep etmelidir. +Exporter sidecar'ı varsayılan olarak yazar (K-quant isteğinde sidecar ara f16 dosyasına aittir; bkz. [`docs/usermanuals/tr/deployment/gguf-export.md`](../usermanuals/tr/deployment/gguf-export.md)); GGUF dosyalarını üçüncü taraflardan alan operatörler sidecar'ı da talep etmelidir. ## Emit edilen audit event'leri @@ -142,7 +142,7 @@ Aynı dosyanın yanında **hiç** sidecar yoksa karar bunun yerine `6` olur — ### İsteğe bağlı bağımlılık yok -`gguf` paketi yüklü değilse meta veri ayrıştırma katmanı sessizce atlanır — magic + sidecar kontrolleri yük taşımaya devam eder: +`gguf` paketi yüklü değilse meta veri ayrıştırma katmanı atlanır (`metadata_parsed: False`). O zaman denetlenen yalnızca magic ve sidecar'dır; sidecar da yoksa temiz bir çıkış yalnızca magic header'ı kanıtlar: ```shell $ pip uninstall -y gguf diff --git a/docs/reference/verify_gguf_subcommand.md b/docs/reference/verify_gguf_subcommand.md index c91506ca..55a970cf 100644 --- a/docs/reference/verify_gguf_subcommand.md +++ b/docs/reference/verify_gguf_subcommand.md @@ -43,7 +43,7 @@ The codes are emitted by `forgelm/cli/subcommands/_verify_gguf.py::_run_verify_g | **Metadata block** | When the optional `gguf` package is installed. Parses the metadata + tensor descriptors via the upstream reader. | Reader raises mid-parse → the exit code depends on what the sidecar can prove, because the parse error alone cannot tell a corrupted file apart from a `gguf` package too old for this file's format revision. **No sidecar** → exit `6` (nothing available to rule out corruption; treat the file as truncated / damaged). **Matching sidecar** → exit `1` (the file is byte-identical to what was exported — upgrade `gguf` and re-run; do not page the artifact owner). **Mismatching sidecar** → exit `6`, reported as the sidecar mismatch (the checksum is the stronger evidence and dominates). Package absent → check is skipped (the magic + sidecar checks remain load-bearing). | | **SHA-256 sidecar** | When `.sha256` exists. Recomputes file SHA-256 and compares against the sidecar's first whitespace-separated token (sha256sum format ` *` is supported). | Well-formed digest that disagrees → exit `6` (file changed after export). Sidecar present but contents are not a 64-character hex digest → exit `1` (fail closed against malformed-sidecar masquerade — nothing was compared). Sidecar absent → check is skipped silently. | -The exporter writes the sidecar by default (see [`docs/usermanuals/en/deployment/gguf-export.md`](../usermanuals/en/deployment/gguf-export.md)); operators receiving GGUF files from third parties should request the sidecar alongside. +The exporter writes the sidecar by default (for a K-quant request it belongs to the intermediate f16 file; see [`docs/usermanuals/en/deployment/gguf-export.md`](../usermanuals/en/deployment/gguf-export.md)); operators receiving GGUF files from third parties should request the sidecar alongside. ## Audit events emitted @@ -142,7 +142,7 @@ With **no** sidecar beside the same file the verdict is `6` instead — nothing ### Optional dependency absent -When the `gguf` package is not installed, the metadata-parse layer is skipped silently — the magic + sidecar checks remain load-bearing: +When the `gguf` package is not installed, the metadata-parse layer is skipped (`metadata_parsed: False`). Magic and sidecar are then all that is checked, and with no sidecar either, a clean exit proves only the magic header: ```shell $ pip uninstall -y gguf diff --git a/docs/roadmap/completed-phases.md b/docs/roadmap/completed-phases.md index f266a662..99679f0c 100644 --- a/docs/roadmap/completed-phases.md +++ b/docs/roadmap/completed-phases.md @@ -1,6 +1,8 @@ # Completed Phases — Archive > **Note:** Historical archive of ForgeLM's Phase 1-10 completed work. Kept for reference and onboarding. For active planning, see [../roadmap.md](../roadmap.md). +> +> **Dates.** Where this archive says the EU AI Act is "fully enforced" in "August 2026", that was the planning assumption when the phase was written and is kept as the record. The application dates have since changed and differ by obligation; the current statement is in [../product_strategy.md](../product_strategy.md), and there is no single full-enforcement date. ## Summary table @@ -661,18 +663,18 @@ graph TD ``` 2. [x] **`forgelm chat` — interactive terminal loop** - Terminal REPL in `forgelm/chat.py`: streaming output (default), `/reset`, `/save [file]`, `/temperature 0.x`, `/system `, `/help` commands. `rich` optional rendering. History management with 50-pair cap. The `--safety` per-turn screen flag is planned for v0.6.0+ Pro CLI (today's safety pipeline runs through the YAML `safety:` block during training/eval). + Terminal REPL in `forgelm/chat.py`: streaming output (default), `/reset`, `/save [file]`, `/temperature 0.x`, `/system `, `/help` commands. `rich` optional rendering. History management with 50-pair cap. `forgelm chat` performs no safety screening and reads no YAML config; the `--safety` per-turn screen flag is not implemented. Screen a model with `forgelm safety-eval --model --default-probes`, or with the `evaluation.safety` block during training, before exposing it. ```bash forgelm chat ./outputs/my_run ``` `forgelm chat`'s shipping form takes a positional checkpoint path - plus optional `--adapter`. The per-turn Llama Guard screen via - `--safety` is planned for v0.6.0+ Pro CLI (see Phase 13 roadmap); - today operators get the same screening by enabling - `safety: enabled: true` in the YAML config the chat REPL reads. - The flag-form preview (`forgelm chat ... --safety`) is NOT - runnable today. + plus optional `--adapter`. It performs no safety screening and does + not read a YAML config: a per-turn Llama Guard screen via `--safety` + is not implemented, and the flag-form preview + (`forgelm chat ... --safety`) is NOT runnable. Screen a model with + `forgelm safety-eval --model --default-probes` (or the + `evaluation.safety` block during training) before exposing it. 3. [x] **`forgelm export` — HF → GGUF conversion** Wraps `llama-cpp-python`'s `convert_hf_to_gguf.py`; no reimplementation. Handles adapter merge (LoRA + base → merged fp16) before conversion. Supports quants: `q2_k`, `q3_k_m`, `q4_k_m`, `q5_k_m`, `q8_0`, `f16`. SHA-256 of output artifact appended to `model_integrity.json`. Optional dependency: `pip install forgelm[export]`. diff --git a/docs/roadmap/risks-and-decisions.md b/docs/roadmap/risks-and-decisions.md index 39b42bd0..e67a0ed4 100644 --- a/docs/roadmap/risks-and-decisions.md +++ b/docs/roadmap/risks-and-decisions.md @@ -8,10 +8,10 @@ | Risk | Impact | Likelihood | Mitigation | |------|--------|------------|------------| | **Dependency Breaking Changes** (TRL, PEFT, Unsloth) | Training pipeline breaks without warning | High | Version pinning with upper bounds, CI nightly builds against latest deps, compatibility matrix | -| **EU AI Act Non-Compliance** (August 2026 deadline) | Enterprise customers cannot adopt ForgeLM for high-risk AI | High | Phase 8 deep compliance: Annex IV docs, audit log, risk assessment, human gate, data governance | +| **EU AI Act Non-Compliance** (Chapter III, Sections 1–3, except Article 6(5), apply from 2 December 2027 to high-risk AI systems under Article 6(2) and Annex III, and from 2 August 2028 to systems under Article 6(1) and Annex I; see [product_strategy.md](../product_strategy.md)) | Enterprise customers cannot adopt ForgeLM for high-risk AI | High | Phase 8 deep compliance: Annex IV docs, audit log, risk assessment, human gate, data governance | | **Safety Degradation from Fine-Tuning** | Fine-tuned models lose alignment, enterprise liability | High | Phase 6 safety evaluation pipeline, auto-revert on safety regression | | **Alignment Method Lock-In** | ForgeLM supports only ORPO while market demands DPO/GRPO | High | Phase 5 is top priority — critical market expectation | -| **Compliance window closing** (Aug 2, 2026) | Enterprise leads not converted before competitors add compliance features | High | Enterprise outreach Day 1; Turkey market as first pilot; deep compliance features (Audit API, managed cloud) as moat | +| **Compliance window** (Annex III from 2 December 2027, Annex I from 2 August 2028) | Enterprise leads not converted before competitors add compliance features | High | Enterprise outreach Day 1; Turkey market as first pilot; deep compliance features (Audit API, managed cloud) as moat | | **Community flywheel not started** | 0 stars → 0 enterprise credibility → missed EU AI Act window | High | Phase 10.5 Quickstart prioritized; YouTube Academy; Show HN + Reddit launch | ### Medium Severity @@ -39,7 +39,7 @@ 2. **Safety-as-a-Feature (Phase 6)** — No competitor integrates safety evaluation into the training pipeline. ForgeLM can own "the safest way to fine-tune an LLM." ### Medium-Term Opportunities -3. **EU AI Act Compliance** — August 2026 deadline creates urgent demand. ForgeLM as the only tool generating compliance artifacts is a powerful enterprise sales argument. +3. **EU AI Act Compliance** — the staggered high-risk application dates (2 December 2027 for Annex III, 2 August 2028 for Annex I) create demand. ForgeLM as the only tool generating compliance artifacts is a powerful enterprise sales argument. 4. **MoE Fine-Tuning** — Qwen3, DeepSeek-V3 dominance means MoE support is becoming table stakes. 5. **Cost Transparency** — GPU cost tracking per run enables enterprise budget planning and optimization. Simple to implement, high perceived value. diff --git a/docs/standards/error-handling.md b/docs/standards/error-handling.md index 1a8860f5..97a645b6 100644 --- a/docs/standards/error-handling.md +++ b/docs/standards/error-handling.md @@ -19,7 +19,7 @@ EXIT_INTEGRITY_FAILURE = 6 | Code | When | Who reads it | |---|---|---| -| **0** | Training completed. With `auto_revert: true` (the compliance default for high-risk tiers) this also means every gate passed; with the **shipped default** `auto_revert: false` a failed benchmark/safety/judge gate is *recorded* (`benchmark`/`safety`/`judge` block in the JSON, `*_passed: false`) but does **not** change the exit code — parse those blocks, don't trust exit 0 alone | CI/CD success | +| **0** | Training completed. With `auto_revert: true` (the compliance default for high-risk tiers) this also means every gate that ran to a verdict passed (a safety evaluation that produced no verdict — `safety.evaluation_completed: false` — keeps the model and exits 0 regardless of `auto_revert`); with the **shipped default** `auto_revert: false` a failed benchmark/safety/judge gate is *recorded* (`benchmark`/`safety`/`judge` block in the JSON, `*_passed: false`) but does **not** change the exit code — parse those blocks, don't trust exit 0 alone | CI/CD success | | **1** | Config validation failed (YAML schema, Pydantic error). Also used by the `verify-*` subcommands for an operator-actionable input error — the artefact could not be read at all, so nothing was ever compared (plus the one case where everything comparable *was* compared and came out clean; see the 1-vs-6 note below) | CI/CD "fail fast"; user fixes YAML or the verifier's target path | | **2** | Training crashed or failed mid-run (OOM, CUDA error, unhandled exception). Also the `verify-*` subcommands' code for a genuine runtime I/O failure on a reachable path (permission denied mid-read, disk error) | CI/CD retry logic | | **3** | Training completed but eval/safety/benchmark threshold failed, and auto-revert happened | CI/CD decision: do not deploy | diff --git a/docs/usermanuals/en/compliance/audit-log.md b/docs/usermanuals/en/compliance/audit-log.md index faddaaa7..09125004 100644 --- a/docs/usermanuals/en/compliance/audit-log.md +++ b/docs/usermanuals/en/compliance/audit-log.md @@ -5,7 +5,7 @@ description: Append-only event log over training, evaluation, and revert decisio # Audit Log -EU AI Act Article 12 requires high-risk AI systems to maintain logs of operationally relevant events. ForgeLM's `audit_log.jsonl` is an append-only, SHA-256-anchored sequence of events covering training start, evaluation gates, auto-revert decisions, and model export. +EU AI Act Article 12 requires high-risk AI systems to maintain logs of operationally relevant events. ForgeLM's `audit_log.jsonl` is an append-only, SHA-256-anchored sequence of events covering training start, evaluation gates, auto-revert decisions, and model export. It records the training pipeline, which is development-time evidence that can support your technical documentation; the logging Article 12 asks for is a capability of the deployed system, which ForgeLM does not provide. ## Format diff --git a/docs/usermanuals/en/compliance/gdpr-erasure.md b/docs/usermanuals/en/compliance/gdpr-erasure.md index 6cf43cb9..359a8561 100644 --- a/docs/usermanuals/en/compliance/gdpr-erasure.md +++ b/docs/usermanuals/en/compliance/gdpr-erasure.md @@ -42,7 +42,7 @@ Six events ride the audit chain (see the [Audit Event Catalog on GitHub](https:/ - `data.erasure_requested` — emitted FIRST, before any deletion. - `data.erasure_completed` — emitted LAST after the disk operation succeeded. - `data.erasure_failed` — emitted instead when the disk operation raised, or no matching row/run was found. -- `data.erasure_warning_memorisation` — corpus-row erasure × `final_model/` exists for any run that consumed this corpus (the row is gone from disk but may still be memorised in the trained weights). +- `data.erasure_warning_memorisation` — corpus-row erasure × a `final_model/` exists directly under `--output-dir` (the row is gone from disk but may still be memorised in the trained weights). - `data.erasure_warning_synthetic_data_present` — corpus-row erasure × `synthetic_data*.jsonl` exists in `output_dir`. - `data.erasure_warning_external_copies` — loaded config has a non-empty `webhook` block; downstream consumers may have received notices. @@ -110,7 +110,7 @@ Codes 3 (`EXIT_EVAL_FAILURE`) and 4 (`EXIT_AWAITING_APPROVAL`) are not part of e ::: :::warn -**Running `forgelm purge` without checking for memorisation.** If a `final_model/` exists for any run that consumed the corpus, the row is gone from disk but may still be encoded in the weights. The `data.erasure_warning_memorisation` event surfaces the gap; act on it (re-train without the erased row, or document the residual risk in your DPIA). +**Running `forgelm purge` without checking for memorisation.** If a trained model exists, the row is gone from disk but may still be encoded in the weights. The `data.erasure_warning_memorisation` event surfaces the gap only when a `final_model/` sits directly under `--output-dir` — which defaults to the corpus's directory, so pass your training output directory; act on it (re-train without the erased row, or document the residual risk in your DPIA). ::: :::tip diff --git a/docs/usermanuals/en/compliance/gdpr.md b/docs/usermanuals/en/compliance/gdpr.md index d8783dbe..d0f37a87 100644 --- a/docs/usermanuals/en/compliance/gdpr.md +++ b/docs/usermanuals/en/compliance/gdpr.md @@ -5,7 +5,7 @@ description: Data protection compliance — PII minimisation, data subject right # GDPR / KVKK -GDPR (Europe) and KVKK (Türkiye) impose data protection requirements on personal data used in training. ForgeLM's role: prevent personal data from entering training in the first place, document what gets ingested, and produce evidence for data subject requests. +GDPR (Europe) and KVKK (Türkiye) impose data protection requirements on personal data used in training. ForgeLM's role is narrower than keeping personal data out of training: it masks the identifier formats it can detect, reports what an audit found, and gives you commands for access and erasure requests. It does not decide your lawful basis, it does not find names or other free-text personal data, and it does not make a trained model anonymous. ## What GDPR / KVKK require for training data @@ -25,40 +25,35 @@ KVKK (Türkiye) mirrors these principles closely. ### Data minimisation (Art. 5(1)(c)) -PII masking at ingest replaces personal identifiers with placeholders: +PII masking at ingest replaces the identifier formats ForgeLM detects with a placeholder: -```yaml -ingestion: - pii_mask: - enabled: true - locale: "tr" - categories: ["email", "phone", "iban", "id_tr"] +```shell +$ forgelm ingest ./policies/ --recursive --strategy markdown --pii-mask --output data/policies.jsonl ``` -By the time data lands in JSONL, identifiable subjects are removed. See [PII Masking](#/data/pii-masking). +The detectors cover email, phone, credit card, IBAN, and the Turkish, German, French and US national-ID formats; every match becomes `[REDACTED]`. **Names, addresses, dates of birth, health data and other free text are not detected**, and the patterns are tuned to miss a borderline match rather than redact ordinary prose. Masking reduces the personal data in the JSONL; it does not remove identifiable subjects from it. See [PII Masking](#/data/pii-masking). ### Accountability (Art. 5(2)) -Every audit produces `data_audit_report.json` documenting: +Every `forgelm audit` run writes `data_audit_report.json`. It records: -- Detected PII categories and counts (before masking). -- Source attribution (which document each row came from). -- Quality and language distribution. -- SHA-256 manifest for tamper-evidence. +- Detected PII and secret categories with their counts, for the data you audited. Audit the unmasked source to see what a masker has to catch; audit the masked output to see what got through. +- Near-duplicate, quality and cross-split-overlap summaries, and an optional Croissant card. +- The audit's own metadata: when it ran, and the path and literal input it was given. -These reports flow into the Annex IV bundle. When a regulator asks "what personal data was in your training set?", you have a structured answer. +The report does not attribute rows to source documents, carries no hash manifest, and is a plain JSON file: it is not itself written to the audit log. Erasure and access requests (`forgelm purge`, `forgelm reverse-pii`) are, in the hash-chained log that `forgelm verify-audit` checks. When the report sits in `training.output_dir`, it feeds the Article 10 data-governance section of the Annex IV bundle, so a regulator's "what personal data was in your training set?" has a structured answer: what the detectors found, not a statement that none exists. ### Storage limitation (Art. 5(1)(e)) -ForgeLM doesn't retain raw user data — it produces JSONL artifacts you control. For automated retention enforcement: +ForgeLM doesn't retain raw user data — it produces JSONL artifacts you control. Retention horizons can be declared and are **reported, not enforced**: ```yaml retention: - raw_documents_retention_days: 90 # auto-delete originals after N days + raw_documents_retention_days: 90 ephemeral_artefact_retention_days: 365 ``` -(The actual deletion is your storage layer's responsibility; ForgeLM just records the intended TTL in the audit log.) +`forgelm purge --check-policy --config config.yaml` lists artefacts that are past their horizon and always exits 0. Nothing is deleted automatically and nothing is written to the audit log for it; deleting what is overdue is your storage layer's job. (`retention.enforce` is reserved: no command acts on it yet.) ## Data subject requests @@ -68,30 +63,57 @@ Most-common request types and how ForgeLM helps: "What personal data do you hold about me?" -Run reverse-PII on your training data: +Search your training data for the person's identifier: ```shell $ forgelm reverse-pii --query "ali@example.com" data/*.jsonl -No matches found in masked data. ``` -Because PII was masked at ingest, no specific person's data is recoverable from the model. The audit report confirms this. +The scan reports the lines where the identifier appears; zero matches is a valid answer. It finds what is literally in the files you point it at. Masking only removes the identifier formats it detects, so a name or an undetected variant can remain, and the raw source documents are not covered by a scan of the masked JSONL: search those separately. ForgeLM cannot tell you what a trained model remembers. ### Right to erasure (Art. 17) "Delete my data." If a person's data was in your training set: -1. The masked JSONL doesn't contain their identifying info — already minimised. -2. Source documents may still — drop them from your raw store and re-ingest if necessary. +1. Find the rows with `forgelm reverse-pii`, then remove them with `forgelm purge --row-id --corpus `; every erasure is written to the audit log. `--row-id` is matched against the row's `id` or `row_id` field and never against a line number, and `forgelm ingest` does not write either field, so give every row a stable id **before you train** (see below). A corpus without ids cannot be purged row by row. +2. Source documents may still hold it — drop them from your raw store and re-ingest if necessary. 3. The model may have memorised some details — see "model-level erasure" below. +To make rows addressable, give each JSONL row a unique `id` once, when the corpus is created, and keep it when the file is later filtered or re-ordered. Rows with a missing, null or empty `id` get a new one; a row whose `id` (or `row_id`, which `purge` also matches) another row already uses stops the script, because `forgelm purge --row-id` would see both rows as one match (it refuses by default, and with `--row-matches all` it would erase every match) and you have to decide which row keeps the id. The output is written to a temporary file and moved into place only after every row passed, so a failed run leaves an existing output file untouched: + +```python +import json, os, uuid + +SRC, DST = "data/train.jsonl", "data/train.with-ids.jsonl" +tmp, seen = DST + ".tmp", set() +try: + with open(SRC, encoding="utf-8") as src, open(tmp, "w", encoding="utf-8") as dst: + for line_no, line in enumerate(src, start=1): + if not line.strip(): + continue + row = json.loads(line) + if row.get("id") in (None, ""): + row["id"] = str(uuid.uuid4()) + ids = {str(row[key]) for key in ("id", "row_id") if row.get(key) not in (None, "")} + if ids & seen: + raise SystemExit(f"line {line_no}: id {sorted(ids & seen)[0]!r} is already used by another row") + seen |= ids + dst.write(json.dumps(row, ensure_ascii=False) + "\n") + os.replace(tmp, DST) # reached only when every row passed +finally: + if os.path.exists(tmp): + os.remove(tmp) +``` + +Train from the file with ids. `forgelm reverse-pii` reports the file and line where the identifier occurs; read that row's `id` from the file and pass it to `forgelm purge --row-id`. Re-assigning ids after training breaks the link between the audit log and the corpus, so assign once. + ### Model-level erasure LLMs can memorise rare strings from training. Even with PII masking, removing all traces from a deployed model is hard. ForgeLM's defences: -- **Prevent memorisation:** PII masking, deduplication (memorised data is usually duplicated data). -- **Detect memorisation:** the audit step flags rows that overlap with known PII patterns. +- **Reduce what goes in:** PII masking and deduplication (memorised data is usually duplicated data). +- **Memorisation is not measured:** ForgeLM does not test whether a trained model reproduces a given string. - **Re-train as last resort:** if a specific subject's data leaked despite masking, re-train without that source. For zero-tolerance scenarios (medical, legal), pair PII masking with manual review before training. @@ -122,7 +144,7 @@ ForgeLM has no YAML surface for GDPR legal metadata (lawful basis, data controll ::: :::warn -**International transfers.** If your training data crosses jurisdiction boundaries (Turkish data trained in EU, EU data trained in US), additional safeguards apply. Set `international_transfers.enabled: true` and document the safeguards. +**International transfers.** If your training data crosses jurisdiction boundaries (Turkish data trained in EU, EU data trained in US), additional safeguards apply. ForgeLM has no setting for transfers; record the safeguards in your DPIA. ::: :::tip diff --git a/docs/usermanuals/en/compliance/human-oversight.md b/docs/usermanuals/en/compliance/human-oversight.md index 373f7db2..48fa2f2a 100644 --- a/docs/usermanuals/en/compliance/human-oversight.md +++ b/docs/usermanuals/en/compliance/human-oversight.md @@ -5,7 +5,7 @@ description: Block model promotion until a human reviews and signs off — Artic # Human Oversight -EU AI Act Article 14 requires high-risk AI systems to provide for human oversight. ForgeLM implements this as an optional config gate: when `evaluation.require_human_approval: true`, model promotion blocks until a human signs an approval. +EU AI Act Article 14 requires high-risk AI systems to provide for human oversight. ForgeLM offers an optional pre-promotion approval gate (`evaluation.require_human_approval: true`) that blocks model promotion until a human signs an approval and records that sign-off. This supports design-time oversight documentation; it does not implement the in-use oversight measures Article 14 requires of the deployed system (monitoring, interpreting, overriding or stopping it), which remain the deployer's. ## How the gate works @@ -48,12 +48,8 @@ There is **no** `compliance.human_approval` field, no `approval.*` block, no `si The trainer halts after eval and prints: ```text -[2026-04-29 14:33:10] Human approval required. - Run ID: abc123 - Bundle: checkpoints/run/artifacts/ - - To approve: forgelm approve abc123 --output-dir checkpoints/run --comment "..." - To reject: forgelm reject abc123 --output-dir checkpoints/run --comment "..." +2026-04-29 14:33:10 [INFO] forgelm.trainer: Human approval required. Model staged at: checkpoints/run/final_model.staging.abc123 +2026-04-29 14:33:10 [INFO] forgelm.trainer: Review results in checkpoints/run/compliance/ and run `forgelm approve abc123 --output-dir checkpoints/run` to promote, or `forgelm reject abc123 --output-dir checkpoints/run` to preserve for forensic review. ``` The reviewer runs the approval command from any machine with access to the audit-log + staging directory. ForgeLM resolves their identity from `FORGELM_OPERATOR`, HMAC-chains the decision event, and (on `approve`) renames the staging directory to the canonical `final_model/` path. diff --git a/docs/usermanuals/en/concepts/alignment-overview.md b/docs/usermanuals/en/concepts/alignment-overview.md index 7c5d123a..fc3bf89f 100644 --- a/docs/usermanuals/en/concepts/alignment-overview.md +++ b/docs/usermanuals/en/concepts/alignment-overview.md @@ -53,7 +53,7 @@ After SFT, the model produces well-formatted outputs but quality varies — some The differences: - **DPO** — learns from `(chosen, rejected)` pairs against a reference model. Best-studied, most stable. - **SimPO** — same as DPO but skips the reference model. Lower memory, slightly less stable. -- **KTO** — learns from binary `(prompt, response, good?)` signals. Use when you only have thumbs-up/down feedback. +- **KTO** — learns from binary `(prompt, completion, label)` rows, where `label` is `true` for a desirable response and `false` for an undesirable one. Use when you only have thumbs-up/down feedback. - **ORPO** — combines SFT and preference loss in a single pass. Faster wall-clock; less flexible than separate stages. ### Problem 3: Reasoning diff --git a/docs/usermanuals/en/concepts/choosing-trainer.md b/docs/usermanuals/en/concepts/choosing-trainer.md index 16d8b971..801b7a03 100644 --- a/docs/usermanuals/en/concepts/choosing-trainer.md +++ b/docs/usermanuals/en/concepts/choosing-trainer.md @@ -48,7 +48,7 @@ flowchart TD |---|---| | `{prompt, completion}` rows only | [SFT](#/training/sft) | | `{prompt, chosen, rejected}` rows | [DPO](#/training/dpo) (or [SimPO](#/training/simpo) if VRAM-tight) | -| `{prompt, response, label: bool}` rows | [KTO](#/training/kto) | +| `{prompt, completion, label: bool}` rows | [KTO](#/training/kto) | | Mix of SFT and preference rows | [SFT](#/training/sft) → [DPO](#/training/dpo) sequence, or [ORPO](#/training/orpo) for one-stage | | Programmatic reward (math grader, code tests) | [GRPO](#/training/grpo) | diff --git a/docs/usermanuals/en/deployment/gguf-export.md b/docs/usermanuals/en/deployment/gguf-export.md index 0303839f..a18f8c2f 100644 --- a/docs/usermanuals/en/deployment/gguf-export.md +++ b/docs/usermanuals/en/deployment/gguf-export.md @@ -5,21 +5,20 @@ description: Quantise to GGUF for local inference with llama.cpp, Ollama, or LM # GGUF Export -GGUF (GPT-Generated Unified Format) is the de facto file format for local CPU/GPU inference via `llama.cpp`. ForgeLM exports any fine-tuned checkpoint to GGUF with six quantisation levels and a SHA-256 manifest. +GGUF (GPT-Generated Unified Format) is the de facto file format for local CPU/GPU inference via `llama.cpp`. ForgeLM exports a fine-tuned checkpoint to GGUF and writes a SHA-256 sidecar next to the file. `f16` and `q8_0` come out of a single command; the K-quant levels (`q2_k`, `q3_k_m`, `q4_k_m`, `q5_k_m`) need one more `llama-quantize` step, described below. ## Quick example ```shell $ forgelm export ./checkpoints/customer-support \ --output model.gguf \ - --quant q4_k_m -✓ merged LoRA into base -✓ converted to GGUF -✓ quantised to q4_k_m (4.1 GB → 4.1 GB) -✓ wrote model.gguf and model.gguf.sha256 + --quant q8_0 +Export complete: model.gguf (quant=q8_0, sha256=3f2a91c0b4d7…) ``` -The output is a single `.gguf` file plus a `.sha256` manifest. +(Log output, abridged.) The result is `model.gguf` plus `model.gguf.sha256` next to it. The sidecar is in `sha256sum` format (` *model.gguf`), so both `sha256sum -c model.gguf.sha256` and [`forgelm verify-gguf`](#/deployment/verify-gguf) accept it. When the checkpoint directory already holds a `model_integrity.json`, ForgeLM also tries to record the digest there under `exported_artifacts` (best effort: if the file is not valid JSON or cannot be written, a warning is logged and the export still succeeds; `--no-integrity-update` skips this). + +The sidecar detects corruption and accidental change. It sits beside the file, so it is **not** tamper evidence: whoever can replace the GGUF can replace the sidecar too. If you need that, sign the file or keep the digest where the file's writer cannot reach it. ## Quantisation levels @@ -34,51 +33,68 @@ The output is a single `.gguf` file plus a `.sha256` manifest. `q4_k_m` is the sweet spot — fits easily on consumer hardware, minimal quality loss versus full precision. +### K-quants take one more step + +The converter ForgeLM calls writes `f16` and `q8_0` directly. For a K-quant (`q2_k`, `q3_k_m`, `q4_k_m` — the default — and `q5_k_m`) `forgelm export` writes an intermediate `f16` GGUF, named like the output with its `.gguf` suffix replaced by `.f16.gguf` (`model.gguf` becomes `model.f16.gguf`; an output without the suffix gets `.f16.gguf` appended), with its own `.sha256`. It reports the substitution (`requested_quant`, `manual_step_required` and the exact `followup_command` in `--output-format json`) and logs the `llama-quantize` command that finishes the job: + +```shell +$ forgelm export ./checkpoints/customer-support --output model.gguf --quant q4_k_m +Export complete: model.f16.gguf (quant=f16, sha256=9c1e47ab02d5…) +Manual quantization step required — run: llama-quantize model.f16.gguf model.gguf Q4_K_M +$ llama-quantize model.f16.gguf model.gguf Q4_K_M +$ sha256sum model.gguf > model.gguf.sha256 +``` + +`llama-quantize` comes from llama.cpp; ForgeLM does not install it. `forgelm export` writes no sidecar for the quantised file, so the last command creates one and `forgelm verify-gguf model.gguf` has a digest to compare. + GGUF export is always a separate, explicit `forgelm export` invocation — there is no config-driven auto-export toggle. `forgelm/config.py` has no `output:` block and no `gguf` field anywhere in the schema, so a training YAML cannot trigger export as a side effect of a passing eval gate; run `forgelm export` yourself once training finishes. ## Multi-quant export `forgelm export --quant` accepts a single value per invocation (one of `{q2_k, q3_k_m, q4_k_m, q5_k_m, q8_0, f16}`). To produce several -quantisations from one checkpoint, run the command once per quant: +quantisations from one checkpoint, run the command once per quant. For +the two levels the converter writes directly: ```shell -$ for q in q4_k_m q5_k_m q8_0; do +$ for q in q8_0 f16; do forgelm export ./checkpoints/run \ --output "./gguf/model.${q}.gguf" \ --quant "${q}" done -✓ wrote gguf/model.q4_k_m.gguf (4.1 GB) -✓ wrote gguf/model.q5_k_m.gguf (4.8 GB) -✓ wrote gguf/model.q8_0.gguf (7.2 GB) +Export complete: ./gguf/model.q8_0.gguf (quant=q8_0, sha256=3f2a91c0b4d7…) +Export complete: ./gguf/model.f16.gguf (quant=f16, sha256=b81d05e6c392…) ``` -The manifest: +Each run writes its own sidecar (`model.q8_0.gguf.sha256`, `model.f16.gguf.sha256`). For K-quants, run the loop with the K-quant names and finish each intermediate file with `llama-quantize`, as above. + +When the checkpoint directory holds a `model_integrity.json`, each run tries to append an entry to its `exported_artifacts` list (best effort, as above): ```json { - "base_model": "Qwen/Qwen2.5-7B-Instruct", - "fine_tune_run_id": "abc123", - "quants": { - "q4_k_m": {"size_bytes": 4100000000, "sha256": "..."}, - "q5_k_m": {"size_bytes": 4800000000, "sha256": "..."}, - "q8_0": {"size_bytes": 7200000000, "sha256": "..."} - }, - "exported_at": "2026-04-29T15:00:00Z" + "exported_artifacts": [ + {"file": "model.q8_0.gguf", "format": "gguf", "quant": "q8_0", "sha256": "…", "size_bytes": 7200000000}, + {"file": "model.f16.gguf", "format": "gguf", "quant": "f16", "sha256": "…", "size_bytes": 13000000000} + ] } ``` ## Verifying GGUF integrity ```shell -$ forgelm verify-gguf model.q4_k_m.gguf -✓ valid GGUF magic -✓ vocab size matches base -✓ sha256 matches manifest -✓ tokenizer round-trip OK +$ forgelm verify-gguf model.q8_0.gguf +OK: model.q8_0.gguf + GGUF magic OK, metadata parsed, SHA-256 sidecar match + magic_ok: True + metadata_parsed: True + sidecar_present: True + sidecar_match: True + tensor_count: 291 + sha256_actual: a4c1f2… + sha256_expected: a4c1f2… ``` -This catches corruption from bad transfers, silent disk errors, and the occasional `llama.cpp` upstream incompatibility. +This checks the GGUF magic header, parses the metadata block when the optional `gguf` package is installed, and compares the `.sha256` sidecar when it exists. It does not compare the vocabulary or the tokenizer with the base model. It catches corruption from bad transfers, silent disk errors and a file edited after export; it does not show who produced the file. [Verify GGUF](#/deployment/verify-gguf) says what each layer proves. ## Loading GGUF in popular tools @@ -117,7 +133,7 @@ The result is full-precision GGUF, ~13 GB for a 7B model. ## Common pitfalls :::warn -**Exporting LoRA without merging.** `forgelm export` always merges LoRA adapters into the base before quantising. If you want adapter-only inference, you don't want GGUF — use `forgelm chat` against the adapter directly or load with PEFT. +**Forgetting `--adapter`.** `forgelm export` merges a LoRA adapter into the base model only when you pass `--adapter `; without it, the model path must already be a full model directory. If you want adapter-only inference, you don't want GGUF — use `forgelm chat` against the adapter directly or load with PEFT. ::: :::warn diff --git a/docs/usermanuals/en/deployment/verify-gguf.md b/docs/usermanuals/en/deployment/verify-gguf.md index 8fecc66a..ddbf4fc7 100644 --- a/docs/usermanuals/en/deployment/verify-gguf.md +++ b/docs/usermanuals/en/deployment/verify-gguf.md @@ -5,11 +5,11 @@ description: Three-layer integrity check for exported GGUF model files before se # Verify GGUF -`forgelm verify-gguf` is the read-only verifier paired with the GGUF export step. It runs a three-layer integrity check on a GGUF model file: the 4-byte `GGUF` magic header, the metadata block (when the optional `gguf` Python package is installed), and a SHA-256 comparison against the `.sha256` sidecar (when present). Use it as the final pre-serve gate so a corrupted or tampered GGUF never reaches `llama.cpp`, Ollama, vLLM, or LM Studio. +`forgelm verify-gguf` is the read-only verifier paired with the GGUF export step. It runs a three-layer integrity check on a GGUF model file: the 4-byte `GGUF` magic header, the metadata block (when the optional `gguf` Python package is installed), and a SHA-256 comparison against the `.sha256` sidecar (when present). Use it as the final pre-serve gate so a corrupted or accidentally changed GGUF does not reach `llama.cpp`, Ollama, vLLM, or LM Studio. The sidecar sits next to the file, so this is not tamper evidence (see the pitfalls below). ## When to use it -- **Before loading a GGUF into a serving runtime.** A clean `verify-gguf` exit is the minimum signal the file is what the exporter wrote. +- **Before loading a GGUF into a serving runtime.** A clean `verify-gguf` exit proves only what ran: with neither the `gguf` package nor a `.sha256` sidecar it proves that the first four bytes are `GGUF` and nothing else, not that the file is complete or unmodified. - **In CI/CD release gates after GGUF export.** Run after every `forgelm export` step and **fail the release on any non-zero exit code** (the public contract is the `0/1/2/3/4/5/6` set; see the in-manual [Exit Codes](#/reference/exit-codes) reference. `1` is the most common gate trip for a bad path or malformed sidecar — nothing was compared, or what was compared came out clean — while `6` means the file *is* a GGUF and a comparison genuinely disagreed; `2` covers genuine I/O failures on existing files, and other non-zero codes propagate from upstream pipelines). - **When receiving a GGUF from a third-party trainer or model hub.** Recompute the magic + metadata + SHA-256 layers to detect drift between what they sent and what they signed. - **After moving a GGUF between machines.** Any byte-level corruption in transit shows up as a SHA-256 mismatch. @@ -100,7 +100,7 @@ Pipe to `jq` to filter on the `valid` flag without parsing the human-readable te | **Metadata block** | When the optional `gguf` Python package is installed. | The reader raised mid-parse. That alone cannot tell a truncated file apart from a `gguf` package too old for this file's format revision, so the verdict is decided by what the sidecar can prove — see [the three metadata outcomes](#metadata-parse-failure-depends-on-the-sidecar) below. | `6` (no sidecar) / `1` (matching sidecar) / `6` (mismatching sidecar) | | **SHA-256 sidecar** | When `.sha256` exists alongside the file. | Either the file was modified after export (a well-formed digest that disagrees — integrity failure), OR the sidecar itself is malformed (non-hex / wrong length / `TODO` placeholder — nothing was compared). The verifier fails closed on a malformed sidecar so a corrupted manifest can never silently masquerade as "verified". | `6` (modified) / `1` (malformed sidecar) | -The exporter writes the sidecar by default. When you receive a GGUF from a third party, request the sidecar alongside it; without one, only the magic + metadata layers run. +The exporter writes the sidecar by default (for a K-quant request it belongs to the intermediate f16 file; create one for the quantised file with `sha256sum model.gguf > model.gguf.sha256`). When you receive a GGUF from a third party, request the sidecar alongside it; without one, only the magic + metadata layers run. ### Metadata parse failure depends on the sidecar @@ -116,7 +116,7 @@ The middle row is the one to internalise in CI: exit `6` means "page whoever own ### Optional dependency: the `gguf` package -The metadata parse uses the upstream `gguf` Python package. When the package is not installed, the layer is skipped silently — the magic + sidecar checks remain load-bearing: +The metadata parse uses the upstream `gguf` Python package. When the package is not installed, the layer is skipped (the text output shows `metadata_parsed: False`). The magic and the sidecar are then all that is checked, and with no sidecar either, a clean exit means only that the file starts with `GGUF`: ```shell $ pip uninstall -y gguf diff --git a/docs/usermanuals/en/getting-started/first-run.md b/docs/usermanuals/en/getting-started/first-run.md index 99a2b6ee..d3f51f9e 100644 --- a/docs/usermanuals/en/getting-started/first-run.md +++ b/docs/usermanuals/en/getting-started/first-run.md @@ -29,22 +29,23 @@ flowchart LR Before anything else, run `forgelm doctor` to check that Python, PyTorch, CUDA, and any optional dependencies are wired up correctly: ```shell -$ forgelm doctor -forgelm doctor — environment check - - [✓ pass] python.version Python 3.11.4 (CPython). - [✓ pass] torch.cuda torch 2.4.0 with CUDA 12.4. - [✓ pass] gpu.inventory 1 GPU(s) — GPU0: NVIDIA RTX 4090 (24.0 GiB). - [✓ pass] extras.qlora Installed (module bitsandbytes, purpose: 4-bit / 8-bit QLoRA training (Linux GPUs only)). - [✓ pass] extras.unsloth Installed (module unsloth, purpose: Unsloth-accelerated training (Linux GPUs only)). - [! warn] extras.distributed Optional extra missing — install with: pip install 'forgelm[distributed]' (purpose: DeepSpeed ZeRO + offload distributed training (Linux + macOS only)). - [✓ pass] hf_hub.reachable HuggingFace Hub reachable (HTTP 200). - [✓ pass] disk.workspace Workspace /home/me/forgelm — 387.0 GiB free of 500.0 GiB. - [! warn] operator.identity FORGELM_OPERATOR not set; audit events will fall back to 'me@workstation'. Pin FORGELM_OPERATOR= for CI / pipeline runs. - -Summary: 7 pass, 2 warn, 0 fail. +$ forgelm doctor --offline +forgelm doctor - environment check + + [+ pass] python.version Python 3.11.4 (CPython). + [! warn] torch.cuda torch 2.7.0 installed but CUDA is unavailable. CPU-only runs are supported but training will be very slow; expect to use this only for tiny experiments or smoke tests. + [+ pass] numpy.torch_abi torch 2.7.0 + numpy 2.1.0 are ABI-compatible. + [! warn] gpu.inventory No CUDA devices visible; CPU-only mode. + [! warn] extras.distributed Optional extra missing — install with: pip install 'forgelm[distributed]' (purpose: DeepSpeed ZeRO + offload distributed training (Linux + macOS only)). + [+ pass] disk.workspace Workspace /home/me/forgelm — 387.0 GiB free of 500.0 GiB. + [! warn] operator.identity FORGELM_OPERATOR not set; audit events will fall back to 'me@workstation'. Pin FORGELM_OPERATOR= for CI / pipeline runs so the audit log identifies a stable identity. + ... + +Summary: 7 pass, 12 warn, 0 fail. ``` +This is an excerpt: the real output has around 19 rows (it also checks the ForgeLM install, every optional extra, and the HF cache or Hub), and the rows, values and counts depend on your machine. A `pass` on an extra means the module is discoverable, not that it was import-tested; `[+ pass]`, `[! warn]` and `[x fail]` are the status markers. + `--output-format json` returns a structured envelope (`{"success": bool, "checks": [...], "summary": {...}}`) so CI can filter on individual probe results without parsing the table. Pass `--offline` to skip the HF Hub network probe and inspect the local cache instead — useful for air-gapped deployments. :::tip @@ -137,43 +138,50 @@ Possible verdicts: ```shell $ forgelm --config configs/quickstart-customer-support.yaml -[2026-04-28 14:01:32] config validated -[2026-04-28 14:01:33] auditing data/customer-support.jsonl (12,400 rows, 3 splits) -[2026-04-28 14:01:35] PII flags: 0 critical, 5 medium · cross-split overlap: 0 -[2026-04-28 14:01:37] SFT epoch 1/3 · loss=2.31 → 1.42 -[2026-04-28 14:18:55] DPO preference pass · β=0.1 · KL=4.2 -[2026-04-28 14:32:11] benchmark hellaswag=0.62 truthfulqa=0.48 -[2026-04-28 14:33:02] Llama Guard S1-S14: clean -[2026-04-28 14:33:04] Annex IV → checkpoints/customer-support/artifacts/annex_iv_metadata.json -[2026-04-28 14:33:04] ✔ finished, exit 0 ``` +The output is a stream of timestamped `[INFO]` lines from the `forgelm.*` loggers, with the progress bar and the per-step loss lines that transformers and TRL print. The `customer-support` template is an SFT run with no `evaluation:` block, so the stages you get are: config validation, training, then the files below being written. Stages such as the benchmark, the safety evaluation, the judge and the human-approval gate run only when the YAML enables them. The command ends with exit code `0` on success (see [Exit codes](#/reference/exit-codes)). + ## 6. Try the model ```shell -$ forgelm chat ./checkpoints/customer-support +$ forgelm chat Qwen/Qwen2.5-7B-Instruct --adapter ./checkpoints/customer-support/final_model forgelm> how do I cancel my subscription? You can cancel from Settings → Billing → Cancel subscription. Your access continues until the end of the current billing period… ``` +The positional argument is the base model from the `Model` line of the run summary; `--adapter` points at the LoRA weights in `final_model/` and merges them into it for the session. If your run saved a merged checkpoint instead (`final_model/` holds full weights and no `adapter_model.safetensors`), pass that directory as the only argument: `forgelm chat ./checkpoints/customer-support/final_model`. + ## What you got on disk ```text -checkpoints/customer-support/ -├── artifacts/ -│ ├── annex_iv_metadata.json ← Article 11 technical documentation -│ ├── audit_log.jsonl ← Article 12 append-only event log -│ ├── data_audit_report.json ← Article 10 data governance evidence -│ ├── safety_report.json ← Llama Guard verdict -│ ├── benchmark_results.json ← Per-task accuracy -│ └── manifest.json ← SHA-256 over every artifact -├── README.md ← Article 13 model card -├── config_snapshot.yaml ← The exact YAML this run used -└── adapter_model.safetensors ← LoRA weights (or merged checkpoint) +checkpoints/customer-support/ ← training.output_dir +├── audit_log.jsonl ← Article 12 append-only event log, hash-chained +├── audit_log.jsonl.manifest.json ← genesis pin: lets verify-audit detect truncation +├── compliance/ +│ ├── compliance_report.json ← Article 11 manifest (the canonical machine-readable bundle) +│ ├── training_manifest.yaml ← operator-readable summary of the run +│ ├── data_provenance.json ← Article 10 dataset fingerprint +│ └── data_governance_report.json ← Article 10 data-governance evidence +├── checkpoint-/ ← intermediate Trainer checkpoints +└── final_model/ + ├── adapter_model.safetensors ← LoRA weights (or merged checkpoint) + ├── README.md ← Article 13 model card + ├── deployer_instructions.md ← notes for whoever deploys the model + └── model_integrity.json ← SHA-256 checksums of the files in final_model/ ``` -That `artifacts/` directory is the deliverable for compliance reviews. Every file inside is hashed in `manifest.json` for tamper-evidence. +Some files appear only when the YAML asks for them: + +| File | Appears when | +|---|---| +| `compliance/annex_iv_metadata.json` | `compliance.provider_name`, `system_name` or `intended_purpose` is set | +| `safety/safety_results.json` | the safety evaluation is enabled | +| `benchmark/benchmark_results.json` | the benchmark gate is enabled | +| `final_model.staging./` instead of `final_model/` | `evaluation.require_human_approval` is on, until `forgelm approve` promotes it | + +There is no `artifacts/` directory and no single manifest over the whole bundle. Two things are tamper-checked: the audit log by its hash chain (`forgelm verify-audit`) and `final_model/` by `model_integrity.json` (`forgelm verify-integrity`). Nothing hashes `compliance/`, so keep those files where an edit would show, for example in version control or write-once storage. ## Next steps diff --git a/docs/usermanuals/en/getting-started/installation.md b/docs/usermanuals/en/getting-started/installation.md index 591e25f0..9a1677b1 100644 --- a/docs/usermanuals/en/getting-started/installation.md +++ b/docs/usermanuals/en/getting-started/installation.md @@ -22,7 +22,7 @@ ForgeLM ships as a single PyPI package with optional dependency groups (called * ::: :::note -**Intel Macs (x86_64) are not supported.** ForgeLM requires `torch>=2.4.0` (pulled in by `transformers>=5.3.0`), and PyPI publishes no `torch>=2.4` wheel for x86_64 macOS. Use Apple Silicon, Linux, or Windows. +**Intel Macs (x86_64) are not supported.** ForgeLM requires `torch>=2.7.0` (the first release with a dtype `transformers` 5.10 imports), and PyPI publishes no `torch>=2.7` wheel for x86_64 macOS. Use Apple Silicon, Linux, or Windows. ::: ## Base install @@ -75,7 +75,16 @@ Adds MinHash LSH (via `datasketch`) for near-duplicate detection on corpora bigg $ pip install 'forgelm[export]' ``` -Adds quantized GGUF export for local inference (Ollama, llama.cpp). Pulls in the `gguf` writer and supporting libraries. Optional because not every workflow ends in GGUF — many users hand off to vLLM or TGI directly. +Adds quantized GGUF export for local inference (Ollama, llama.cpp). Pulls in `llama-cpp-python`. Optional because not every workflow ends in GGUF — many users hand off to vLLM or TGI directly. + +**The conversion script is not included.** `forgelm export` runs llama.cpp's `convert_hf_to_gguf.py` with ForgeLM's own Python. No `llama-cpp-python` wheel puts that script where ForgeLM looks, so with only the extra installed `forgelm export` stops with `convert_hf_to_gguf.py not found inside the llama-cpp-python installation` (exit 2). Clone llama.cpp and point `FORGELM_GGUF_CONVERTER` at the script in that checkout: + +```shell +$ git clone https://github.com/ggml-org/llama.cpp +$ export FORGELM_GGUF_CONVERTER="$PWD/llama.cpp/convert_hf_to_gguf.py" +``` + +Use the copy inside the checkout: current versions import sibling modules (`conversion`, `gguf`) that sit beside it, and the script needs the Python packages listed in the checkout's `requirements/` folder importable in the environment ForgeLM runs in. K-quant levels additionally need llama.cpp's `llama-quantize` binary (see [GGUF export](#/deployment/gguf-export)). ### Distributed training (`[distributed]`) diff --git a/docs/usermanuals/en/getting-started/project-layout.md b/docs/usermanuals/en/getting-started/project-layout.md index 05b26f00..6883382c 100644 --- a/docs/usermanuals/en/getting-started/project-layout.md +++ b/docs/usermanuals/en/getting-started/project-layout.md @@ -22,10 +22,9 @@ my-finetune/ │ └── data_audit_report.json ├── checkpoints/ — model outputs (gitignored) │ └── customer-support/ -│ ├── adapter_model.safetensors -│ ├── README.md — model card (Article 13) -│ ├── config_snapshot.yaml — frozen copy of the config used -│ └── artifacts/ — compliance evidence bundle +│ ├── audit_log.jsonl — append-only, hash-chained event log +│ ├── compliance/ — Article 10/11 evidence (JSON + YAML) +│ └── final_model/ — weights, model card (README.md), model_integrity.json ├── ingested/ — raw documents → JSONL via `forgelm ingest` └── .forgelm/ — local cache (also gitignored) ``` @@ -50,11 +49,11 @@ ForgeLM's defaults play well with version control if you commit the right things **Commit:** - `configs/*.yaml` — your runs are configurations; the YAML is the source of truth. - `audit/*.json` — small, machine-readable, useful in PR reviews ("did the audit numbers regress?"). -- `checkpoints/*/README.md` and `checkpoints/*/artifacts/` (model card and audit bundle, both small). +- `checkpoints/*/final_model/README.md` and `checkpoints/*/compliance/` (model card and compliance evidence, both small). **Gitignore:** - `data/` — usually too large; track via DVC or your preferred dataset registry instead. -- `checkpoints/*/adapter_model.safetensors` and other weight files — too large for git, push to HuggingFace Hub or model registry. +- `checkpoints/*/final_model/*.safetensors`, the `checkpoint-*` directories and other weight files — too large for git, push to HuggingFace Hub or model registry. - `ingested/` — re-buildable from raw documents. - `.forgelm/` — local cache. @@ -125,10 +124,10 @@ Run the dev variant during development, the full one for release. CI runs the fu ## Where compliance artifacts go -Every successful (or failed) training run creates a `checkpoints//artifacts/` directory containing the evidence bundle described in [Compliance Overview](#/compliance/overview). This is the directory you hand to a regulator or pin in your CI artifacts. +A training run writes its evidence into `training.output_dir`: `audit_log.jsonl` (with its `.manifest.json` genesis pin) at the top level and a `compliance/` directory next to `final_model/`; [Your First Run](#/getting-started/first-run) shows the full tree and which files depend on the YAML. There is no `artifacts/` directory. Hand `compliance/` and `audit_log.jsonl` to a reviewer, or pin them in your CI artifacts. :::warn -**Don't merge the artifacts/ directory across runs.** Each run is a separate evidence bundle; mixing them breaks the SHA-256 manifest and undermines tamper-evidence. +**Keep each run's output directory separate.** Each run has its own audit chain and run id; copying two runs' files into one directory mixes them. Only the audit log (`forgelm verify-audit`) and `final_model/` (`forgelm verify-integrity`) are tamper-checked; nothing hashes `compliance/`. ::: ## See also diff --git a/docs/usermanuals/en/operations/air-gap.md b/docs/usermanuals/en/operations/air-gap.md index aacf64e5..aab61050 100644 --- a/docs/usermanuals/en/operations/air-gap.md +++ b/docs/usermanuals/en/operations/air-gap.md @@ -59,11 +59,11 @@ ERROR: model "meta-llama/Llama-Guard-3-8B" not in local cache. | Subsystem | Flag effect or required operator preparation | |---|---| | HuggingFace Hub downloads | `--offline` sets the HF offline flags; pre-cache the required artifacts. | -| W&B / MLflow / Comet | Disable reporting or configure local-only MLflow with a file URI; `--offline` does not disable these integrations. | +| W&B / MLflow | Set `training.report_to` to `none` (or `tensorboard`), or configure local-only MLflow with a file URI; `--offline` does not disable these integrations. | | OpenAI / Anthropic judges | Disable API-backed evaluation or use a local judge; `--offline` does not disable API clients. | | Webhooks | `--offline` does not disable configured delivery. Omit the `webhook:` block, or leave `url` / `url_env` unset, to disable it; `allow_private_destinations` permits private destinations in addition to public ones. | | pip install (during runtime) | Install required dependencies before disconnecting; `--offline` does not control pip. | -| Telemetry | ForgeLM has no own telemetry service; configure external integrations separately. | +| Telemetry | ForgeLM has no own telemetry service. The `forgelm` CLI sets `HF_HUB_DISABLE_TELEMETRY=1` (unless you set it yourself) so TRL and `huggingface_hub` send no anonymous usage ping; using ForgeLM as a library does not set it. Configure external integrations separately. | ## Local synthetic data @@ -111,7 +111,7 @@ $ forgelm doctor --offline forgelm doctor — environment check [✓ pass] python.version Python 3.11.4 (CPython). - [✓ pass] torch.cuda torch 2.4.0 with CUDA 12.4. + [✓ pass] torch.cuda torch 2.7.0 with CUDA 12.6. [✓ pass] gpu.inventory 1 GPU(s) — GPU0: NVIDIA A100 (80.0 GiB). [✓ pass] extras.qlora Installed (module bitsandbytes, purpose: 4-bit / 8-bit QLoRA training). [✓ pass] extras.eval Installed (module lm_eval, purpose: lm-evaluation-harness benchmark scoring). diff --git a/docs/usermanuals/en/operations/iso-soc2-deployer.md b/docs/usermanuals/en/operations/iso-soc2-deployer.md index 862c2748..d53546d6 100644 --- a/docs/usermanuals/en/operations/iso-soc2-deployer.md +++ b/docs/usermanuals/en/operations/iso-soc2-deployer.md @@ -14,16 +14,16 @@ Four pillars an ISO / SOC 2 auditor cares about most have direct ForgeLM evidenc 1. **Audit trail** — `audit_log.jsonl` per training run, append-only with HMAC + SHA-256 hash chain + genesis manifest sidecar. `forgelm verify-audit` validates the chain end-to-end. 2. **Change control** — Article 14 staging gate (`forgelm approve` / `reject`) + `human_approval.required/granted/rejected` audit events + `config_hash` (per-run manifest sidecar field) stamped per run. 3. **Data lineage** — `data_provenance.json` (SHA-256 fingerprint + size + mtime, plus the dataset's HF Hub commit SHA **graded by `hf_revision_source`**) + `data_governance_report.json` (collection_method, annotation_process, known_biases, personal_data_included, dpia_completed). Read the grade before the SHA: only `hf_revision_source: loaded` is evidence of the corpus that trained the model. -4. **Supply chain** — CycloneDX 1.5 SBOM emitted per release, `pip-audit` nightly, `bandit` CI for static + dynamic security scanning. +4. **Supply chain** — CycloneDX 1.5 SBOM emitted per release (a workflow-run artifact kept 90 days, not a GitHub release asset), `pip-audit` nightly, `bandit` CI for static + dynamic security scanning. ## The eight audit-floor questions The deep guide answers each with the exact `jq` / CLI command + the artefact it returns: 1. "Show me the audit trail for every model promotion in the past 90 days" → `forgelm verify-audit` + `jq 'select(.event == "human_approval.granted")'`. -2. "Show me the change-control evidence — who approved this model?" → cross-reference `training.started` + `human_approval.granted` events; two distinct operator IDs prove segregation of duties (ISO A.5.3, SOC 2 CC1.5). +2. "Show me the change-control evidence — who approved this model?" → cross-reference `training.started` + `human_approval.granted` events; compare the two identities; they are self-asserted (`FORGELM_OPERATOR`, else the OS user), so segregation of duties (ISO A.5.3, SOC 2 CC1.5) holds only if your identity provider makes them different people. 3. "Show me the data lineage" → `data_provenance.json`; `sha256` + `hf_revision` pin the corpus — but only when `hf_revision_source` reads `loaded`. `unverified` means the SHA is a Hub lookup made when the bundle was written, with no coupling to the load; `unresolved` means no SHA was obtained and none is claimed. -4. "Show me the supply chain" → `gh release download "$(gh release view --json tagName -q .tagName)" --pattern 'sbom-*'`; CycloneDX 1.5 JSON. +4. "Show me the supply chain" → `gh run list --repo HodeTech/ForgeLM --workflow publish.yml --branch v --status success`, then `gh run download --repo HodeTech/ForgeLM --pattern 'sbom-*'` within 90 days; CycloneDX 1.5 JSON (the CI runner's package list, test tooling included, no licence data). 5. "Show me the access controls" → IdP audit log + `FORGELM_OPERATOR` cross-reference. 6. "Show me the encryption posture" → deployer-side substrate (KMS audit log + `data_governance_report.json`). 7. "Show me the incident response" → `audit.classifier_load_failed` + F-compliance-110 strict gate. diff --git a/docs/usermanuals/en/operations/supply-chain.md b/docs/usermanuals/en/operations/supply-chain.md index 7ec468ae..7641fa43 100644 --- a/docs/usermanuals/en/operations/supply-chain.md +++ b/docs/usermanuals/en/operations/supply-chain.md @@ -5,28 +5,34 @@ description: Operator surface for ForgeLM's SBOM + pip-audit + bandit pipeline # Supply Chain -ForgeLM emits a CycloneDX 1.5 SBOM per release tag, and runs `pip-audit` + `bandit` nightly via the `nightly.yml` workflow (also re-run on every release-tag publish). This page is the operator's mental model: what runs when, where to get the artefacts, and how to mirror the same checks locally. +ForgeLM emits a CycloneDX 1.5 SBOM per release tag (as a workflow-run artifact, not a GitHub release asset), and runs `pip-audit` + `bandit` nightly via the `nightly.yml` workflow (also re-run on every release-tag publish). This page is the operator's mental model: what runs when, where to get the artefacts, and how to mirror the same checks locally. ## What runs when | Trigger | Tool | Result | Failure policy | |---|---|---|---| -| Release tag (`v*`) | `tools/generate_sbom.py` | One CycloneDX 1.5 JSON per (OS × Python-version) cell, attached to the GitHub release | SBOM step is pure-stdlib; cannot silently degrade a green release matrix | -| Nightly 03:00 UTC | `pip-audit` (via `tools/check_pip_audit.py`) | OSV / GHSA scan against installed deps | HIGH/CRITICAL → exit 1 + GitHub issue; MEDIUM → `::warning::`; LOW → silent | +| Release tag (`v*`) | `tools/generate_sbom.py` | One CycloneDX 1.5 JSON per (OS × Python-version) cell, uploaded as a workflow-run artifact of the tag's publish run (kept 90 days; not attached to the GitHub release) | SBOM step is pure-stdlib; cannot silently degrade a green release matrix | +| Nightly 03:00 UTC | `pip-audit` (via `tools/check_pip_audit.py`) | PyPI vulnerability-data scan against installed deps | HIGH/CRITICAL or UNKNOWN severity → exit 1 + GitHub issue; MEDIUM → `::warning::`; LOW → silent | | Nightly 03:00 UTC + release tag | `bandit` (via `tools/check_bandit.py`) | Static security scan of `forgelm/` (excludes `tests/`) | HIGH → exit 1; MEDIUM → `::warning::`; LOW → silent | ## Where to get the SBOM ```bash -# All SBOMs for a release. -gh release download "$(gh release view --json tagName -q .tagName)" --pattern 'sbom-*' - -# Pretty-print one. +# Find the successful publish run for a tag, then download one SBOM into the +# current directory. Run artifacts expire after 90 days. (--pattern 'sbom-*' +# would fetch every SBOM instead, one sub-directory per artifact.) +gh run list --repo HodeTech/ForgeLM --workflow publish.yml --branch v --status success \ + --json databaseId,createdAt +gh run download --repo HodeTech/ForgeLM --name sbom-ubuntu-latest-py3.11 + +# Pretty-print it. jq . sbom-ubuntu-latest-py3.11.json | less -# Diff between two releases (lists added/removed dependencies). -diff <(jq -S '.components | sort_by(.purl)' sbom-prev.json) \ - <(jq -S '.components | sort_by(.purl)' sbom.json) +# Diff between two releases (lists added/removed dependencies): download the +# previous release's SBOM into its own directory, then compare. +gh run download --repo HodeTech/ForgeLM --name sbom-ubuntu-latest-py3.11 --dir previous +diff <(jq -S '.components | sort_by(.purl)' previous/sbom-ubuntu-latest-py3.11.json) \ + <(jq -S '.components | sort_by(.purl)' sbom-ubuntu-latest-py3.11.json) # Ingest into Dependency-Track for CVE correlation. curl -X POST -H "X-Api-Key: $DT_KEY" -H "Content-Type: application/octet-stream" \ @@ -34,7 +40,7 @@ curl -X POST -H "X-Api-Key: $DT_KEY" -H "Content-Type: application/octet-stream" https://deptrack.example.com/api/v1/bom ``` -The SBOM is determinism-pinned — two consecutive emissions on the same Python environment produce content-identical JSON (modulo `serialNumber` and `metadata.timestamp`, both intentionally varying per CycloneDX semantics). +The SBOM is determinism-pinned — two consecutive emissions on the same Python environment produce content-identical JSON (modulo `serialNumber` and `metadata.timestamp`, both intentionally varying per CycloneDX semantics). That is a same-environment property: a release SBOM is the CI runner's `pip list` (test tooling such as `pytest` included, no dependency graph, no licence data), so re-emitting it later from the same tag will not reproduce it. To describe your own environment, run `python3 tools/generate_sbom.py` from a ForgeLM checkout there. ## Mirror the nightly check locally @@ -46,7 +52,7 @@ pip-audit --strict --format json --output /tmp/pip-audit.json python3 tools/check_pip_audit.py /tmp/pip-audit.json ``` -Exit 0 means the same severity policy ForgeLM CI applies passed. Exit 1 means a HIGH/CRITICAL vulnerability needs addressing before the nightly fires. +Exit 0 means the same severity policy ForgeLM CI applies passed. Exit 1 means a HIGH/CRITICAL vulnerability, or a finding whose severity pip-audit does not report (UNKNOWN, which is most of them), needs addressing before the nightly fires. ## Mirror the bandit check locally diff --git a/docs/usermanuals/en/operations/webhooks.md b/docs/usermanuals/en/operations/webhooks.md index d7cf7107..ffccf7bf 100644 --- a/docs/usermanuals/en/operations/webhooks.md +++ b/docs/usermanuals/en/operations/webhooks.md @@ -1,11 +1,11 @@ --- title: Webhooks -description: Slack and Teams notifications on training events — start, success, failure, auto-revert. +description: Slack-format webhook notifications on training events — start, success, failure, auto-revert. Teams and Discord need a relay. --- # Webhooks -ForgeLM fires structured webhooks on training milestones. Wire them into Slack, Teams, or any incident tool that accepts JSON payloads — get the right context to the right humans without anyone watching a log. +ForgeLM fires structured webhooks on training milestones. Wire them into Slack, or into any incident tool or relay that accepts this JSON payload (Microsoft Teams and Discord need a relay; see below) — get the right context to the right humans without anyone watching a log. ## Quick example @@ -17,8 +17,9 @@ webhook: notify_on_failure: true # default true (covers training.failure + training.reverted) ``` -The notifier emits a generic JSON payload — Slack and Teams ingest it directly via -their incoming-webhook endpoints. Per-event subscription is not currently +The notifier emits one generic JSON payload with a Slack-style `attachments` block. +Slack's incoming webhooks accept it; Microsoft Teams and Discord do not (see +"Slack / Teams / Discord ingestion" below). Per-event subscription is not currently configurable; toggle the three `notify_on_*` flags to coarse-control which lifecycle events fire. @@ -50,40 +51,56 @@ surface mirrored in the ## Payload shape -Single generic JSON shape — Slack / Teams / Discord all accept it -directly via incoming-webhook endpoints; ForgeLM does **not** wrap it in -provider-specific templates: +One generic JSON shape. Every payload carries `event`, `run_name`, `status`, `metrics`, `reason`, +`model_path` and an `attachments` array with one Slack-style block (`title`, `text`, `color`). +ForgeLM does **not** wrap it in provider-specific templates, and it has no top-level `text`, +`content` or `embeds` field: ```json { "event": "training.reverted", - "run_name": "customer-support-v1.2.0", + "run_name": "llama3-support-sft", "status": "reverted", - "reason": "safety regression: S5 hate-speech +0.18 over baseline" + "metrics": {}, + "reason": "Safety gate failed: unsafe_rate 0.08 exceeds max_unsafe_rate 0.05", + "model_path": null, + "attachments": [ + { + "title": "Training Reverted: llama3-support-sft", + "text": "Auto-revert fired. Generated artifacts were deleted because a post-training gate (evaluation, safety, judge, or benchmark) rejected the run.\n\nReason: Safety gate failed: unsafe_rate 0.08 exceeds max_unsafe_rate 0.05", + "color": "#ff9900" + } + ] } ``` -Payload keys vary by event; the full per-event field list is in the -[Audit Event Catalog on GitHub](https://github.com/HodeTech/ForgeLM/blob/main/docs/reference/audit_event_catalog.md) -under the *Webhook lifecycle events* table. +The `pipeline.*` events add a few fields. The full per-event field list is in the +[Webhook Payload Schema on GitHub](https://github.com/HodeTech/ForgeLM/blob/main/docs/reference/webhook_schema.md). ## Slack / Teams / Discord ingestion -The single generic JSON payload shown above is what Slack, Teams, and Discord -all expect on their incoming-webhook endpoints. There are **no per-provider +The payload above is a Slack-style body. There are **no per-provider templates** in `WebhookConfig` (no `template:`, no `events:` allow-list, no `channel:` / `mention_on_failure:` formatting knobs, no per-destination fan-out array). Routing and formatting happen on the receiving side: -- **Slack** — paste the payload into a Slack workflow or incoming-webhook - app; Slack renders the JSON's top-level fields. To get a richer - formatted card, point the webhook at a relay (Slack workflow / AWS - Lambda / your own gateway) that translates the ForgeLM payload into - Slack Block Kit. -- **Microsoft Teams** — similar pattern. Teams renders incoming JSON - natively but the visual is plain; for MessageCard / Adaptive Card - formatting, run a relay. -- **Discord** — accepts the JSON directly via the bot/webhook URL. +- **Slack** — an incoming webhook accepts the body and shows the + `attachments` block (title, text, colour). That is the legacy attachment + format, which Slack treats as secondary; for a richer card point the + webhook at a relay (Slack workflow / AWS Lambda / your own gateway) that + translates the ForgeLM payload into Slack Block Kit. +- **Microsoft Teams** — not accepted directly. Office 365 Connectors were + retired in May 2026, and a Teams Workflows (Power Automate) webhook expects + an Adaptive Card message, which this payload is not. Run a relay that wraps + the payload in an Adaptive Card. +- **Discord** — not accepted directly. A Discord webhook needs `content` or + `embeds` (or a file or poll) in the body, and ForgeLM sends none of them, so + Discord answers with HTTP 400. Run a relay that maps `attachments[0]` to + `content` or an embed. + +A receiver that rejects a delivery does not fail the run, and nothing is paged: +the rejection is a `WARNING` in the run's log (see *Common pitfalls*). Send a +test event to a new destination before relying on it. For multiple destinations, run multiple separate ForgeLM training configs (each pinned to its own `webhook.url_env`) or fan out from a @@ -97,7 +114,7 @@ Real `WebhookConfig` (see `forgelm/config.py::WebhookConfig`): | Field | Default | Notes | |---|---|---| | `url` | `null` | Inline URL — prefer `url_env` for secret hygiene. | -| `url_env` | `null` | Env-var name carrying the URL. Overrides `url` when set. | +| `url_env` | `null` | Env-var name carrying the URL. Read only when `url` is empty; when both are set, `url` wins. | | `notify_on_start` | `true` | Gates `training.start` AND `pipeline.started`. | | `notify_on_success` | `true` | Gates `training.success`, `approval.required`, AND a successful `pipeline.completed`. | | `notify_on_failure` | `true` | Gates `training.failure`, `training.reverted`, `pipeline.stage_reverted`, AND a failing `pipeline.completed`. | @@ -117,7 +134,7 @@ live outside ForgeLM. - **TLS strongly recommended.** ForgeLM permits both HTTPS and HTTP webhook URLs — HTTP destinations log a `Webhook URL uses HTTP (not HTTPS). Data will be sent unencrypted.` warning but are not rejected by default (see `forgelm/webhook.py` `_send`). On a regulated estate set `webhook.require_https: true` to make a plaintext `http://` URL a hard failure (the delivery is refused, not sent). Pin `https://` URLs in production. - **Curated payload.** ForgeLM never includes raw training data, full configs, or unredacted PII in webhook payloads. The notifier wraps a fixed-shape JSON; there is no `webhook.redact` toggle because there's nothing user-controllable to redact. - **Server-Side Request Forgery (SSRF) guard.** ForgeLM blocks webhook URLs pointing at internal IPs — RFC 1918, loopback, link-local (incl. cloud IMDS at `169.254.169.254`), RFC 6598 Shared Address Space / Carrier-Grade-NAT (`100.64.0.0/10`, which also covers Alibaba Cloud ECS's IMDS at `100.100.100.200` — the stdlib `ipaddress` predicates don't flag that block as private or reserved on their own), plus reserved and multicast destinations — unless you explicitly opt-in with `webhook.allow_private_destinations: true`. This prevents misconfigured runs from probing your internal network or leaking the payload to a cloud metadata service. -- **No HMAC body signing.** ForgeLM does not sign webhook bodies — destination-side authenticity falls to TLS + URL secrecy via `url_env` plus the receiving system's bearer-token / signed-request controls (Slack signing secret, Teams connector token). +- **No HMAC body signing.** ForgeLM does not sign webhook bodies — destination-side authenticity falls to TLS + URL secrecy via `url_env` plus whatever authentication the receiver applies itself. A Slack incoming webhook has none beyond its secret URL (Slack's signing secret verifies requests *from* Slack, not requests to a webhook URL), so put your own authentication in front of a custom relay: ForgeLM sends no signature. ## Common pitfalls diff --git a/docs/usermanuals/en/reference/cli.md b/docs/usermanuals/en/reference/cli.md index a9423316..5ee639a8 100644 --- a/docs/usermanuals/en/reference/cli.md +++ b/docs/usermanuals/en/reference/cli.md @@ -44,7 +44,7 @@ Run `forgelm --help` for any of these. | `--dry-run` | Validate configuration and check model/dataset access; no training. | | `--fit-check` | Estimate peak training VRAM; no model load. Requires `--config`. | | `--resume [PATH]` | Resume training. Bare `--resume` auto-detects last checkpoint; `--resume PATH` resumes from a specific one. | -| `--offline` | Air-gapped mode: disable all HF Hub network calls. Models and datasets must be available locally. | +| `--offline` | Offline mode: set the Hugging Face offline variables so Hub, transformers and datasets make no network calls. Models and datasets must be available locally. Does not stop webhooks, judge or synthetic-data API clients, or experiment trackers. | | `--benchmark-only MODEL_PATH` | Run benchmark evaluation on an existing model without training. Requires `evaluation.benchmark` config. | | `--merge` | Run model merging from the `merge:` config block. No training. | | `--stage NAME` | Multi-stage pipelines only: run a single named stage in isolation. Non-first stages need the previous stage's on-disk output, or `--input-model`. | @@ -327,7 +327,7 @@ See [Exit Codes](#/reference/exit-codes) for the full contract. | `FORGELM_OPERATOR` | Operator identity recorded in audit events (overrides `getpass.getuser()@hostname`) | | `FORGELM_ALLOW_ANONYMOUS_OPERATOR` | When `1`, permit the audit log to record an anonymous operator (otherwise an unresolved identity is an error) | | `FORGELM_AUDIT_SECRET` | HMAC signing key for the audit log chain (enables tamper-detection) | -| `FORGELM_GGUF_CONVERTER` | Path to a custom `convert-hf-to-gguf.py` script | +| `FORGELM_GGUF_CONVERTER` | Path to `convert_hf_to_gguf.py` inside a llama.cpp checkout. In practice required: no `llama-cpp-python` wheel installs the script where ForgeLM searches (see [Installation](#/getting-started/installation)) | ## Common patterns diff --git a/docs/usermanuals/en/reference/configuration.md b/docs/usermanuals/en/reference/configuration.md index c4ba73c2..ae2a455b 100644 --- a/docs/usermanuals/en/reference/configuration.md +++ b/docs/usermanuals/en/reference/configuration.md @@ -228,11 +228,11 @@ The merge field is `method` (not `algorithm`), and there is no `dare_ties` choic ```yaml distributed: strategy: null # null (single-GPU, no distributed wrapping) | deepspeed | fsdp - deepspeed_config: null # path to a DeepSpeed JSON, or preset name: zero2 | zero3 | zero3_offload - fsdp_strategy: "full_shard" # full_shard | shard_grad_op | no_shard | hybrid_shard - fsdp_auto_wrap: true # auto-wrap transformer layers (recommended) + deepspeed_config: null # path to a DeepSpeed JSON (preset names zero2 | zero3 | zero3_offload need a source checkout) + fsdp_strategy: "full_shard" # full_shard | shard_grad_op + fsdp_auto_wrap: true # ignored on transformers 5.10+ (FSDP2 always wraps transformer layers) fsdp_offload: false # offload FSDP parameters to CPU between forward/backward - fsdp_backward_prefetch: "backward_pre" # backward_pre | backward_post + fsdp_backward_prefetch: "backward_pre" # backward_pre | backward_post; ignored on transformers 5.10+ fsdp_state_dict_type: "FULL_STATE_DICT" # FULL_STATE_DICT | SHARDED_STATE_DICT ``` @@ -262,8 +262,8 @@ compliance: ```yaml webhook: - url: null # Slack / Teams / Discord / custom; prefer url_env - url_env: null # env var carrying the webhook URL + url: null # Slack incoming webhook, or a relay for Teams / Discord; prefer url_env + url_env: null # env var carrying the webhook URL; read only when url is empty notify_on_start: true notify_on_success: true notify_on_failure: true @@ -363,7 +363,7 @@ retention: staging_ttl_days: 7 # days to retain staging model after forgelm reject ephemeral_artefact_retention_days: 90 # compliance bundles, audit reports raw_documents_retention_days: 90 # ingested PDF/DOCX/EPUB/TXT/Markdown - enforce: "log_only" # log_only | warn_on_excess | block_on_excess + enforce: "log_only" # reserved: no command acts on it yet (log_only | warn_on_excess | block_on_excess) ``` ## `pipeline:` diff --git a/docs/usermanuals/en/reference/exit-codes.md b/docs/usermanuals/en/reference/exit-codes.md index 8780f19e..e462872d 100644 --- a/docs/usermanuals/en/reference/exit-codes.md +++ b/docs/usermanuals/en/reference/exit-codes.md @@ -11,7 +11,7 @@ ForgeLM's exit codes are a public contract. CI/CD pipelines, schedulers, and das | Exit | Constant | Meaning | Typical CI action | |---|---|---|---| -| **0** | `EXIT_SUCCESS` | Run completed and the checkpoint was promoted. With `evaluation.auto_revert: true` every gate also passed; with the shipped default `auto_revert: false` a failed benchmark/safety/judge gate is **recorded in the JSON output but does not change the exit code** — see ["What exit 0 actually guarantees"](#what-exit-0-actually-guarantees) below. | Continue pipeline (parse gate blocks if `auto_revert` is off) | +| **0** | `EXIT_SUCCESS` | Run completed and the checkpoint was promoted. With `evaluation.auto_revert: true` every gate that produced a verdict also passed (a safety evaluation with no verdict keeps the model and exits 0 too); with the shipped default `auto_revert: false` a failed benchmark/safety/judge gate is **recorded in the JSON output but does not change the exit code** — see ["What exit 0 actually guarantees"](#what-exit-0-actually-guarantees) below. | Continue pipeline (parse gate blocks if `auto_revert` is off) | | **1** | `EXIT_CONFIG_ERROR` | YAML invalid, file missing, env var unset, or argument malformed. | Fail fast | | **2** | `EXIT_TRAINING_ERROR` | Training-time runtime error (any unhandled exception that isn't a config or eval-gate failure: data load, OOM, NaN loss, I/O failure, mid-stream audit-iteration OSError). | Investigate; surface logs | | **3** | `EXIT_EVAL_FAILURE` | A benchmark/safety/judge gate failed **and** the model was auto-reverted (requires `evaluation.auto_revert: true`). With `auto_revert: false` a failed gate does not produce exit 3 — the run exits 0 with the failure recorded in the JSON gate blocks. | Investigate; do NOT promote | @@ -112,7 +112,7 @@ A run that exits 0 has: **Gates and exit 0.** Whether a *passed* benchmark/safety/judge gate is part of the exit-0 guarantee depends on `evaluation.auto_revert`: -- With `evaluation.auto_revert: true` (the EU AI Act high-risk default), a failed gate auto-reverts the model and exits **3** — so exit 0 *does* mean every configured gate passed. +- With `evaluation.auto_revert: true` (the EU AI Act high-risk default), a failed gate auto-reverts the model and exits **3** — so exit 0 means every configured gate that produced a verdict passed. **One exception:** a safety evaluation that produced no verdict (`safety.evaluation_completed: false` — the classifier failed to load, the probes file is missing or empty, or most verdicts were unreadable) keeps the model and exits 0 (4 with the approval gate) even with `auto_revert: true`; check `safety.passed` and `safety.evaluation_completed` in the JSON output. - With the shipped default `evaluation.auto_revert: false`, a failed gate is **recorded** (the `benchmark` / `safety` / `judge` block in the JSON output carries `*_passed: false`) but the model is still promoted and the run exits **0**. Read those JSON blocks; do not infer gate success from exit 0 alone. There is no "partial success" exit code by design — turn on `auto_revert` if you want a failing gate to change the exit code. diff --git a/docs/usermanuals/en/reference/json-output.md b/docs/usermanuals/en/reference/json-output.md index 7132bc10..c0ebbabe 100644 --- a/docs/usermanuals/en/reference/json-output.md +++ b/docs/usermanuals/en/reference/json-output.md @@ -178,7 +178,7 @@ When training runs to completion the pipeline emits a result envelope on **stdou | Key | Type | Notes | |---|---|---| -| `success` | bool | `true` when the run completed (including a staged, awaiting-approval run). `false` when a gate auto-reverted the model. | +| `success` | bool | `true` when the run completed (including a staged, awaiting-approval run, and a run whose safety evaluation produced no verdict: check `safety.evaluation_completed`). `false` when a gate auto-reverted the model. | | `error` | string | Present **iff** `success` is `false`, never on the success path. Carries the gate's own reason — which gate fired and against what threshold — because every gate exits `3`, so the exit code alone cannot tell them apart. If a failure reaches the envelope with no reason recorded, the field says so explicitly and points at `audit_log.jsonl` rather than being omitted. | | `metrics` | object | Training/eval/gate metrics (e.g. `eval_loss`, `benchmark/average`, `safety/safe_ratio`). Values are numbers, **except** a measurement that came out non-finite, which is written as the string `"nan"` / `"inf"` / `"-inf"` — JSON has no non-finite number literal, so the alternative was output no strict parser accepts. Type these fields as number-or-string. | | `final_model_path` | str | Where the model artefacts live. For an awaiting-approval run this is the **staging** directory until `forgelm approve` promotes it. | diff --git a/docs/usermanuals/en/reference/yaml-templates.md b/docs/usermanuals/en/reference/yaml-templates.md index 5fb6481a..75341a39 100644 --- a/docs/usermanuals/en/reference/yaml-templates.md +++ b/docs/usermanuals/en/reference/yaml-templates.md @@ -23,7 +23,7 @@ The five templates and the user journey each one optimises for: | `customer-support` | SFT | Multi-turn helpful + safe assistant — the most common production pattern. | | `code-assistant` | SFT | Code-completion / pair-programming model. | | `domain-expert` | SFT | BYOD (Bring Your Own Dataset) on a domain-specific corpus. | -| `medical-qa-tr` | SFT | Turkish-language medical QA (regulated domain — ships with `risk_assessment` + safety eval pre-wired). | +| `medical-qa-tr` | SFT | Turkish-language medical QA (regulated domain — the sample answers carry a safety disclaimer, but the template enables no safety evaluation and has no `risk_assessment` metadata. It is not for clinical decision support, and adding these does not change that; before any use beyond a demo, set `evaluation.safety.enabled: true` and fill in the top-level `risk_assessment:` block accurately). | | `grpo-math` | GRPO | Reward-model fine-tuning on math benchmarks (built-in format/length reward shaping). | The canonical template content is **the source of truth** — it is what diff --git a/docs/usermanuals/en/training/distributed.md b/docs/usermanuals/en/training/distributed.md index 7aa622ff..72321962 100644 --- a/docs/usermanuals/en/training/distributed.md +++ b/docs/usermanuals/en/training/distributed.md @@ -66,7 +66,7 @@ ZeRO-2 shards the optimiser state (the heaviest VRAM component for adaptive opti ```yaml distributed: strategy: "deepspeed" - deepspeed_config: "zero2" # preset name (or a filesystem path to a DeepSpeed JSON) + deepspeed_config: "./ds_zero2.json" # a filesystem path to a DeepSpeed JSON (preset names such as "zero2" need a source checkout) training: gradient_accumulation_steps: 4 @@ -95,7 +95,7 @@ training: gradient_accumulation_steps: 8 ``` -The `zero3_offload` preset fits 70B on 8×24 GB by offloading optimiser state and parameters to CPU. ZeRO-Infinity's NVMe offload isn't a built-in preset — point `deepspeed_config` at a custom DeepSpeed JSON with `offload_param`/`offload_optimizer` set to `device: nvme` for that case. +The `zero3_offload` preset offloads optimiser state and parameters to CPU, trading speed for GPU memory; whether a given model fits depends on your GPU memory and CPU RAM. The preset names `zero2`, `zero3` and `zero3_offload` resolve only from a source checkout (the JSON files live in the repository's `configs/deepspeed/` folder, not in the pip wheel or the Docker image), and the `zero2` preset is currently broken ([#410](https://github.com/HodeTech/ForgeLM/issues/410)), so give `deepspeed_config` the path of a JSON file you control. ZeRO-Infinity's NVMe offload isn't a preset — point `deepspeed_config` at a custom DeepSpeed JSON with `offload_param`/`offload_optimizer` set to `device: nvme` for that case. | Model | GPUs | ZeRO-3 + offload? | |---|---|---| @@ -111,12 +111,14 @@ FSDP shards similarly to ZeRO-3 but uses PyTorch's native FullyShardedDataParall ```yaml distributed: strategy: "fsdp" - fsdp_strategy: "full_shard" # full_shard | shard_grad_op | no_shard | hybrid_shard - fsdp_auto_wrap: true # auto-wrap transformer layers (recommended) + fsdp_strategy: "full_shard" # full_shard | shard_grad_op + fsdp_auto_wrap: true # ignored on transformers 5.10+ (FSDP2 always wraps transformer layers) fsdp_offload: false # offload parameters to CPU between forward/backward fsdp_state_dict_type: "FULL_STATE_DICT" # FULL_STATE_DICT | SHARDED_STATE_DICT ``` +`no_shard` and `hybrid_shard` are rejected when the config loads: transformers 5.10+, which ForgeLM requires, runs FSDP2, and FSDP2 supports neither. `fsdp_auto_wrap: false` and `fsdp_backward_prefetch: backward_post` are still accepted but change nothing under FSDP2, so ForgeLM logs a warning when either is set. + `DistributedConfig` has no `fsdp_auto_wrap_policy` or `fsdp_offload_params` field — auto-wrap is the plain boolean `fsdp_auto_wrap`, and CPU offload is `fsdp_offload` (no `_params` suffix). ## Gradient accumulation diff --git a/docs/usermanuals/tr/compliance/audit-log.md b/docs/usermanuals/tr/compliance/audit-log.md index 1c435f7e..8f262f99 100644 --- a/docs/usermanuals/tr/compliance/audit-log.md +++ b/docs/usermanuals/tr/compliance/audit-log.md @@ -5,7 +5,7 @@ description: Eğitim, değerlendirme ve geri alma kararlarını append-only even # Audit Log -EU AI Act Madde 12, yüksek-riskli AI sistemlerinin operasyonel olarak ilgili olayların log'unu tutmasını gerektirir. ForgeLM'in `audit_log.jsonl` dosyası; eğitim başlangıcı, eval kapıları, otomatik geri alma kararları ve model export'unu kapsayan, append-only ve SHA-256-anchored bir olay dizisidir. +EU AI Act Madde 12, yüksek-riskli AI sistemlerinin operasyonel olarak ilgili olayların log'unu tutmasını gerektirir. ForgeLM'in `audit_log.jsonl` dosyası; eğitim başlangıcı, eval kapıları, otomatik geri alma kararları ve model export'unu kapsayan, append-only ve SHA-256-anchored bir olay dizisidir. Eğitim hattını kaydeder; bu, teknik dokümantasyonunuzu destekleyebilecek geliştirme-zamanı kanıtıdır. Madde 12'nin istediği loglama dağıtılmış sistemin bir yeteneğidir ve ForgeLM bunu sağlamaz. ## Format diff --git a/docs/usermanuals/tr/compliance/gdpr-erasure.md b/docs/usermanuals/tr/compliance/gdpr-erasure.md index f430e5c0..77e6b09a 100644 --- a/docs/usermanuals/tr/compliance/gdpr-erasure.md +++ b/docs/usermanuals/tr/compliance/gdpr-erasure.md @@ -42,7 +42,7 @@ Audit zincirine altı event yazılır (bkz. [GitHub'daki Audit Event Kataloğu]( - `data.erasure_requested` — herhangi bir silmeden ÖNCE, ilk yayılır. - `data.erasure_completed` — disk operasyonu başarılı sonra son yayılır. - `data.erasure_failed` — disk operasyonu raise ettiğinde veya eşleşen satır/koşum bulunmadığında bunun yerine yayılır. -- `data.erasure_warning_memorisation` — corpus-satır silme × bu corpus'u tüketen herhangi bir koşum için `final_model/` mevcut (satır diskten gitti ama eğitilmiş ağırlıklarda hâlâ memorize edilmiş olabilir). +- `data.erasure_warning_memorisation` — corpus-satır silme × `--output-dir` altında doğrudan bir `final_model/` mevcut (satır diskten gitti ama eğitilmiş ağırlıklarda hâlâ memorize edilmiş olabilir). - `data.erasure_warning_synthetic_data_present` — corpus-satır silme × `output_dir`'de `synthetic_data*.jsonl` mevcut. - `data.erasure_warning_external_copies` — yüklü config boş-olmayan `webhook` block'u içeriyor; downstream tüketiciler bildirim almış olabilir. @@ -110,7 +110,7 @@ Kod 3 (`EXIT_EVAL_FAILURE`) ve 4 (`EXIT_AWAITING_APPROVAL`) iki subcommand'ın d ::: :::warn -**Memorization'ı kontrol etmeden `forgelm purge` çalıştırmak.** Corpus'u tüketen herhangi bir koşum için `final_model/` mevcutsa, satır diskten gitti ama ağırlıklarda hâlâ encoded olabilir. `data.erasure_warning_memorisation` event'ı boşluğu yüzeyselleştirir; harekete geçin (silinmiş satır olmadan yeniden eğitin veya DPIA'nızda residual riski belgeleyin). +**Memorization'ı kontrol etmeden `forgelm purge` çalıştırmak.** Eğitilmiş bir model varsa, satır diskten gitti ama ağırlıklarda hâlâ encoded olabilir. `data.erasure_warning_memorisation` event'ı boşluğu yalnızca `--output-dir` altında doğrudan bir `final_model/` varsa yüzeyselleştirir — `--output-dir` varsayılan olarak corpus'un dizinidir, bu yüzden eğitim çıktı dizininizi verin; harekete geçin (silinmiş satır olmadan yeniden eğitin veya DPIA'nızda residual riski belgeleyin). ::: :::tip diff --git a/docs/usermanuals/tr/compliance/gdpr.md b/docs/usermanuals/tr/compliance/gdpr.md index d8f425a3..152a5c50 100644 --- a/docs/usermanuals/tr/compliance/gdpr.md +++ b/docs/usermanuals/tr/compliance/gdpr.md @@ -5,7 +5,7 @@ description: Veri koruma uyumluluğu — PII minimizasyonu, veri sahibi hakları # GDPR / KVKK -GDPR (Avrupa) ve KVKK (Türkiye), eğitimde kullanılan kişisel verilere koruma gereklilikleri getirir. ForgeLM'in rolü: kişisel verinin eğitime girmesini en başta engellemek, ne girdiğini belgelemek ve veri sahibi talepleri için kanıt üretmek. +GDPR (Avrupa) ve KVKK (Türkiye), eğitimde kullanılan kişisel verilere koruma gereklilikleri getirir. ForgeLM'in rolü, kişisel veriyi eğitimden uzak tutmaktan daha dardır: tespit edebildiği tanımlayıcı biçimlerini maskeler, bir audit'in bulduğunu raporlar ve erişim ile silme talepleri için size komutlar verir. Hukuki dayanağınızı belirlemez, isimleri ya da diğer serbest-metin kişisel verileri bulmaz ve eğitilmiş bir modeli anonim yapmaz. ## GDPR / KVKK eğitim verisinde ne ister @@ -25,40 +25,35 @@ KVKK (Türkiye) bu ilkeleri yakından yansıtır. ### Veri minimizasyonu (Md. 5(1)(c)) -Ingest'te PII maskeleme kişisel tanımlayıcıları placeholder'larla değiştirir: +Ingest'te PII maskeleme, ForgeLM'in tespit ettiği tanımlayıcı biçimlerini bir placeholder ile değiştirir: -```yaml -ingestion: - pii_mask: - enabled: true - locale: "tr" - categories: ["email", "phone", "iban", "id_tr"] +```shell +$ forgelm ingest ./policies/ --recursive --strategy markdown --pii-mask --output data/policies.jsonl ``` -Veri JSONL'a düştüğünde tanımlanabilir özneler kaldırılmıştır. Bkz. [PII Maskeleme](#/data/pii-masking). +Dedektörler e-posta, telefon, kredi kartı, IBAN ile Türk, Alman, Fransız ve ABD ulusal kimlik biçimlerini kapsar; her eşleşme `[REDACTED]` olur. **İsimler, adresler, doğum tarihleri, sağlık verisi ve diğer serbest metin tespit edilmez**; desenler sıradan düzyazıyı maskelemek yerine sınırda bir eşleşmeyi kaçırmaya ayarlıdır. Maskeleme JSONL'daki kişisel veriyi azaltır; tanımlanabilir özneleri ondan kaldırmaz. Bkz. [PII Maskeleme](#/data/pii-masking). ### Hesap verebilirlik (Md. 5(2)) -Her audit aşağıdakileri belgeleyen `data_audit_report.json` üretir: +Her `forgelm audit` çalıştırması `data_audit_report.json` yazar. İçeriği: -- Tespit edilen PII kategorileri ve sayıları (maskelemeden önce). -- Kaynak atfı (her satırın hangi dokümandan geldiği). -- Kalite ve dil dağılımı. -- Tamper-evidence için SHA-256 manifest. +- Denetlediğiniz veri için tespit edilen PII ve gizli bilgi kategorileri ve sayıları. Bir maskelemenin neyi yakalaması gerektiğini görmek için maskelenmemiş kaynağı, neyin sızdığını görmek için maskelenmiş çıktıyı denetleyin. +- Near-duplicate, kalite ve split-arası örtüşme özetleri ile isteğe bağlı bir Croissant kartı. +- Audit'in kendi üst verisi: ne zaman çalıştığı, ona verilen yol ve girdi. -Bu raporlar Annex IV paketine akar. Regülatör "eğitim setinizde hangi kişisel veriler vardı?" diye sorduğunda yapılandırılmış cevabınız olur. +Rapor satırları kaynak dokümanlara atfetmez, hash manifest taşımaz ve düz bir JSON dosyasıdır: kendisi audit log'a yazılmaz. Silme ve erişim talepleri (`forgelm purge`, `forgelm reverse-pii`) ise `forgelm verify-audit`'in denetlediği hash-zincirli log'a yazılır. Rapor `training.output_dir` içindeyse Annex IV paketinin Madde 10 veri yönetişimi bölümünü besler; böylece regülatörün "eğitim setinizde hangi kişisel veriler vardı?" sorusunun yapılandırılmış bir cevabı olur: dedektörlerin bulduğu, hiç olmadığına dair bir beyan değil. ### Saklama sınırlaması (Md. 5(1)(e)) -ForgeLM ham kullanıcı verisini saklamaz — sizin kontrol ettiğiniz JSONL artifact'lar üretir. Otomatik saklama uygulanması için: +ForgeLM ham kullanıcı verisini saklamaz — sizin kontrol ettiğiniz JSONL artifact'lar üretir. Saklama ufukları tanımlanabilir ve **raporlanır, zorlanmaz**: ```yaml retention: - raw_documents_retention_days: 90 # N gün sonra orijinalleri otomatik sil + raw_documents_retention_days: 90 ephemeral_artefact_retention_days: 365 ``` -(Asıl silme depolama katmanınızın sorumluluğu; ForgeLM sadece amaçlanan TTL'yi audit log'a kaydeder.) +`forgelm purge --check-policy --config config.yaml` ufku aşmış artefaktları listeler ve her zaman 0 ile çıkar. Hiçbir şey otomatik silinmez ve bunun için audit log'a bir şey yazılmaz; süresi geçeni silmek depolama katmanınızın işidir. (`retention.enforce` ayrılmıştır: henüz hiçbir komut onu kullanmaz.) ## Veri sahibi talepleri @@ -68,30 +63,57 @@ En sık talep tipleri ve ForgeLM nasıl yardım eder: "Hakkımda hangi kişisel verileri tutuyorsunuz?" -Eğitim verinizde reverse-PII koşturun: +Kişinin tanımlayıcısını eğitim verinizde arayın: ```shell $ forgelm reverse-pii --query "ali@example.com" data/*.jsonl -Maskelenmiş veride eşleşme bulunamadı. ``` -PII ingest'te maskelendiği için modelden belirli birinin verisi geri çıkarılamaz. Audit raporu bunu teyit eder. +Tarama, tanımlayıcının geçtiği satırları raporlar; sıfır eşleşme geçerli bir cevaptır. Yalnızca işaret ettiğiniz dosyalarda gerçekten bulunanı bulur. Maskeleme yalnızca tespit ettiği tanımlayıcı biçimlerini kaldırır; bir isim ya da tespit edilemeyen bir varyant kalabilir ve maskelenmiş JSONL taraması ham kaynak dokümanları kapsamaz: onları ayrıca arayın. ForgeLM eğitilmiş bir modelin neyi hatırladığını söyleyemez. ### Silme hakkı (Md. 17) "Verilerimi silin." Bir kişinin verisi eğitim setinizdeydi: -1. Maskelenmiş JSONL kimlik bilgilerini içermiyor — zaten minimize edilmiş. +1. Satırları `forgelm reverse-pii` ile bulun, sonra `forgelm purge --row-id --corpus ` ile kaldırın; her silme audit log'a yazılır. `--row-id` satırın `id` veya `row_id` alanıyla eşleştirilir, satır numarasıyla asla; `forgelm ingest` bu alanlardan hiçbirini yazmaz. Bu yüzden **eğitimden önce** her satıra kalıcı bir id verin (aşağıya bakın). Id'si olmayan bir corpus satır satır silinemez. 2. Ham kaynak dokümanlar hâlâ içerebilir — ham deponuzdan düşürün ve gerekirse yeniden ingest edin. 3. Model bazı detayları ezberlemiş olabilir — aşağıdaki "model-seviyesi silme"ye bakın. +Satırları adreslenebilir kılmak için corpus oluşturulurken her JSONL satırına bir kez benzersiz bir `id` verin ve dosya sonradan filtrelenir ya da yeniden sıralanırsa bu id'yi koruyun. `id`'si eksik, null ya da boş olan satırlar yeni bir id alır; `id`'sini (ya da `purge`'ın da eşleştirdiği `row_id`'yi) başka bir satırın da kullandığı satırda betik durur, çünkü `forgelm purge --row-id` iki satırı tek eşleşme olarak görür (varsayılanda reddeder, `--row-matches all` ile her eşleşmeyi siler) ve id'yi hangi satırın koruyacağına sizin karar vermeniz gerekir. Çıktı önce geçici bir dosyaya yazılır ve yalnızca her satır doğrulandıktan sonra yerine taşınır; böylece başarısız bir çalıştırma mevcut çıktı dosyasına dokunmaz: + +```python +import json, os, uuid + +SRC, DST = "data/train.jsonl", "data/train.with-ids.jsonl" +tmp, seen = DST + ".tmp", set() +try: + with open(SRC, encoding="utf-8") as src, open(tmp, "w", encoding="utf-8") as dst: + for line_no, line in enumerate(src, start=1): + if not line.strip(): + continue + row = json.loads(line) + if row.get("id") in (None, ""): + row["id"] = str(uuid.uuid4()) + ids = {str(row[key]) for key in ("id", "row_id") if row.get(key) not in (None, "")} + if ids & seen: + raise SystemExit(f"line {line_no}: id {sorted(ids & seen)[0]!r} is already used by another row") + seen |= ids + dst.write(json.dumps(row, ensure_ascii=False) + "\n") + os.replace(tmp, DST) # reached only when every row passed +finally: + if os.path.exists(tmp): + os.remove(tmp) +``` + +Eğitimi id'li dosyadan yapın. `forgelm reverse-pii` tanımlayıcının geçtiği dosyayı ve satırı raporlar; o satırın `id` değerini dosyadan okuyup `forgelm purge --row-id`'ye verin. Eğitimden sonra id'leri yeniden atamak audit log ile corpus arasındaki bağı koparır; bu yüzden bir kez atayın. + ### Model-seviyesi silme LLM'ler eğitimden nadir string'leri ezberleyebilir. PII maskelemeyle bile deploy edilmiş bir modelden tüm izleri kaldırmak zordur. ForgeLM'in savunmaları: -- **Ezberi önle:** PII maskeleme, deduplikasyon (ezberlenen veri genelde tekrarlanan veridir). -- **Ezberi tespit et:** Audit aşaması bilinen PII pattern'leriyle örtüşen satırları flagler. +- **Girenleri azalt:** PII maskeleme ve deduplikasyon (ezberlenen veri genelde tekrarlanan veridir). +- **Ezber ölçülmez:** ForgeLM, eğitilmiş bir modelin belirli bir string'i üretip üretmediğini test etmez. - **Son çare yeniden eğitim:** Maskelemeye rağmen belirli bir öznenin verisi sızdıysa, o kaynak olmadan yeniden eğitin. Sıfır-tolerans senaryolarda (sağlık, hukuk) PII maskelemeyi eğitimden önce manuel inceleme ile birleştirin. @@ -122,7 +144,7 @@ ForgeLM'in GDPR hukuki meta verisi (hukuki dayanak, veri sorumlusu vb.) için YA ::: :::warn -**Uluslararası transferler.** Eğitim veriniz yargı sınırlarını aşıyorsa (Türk verisi AB'de eğitiliyor, AB verisi ABD'de eğitiliyor), ek koruma önlemleri uygulanır. `international_transfers.enabled: true` ayarlayın ve koruma önlemlerini belgeleyin. +**Uluslararası transferler.** Eğitim veriniz yargı sınırlarını aşıyorsa (Türk verisi AB'de eğitiliyor, AB verisi ABD'de eğitiliyor), ek koruma önlemleri uygulanır. ForgeLM'in transferler için bir ayarı yoktur; koruma önlemlerini DPIA'nıza kaydedin. ::: :::tip diff --git a/docs/usermanuals/tr/compliance/human-oversight.md b/docs/usermanuals/tr/compliance/human-oversight.md index 65a1f336..98bbbe15 100644 --- a/docs/usermanuals/tr/compliance/human-oversight.md +++ b/docs/usermanuals/tr/compliance/human-oversight.md @@ -5,7 +5,7 @@ description: Bir insan inceleyip imzalayana kadar model terfisini engelleyin — # İnsan Gözetimi -EU AI Act Madde 14, yüksek-riskli AI sistemlerinin insan gözetimi imkânı sağlamasını gerektirir. ForgeLM bunu opsiyonel bir config kapısı olarak uygular: `evaluation.require_human_approval: true` olduğunda model terfisi bir insan onay imzalayana kadar engellenir. +EU AI Act Madde 14, yüksek-riskli AI sistemlerinin insan gözetimi imkânı sağlamasını gerektirir. ForgeLM, `evaluation.require_human_approval: true` ile açılan opsiyonel bir terfi-öncesi onay kapısı sunar: model terfisi bir insan onay imzalayana kadar engellenir ve bu imza kaydedilir. Bu, tasarım-zamanı gözetim dokümantasyonunu destekler; dağıtılmış sistemden Madde 14'ün istediği kullanım-sırası gözetim önlemlerini (izleme, yorumlama, geçersiz kılma veya durdurma) uygulamaz; bunlar dağıtıcıda kalır. ## Kapı nasıl çalışır @@ -48,12 +48,8 @@ webhook: Trainer eval'den sonra durur ve yazdırır: ```text -[2026-04-29 14:33:10] Human approval required. - Run ID: abc123 - Bundle: checkpoints/run/artifacts/ - - To approve: forgelm approve abc123 --output-dir checkpoints/run --comment "..." - To reject: forgelm reject abc123 --output-dir checkpoints/run --comment "..." +2026-04-29 14:33:10 [INFO] forgelm.trainer: Human approval required. Model staged at: checkpoints/run/final_model.staging.abc123 +2026-04-29 14:33:10 [INFO] forgelm.trainer: Review results in checkpoints/run/compliance/ and run `forgelm approve abc123 --output-dir checkpoints/run` to promote, or `forgelm reject abc123 --output-dir checkpoints/run` to preserve for forensic review. ``` Reviewer audit-log + staging dizinine erişimi olan herhangi bir makineden onay komutunu çalıştırır. ForgeLM kimliği `FORGELM_OPERATOR`'dan resolve eder, kararı HMAC ile zincirler ve (`approve`'da) staging dizinini kanonik `final_model/` yoluna rename eder. diff --git a/docs/usermanuals/tr/concepts/alignment-overview.md b/docs/usermanuals/tr/concepts/alignment-overview.md index 5afd56fa..50011b97 100644 --- a/docs/usermanuals/tr/concepts/alignment-overview.md +++ b/docs/usermanuals/tr/concepts/alignment-overview.md @@ -53,7 +53,7 @@ SFT'den sonra model iyi formatta çıktı üretir ama kalite değişken — baze Farklar: - **DPO** — referans modele karşı `(chosen, rejected)` çiftlerinden öğrenir. En çok çalışılmış, en kararlı. - **SimPO** — DPO ile aynı ama referans modeli atlar. Daha az bellek, biraz daha az kararlılık. -- **KTO** — ikili `(prompt, response, iyi mi?)` sinyallerinden öğrenir. Sadece thumbs-up/down geri bildiriminiz varsa kullanın. +- **KTO** — ikili `(prompt, completion, label)` satırlarından öğrenir; `label`, istenen yanıt için `true`, istenmeyen yanıt için `false` olur. Sadece thumbs-up/down geri bildiriminiz varsa kullanın. - **ORPO** — SFT ve tercih kaybını tek geçişte birleştirir. Daha hızlı wall-clock; ayrı aşamalardan daha az esneklik. ### Problem 3: Akıl yürütme diff --git a/docs/usermanuals/tr/concepts/choosing-trainer.md b/docs/usermanuals/tr/concepts/choosing-trainer.md index e56f9a86..b3f0db67 100644 --- a/docs/usermanuals/tr/concepts/choosing-trainer.md +++ b/docs/usermanuals/tr/concepts/choosing-trainer.md @@ -48,7 +48,7 @@ flowchart TD |---|---| | Sadece `{prompt, completion}` satırları | [SFT](#/training/sft) | | `{prompt, chosen, rejected}` satırları | [DPO](#/training/dpo) (VRAM dar ise [SimPO](#/training/simpo)) | -| `{prompt, response, label: bool}` satırları | [KTO](#/training/kto) | +| `{prompt, completion, label: bool}` satırları | [KTO](#/training/kto) | | SFT ve tercih satırlarının karışımı | [SFT](#/training/sft) → [DPO](#/training/dpo) sıralı, veya tek aşama için [ORPO](#/training/orpo) | | Programatik reward (math grader, kod testi) | [GRPO](#/training/grpo) | diff --git a/docs/usermanuals/tr/deployment/gguf-export.md b/docs/usermanuals/tr/deployment/gguf-export.md index 5832e786..24c77d40 100644 --- a/docs/usermanuals/tr/deployment/gguf-export.md +++ b/docs/usermanuals/tr/deployment/gguf-export.md @@ -5,21 +5,20 @@ description: llama.cpp, Ollama veya LM Studio ile yerel inference için GGUF'a k # GGUF Export -GGUF (GPT-Generated Unified Format), `llama.cpp` üzerinden yerel CPU/GPU inference için fiili dosya formatıdır. ForgeLM herhangi bir fine-tuned checkpoint'i altı kuantizasyon seviyesi ve SHA-256 manifest ile GGUF'a export eder. +GGUF (GPT-Generated Unified Format), `llama.cpp` üzerinden yerel CPU/GPU inference için fiili dosya formatıdır. ForgeLM fine-tuned bir checkpoint'i GGUF'a export eder ve dosyanın yanına bir SHA-256 sidecar yazar. `f16` ve `q8_0` tek komutla çıkar; K-quant seviyeleri (`q2_k`, `q3_k_m`, `q4_k_m`, `q5_k_m`) aşağıda anlatılan bir `llama-quantize` adımı daha ister. ## Hızlı örnek ```shell $ forgelm export ./checkpoints/customer-support \ --output model.gguf \ - --quant q4_k_m -✓ LoRA base'e merge edildi -✓ GGUF'a dönüştürüldü -✓ q4_k_m'a kuantize edildi (4.1 GB → 4.1 GB) -✓ model.gguf ve model.gguf.sha256 yazıldı + --quant q8_0 +Export complete: model.gguf (quant=q8_0, sha256=3f2a91c0b4d7…) ``` -Çıktı tek `.gguf` dosyası artı `.sha256` manifest. +(Log çıktısı, kısaltılmış.) Sonuç `model.gguf` ve yanında `model.gguf.sha256`'dır. Sidecar `sha256sum` formatındadır (` *model.gguf`); bu yüzden hem `sha256sum -c model.gguf.sha256` hem de [`forgelm verify-gguf`](#/deployment/verify-gguf) onu kabul eder. Checkpoint dizininde zaten bir `model_integrity.json` varsa ForgeLM digest'i oraya da `exported_artifacts` altında yazmayı dener (best-effort: dosya geçerli JSON değilse ya da yazılamıyorsa bir uyarı loglanır ve export yine de başarılı olur; `--no-integrity-update` bunu atlar). + +Sidecar bozulmayı ve kazara değişikliği yakalar. Dosyanın yanında durduğu için kurcalama kanıtı **değildir**: GGUF'u değiştirebilen sidecar'ı da değiştirebilir. Buna ihtiyacınız varsa dosyayı imzalayın ya da digest'i dosyanın yazarının erişemediği bir yerde tutun. ## Kuantizasyon seviyeleri @@ -34,6 +33,20 @@ $ forgelm export ./checkpoints/customer-support \ `q4_k_m` tatlı nokta — consumer donanıma rahat sığar, full precision'a karşı minimum kalite kaybı. +### K-quant'lar bir adım daha ister + +ForgeLM'in çağırdığı dönüştürücü `f16` ve `q8_0`'ı doğrudan yazar. Bir K-quant için (`q2_k`, `q3_k_m`, `q4_k_m` — varsayılan — ve `q5_k_m`) `forgelm export`, kendi `.sha256`'sıyla birlikte, çıktı adındaki `.gguf` uzantısı `.f16.gguf` ile değiştirilerek adlandırılan (`model.gguf` → `model.f16.gguf`; bu uzantıyla bitmeyen bir çıktıya `.f16.gguf` eklenir) ara bir `f16` GGUF yazar. Değişikliği bildirir (`--output-format json`'da `requested_quant`, `manual_step_required` ve tam `followup_command`) ve işi bitiren `llama-quantize` komutunu loglar: + +```shell +$ forgelm export ./checkpoints/customer-support --output model.gguf --quant q4_k_m +Export complete: model.f16.gguf (quant=f16, sha256=9c1e47ab02d5…) +Manual quantization step required — run: llama-quantize model.f16.gguf model.gguf Q4_K_M +$ llama-quantize model.f16.gguf model.gguf Q4_K_M +$ sha256sum model.gguf > model.gguf.sha256 +``` + +`llama-quantize` llama.cpp'den gelir; ForgeLM onu kurmaz. `forgelm export` kuantize edilmiş dosya için sidecar yazmaz; son komut onu oluşturur ve `forgelm verify-gguf model.gguf` karşılaştıracak bir digest bulur. + GGUF export her zaman ayrı, açık bir `forgelm export` çağrısıdır — config-driven bir otomatik export toggle'ı yoktur. `forgelm/config.py`'de bir `output:` bloğu ve şemada hiçbir yerde `gguf` alanı yoktur, dolayısıyla bir eğitim YAML'ı eval kapısının geçmesinin yan etkisi olarak export'u tetikleyemez; eğitim bittiğinde `forgelm export`'u kendiniz çalıştırın. ## Çoklu-quant export @@ -41,45 +54,48 @@ GGUF export her zaman ayrı, açık bir `forgelm export` çağrısıdır — con `forgelm export --quant` invocation başına tek değer kabul eder (seçenekler: `{q2_k, q3_k_m, q4_k_m, q5_k_m, q8_0, f16}`). Tek bir checkpoint'ten birden çok quantisation üretmek için komutu her -quant için bir kez çalıştırın: +quant için bir kez çalıştırın. Dönüştürücünün doğrudan yazdığı iki +seviye için: ```shell -$ for q in q4_k_m q5_k_m q8_0; do +$ for q in q8_0 f16; do forgelm export ./checkpoints/run \ --output "./gguf/model.${q}.gguf" \ --quant "${q}" done -✓ gguf/model.q4_k_m.gguf yazıldı (4.1 GB) -✓ gguf/model.q5_k_m.gguf yazıldı (4.8 GB) -✓ gguf/model.q8_0.gguf yazıldı (7.2 GB) +Export complete: ./gguf/model.q8_0.gguf (quant=q8_0, sha256=3f2a91c0b4d7…) +Export complete: ./gguf/model.f16.gguf (quant=f16, sha256=b81d05e6c392…) ``` -Manifest: +Her çalıştırma kendi sidecar'ını yazar (`model.q8_0.gguf.sha256`, `model.f16.gguf.sha256`). K-quant'lar için döngüyü K-quant adlarıyla çalıştırın ve her ara dosyayı yukarıdaki gibi `llama-quantize` ile tamamlayın. + +Checkpoint dizininde bir `model_integrity.json` varsa her çalıştırma onun `exported_artifacts` listesine (yukarıdaki gibi best-effort) bir kayıt eklemeyi dener: ```json { - "base_model": "Qwen/Qwen2.5-7B-Instruct", - "fine_tune_run_id": "abc123", - "quants": { - "q4_k_m": {"size_bytes": 4100000000, "sha256": "..."}, - "q5_k_m": {"size_bytes": 4800000000, "sha256": "..."}, - "q8_0": {"size_bytes": 7200000000, "sha256": "..."} - }, - "exported_at": "2026-04-29T15:00:00Z" + "exported_artifacts": [ + {"file": "model.q8_0.gguf", "format": "gguf", "quant": "q8_0", "sha256": "…", "size_bytes": 7200000000}, + {"file": "model.f16.gguf", "format": "gguf", "quant": "f16", "sha256": "…", "size_bytes": 13000000000} + ] } ``` ## GGUF bütünlük doğrulama ```shell -$ forgelm verify-gguf model.q4_k_m.gguf -✓ valid GGUF magic -✓ vocab boyutu base ile eşleşiyor -✓ sha256 manifest ile eşleşiyor -✓ tokenizer round-trip OK +$ forgelm verify-gguf model.q8_0.gguf +OK: model.q8_0.gguf + GGUF magic OK, metadata parsed, SHA-256 sidecar match + magic_ok: True + metadata_parsed: True + sidecar_present: True + sidecar_match: True + tensor_count: 291 + sha256_actual: a4c1f2… + sha256_expected: a4c1f2… ``` -Bu kötü transferden bozulmaları, sessiz disk hatalarını ve ara sıra `llama.cpp` upstream uyumsuzluklarını yakalar. +Bu komut GGUF magic header'ını denetler, isteğe bağlı `gguf` paketi kuruluysa meta veri bloğunu ayrıştırır ve varsa `.sha256` sidecar'ını karşılaştırır. Kelime dağarcığını ya da tokenizer'ı base modelle karşılaştırmaz. Kötü transferden bozulmaları, sessiz disk hatalarını ve export sonrası düzenlenmiş bir dosyayı yakalar; dosyayı kimin ürettiğini göstermez. Her katmanın neyi kanıtladığı için [Verify GGUF](#/deployment/verify-gguf) sayfasına bakın. ## Yaygın araçlarda GGUF yükleme @@ -118,7 +134,7 @@ Sonuç full-precision GGUF, 7B model için ~13 GB. ## Sık hatalar :::warn -**LoRA'yı merge etmeden export.** `forgelm export` her zaman LoRA adapter'ı kuantizasyondan önce base'e merge eder. Adapter-only inference istiyorsanız GGUF istemiyorsunuz — adapter'a karşı doğrudan `forgelm chat` kullanın veya PEFT ile yükleyin. +**`--adapter`'ı unutmak.** `forgelm export` bir LoRA adapter'ı base modele yalnızca `--adapter ` verdiğinizde merge eder; vermezseniz model yolu zaten tam bir model dizini olmalıdır. Adapter-only inference istiyorsanız GGUF istemiyorsunuz — adapter'a karşı doğrudan `forgelm chat` kullanın veya PEFT ile yükleyin. ::: :::warn diff --git a/docs/usermanuals/tr/deployment/verify-gguf.md b/docs/usermanuals/tr/deployment/verify-gguf.md index de55e2a3..37383a20 100644 --- a/docs/usermanuals/tr/deployment/verify-gguf.md +++ b/docs/usermanuals/tr/deployment/verify-gguf.md @@ -5,11 +5,11 @@ description: llama.cpp, Ollama, vLLM ya da LM Studio üzerinden servis etmeden # GGUF Doğrulama -`forgelm verify-gguf`, GGUF export adımıyla eşleşen salt-okunur doğrulayıcıdır. Bir GGUF model dosyası üzerinde üç katmanlı bütünlük kontrolü yapar: 4-baytlık `GGUF` magic header, isteğe bağlı `gguf` Python paketi yüklüyse meta veri bloğu ve mevcutsa `.sha256` sidecar'ına karşı SHA-256 karşılaştırması. Bozulmuş veya tahrif edilmiş bir GGUF'un asla `llama.cpp`, Ollama, vLLM ya da LM Studio'ya ulaşmaması için son ön-servis kapısı olarak kullanın. +`forgelm verify-gguf`, GGUF export adımıyla eşleşen salt-okunur doğrulayıcıdır. Bir GGUF model dosyası üzerinde üç katmanlı bütünlük kontrolü yapar: 4-baytlık `GGUF` magic header, isteğe bağlı `gguf` Python paketi yüklüyse meta veri bloğu ve mevcutsa `.sha256` sidecar'ına karşı SHA-256 karşılaştırması. Bozulmuş veya kazara değişmiş bir GGUF'un `llama.cpp`, Ollama, vLLM ya da LM Studio'ya ulaşmaması için son ön-servis kapısı olarak kullanın. Sidecar dosyanın yanında durduğu için bu bir kurcalama kanıtı değildir (aşağıdaki tuzaklara bakın). ## Ne zaman kullanılır -- **Bir GGUF'u servis runtime'ına yüklemeden önce.** Temiz bir `verify-gguf` çıkışı, dosyanın exporter'ın yazdığı şey olduğuna dair minimum sinyaldir. +- **Bir GGUF'u servis runtime'ına yüklemeden önce.** Temiz bir `verify-gguf` çıkışı yalnızca çalışan kontrolü kanıtlar: ne `gguf` paketi ne de `.sha256` sidecar'ı varsa ilk dört baytın `GGUF` olduğunu kanıtlar, başka hiçbir şeyi değil; dosyanın eksiksiz ya da değiştirilmemiş olduğunu değil. - **GGUF export sonrası CI/CD yayın kapılarında.** Her `forgelm export` adımından sonra çalıştırın ve **sıfırdan farklı herhangi bir çıkış kodunda yayını başarısız sayın** (kamuya açık sözleşme `0/1/2/3/4/5/6` setidir; manuel içindeki [Çıkış Kodları](#/reference/exit-codes) referansına bakın. Bozuk bir yol veya bozuk sidecar için `1` en sık kapı tripi'dir — hiçbir şey karşılaştırılmadı ya da karşılaştırılan şey temiz çıktı — `6` ise dosyanın *gerçekten* bir GGUF olduğu ve bir karşılaştırmanın gerçekten uyuşmadığı anlamına gelir; `2` mevcut dosyalardaki gerçek I/O hatalarını kapsar, üst pipeline'lardan diğer sıfırdan farklı kodlar da iletilebilir). - **Üçüncü-taraf bir trainer'dan veya model hub'ından GGUF alındığında.** Gönderilen ile imzalanan arasındaki sapmayı tespit etmek için magic + meta veri + SHA-256 katmanlarını yeniden hesaplayın. - **Bir GGUF'u makineler arasında taşıdıktan sonra.** Aktarımda oluşan herhangi bir bayt-seviyesi bozulma SHA-256 uyuşmazlığı olarak ortaya çıkar. @@ -100,7 +100,7 @@ $ forgelm verify-gguf --output-format json \ | **Meta veri bloğu** | İsteğe bağlı `gguf` Python paketi yüklüyse. | Okuyucu ayrıştırma sırasında istisna fırlattı. Bu tek başına, kesilmiş bir dosyayı bu dosyanın format revizyonunu okuyamayacak kadar eski bir `gguf` paketinden ayırt edemez; dolayısıyla kararı sidecar'ın kanıtladığı şey belirler — aşağıdaki [üç meta veri sonucuna](#meta-veri-ayrıştırma-hatası-sidecara-bağlıdır) bakın. | `6` (sidecar yok) / `1` (eşleşen sidecar) / `6` (uyuşmayan sidecar) | | **SHA-256 sidecar** | Dosyanın yanında `.sha256` mevcutsa. | Ya dosya export sonrası değiştirildi (uyuşmayan iyi-biçimli bir digest — bütünlük arızası) YA DA sidecar'ın kendisi bozuk (hex değil / yanlış uzunluk / `TODO` placeholder — hiçbir şey karşılaştırılmadı). Doğrulayıcı bozuk sidecar'da kapalı başarısız olur; böylece bozuk bir manifest asla "doğrulanmış" gibi sessizce maskelenemez. | `6` (değiştirilmiş) / `1` (bozuk sidecar) | -Exporter sidecar'ı varsayılan olarak yazar. Üçüncü taraftan bir GGUF aldığınızda yanında sidecar'ı da talep edin; sidecar olmazsa yalnızca magic + meta veri katmanları çalışır. +Exporter sidecar'ı varsayılan olarak yazar (K-quant isteğinde sidecar ara f16 dosyasına aittir; kuantize edilmiş dosya için `sha256sum model.gguf > model.gguf.sha256` ile bir tane oluşturun). Üçüncü taraftan bir GGUF aldığınızda yanında sidecar'ı da talep edin; sidecar olmazsa yalnızca magic + meta veri katmanları çalışır. ### Meta veri ayrıştırma hatası sidecar'a bağlıdır @@ -116,7 +116,7 @@ CI'da içselleştirilmesi gereken satır ortadaki: çıkış `6` "artefakt sahib ### İsteğe bağlı bağımlılık: `gguf` paketi -Meta veri ayrıştırması üst kaynak `gguf` Python paketini kullanır. Paket yüklü değilse katman sessizce atlanır — magic + sidecar kontrolleri yük taşımaya devam eder: +Meta veri ayrıştırması üst kaynak `gguf` Python paketini kullanır. Paket yüklü değilse katman atlanır (metin çıktısı `metadata_parsed: False` gösterir). O zaman denetlenen yalnızca magic ve sidecar'dır; sidecar da yoksa temiz bir çıkış yalnızca dosyanın `GGUF` ile başladığı anlamına gelir: ```shell $ pip uninstall -y gguf diff --git a/docs/usermanuals/tr/getting-started/first-run.md b/docs/usermanuals/tr/getting-started/first-run.md index 1f65a3eb..7fc6d6b4 100644 --- a/docs/usermanuals/tr/getting-started/first-run.md +++ b/docs/usermanuals/tr/getting-started/first-run.md @@ -29,22 +29,23 @@ flowchart LR Her şeyden önce Python, PyTorch, CUDA ve opsiyonel bağımlılıkların doğru bağlandığını kontrol etmek için `forgelm doctor` çalıştırın: ```shell -$ forgelm doctor -forgelm doctor — environment check - - [✓ pass] python.version Python 3.11.4 (CPython). - [✓ pass] torch.cuda torch 2.4.0 with CUDA 12.4. - [✓ pass] gpu.inventory 1 GPU(s) — GPU0: NVIDIA RTX 4090 (24.0 GiB). - [✓ pass] extras.qlora Installed (module bitsandbytes, purpose: 4-bit / 8-bit QLoRA training (Linux GPUs only)). - [✓ pass] extras.unsloth Installed (module unsloth, purpose: Unsloth-accelerated training (Linux GPUs only)). - [! warn] extras.distributed Optional extra missing — install with: pip install 'forgelm[distributed]' (purpose: DeepSpeed ZeRO + offload distributed training (Linux + macOS only)). - [✓ pass] hf_hub.reachable HuggingFace Hub reachable (HTTP 200). - [✓ pass] disk.workspace Workspace /home/me/forgelm — 387.0 GiB free of 500.0 GiB. - [! warn] operator.identity FORGELM_OPERATOR not set; audit events will fall back to 'me@workstation'. Pin FORGELM_OPERATOR= for CI / pipeline runs. - -Summary: 7 pass, 2 warn, 0 fail. +$ forgelm doctor --offline +forgelm doctor - environment check + + [+ pass] python.version Python 3.11.4 (CPython). + [! warn] torch.cuda torch 2.7.0 installed but CUDA is unavailable. CPU-only runs are supported but training will be very slow; expect to use this only for tiny experiments or smoke tests. + [+ pass] numpy.torch_abi torch 2.7.0 + numpy 2.1.0 are ABI-compatible. + [! warn] gpu.inventory No CUDA devices visible; CPU-only mode. + [! warn] extras.distributed Optional extra missing — install with: pip install 'forgelm[distributed]' (purpose: DeepSpeed ZeRO + offload distributed training (Linux + macOS only)). + [+ pass] disk.workspace Workspace /home/me/forgelm — 387.0 GiB free of 500.0 GiB. + [! warn] operator.identity FORGELM_OPERATOR not set; audit events will fall back to 'me@workstation'. Pin FORGELM_OPERATOR= for CI / pipeline runs so the audit log identifies a stable identity. + ... + +Summary: 7 pass, 12 warn, 0 fail. ``` +Bu bir alıntıdır: gerçek çıktı yaklaşık 19 satırdır (ForgeLM kurulumunu, her opsiyonel extra'yı ve HF cache'i ya da Hub'ı da denetler); satırlar, değerler ve sayılar makinenize göre değişir. Bir extra için `pass`, modülün bulunabildiği anlamına gelir, import testinden geçtiği anlamına gelmez; durum işaretleri `[+ pass]`, `[! warn]` ve `[x fail]`'dir. + `--output-format json` yapısal bir zarf döner (`{"success": bool, "checks": [...], "summary": {...}}`); CI ayrı ayrı probe sonuçlarını tabloyu parse etmeden filtreleyebilir. `--offline` HF Hub ağ probe'unu atlar ve onun yerine yerel cache'i inceler — air-gap dağıtımları için faydalı. :::tip @@ -137,43 +138,50 @@ Olası verdict'ler: ```shell $ forgelm --config configs/quickstart-customer-support.yaml -[2026-04-29 14:01:32] config validated -[2026-04-29 14:01:33] auditing data/customer-support.jsonl (12,400 rows, 3 splits) -[2026-04-29 14:01:35] PII flags: 0 critical, 5 medium · cross-split overlap: 0 -[2026-04-29 14:01:37] SFT epoch 1/3 · loss=2.31 → 1.42 -[2026-04-29 14:18:55] DPO preference pass · β=0.1 · KL=4.2 -[2026-04-29 14:32:11] benchmark hellaswag=0.62 truthfulqa=0.48 -[2026-04-29 14:33:02] Llama Guard S1-S14: clean -[2026-04-29 14:33:04] Annex IV → checkpoints/customer-support/artifacts/annex_iv_metadata.json -[2026-04-29 14:33:04] ✔ finished, exit 0 ``` +Çıktı, `forgelm.*` logger'larından gelen zaman damgalı `[INFO]` satırlarının ve transformers ile TRL'in bastığı ilerleme çubuğu ile adım başı loss satırlarının akışıdır. `customer-support` şablonu `evaluation:` bloğu olmayan bir SFT koşusudur; bu yüzden göreceğiniz aşamalar şunlardır: config doğrulama, eğitim, ardından aşağıdaki dosyaların yazılması. Benchmark, güvenlik değerlendirmesi, judge ve insan-onay kapısı gibi aşamalar yalnızca YAML bunları açtığında çalışır. Komut başarıda `0` çıkış koduyla biter ([Çıkış kodları](#/reference/exit-codes)). + ## 6. Modeli dene ```shell -$ forgelm chat ./checkpoints/customer-support +$ forgelm chat Qwen/Qwen2.5-7B-Instruct --adapter ./checkpoints/customer-support/final_model forgelm> aboneliği nasıl iptal ederim? Aboneliğinizi Ayarlar → Faturalandırma → Aboneliği İptal Et adımlarıyla iptal edebilirsiniz. Erişiminiz mevcut faturalama döneminin sonuna kadar devam eder… ``` +Konumsal argüman, çalıştırma özetindeki `Model` satırında yazan temel modeldir; `--adapter`, `final_model/` içindeki LoRA ağırlıklarını gösterir ve oturum için temel modele birleştirir. Çalıştırmanız bunun yerine birleştirilmiş bir checkpoint kaydettiyse (`final_model/` tam ağırlıkları içerir ve `adapter_model.safetensors` yoktur), o dizini tek argüman olarak verin: `forgelm chat ./checkpoints/customer-support/final_model`. + ## Diskte ne var ```text -checkpoints/customer-support/ -├── artifacts/ -│ ├── annex_iv_metadata.json ← Article 11 teknik dokümantasyon -│ ├── audit_log.jsonl ← Article 12 append-only event log -│ ├── data_audit_report.json ← Article 10 veri yönetimi kanıtı -│ ├── safety_report.json ← Llama Guard verdict -│ ├── benchmark_results.json ← Görev başı doğruluk -│ └── manifest.json ← Her artifact üzerinde SHA-256 -├── README.md ← Article 13 model card -├── config_snapshot.yaml ← Bu koşunun kullandığı tam YAML -└── adapter_model.safetensors ← LoRA ağırlıkları (veya birleştirilmiş checkpoint) +checkpoints/customer-support/ ← training.output_dir +├── audit_log.jsonl ← Article 12 append-only event log, hash zincirli +├── audit_log.jsonl.manifest.json ← genesis pin: verify-audit'in kesilmeyi tespit etmesini sağlar +├── compliance/ +│ ├── compliance_report.json ← Article 11 manifesti (kanonik makine-okunur paket) +│ ├── training_manifest.yaml ← koşunun operatör-okunur özeti +│ ├── data_provenance.json ← Article 10 veri seti parmak izi +│ └── data_governance_report.json ← Article 10 veri yönetimi kanıtı +├── checkpoint-/ ← ara Trainer checkpoint'leri +└── final_model/ + ├── adapter_model.safetensors ← LoRA ağırlıkları (veya birleştirilmiş checkpoint) + ├── README.md ← Article 13 model card + ├── deployer_instructions.md ← modeli dağıtacak kişi için notlar + └── model_integrity.json ← final_model/ içindeki dosyaların SHA-256 özetleri ``` -O `artifacts/` dizini compliance incelemeleri için teslim edilebilirdir. İçindeki her dosya `manifest.json`'da tamper-evidence için hashlenmiştir. +Bazı dosyalar yalnızca YAML istediğinde oluşur: + +| Dosya | Ne zaman oluşur | +|---|---| +| `compliance/annex_iv_metadata.json` | `compliance.provider_name`, `system_name` veya `intended_purpose` ayarlıysa | +| `safety/safety_results.json` | güvenlik değerlendirmesi açıksa | +| `benchmark/benchmark_results.json` | benchmark kapısı açıksa | +| `final_model/` yerine `final_model.staging./` | `evaluation.require_human_approval` açıksa, `forgelm approve` onu terfi ettirene kadar | + +`artifacts/` dizini ve tüm paketi kapsayan tek bir manifest yoktur. İki şey tamper-check'lidir: audit log kendi hash zinciriyle (`forgelm verify-audit`) ve `final_model/` `model_integrity.json` ile (`forgelm verify-integrity`). `compliance/` dosyalarını hiçbir şey hash'lemez; bu yüzden onları bir değişikliğin görüneceği bir yerde tutun, örneğin sürüm kontrolünde ya da bir kez yazılabilir depolamada. ## Sonraki adımlar diff --git a/docs/usermanuals/tr/getting-started/installation.md b/docs/usermanuals/tr/getting-started/installation.md index 5c5517a5..f5e4c7cd 100644 --- a/docs/usermanuals/tr/getting-started/installation.md +++ b/docs/usermanuals/tr/getting-started/installation.md @@ -22,7 +22,7 @@ ForgeLM, PyPI'da tek bir paket olarak gelir; ağır özellikler için opsiyonel ::: :::note -**Intel Mac'ler (x86_64) desteklenmiyor.** ForgeLM `torch>=2.4.0` gerektirir (`transformers>=5.3.0` üzerinden gelir) ve PyPI'da x86_64 macOS için `torch>=2.4` wheel'i yok. Apple Silicon, Linux veya Windows kullanın. +**Intel Mac'ler (x86_64) desteklenmiyor.** ForgeLM `torch>=2.7.0` gerektirir (`transformers` 5.10'un import ettiği bir dtype'a sahip ilk sürüm) ve PyPI'da x86_64 macOS için `torch>=2.7` wheel'i yok. Apple Silicon, Linux veya Windows kullanın. ::: ## Temel kurulum @@ -75,7 +75,16 @@ $ pip install 'forgelm[ingestion-scale]' $ pip install 'forgelm[export]' ``` -Yerel inference için (Ollama, llama.cpp) kuantize GGUF export'u ekler. `gguf` writer'ı ve destekleyici kütüphaneleri kurar. Opsiyonel çünkü her iş akışı GGUF ile bitmez — birçok kullanıcı doğrudan vLLM veya TGI'a teslim eder. +Yerel inference için (Ollama, llama.cpp) kuantize GGUF export'u ekler. `llama-cpp-python`'ı kurar. Opsiyonel çünkü her iş akışı GGUF ile bitmez — birçok kullanıcı doğrudan vLLM veya TGI'a teslim eder. + +**Dönüştürme script'i pakete dahil değildir.** `forgelm export`, llama.cpp'nin `convert_hf_to_gguf.py` script'ini ForgeLM'in kendi Python'uyla çalıştırır. Hiçbir `llama-cpp-python` wheel'i o script'i ForgeLM'in aradığı yere koymaz; bu yüzden yalnızca extra kuruluyken `forgelm export` `convert_hf_to_gguf.py not found inside the llama-cpp-python installation` hatasıyla (exit 2) durur. llama.cpp'yi klonlayın ve `FORGELM_GGUF_CONVERTER`'ı o checkout'taki script'e yöneltin: + +```shell +$ git clone https://github.com/ggml-org/llama.cpp +$ export FORGELM_GGUF_CONVERTER="$PWD/llama.cpp/convert_hf_to_gguf.py" +``` + +Checkout'un içindeki kopyayı kullanın: güncel sürümler yanında duran kardeş modülleri (`conversion`, `gguf`) import eder ve script, checkout'un `requirements/` klasöründe listelenen Python paketlerinin ForgeLM'in çalıştığı ortamda import edilebilir olmasını ister. K-quant seviyeleri ayrıca llama.cpp'nin `llama-quantize` binary'sini ister ([GGUF export](#/deployment/gguf-export) sayfasına bakın). ### Dağıtık eğitim (`[distributed]`) diff --git a/docs/usermanuals/tr/getting-started/project-layout.md b/docs/usermanuals/tr/getting-started/project-layout.md index e24bf6ef..568043f9 100644 --- a/docs/usermanuals/tr/getting-started/project-layout.md +++ b/docs/usermanuals/tr/getting-started/project-layout.md @@ -22,10 +22,9 @@ my-finetune/ │ └── data_audit_report.json ├── checkpoints/ — model çıktıları (gitignored) │ └── customer-support/ -│ ├── adapter_model.safetensors -│ ├── README.md — model card (Article 13) -│ ├── config_snapshot.yaml — kullanılan config'in dondurulmuş kopyası -│ └── artifacts/ — compliance kanıt paketi +│ ├── audit_log.jsonl — append-only, hash zincirli olay günlüğü +│ ├── compliance/ — Article 10/11 kanıtı (JSON + YAML) +│ └── final_model/ — ağırlıklar, model card (README.md), model_integrity.json ├── ingested/ — `forgelm ingest` üzerinden ham döküman → JSONL └── .forgelm/ — yerel cache (gitignored) ``` @@ -50,11 +49,11 @@ ForgeLM varsayılanları doğru şeyleri commit'leyip gerisini ignore'larsanız **Commit:** - `configs/*.yaml` — koşularınız konfigürasyon; YAML kaynak doğrudur. - `audit/*.json` — küçük, makine-okunur, PR review'larında faydalı ("audit numaraları geriledi mi?"). -- `checkpoints/*/README.md` ve `checkpoints/*/artifacts/` (model card ve audit paketi, ikisi de küçük). +- `checkpoints/*/final_model/README.md` ve `checkpoints/*/compliance/` (model card ve compliance kanıtı, ikisi de küçük). **Gitignore:** - `data/` — genelde fazla büyük; DVC veya tercih ettiğiniz dataset registry'sine alın. -- `checkpoints/*/adapter_model.safetensors` ve diğer ağırlık dosyaları — git için fazla büyük; HuggingFace Hub veya model registry'ye gönderin. +- `checkpoints/*/final_model/*.safetensors`, `checkpoint-*` dizinleri ve diğer ağırlık dosyaları — git için fazla büyük; HuggingFace Hub veya model registry'ye gönderin. - `ingested/` — ham dokümanlardan yeniden inşa edilebilir. - `.forgelm/` — yerel cache. @@ -123,10 +122,10 @@ Geliştirme sırasında dev varyantı, release için tam olanını koşturun. CI ## Compliance artifact'ları nereye gider -Her başarılı (veya başarısız) eğitim koşusu, [Compliance Genel Bakış](#/compliance/overview) bölümünde anlatılan kanıt paketini içeren bir `checkpoints//artifacts/` dizini oluşturur. Regülatöre veya CI artifact'larında pinleyeceğiniz dizin budur. +Bir eğitim koşusu kanıtını `training.output_dir` içine yazar: en üstte `audit_log.jsonl` (`.manifest.json` genesis pin'iyle) ve `final_model/` yanında bir `compliance/` dizini; tam ağacı ve hangi dosyaların YAML'a bağlı olduğunu [İlk Koşunuz](#/getting-started/first-run) gösterir. `artifacts/` dizini yoktur. `compliance/` ve `audit_log.jsonl`'i bir incelemeciye teslim edin ya da CI artifact'larında pinleyin. :::warn -**`artifacts/` dizinini koşular arasında birleştirmeyin.** Her koşu ayrı bir kanıt paketidir; karıştırmak SHA-256 manifesti bozar ve tamper-evidence'ı baltalar. +**Her koşunun çıktı dizinini ayrı tutun.** Her koşunun kendi audit zinciri ve run id'si vardır; iki koşunun dosyalarını tek dizine kopyalamak onları karıştırır. Yalnızca audit log (`forgelm verify-audit`) ve `final_model/` (`forgelm verify-integrity`) tamper-check'lidir; `compliance/`'ı hiçbir şey hash'lemez. ::: ## Bkz. diff --git a/docs/usermanuals/tr/operations/air-gap.md b/docs/usermanuals/tr/operations/air-gap.md index 58dc233e..54792a27 100644 --- a/docs/usermanuals/tr/operations/air-gap.md +++ b/docs/usermanuals/tr/operations/air-gap.md @@ -59,11 +59,11 @@ ERROR: model "meta-llama/Llama-Guard-3-8B" not in local cache. | Alt sistem | Bayrağın etkisi veya operatörün yapması gereken hazırlık | |---|---| | HuggingFace Hub indirmeleri | `--offline`, HF çevrimdışı ayarlarını yapar; gerekli artefaktları önceden önbelleğe alın. | -| W&B / MLflow / Comet | Raporlamayı kapatın veya MLflow'u file URI ile yalnızca yerel çalışacak şekilde yapılandırın; `--offline` bu entegrasyonları kapatmaz. | +| W&B / MLflow | `training.report_to`'yu `none` (veya `tensorboard`) yapın ya da MLflow'u file URI ile yalnızca yerel çalışacak şekilde yapılandırın; `--offline` bu entegrasyonları kapatmaz. | | OpenAI / Anthropic judge'lar | API tabanlı değerlendirmeyi kapatın veya yerel judge kullanın; `--offline` API istemcilerini kapatmaz. | | Webhook'lar | `--offline`, yapılandırılmış teslimatları kapatmaz. Kapatmak için `webhook:` bloğunu çıkarın veya `url` / `url_env` değerlerini boş bırakın; `allow_private_destinations`, genel hedeflerin yanında özel hedeflere de izin verir. | | Runtime'da pip install | Gerekli bağımlılıkları bağlantıyı kesmeden kurun; `--offline`, pip'i yönetmez. | -| Telemetry | ForgeLM'in kendi telemetri servisi yoktur; harici entegrasyonları ayrıca yapılandırın. | +| Telemetry | ForgeLM'in kendi telemetri servisi yoktur. `forgelm` CLI'ı `HF_HUB_DISABLE_TELEMETRY=1`'i (siz ayarlamadıysanız) ayarlar; böylece TRL ve `huggingface_hub` anonim kullanım ping'i göndermez. ForgeLM'i kütüphane olarak kullanmak bunu ayarlamaz. Harici entegrasyonları ayrıca yapılandırın. | ## Yerel sentetik veri @@ -111,7 +111,7 @@ $ forgelm doctor --offline forgelm doctor — environment check [✓ pass] python.version Python 3.11.4 (CPython). - [✓ pass] torch.cuda torch 2.4.0 with CUDA 12.4. + [✓ pass] torch.cuda torch 2.7.0 with CUDA 12.6. [✓ pass] gpu.inventory 1 GPU(s) — GPU0: NVIDIA A100 (80.0 GiB). [✓ pass] extras.qlora Installed (module bitsandbytes, purpose: 4-bit / 8-bit QLoRA training). [✓ pass] extras.eval Installed (module lm_eval, purpose: lm-evaluation-harness benchmark scoring). diff --git a/docs/usermanuals/tr/operations/iso-soc2-deployer.md b/docs/usermanuals/tr/operations/iso-soc2-deployer.md index d96a3898..55e8a0ab 100644 --- a/docs/usermanuals/tr/operations/iso-soc2-deployer.md +++ b/docs/usermanuals/tr/operations/iso-soc2-deployer.md @@ -14,16 +14,16 @@ ISO / SOC 2 denetçisinin en çok önem verdiği dört sütun, doğrudan ForgeLM 1. **Denetim izi** — eğitim çalışması başına `audit_log.jsonl`, HMAC + SHA-256 hash chain + genesis manifest sidecar ile append-only. `forgelm verify-audit` zinciri uçtan uca doğrular. 2. **Değişiklik kontrolü** — Article 14 staging gate'i (`forgelm approve` / `reject`) + `human_approval.required/granted/rejected` audit olayları + run başına damgalanmış `config_hash` (per-run manifest sidecar field). 3. **Veri lineage** — `data_provenance.json` (SHA-256 fingerprint + size + mtime, ayrıca **`hf_revision_source` ile derecelendirilmiş** dataset HF Hub commit SHA'sı) + `data_governance_report.json` (collection_method, annotation_process, known_biases, personal_data_included, dpia_completed). Dereceyi SHA'dan önce okuyun: modeli eğiten korpusun kanıtı yalnızca `hf_revision_source: loaded`'dır. -4. **Tedarik zinciri** — sürüm başına yayınlanan CycloneDX 1.5 SBOM, gecelik `pip-audit`, statik + dinamik güvenlik tarama için CI'da `bandit`. +4. **Tedarik zinciri** — sürüm başına üretilen CycloneDX 1.5 SBOM (90 gün saklanan bir workflow-run artifact'ı, GitHub release asset'i değil), gecelik `pip-audit`, statik + dinamik güvenlik tarama için CI'da `bandit`. ## Sekiz denetim katı sorusu Derin rehber her birini tam `jq` / CLI komutu + döndürdüğü artefakt ile yanıtlar: 1. "Son 90 gündeki her model promotion için audit trail'i göster" → `forgelm verify-audit` + `jq 'select(.event == "human_approval.granted")'`. -2. "Bana change-control kanıtını göster — bu modeli kim onayladı?" → `training.started` + `human_approval.granted` olaylarını çapraz referans alın; iki farklı operator ID, görev ayrılığını (segregation of duties) kanıtlar (ISO A.5.3, SOC 2 CC1.5). +2. "Bana change-control kanıtını göster — bu modeli kim onayladı?" → `training.started` + `human_approval.granted` olaylarını çapraz referans alın; iki kimliği karşılaştırın; ikisi de kendi beyanıdır (`FORGELM_OPERATOR`, yoksa OS kullanıcısı), dolayısıyla görev ayrılığı (segregation of duties; ISO A.5.3, SOC 2 CC1.5) ancak kimlik sağlayıcınız bunları farklı kişiler yapıyorsa geçerlidir. 3. "Veri lineage'ını göster" → `data_provenance.json`; `sha256` + `hf_revision` corpus'u pinler — ancak yalnızca `hf_revision_source` `loaded` olduğunda. `unverified`, SHA'nın paket yazılırken yapılan ve yüklemeyle hiçbir bağı olmayan bir Hub sorgusu olduğu anlamına gelir; `unresolved` ise hiçbir SHA alınmadığı ve hiçbir iddiada bulunulmadığı anlamına gelir. -4. "Tedarik zincirini göster" → `gh release download "$(gh release view --json tagName -q .tagName)" --pattern 'sbom-*'`; CycloneDX 1.5 JSON. +4. "Tedarik zincirini göster" → `gh run list --repo HodeTech/ForgeLM --workflow publish.yml --branch v --status success`, ardından 90 gün içinde `gh run download --repo HodeTech/ForgeLM --pattern 'sbom-*'`; CycloneDX 1.5 JSON (CI runner'ının paket listesi, test araçları dahil, lisans verisi yok). 5. "Erişim kontrollerini göster" → IdP audit log + `FORGELM_OPERATOR` çapraz referansı. 6. "Şifreleme duruşunu göster" → operatör-tarafı substrate (KMS audit log + `data_governance_report.json`). 7. "Olay müdahalesini göster" → `audit.classifier_load_failed` + F-compliance-110 strict gate. diff --git a/docs/usermanuals/tr/operations/supply-chain.md b/docs/usermanuals/tr/operations/supply-chain.md index 00a82beb..bf42375f 100644 --- a/docs/usermanuals/tr/operations/supply-chain.md +++ b/docs/usermanuals/tr/operations/supply-chain.md @@ -5,28 +5,34 @@ description: ForgeLM SBOM + pip-audit + bandit pipeline'ı için operatör yüze # Tedarik Zinciri -ForgeLM sürüm tag'i başına bir CycloneDX 1.5 SBOM yayınlar ve `pip-audit` + `bandit`'i `nightly.yml` workflow'u üzerinden gecelik çalıştırır (her sürüm-tag yayınında da yeniden koşar). Bu sayfa operatörün zihinsel modelidir: ne zaman ne çalışır, artefaktlar nereden alınır ve aynı kontroller lokalde nasıl yansıtılır. +ForgeLM sürüm tag'i başına bir CycloneDX 1.5 SBOM üretir (workflow-run artifact'ı olarak, GitHub release asset'i değil) ve `pip-audit` + `bandit`'i `nightly.yml` workflow'u üzerinden gecelik çalıştırır (her sürüm-tag yayınında da yeniden koşar). Bu sayfa operatörün zihinsel modelidir: ne zaman ne çalışır, artefaktlar nereden alınır ve aynı kontroller lokalde nasıl yansıtılır. ## Ne zaman ne çalışır | Tetikleyici | Araç | Sonuç | Hata politikası | |---|---|---|---| -| Sürüm tag'i (`v*`) | `tools/generate_sbom.py` | (OS × Python-sürüm) hücresi başına bir CycloneDX 1.5 JSON, GitHub release'e iliştirilir | SBOM adımı pure-stdlib; yeşil sürüm matrisini sessizce bozamaz | -| Gecelik 03:00 UTC | `pip-audit` (`tools/check_pip_audit.py` üzerinden) | Kurulu bağımlılıklara karşı OSV / GHSA taraması | HIGH/CRITICAL → exit 1 + GitHub issue; MEDIUM → `::warning::`; LOW → sessiz | +| Sürüm tag'i (`v*`) | `tools/generate_sbom.py` | (OS × Python-sürüm) hücresi başına bir CycloneDX 1.5 JSON, tag'in publish çalıştırmasının workflow-run artifact'ı olarak yüklenir (90 gün saklanır; GitHub release'e iliştirilmez) | SBOM adımı pure-stdlib; yeşil sürüm matrisini sessizce bozamaz | +| Gecelik 03:00 UTC | `pip-audit` (`tools/check_pip_audit.py` üzerinden) | Kurulu bağımlılıklara karşı PyPI güvenlik açığı verisi taraması | HIGH/CRITICAL ya da UNKNOWN severity → exit 1 + GitHub issue; MEDIUM → `::warning::`; LOW → sessiz | | Gecelik 03:00 UTC + sürüm tag'i | `bandit` (`tools/check_bandit.py` üzerinden) | `forgelm/` üzerinde statik güvenlik taraması (`tests/` hariç) | HIGH → exit 1; MEDIUM → `::warning::`; LOW → sessiz | ## SBOM nereden alınır ```bash -# Bir sürüm için tüm SBOM'lar. -gh release download "$(gh release view --json tagName -q .tagName)" --pattern 'sbom-*' - -# Birini güzel-yazdır. +# Bir tag için başarılı publish çalıştırmasını bulun, sonra bir SBOM'u mevcut +# dizine indirin. Run artifact'ları 90 günde silinir. (--pattern 'sbom-*' +# bunun yerine tüm SBOM'ları indirir, artifact başına bir alt dizinle.) +gh run list --repo HodeTech/ForgeLM --workflow publish.yml --branch v --status success \ + --json databaseId,createdAt +gh run download --repo HodeTech/ForgeLM --name sbom-ubuntu-latest-py3.11 + +# Güzel-yazdır. jq . sbom-ubuntu-latest-py3.11.json | less -# İki sürüm arası diff (eklenen/kaldırılan bağımlılıkları listeler). -diff <(jq -S '.components | sort_by(.purl)' sbom-prev.json) \ - <(jq -S '.components | sort_by(.purl)' sbom.json) +# İki sürüm arası diff (eklenen/kaldırılan bağımlılıkları listeler): önceki +# sürümün SBOM'unu kendi dizinine indirip karşılaştırın. +gh run download --repo HodeTech/ForgeLM --name sbom-ubuntu-latest-py3.11 --dir previous +diff <(jq -S '.components | sort_by(.purl)' previous/sbom-ubuntu-latest-py3.11.json) \ + <(jq -S '.components | sort_by(.purl)' sbom-ubuntu-latest-py3.11.json) # CVE korelasyonu için Dependency-Track'a yükle. curl -X POST -H "X-Api-Key: $DT_KEY" -H "Content-Type: application/octet-stream" \ @@ -34,7 +40,7 @@ curl -X POST -H "X-Api-Key: $DT_KEY" -H "Content-Type: application/octet-stream" https://deptrack.example.com/api/v1/bom ``` -SBOM determinizm-pinli — aynı Python ortamında ardışık iki yayın content-identical JSON üretir (CycloneDX semantiği gereği kasıtlı olarak değişen `serialNumber` ve `metadata.timestamp` hariç). +SBOM determinizm-pinli — aynı Python ortamında ardışık iki yayın content-identical JSON üretir (CycloneDX semantiği gereği kasıtlı olarak değişen `serialNumber` ve `metadata.timestamp` hariç). Bu, aynı ortama özgü bir özelliktir: release SBOM'u CI runner'ının `pip list`'idir (`pytest` gibi test araçları dahil, bağımlılık grafiği ve lisans verisi yok); aynı tag'den sonradan yeniden emit etmek onu tekrarlamaz. Kendi ortamınızı tanımlamak için orada bir ForgeLM checkout'undan `python3 tools/generate_sbom.py` çalıştırın. ## Gecelik kontrolü lokalde yansıtın @@ -46,7 +52,7 @@ pip-audit --strict --format json --output /tmp/pip-audit.json python3 tools/check_pip_audit.py /tmp/pip-audit.json ``` -Exit 0, ForgeLM CI'nın uyguladığı aynı severity politikasının geçtiği anlamına gelir. Exit 1, gecelik ateşlenmeden önce bir HIGH/CRITICAL açığın ele alınması gerektiği anlamına gelir. +Exit 0, ForgeLM CI'nın uyguladığı aynı severity politikasının geçtiği anlamına gelir. Exit 1, gecelik ateşlenmeden önce bir HIGH/CRITICAL açığın ya da pip-audit'in severity bildirmediği (UNKNOWN; çoğu bulgu böyledir) bir bulgunun ele alınması gerektiği anlamına gelir. ## Bandit kontrolünü lokalde yansıtın diff --git a/docs/usermanuals/tr/operations/webhooks.md b/docs/usermanuals/tr/operations/webhooks.md index 6f80a377..722e15d5 100644 --- a/docs/usermanuals/tr/operations/webhooks.md +++ b/docs/usermanuals/tr/operations/webhooks.md @@ -1,11 +1,11 @@ --- title: Webhook'lar -description: Eğitim olaylarında Slack ve Teams bildirimleri — başlangıç, başarı, hata, otomatik geri alma. +description: Eğitim olaylarında Slack biçiminde webhook bildirimleri — başlangıç, başarı, hata, otomatik geri alma. Teams ve Discord için relay gerekir. --- # Webhook'lar -ForgeLM eğitim dönüm noktalarında yapılandırılmış webhook fırlatır. Bunları Slack, Teams veya JSON payload kabul eden herhangi bir incident aracına bağlayın — hiç kimsenin log izlemesine gerek kalmadan doğru bağlamı doğru insanlara getirin. +ForgeLM eğitim dönüm noktalarında yapılandırılmış webhook fırlatır. Bunları Slack'e ya da bu JSON payload'u kabul eden herhangi bir incident aracına veya relay'e bağlayın (Microsoft Teams ve Discord için relay gerekir; aşağıya bakın) — hiç kimsenin log izlemesine gerek kalmadan doğru bağlamı doğru insanlara getirin. ## Hızlı örnek @@ -17,8 +17,9 @@ webhook: notify_on_failure: true # default true (training.failure + training.reverted'i kapsar) ``` -Notifier generic JSON payload yayar — Slack ve Teams bunu incoming-webhook -uçlarından doğrudan kabul eder. Per-event abonelik şu an konfigüre +Notifier, Slack tarzı bir `attachments` bloğu taşıyan tek bir generic JSON payload yayar. +Slack'in incoming webhook'ları bunu kabul eder; Microsoft Teams ve Discord etmez +(aşağıdaki "Slack / Teams / Discord ingest" bölümüne bakın). Per-event abonelik şu an konfigüre edilemiyor; üç `notify_on_*` flag'iyle hangi yaşam-döngüsü olaylarının fırlatılacağı kaba ayarlanır. @@ -50,41 +51,58 @@ ile aynalanan kanonik yüzeydir: ## Payload yapısı -Tek generic JSON şekil — Slack / Teams / Discord hepsi bunu incoming-webhook -uçlarından doğrudan kabul eder; ForgeLM provider'a-özel template'lerle -sarmalama **yapmaz**: +Tek generic JSON şekil. Her payload `event`, `run_name`, `status`, `metrics`, `reason`, +`model_path` ve tek bir Slack tarzı blok (`title`, `text`, `color`) içeren bir `attachments` +dizisi taşır. ForgeLM provider'a-özel template'lerle sarmalama **yapmaz** ve üst düzeyde +`text`, `content` veya `embeds` alanı yoktur: ```json { "event": "training.reverted", - "run_name": "customer-support-v1.2.0", + "run_name": "llama3-support-sft", "status": "reverted", - "reason": "safety regression: S5 hate-speech +0.18 over baseline" + "metrics": {}, + "reason": "Safety gate failed: unsafe_rate 0.08 exceeds max_unsafe_rate 0.05", + "model_path": null, + "attachments": [ + { + "title": "Training Reverted: llama3-support-sft", + "text": "Auto-revert fired. Generated artifacts were deleted because a post-training gate (evaluation, safety, judge, or benchmark) rejected the run.\n\nReason: Safety gate failed: unsafe_rate 0.08 exceeds max_unsafe_rate 0.05", + "color": "#ff9900" + } + ] } ``` -Payload anahtarları event'e göre değişir; tam per-event alan listesi -[GitHub'daki Audit Event Kataloğu](https://github.com/HodeTech/ForgeLM/blob/main/docs/reference/audit_event_catalog.md) -*Webhook lifecycle events* tablosundadır. +`pipeline.*` event'leri birkaç alan ekler. Tam per-event alan listesi +[GitHub'daki Webhook Payload Şeması](https://github.com/HodeTech/ForgeLM/blob/main/docs/reference/webhook_schema.md) +sayfasındadır. ## Slack / Teams / Discord ingest -Yukarıdaki tek generic JSON payload, Slack, Teams ve Discord'un -incoming-webhook endpoint'lerinde beklediği şeydir. `WebhookConfig`'te +Yukarıdaki payload Slack tarzı bir gövdedir. `WebhookConfig`'te **provider başına template yoktur** (no `template:`, no `events:` allow-list, no `channel:` / `mention_on_failure:` formatlama knob'ları, no per-destination fan-out array). Routing ve formatlama alıcı tarafta yapılır: -- **Slack** — payload'u Slack workflow veya incoming-webhook - uygulamasına yapıştırın; Slack JSON'un top-level alanlarını render - eder. Daha zengin formatlanmış bir kart için, webhook'u ForgeLM - payload'unu Slack Block Kit'e çeviren bir relay'e (Slack +- **Slack** — incoming webhook gövdeyi kabul eder ve `attachments` + bloğunu (başlık, metin, renk) gösterir. Bu eski attachment biçimidir + ve Slack onu ikincil sayar; daha zengin bir kart için webhook'u + ForgeLM payload'unu Slack Block Kit'e çeviren bir relay'e (Slack workflow / AWS Lambda / kendi gateway'iniz) yönlendirin. -- **Microsoft Teams** — benzer pattern. Teams gelen JSON'u natively - render eder ama görsel düz; MessageCard / Adaptive Card - formatlama için bir relay çalıştırın. -- **Discord** — JSON'u doğrudan bot/webhook URL üzerinden kabul eder. +- **Microsoft Teams** — doğrudan kabul edilmez. Office 365 Connector'lar + Mayıs 2026'da kapatıldı ve bir Teams Workflows (Power Automate) + webhook'u Adaptive Card mesajı bekler; bu payload öyle değildir. + Payload'u Adaptive Card'a saran bir relay çalıştırın. +- **Discord** — doğrudan kabul edilmez. Discord webhook'u gövdede + `content` veya `embeds` (ya da dosya ya da anket) ister; ForgeLM bunların + hiçbirini göndermez, bu yüzden Discord HTTP 400 döner. `attachments[0]`'ı + `content`'e veya bir embed'e eşleyen bir relay çalıştırın. + +Teslimatı reddeden bir alıcı run'ı başarısız etmez ve kimseye alarm gitmez: +ret, run log'unda bir `WARNING`'dir (bkz. *Sık hatalar*). Yeni bir hedefe güvenmeden +önce ona bir test event'i gönderin. Birden çok hedef için, her biri kendi `webhook.url_env`'ine pinlenmiş birden çok ayrı ForgeLM training config'i çalıştırın @@ -99,7 +117,7 @@ Gerçek `WebhookConfig` (bkz. `forgelm/config.py::WebhookConfig`): | Alan | Vars. | Notlar | |---|---|---| | `url` | `null` | Inline URL — secret hijyeni için `url_env`'i tercih edin. | -| `url_env` | `null` | URL'i taşıyan env var adı. Set edildiğinde `url`'i override eder. | +| `url_env` | `null` | URL'i taşıyan env var adı. Yalnızca `url` boşken okunur; ikisi de set ise `url` kazanır. | | `notify_on_start` | `true` | `training.start` VE `pipeline.started`'ı gate'ler. | | `notify_on_success` | `true` | `training.success`, `approval.required` VE başarılı bir `pipeline.completed`'ı gate'ler. | | `notify_on_failure` | `true` | `training.failure`, `training.reverted`, `pipeline.stage_reverted` VE başarısız bir `pipeline.completed`'ı gate'ler. | @@ -118,7 +136,7 @@ payload zaten curated), ve routing hepsi ForgeLM'in dışında yaşar. - **TLS şiddetle önerilir.** ForgeLM hem HTTPS hem HTTP webhook URL'lerine izin verir — HTTP hedefleri `Webhook URL uses HTTP (not HTTPS). Data will be sent unencrypted.` uyarısı loglar ama varsayılan olarak reddedilmez (bkz. `forgelm/webhook.py` `_send`). Regüle bir ortamda `webhook.require_https: true` ile plaintext bir `http://` URL'ini hard failure yapın (teslimat reddedilir, gönderilmez). Üretimde `https://` URL'leri pinleyin. - **Curated payload.** ForgeLM webhook payload'larına asla raw eğitim verisi, tam config'ler veya unredacted PII koymaz. Notifier sabit-şekilli bir JSON sarar; `webhook.redact` toggle'ı yoktur çünkü kullanıcı-kontrollü redakte edilecek bir şey yok. - **SSRF guard.** ForgeLM iç IP'lere işaret eden webhook URL'lerini engeller — RFC 1918, loopback, link-local (bulut IMDS `169.254.169.254` dahil), RFC 6598 Shared Address Space / Carrier-Grade-NAT (`100.64.0.0/10`; bu blok, Alibaba Cloud ECS'in IMDS'ini de kapsar — `100.100.100.200` — stdlib `ipaddress` predicate'leri bu bloğu kendiliğinden private veya reserved olarak işaretlemez), ayrıca reserved ve multicast hedefler — `webhook.allow_private_destinations: true` ile açıkça opt-in olmadıkça. Yanlış konfigüre koşuların iç ağınızı sondalamasını veya payload'ı bir bulut metadata servisine sızdırmasını önler. -- **HMAC body imzalama yok.** ForgeLM webhook gövdelerini imzalamaz — hedef-tarafı authenticity TLS + `url_env` üzerinden URL gizliliği artı alıcı sistemin bearer-token / signed-request kontrollerine (Slack signing secret, Teams connector token) düşer. +- **HMAC body imzalama yok.** ForgeLM webhook gövdelerini imzalamaz — hedef-tarafı authenticity TLS + `url_env` üzerinden URL gizliliği artı alıcının kendi uyguladığı kimlik doğrulamaya düşer. Bir Slack incoming webhook'unun gizli URL'sinden başka doğrulaması yoktur (Slack'in signing secret'ı Slack'ten *gelen* istekleri doğrular, bir webhook URL'sine gelenleri değil); bu yüzden özel bir relay'in önüne kendi kimlik doğrulamanızı koyun: ForgeLM imza göndermez. ## Sık hatalar diff --git a/docs/usermanuals/tr/reference/cli.md b/docs/usermanuals/tr/reference/cli.md index e1da7e23..c552e54b 100644 --- a/docs/usermanuals/tr/reference/cli.md +++ b/docs/usermanuals/tr/reference/cli.md @@ -44,7 +44,7 @@ Bunlardan herhangi biri için `forgelm --help`. | `--dry-run` | Config'i ve model/dataset erişimini doğrula; eğitim yok. | | `--fit-check` | Eğitim VRAM tahmini; model yüklenmez. `--config` gerektirir. | | `--resume [PATH]` | Eğitime kaldığı yerden devam. Çıplak `--resume` son checkpoint'i otomatik bulur; `--resume PATH` belirli bir yerden. | -| `--offline` | Air-gap modu: tüm HF Hub ağ çağrılarını kapat. Modeller ve dataset'ler yerel olarak mevcut olmalı. | +| `--offline` | Offline mod: Hugging Face offline değişkenlerini ayarlar; Hub, transformers ve datasets ağ çağrısı yapmaz. Modeller ve dataset'ler yerel olarak mevcut olmalı. Webhook'ları, judge veya sentetik-veri API istemcilerini ve deney izleyicilerini durdurmaz. | | `--benchmark-only MODEL_PATH` | Mevcut bir model üzerinde benchmark koştur, eğitim yok. `evaluation.benchmark` config'i gerektirir. | | `--merge` | Config'in `merge:` bloğundan model birleştirmeyi koştur. Eğitim yok. | | `--stage NAME` | Yalnızca çok aşamalı pipeline'lar: adı verilen tek bir stage'i izole koştur. İlk olmayan stage'ler önceki stage'in on-disk çıktısına ya da `--input-model` override'ına ihtiyaç duyar. | @@ -328,7 +328,7 @@ Tam kontrat için bkz. [Exit Kodları](#/reference/exit-codes). | `FORGELM_OPERATOR` | Audit event'lerinde kaydedilen operatör kimliği (`getpass.getuser()@hostname`'i override eder) | | `FORGELM_ALLOW_ANONYMOUS_OPERATOR` | `1` olduğunda audit log'un anonim operatör kaydetmesine izin verir (aksi halde çözülemeyen kimlik hatası) | | `FORGELM_AUDIT_SECRET` | Audit log chain için HMAC imza anahtarı (tahrif tespitini açar) | -| `FORGELM_GGUF_CONVERTER` | Özel `convert-hf-to-gguf.py` script'inin yolu | +| `FORGELM_GGUF_CONVERTER` | Bir llama.cpp checkout'u içindeki `convert_hf_to_gguf.py`'nin yolu. Pratikte zorunlu: hiçbir `llama-cpp-python` wheel'i script'i ForgeLM'in aradığı yere koymaz ([Kurulum](#/getting-started/installation) sayfasına bakın) | ## Sık pattern'ler diff --git a/docs/usermanuals/tr/reference/configuration.md b/docs/usermanuals/tr/reference/configuration.md index 39f8e44b..753f164c 100644 --- a/docs/usermanuals/tr/reference/configuration.md +++ b/docs/usermanuals/tr/reference/configuration.md @@ -230,11 +230,11 @@ Merge alanı `algorithm` değil `method`'dur, `dare_ties` seçeneği, `base_mode ```yaml distributed: strategy: null # null (tek GPU, dağıtık sarmalama yok) | deepspeed | fsdp - deepspeed_config: null # bir DeepSpeed JSON'a yol, veya preset adı: zero2 | zero3 | zero3_offload - fsdp_strategy: "full_shard" # full_shard | shard_grad_op | no_shard | hybrid_shard - fsdp_auto_wrap: true # transformer katmanlarını otomatik sarmala (önerilir) + deepspeed_config: null # bir DeepSpeed JSON'a yol (preset adları zero2 | zero3 | zero3_offload kaynak checkout'u ister) + fsdp_strategy: "full_shard" # full_shard | shard_grad_op + fsdp_auto_wrap: true # transformers 5.10+'ta yok sayılır (FSDP2 transformer katmanlarını hep sarar) fsdp_offload: false # forward/backward arasında FSDP parametrelerini CPU'ya offload et - fsdp_backward_prefetch: "backward_pre" # backward_pre | backward_post + fsdp_backward_prefetch: "backward_pre" # backward_pre | backward_post; transformers 5.10+'ta yok sayılır fsdp_state_dict_type: "FULL_STATE_DICT" # FULL_STATE_DICT | SHARDED_STATE_DICT ``` @@ -264,8 +264,8 @@ compliance: ```yaml webhook: - url: null # Slack / Teams / Discord / özel; url_env tercihli - url_env: null # webhook URL'sini taşıyan env değişkeni + url: null # Slack incoming webhook'u ya da Teams / Discord için relay; url_env tercihli + url_env: null # webhook URL'sini taşıyan env değişkeni; yalnızca url boşken okunur notify_on_start: true notify_on_success: true notify_on_failure: true @@ -371,7 +371,7 @@ retention: staging_ttl_days: 7 # forgelm reject sonrası staging modelini saklama süresi ephemeral_artefact_retention_days: 90 # uyumluluk paketleri, denetim raporları raw_documents_retention_days: 90 # ingest edilen PDF/DOCX/EPUB/TXT/Markdown - enforce: "log_only" # log_only | warn_on_excess | block_on_excess + enforce: "log_only" # ayrılmış: henüz hiçbir komut onu kullanmaz (log_only | warn_on_excess | block_on_excess) ``` ## `pipeline:` diff --git a/docs/usermanuals/tr/reference/exit-codes.md b/docs/usermanuals/tr/reference/exit-codes.md index fd21ec0e..6a0fc562 100644 --- a/docs/usermanuals/tr/reference/exit-codes.md +++ b/docs/usermanuals/tr/reference/exit-codes.md @@ -11,7 +11,7 @@ ForgeLM'in exit kodları kamuya açık bir kontrattır. CI/CD hatları, schedule | Exit | Sabit | Anlam | Tipik CI aksiyonu | |---|---|---|---| -| **0** | `EXIT_SUCCESS` | Koşu tamamlandı ve checkpoint terfi etti. `evaluation.auto_revert: true` ile tüm kapılar da geçti; sevk edilen varsayılan `auto_revert: false` ile başarısız bir benchmark/güvenlik/judge kapısı **JSON çıktısına kaydedilir ama exit kodunu değiştirmez** — aşağıdaki ["exit 0 tam olarak ne garanti eder"](#exit-0-tam-olarak-ne-garanti-eder) bölümüne bakın. | Hattı sürdür (`auto_revert` kapalıysa kapı bloklarını parse et) | +| **0** | `EXIT_SUCCESS` | Koşu tamamlandı ve checkpoint terfi etti. `evaluation.auto_revert: true` ile karara varan tüm kapılar da geçti (karar üretemeyen bir güvenlik değerlendirmesi de modeli korur ve 0 ile çıkar); sevk edilen varsayılan `auto_revert: false` ile başarısız bir benchmark/güvenlik/judge kapısı **JSON çıktısına kaydedilir ama exit kodunu değiştirmez** — aşağıdaki ["exit 0 tam olarak ne garanti eder"](#exit-0-tam-olarak-ne-garanti-eder) bölümüne bakın. | Hattı sürdür (`auto_revert` kapalıysa kapı bloklarını parse et) | | **1** | `EXIT_CONFIG_ERROR` | YAML geçersiz, dosya yok, env var ayarsız veya argüman bozuk. | Hızlı başarısız | | **2** | `EXIT_TRAINING_ERROR` | Eğitim sırasında runtime hatası (config veya değerlendirme kapısı dışı her ele alınmamış istisna: data yükleme, OOM, NaN loss, I/O başarısızlığı, mid-stream audit-iteration OSError). | İncele; logları yüzeyle | | **3** | `EXIT_EVAL_FAILURE` | Bir benchmark/güvenlik/judge kapısı geçemedi **ve** model otomatik geri alındı (`evaluation.auto_revert: true` gerektirir). `auto_revert: false` ile başarısız bir kapı exit 3 üretmez — koşu, JSON kapı bloklarına kaydedilen başarısızlıkla 0 çıkar. | İncele; terfi ETTİRME | @@ -112,7 +112,7 @@ Exit kodu kontrat tek başına yeterli — POSIX kabuklarda `$?`, cmd'de `%ERROR **Kapılar ve exit 0.** *Geçen* bir benchmark/güvenlik/judge kapısının exit-0 garantisinin parçası olup olmadığı `evaluation.auto_revert`'e bağlıdır: -- `evaluation.auto_revert: true` ile (EU AI Act yüksek-riskli varsayılanı), başarısız bir kapı modeli otomatik geri alır ve **3** ile çıkar — yani exit 0 *gerçekten* tüm konfigüre kapıların geçtiği anlamına gelir. +- `evaluation.auto_revert: true` ile (EU AI Act yüksek-riskli varsayılanı), başarısız bir kapı modeli otomatik geri alır ve **3** ile çıkar — yani exit 0, karara varan her konfigüre kapının geçtiği anlamına gelir. **Tek istisna:** karar üretemeyen bir güvenlik değerlendirmesi (`safety.evaluation_completed: false` — sınıflandırıcı yüklenemedi, probe dosyası yok ya da boş, ya da çoğu karar okunamadı) `auto_revert: true` olsa bile modeli korur ve 0 ile çıkar (onay kapısıyla 4); JSON çıktısındaki `safety.passed` ve `safety.evaluation_completed`'e bakın. - Sevk edilen varsayılan `evaluation.auto_revert: false` ile, başarısız bir kapı **kaydedilir** (JSON çıktısındaki `benchmark` / `safety` / `judge` bloğu `*_passed: false` taşır) ama model yine terfi eder ve koşu **0** ile çıkar. Bu JSON bloklarını okuyun; kapı başarısını yalnızca exit 0'dan çıkarsamayın. Tasarım gereği "kısmi başarı" exit kodu yok — başarısız bir kapının exit kodunu değiştirmesini istiyorsanız `auto_revert`'i açın. diff --git a/docs/usermanuals/tr/reference/json-output.md b/docs/usermanuals/tr/reference/json-output.md index a4421064..e1f3a911 100644 --- a/docs/usermanuals/tr/reference/json-output.md +++ b/docs/usermanuals/tr/reference/json-output.md @@ -178,7 +178,7 @@ Eğitim tamamlanana kadar çalıştığında pipeline **stdout**'a bir sonuç en | Anahtar | Tip | Not | |---|---|---| -| `success` | bool | Çalışma tamamlandığında (stage'lenmiş, onay-bekleyen çalışma dahil) `true`. Bir gate modeli auto-revert ettiğinde `false`. | +| `success` | bool | Çalışma tamamlandığında (stage'lenmiş, onay-bekleyen çalışma ve güvenlik değerlendirmesi karar üretemeyen çalışma dahil; `safety.evaluation_completed`'e bakın) `true`. Bir gate modeli auto-revert ettiğinde `false`. | | `error` | string | **Yalnızca** `success` `false` iken bulunur, başarı yolunda asla. Gate'in kendi gerekçesini taşır — hangi gate'in hangi eşiğe karşı tetiklendiğini — çünkü her gate `3` ile çıkar, dolayısıyla exit kodu tek başına ayırt edemez. Bir başarısızlık zarfa gerekçesiz ulaşırsa alan bunu açıkça söyler ve `audit_log.jsonl`'a yönlendirir; atlanmaz. | | `metrics` | object | Eğitim/eval/gate metrikleri (ör. `eval_loss`, `benchmark/average`, `safety/safe_ratio`). Değerler sayıdır; **istisna**, sonlu çıkmayan bir ölçüm `"nan"` / `"inf"` / `"-inf"` string'i olarak yazılır — JSON'da sonlu olmayan sayı literali yoktur, alternatifi hiçbir katı ayrıştırıcının kabul etmediği bir çıktıydı. Bu alanları sayı-veya-string olarak tipleyin. | | `final_model_path` | str | Model artefactlarının yaşadığı yer. Onay-bekleyen bir çalışma için bu, `forgelm approve` promote edene kadar **staging** dizinidir. | diff --git a/docs/usermanuals/tr/reference/yaml-templates.md b/docs/usermanuals/tr/reference/yaml-templates.md index 9288a614..5baf2e0d 100644 --- a/docs/usermanuals/tr/reference/yaml-templates.md +++ b/docs/usermanuals/tr/reference/yaml-templates.md @@ -23,7 +23,7 @@ Beş şablon ve her birinin optimize ettiği kullanıcı yolu: | `customer-support` | SFT | Multi-turn yardımcı + güvenli asistan — en yaygın production pattern. | | `code-assistant` | SFT | Kod-tamamlama / pair-programming modeli. | | `domain-expert` | SFT | Domain-spesifik corpus üzerinde BYOD (Bring Your Own Dataset). | -| `medical-qa-tr` | SFT | Türkçe tıbbi soru-cevap (regüle alan — `risk_assessment` + safety eval ön-bağlı ship olur). | +| `medical-qa-tr` | SFT | Türkçe tıbbi soru-cevap (regüle alan — örnek cevaplar bir güvenlik uyarısı taşır, ama şablon güvenlik değerlendirmesini etkinleştirmez ve `risk_assessment` metadata'sı içermez. Klinik karar desteği için değildir ve bunları eklemek bunu değiştirmez; demo dışındaki bir kullanımdan önce `evaluation.safety.enabled: true` yap ve en üstteki `risk_assessment:` bloğunu doğru biçimde doldur). | | `grpo-math` | GRPO | Matematik benchmark'ları üzerinde reward-model fine-tuning (yerleşik format/length reward shaping). | Kanonik şablon içeriği **doğruluk kaynağıdır** — `forgelm quickstart`'ın diff --git a/docs/usermanuals/tr/training/distributed.md b/docs/usermanuals/tr/training/distributed.md index 4dcfd239..7c1dc991 100644 --- a/docs/usermanuals/tr/training/distributed.md +++ b/docs/usermanuals/tr/training/distributed.md @@ -66,7 +66,7 @@ ZeRO-2 optimizer state'i sharder (Adam gibi adaptif optimizer'larda en ağır VR ```yaml distributed: strategy: "deepspeed" - deepspeed_config: "zero2" # preset adı (veya bir DeepSpeed JSON dosya yolu) + deepspeed_config: "./ds_zero2.json" # bir DeepSpeed JSON dosya yolu ("zero2" gibi preset adları kaynak checkout'u ister) training: gradient_accumulation_steps: 4 @@ -95,7 +95,7 @@ training: gradient_accumulation_steps: 8 ``` -`zero3_offload` preset'i, optimizer state ve parametreleri CPU'ya boşaltarak 70B'i 8×24 GB'a sığdırır. ZeRO-Infinity'nin NVMe offload'u hazır bir preset değildir — bunun için `deepspeed_config`'i, `offload_param`/`offload_optimizer` alanları `device: nvme` olarak ayarlanmış özel bir DeepSpeed JSON dosyasına yönlendirin. +`zero3_offload` preset'i optimizer state ve parametreleri CPU'ya boşaltır, hızdan GPU belleği kazandırır; belirli bir modelin sığıp sığmayacağı GPU belleğinize ve CPU RAM'inize bağlıdır. `zero2`, `zero3` ve `zero3_offload` preset adları yalnızca kaynak checkout'unda çözülür (JSON dosyaları pip wheel'inde ya da Docker imajında değil, deponun `configs/deepspeed/` klasöründedir) ve `zero2` preset'i şu an bozuktur ([#410](https://github.com/HodeTech/ForgeLM/issues/410)); bu yüzden `deepspeed_config`'e kontrolünüzdeki bir JSON dosyasının yolunu verin. ZeRO-Infinity'nin NVMe offload'u bir preset değildir — bunun için `deepspeed_config`'i, `offload_param`/`offload_optimizer` alanları `device: nvme` olarak ayarlanmış özel bir DeepSpeed JSON dosyasına yönlendirin. | Model | GPU | ZeRO-3 + offload? | |---|---|---| @@ -111,12 +111,14 @@ FSDP, ZeRO-3 gibi sharder ama PyTorch'un yerli FullyShardedDataParallel'ini kull ```yaml distributed: strategy: "fsdp" - fsdp_strategy: "full_shard" # full_shard | shard_grad_op | no_shard | hybrid_shard - fsdp_auto_wrap: true # transformer katmanlarını otomatik sar (önerilir) + fsdp_strategy: "full_shard" # full_shard | shard_grad_op + fsdp_auto_wrap: true # transformers 5.10+'ta yok sayılır (FSDP2 transformer katmanlarını hep sarar) fsdp_offload: false # forward/backward arasında parametreleri CPU'ya boşalt fsdp_state_dict_type: "FULL_STATE_DICT" # FULL_STATE_DICT | SHARDED_STATE_DICT ``` +`no_shard` ve `hybrid_shard` config yüklenirken reddedilir: ForgeLM'in gerektirdiği transformers 5.10+ FSDP2 çalıştırır ve FSDP2 bu ikisini desteklemez. `fsdp_auto_wrap: false` ve `fsdp_backward_prefetch: backward_post` hâlâ kabul edilir ancak FSDP2'de hiçbir şeyi değiştirmez; bu yüzden ikisinden biri ayarlandığında ForgeLM bir uyarı loglar. + `DistributedConfig`'te `fsdp_auto_wrap_policy` veya `fsdp_offload_params` alanı yoktur — auto-wrap düz bir `fsdp_auto_wrap` boolean'ıdır, CPU offload ise `fsdp_offload`'dur (`_params` eki yoktur). ## Gradient accumulation diff --git a/forgelm/cli/_config_load.py b/forgelm/cli/_config_load.py index 4e5ac477..04c846e7 100644 --- a/forgelm/cli/_config_load.py +++ b/forgelm/cli/_config_load.py @@ -14,6 +14,29 @@ from ._logging import logger +def _warn_reserved_retention_enforce(config) -> None: + """``retention.enforce`` is validated and offered by both wizards, but no command reads it. + + A config that sets ``warn_on_excess`` or ``block_on_excess`` is asking for a gate that does not exist; say so, + rather than accept it silently. (This lives here, not in the schema, to keep ``config.py`` within its + deferred-split line budget.) + """ + mode = getattr(getattr(config, "retention", None), "enforce", "log_only") + if mode != "log_only": + logger.warning( + "retention.enforce=%r is accepted but no command acts on it yet: retention horizons are only reported " + "by `forgelm purge --check-policy`, and the trainer is not warned or blocked.", + mode, + ) + + +def _reject_unsloth_unsupported_overrides(config) -> None: + """Fail at config load (exit 1, also under ``--dry-run``) when the unsloth backend cannot apply an override.""" + from ..model import _reject_unsloth_unsupported_overrides as reject + + reject(config) + + def _load_config_or_exit(config_path: str, json_output: bool): """Load config and translate exceptions into the right exit code. @@ -25,7 +48,10 @@ def _load_config_or_exit(config_path: str, json_output: bool): """ try: logger.info("Loading configuration from %s...", config_path) - return load_config(config_path) + config = load_config(config_path) + _warn_reserved_retention_enforce(config) + _reject_unsloth_unsupported_overrides(config) + return config except FileNotFoundError as e: msg = f"Config file not found: {e}" except ConfigError as e: @@ -47,6 +73,20 @@ def _load_config_or_exit(config_path: str, json_output: bool): sys.exit(EXIT_CONFIG_ERROR) +def _disable_hub_telemetry() -> None: + """Keep the ``forgelm`` CLI from sending Hugging Face's anonymous usage ping. + + A training run makes ``huggingface_hub`` send a small usage ping to huggingface.co: TRL's trainers call + ``send_telemetry`` when they are built, and ``huggingface_hub`` skips it only when ``HF_HUB_DISABLE_TELEMETRY`` is + true or offline mode is on. ForgeLM's privacy page says the tool sends no usage data anywhere, so the CLI sets + the variable by default. ``setdefault`` leaves an operator's own value alone (``HF_HUB_DISABLE_TELEMETRY=0`` + opts back in). It must run before ``huggingface_hub`` is imported, which reads it once at import; importing + ``forgelm.cli`` does not import it, and ``main()`` calls this first. Using ForgeLM as a library does not go + through ``main()``, so a library user sets the variable themselves. + """ + os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1") + + def _apply_offline_flag(config, offline_arg: bool) -> None: if offline_arg: config.model.offline = True @@ -55,3 +95,27 @@ def _apply_offline_flag(config, offline_arg: bool) -> None: os.environ["TRANSFORMERS_OFFLINE"] = "1" os.environ["HF_DATASETS_OFFLINE"] = "1" logger.info("Offline mode enabled. All HF Hub network calls are disabled.") + + +def _warn_inert_fsdp_options(config) -> None: + """Say so when an FSDP knob that transformers' FSDP2 default ignores is set away from its default. + + ``fsdp_auto_wrap: false`` and ``fsdp_backward_prefetch: backward_post`` are still accepted, but under + transformers 5.10+ (FSDP2) the transformer-layer wrap policy is always applied and no backward prefetch is + passed on, so the run behaves as if the defaults had been kept. (This lives here, not in the schema, to keep + ``config.py`` within its deferred-split line budget.) + """ + dist = getattr(config, "distributed", None) + if getattr(dist, "strategy", None) != "fsdp": + return + if not dist.fsdp_auto_wrap: + logger.warning( + "distributed.fsdp_auto_wrap=false has no effect: transformers 5.10+ runs FSDP2, which always wraps " + "transformer layers." + ) + if dist.fsdp_backward_prefetch != "backward_pre": + logger.warning( + "distributed.fsdp_backward_prefetch=%r has no effect: transformers 5.10+ runs FSDP2, which does not take " + "a backward-prefetch policy.", + dist.fsdp_backward_prefetch, + ) diff --git a/forgelm/cli/_dispatch.py b/forgelm/cli/_dispatch.py index e4b64387..972ee75d 100644 --- a/forgelm/cli/_dispatch.py +++ b/forgelm/cli/_dispatch.py @@ -10,7 +10,7 @@ import sys from .._strict_json import dumps_strict -from ._config_load import _apply_offline_flag, _load_config_or_exit +from ._config_load import _apply_offline_flag, _disable_hub_telemetry, _load_config_or_exit, _warn_inert_fsdp_options from ._exit_codes import EXIT_CONFIG_ERROR, EXIT_SUCCESS, EXIT_TRAINING_ERROR, _clamp_exit_code from ._logging import _setup_logging, logger from ._no_train_modes import _maybe_run_no_train_mode @@ -163,6 +163,7 @@ def main(): # (= 128+SIGINT) — outside the documented public 0/1/2/3/4 surface # and surprising to CI/CD scripts that branch on exit code 2. _force_utf8_console_streams() + _disable_hub_telemetry() try: return _main_inner() except KeyboardInterrupt: @@ -241,6 +242,7 @@ def _main_inner() -> None: config = _load_config_or_exit(args.config, json_output) _apply_offline_flag(config, args.offline) + _warn_inert_fsdp_options(config) if config.pipeline is not None: _dispatch_pipeline_mode(config, args, json_output) diff --git a/forgelm/cli/_parser.py b/forgelm/cli/_parser.py index 562c6343..4bd75f23 100644 --- a/forgelm/cli/_parser.py +++ b/forgelm/cli/_parser.py @@ -118,7 +118,9 @@ def _add_deploy_subcommand(subparsers) -> None: choices=["ollama", "vllm", "tgi", "hf-endpoints"], help="Target serving runtime.", ) - p.add_argument("--output", type=str, default=None, metavar="FILE", help="Output file path (default: auto).") + p.add_argument( + "--output", type=str, default=argparse.SUPPRESS, metavar="FILE", help="Output file path (default: auto)." + ) p.add_argument("--system", type=str, default=None, metavar="PROMPT", help="System prompt (Ollama only).") p.add_argument("--max-length", type=_positive_int, default=4096, help="Context window length (default: 4096).") p.add_argument( @@ -174,13 +176,14 @@ def _add_quickstart_subcommand(subparsers) -> None: p.add_argument( "--output", type=str, - default=None, + default=argparse.SUPPRESS, metavar="FILE", help="Where to write the generated YAML (default: ./configs/