diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..8a6c241 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Preserve upstream document bytes and original whitespace on every platform. +upstream/** -text -whitespace diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..76aab35 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,90 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + mcpp: + name: mcpp (${{ matrix.os }}, ${{ matrix.toolchain }}) + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-24.04, toolchain: 'gcc@16.1.0' } + - { os: ubuntu-24.04, toolchain: 'llvm@23.1.3' } + - { os: macos-15, toolchain: 'llvm@23.1.3' } + - { os: windows-2025, toolchain: 'llvm@23.1.3' } + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + defaults: + run: + shell: bash + env: + XLINGS_NON_INTERACTIVE: 1 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Mwrap identity and bilingual README contract + run: python3 tools/check_mwrap_release.py + + - name: Install Xlings (Linux/macOS) + if: runner.os != 'Windows' + run: | + curl -fsSL https://raw.githubusercontent.com/openxlings/xlings/refs/heads/main/tools/other/quick_install.sh | bash + echo "$HOME/.xlings/subos/current/bin" >> "$GITHUB_PATH" + + - name: Install Xlings (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + irm https://raw.githubusercontent.com/openxlings/xlings/main/tools/other/quick_install.ps1 | iex + "$env:USERPROFILE\.xlings\subos\current\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Install project tools (.xlings.json) + run: | + for attempt in 1 2 3; do + xlings update > /dev/null 2>&1 || true + if xlings install; then break; fi + [ "$attempt" = 3 ] && exit 1 + sleep 30 + done + mcpp --version + + - name: Select the toolchain + run: | + name="${{ matrix.toolchain }}" + mcpp toolchain install "${name%@*}" "${name#*@}" + mcpp toolchain default "$name" + + - name: Build the modules (and with the c-api feature) + run: | + mcpp build + mcpp build --features c-api + + - name: Tests (the queues through the module from many threads, the constants, header mixing) + run: mcpp test + + - name: Tests with the c-api and no-thread-local features (the C API; gtest beside the queue) + working-directory: tests/c_api + run: mcpp test + + - name: Export audit (every declaration of moodycamel, every helper of details, the C API, every macro) + run: python3 tools/check_exports.py + + - name: Upstream's tests/unittests built against the module + run: python3 tools/run_conformance.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f0c6447 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,49 @@ +name: Release +on: + push: + tags: ["mwrap-*"] +permissions: + contents: read +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write + actions: read + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Check tag, upstream version and bilingual entry pages + run: python3 tools/check_mwrap_release.py --tag "$GITHUB_REF_NAME" + - name: Require successful source CI at this exact commit + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + result=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml --commit "$GITHUB_SHA" --limit 1 --json status,conclusion --jq '.[0] | .status == "completed" and .conclusion == "success"') + test "$result" = true || { echo 'No successful source CI at the tag commit'; exit 1; } + - name: Build immutable source asset and publish + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + version="${GITHUB_REF_NAME#mwrap-}" + repo="${GITHUB_REPOSITORY#*/}" + asset="${repo}-${version}.tar.gz" + git archive --format=tar.gz --prefix="${repo}-${version}/" "$GITHUB_SHA" > "$RUNNER_TEMP/$asset" + (cd "$RUNNER_TEMP" && sha256sum "$asset" > "$asset.sha256") + if gh release view "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + echo 'Release already exists; refusing to change immutable assets'; exit 1 + fi + extra=() + if [[ "$version" == *-rc* ]]; then extra+=(--prerelease); fi + gh release create "$GITHUB_REF_NAME" "$RUNNER_TEMP/$asset" "$RUNNER_TEMP/$asset.sha256" \ + --repo "$GITHUB_REPOSITORY" --verify-tag --title "$repo $version" \ + --notes "Module wrapper for upstream $version. New mwrap package identities become installable after mcpp-index publication; see the bilingual README for the tested support scope." "${extra[@]}" diff --git a/.github/workflows/templates.yml b/.github/workflows/templates.yml new file mode 100644 index 0000000..f919613 --- /dev/null +++ b/.github/workflows/templates.yml @@ -0,0 +1,28 @@ +name: Actual project templates +on: + push: + branches: [main] + pull_request: + workflow_dispatch: +permissions: + contents: read +concurrency: + group: templates-${{ github.ref }} + cancel-in-progress: true +env: + MCPP_VERSION: "2026.10.26.1" +jobs: + templates: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - name: Install the pinned mcpp + shell: bash + run: | + archive="mcpp-${MCPP_VERSION}-linux-x86_64.tar.gz" + curl -L -fsS -o "$archive" "https://github.com/mcpp-community/mcpp/releases/download/v${MCPP_VERSION}/$archive" + tar -xzf "$archive" + echo "$PWD/mcpp-${MCPP_VERSION}-linux-x86_64/bin" >> "$GITHUB_PATH" + - name: Actual scaffold, strict build and run of the default template + run: python3 tools/test_template.py --toolchain gcc@16.1.0 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b6dfbae --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +target +compile_commands.json +mcpp.lock +.mcpp/ +.xlings/ +doctor.log +__pycache__/ diff --git a/.xlings.json b/.xlings.json new file mode 100644 index 0000000..730a8ff --- /dev/null +++ b/.xlings.json @@ -0,0 +1,5 @@ +{ + "workspace": { + "xim:mcpp": "2026.10.26.1" + } +} diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..dccd076 --- /dev/null +++ b/NOTICE @@ -0,0 +1,16 @@ +concurrentqueue-m: C++ module wrapper +Copyright (c) 2026 mcpp-community contributors + +The wrapper contributions are licensed under the Apache License, Version 2.0. +Upstream code, copied interfaces and dependencies retain their original +licenses and copyright notices. moodycamel::ConcurrentQueue is distributed +under the Simplified BSD license or, at your option, the Boost Software +License 1.0; its lightweight semaphore embeds Jeff Preshing's semaphore under +the zlib license: + + Copyright (c) 2013-2016, Cameron Desrochers. + Copyright (c) 2015 Jeff Preshing. + +Its original README and license are retained in upstream/concurrentqueue/. +The headers themselves are provided by compat.concurrentqueue from the same +release. Provenance and checksums are recorded in upstream/documents.json. diff --git a/README.md b/README.md index b915dad..11bfefe 100644 --- a/README.md +++ b/README.md @@ -1 +1,94 @@ -# concurrentqueue-m — moodycamel::ConcurrentQueue as C++23 modules (mcpplibs.mwrap.concurrentqueue) +# concurrentqueue + +> moodycamel::ConcurrentQueue, the lock-free multi-producer multi-consumer queue, through C++23 modules. + +**English** · [简体中文](README.zh.md) · [Upstream](https://github.com/cameron314/concurrentqueue) · [Releases](https://github.com/mcpplibs/concurrentqueue-m/releases) · [mcpp](https://github.com/mcpp-community/mcpp) · [Index](https://github.com/mcpplibs/mcpp-index) + +[![C++23](https://img.shields.io/badge/C%2B%2B-23-blue.svg)](https://en.cppreference.com/w/cpp/23) +[![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) +[![mcpp-community](https://img.shields.io/badge/mcpp--community-mwrap-blue)](https://github.com/mcpp-community) +[![CI](https://github.com/mcpplibs/concurrentqueue-m/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/mcpplibs/concurrentqueue-m/actions/workflows/ci.yml) + +The upstream API is unchanged: `moodycamel::ConcurrentQueue`, +`BlockingConcurrentQueue`, `ProducerToken`, `ConsumerToken` and +`LightweightSemaphore` are the upstream entities of the three headers, +templates over your element and `Traits` types. The headers are +`compat.concurrentqueue` 1.0.5. The value macros are constants. Upstream's +own `tests/unittests` (all 45 tests) pass on every CI run, built against the +module instead of the headers. + +## Modules + +- `import mcpplibs.mwrap.concurrentqueue;`: the queues, tokens, traits, semaphore and `swap` in `moodycamel`, re-exporting the one below +- `import mcpplibs.mwrap.concurrentqueue.macros;`: `MOODYCAMEL_EXCEPTIONS_ENABLED`, `MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED` … in `mcpplibs::mwrap::concurrentqueue::macros` +- `import mcpplibs.mwrap.concurrentqueue.c_api;`: the C API (`moodycamel_cq_*`, `moodycamel_bcq_*`), with the `c-api` feature + +> More Modules API -> [Usage and API](docs/usage.md) + +## Getting started + +```sh +mcpp new pipeline --template mcpplibs.mwrap.concurrentqueue@1.0.5 +cd pipeline +mcpp run +``` + +The default template runs a producer-consumer pipeline. Producer threads +enqueue numbers in bulk through their tokens. Consumers wait on a blocking +queue with a timeout and count primes. + +```toml +[dependencies] +"mcpplibs.mwrap.concurrentqueue" = "1.0.5" +``` + +```cpp +import std; +import mcpplibs.mwrap.concurrentqueue; + +int main() { + moodycamel::ConcurrentQueue> tasks; + std::jthread worker([&] { tasks.enqueue([] { std::println("ran on the main thread"); }); }); + worker.join(); + std::function task; + while (tasks.try_dequeue(task)) task(); +} +``` + +| Feature | Effect | +|---|---| +| `c-api` | upstream's C API: compat.concurrentqueue's sources and the `c_api` module | +| `no-thread-local` | `MOODYCAMEL_NO_THREAD_LOCAL`: no `thread_local`, implicit producers not recycled at thread exit | + +## Configuration and macros + +Configuration macros must be set before the module is built. The module +uses upstream's defaults, with `MOODYCAMEL_NOEXCEPT_CTOR` / `_ASSIGN` spelled +as the equivalent type traits so that GCC accepts the interface +([config.hpp](include/mwrap/concurrentqueue/config.hpp)). Per-queue settings +(block size, index sizes, allocation) are the `Traits` argument, as upstream +documents. The syntax macros (`MOODYCAMEL_NOEXCEPT`, +`MOODYCAMEL_DELETE_FUNCTION`, `MOODYCAMEL_TRY` …) stay in +``, included before the imports. With GCC, +a unit that includes `` (gtest does) cannot use the queue's +thread-exit path: use `no-thread-local` there. Details: +[docs/usage.md](docs/usage.md). + +## Build and testing + +Linux, macOS and Windows CI build the modules and run the tests: +multi-producer multi-consumer traffic, tokens, bulk operations, timed waits, +user types and traits, the semaphore, the C API and header mixing. CI also +runs the export audit ([tools/check_exports.py](tools/check_exports.py): every +declaration exported or listed with its reason, every macro mapped or kept) +and upstream's `tests/unittests` against the module +([tools/run_conformance.py](tools/run_conformance.py)). A separate job +scaffolds the default template as an installed package and runs it. + +## License + +[Apache-2.0](LICENSE) for the wrapper, by mcpp-community ([NOTICE](NOTICE)). +concurrentqueue remains under the Simplified BSD license or the Boost Software +License, and its semaphore under zlib +([upstream/concurrentqueue/LICENSE.md](upstream/concurrentqueue/LICENSE.md), +[README](upstream/concurrentqueue/README.md)). diff --git a/README.zh.md b/README.zh.md new file mode 100644 index 0000000..0740ef0 --- /dev/null +++ b/README.zh.md @@ -0,0 +1,86 @@ +# concurrentqueue + +> 通过 C++23 模块使用 moodycamel::ConcurrentQueue（无锁的多生产者多消费者队列）。 + +[English](README.md) · **简体中文** · [上游](https://github.com/cameron314/concurrentqueue) · [发布](https://github.com/mcpplibs/concurrentqueue-m/releases) · [mcpp](https://github.com/mcpp-community/mcpp) · [索引](https://github.com/mcpplibs/mcpp-index) + +[![C++23](https://img.shields.io/badge/C%2B%2B-23-blue.svg)](https://en.cppreference.com/w/cpp/23) +[![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) +[![mcpp-community](https://img.shields.io/badge/mcpp--community-mwrap-blue)](https://github.com/mcpp-community) +[![CI](https://github.com/mcpplibs/concurrentqueue-m/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/mcpplibs/concurrentqueue-m/actions/workflows/ci.yml) + +上游 API 保持不变：`moodycamel::ConcurrentQueue`、`BlockingConcurrentQueue`、 +`ProducerToken`、`ConsumerToken`、`LightweightSemaphore` 就是三个头文件里的上游 +实体，是以你的元素类型和 `Traits` 类型为参数的模板。头文件由 +`compat.concurrentqueue` 1.0.5 提供，表示值的宏映射为常量。每次 CI 都会把上游自带的 +`tests/unittests`（共 45 个测试）改为基于本模块构建（而不是头文件），并且全部通过。 + +## 模块 + +- `import mcpplibs.mwrap.concurrentqueue;`：`moodycamel` 中的队列、令牌、traits、信号量和 `swap`，并重新导出下面这个模块 +- `import mcpplibs.mwrap.concurrentqueue.macros;`：`MOODYCAMEL_EXCEPTIONS_ENABLED`、`MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED` 等，位于 `mcpplibs::mwrap::concurrentqueue::macros` +- `import mcpplibs.mwrap.concurrentqueue.c_api;`：C API（`moodycamel_cq_*`、`moodycamel_bcq_*`），需要 `c-api` 特性 + +> 更多模块 API -> [用法和 API](docs/usage.md) + +## 快速开始 + +```sh +mcpp new pipeline --template mcpplibs.mwrap.concurrentqueue@1.0.5 +cd pipeline +mcpp run +``` + +默认模板运行一条生产者-消费者流水线：生产者线程通过各自的令牌批量入队数字， +消费者在阻塞队列上带超时地等待，并统计其中的素数。 + +```toml +[dependencies] +"mcpplibs.mwrap.concurrentqueue" = "1.0.5" +``` + +```cpp +import std; +import mcpplibs.mwrap.concurrentqueue; + +int main() { + moodycamel::ConcurrentQueue> tasks; + std::jthread worker([&] { tasks.enqueue([] { std::println("ran on the main thread"); }); }); + worker.join(); + std::function task; + while (tasks.try_dequeue(task)) task(); +} +``` + +| 特性 | 作用 | +|---|---| +| `c-api` | 上游的 C API：compat.concurrentqueue 中的源文件和 `c_api` 模块 | +| `no-thread-local` | `MOODYCAMEL_NO_THREAD_LOCAL`：不使用 `thread_local`，线程退出时不回收隐式生产者 | + +## 配置与宏 + +配置宏必须在模块构建之前设定。模块使用上游默认配置，只是把 +`MOODYCAMEL_NOEXCEPT_CTOR` / `_ASSIGN` 写成等价的类型特征，这样 GCC 才能接受这个 +接口（[config.hpp](include/mwrap/concurrentqueue/config.hpp)）。单个队列的设置 +（块大小、索引大小、内存分配）按上游文档通过 `Traits` 参数传入。语法类的宏 +（`MOODYCAMEL_NOEXCEPT`、`MOODYCAMEL_DELETE_FUNCTION`、`MOODYCAMEL_TRY` 等）保留在 +`` 中，需要在 import 之前包含。使用 GCC 时，包含了 +`` 的翻译单元（gtest 会包含它）不能使用队列在线程退出时的回收路径， +这时请启用 `no-thread-local`。详见 [docs/usage.md](docs/usage.md)。 + +## 构建与测试 + +Linux、macOS 和 Windows 的 CI 会构建模块并运行测试，覆盖：多生产者多消费者流量、 +令牌、批量操作、带超时的等待、用户类型与 traits、信号量、C API，以及头文件与模块 +混用。CI 还会运行导出审计（[tools/check_exports.py](tools/check_exports.py)：每个 +声明都已导出或注明原因，每个宏都已映射或保留为宏），并基于本模块运行上游的 +`tests/unittests`（[tools/run_conformance.py](tools/run_conformance.py)）。另一个任务 +把默认模板当作已安装的包生成出来并运行。 + +## 许可证 + +封装部分采用 [Apache-2.0](LICENSE)，作者为 mcpp-community（[NOTICE](NOTICE)）。 +concurrentqueue 仍采用 Simplified BSD 许可证或 Boost Software License，其中的 +信号量部分采用 zlib 许可证 +（[upstream/concurrentqueue/LICENSE.md](upstream/concurrentqueue/LICENSE.md)、 +[README](upstream/concurrentqueue/README.md)）。 diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000..337c1df --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,85 @@ +# Using mcpplibs.mwrap.concurrentqueue + +`import mcpplibs.mwrap.concurrentqueue;` declares moodycamel::ConcurrentQueue 1.0.5 as ``, `` and `` do. The exported names are the upstream entities in namespace `moodycamel`, so a translation unit may include the headers too. The headers are `compat.concurrentqueue` 1.0.5's. + +## Modules + +| Module | What it holds | +|---|---| +| `mcpplibs.mwrap.concurrentqueue` | `moodycamel::ConcurrentQueue`, `BlockingConcurrentQueue`, `ConcurrentQueueDefaultTraits`, `ProducerToken`, `ConsumerToken`, `LightweightSemaphore`, every `swap` overload, and `ConcurrentQueueTests` (the class the queue befriends for upstream's tests); the adapted helpers of `moodycamel::details` (below); re-exports the macros module | +| `mcpplibs.mwrap.concurrentqueue.macros` | the value macros, in `mcpplibs::mwrap::concurrentqueue::macros` | +| `mcpplibs.mwrap.concurrentqueue.c_api` | with the `c-api` feature: upstream's C API, `moodycamel_cq_*` and `moodycamel_bcq_*` with `MoodycamelCQHandle`, `MoodycamelBCQHandle`, `MoodycamelValue`. Import it by name: mcpp's scanner admits no import under a feature test, so the root cannot re-export it. | + +The queues are templates over the element type and a `Traits` type (block size, index sizes, `malloc` / `free` …). Both are the importer's own types, instantiated in the importer as with the headers. + +## Features + +| Feature | Effect | +|---|---| +| `c-api` | builds compat.concurrentqueue's two C API sources and the `c_api` module | +| `no-thread-local` | upstream's `MOODYCAMEL_NO_THREAD_LOCAL`, for the module and its consumers: thread ids from `std::this_thread`, no `thread_local`, so implicit producers are not recycled at thread exit (as upstream always builds on ARM and with Clang in GNU mode) | + +Neither is on by default. + +## Configuration + +Configuration macros must be set before the module is built. The module is built with upstream's defaults, plus one adjustment, in ``. Upstream defines `MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr)` as `noexcept(expr)`, an expression that names `details::deref_noexcept`, a `static` function. In a template argument (`details::nomove_if<…>` in `enqueue_bulk`), that makes the queue expose a TU-local entity, which GCC rejects in a module interface. config.hpp spells the two macros with the type traits that give the same answers: `std::is_nothrow_constructible` and `std::is_nothrow_assignable`. This uses upstream's own configuration point: both macros are defined by concurrentqueue.h only when `MOODYCAMEL_NOEXCEPT` is not. Without exceptions, upstream's definitions stand. + +Per-queue configuration is the `Traits` argument. It needs no macro and works through the module unchanged: + +```cpp +struct SmallBlocks : moodycamel::ConcurrentQueueDefaultTraits { + static constexpr std::size_t BLOCK_SIZE = 4; +}; +moodycamel::ConcurrentQueue queue; +``` + +## Macros + +A named module cannot export a macro. `mcpplibs::mwrap::concurrentqueue::macros` holds the value macros, under their original names, as `inline constexpr bool`. Each is true exactly when the macro is defined where the module is built. + +- Derived by the headers: `MOODYCAMEL_EXCEPTIONS_ENABLED`, `MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED` (implicit producers are recycled at thread exit; upstream enables it for GCC 4.8+ and MSVC 2015+, but not for Clang in GNU mode, which reports `__GNUC__` 4.2, nor on ARM), and `MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC` (timed waits on `CLOCK_MONOTONIC`, glibc 2.30+). +- The configuration inputs, as the module was built: `MOODYCAMEL_NO_THREAD_LOCAL`, `MOODYCAMEL_QUEUE_INTERNAL_DEBUG`, `MCDBGQ_USE_RELACY`, `MCDBGQ_TRACKMEM`, `MCDBGQ_NOLOCKFREE_FREELIST`, `MCDBGQ_USEDEBUGFREELIST`, `MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX`, `MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH`. + +The syntax macros stay macros, in ``. That header is config.hpp plus the three headers, so it is the configuration the module has: + +| Macro | Why | +|---|---| +| `MOODYCAMEL_NOEXCEPT`, `MOODYCAMEL_NOEXCEPT_CTOR`, `MOODYCAMEL_NOEXCEPT_ASSIGN` | a specifier; expressions over an argument expression | +| `MOODYCAMEL_DELETE_FUNCTION` | `= delete` | +| `MOODYCAMEL_ALIGNAS`, `MOODYCAMEL_ALIGNOF`, `MOODYCAMEL_ALIGNED_TYPE_LIKE` | alignment specifiers and operators | +| `MOODYCAMEL_TRY`, `MOODYCAMEL_CATCH`, `MOODYCAMEL_RETHROW`, `MOODYCAMEL_THROW`, `MOODYCAMEL_CONSTEXPR_IF` | control flow | +| `MOODYCAMEL_MAYBE_UNUSED`, `MOODYCAMEL_NO_TSAN`, `MOODYCAMEL_THREADLOCAL` | attributes, a storage class | +| `MOODYCAMEL_EXPORT` | the C API's linkage annotation | + +Include the side header before the imports. While a macro is defined it expands even inside a qualified name, so `macros::MOODYCAMEL_EXCEPTIONS_ENABLED` is reachable there only with the macro set aside (`#pragma push_macro`, see [tests/header_mix_test.cpp](../tests/header_mix_test.cpp)). To mix a unit that includes the headers with the module, include `` rather than `` alone, so both see one configuration. + +## moodycamel::details + +`moodycamel::details` is the queue's implementation and is not exported. Its `static` (TU-local) helpers cannot be exported. Those that are not templates over arbitrary types are adapted in `mcpplibs::mwrap::concurrentqueue::details`, with the same names, and each calls upstream's body: `thread_id`, `thread_id_t`, `likely`, `unlikely`, `hash_thread_id`, and `circular_less_than` and `ceil_to_pow_2`, with one overload per unsigned integer type. `align_for`, `swap_relaxed`, `nomove` and `deref_noexcept` are templates over any type. The queue instantiates upstream's own, so they are not adapted. [tools/check_exports.py](../tools/check_exports.py) lists every name of the namespace with its reason. + +## GCC + +GCC 16.1 has three module behaviours the wrapper works with: + +- The queue's member templates call upstream's `static` helpers, so they expose TU-local entities from the global module fragment. GCC accepts this as an extension, with one warning per template. The module silences that warning (src/module_prologue.hpp), and importers instantiate the templates normally. +- GCC binds a call that does not depend on the queue's `T` when it reads the template, and with it the specialization of a `static` function template (`details::swap_relaxed` on the queue's counters). It emits that body only where a member that calls it is instantiated. The module instantiates the queues' members once, so the bodies are in its interface and every importer emits its own. Without this, `swap` failed to link. +- A translation unit that includes `` (gtest does) and instantiates the queue's thread-exit path fails with GCC: "conflicting 'noexcept' specifier for imported declaration `__cxa_thread_atexit`". This is GCC's handling of a function-local `thread_local` from an imported global module fragment. Clang has neither thread_local recycling nor the error. Keep such code in a unit without ``, or use the `no-thread-local` feature, as [tests/c_api](../tests/c_api) does with gtest. + +## Checks + +- `mcpp test`: the queues through the module (the module is the only source of the queue's declarations): + - multi-producer multi-consumer traffic on threads, implicit and with tokens and bulk operations, checking that every item arrives once and in per-producer order; + - bounded `try_enqueue`; + - move-only, throwing and over-aligned element types; + - a counting `Traits`; + - `swap` and moves; + - `try_dequeue_from_producer`; + - the blocking queue's waits with and without timeouts; + - `LightweightSemaphore`; + - the adapted helpers; + - the constants; + - the side header mixed with the import. +- `tests/c_api` (its own project, with `c-api` and `no-thread-local`): the C API from threads, and gtest beside the queue. +- `python3 tools/check_exports.py`: every declaration of `moodycamel`, every name of `moodycamel::details`, the C API and every macro, against the pinned headers. +- `python3 tools/run_conformance.py`: upstream's `tests/unittests` (all 45 registered tests, including the C API and `corealgos.h` tests), built against the module instead of the headers. The tool documents the mechanical edits and applies them to the pinned tarball's files. The run must end with upstream's "All tests passed.". diff --git a/include/mwrap/concurrentqueue/config.hpp b/include/mwrap/concurrentqueue/config.hpp new file mode 100644 index 0000000..187f21f --- /dev/null +++ b/include/mwrap/concurrentqueue/config.hpp @@ -0,0 +1,27 @@ +// — the configuration mcpplibs.mwrap.concurrentqueue +// is built with, applied before concurrentqueue.h. Every other configuration +// macro keeps upstream's default. +// +// MOODYCAMEL_NOEXCEPT_CTOR / MOODYCAMEL_NOEXCEPT_ASSIGN: upstream's +// configuration point (both are only defined by concurrentqueue.h when +// MOODYCAMEL_NOEXCEPT is not), here spelled with the type traits that give the +// same answer as upstream's `noexcept(expr)`. Upstream's expression names +// details::deref_noexcept, a `static` (TU-local) function; inside a template +// argument (details::nomove_if<…> in enqueue_bulk) that makes the class +// template's declaration expose a TU-local entity, which GCC rejects in a +// module interface. The traits name nothing TU-local. +// +// noexcept(new (p) T(details::deref_noexcept(it))) == std::is_nothrow_constructible +// noexcept(element = std::move(el)) == std::is_nothrow_assignable +// +// Without exceptions (upstream's own test) upstream's definitions stand. +#pragma once +#include + +#if !defined(MOODYCAMEL_NOEXCEPT) && !defined(MOODYCAMEL_EXCEPTIONS_ENABLED) && \ + ((defined(_MSC_VER) && defined(_CPPUNWIND)) || (defined(__GNUC__) && defined(__EXCEPTIONS)) || \ + (!defined(_MSC_VER) && !defined(__GNUC__))) +#define MOODYCAMEL_NOEXCEPT noexcept +#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) (::std::is_nothrow_constructible::value) +#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) (::std::is_nothrow_assignable::value) +#endif diff --git a/include/mwrap/concurrentqueue/macros.hpp b/include/mwrap/concurrentqueue/macros.hpp new file mode 100644 index 0000000..916d282 --- /dev/null +++ b/include/mwrap/concurrentqueue/macros.hpp @@ -0,0 +1,14 @@ +// — moodycamel's headers with their macros, +// configured as mcpplibs.mwrap.concurrentqueue is (). +// Include it, before `import mcpplibs.mwrap.concurrentqueue;`, for what a +// mapping cannot be: the syntax macros (MOODYCAMEL_NOEXCEPT, +// MOODYCAMEL_DELETE_FUNCTION, MOODYCAMEL_ALIGNAS, MOODYCAMEL_TRY / CATCH / +// THROW …) and `#if` tests of the configuration (MOODYCAMEL_EXCEPTIONS_ENABLED, +// MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED). The declarations it brings are the +// very entities the module exports. While a macro is defined it expands even +// inside `mcpplibs::mwrap::concurrentqueue::macros::NAME`. +#pragma once +#include +#include +#include +#include diff --git a/mcpp.toml b/mcpp.toml new file mode 100644 index 0000000..f6f102c --- /dev/null +++ b/mcpp.toml @@ -0,0 +1,56 @@ +# Module wrapper; upstream API preserved. Policy: mcpp-index/docs/mwrap.md. +[package] +namespace = "mcpplibs.mwrap" +name = "concurrentqueue" +version = "1.0.5" +description = "moodycamel::ConcurrentQueue 1.0.5 C++ modules for mcpp (import mcpplibs.mwrap.concurrentqueue;): the lock-free MPMC queue, its blocking variant and semaphore over compat.concurrentqueue, the C API behind the c-api feature" +license = "Apache-2.0" +authors = ["mcpp-community"] +repo = "https://github.com/mcpplibs/concurrentqueue-m" +platforms = ["linux", "macos", "windows"] + +[targets.concurrentqueue] +kind = "lib" + +[build] +sources = ["src/*.cppm", "src/*.cpp"] +# (the configuration the module is built +# with) and (the headers with their macros). +include_dirs = ["include"] + +[features] +default = [] +# Upstream's C API (moodycamel_cq_*, moodycamel_bcq_*): its two sources in +# compat.concurrentqueue, and the module that exports it. +c-api = { sources = ["src/c_api/*.cppm"], forward = ["compat.concurrentqueue/c-api"] } +# Upstream's MOODYCAMEL_NO_THREAD_LOCAL: thread ids from std::this_thread and +# no thread_local, so implicit producers are not recycled at thread exit (as +# upstream always builds on ARM and with Clang in GNU mode). With GCC it lets a +# translation unit that includes (gtest does) use the queue. +no-thread-local = { defines = ["MOODYCAMEL_NO_THREAD_LOCAL"] } + +[dependencies] +compat.concurrentqueue = "1.0.5" + +[test] +# tests/c_api is its own project (it needs the c-api feature). +discover = ["tests/*.cpp"] + +[package.metadata.mwrap] +upstream-version = "1.0.5" + +[[runtime.deploy]] +from = "LICENSE" +to = "licenses/concurrentqueue" + +[[runtime.deploy]] +from = "NOTICE" +to = "licenses/concurrentqueue" + +[[runtime.deploy]] +from = "upstream/concurrentqueue/README.md" +to = "licenses/concurrentqueue/upstream/concurrentqueue" + +[[runtime.deploy]] +from = "upstream/concurrentqueue/LICENSE.md" +to = "licenses/concurrentqueue/upstream/concurrentqueue" diff --git a/src/c_api/c_api.cppm b/src/c_api/c_api.cppm new file mode 100644 index 0000000..1d52602 --- /dev/null +++ b/src/c_api/c_api.cppm @@ -0,0 +1,30 @@ +// mcpplibs.mwrap.concurrentqueue.c_api — upstream's flat C API over both +// queues (c_api/concurrentqueue.h: moodycamel_cq_*, moodycamel_bcq_*), with +// the `c-api` feature, which builds compat.concurrentqueue's two C API +// sources. Import it by name: the root does not re-export it. +module; +// A static library on Windows: no dllimport (compat.concurrentqueue defines +// it for its own sources and consumers alike). +#ifndef MOODYCAMEL_STATIC +#define MOODYCAMEL_STATIC +#endif +#include + +export module mcpplibs.mwrap.concurrentqueue.c_api; + +// types (3) +export using ::MoodycamelCQHandle; +export using ::MoodycamelBCQHandle; +export using ::MoodycamelValue; + +// functions (10) +export using ::moodycamel_cq_create; +export using ::moodycamel_cq_destroy; +export using ::moodycamel_cq_enqueue; +export using ::moodycamel_cq_try_dequeue; +export using ::moodycamel_cq_size_approx; +export using ::moodycamel_bcq_create; +export using ::moodycamel_bcq_destroy; +export using ::moodycamel_bcq_enqueue; +export using ::moodycamel_bcq_wait_dequeue; +export using ::moodycamel_bcq_try_dequeue; diff --git a/src/concurrentqueue.cppm b/src/concurrentqueue.cppm new file mode 100644 index 0000000..9688792 --- /dev/null +++ b/src/concurrentqueue.cppm @@ -0,0 +1,123 @@ +// mcpplibs.mwrap.concurrentqueue — moodycamel::ConcurrentQueue, the lock-free +// multi-producer multi-consumer queue, with its blocking variant and +// lightweight semaphore, as a C++ module. Every exported name is the upstream +// entity of , and +// , unchanged; the root also re-exports the macro +// mappings. The headers are compat.concurrentqueue's, configured by +// . The C API is its own module, +// mcpplibs.mwrap.concurrentqueue.c_api, with the `c-api` feature (mcpp's +// scanner admits no import under a feature test, so the root cannot +// re-export it conditionally). +// +// moodycamel::details is the queue's implementation and is not exported. Its +// `static` (TU-local, not exportable) helpers that are not templates over +// arbitrary types are adapted in mcpplibs::mwrap::concurrentqueue::details +// under the same names, each calling upstream's body (src/details.cpp). +// tools/check_exports.py audits both lists against the headers. +module; +#include "module_prologue.hpp" // GCC: the exposures of upstream's static helpers +#include +#include +#include +#include + +export module mcpplibs.mwrap.concurrentqueue; + +export import mcpplibs.mwrap.concurrentqueue.macros; + +export namespace moodycamel { +// +using moodycamel::ConcurrentQueue; +using moodycamel::ConcurrentQueueDefaultTraits; +using moodycamel::ProducerToken; +using moodycamel::ConsumerToken; +using moodycamel::ConcurrentQueueTests; // forward-declared, befriended by the queue: upstream's test class +// +using moodycamel::BlockingConcurrentQueue; +// +using moodycamel::LightweightSemaphore; +// every overload: ConcurrentQueue, BlockingConcurrentQueue, ProducerToken, +// ConsumerToken, ConcurrentQueue::ImplicitProducerKVP +using moodycamel::swap; +} // namespace moodycamel + +// The TU-local helpers of moodycamel::details, adapted (implementation +// details; upstream's own tests call circular_less_than). +export namespace mcpplibs::mwrap::concurrentqueue::details { +using thread_id_t = ::moodycamel::details::thread_id_t; +thread_id_t thread_id(); +bool likely(bool x); +bool unlikely(bool x); +std::size_t hash_thread_id(thread_id_t id); +// Templates over the unsigned integer types: one overload per type. +bool circular_less_than(unsigned char a, unsigned char b); +bool circular_less_than(unsigned short a, unsigned short b); +bool circular_less_than(unsigned int a, unsigned int b); +bool circular_less_than(unsigned long a, unsigned long b); +bool circular_less_than(unsigned long long a, unsigned long long b); +unsigned char ceil_to_pow_2(unsigned char x); +unsigned short ceil_to_pow_2(unsigned short x); +unsigned int ceil_to_pow_2(unsigned int x); +unsigned long ceil_to_pow_2(unsigned long x); +unsigned long long ceil_to_pow_2(unsigned long long x); +} // namespace mcpplibs::mwrap::concurrentqueue::details + +// GCC binds a call that does not depend on the queue's T when it reads the +// template, and with it the specialization of upstream's `static` function +// template the call names (details::swap_relaxed on the std::uint32_t, +// thread-id and std::size_t members swap() exchanges), but emits that +// specialization's body only where a member that calls it is instantiated. +// An importer instantiating swap() would then reference a TU-local +// specialization that no object defines. Instantiating the queues' members +// here, once, puts those bodies in the module interface, and every importer +// emits its own copy (tests/api_test.cpp swaps the queues and tokens). +namespace mcpplibs::mwrap::concurrentqueue::instantiation { +[[maybe_unused]] void bind_non_dependent_helpers() { + moodycamel::ConcurrentQueue a, b; + moodycamel::ProducerToken producer(a); + moodycamel::ConsumerToken consumer(a); + int items[2] = {1, 2}, item = 0; + a.enqueue(item); + a.enqueue(producer, item); + a.try_enqueue(item); + a.try_enqueue(producer, item); + a.enqueue_bulk(items, 2); + a.enqueue_bulk(producer, items, 2); + a.try_enqueue_bulk(items, 2); + a.try_enqueue_bulk(producer, items, 2); + a.try_dequeue(item); + a.try_dequeue(consumer, item); + a.try_dequeue_non_interleaved(item); + a.try_dequeue_from_producer(producer, item); + a.try_dequeue_bulk(items, 2); + a.try_dequeue_bulk(consumer, items, 2); + a.try_dequeue_bulk_from_producer(producer, items, 2); + (void)a.size_approx(); + a.swap(b); + swap(a, b); + moodycamel::ConcurrentQueue moved(std::move(a)); + b = std::move(moved); + moodycamel::BlockingConcurrentQueue c, d; + moodycamel::ProducerToken blocking_producer(c); + moodycamel::ConsumerToken blocking_consumer(c); + c.enqueue(item); + c.enqueue(blocking_producer, item); + c.try_enqueue(item); + c.enqueue_bulk(items, 2); + c.try_enqueue_bulk(blocking_producer, items, 2); + c.try_dequeue(item); + c.try_dequeue(blocking_consumer, item); + c.wait_dequeue(item); + c.wait_dequeue(blocking_consumer, item); + c.wait_dequeue_timed(item, 0); + c.wait_dequeue_timed(blocking_consumer, item, 0); + c.wait_dequeue_bulk(items, 2); + c.wait_dequeue_bulk_timed(blocking_consumer, items, 2, 0); + c.try_dequeue_bulk(items, 2); + (void)c.size_approx(); + c.swap(d); + swap(c, d); + moodycamel::swap(producer, blocking_producer); + moodycamel::swap(consumer, blocking_consumer); +} +} // namespace mcpplibs::mwrap::concurrentqueue::instantiation diff --git a/src/details.cpp b/src/details.cpp new file mode 100644 index 0000000..198d366 --- /dev/null +++ b/src/details.cpp @@ -0,0 +1,26 @@ +// The adaptations of moodycamel::details' TU-local helpers: each calls +// upstream's own `static` function (or its instantiation for the type). +module; +#include +#include + +module mcpplibs.mwrap.concurrentqueue; + +namespace mcpplibs::mwrap::concurrentqueue::details { + +thread_id_t thread_id() { return ::moodycamel::details::thread_id(); } +bool likely(bool x) { return (::moodycamel::details::likely)(x); } +bool unlikely(bool x) { return (::moodycamel::details::unlikely)(x); } +std::size_t hash_thread_id(thread_id_t id) { return ::moodycamel::details::hash_thread_id(id); } +bool circular_less_than(unsigned char a, unsigned char b) { return ::moodycamel::details::circular_less_than(a, b); } +bool circular_less_than(unsigned short a, unsigned short b) { return ::moodycamel::details::circular_less_than(a, b); } +bool circular_less_than(unsigned int a, unsigned int b) { return ::moodycamel::details::circular_less_than(a, b); } +bool circular_less_than(unsigned long a, unsigned long b) { return ::moodycamel::details::circular_less_than(a, b); } +bool circular_less_than(unsigned long long a, unsigned long long b) { return ::moodycamel::details::circular_less_than(a, b); } +unsigned char ceil_to_pow_2(unsigned char x) { return ::moodycamel::details::ceil_to_pow_2(x); } +unsigned short ceil_to_pow_2(unsigned short x) { return ::moodycamel::details::ceil_to_pow_2(x); } +unsigned int ceil_to_pow_2(unsigned int x) { return ::moodycamel::details::ceil_to_pow_2(x); } +unsigned long ceil_to_pow_2(unsigned long x) { return ::moodycamel::details::ceil_to_pow_2(x); } +unsigned long long ceil_to_pow_2(unsigned long long x) { return ::moodycamel::details::ceil_to_pow_2(x); } + +} // namespace mcpplibs::mwrap::concurrentqueue::details diff --git a/src/macros.cppm b/src/macros.cppm new file mode 100644 index 0000000..53b60f2 --- /dev/null +++ b/src/macros.cppm @@ -0,0 +1,118 @@ +// mcpplibs.mwrap.concurrentqueue.macros — the macros of moodycamel's headers +// that are values (a named module cannot export a macro), under their +// original names in mcpplibs::mwrap::concurrentqueue::macros, each an +// `inline constexpr bool` that is true exactly when the macro is defined +// where the module is built: +// +// - what the headers derive: MOODYCAMEL_EXCEPTIONS_ENABLED, +// MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED (implicit producers are +// recycled at thread exit), MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC +// (timed waits on CLOCK_MONOTONIC, glibc 2.30 and later); +// - the configuration inputs they read, as the module was built (upstream's +// defaults: all off): MOODYCAMEL_NO_THREAD_LOCAL, +// MOODYCAMEL_QUEUE_INTERNAL_DEBUG and the MCDBGQ_* debugging switches. +// +// The syntax macros (MOODYCAMEL_NOEXCEPT, MOODYCAMEL_DELETE_FUNCTION, +// MOODYCAMEL_ALIGNAS, MOODYCAMEL_TRY / CATCH / RETHROW / THROW, +// MOODYCAMEL_THREADLOCAL …) cannot be values: they stay macros, in +// . Per-queue configuration is the Traits +// template argument, which a module passes through as it is. +// tools/check_exports.py accounts for every macro. +module; +#include +#include +#include +#include + +namespace mwrap_concurrentqueue_capture { +#ifdef MOODYCAMEL_EXCEPTIONS_ENABLED +inline constexpr bool value_MOODYCAMEL_EXCEPTIONS_ENABLED = true; +#else +inline constexpr bool value_MOODYCAMEL_EXCEPTIONS_ENABLED = false; +#endif +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED +inline constexpr bool value_MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED = true; +#else +inline constexpr bool value_MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED = false; +#endif +#ifdef MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC +inline constexpr bool value_MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC = true; +#else +inline constexpr bool value_MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC = false; +#endif +#ifdef MOODYCAMEL_NO_THREAD_LOCAL +inline constexpr bool value_MOODYCAMEL_NO_THREAD_LOCAL = true; +#else +inline constexpr bool value_MOODYCAMEL_NO_THREAD_LOCAL = false; +#endif +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG +inline constexpr bool value_MOODYCAMEL_QUEUE_INTERNAL_DEBUG = true; +#else +inline constexpr bool value_MOODYCAMEL_QUEUE_INTERNAL_DEBUG = false; +#endif +#ifdef MCDBGQ_USE_RELACY +inline constexpr bool value_MCDBGQ_USE_RELACY = true; +#else +inline constexpr bool value_MCDBGQ_USE_RELACY = false; +#endif +#ifdef MCDBGQ_TRACKMEM +inline constexpr bool value_MCDBGQ_TRACKMEM = true; +#else +inline constexpr bool value_MCDBGQ_TRACKMEM = false; +#endif +#ifdef MCDBGQ_NOLOCKFREE_FREELIST +inline constexpr bool value_MCDBGQ_NOLOCKFREE_FREELIST = true; +#else +inline constexpr bool value_MCDBGQ_NOLOCKFREE_FREELIST = false; +#endif +#ifdef MCDBGQ_USEDEBUGFREELIST +inline constexpr bool value_MCDBGQ_USEDEBUGFREELIST = true; +#else +inline constexpr bool value_MCDBGQ_USEDEBUGFREELIST = false; +#endif +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX +inline constexpr bool value_MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX = true; +#else +inline constexpr bool value_MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX = false; +#endif +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH +inline constexpr bool value_MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH = true; +#else +inline constexpr bool value_MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH = false; +#endif +} // namespace mwrap_concurrentqueue_capture + +// Every mapped name, so the definitions below are not rewritten. +#undef MOODYCAMEL_EXCEPTIONS_ENABLED +#undef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED +#undef MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC +#undef MOODYCAMEL_NO_THREAD_LOCAL +#undef MOODYCAMEL_QUEUE_INTERNAL_DEBUG +#undef MCDBGQ_USE_RELACY +#undef MCDBGQ_TRACKMEM +#undef MCDBGQ_NOLOCKFREE_FREELIST +#undef MCDBGQ_USEDEBUGFREELIST +#undef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX +#undef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH + +export module mcpplibs.mwrap.concurrentqueue.macros; + +namespace c = mwrap_concurrentqueue_capture; + +export namespace mcpplibs::mwrap::concurrentqueue::macros { + +// derived by the headers +inline constexpr bool MOODYCAMEL_EXCEPTIONS_ENABLED = c::value_MOODYCAMEL_EXCEPTIONS_ENABLED; +inline constexpr bool MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED = c::value_MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED; +inline constexpr bool MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC = c::value_MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC; +// configuration inputs, as the module was built +inline constexpr bool MOODYCAMEL_NO_THREAD_LOCAL = c::value_MOODYCAMEL_NO_THREAD_LOCAL; +inline constexpr bool MOODYCAMEL_QUEUE_INTERNAL_DEBUG = c::value_MOODYCAMEL_QUEUE_INTERNAL_DEBUG; +inline constexpr bool MCDBGQ_USE_RELACY = c::value_MCDBGQ_USE_RELACY; +inline constexpr bool MCDBGQ_TRACKMEM = c::value_MCDBGQ_TRACKMEM; +inline constexpr bool MCDBGQ_NOLOCKFREE_FREELIST = c::value_MCDBGQ_NOLOCKFREE_FREELIST; +inline constexpr bool MCDBGQ_USEDEBUGFREELIST = c::value_MCDBGQ_USEDEBUGFREELIST; +inline constexpr bool MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX = c::value_MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX; +inline constexpr bool MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH = c::value_MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH; + +} // namespace mcpplibs::mwrap::concurrentqueue::macros diff --git a/src/module_prologue.hpp b/src/module_prologue.hpp new file mode 100644 index 0000000..c50e6ec --- /dev/null +++ b/src/module_prologue.hpp @@ -0,0 +1,11 @@ +// Included first in the global module fragment of src/concurrentqueue.cppm +// (a fragment holds only inclusions). +// +// The queue's member templates call upstream's `static` helpers +// (details::likely, details::circular_less_than …), so exporting them exposes +// TU-local entities of the global module fragment. GCC accepts this as an +// extension, with a warning per template, and importers instantiate the +// templates (every test here does); Clang accepts it as is. +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic ignored "-Wexpose-global-module-tu-local" +#endif diff --git a/templates/pipeline/mcpp.toml.in b/templates/pipeline/mcpp.toml.in new file mode 100644 index 0000000..a637560 --- /dev/null +++ b/templates/pipeline/mcpp.toml.in @@ -0,0 +1,10 @@ +[package] +name = "{{project.name}}" +version = "0.1.0" + +[targets.{{project.name}}] +kind = "bin" +main = "src/main.cpp" + +[dependencies] +"{{self.name}}" = "{{self.version}}" diff --git a/templates/pipeline/src/main.cpp.in b/templates/pipeline/src/main.cpp.in new file mode 100644 index 0000000..5edfd5c --- /dev/null +++ b/templates/pipeline/src/main.cpp.in @@ -0,0 +1,62 @@ +// {{project.name}} — a producer-consumer pipeline over +// import mcpplibs.mwrap.concurrentqueue: producer threads enqueue numbers in +// bulk through their tokens, consumer threads wait on a blocking queue (with a +// timeout, so they notice the end) and count the primes they are handed. +// Arguments: producers, consumers, numbers (4, 4, 200000). +import std; +import mcpplibs.mwrap.concurrentqueue; + +namespace { + +bool is_prime(std::uint64_t n) { + if (n < 2) return false; + for (std::uint64_t d = 2; d * d <= n; ++d) + if (n % d == 0) return false; + return true; +} + +unsigned arg(int argc, char** argv, int i, unsigned fallback) { + unsigned value = fallback; + if (argc > i) std::from_chars(argv[i], argv[i] + std::strlen(argv[i]), value); + return value == 0 ? fallback : value; +} + +} // namespace + +int main(int argc, char** argv) { + const unsigned producers = arg(argc, argv, 1, 4), consumers = arg(argc, argv, 2, 4), items = arg(argc, argv, 3, 200000); + moodycamel::BlockingConcurrentQueue queue; + std::atomic consumed{0}, primes{0}; + const auto start = std::chrono::steady_clock::now(); + { + std::vector threads; + for (unsigned p = 0; p < producers; ++p) + threads.emplace_back([&, p] { + moodycamel::ProducerToken token(queue); + std::array batch; + for (unsigned i = p; i < items;) { + std::size_t n = 0; + for (; n < batch.size() && i < items; ++n, i += producers) batch[n] = i; + queue.enqueue_bulk(token, batch.begin(), n); + } + }); + for (unsigned c = 0; c < consumers; ++c) + threads.emplace_back([&] { + moodycamel::ConsumerToken token(queue); + std::array batch; + // Until every number is counted: the timeout lets a consumer + // with nothing left to take see that the others finished. + while (consumed.load() < items) { + const std::size_t n = queue.wait_dequeue_bulk_timed(token, batch.begin(), batch.size(), std::chrono::milliseconds(5)); + std::uint64_t found = 0; + for (std::size_t k = 0; k < n; ++k) found += is_prime(batch[k]); + primes.fetch_add(found); + consumed.fetch_add(n); + } + }); + } + const std::chrono::duration took = std::chrono::steady_clock::now() - start; + std::println("{} producers -> {} consumers: {} of {} numbers, {} primes, {:.0f} items/s", producers, consumers, + consumed.load(), items, primes.load(), consumed.load() / took.count()); + return consumed.load() == items ? 0 : 1; +} diff --git a/templates/pipeline/template.toml b/templates/pipeline/template.toml new file mode 100644 index 0000000..41534e0 --- /dev/null +++ b/templates/pipeline/template.toml @@ -0,0 +1,4 @@ +[template] +description = "A producer-consumer pipeline with import mcpplibs.mwrap.concurrentqueue: producer threads, a blocking queue, consumers that wait with a timeout" +default = true +post_message = "Run `mcpp run`, or `mcpp run -- `." diff --git a/tests/api_test.cpp b/tests/api_test.cpp new file mode 100644 index 0000000..321415e --- /dev/null +++ b/tests/api_test.cpp @@ -0,0 +1,366 @@ +// moodycamel's queues through the module: multi-producer multi-consumer +// traffic on threads (every item exactly once, per-producer order kept), +// tokens, bulk operations, bounded try_enqueue, user types (move-only, +// throwing, over-aligned) and Traits, swap and move, the blocking queue's +// waits with and without timeouts, and the lightweight semaphore — spelled as +// upstream's README documents them. +#include "support/check.hpp" + +import std; +import mcpplibs.mwrap.concurrentqueue; + +using moodycamel::BlockingConcurrentQueue; +using moodycamel::ConcurrentQueue; +using moodycamel::ConsumerToken; +using moodycamel::LightweightSemaphore; +using moodycamel::ProducerToken; + +namespace { + +constexpr int producers = 4, consumers = 4, per_producer = 20000; + +struct Item { + int producer = -1, seq = -1; +}; + +// Every item dequeued once; each consumer sees each producer's items in order. +void check_traffic(const std::vector>& seen) { + std::vector> got(producers, std::vector(per_producer, 0)); + for (const auto& one : seen) { + std::vector last(producers, -1); + for (const Item& item : one) { + if (item.producer < 0 || item.producer >= producers || item.seq < 0 || item.seq >= per_producer) { + CHECK(!"an item that was never enqueued"); + return; + } + CHECK(item.seq > last[item.producer]); // per-producer FIFO + last[item.producer] = item.seq; + CHECK(got[item.producer][item.seq]++ == 0); // not dequeued twice + } + } + std::size_t lost = 0; + for (const auto& p : got) lost += static_cast(std::ranges::count(p, 0)); + CHECK(lost == 0); +} + +void ConcurrentQueue_MpmcImplicitProducers() { + ConcurrentQueue q; + std::atomic done{0}, taken{0}; + std::vector> seen(consumers); + { + std::vector threads; + for (int p = 0; p < producers; ++p) + threads.emplace_back([&, p] { + for (int i = 0; i < per_producer; ++i) CHECK(q.enqueue(Item{p, i})); + done.fetch_add(1); + }); + for (int c = 0; c < consumers; ++c) + threads.emplace_back([&, c] { + Item item; + while (taken.load() < producers * per_producer) + if (q.try_dequeue(item)) { + seen[c].push_back(item); + taken.fetch_add(1); + } + }); + } + Item rest; + CHECK(!(q.try_dequeue(rest))); + check_traffic(seen); +} + +void ConcurrentQueue_MpmcTokensAndBulk() { + ConcurrentQueue q(64); + std::atomic done{0}, taken{0}; + std::vector> seen(consumers); + { + std::vector threads; + for (int p = 0; p < producers; ++p) + threads.emplace_back([&, p] { + ProducerToken token(q); + std::array batch; + for (int i = 0; i < per_producer; i += static_cast(batch.size())) { + for (std::size_t k = 0; k < batch.size(); ++k) batch[k] = Item{p, i + static_cast(k)}; + CHECK(q.enqueue_bulk(token, batch.begin(), batch.size())); + } + done.fetch_add(1); + }); + for (int c = 0; c < consumers; ++c) + threads.emplace_back([&, c] { + ConsumerToken token(q); + std::array out; + while (taken.load() < producers * per_producer) { + const std::size_t n = q.try_dequeue_bulk(token, out.begin(), out.size()); + seen[c].insert(seen[c].end(), out.begin(), out.begin() + static_cast(n)); + taken.fetch_add(static_cast(n)); + } + }); + } + check_traffic(seen); +} + +void ConcurrentQueue_TryEnqueueIsBounded() { + // try_enqueue never allocates: with no pre-allocated room it fails. + ConcurrentQueue empty(0); + CHECK(!(empty.try_enqueue(1))); + ConcurrentQueue q(ConcurrentQueue::BLOCK_SIZE); + std::size_t accepted = 0; + while (q.try_enqueue(static_cast(accepted))) ++accepted; + CHECK((accepted) >= (ConcurrentQueue::BLOCK_SIZE)); + CHECK(q.enqueue(-1)); // enqueue allocates + CHECK((q.size_approx()) == (accepted + 1)); + int v = 0; + CHECK(q.try_dequeue(v)); + CHECK((v) == (0)); +} + +void ConcurrentQueue_UserTypes() { + // Move-only. + ConcurrentQueue> owned; + owned.enqueue(std::make_unique("moved")); + std::unique_ptr out; + CHECK(owned.try_dequeue(out)); + CHECK((*out) == ("moved")); + // A braced element. + ConcurrentQueue> pairs; + pairs.enqueue({7, "seven"}); + std::pair p; + CHECK(pairs.try_dequeue(p)); + CHECK((p.second) == ("seven")); + // Over-aligned. + struct alignas(64) Wide { + std::array lanes{}; + }; + ConcurrentQueue wide; + for (std::uint64_t i = 0; i < 1000; ++i) { + Wide w; + w.lanes.fill(i); + wide.enqueue(w); + } + Wide w; + for (std::uint64_t i = 0; i < 1000; ++i) { + CHECK(wide.try_dequeue(w)); + CHECK((reinterpret_cast(&w) % 64) == (0u)); + CHECK((w.lanes[7]) == (i)); + } +} + +struct Throwing { + static inline int copies_left = 0; + int value = 0; + Throwing() = default; + explicit Throwing(int v) : value(v) {} + Throwing(const Throwing& o) : value(o.value) { + if (--copies_left < 0) throw std::runtime_error("copy"); + } + Throwing& operator=(const Throwing& o) { + if (--copies_left < 0) throw std::runtime_error("assign"); + value = o.value; + return *this; + } +}; + +void ConcurrentQueue_ExceptionsLeaveTheQueueConsistent() { + static_assert(mcpplibs::mwrap::concurrentqueue::macros::MOODYCAMEL_EXCEPTIONS_ENABLED); + ConcurrentQueue q; + Throwing one(1), two(2); + Throwing::copies_left = 1; + CHECK(q.enqueue(one)); + { bool thrown = false; try { q.enqueue(two); } catch (const std::runtime_error&) { thrown = true; } CHECK(thrown); } // the element's copy throws: nothing is enqueued + CHECK((q.size_approx()) == (1u)); + Throwing::copies_left = 100; + Throwing out; + CHECK(q.try_dequeue(out)); + CHECK((out.value) == (1)); + CHECK(!(q.try_dequeue(out))); +} + +// Traits: a small block size and allocation through the traits' malloc/free. +struct CountingTraits : moodycamel::ConcurrentQueueDefaultTraits { + static constexpr std::size_t BLOCK_SIZE = 4; + static constexpr std::size_t EXPLICIT_INITIAL_INDEX_SIZE = 4; + static constexpr std::size_t IMPLICIT_INITIAL_INDEX_SIZE = 4; + static inline std::atomic live{0}; + static void* malloc(std::size_t size) { + ++live; + return std::malloc(size); + } + static void free(void* p) { + if (p) --live; + std::free(p); + } +}; + +void ConcurrentQueue_TraitsSwapAndMove() { + { + ConcurrentQueue a, b; + CHECK((CountingTraits::live.load()) > (0)); + for (int i = 0; i < 100; ++i) a.enqueue(i); // many 4-element blocks + b.enqueue(-1); + swap(a, b); // moodycamel::swap, by argument-dependent lookup + int v = 0; + CHECK(a.try_dequeue(v)); + CHECK((v) == (-1)); + CHECK(b.try_dequeue(v)); + CHECK((v) == (0)); + ConcurrentQueue c(std::move(b)); + CHECK((c.size_approx()) == (99u)); + ProducerToken t1(c), t2(c); + moodycamel::swap(t1, t2); + CHECK(t1.valid() && t2.valid()); + ConsumerToken k1(c), k2(c); + moodycamel::swap(k1, k2); + CHECK(c.try_dequeue(k1, v)); + CHECK((v) == (1)); + } + CHECK((CountingTraits::live.load()) == (0)); // every block freed through the traits + static_assert(ConcurrentQueue::BLOCK_SIZE == 4); + static_assert(ConcurrentQueue::BLOCK_SIZE == moodycamel::ConcurrentQueueDefaultTraits::BLOCK_SIZE); + CHECK(ConcurrentQueue::is_lock_free() || !ConcurrentQueue::is_lock_free()); +} + +void ConcurrentQueue_ProducerThatExitsAndTheDequeueFromProducer() { + ConcurrentQueue q; + ProducerToken token(q); + for (int i = 0; i < 10; ++i) q.enqueue(token, i); + // A thread's implicit producer outlives the thread. + std::jthread([&] { q.enqueue(100); }).join(); + int v = 0; + CHECK(q.try_dequeue_from_producer(token, v)); + CHECK((v) == (0)); + std::vector rest; + while (q.try_dequeue(v)) rest.push_back(v); + CHECK((rest.size()) == (10u)); + CHECK(std::ranges::find(rest, 100) != rest.end()); +} + +// ── BlockingConcurrentQueue ────────────────────────────────────────────── + +void BlockingConcurrentQueue_WaitDequeueAcrossThreads() { + // Consumers block in wait_dequeue_timed until everything is taken. The + // queue is not globally ordered, so a sentinel item could overtake real + // ones: the consumers count instead. + BlockingConcurrentQueue q; + std::vector> seen(consumers); + std::atomic taken{0}; + { + std::vector threads; + for (int c = 0; c < consumers; ++c) + threads.emplace_back([&, c] { + ConsumerToken token(q); + Item item; + while (taken.load() < producers * per_producer) + if (q.wait_dequeue_timed(token, item, std::chrono::milliseconds(1))) { + seen[c].push_back(item); + taken.fetch_add(1); + } + }); + for (int p = 0; p < producers; ++p) + threads.emplace_back([&, p] { + ProducerToken token(q); + for (int i = 0; i < per_producer; ++i) q.enqueue(token, Item{p, i}); + }); + } + check_traffic(seen); + // wait_dequeue blocks until an item arrives. + std::jthread later([&] { q.enqueue(Item{0, 0}); }); + Item item; + q.wait_dequeue(item); + CHECK(item.producer == 0); +} + +void BlockingConcurrentQueue_TimedWaits() { + BlockingConcurrentQueue q; + int v = 0; + auto start = std::chrono::steady_clock::now(); + CHECK(!(q.wait_dequeue_timed(v, std::chrono::milliseconds(30)))); + CHECK((std::chrono::steady_clock::now() - start) >= (std::chrono::milliseconds(25))); + CHECK(!(q.wait_dequeue_timed(v, 1000))); // microseconds + std::jthread later([&] { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + q.enqueue(42); + }); + CHECK(q.wait_dequeue_timed(v, std::chrono::seconds(10))); + CHECK((v) == (42)); + // Bulk waits. + int many[] = {1, 2, 3, 4, 5}; + q.enqueue_bulk(many, 5); + int out[8]; + CHECK(q.wait_dequeue_bulk(out, 8) == 5u); + CHECK(q.wait_dequeue_bulk_timed(out, 8, std::chrono::milliseconds(5)) == 0u); + ConsumerToken token(q); + q.enqueue(9); + CHECK(q.wait_dequeue_bulk_timed(token, out, 8, 1000000) == 1u); + CHECK((out[0]) == (9)); + CHECK(!(q.try_dequeue(v))); + CHECK((q.size_approx()) == (0u)); + BlockingConcurrentQueue other; + other.enqueue(5); + swap(q, other); + CHECK(q.try_dequeue(v)); + CHECK((v) == (5)); +} + +// ── LightweightSemaphore ───────────────────────────────────────────────── + +void LightweightSemaphore_SignalWaitAndTimeouts() { + LightweightSemaphore sema(2); + CHECK((sema.availableApprox()) == (2u)); + CHECK(sema.tryWait()); + CHECK(sema.wait()); + CHECK(!(sema.tryWait())); + CHECK(!(sema.wait(2000))); // microseconds + sema.signal(5); + CHECK((sema.tryWaitMany(3)) == (3)); + CHECK((sema.waitMany(10)) == (2)); + CHECK(sema.waitMany(10, 1000) == 0); + std::atomic woken{0}; + { + std::vector waiters; + for (int i = 0; i < 4; ++i) + waiters.emplace_back([&] { + sema.wait(); + ++woken; + }); + sema.signal(4); + } + CHECK((woken.load()) == (4)); + static_assert(std::is_same_v>); +} + +// ── the adapted helpers of moodycamel::details ─────────────────────────── + +void Details_AdaptedHelpers() { + namespace d = mcpplibs::mwrap::concurrentqueue::details; + CHECK(d::circular_less_than(std::uint32_t{0}, std::uint32_t{100})); + CHECK(d::circular_less_than(std::uint32_t{0xFFFFFFF0u}, std::uint32_t{5})); // wraps around + CHECK(!(d::circular_less_than(std::uint16_t{5}, std::uint16_t{5}))); + CHECK(d::circular_less_than(std::uint64_t{1}, std::uint64_t{2})); + CHECK(d::circular_less_than(std::uint8_t{250}, std::uint8_t{3})); + CHECK((d::ceil_to_pow_2(std::size_t{33})) == (64u)); + CHECK((d::ceil_to_pow_2(std::uint8_t{3})) == (4u)); + CHECK((d::ceil_to_pow_2(std::uint32_t{1024})) == (1024u)); + CHECK(d::likely(true) && !d::unlikely(false)); + const d::thread_id_t here = d::thread_id(); + d::thread_id_t there{}; + std::jthread([&] { there = d::thread_id(); }).join(); + CHECK((here) != (there)); + CHECK((d::hash_thread_id(here)) == (d::hash_thread_id(d::thread_id()))); +} + +} // namespace + +int main() { + check::run("ConcurrentQueue.MpmcImplicitProducers", ConcurrentQueue_MpmcImplicitProducers); + check::run("ConcurrentQueue.MpmcTokensAndBulk", ConcurrentQueue_MpmcTokensAndBulk); + check::run("ConcurrentQueue.TryEnqueueIsBounded", ConcurrentQueue_TryEnqueueIsBounded); + check::run("ConcurrentQueue.UserTypes", ConcurrentQueue_UserTypes); + check::run("ConcurrentQueue.ExceptionsLeaveTheQueueConsistent", ConcurrentQueue_ExceptionsLeaveTheQueueConsistent); + check::run("ConcurrentQueue.TraitsSwapAndMove", ConcurrentQueue_TraitsSwapAndMove); + check::run("ConcurrentQueue.ProducerThatExitsAndTheDequeueFromProducer", ConcurrentQueue_ProducerThatExitsAndTheDequeueFromProducer); + check::run("BlockingConcurrentQueue.WaitDequeueAcrossThreads", BlockingConcurrentQueue_WaitDequeueAcrossThreads); + check::run("BlockingConcurrentQueue.TimedWaits", BlockingConcurrentQueue_TimedWaits); + check::run("LightweightSemaphore.SignalWaitAndTimeouts", LightweightSemaphore_SignalWaitAndTimeouts); + check::run("Details.AdaptedHelpers", Details_AdaptedHelpers); + return check::result(); +} diff --git a/tests/c_api/mcpp.toml b/tests/c_api/mcpp.toml new file mode 100644 index 0000000..6f66556 --- /dev/null +++ b/tests/c_api/mcpp.toml @@ -0,0 +1,12 @@ +# The C API through the module: its own project, for the c-api feature (and, +# here, gtest with no-thread-local, which GCC needs beside ). +[package] +name = "concurrentqueue-c-api-tests" +version = "0.1.0" +description = "mcpplibs.mwrap.concurrentqueue with the c-api and no-thread-local features" + +[dependencies] +"mcpplibs.mwrap.concurrentqueue" = { path = "../..", features = ["c-api", "no-thread-local"] } + +[dev-dependencies] +compat.gtest = "1.15.2" diff --git a/tests/c_api/tests/c_api_test.cpp b/tests/c_api/tests/c_api_test.cpp new file mode 100644 index 0000000..3720026 --- /dev/null +++ b/tests/c_api/tests/c_api_test.cpp @@ -0,0 +1,72 @@ +// Upstream's C API (c_api/concurrentqueue.h) through +// import mcpplibs.mwrap.concurrentqueue.c_api, and the queue through gtest: +// with no-thread-local the module has no thread_local, so a translation unit +// that includes (gtest does) instantiates it with GCC too. +#include + +import std; +import mcpplibs.mwrap.concurrentqueue; +import mcpplibs.mwrap.concurrentqueue.c_api; + +namespace macros = mcpplibs::mwrap::concurrentqueue::macros; + +TEST(CApi, ConcurrentQueue) { + MoodycamelCQHandle q = nullptr; + ASSERT_TRUE(moodycamel_cq_create(&q)); + int values[3] = {1, 2, 3}; + for (int& v : values) ASSERT_TRUE(moodycamel_cq_enqueue(q, &v)); + EXPECT_EQ(moodycamel_cq_size_approx(q), 3u); + MoodycamelValue out = nullptr; + int sum = 0; + while (moodycamel_cq_try_dequeue(q, &out)) sum += *static_cast(out); + EXPECT_EQ(sum, 6); + EXPECT_TRUE(moodycamel_cq_destroy(q)); +} + +TEST(CApi, BlockingQueueAcrossThreads) { + MoodycamelBCQHandle q = nullptr; + ASSERT_TRUE(moodycamel_bcq_create(&q)); + static int values[1000]; + std::jthread producer([&] { + for (int i = 0; i < 1000; ++i) { + values[i] = i; + moodycamel_bcq_enqueue(q, &values[i]); + } + }); + long sum = 0; + for (int i = 0; i < 1000; ++i) { + MoodycamelValue out = nullptr; + ASSERT_TRUE(moodycamel_bcq_wait_dequeue(q, &out)); + sum += *static_cast(out); + } + producer.join(); + MoodycamelValue none = nullptr; + EXPECT_FALSE(moodycamel_bcq_try_dequeue(q, &none)); + EXPECT_EQ(sum, 999L * 1000 / 2); + EXPECT_TRUE(moodycamel_bcq_destroy(q)); +} + +// The feature defines MOODYCAMEL_NO_THREAD_LOCAL for its consumers too (so the +// headers, where a unit includes them, agree with the module): set it aside +// to read the mapped constant. +#ifndef MOODYCAMEL_NO_THREAD_LOCAL +#error "no-thread-local defines MOODYCAMEL_NO_THREAD_LOCAL for consumers" +#endif +#pragma push_macro("MOODYCAMEL_NO_THREAD_LOCAL") +#undef MOODYCAMEL_NO_THREAD_LOCAL +static_assert(macros::MOODYCAMEL_NO_THREAD_LOCAL && !macros::MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED); +#pragma pop_macro("MOODYCAMEL_NO_THREAD_LOCAL") + +TEST(NoThreadLocal, QueueBesideGtest) { + moodycamel::ConcurrentQueue q; + std::vector threads; + for (int t = 0; t < 4; ++t) + threads.emplace_back([&, t] { + for (int i = 0; i < 100; ++i) q.enqueue(std::to_string(t * 1000 + i)); + }); + threads.clear(); + std::string s; + int n = 0; + while (q.try_dequeue(s)) ++n; + EXPECT_EQ(n, 400); +} diff --git a/tests/header_mix_test.cpp b/tests/header_mix_test.cpp new file mode 100644 index 0000000..5188a9e --- /dev/null +++ b/tests/header_mix_test.cpp @@ -0,0 +1,86 @@ +// The headers and the import in one translation unit (the side header first, +// the canonical order): the module's names are the very same entities, the +// syntax macros work beside them, and the header's configuration equals the +// mapped constants. While a macro is defined it expands even inside a +// qualified name, so each mapped constant is read with its macro set aside. +#include +#include "support/check.hpp" + +import std; +import mcpplibs.mwrap.concurrentqueue; + +namespace macros = mcpplibs::mwrap::concurrentqueue::macros; + +#ifdef MOODYCAMEL_EXCEPTIONS_ENABLED +#pragma push_macro("MOODYCAMEL_EXCEPTIONS_ENABLED") +#undef MOODYCAMEL_EXCEPTIONS_ENABLED +static_assert(macros::MOODYCAMEL_EXCEPTIONS_ENABLED); +#pragma pop_macro("MOODYCAMEL_EXCEPTIONS_ENABLED") +#else +static_assert(!macros::MOODYCAMEL_EXCEPTIONS_ENABLED); +#endif +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED +#pragma push_macro("MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED") +#undef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED +static_assert(macros::MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED); +#pragma pop_macro("MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED") +#else +static_assert(!macros::MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED); +#endif +#ifdef MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC +#pragma push_macro("MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC") +#undef MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC +static_assert(macros::MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC); +#pragma pop_macro("MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC") +#else +static_assert(!macros::MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC); +#endif + +namespace { +// The syntax macros, as upstream's tests use them. +struct MOODYCAMEL_ALIGNAS(32) Aligned { + int value = 0; + Aligned() MOODYCAMEL_NOEXCEPT = default; + explicit Aligned(int v) MOODYCAMEL_NOEXCEPT : value(v) {} + Aligned(Aligned&&) MOODYCAMEL_NOEXCEPT = default; + Aligned& operator=(Aligned&&) MOODYCAMEL_NOEXCEPT = default; + Aligned(const Aligned&) MOODYCAMEL_DELETE_FUNCTION; + Aligned& operator=(const Aligned&) MOODYCAMEL_DELETE_FUNCTION; +}; +} // namespace + +void same_entities() { + // The header's class template and the module's are one entity: an object + // made with one is the other. + moodycamel::ConcurrentQueue q; + ::moodycamel::ConcurrentQueue& same = q; + static_assert(std::is_same_v&>); + static_assert(alignof(Aligned) == 32 && !std::is_copy_constructible_v); + // The configured noexcept: a nothrow-movable element's bulk path. + Aligned items[3] = {Aligned(1), Aligned(2), Aligned(3)}; + CHECK(q.enqueue_bulk(std::make_move_iterator(items), 3)); + Aligned out[3]; + CHECK(q.try_dequeue_bulk(out, 3) == 3u); + CHECK(out[2].value == 3); + // MOODYCAMEL_TRY / MOODYCAMEL_CATCH, upstream's exception spelling. + bool caught = false; + MOODYCAMEL_TRY { MOODYCAMEL_THROW(std::runtime_error("x")); } + MOODYCAMEL_CATCH(const std::runtime_error&) { caught = true; } + CHECK(caught); + // The blocking queue and the semaphore through both spellings. + moodycamel::BlockingConcurrentQueue b; + b.enqueue(5); + int v = 0; + CHECK(b.wait_dequeue_timed(v, 1000)); + CHECK(v == 5); + moodycamel::LightweightSemaphore s(1); + CHECK(s.tryWait()); + // The adapted helper agrees with the header's own. + CHECK(mcpplibs::mwrap::concurrentqueue::details::circular_less_than(3u, 7u) == moodycamel::details::circular_less_than(3u, 7u)); + CHECK(mcpplibs::mwrap::concurrentqueue::details::ceil_to_pow_2(17u) == moodycamel::details::ceil_to_pow_2(17u)); +} + +int main() { + check::run("HeaderMix.SameEntities", same_entities); + return check::result(); +} diff --git a/tests/macros.cpp b/tests/macros.cpp new file mode 100644 index 0000000..4ed3433 --- /dev/null +++ b/tests/macros.cpp @@ -0,0 +1,28 @@ +// The value macros, at compile time: the macros module alone. +#include // the C library's version macros (__GLIBC__), for the expectations below + +import mcpplibs.mwrap.concurrentqueue.macros; +namespace macros = mcpplibs::mwrap::concurrentqueue::macros; +// A C++ build with exceptions: try / catch, not the exception-free spelling. +static_assert(macros::MOODYCAMEL_EXCEPTIONS_ENABLED); +// Upstream's defaults: no debugging switches, thread-locals allowed. +static_assert(!macros::MOODYCAMEL_NO_THREAD_LOCAL && !macros::MOODYCAMEL_QUEUE_INTERNAL_DEBUG); +static_assert(!macros::MCDBGQ_USE_RELACY && !macros::MCDBGQ_TRACKMEM && !macros::MCDBGQ_NOLOCKFREE_FREELIST); +static_assert(!macros::MCDBGQ_USEDEBUGFREELIST && !macros::MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX && !macros::MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH); +// Derived: implicit producers are recycled at thread exit through C++11 +// thread_local where upstream's test admits it (not Clang in GNU mode, which +// reports __GNUC__ 4.2, nor ARM). The condition is upstream's, as it is. +#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && (!defined(__MINGW32__) && !defined(__MINGW64__) || !defined(__WINPTHREADS_VERSION)) && \ + (!defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && !defined(__arm__) && !defined(_M_ARM) && \ + !defined(__aarch64__) && !defined(__MVS__) +static_assert(macros::MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED); +#else +static_assert(!macros::MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED); +#endif +// CLOCK_MONOTONIC timed waits: glibc 2.30 and later (with _GNU_SOURCE, which C++ implies). +#if defined(__GLIBC__) && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 30)) +static_assert(macros::MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC); +#elif !defined(__linux__) +static_assert(!macros::MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC); +#endif +int main() { return 0; } diff --git a/tests/support/check.hpp b/tests/support/check.hpp new file mode 100644 index 0000000..69cf8cc --- /dev/null +++ b/tests/support/check.hpp @@ -0,0 +1,28 @@ +// A small checker for the tests. Not gtest: with GCC 16.1, a translation unit +// that includes (gtest does) cannot instantiate the queue's +// thread-exit path from the module (docs/usage.md, "GCC and "). +#pragma once +#include +#include +#include + +namespace check { +inline int failures = 0; +inline void that(bool ok, const char* what, std::source_location at = std::source_location::current()) { + if (ok) return; + ++failures; + std::fprintf(stderr, "%s:%u: check failed: %s\n", at.file_name(), static_cast(at.line()), what); +} +template +void run(const char* name, Test test) { + const int before = failures; + test(); + std::printf("[%s] %s\n", failures == before ? " OK " : "FAILED", name); +} +inline int result() { + std::printf("%s (%d failed checks)\n", failures ? "FAILED" : "all passed", failures); + return failures ? 1 : 0; +} +} // namespace check + +#define CHECK(...) ::check::that(static_cast(__VA_ARGS__), #__VA_ARGS__) diff --git a/tools/check_exports.py b/tools/check_exports.py new file mode 100644 index 0000000..dafd1f8 --- /dev/null +++ b/tools/check_exports.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Audit the modules against concurrentqueue's headers (the pinned release's). + +- src/concurrentqueue.cppm exports every entity of namespace moodycamel + (outside moodycamel::details) that concurrentqueue.h, + blockingconcurrentqueue.h and lightweightsemaphore.h declare, every + overload of swap included, and nothing else. +- Every name of moodycamel::details is listed below as adapted (its TU-local + helpers, declared in mcpplibs::mwrap::concurrentqueue::details) or as the + queue's implementation, with the reason it is not exported. +- src/c_api/c_api.cppm exports every type and function of c_api/concurrentqueue.h. +- Every macro the headers define is mapped in src/macros.cppm or kept as a + macro in with its reason; every + configuration input they test is mapped. +""" +from pathlib import Path +import re +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from inventory import fetch, surface # noqa: E402 + +SIDE_HEADER = { + 'MOODYCAMEL_NOEXCEPT': 'a specifier (noexcept, or nothing without exceptions)', + 'MOODYCAMEL_NOEXCEPT_CTOR': 'an expression over its argument expression (the configuration point, see config.hpp)', + 'MOODYCAMEL_NOEXCEPT_ASSIGN': 'an expression over its argument expression (the configuration point, see config.hpp)', + 'MOODYCAMEL_DELETE_FUNCTION': 'a function definition (= delete)', + 'MOODYCAMEL_ALIGNAS': 'an attribute-specifier (alignas)', + 'MOODYCAMEL_ALIGNOF': 'an operator applied to a type (alignof)', + 'MOODYCAMEL_ALIGNED_TYPE_LIKE': 'a type with an alignment specifier', + 'MOODYCAMEL_TRY': 'control flow (try, or if constexpr (true))', + 'MOODYCAMEL_CATCH': 'control flow (catch, or else if constexpr (false))', + 'MOODYCAMEL_RETHROW': 'control flow (throw;)', + 'MOODYCAMEL_THROW': 'control flow (throw expr)', + 'MOODYCAMEL_CONSTEXPR_IF': 'control flow (if constexpr)', + 'MOODYCAMEL_MAYBE_UNUSED': 'an attribute ([[maybe_unused]])', + 'MOODYCAMEL_THREADLOCAL': 'a storage class (__thread / thread_local)', + 'MOODYCAMEL_NO_TSAN': 'an attribute (no_sanitize("thread"))', + 'MOODYCAMEL_EXPORT': "the C API's linkage annotation (dllimport / dllexport)", +} + +# moodycamel::details: the TU-local helpers adapted under the same names. +ADAPTED = {'thread_id', 'likely', 'unlikely', 'hash_thread_id', 'circular_less_than', 'ceil_to_pow_2', 'thread_id_t'} + +NOT_EXPORTED = { + 'align_for': 'TU-local template over any element type: the queue instantiates upstream\'s own', + 'swap_relaxed': 'TU-local template over any atomic: the queue instantiates upstream\'s own', + 'nomove': 'TU-local template over any type: the queue instantiates upstream\'s own', + 'deref_noexcept': 'TU-local template over any iterator: the queue instantiates upstream\'s own', + 'nomove_if': "the queue's implementation (bulk enqueue's copy-or-move)", + 'ConcurrentQueueProducerTypelessBase': "the queue's implementation (the producers' base, reachable through the tokens)", + 'ThreadExitListener': "the queue's implementation (thread-exit recycling of implicit producers)", + 'ThreadExitNotifier': "the queue's implementation (thread-exit recycling of implicit producers)", + 'Semaphore': "LightweightSemaphore's implementation (the OS semaphore)", + 'thread_id_converter': "the queue's implementation (thread-id hashing)", + 'hash_32_or_64': "the queue's implementation (thread-id hashing)", + '_hash_32_or_64': "the queue's implementation (thread-id hashing)", + 'const_numeric_max': "the queue's implementation (index arithmetic)", + 'static_is_lock_free': "the queue's implementation (is_lock_free())", + 'static_is_lock_free_num': "the queue's implementation (is_lock_free())", + 'is_trivially_destructible': "the queue's implementation (a type-trait shim)", + 'identity': "the queue's implementation (a type identity)", + 'max_align_t': "the queue's implementation (block alignment)", + 'std_max_align_t': "the queue's implementation (block alignment)", + 'invalid_thread_id': "TU-local constant of the queue's implementation (the empty hash slot)", + 'invalid_thread_id2': "TU-local constant of the queue's implementation (the empty hash slot)", +} + + +def exported(text: str, pattern: str) -> set[str]: + return set(re.findall(pattern, text[text.index('export module'):], flags=re.M)) + + +def main(): + root = Path(__file__).resolve().parents[1] + sf = surface(fetch(root)) + problems = [] + cppm = (root / 'src/concurrentqueue.cppm').read_text(encoding='utf-8') + have = exported(cppm, r'^using moodycamel::(\w+);') + if set(sf.entities) - have: + problems.append('src/concurrentqueue.cppm lacks: ' + ', '.join(sorted(set(sf.entities) - have))) + if have - set(sf.entities): + problems.append('src/concurrentqueue.cppm exports what the headers do not declare: ' + ', '.join(sorted(have - set(sf.entities)))) + adaptations = cppm[cppm.index('export namespace mcpplibs::mwrap::concurrentqueue::details'):] + adaptations = adaptations[:adaptations.index('} // namespace')] + adapted = set(re.findall(r'\b(\w+)\s*\(', adaptations)) | set(re.findall(r'using\s+(\w+)\s*=', adaptations)) + if adapted != ADAPTED: + problems.append(f'adapted helpers differ from the list: {sorted(adapted ^ ADAPTED)}') + details = set(sf.details) + unaccounted = details - ADAPTED - NOT_EXPORTED.keys() + if unaccounted: + problems.append('moodycamel::details names neither adapted nor listed: ' + ', '.join(sorted(unaccounted))) + stale = (ADAPTED | NOT_EXPORTED.keys()) - details + if stale: + problems.append('listed but not in moodycamel::details: ' + ', '.join(sorted(stale))) + tu_local = set(sf.tu_local) - ADAPTED + if tu_local - NOT_EXPORTED.keys(): + problems.append('TU-local helpers neither adapted nor listed: ' + ', '.join(sorted(tu_local - NOT_EXPORTED.keys()))) + c_api = exported((root / 'src/c_api/c_api.cppm').read_text(encoding='utf-8'), r'export using ::(\w+);') + if c_api != set(sf.c_api): + problems.append(f'the C API module differs from c_api/concurrentqueue.h: {sorted(c_api ^ set(sf.c_api))}') + mapped = exported((root / 'src/macros.cppm').read_text(encoding='utf-8'), r'inline constexpr bool (\w+) =') + macros = set(sf.macros) + unaccounted = macros - mapped - SIDE_HEADER.keys() + if unaccounted: + problems.append('macros neither mapped nor kept as macros: ' + ', '.join(sorted(unaccounted))) + both = SIDE_HEADER.keys() & mapped + if both: + problems.append('kept as macros yet mapped: ' + ', '.join(sorted(both))) + stale = SIDE_HEADER.keys() - macros + if stale: + problems.append('listed but not defined by the headers: ' + ', '.join(sorted(stale))) + if sf.read_macros - mapped: + problems.append('configuration inputs not mapped: ' + ', '.join(sorted(sf.read_macros - mapped))) + if mapped - macros - sf.read_macros: + problems.append('mapped but neither defined nor tested by the headers: ' + ', '.join(sorted(mapped - macros - sf.read_macros))) + if problems: + sys.exit('\n'.join(problems)) + print(f'{len(sf.entities)} names of moodycamel exported ({", ".join(sf.entities)}); moodycamel::details: ' + f'{len(details & ADAPTED)} adapted, {len(details - ADAPTED)} implementation; C API: {len(c_api)} names; ' + f'{len(macros)} macros: {len(macros & mapped)} mapped, {len(SIDE_HEADER)} kept as macros; ' + f'{len(sf.read_macros)} configuration inputs mapped') + + +if __name__ == '__main__': + main() diff --git a/tools/check_mwrap_release.py b/tools/check_mwrap_release.py new file mode 100644 index 0000000..9bedddb --- /dev/null +++ b/tools/check_mwrap_release.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Check the immutable mwrap release coordinate and bilingual entry pages.""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import tomllib + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--tag', help='release tag; omitted during PR checks') + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + package = tomllib.loads((root / 'mcpp.toml').read_text(encoding="utf-8"))['package'] + assert package['namespace'] == 'mcpplibs.mwrap', package + assert re.fullmatch(r'[A-Za-z_][A-Za-z_0-9]*', package['name']), package['name'] + upstream = package['metadata']['mwrap']['upstream-version'] + version = package['version'] + assert version == upstream or re.fullmatch(re.escape(upstream) + r'\.[1-9][0-9]*', version), (version, upstream) + if args.tag: + assert args.tag == 'mwrap-' + version, (args.tag, version) + assert package['license'] == 'Apache-2.0', package['license'] + assert package['authors'] == ['mcpp-community'], package.get('authors') + license_text = (root / 'LICENSE').read_text(encoding='utf-8') + assert 'Apache License' in license_text and 'Version 2.0' in license_text + assert 'mcpp-community' in (root / 'NOTICE').read_text(encoding='utf-8') + documents = json.loads((root / 'upstream/documents.json').read_text(encoding='utf-8'))['documents'] + assert any(Path(d['path']).name.startswith('README') for d in documents) + assert any(Path(d['path']).name.startswith(('LICENSE', 'COPYING')) for d in documents) + deployed = {entry['from'] for entry in tomllib.loads((root / 'mcpp.toml').read_text(encoding='utf-8')).get('runtime', {}).get('deploy', [])} + assert {'LICENSE', 'NOTICE'} <= deployed + for document in documents: + path = Path(document['path']) + assert not path.is_absolute() and '..' not in path.parts and path.parts[0] == 'upstream' + assert hashlib.sha256((root / path).read_bytes()).hexdigest() == document['sha256'], path + assert document['source'] and path.as_posix() in deployed, path + identity = package['namespace'] + '.' + package['name'] + for name, other in [('README.md', 'README.zh.md'), ('README.zh.md', 'README.md')]: + text = (root / name).read_text(encoding="utf-8") + assert len(text.splitlines()) <= 150, name + assert '](' + other + ')' in '\n'.join(text.splitlines()[:12]), name + assert identity in text and version in text, name + assert text.count('```') % 2 == 0, name + print(identity + '@' + version) + + +if __name__ == '__main__': + main() diff --git a/tools/inventory.py b/tools/inventory.py new file mode 100644 index 0000000..071bcfe --- /dev/null +++ b/tools/inventory.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""The public surface of concurrentqueue's headers, from the pinned release. + +Read by tools/check_exports.py, which audits the modules against it. The +headers come from the upstream tarball, downloaded once into target/upstream +and checked against upstream/documents.json: the same bytes +compat.concurrentqueue provides. + +The headers choose between platform branches with the preprocessor (the +thread id: Win32, std::thread::id or a thread_local address; the semaphore: +Win32, Mach or POSIX). Their conditionals are evaluated here for one target +(x86-64 Linux, GCC, exceptions on, upstream's defaults); the entities they +declare are the same on every target, only their definitions differ. Every +`#define`, in any branch, is listed. +""" +from dataclasses import dataclass, field +import hashlib +import json +from pathlib import Path +import re +import sys +import tarfile +import urllib.request + +HEADERS = ('concurrentqueue.h', 'blockingconcurrentqueue.h', 'lightweightsemaphore.h') +C_API = 'c_api/concurrentqueue.h' + +TARGET = {'__GNUC__': '16', '__GNUC_MINOR__': '1', '__GNUG__': '16', '__cplusplus': '202302L', '__EXCEPTIONS': '1', + '__linux__': '1', '__unix__': '1', '__x86_64__': '1', '__amd64__': '1', '__GLIBCXX__': '20260101', + '__GLIBC__': '2', '__GLIBC_MINOR__': '39', '_GNU_SOURCE': '1', '__has_feature': ''} + + +def fetch(root: Path) -> Path: + pins = json.loads((root / 'upstream/documents.json').read_text(encoding='utf-8')) + cache = root / 'target/upstream' + cache.mkdir(parents=True, exist_ok=True) + archive = cache / 'concurrentqueue-release.tar.gz' + if not archive.exists() or hashlib.sha256(archive.read_bytes()).hexdigest() != pins['upstream_sha256']: + print('+ fetch', pins['upstream'], flush=True) + with urllib.request.urlopen(pins['upstream'], timeout=120) as response: + archive.write_bytes(response.read()) + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + if digest != pins['upstream_sha256']: + sys.exit(f'upstream archive checksum {digest} != {pins["upstream_sha256"]}') + with tarfile.open(archive) as tar: + top = tar.getmembers()[0].name.split('/')[0] + if not (cache / top).exists(): + tar.extractall(cache, filter='data') + return cache / top + + +def strip_comments(text: str) -> str: + text = re.sub(r'/\*.*?\*/', lambda m: ' ' + '\n' * m.group(0).count('\n'), text, flags=re.S) + return re.sub(r'//[^\n]*', '', text) + + +class Preprocessor: + """Conditionals and #define / #undef only; nothing is expanded in the text.""" + + def __init__(self, predefined: dict[str, str]): + self.defines = dict(predefined) + self.active: list[str] = [] + self.defined_anywhere: set[str] = set() + + def value(self, expr: str, depth: int = 0) -> int: + expr = re.sub(r'\bdefined\s*\(\s*(\w+)\s*\)|\bdefined\s+(\w+)', + lambda m: '1' if (m.group(1) or m.group(2)) in self.defines else '0', expr) + expr = re.sub(r'\b\w+\s*\([^()]*\)', '0', expr) # __has_feature(…), __GLIBC_PREREQ(…) + + def name(m): + word = m.group(0) + if word in self.defines and depth < 8: + body = self.defines[word].strip() + return f'({self.value(body, depth + 1)})' if body else '1' + return '0' + expr = re.sub(r'\b[A-Za-z_]\w*\b', name, expr) + expr = re.sub(r'\b(0x[0-9A-Fa-f]+|\d+)[uUlL]+\b', r'\1', expr) + expr = expr.replace('&&', ' and ').replace('||', ' or ') + expr = re.sub(r'!(?!=)', ' not ', expr) + return int(eval(expr, {'__builtins__': {}})) # noqa: S307 - integer constant expressions + + def run(self, text: str): + stack: list[list[bool]] = [] + active = True + for line in strip_comments(text).replace('\\\n', ' ').split('\n'): + m = re.match(r'\s*#\s*(\w+)\s*(.*)', line) + if not m: + if active: + self.active.append(line) + continue + directive, rest = m.group(1), m.group(2).strip() + if directive in ('if', 'ifdef', 'ifndef'): + if directive == 'if': + cond = active and bool(self.value(rest)) + elif directive == 'ifdef': + cond = active and rest.split()[0] in self.defines + else: + cond = active and rest.split()[0] not in self.defines + stack.append([cond, cond, active]) + active = cond + elif directive == 'elif': + frame = stack[-1] + cond = frame[2] and not frame[1] and bool(self.value(rest)) + frame[0], frame[1] = cond, frame[1] or cond + active = cond + elif directive == 'else': + frame = stack[-1] + cond = frame[2] and not frame[1] + frame[0], frame[1] = cond, True + active = cond + elif directive == 'endif': + active = stack.pop()[2] + elif directive == 'define': + d = re.match(r'(\w+)(\([^)]*\))?\s*(.*)', rest) + self.defined_anywhere.add(d.group(1)) + if active: + self.defines[d.group(1)] = '' if d.group(2) else d.group(3) + elif directive == 'undef' and active: + self.defines.pop(rest.split()[0], None) + + +@dataclass +class Surface: + entities: list[str] = field(default_factory=list) # namespace moodycamel, outside details + details: list[str] = field(default_factory=list) # moodycamel::details: classes, typedefs, functions, variables + tu_local: list[str] = field(default_factory=list) # … of which `static` functions (TU-local) + tu_local_templates: list[str] = field(default_factory=list) # … of which function templates + c_api: list[str] = field(default_factory=list) # c_api/concurrentqueue.h: types and functions + macros: dict[str, str] = field(default_factory=dict) # every #define -> header + read_macros: set[str] = field(default_factory=set) # configuration inputs the headers test + + +def scan(code: str, s: Surface): + """Namespace-scope declarations of moodycamel and moodycamel::details.""" + stack: list[str | None] = [] # namespace names; None for any other brace + last = 0 + for m in re.finditer(r'\bnamespace\s+(\w+)\s*\{|\{|\}|;', code): + segment = ' '.join(code[last:m.start()].split()) + inside = [n for n in stack] + at_ns = all(n is not None for n in inside) + where = '::'.join(inside) if at_ns else None + token = m.group(0) + if token.startswith('namespace'): + stack.append(m.group(1)) + elif token == '{': + if where in ('moodycamel', 'moodycamel::details'): + record(segment, where, s, definition=True) + stack.append(None) + elif token == '}': + stack.pop() + elif where in ('moodycamel', 'moodycamel::details'): + record(segment, where, s, definition=False) + last = m.end() + + +def record(segment: str, where: str, s: Surface, definition: bool): + segment = re.sub(r'^(?:template\s*<.*?>\s*)+', lambda m: m.group(0), segment) + is_template = segment.startswith('template') + body = re.sub(r'^template\s*<[^{};]*?>\s*', '', segment) + name = None + m = re.match(r'(?:struct|class|union)\s+(?:\w+\s+)*?(\w+)\s*(?:<[^{]*>)?\s*(?::[^{]*)?$', body) + if body.startswith('typedef') and definition: + return # `typedef union { … } name;`: the name follows the body + if re.match(r'^\w+$', body) and not definition: + name = body # … here + elif m: + name = m.group(1) + elif body.startswith('typedef'): + name = re.findall(r'(\w+)\s*$', body)[0] + elif body.startswith('using '): + name = body.split()[1] + else: + # A declarator's name: not a member defined out of its class + # (`ProducerToken::ProducerToken(…) : producer(…)`). + head = body.split(') :')[0] + ')' if ') :' in body else body + f = re.search(r'(?.*)?$', head) + if f and re.search(r'\b\w+::' + f.group(1) + r'\s*\(', head): + return + if f: + name = f.group(1) + if where == 'moodycamel::details' and re.match(r'static\b', body): + s.tu_local.append(name) + if is_template: + s.tu_local_templates.append(name) + else: + v = re.match(r'(?:static\s+)?(?:const\s+)?[\w:<>]+\s+(\w+)\s*(?:=.*)?$', body) + if v: + name = v.group(1) + if not name or name in ('MOODYCAMEL_ALIGNAS',): + return + (s.entities if where == 'moodycamel' else s.details).append(name) + + +def surface(upstream: Path) -> Surface: + s = Surface() + pp = Preprocessor(TARGET) + for name in HEADERS: + text = (upstream / name).read_text(encoding='utf-8') + before = set(pp.defined_anywhere) + pp.run(text) + for macro in pp.defined_anywhere - before: + s.macros.setdefault(macro, name) + s.read_macros |= set(re.findall(r'#\s*(?:if|ifdef|ifndef|elif)\b[^\n]*?\b((?:MOODYCAMEL|MCDBGQ)_\w+)', text)) + scan('\n'.join(pp.active), s) + c_api = strip_comments((upstream / C_API).read_text(encoding='utf-8')) + for macro in re.findall(r'^\s*#\s*define\s+(\w+)', c_api, flags=re.M): + s.macros.setdefault(macro, C_API) + s.c_api = sorted(set(re.findall(r'typedef\s+[\w\s\*]+?\b(\w+)\s*;', c_api)) | + set(re.findall(r'MOODYCAMEL_EXPORT\s+[\w\s\*]+?\b(\w+)\s*\(', c_api))) + s.entities = sorted(set(s.entities)) + s.details = sorted(set(s.details)) + s.tu_local = sorted(set(s.tu_local)) + s.tu_local_templates = sorted(set(s.tu_local_templates)) + s.read_macros -= set(s.macros) + return s + + +if __name__ == '__main__': + root = Path(__file__).resolve().parents[1] + sf = surface(fetch(root)) + print('moodycamel:', ' '.join(sf.entities)) + print('details:', ' '.join(sf.details)) + print('TU-local:', ' '.join(sf.tu_local), '| templates:', ' '.join(sf.tu_local_templates)) + print('C API:', ' '.join(sf.c_api)) + print('macros:', ' '.join(sorted(sf.macros))) + print('configuration inputs:', ' '.join(sorted(sf.read_macros))) diff --git a/tools/run_conformance.py b/tools/run_conformance.py new file mode 100644 index 0000000..a578408 --- /dev/null +++ b/tools/run_conformance.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Run concurrentqueue's own unit tests (tests/unittests of the release) +built against import mcpplibs.mwrap.concurrentqueue instead of the headers. + +The release tarball is downloaded once into target/upstream and checked +against upstream/documents.json. Its test harness is copied unchanged into +target/conformance/unittests (tests/common/*, tests/corealgos.h), except for +these mechanical changes, each written out below: + +- unittests.cpp and mallocmacro.cpp: the three `#include "../../…h"` of the + queue become `import mcpplibs.mwrap.concurrentqueue;` (and `.c_api`, built + with the c-api feature), after the standard headers those three included, + so nothing of the queue is declared by a header; +- the syntax macros the tests use are spelled as upstream defines them in + this configuration (MOODYCAMEL_NOEXCEPT -> noexcept, + MOODYCAMEL_DELETE_FUNCTION -> = delete, MOODYCAMEL_ALIGNAS(x) -> + alignas(x)), and the one `#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED` + block becomes `if constexpr` on the module's constant of that name; +- details::circular_less_than, a `static` helper the tests reach into, is the + module's adaptation, mcpplibs::mwrap::concurrentqueue::details::circular_less_than; +- `#if !defined(TARGET_OS_IPHONE)` before main() also admits macOS, where + TargetConditionals.h (which tests/corealgos.h includes) defines it as 0: + upstream's file has no main() there; +- minitest.h declares abi::__cxa_demangle, the one function it uses, instead + of including : with GCC 16, a unit that includes + cannot instantiate the module's thread-exit path (docs/usage.md). + +Every test runs (`--iterations`, default 2, bounds the time; upstream's +default is 8) and the run must end with upstream's "All tests passed.". +""" +import argparse +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from inventory import fetch # noqa: E402 + +# The standard headers concurrentqueue.h, blockingconcurrentqueue.h and +# lightweightsemaphore.h include, in their order: what the tests had through +# them. They come before the imports, so tests/corealgos.h (included later) +# finds them already included. +STANDARD = ['atomic', 'cassert', 'cstddef', 'cstdint', 'cstdlib', 'type_traits', 'algorithm', 'utility', 'limits', + 'climits', 'array', 'thread', 'mutex', 'memory', 'chrono', 'ctime', 'new'] + +IMPORTS = '''// [mwrap] the queue, from the module (was: the three headers). +{includes} +import mcpplibs.mwrap.concurrentqueue; +{c_api}''' + +MANIFEST = '''[package] +name = "concurrentqueue-unittests" +version = "1.0.5" +description = "concurrentqueue 1.0.5's tests/unittests, built against import mcpplibs.mwrap.concurrentqueue (tools/run_conformance.py)" + +[targets.unittests] +kind = "bin" +main = "tests/unittests/unittests.cpp" + +[build] +sources = ["tests/common/*.cpp", "tests/unittests/mallocmacro.cpp"] + +[dependencies] +"mcpplibs.mwrap.concurrentqueue" = {{ path = {root}, features = ["c-api"] }} +''' + + +def port_unittests(text: str) -> str: + out, inserted = [], False + for line in text.split('\n'): + m = re.match(r'\s*#\s*include\s+"\.\./\.\./(concurrentqueue|blockingconcurrentqueue|c_api/concurrentqueue)\.h"', line) + if m: + if not inserted: + out.append(IMPORTS.format(includes='\n'.join(f'#include <{h}>' for h in STANDARD), + c_api='import mcpplibs.mwrap.concurrentqueue.c_api;')) + inserted = True + continue + out.append(line) + assert inserted + text = '\n'.join(out) + text = text.replace('MOODYCAMEL_DELETE_FUNCTION', '= delete') + text = re.sub(r'\bMOODYCAMEL_NOEXCEPT\b', 'noexcept', text) + text = re.sub(r'\bMOODYCAMEL_ALIGNAS\(', 'alignas(', text) + text = re.sub(r'\bdetails::circular_less_than\b', 'mcpplibs::mwrap::concurrentqueue::details::circular_less_than', text) + # main() is upstream's except on iOS, which it tests with !defined(TARGET_OS_IPHONE); + # corealgos.h includes TargetConditionals.h on Apple, where macOS defines it as 0. + old = '#if !defined(TARGET_OS_IPHONE)' + assert text.count(old) == 1 + text = text.replace(old, '#if !defined(TARGET_OS_IPHONE) || !TARGET_OS_IPHONE // [mwrap] macOS defines it as 0') + # The one #ifdef block on the configuration: if constexpr on the mapped constant. + lines = text.split('\n') + start = next(i for i, l in enumerate(lines) if l.strip() == '#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED') + end = next(i for i in range(start + 1, len(lines)) if lines[i].strip().startswith('#')) + assert lines[end].strip() == '#endif', lines[end] + lines[start] = '\t\tif constexpr (mcpplibs::mwrap::concurrentqueue::macros::MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED) { // [mwrap] was #ifdef' + lines[end] = '\t\t} // [mwrap] was #endif' + text = '\n'.join(lines) + leftover = re.findall(r'(? str: + return re.sub(r'#include "\.\./\.\./blockingconcurrentqueue\.h"', + IMPORTS.format(includes='', c_api='').strip() + '\n', text) + + +def port_minitest(text: str) -> str: + old = '#include ' + assert old in text + return text.replace(old, '// [mwrap] abi::__cxa_demangle, declared as declares it (see the module docs)\n' + '#include \n' + 'namespace __cxxabiv1 { extern "C" char* __cxa_demangle(const char*, char*, std::size_t*, int*); }\n' + 'namespace abi = __cxxabiv1;') + + +def build(project: Path, mcpp: str, toolchain: str | None) -> Path: + command = [mcpp, 'build', '--profile', 'release'] + (['--toolchain', toolchain] if toolchain else []) + print('+', ' '.join(command), flush=True) + subprocess.run(command, cwd=project, check=True) + exe = 'unittests' + ('.exe' if os.name == 'nt' else '') + found = sorted((p for p in (project / 'target').rglob(exe) if p.parent.name == 'bin'), key=lambda p: p.stat().st_mtime) + if not found: + sys.exit(f'{project} was not built') + return found[-1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--mcpp', default='mcpp') + parser.add_argument('--toolchain') + parser.add_argument('--iterations', type=int, default=2) + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + upstream = fetch(root) + project = root / 'target/conformance/unittests' + if (project / 'tests').exists(): + shutil.rmtree(project / 'tests') + (project / 'tests/unittests').mkdir(parents=True) + shutil.copytree(upstream / 'tests/common', project / 'tests/common') + shutil.copyfile(upstream / 'tests/corealgos.h', project / 'tests/corealgos.h') + src = upstream / 'tests/unittests' + (project / 'tests/unittests/unittests.cpp').write_text(port_unittests(src.joinpath('unittests.cpp').read_text(encoding='utf-8')), encoding='utf-8') + (project / 'tests/unittests/mallocmacro.cpp').write_text(port_mallocmacro(src.joinpath('mallocmacro.cpp').read_text(encoding='utf-8')), encoding='utf-8') + (project / 'tests/unittests/minitest.h').write_text(port_minitest(src.joinpath('minitest.h').read_text(encoding='utf-8')), encoding='utf-8') + (project / 'mcpp.toml').write_text(MANIFEST.format(root='"' + root.as_posix() + '"'), encoding='utf-8') + exe = build(project, args.mcpp, args.toolchain) + command = [str(exe), '--disable-prompt', '--iterations', str(args.iterations)] + print('+', ' '.join(command), flush=True) + run = subprocess.run(command, cwd=project, capture_output=True, text=True, timeout=3600) + output = run.stdout + run.stderr + print(output, flush=True) + registered = re.findall(r'^\s*REGISTER_TEST\((.+?)\);', src.joinpath('unittests.cpp').read_text(encoding='utf-8'), flags=re.M) + # The class name is typeid's, demangled where the ABI allows (MSVC's reads `class moodycamel::…`). + ran = re.findall(r'^(?:class |struct )?(?:moodycamel::)?ConcurrentQueueTests::(.+?)\.\.\. *\n\s*passed\s*$', output.replace('\r\n', '\n'), flags=re.M) + missing = [name for name in registered if name.replace(' ', '') not in {r.replace(' ', '') for r in ran}] + print(f'{len(ran)} of the {len(registered)} registered tests passed, {args.iterations} iteration(s) each; exit {run.returncode}') + if missing: + print('not run or not passed:', ', '.join(missing)) + if run.returncode != 0 or 'All tests passed.' not in output or missing: + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/tools/test_template.py b/tools/test_template.py new file mode 100644 index 0000000..81ea805 --- /dev/null +++ b/tools/test_template.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Render an installed candidate's default mcpp template, then build and run it.""" +import argparse +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import tomllib + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mcpp", default="mcpp") + parser.add_argument("--namespace", help="index namespace for legacy manifests without package.namespace") + parser.add_argument("--toolchain", default="llvm@23.1.3") + parser.add_argument("--template", help="named template; omit for the package default") + parser.add_argument("--run-arg", action="append", default=[]) + parser.add_argument("--index", type=Path, help="candidate SDK/wrapper index for unpublished feature dependencies") + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + metadata = tomllib.loads((root / "mcpp.toml").read_text())["package"] + namespace = metadata.get("namespace", args.namespace) + assert namespace, "Supply the published index namespace for this legacy manifest" + name, version = metadata["name"], metadata["version"] + identity = namespace + "." + name + with tempfile.TemporaryDirectory(prefix=name + "-template-") as directory: + temporary = Path(directory) + home = temporary / "home" + package = home / "registry/data/xpkgs" / (namespace + "-x-" + name) / version / (name + "-" + version) + shutil.copytree(root, package, ignore=shutil.ignore_patterns( + ".git", "target", ".mcpp", ".xlings", ".candidate-index", "__pycache__", "mcpp.lock", "compile_commands.json", "mcpp-2026.*")) + index = home / "indices/mcpplibs/pkgs/mwrap" + index.mkdir(parents=True) + entry = '{url="https://example.invalid/candidate.tar.gz",sha256="fixture"}' + descriptor = ('package={spec="1",namespace=' + json.dumps(namespace) + ',name=' + json.dumps(name) + + ',description="Installed template fixture",type="package",mcpp="mcpp.toml",xpm={') + descriptor += ",".join(platform + "={[" + json.dumps(version) + "]=" + entry + "}" + for platform in ("linux", "macosx", "windows")) + "}}\n" + (index / (identity + ".lua")).write_text(descriptor) + env = dict(os.environ, MCPP_HOME=str(home)) + selector = identity + "@" + version + for command in ([args.mcpp, "new", "--list-templates", selector], + [args.mcpp, "new", "quickstart", "--template", selector + (":" + args.template if args.template else "")]): + print("+", " ".join(command), flush=True) + subprocess.run(command, cwd=temporary, env=env, check=True) + project = temporary / "quickstart" + manifest = project / "mcpp.toml" + text = manifest.read_text() + dependency = tomllib.loads(text)["dependencies"][identity] + options = {"version": dependency} if isinstance(dependency, str) else dependency + assert options["version"] == version, options + replacement = '{ path = ' + json.dumps(package.as_posix()) + if options.get("features"): + replacement += ', features = ' + json.dumps(options["features"]) + replacement += ' }' + lines = text.splitlines() + matches = [i for i, line in enumerate(lines) if line.startswith(json.dumps(identity) + " =")] + assert len(matches) == 1, "Expected exactly one qualified dependency from self.name" + lines[matches[0]] = json.dumps(identity) + " = " + replacement + manifest.write_text("\n".join(lines) + "\n") + if args.index: + candidate_index = args.index.resolve() + subprocess.run([os.sys.executable, str(candidate_index / "tests/prepare_mwrap_indices.py")], + cwd=candidate_index, check=True) + with manifest.open("a") as out: + out.write("\n[indices]\n") + for namespace in ("compat", "mcpplibs.mwrap"): + view = candidate_index / "tests/mwrap/.indices" / namespace + out.write(json.dumps(namespace) + ' = { path = ' + json.dumps(view.as_posix()) + ' }\n') + command = [args.mcpp, "build", "--toolchain", args.toolchain, "--strict"] + print("+", " ".join(command), flush=True) + subprocess.run(command, cwd=project, check=True) + command = [args.mcpp, "run", "--toolchain", args.toolchain] + if args.run_arg: + command += ["--", *args.run_arg] + print("+", " ".join(command), flush=True) + subprocess.run(command, cwd=project, check=True) + + +if __name__ == "__main__": + main() diff --git a/upstream/concurrentqueue/LICENSE.md b/upstream/concurrentqueue/LICENSE.md new file mode 100644 index 0000000..5193389 --- /dev/null +++ b/upstream/concurrentqueue/LICENSE.md @@ -0,0 +1,62 @@ +This license file applies to everything in this repository except that which +is explicitly annotated as being written by other authors, i.e. the Boost +queue (included in the benchmarks for comparison), Intel's TBB library (ditto), +dlib::pipe (ditto), +the CDSChecker tool (used for verification), the Relacy model checker (ditto), +and Jeff Preshing's semaphore implementation (used in the blocking queue) which +has a zlib license (embedded in lightweightsempahore.h). + +--- + +Simplified BSD License: + +Copyright (c) 2013-2016, Cameron Desrochers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +- Redistributions of source code must retain the above copyright notice, this list of +conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright notice, this list of +conditions and the following disclaimer in the documentation and/or other materials +provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +I have also chosen to dual-license under the Boost Software License as an alternative to +the Simplified BSD license above: + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/upstream/concurrentqueue/README.md b/upstream/concurrentqueue/README.md new file mode 100644 index 0000000..c444a36 --- /dev/null +++ b/upstream/concurrentqueue/README.md @@ -0,0 +1,533 @@ +# moodycamel::ConcurrentQueue + +An industrial-strength lock-free queue for C++. + +Note: If all you need is a single-producer, single-consumer queue, I have [one of those too][spsc]. + +## Features + +- Knock-your-socks-off [blazing fast performance][benchmarks]. +- Single-header implementation. Just drop it in your project. +- Fully thread-safe lock-free queue. Use concurrently from any number of threads. +- C++11 implementation -- elements are moved (instead of copied) where possible. +- Templated, obviating the need to deal exclusively with pointers -- memory is managed for you. +- No artificial limitations on element types or maximum count. +- Memory can be allocated once up-front, or dynamically as needed. +- Fully portable (no assembly; all is done through standard C++11 primitives). +- Supports super-fast bulk operations. +- Includes a low-overhead blocking version (BlockingConcurrentQueue). +- Exception safe. + +## Reasons to use + +There are not that many full-fledged lock-free queues for C++. Boost has one, but it's limited to objects with trivial +assignment operators and trivial destructors, for example. +Intel's TBB queue isn't lock-free, and requires trivial constructors too. +There're many academic papers that implement lock-free queues in C++, but usable source code is +hard to find, and tests even more so. + +This queue not only has less limitations than others (for the most part), but [it's also faster][benchmarks]. +It's been fairly well-tested, and offers advanced features like **bulk enqueueing/dequeueing** +(which, with my new design, is much faster than one element at a time, approaching and even surpassing +the speed of a non-concurrent queue even under heavy contention). + +In short, there was a lock-free queue shaped hole in the C++ open-source universe, and I set out +to fill it with the fastest, most complete, and well-tested design and implementation I could. +The result is `moodycamel::ConcurrentQueue` :-) + +## Reasons *not* to use + +The fastest synchronization of all is the kind that never takes place. Fundamentally, +concurrent data structures require some synchronization, and that takes time. Every effort +was made, of course, to minimize the overhead, but if you can avoid sharing data between +threads, do so! + +Why use concurrent data structures at all, then? Because they're gosh darn convenient! (And, indeed, +sometimes sharing data concurrently is unavoidable.) + +My queue is **not linearizable** (see the next section on high-level design). The foundations of +its design assume that producers are independent; if this is not the case, and your producers +co-ordinate amongst themselves in some fashion, be aware that the elements won't necessarily +come out of the queue in the same order they were put in *relative to the ordering formed by that co-ordination* +(but they will still come out in the order they were put in by any *individual* producer). If this affects +your use case, you may be better off with another implementation; either way, it's an important limitation +to be aware of. + +My queue is also **not NUMA aware**, and does a lot of memory re-use internally, meaning it probably doesn't +scale particularly well on NUMA architectures; however, I don't know of any other lock-free queue that *is* +NUMA aware (except for [SALSA][salsa], which is very cool, but has no publicly available implementation that I know of). + +Finally, the queue is **not sequentially consistent**; there *is* a happens-before relationship between when an element is put +in the queue and when it comes out, but other things (such as pumping the queue until it's empty) require more thought +to get right in all eventualities, because explicit memory ordering may have to be done to get the desired effect. In other words, +it can sometimes be difficult to use the queue correctly. This is why it's a good idea to follow the [samples][samples.md] where possible. +On the other hand, the upside of this lack of sequential consistency is better performance. + +## High-level design + +Elements are stored internally using contiguous blocks instead of linked lists for better performance. +The queue is made up of a collection of sub-queues, one for each producer. When a consumer +wants to dequeue an element, it checks all the sub-queues until it finds one that's not empty. +All of this is largely transparent to the user of the queue, however -- it mostly just worksTM. + +One particular consequence of this design, however, (which seems to be non-intuitive) is that if two producers +enqueue at the same time, there is no defined ordering between the elements when they're later dequeued. +Normally this is fine, because even with a fully linearizable queue there'd be a race between the producer +threads and so you couldn't rely on the ordering anyway. However, if for some reason you do extra explicit synchronization +between the two producer threads yourself, thus defining a total order between enqueue operations, you might expect +that the elements would come out in the same total order, which is a guarantee my queue does not offer. At that +point, though, there semantically aren't really two separate producers, but rather one that happens to be spread +across multiple threads. In this case, you can still establish a total ordering with my queue by creating +a single producer token, and using that from both threads to enqueue (taking care to synchronize access to the token, +of course, but there was already extra synchronization involved anyway). + +I've written a more detailed [overview of the internal design][blog], as well as [the full +nitty-gritty details of the design][design], on my blog. Finally, the +[source][source] itself is available for perusal for those interested in its implementation. + +## Basic use + +The entire queue's implementation is contained in **one header**, [`concurrentqueue.h`][concurrentqueue.h]. +Simply download and include that to use the queue. The blocking version is in a separate header, +[`blockingconcurrentqueue.h`][blockingconcurrentqueue.h], that depends on [`concurrentqueue.h`][concurrentqueue.h] and +[`lightweightsemaphore.h`][lightweightsemaphore.h]. The implementation makes use of certain key C++11 features, +so it requires a relatively recent compiler (e.g. VS2012+ or g++ 4.8; note that g++ 4.6 has a known bug with `std::atomic` +and is thus not supported). The algorithm implementations themselves are platform independent. + +Use it like you would any other templated queue, with the exception that you can use +it from many threads at once :-) + +Simple example: + +```C++ +#include "concurrentqueue.h" + +moodycamel::ConcurrentQueue q; +q.enqueue(25); + +int item; +bool found = q.try_dequeue(item); +assert(found && item == 25); +``` + +Description of basic methods: +- `ConcurrentQueue(size_t initialSizeEstimate)` + Constructor which optionally accepts an estimate of the number of elements the queue will hold +- `enqueue(T&& item)` + Enqueues one item, allocating extra space if necessary +- `try_enqueue(T&& item)` + Enqueues one item, but only if enough memory is already allocated +- `try_dequeue(T& item)` + Dequeues one item, returning true if an item was found or false if the queue appeared empty + +Note that it is up to the user to ensure that the queue object is completely constructed before +being used by any other threads (this includes making the memory effects of construction +visible, possibly via a memory barrier). Similarly, it's important that all threads have +finished using the queue (and the memory effects have fully propagated) before it is +destructed. + +There's usually two versions of each method, one "explicit" version that takes a user-allocated per-producer or +per-consumer token, and one "implicit" version that works without tokens. Using the explicit methods is almost +always faster (though not necessarily by a huge factor). Apart from performance, the primary distinction between them +is their sub-queue allocation behaviour for enqueue operations: Using the implicit enqueue methods causes an +automatically-allocated thread-local producer sub-queue to be allocated. +Explicit producers, on the other hand, are tied directly to their tokens' lifetimes (but are recycled internally). + +In order to avoid the number of sub-queues growing without bound, implicit producers are marked for reuse once +their thread exits. However, this is not supported on all platforms. If using the queue from short-lived threads, +it is recommended to use explicit producer tokens instead. + +Full API (pseudocode): + + # Allocates more memory if necessary + enqueue(item) : bool + enqueue(prod_token, item) : bool + enqueue_bulk(item_first, count) : bool + enqueue_bulk(prod_token, item_first, count) : bool + + # Fails if not enough memory to enqueue + try_enqueue(item) : bool + try_enqueue(prod_token, item) : bool + try_enqueue_bulk(item_first, count) : bool + try_enqueue_bulk(prod_token, item_first, count) : bool + + # Attempts to dequeue from the queue (never allocates) + try_dequeue(item&) : bool + try_dequeue(cons_token, item&) : bool + try_dequeue_bulk(item_first, max) : size_t + try_dequeue_bulk(cons_token, item_first, max) : size_t + + # If you happen to know which producer you want to dequeue from + try_dequeue_from_producer(prod_token, item&) : bool + try_dequeue_bulk_from_producer(prod_token, item_first, max) : size_t + + # A not-necessarily-accurate count of the total number of elements + size_approx() : size_t + +## Blocking version + +As mentioned above, a full blocking wrapper of the queue is provided that adds +`wait_dequeue` and `wait_dequeue_bulk` methods in addition to the regular interface. +This wrapper is extremely low-overhead, but slightly less fast than the non-blocking +queue (due to the necessary bookkeeping involving a lightweight semaphore). + +There are also timed versions that allow a timeout to be specified (either in microseconds +or with a `std::chrono` object). + +The only major caveat with the blocking version is that you must be careful not to +destroy the queue while somebody is waiting on it. This generally means you need to +know for certain that another element is going to come along before you call one of +the blocking methods. (To be fair, the non-blocking version cannot be destroyed while +in use either, but it can be easier to coordinate the cleanup.) + +Blocking example: + +```C++ +#include "blockingconcurrentqueue.h" + +moodycamel::BlockingConcurrentQueue q; +std::thread producer([&]() { + for (int i = 0; i != 100; ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(i % 10)); + q.enqueue(i); + } +}); +std::thread consumer([&]() { + for (int i = 0; i != 100; ++i) { + int item; + q.wait_dequeue(item); + assert(item == i); + + if (q.wait_dequeue_timed(item, std::chrono::milliseconds(5))) { + ++i; + assert(item == i); + } + } +}); +producer.join(); +consumer.join(); + +assert(q.size_approx() == 0); +``` + +## Advanced features + +#### Tokens + +The queue can take advantage of extra per-producer and per-consumer storage if +it's available to speed up its operations. This takes the form of "tokens": +You can create a consumer token and/or a producer token for each thread or task +(tokens themselves are not thread-safe), and use the methods that accept a token +as their first parameter: + +```C++ +moodycamel::ConcurrentQueue q; + +moodycamel::ProducerToken ptok(q); +q.enqueue(ptok, 17); + +moodycamel::ConsumerToken ctok(q); +int item; +q.try_dequeue(ctok, item); +assert(item == 17); +``` + +If you happen to know which producer you want to consume from (e.g. in +a single-producer, multi-consumer scenario), you can use the `try_dequeue_from_producer` +methods, which accept a producer token instead of a consumer token, and cut some overhead. + +Note that tokens work with the blocking version of the queue too. + +When producing or consuming many elements, the most efficient way is to: + +1. Use the bulk methods of the queue with tokens +2. Failing that, use the bulk methods without tokens +3. Failing that, use the single-item methods with tokens +4. Failing that, use the single-item methods without tokens + +Having said that, don't create tokens willy-nilly -- ideally there would be +one token (of each kind) per thread. The queue will work with what it is +given, but it performs best when used with tokens. + +Note that tokens aren't actually tied to any given thread; it's not technically +required that they be local to the thread, only that they be used by a single +producer/consumer at a time. + +#### Bulk operations + +Thanks to the [novel design][blog] of the queue, it's just as easy to enqueue/dequeue multiple +items as it is to do one at a time. This means that overhead can be cut drastically for +bulk operations. Example syntax: + +```C++ +moodycamel::ConcurrentQueue q; + +int items[] = { 1, 2, 3, 4, 5 }; +q.enqueue_bulk(items, 5); + +int results[5]; // Could also be any iterator +size_t count = q.try_dequeue_bulk(results, 5); +for (size_t i = 0; i != count; ++i) { + assert(results[i] == items[i]); +} +``` + +#### Preallocation (correctly using `try_enqueue`) + +`try_enqueue`, unlike just plain `enqueue`, will never allocate memory. If there's not enough room in the +queue, it simply returns false. The key to using this method properly, then, is to ensure enough space is +pre-allocated for your desired maximum element count. + +The constructor accepts a count of the number of elements that it should reserve space for. Because the +queue works with blocks of elements, however, and not individual elements themselves, the value to pass +in order to obtain an effective number of pre-allocated element slots is non-obvious. + +First, be aware that the count passed is rounded up to the next multiple of the block size. Note that the +default block size is 32 (this can be changed via the traits). Second, once a slot in a block has been +enqueued to, that slot cannot be re-used until the rest of the block has been completely filled +up and then completely emptied. This affects the number of blocks you need in order to account for the +overhead of partially-filled blocks. Third, each producer (whether implicit or explicit) claims and recycles +blocks in a different manner, which again affects the number of blocks you need to account for a desired number of +usable slots. + +Suppose you want the queue to be able to hold at least `N` elements at any given time. Without delving too +deep into the rather arcane implementation details, here are some simple formulas for the number of elements +to request for pre-allocation in such a case. Note the division is intended to be arithmetic division and not +integer division (in order for `ceil()` to work). + +For explicit producers (using tokens to enqueue): + +```C++ +(ceil(N / BLOCK_SIZE) + 1) * MAX_NUM_PRODUCERS * BLOCK_SIZE +``` + +For implicit producers (no tokens): + +```C++ +(ceil(N / BLOCK_SIZE) - 1 + 2 * MAX_NUM_PRODUCERS) * BLOCK_SIZE +``` + +When using mixed producer types: + +```C++ +((ceil(N / BLOCK_SIZE) - 1) * (MAX_EXPLICIT_PRODUCERS + 1) + 2 * (MAX_IMPLICIT_PRODUCERS + MAX_EXPLICIT_PRODUCERS)) * BLOCK_SIZE +``` + +If these formulas seem rather inconvenient, you can use the constructor overload that accepts the minimum +number of elements (`N`) and the maximum number of explicit and implicit producers directly, and let it do the +computation for you. + +In addition to blocks, there are other internal data structures that require allocating memory if they need to resize (grow). +If using `try_enqueue` exclusively, the initial sizes may be exceeded, causing subsequent `try_enqueue` operations to fail. +Specifically, the `INITIAL_IMPLICIT_PRODUCER_HASH_SIZE` trait limits the number of implicit producers that can be active at once +before the internal hash needs resizing. Along the same lines, the `IMPLICIT_INITIAL_INDEX_SIZE` trait limits the number of +unconsumed elements that an implicit producer can insert before its internal hash needs resizing. Similarly, the +`EXPLICIT_INITIAL_INDEX_SIZE` trait limits the number of unconsumed elements that an explicit producer can insert before its +internal hash needs resizing. In order to avoid hitting these limits when using `try_enqueue`, it is crucial to adjust the +initial sizes in the traits appropriately, in addition to sizing the number of blocks properly as outlined above. + +Finally, it's important to note that because the queue is only eventually consistent and takes advantage of +weak memory ordering for speed, there's always a possibility that under contention `try_enqueue` will fail +even if the queue is correctly pre-sized for the desired number of elements. (e.g. A given thread may think that +the queue's full even when that's no longer the case.) So no matter what, you still need to handle the failure +case (perhaps looping until it succeeds), unless you don't mind dropping elements. + +#### Exception safety + +The queue is exception safe, and will never become corrupted if used with a type that may throw exceptions. +The queue itself never throws any exceptions (operations fail gracefully (return false) if memory allocation +fails instead of throwing `std::bad_alloc`). + +It is important to note that the guarantees of exception safety only hold if the element type never throws +from its destructor, and that any iterators passed into the queue (for bulk operations) never throw either. +Note that in particular this means `std::back_inserter` iterators must be used with care, since the vector +being inserted into may need to allocate and throw a `std::bad_alloc` exception from inside the iterator; +so be sure to reserve enough capacity in the target container first if you do this. + +The guarantees are presently as follows: +- Enqueue operations are rolled back completely if an exception is thrown from an element's constructor. + For bulk enqueue operations, this means that elements are copied instead of moved (in order to avoid + having only some objects moved in the event of an exception). Non-bulk enqueues always use + the move constructor if one is available. +- If the assignment operator throws during a dequeue operation (both single and bulk), the element(s) are + considered dequeued regardless. In such a case, the dequeued elements are all properly destructed before + the exception is propagated, but there's no way to get the elements themselves back. +- Any exception that is thrown is propagated up the call stack, at which point the queue is in a consistent + state. + +Note: If any of your type's copy constructors/move constructors/assignment operators don't throw, be sure +to annotate them with `noexcept`; this will avoid the exception-checking overhead in the queue where possible +(even with zero-cost exceptions, there's still a code size impact that has to be taken into account). + +#### Traits + +The queue also supports a traits template argument which defines various types, constants, +and the memory allocation and deallocation functions that are to be used by the queue. The typical pattern +to providing your own traits is to create a class that inherits from the default traits +and override only the values you wish to change. Example: + +```C++ +struct MyTraits : public moodycamel::ConcurrentQueueDefaultTraits +{ + static const size_t BLOCK_SIZE = 256; // Use bigger blocks +}; + +moodycamel::ConcurrentQueue q; +``` + +#### How to dequeue types without calling the constructor + +The normal way to dequeue an item is to pass in an existing object by reference, which +is then assigned to internally by the queue (using the move-assignment operator if possible). +This can pose a problem for types that are +expensive to construct or don't have a default constructor; fortunately, there is a simple +workaround: Create a wrapper class that copies the memory contents of the object when it +is assigned by the queue (a poor man's move, essentially). Note that this only works if +the object contains no internal pointers. Example: + +```C++ +struct MyObjectMover { + inline void operator=(MyObject&& obj) { + std::memcpy(data, &obj, sizeof(MyObject)); + + // TODO: Cleanup obj so that when it's destructed by the queue + // it doesn't corrupt the data of the object we just moved it into + } + + inline MyObject& obj() { return *reinterpret_cast(data); } + +private: + align(alignof(MyObject)) char data[sizeof(MyObject)]; +}; +``` + +A less dodgy alternative, if moves are cheap but default construction is not, is to use a +wrapper that defers construction until the object is assigned, enabling use of the move +constructor: + +```C++ +struct MyObjectMover { + inline void operator=(MyObject&& x) { + new (data) MyObject(std::move(x)); + created = true; + } + + inline MyObject& obj() { + assert(created); + return *reinterpret_cast(data); + } + + ~MyObjectMover() { + if (created) + obj().~MyObject(); + } + +private: + align(alignof(MyObject)) char data[sizeof(MyObject)]; + bool created = false; +}; +``` + +## Samples + +There are some more detailed samples [here][samples.md]. The source of +the [unit tests][unittest-src] and [benchmarks][benchmark-src] are available for reference as well. + +## Benchmarks + +See my blog post for some [benchmark results][benchmarks] (including versus `boost::lockfree::queue` and `tbb::concurrent_queue`), +or run the benchmarks yourself (requires MinGW and certain GnuWin32 utilities to build on Windows, or a recent +g++ on Linux): + +```Shell +cd build +make benchmarks +bin/benchmarks +``` + +The short version of the benchmarks is that it's so fast (especially the bulk methods), that if you're actually +using the queue to *do* anything, the queue won't be your bottleneck. + +## Tests (and bugs) + +I've written quite a few unit tests as well as a randomized long-running fuzz tester. I also ran the +core queue algorithm through the [CDSChecker][cdschecker] C++11 memory model model checker. Some of the +inner algorithms were tested separately using the [Relacy][relacy] model checker, and full integration +tests were also performed with Relacy. +I've tested +on Linux (Fedora 19) and Windows (7), but only on x86 processors so far (Intel and AMD). The code was +written to be platform-independent, however, and should work across all processors and OSes. + +Due to the complexity of the implementation and the difficult-to-test nature of lock-free code in general, +there may still be bugs. If anyone is seeing buggy behaviour, I'd like to hear about it! (Especially if +a unit test for it can be cooked up.) Just open an issue on GitHub. + +## Using vcpkg +You can download and install `moodycamel::ConcurrentQueue` using the [vcpkg](https://github.com/Microsoft/vcpkg) dependency manager: + +```Shell +git clone https://github.com/Microsoft/vcpkg.git +cd vcpkg +./bootstrap-vcpkg.sh +./vcpkg integrate install +vcpkg install concurrentqueue +``` + +The `moodycamel::ConcurrentQueue` port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository. + +## License + +I'm releasing the source of this repository (with the exception of third-party code, i.e. the Boost queue +(used in the benchmarks for comparison), Intel's TBB library (ditto), CDSChecker, Relacy, and Jeff Preshing's +cross-platform semaphore, which all have their own licenses) +under a simplified BSD license. I'm also dual-licensing under the Boost Software License. +See the [LICENSE.md][license] file for more details. + +Note that lock-free programming is a patent minefield, and this code may very +well violate a pending patent (I haven't looked), though it does not to my present knowledge. +I did design and implement this queue from scratch. + +## Diving into the code + +If you're interested in the source code itself, it helps to have a rough idea of how it's laid out. This +section attempts to describe that. + +The queue is formed of several basic parts (listed here in roughly the order they appear in the source). There's the +helper functions (e.g. for rounding to a power of 2). There's the default traits of the queue, which contain the +constants and malloc/free functions used by the queue. There's the producer and consumer tokens. Then there's the queue's +public API itself, starting with the constructor, destructor, and swap/assignment methods. There's the public enqueue methods, +which are all wrappers around a small set of private enqueue methods found later on. There's the dequeue methods, which are +defined inline and are relatively straightforward. + +Then there's all the main internal data structures. First, there's a lock-free free list, used for recycling spent blocks (elements +are enqueued to blocks internally). Then there's the block structure itself, which has two different ways of tracking whether +it's fully emptied or not (remember, given two parallel consumers, there's no way to know which one will finish first) depending on where it's used. +Then there's a small base class for the two types of internal SPMC producer queues (one for explicit producers that holds onto memory +but attempts to be faster, and one for implicit ones which attempt to recycle more memory back into the parent but is a little slower). +The explicit producer is defined first, then the implicit one. They both contain the same general four methods: One to enqueue, one to +dequeue, one to enqueue in bulk, and one to dequeue in bulk. (Obviously they have constructors and destructors too, and helper methods.) +The main difference between them is how the block handling is done (they both use the same blocks, but in different ways, and map indices +to them in different ways). + +Finally, there's the miscellaneous internal methods: There's the ones that handle the initial block pool (populated when the queue is constructed), +and an abstract block pool that comprises the initial pool and any blocks on the free list. There's ones that handle the producer list +(a lock-free add-only linked list of all the producers in the system). There's ones that handle the implicit producer lookup table (which +is really a sort of specialized TLS lookup). And then there's some helper methods for allocating and freeing objects, and the data members +of the queue itself, followed lastly by the free-standing swap functions. + + +[blog]: http://moodycamel.com/blog/2014/a-fast-general-purpose-lock-free-queue-for-c++ +[design]: http://moodycamel.com/blog/2014/detailed-design-of-a-lock-free-queue +[samples.md]: https://github.com/cameron314/concurrentqueue/blob/master/samples.md +[source]: https://github.com/cameron314/concurrentqueue +[concurrentqueue.h]: https://github.com/cameron314/concurrentqueue/blob/master/concurrentqueue.h +[blockingconcurrentqueue.h]: https://github.com/cameron314/concurrentqueue/blob/master/blockingconcurrentqueue.h +[lightweightsemaphore.h]: https://github.com/cameron314/concurrentqueue/blob/master/lightweightsemaphore.h +[unittest-src]: https://github.com/cameron314/concurrentqueue/tree/master/tests/unittests +[benchmarks]: http://moodycamel.com/blog/2014/a-fast-general-purpose-lock-free-queue-for-c++#benchmarks +[benchmark-src]: https://github.com/cameron314/concurrentqueue/tree/master/benchmarks +[license]: https://github.com/cameron314/concurrentqueue/blob/master/LICENSE.md +[cdschecker]: http://demsky.eecs.uci.edu/c11modelchecker.html +[relacy]: http://www.1024cores.net/home/relacy-race-detector +[spsc]: https://github.com/cameron314/readerwriterqueue +[salsa]: https://iditkeidar.com/wp-content/uploads/files/ftp/spaa049-gidron.pdf diff --git a/upstream/documents.json b/upstream/documents.json new file mode 100644 index 0000000..e1ecdf7 --- /dev/null +++ b/upstream/documents.json @@ -0,0 +1,17 @@ +{ + "upstream": "https://github.com/cameron314/concurrentqueue/archive/refs/tags/v1.0.5.tar.gz", + "upstream_sha256": "4d6368a27492d86011fde5ca0cf386dce7c49cd425aa3d9b063ca6ec373a6ef3", + "pruning": "Nothing of concurrentqueue is vendored: the headers (and, with the c-api feature, the C API sources) are compat.concurrentqueue 1.0.5's, from the same tarball. Its README and license are retained here. The conformance run builds the release's tests/unittests against the module, from the tarball downloaded at test time.", + "documents": [ + { + "path": "upstream/concurrentqueue/README.md", + "source": "https://github.com/cameron314/concurrentqueue/blob/v1.0.5/README.md", + "sha256": "6dd0a6732be4ee5a9a6e2c7da5e4dc135f301e59c7d76bb4cf5d4a3ce269991f" + }, + { + "path": "upstream/concurrentqueue/LICENSE.md", + "source": "https://github.com/cameron314/concurrentqueue/blob/v1.0.5/LICENSE.md", + "sha256": "22331571e5e3dbe4c79ad715e23127bdf3beabd75052c953ddabf191ef799e14" + } + ] +}