diff --git a/.github/workflows/ubuntu_latest.yml b/.github/workflows/ubuntu_latest.yml index 5e6bc75..0a4b676 100644 --- a/.github/workflows/ubuntu_latest.yml +++ b/.github/workflows/ubuntu_latest.yml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v6 - - run: sudo apt-get install iwyu + - run: sudo apt-get update && sudo apt-get install -y iwyu - run: | $CURL_GET -o mkn ${PATH_GET}/mkn_nix diff --git a/README.md b/README.md index 4b8582c..0b40148 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,14 @@ # cpp.iwyu -** Compile stage plugin to to run include-what-you-use on all applicable files ** +**Compile phase module running [include-what-you-use](https://include-what-you-use.org) on project sources** + +Each file is checked with the same flags its real compilation uses. iwyu's suggestions are printed, +they do not fail the build. ## Prerequisites - [maiken](https://github.com/mkn/mkn) + +- [maiken](https://github.com/mkn/mkn) +- `iwyu` or `include-what-you-use` on `PATH` ## Usage @@ -11,24 +16,50 @@ mod: - name: cpp.iwyu compile: - headers: -x c++-header # h files need this for scanning if they are C++ - args: -std=c++17 # additional arguments - inc: inc/1 inc/2 # additional include directories - paths: search/dir # scan directories - ignore: src/python3 # if file found has path that contains string ignore - types: cpp hpp cc # file types checked default: {"cpp", "cxx", "cc", "cc", "h", "hpp"} - + args: -DSOME_DEFINE # additional arguments + headers: -x c++-header # additional arguments for header files (h hh hpp hxx) + inc: inc/1 inc/2 # additional include directories + paths: search/dir # also check files found in these directories + ignore: src/python3 tpp # skip files whose path contains any of these + types: cpp hpp cc # file types checked, default: cpp cxx cc h hpp + fix: --nocomments # apply the suggestions with fix_include, using these arguments + system_last: true # with fix, put system includes after project includes ``` -## Building +Every key is optional, the project's own source files are always checked. Lists are space +separated, `types` may also be `:` separated. + +## Applying suggestions + +With `fix`, iwyu's output is written to `/iwyu.out` and passed to iwyu's `fix_include` +(`fix_includes.py` upstream), which edits the files. `fix: ""` uses `fix_include`'s defaults. Useful arguments: - *nix gcc: +| argument | effect | +|---|---| +| `-n` | print the diffs, do not edit | +| `--nocomments` | no `// for X` comments after includes | +| `--nosafe_headers` | also remove includes from headers, by default headers only get additions | +| `--noreorder` | do not reorder existing includes, for order sensitive includes | +| `--ignore_re REGEX` | do not edit matching files | - mkn clean build -dtOa "-fPIC" -l "-pthread -ldl" +`fix_include` puts system includes first. With `system_last: true` the module then moves them after +the project includes in every file it edited, so a system include cannot hide a project header +missing its own includes. Only contiguous runs of `#include` lines are reordered, blank line separated groups keep +their order, includes in `#if` sections stay where they are. +Edits happen during the compile phase, so the edited files are what get compiled. Running `fix` +from its own profile keeps it out of normal builds. + +## Building + +```bash +mkn clean build -dtOa "-std=c++20 -fPIC" +``` ## Testing - *nix gcc: +Runs iwyu over this module's own source: - mkn clean build -dtOa "-fPIC" -l "-pthread -ldl" -p test run +```bash +mkn clean build -dtOa "-std=c++20 -fPIC" -p iwyu +``` diff --git a/mkn.yaml b/mkn.yaml index a734786..82dc510 100644 --- a/mkn.yaml +++ b/mkn.yaml @@ -1,9 +1,7 @@ +#! clean build -dtOa "-std=c++20 -fPIC" -p iwyu name: cpp.iwyu version: master -property: - maiken_location: ${MKN_HOME}/app/mkn/${version} - maiken_scm: https://github.com/mkn/mkn parent: base src: mod.cpp @@ -11,16 +9,19 @@ mode: shared profile: - name: base - dep: mkn.mod mkn#${version}&${maiken_location}(${maiken_scm})[headers] + dep: mkn.mod if_arg: win_shared: -DYAML_CPP_DLL shared: -DMKN_KUL_SHARED - name: format mod: | - clang.format{init{style: google, paths: .}} + clang.format{init{style: file, paths: .}} +# runs iwyu over this module's own source and applies its suggestions - name: iwyu parent: base + src: mod.cpp + # inline form, the list form resolves the module from the default branch in CI mod: | - cpp.iwyu{init{args: -std=c++17, paths: .}} + cpp.iwyu{compile{fix: --nocomments --nosafe_headers, system_last: true}} diff --git a/mod.cpp b/mod.cpp index 9c33eec..f8fe968 100644 --- a/mod.cpp +++ b/mod.cpp @@ -30,23 +30,29 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "mkn/mod/init.hpp" // IWYU pragma: keep +#include "mkn/mod/def.hpp" -#include // for uint16_t -#include // for string, allocator, operator+, basic_... -#include // for unordered_set -#include // for pair -#include // for vector -#include "mkn/kul/cli.hpp" // for asArgs -#include "mkn/kul/defs.hpp" // for MKN_KUL_PUBLISH -#include "mkn/kul/except.hpp" // for Exception, KEXCEPT, KTHROW -#include "mkn/kul/log.hpp" // for KLOG, KLOG_INF, KLOG_DBG -#include "mkn/kul/map.hpp" // for Map -#include "mkn/kul/os.hpp" // for Dir, File, WHICH, Exception, PushDir -#include "mkn/kul/proc.hpp" // for Process, AProcess, ExitException -#include "mkn/kul/string.hpp" // for String -#include "mkn/kul/yaml.hpp" // for NodeValidator, Validator, yaml -#include "yaml-cpp/node/impl.h" // for Node::operator[], Node::Scalar -#include "yaml-cpp/node/node.h" // for Node +#include "mkn/kul/cli.hpp" +#include "mkn/kul/defs.hpp" +#include "mkn/kul/except.hpp" +#include "mkn/kul/io.hpp" +#include "mkn/kul/lang/cpp/compilers.hpp" +#include "mkn/kul/lang/def.hpp" +#include "mkn/kul/log.hpp" +#include "mkn/kul/os.hpp" +#include "mkn/kul/proc.hpp" +#include "mkn/kul/string.hpp" +#include "mkn/kul/yaml.hpp" +#include "yaml-cpp/yaml.h" + +#include +#include +#include +#include +#include +#include +#include +#include namespace mkn::mod::cpp::iwyu { @@ -62,70 +68,232 @@ class Module : public mkn::mod::Module { KEXCEPT(Exception, "Failed to find valid iwyu binary, check PATH"); } + static std::string find_fix_include() { + std::vector fix{"fix_include", "fix_includes.py", "iwyu-fix-includes"}; + for (auto const& str : fix) + if (kul::env::WHICH(str)) return str; + KEXCEPT(Exception, "Failed to find valid fix_include binary, check PATH"); + } + + // fix_include reads iwyu output from stdin only, so a shell does the redirect + // kul::Process re-splits its arguments, the quotes keep the shell command as one + // only the checked files are passed, iwyu also reports on headers of other projects + // --basedir so fix_include resolves iwyu's relative paths like the passed ones + static void fix_includes(std::string const& args, kul::File const& iwyu_out, + std::vector const& checked, std::string const& basedir) { +#if MKN_KUL_IS_WIN + auto const quote = [](std::string const& s) { return "\"" + s + "\""; }; +#else + auto const quote = [](std::string const& s) { return "'" + s + "'"; }; +#endif + std::string cmd = find_fix_include() + " " + args + " --basedir " + quote(basedir); + for (auto const& file : checked) cmd += " " + quote(file); + cmd += " < " + quote(iwyu_out.full()); +#if MKN_KUL_IS_WIN + kul::Process p("cmd"); + p << "/c" << cmd; +#else + kul::Process p("sh"); + p << "-c" << "\"" + cmd + "\""; +#endif + KLOG(DBG) << p; + try { + p.start(); + } catch (kul::proc::ExitException const&) { + // non-zero with -n when changes are needed + } + } + static void VALIDATE_NODE(YAML::Node const& node) { using namespace kul::yaml; Validator({NodeValidator("inc"), NodeValidator("args"), NodeValidator("ignore"), - NodeValidator("headers"), NodeValidator("paths"), NodeValidator("types")}) + NodeValidator("headers"), NodeValidator("paths"), NodeValidator("types"), + NodeValidator("fix"), NodeValidator("system_last")}) .validate(node); } - void CHECK(std::string const& proc, std::string const& compileCmd, kul::File&& f, - YAML::Node const& node) { + // flags of the real compile command, without compiler prefix (e.g. ccache), + // binary, output and input + static std::vector compile_args(std::string const& compileCmd) { + auto const id = mkn::kul::lang::cpp::Compilers::INSTANCE().identify(compileCmd); + std::vector args; + for (auto const& arg : id.trailing) { + if (arg == "-o") break; + args.push_back(arg); + } + return args; + } + + static std::string read(std::string const& file) { + std::ifstream in(file, std::ios::binary); + std::stringstream ss; + ss << in.rdbuf(); + return ss.str(); + } + + // fix_include puts system includes first, which can hide project headers missing their own + // includes. In each run of #include lines (blank lines allowed in between), move the system + // includes after the others, keeping blank line separated groups and their order. + static std::string system_includes_last(std::string const& text) { + std::string const eol = text.find("\r\n") != std::string::npos ? "\r\n" : "\n"; + std::vector lines; + { + std::stringstream ss(text); + for (std::string line; std::getline(ss, line);) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + lines.push_back(line); + } + } + auto const include_kind = [](std::string const& line) -> char { // '<', '"' or 0 + auto i = line.find_first_not_of(" \t"); + if (i == std::string::npos || line[i] != '#') return 0; + i = line.find_first_not_of(" \t", i + 1); + if (i == std::string::npos || line.compare(i, 7, "include") != 0) return 0; + i = line.find_first_not_of(" \t", i + 7); + return i != std::string::npos && (line[i] == '<' || line[i] == '"') ? line[i] : 0; + }; + auto const blank = [](std::string const& line) { + return line.find_first_not_of(" \t") == std::string::npos; + }; + + std::vector res; + for (std::size_t i = 0; i < lines.size();) { + if (!include_kind(lines[i])) { + res.push_back(lines[i++]); + continue; + } + std::size_t end = i; // one past the last include of this run + for (std::size_t j = i; j < lines.size() && (include_kind(lines[j]) || blank(lines[j])); ++j) + if (include_kind(lines[j])) end = j + 1; + + std::vector> project{{}}, system{{}}; + for (std::size_t j = i; j < end; ++j) { + if (blank(lines[j])) { + if (!project.back().empty()) project.emplace_back(); + if (!system.back().empty()) system.emplace_back(); + continue; + } + (include_kind(lines[j]) == '<' ? system : project).back().push_back(lines[j]); + } + if (project.back().empty()) project.pop_back(); + if (system.back().empty()) system.pop_back(); + if (project.empty() || system.empty()) { // nothing to move + for (std::size_t j = i; j < end; ++j) res.push_back(lines[j]); + } else { + bool first = true; + for (auto const* groups : {&project, &system}) + for (auto const& group : *groups) { + if (!first) res.emplace_back(); + first = false; + res.insert(res.end(), group.begin(), group.end()); + } + } + i = end; + } + + std::string out; + for (auto const& line : res) out += line + eol; + if (!text.empty() && text.back() != '\n' && !out.empty()) out.resize(out.size() - eol.size()); + return out; + } + + // returns iwyu's output, empty if ignored + std::string CHECK(std::string const& proc, std::string const& compileCmd, kul::File&& f, + YAML::Node const& node) { if (node["ignore"]) - if (f.escm().find(node["ignore"].Scalar()) != std::string::npos) return; + for (auto const& ignore : mkn::kul::cli::asArgs(node["ignore"].Scalar())) + if (f.escm().find(ignore) != std::string::npos) return ""; kul::Process p(proc); - auto const firstSpace = compileCmd.find(' '); - auto compileStr = - firstSpace == std::string::npos ? std::string() : compileCmd.substr(firstSpace + 1); - p << compileStr.substr(0, compileStr.rfind(" -o")); - if (node["args"]) p << node["args"].Scalar(); + for (auto const& arg : compile_args(compileCmd)) p << arg; + if (node["args"]) + for (auto const& arg : mkn::kul::cli::asArgs(node["args"].Scalar())) p << arg; if (node["inc"]) - for (const auto& inc : mkn::kul::cli::asArgs(node["inc"].Scalar())) + for (auto const& inc : mkn::kul::cli::asArgs(node["inc"].Scalar())) p << std::string{"-I"} + inc; - if (node["headers"]) p << node["headers"].Scalar(); + if (node["headers"] && is_header(f.name())) + for (auto const& arg : mkn::kul::cli::asArgs(node["headers"].Scalar())) p << arg; p << f.escm(); KLOG(DBG) << p; + mkn::kul::ProcessCapture pc(p); try { p.start(); - } catch (kul::proc::ExitException const& e) { + } catch (kul::proc::ExitException const&) { + // iwyu exits non-zero whenever it has suggestions } + auto const out = pc.outs() + pc.errs(); + KOUT(NON) << out; + return out; + } + + static std::string extension(std::string const& name) { + auto const dot = name.rfind("."); + return dot == std::string::npos ? "" : name.substr(dot + 1); + } + + static bool is_header(std::string const& name) { + static std::unordered_set const headers{"h", "hh", "hpp", "hxx"}; + return headers.count(extension(name)); } + void run(mkn::mod::Context& ctx, YAML::Node const& node) KTHROW(std::exception) { VALIDATE_NODE(node); auto const state = ctx.state(); kul::os::PushDir pushd(state.projectDir); - std::unordered_set types; - if (!node["types"]) { - types = {"cpp", "cxx", "cc", "cc", "h", "hpp"}; - } else - for (const auto& s : kul::String::SPLIT(node["types"].Scalar(), ":")) types.insert(s); + std::unordered_set types{"cpp", "cxx", "cc", "h", "hpp"}; + if (node["types"]) { + types.clear(); + for (auto const& arg : mkn::kul::cli::asArgs(node["types"].Scalar())) + for (auto const& type : kul::String::SPLIT(arg, ":")) types.insert(type); + } std::unordered_set files; - - for (const auto& f : state.sourceFiles) { - auto const dot = f.rfind("."); - if (dot == std::string::npos) continue; - if (types.count(f.substr(dot + 1))) files.insert(f); - } + for (auto const& f : state.sourceFiles) + if (types.count(extension(f))) files.insert(f); if (node["paths"]) { - for (const auto& path : kul::cli::asArgs(node["paths"].Scalar())) { + for (auto const& path : kul::cli::asArgs(node["paths"].Scalar())) { kul::Dir d(path); if (!d) KEXCEPT(kul::fs::Exception, "Directory does not exist: ", d.path()); - for (const auto& file : d.files(1)) { - const std::string name = file.name(); - if (name.find(".") == std::string::npos) continue; - const std::string type = name.substr(name.rfind(".") + 1); - if (types.count(type)) files.insert(file.real()); - } + for (auto const& file : d.files(1)) + if (types.count(extension(file.name()))) files.insert(file.real()); } } auto proc = find_iwyu(); - for (const auto& file : files) - CHECK(proc, ctx.compileCommandFor(kul::File(file).escm() + ".cpp"), kul::File(file), node); + std::string out; + std::vector checked; + // ".cpp" so headers get the C++ compile command too + for (auto const& file : files) { + kul::File f(file); + auto const o = CHECK(proc, ctx.compileCommandFor(f.escm() + ".cpp"), kul::File(file), node); + if (o.empty()) continue; + out += o; + checked.push_back(f.escm()); + } + + if (node["fix"]) { + auto const buildDir = state.extra.count("buildDir") ? state.extra.at("buildDir") : "bin"; + kul::Dir dir(buildDir); + dir.mk(); + kul::File iwyu_out("iwyu.out", dir); + kul::io::Writer(iwyu_out) << out; + + bool const system_last = node["system_last"] && node["system_last"].Scalar() == "true"; + std::map before; + if (system_last) + for (auto const& file : checked) before[file] = read(file); + fix_includes(node["fix"].Scalar(), iwyu_out, checked, kul::Dir(state.projectDir).real()); + for (auto const& [file, text] : before) { + auto const fixed = read(file); + if (fixed == text) continue; + auto const reordered = system_includes_last(fixed); + if (reordered == fixed) continue; + std::ofstream(file, std::ios::binary) << reordered; + KLOG(INF) << "System includes moved last: " << file; + } + } } };