diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 67b351db9..b96c2c7ed 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -52,6 +52,10 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: ./tools/github-actions/setup-deps + - name: Install ripgrep + run: | + sudo apt-get update + sudo apt-get install --yes ripgrep - run: make tools - run: ./mvnw -B install build: diff --git a/argi-orchestration/argi-agent-framework/src/main/java/io/github/agentic/ai/agent/orchestration/extension/file/LocalFilesystemBackend.java b/argi-orchestration/argi-agent-framework/src/main/java/io/github/agentic/ai/agent/orchestration/extension/file/LocalFilesystemBackend.java index 155ddd403..1c1c993e2 100644 --- a/argi-orchestration/argi-agent-framework/src/main/java/io/github/agentic/ai/agent/orchestration/extension/file/LocalFilesystemBackend.java +++ b/argi-orchestration/argi-agent-framework/src/main/java/io/github/agentic/ai/agent/orchestration/extension/file/LocalFilesystemBackend.java @@ -521,18 +521,30 @@ public Object grepRaw(String pattern, String path, String glob) { // Helper methods private Map> ripgrepSearch(String pattern, Path baseFull, String includeGlob) { + boolean explicitFile = Files.isRegularFile(baseFull, LinkOption.NOFOLLOW_LINKS); List cmd = new ArrayList<>(); cmd.add("rg"); cmd.add("--json"); - if (includeGlob != null) { + cmd.add("--max-filesize"); + cmd.add(Long.toString(maxFileSizeBytes)); + if (explicitFile) { + // Explicit file arguments bypass rg's size filter. Select the file through directory traversal instead. + cmd.add("--max-depth"); + cmd.add("1"); + cmd.add("--glob"); + cmd.add(literalRipgrepGlob(baseFull.getFileName().toString())); + } else if (includeGlob != null) { cmd.add("--glob"); cmd.add(includeGlob); } cmd.add("--"); cmd.add(pattern); - cmd.add(baseFull.toString()); + cmd.add((explicitFile ? baseFull.getParent() : baseFull).toString()); try { + if (explicitFile && Files.size(baseFull) > maxFileSizeBytes) { + return Collections.emptyMap(); + } ProcessBuilder pb = new ProcessBuilder(cmd); Process process = pb.start(); @@ -590,6 +602,17 @@ private Map> ripgrepSearch(String pattern, Path baseFull } } + private String literalRipgrepGlob(String filename) { + StringBuilder glob = new StringBuilder(); + for (char character : filename.toCharArray()) { + if ("\\*?[]{}!".indexOf(character) >= 0) { + glob.append('\\'); + } + glob.append(character); + } + return glob.toString(); + } + private Map> javaSearch(String pattern, Path baseFull, String includeGlob) { Pattern regex; try { diff --git a/argi-orchestration/argi-agent-framework/src/test/java/io/github/agentic/ai/agent/orchestration/extension/file/LocalFilesystemBackendFileGrowthTest.java b/argi-orchestration/argi-agent-framework/src/test/java/io/github/agentic/ai/agent/orchestration/extension/file/LocalFilesystemBackendFileGrowthTest.java new file mode 100644 index 000000000..c48b9673c --- /dev/null +++ b/argi-orchestration/argi-agent-framework/src/test/java/io/github/agentic/ai/agent/orchestration/extension/file/LocalFilesystemBackendFileGrowthTest.java @@ -0,0 +1,161 @@ +/* + * Copyright 2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.agentic.ai.agent.orchestration.extension.file; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.condition.EnabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** 验证后端检查文件后、真实 rg 搜索前，文件增长越过大小上限的情况。 */ +@EnabledOnOs({ OS.MAC, OS.LINUX }) +public class LocalFilesystemBackendFileGrowthTest { + + private static final String MARKER = "SIZE_LIMIT_MARKER"; + + @TempDir + Path root; + + /** 验证显式搜索只返回目标文件，兼容隐藏文件、忽略规则和文件名中的特殊字符。 */ + @ParameterizedTest + @ValueSource(strings = { "plain.txt", ".hidden[?]*{x}.txt", "!special.txt", "space name.txt", "back\\slash.txt" }) + void explicitSearchKeepsItsOriginalFileScope(String name) throws Exception { + findRipgrep(); + Files.writeString(this.root.resolve(name), MARKER + "\n"); + Files.writeString(this.root.resolve("other.txt"), MARKER + "\n"); + Files.writeString(Files.createDirectory(this.root.resolve("nested")).resolve(name), MARKER + "\n"); + Files.writeString(this.root.resolve(".ignore"), "*\n"); + final LocalFilesystemBackend backend = new LocalFilesystemBackend(this.root.toString(), true, 1); + final Object result = backend.grepRaw(MARKER, "/" + name, null); + assertTrue(result instanceof List, "Expected a match list"); + final List matches = (List) result; + assertEquals(1, matches.size(), "Explicit search must not include sibling or nested files"); + assertTrue(matches.get(0) instanceof GrepMatch); + assertEquals("/" + name.replace("\\", "/"), ((GrepMatch) matches.get(0)).getPath()); + } + + /** 用信号文件同步扩容与搜索，分别验证目录输入和显式文件输入。 */ + @ParameterizedTest(name = "explicitFile={0}") + @ValueSource(booleans = { false, true }) + void skipsFileThatGrowsAfterBackendCheck(boolean explicitFile) throws Exception { + final Path realRipgrep = findRipgrep(); + final Path workspace = Files.createDirectory(this.root.resolve("workspace")); + final Path growingFile = Files.writeString(workspace.resolve("growing.txt"), "initial content\n"); + assertTrue(Files.size(growingFile) < 1024L * 1024L); + final Path ready = this.root.resolve("rg-ready"); + final Path release = this.root.resolve("rg-release"); + final Path result = this.root.resolve("result.txt"); + final Path log = this.root.resolve("probe.log"); + final Path bin = createRipgrepGate(); + final String java = Path.of(System.getProperty("java.home"), "bin", "java").toString(); + final String classpath = System.getProperty("surefire.test.class.path", System.getProperty("java.class.path")); + final ProcessBuilder builder = new ProcessBuilder(java, "-cp", classpath, + LocalFilesystemBackendFileGrowthTest.class.getName(), workspace.toString(), + Boolean.toString(explicitFile), result.toString()); + builder.environment().put("PATH", bin.toString()); + builder.environment().put("ARGI_GROWTH_READY", ready.toString()); + builder.environment().put("ARGI_GROWTH_RELEASE", release.toString()); + builder.environment().put("ARGI_GROWTH_REAL_RG", realRipgrep.toString()); + builder.environment().remove("RIPGREP_CONFIG_PATH"); + builder.redirectErrorStream(true).redirectOutput(log.toFile()); + final Process process = builder.start(); + try { + awaitRipgrepGate(ready, process, log); + Files.writeString(growingFile, "padding\n".repeat(150_000) + MARKER + "\n", StandardCharsets.UTF_8, + StandardOpenOption.APPEND); + assertTrue(Files.size(growingFile) > 1024L * 1024L); + Files.writeString(release, "continue"); + assertTrue(process.waitFor(15, TimeUnit.SECONDS), "Probe JVM did not finish after releasing rg"); + assertEquals(0, process.exitValue(), Files.readString(log)); + assertTrue(Files.isRegularFile(result), "Probe did not produce a search result"); + assertEquals("matches=0", Files.readString(result), + "The file grew beyond 1 MiB after the backend check; explicitFile=" + explicitFile); + } + finally { + process.descendants().forEach(ProcessHandle::destroyForcibly); + process.destroyForcibly(); + assertTrue(process.waitFor(5, TimeUnit.SECONDS), "Probe JVM did not stop"); + } + } + + /** 定位 PATH 中真实可执行的 rg，缺失时明确指出测试环境不满足要求。 */ + private static Path findRipgrep() throws Exception { + final String path = System.getenv().getOrDefault("PATH", ""); + for (String directory : path.split(File.pathSeparator)) { + final Path candidate = Path.of(directory, "rg"); + if (Files.isRegularFile(candidate) && Files.isExecutable(candidate)) { + return candidate.toRealPath(); + } + } + throw new IllegalStateException("Install ripgrep and add its directory to the test JVM PATH"); + } + + /** 创建测试专用 rg 包装脚本，收到放行信号后以原参数执行真实 rg。 */ + private Path createRipgrepGate() throws Exception { + final Path bin = Files.createDirectory(this.root.resolve("bin")); + final Path wrapper = Files.writeString(bin.resolve("rg"), """ + #!/bin/sh + set -eu + : > "$ARGI_GROWTH_READY" + attempt=0 + while [ ! -f "$ARGI_GROWTH_RELEASE" ]; do + if [ "$attempt" -ge 2000 ]; then + exit 70 + fi + /bin/sleep 0.01 + attempt=$((attempt + 1)) + done + exec "$ARGI_GROWTH_REAL_RG" "$@" + """, StandardCharsets.UTF_8); + Files.setPosixFilePermissions(wrapper, PosixFilePermissions.fromString("rwx------")); + return bin; + } + + /** 等待 rg 包装脚本发出就绪信号，不通过固定延时猜测文件检查时序。 */ + private static void awaitRipgrepGate(Path ready, Process process, Path log) throws Exception { + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(15); + while (!Files.exists(ready)) { + assertTrue(process.isAlive(), "Probe exited before reaching rg: " + Files.readString(log)); + assertTrue(System.nanoTime() < deadline, "Timed out waiting for rg gate"); + Thread.sleep(10); + } + } + + /** 子 JVM 入口：以 1 MiB 上限调用真实后端，并将搜索匹配数量写入结果文件。 */ + public static void main(String[] args) throws Exception { + final LocalFilesystemBackend backend = new LocalFilesystemBackend(args[0], true, 1); + final boolean explicitFile = Boolean.parseBoolean(args[1]); + final Object result = backend.grepRaw(MARKER, explicitFile ? "/growing.txt" : "/", "growing.txt"); + if (!(result instanceof List matches)) { + throw new IllegalStateException("Expected grepRaw to return a match list"); + } + Files.writeString(Path.of(args[2]), "matches=" + matches.size(), StandardCharsets.UTF_8); + } + +} diff --git a/argi-orchestration/argi-agent-framework/src/test/java/io/github/agentic/ai/agent/orchestration/extension/file/LocalFilesystemBackendSizeLimitReproductionTest.java b/argi-orchestration/argi-agent-framework/src/test/java/io/github/agentic/ai/agent/orchestration/extension/file/LocalFilesystemBackendSizeLimitReproductionTest.java new file mode 100644 index 000000000..5f1f0f962 --- /dev/null +++ b/argi-orchestration/argi-agent-framework/src/test/java/io/github/agentic/ai/agent/orchestration/extension/file/LocalFilesystemBackendSizeLimitReproductionTest.java @@ -0,0 +1,113 @@ +/* + * Copyright 2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.agentic.ai.agent.orchestration.extension.file; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** 使用真实文件和两个独立 JVM，复现 #231：rg 搜索绕过单文件大小限制。 */ +public class LocalFilesystemBackendSizeLimitReproductionTest { + + private static final String MARKER = "SIZE_LIMIT_MARKER"; + + @TempDir + Path root; + + /** 验证目录与显式文件输入在大小低于、等于、超过 1 MiB 时的行为。 */ + @ParameterizedTest(name = "explicitFile={0}") + @ValueSource(booleans = { false, true }) + void oversizedFileMustBeSkippedWithAndWithoutRipgrep(boolean explicitFile) throws Exception { + final String marker = "\n" + MARKER + "\n"; + final String padding = "x\n".repeat(524_289); + final String[] names = { "below.txt", "at.txt", "above.txt" }; + for (int i = 0; i < names.length; i++) { + final int size = 1024 * 1024 + i - 1; + final Path file = this.root.resolve(names[i]); + Files.writeString(file, padding.substring(0, size - marker.length()) + marker, StandardCharsets.UTF_8); + assertEquals(size, Files.size(file)); + } + final String fallback = probe(false, explicitFile); + final String ripgrep = probe(true, explicitFile); + assertTrue(ripgrep.contains("rg=true;"), "rg is required on the test JVM PATH: " + ripgrep); + assertTrue(fallback.contains("rg=false;below.txt=1;at.txt=1;above.txt=0"), fallback); + assertTrue(ripgrep.contains("rg=true;below.txt=1;at.txt=1;above.txt=0"), + "Issue #231: both backends must enforce the same size limit.\nJava: " + fallback + "\nrg: " + ripgrep); + } + + /** 在独立 JVM 中运行搜索，通过 PATH 控制是否使用 rg，并返回搜索结果。 */ + private String probe(boolean useRipgrep, boolean explicitFile) throws Exception { + final String java = Path.of(System.getProperty("java.home"), "bin", "java").toString(); + final String classpath = System.getProperty("surefire.test.class.path", System.getProperty("java.class.path")); + final Path log = this.root.resolve(useRipgrep ? "rg.log" : "java.log"); + final ProcessBuilder builder = new ProcessBuilder(java, "-cp", classpath, + LocalFilesystemBackendSizeLimitReproductionTest.class.getName(), this.root.toString(), Boolean.toString(explicitFile)); + if (!useRipgrep) { + builder.environment().put("PATH", Files.createDirectory(this.root.resolve("empty-path")).toString()); + } + builder.redirectErrorStream(true).redirectOutput(log.toFile()); + final Process process = builder.start(); + try { + assertTrue(process.waitFor(30, TimeUnit.SECONDS), "Probe JVM timed out"); + final String output = Files.readString(log); + assertEquals(0, process.exitValue(), output); + return output; + } + finally { + process.destroyForcibly(); + } + } + + /** 子 JVM 入口：创建文件后端，搜索大小文件并输出匹配数量。 */ + public static void main(String[] args) throws Exception { + boolean hasRipgrep = false; + try { + final Process version = new ProcessBuilder("rg", "--version") + .redirectOutput(ProcessBuilder.Redirect.DISCARD) + .redirectError(ProcessBuilder.Redirect.DISCARD) + .start(); + try { + hasRipgrep = version.waitFor(5, TimeUnit.SECONDS) && version.exitValue() == 0; + } + finally { + version.destroyForcibly(); + } + } + catch (java.io.IOException ignored) { + } + final LocalFilesystemBackend backend = new LocalFilesystemBackend(args[0], true, 1); + final boolean explicitFile = Boolean.parseBoolean(args[1]); + final StringBuilder result = new StringBuilder("rg=" + hasRipgrep); + for (String name : List.of("below.txt", "at.txt", "above.txt")) { + final Object matches = backend.grepRaw(MARKER, explicitFile ? "/" + name : "/", name); + if (!(matches instanceof List matchList)) { + throw new IllegalStateException("Expected grepRaw to return a match list"); + } + result.append(';').append(name).append('=').append(matchList.size()); + } + System.out.println(result); + } + +}