diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/ClientApiFunctionalTest.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/ClientApiFunctionalTest.java index 28d8921a51..5c25353e33 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/ClientApiFunctionalTest.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/ClientApiFunctionalTest.java @@ -18,7 +18,6 @@ import com.marklogic.client.io.Format; import com.marklogic.client.io.JacksonHandle; import com.marklogic.client.io.marker.JSONWriteHandle; -import com.marklogic.mgmt.resource.appservers.ServerManager; import okhttp3.Credentials; import okhttp3.OkHttpClient; import okhttp3.Request; @@ -35,8 +34,6 @@ public class ClientApiFunctionalTest extends AbstractFunctionalTest { - private final static String serverName = "java-functest"; - private static DatabaseClient dbclient = null; // Create an identifier for modules document - Client API call will be to endpoint @@ -302,63 +299,6 @@ public void TestE2EModuleNotFound() { } } - // This test sets the App Server concurrent users to be 1 instead of 0 - // (default). Expect to see ForbiddenUser when same user logins multiple times. - @Test - @Disabled("Disabling on 2023-01-03; this succeeds consistently locally, but fairly consistently fails on Jenkins. " + - "Not sure yet if it's due to the sleep time in the DS module or not. This is also not testing the Java Client " + - "but rather it's testing the 'concurrent request limit' feature of a MarkLogic app server.") - public void TestE2ENumberOfConcurrentUsers() throws InterruptedException { - StringBuilder msgEx = new StringBuilder(); - Thread w1, w2, w3; - - System.out.println("Running TestE2ENumberOfConcurrentUsers"); - modifyConcurrentUsersOnHttpServer(serverName, 1); - - class MultipleApiUsers implements Runnable { - - final String msg; - Float f; - - public void run() { - try { - f = TestE2EIntegerParaReturnDouble.on(dbclient).TestE2EItemPriceErrorCond(1000, 1000); - } catch (FailedRequestException ex) { - System.out.println("Exception is from Thread " + msg); - ex.printStackTrace(); - msgEx.append(msg); - msgEx.append("******"); - msgEx.append(ex.getMessage()); - } - } - - MultipleApiUsers(String in) { - msg = in; - } - } - - // Setup 3 API calls - try { - w1 = new Thread(new MultipleApiUsers("m1")); - w2 = new Thread(new MultipleApiUsers("m2")); - w3 = new Thread(new MultipleApiUsers("m3")); - - w1.start(); - w2.start(); - w3.start(); - w1.join(); - w2.join(); - w3.join(); - } finally { - modifyConcurrentUsersOnHttpServer(serverName, 0); - System.out.println("Exception from API responses of call are " + msgEx); - assertTrue( - msgEx.toString().contains("Local message: failed to POST at") && - msgEx.toString().contains("/ext/TestE2EIntegerParamReturnDouble/TestE2EIntegerParamReturnDoubleErrorCond.sjs"), - "Unexpected error message: " + msgEx); - } - } - // Test users with invalid roles @Test public void TestE2EuserWithInvalidRole() { @@ -629,13 +569,6 @@ public void TestOpenApiDirectory() throws Exception { assertTrue( jsNode.path("openapi").asText().contains("3.0.0")); assertTrue( jsNode.path("info").path("title").asText().contains("/ext/TestOpenApi services")); } - - public static void modifyConcurrentUsersOnHttpServer(String restServerName, int numberOfUsers) { - String body = "{\"server-name\":\"" + restServerName + "\", " + - "\"group-name\": \"Default\", " + - "\"concurrent-request-limit\":\"" + numberOfUsers + "\"}"; - new ServerManager(newManageClient()).save(body); - } } /** diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestOpticOnViews.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestOpticOnViews.java index a30b86f590..da34687aa1 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestOpticOnViews.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestOpticOnViews.java @@ -2665,7 +2665,6 @@ public void testQueryDSLJavaScriptFn() { // Similar to testgroupBy @Test - @Disabled("Going to fix this in a separate PR, it has been failing intermittently for a while.") public void testgroupByUnion() { System.out.println("In testgroupByUnion method"); RowManager rowMgr = client.newRowManager(); @@ -2698,18 +2697,27 @@ public void testgroupByUnion() { System.out.println("Results are : " + jsonBindingsNodes); // Should have 10 array nodes returned. assertEquals( 10, jsonBindingsNodes.size()); - assertEquals( "blue", jsonBindingsNodes.get(0).path("color").path("value").asText()); - assertEquals( 3, jsonBindingsNodes.get(0).path("ColorCount").path("value").asInt()); - assertEquals( "20.02", jsonBindingsNodes.get(0).path("AverageAmout").path("value").asText()); + // Rows with the same ColorCount have no guaranteed relative order, so locate rows by content instead of by index. + JsonNode blueRow = findRow(jsonBindingsNodes, "color", "value", "blue"); + assertNotNull(blueRow, "Expected a row for the blue color group"); + assertEquals( 3, blueRow.path("ColorCount").path("value").asInt()); + assertEquals( "20.02", blueRow.path("AverageAmout").path("value").asText()); - assertEquals( "green", jsonBindingsNodes.get(1).path("color").path("value").asText()); - assertEquals( 3, jsonBindingsNodes.get(1).path("ColorCount").path("value").asInt()); - assertEquals( "50.05", jsonBindingsNodes.get(1).path("AverageAmout").path("value").asText()); - // Assert for null in both grouping columns. We will have one row with nulls + JsonNode greenRow = findRow(jsonBindingsNodes, "color", "value", "green"); + assertNotNull(greenRow, "Expected a row for the green color group"); + assertEquals( 3, greenRow.path("ColorCount").path("value").asInt()); + assertEquals( "50.05", greenRow.path("AverageAmout").path("value").asText()); - assertEquals( "null", jsonBindingsNodes.get(9).path("color").path("type").asText()); - assertEquals( 0, jsonBindingsNodes.get(9).path("ColorCount").path("value").asInt()); - assertEquals( "null", jsonBindingsNodes.get(9).path("AverageAmout").path("type").asText()); + // Assert for null in both grouping columns. We will have one row with nulls + boolean nullRowFound = false; + for (JsonNode row : jsonBindingsNodes) { + if ("null".equals(row.path("color").path("type").asText()) + && "null".equals(row.path("AverageAmout").path("type").asText())) { + assertEquals( 0, row.path("ColorCount").path("value").asInt()); + nullRowFound = true; + } + } + assertTrue(nullRowFound, "Expected a row with null color and null average amount"); // Use date in aggregate ModifyPlan plan4 = plan1.union(plan2) .select(p.as("MasterDate", p.schemaCol("opticFunctionalTest", "master", "date")), @@ -2761,11 +2769,21 @@ public void testgroupByUnion() { assertEquals( 9, jsonBindingsNodesColOrd.size()); assertEquals( "2015-12-02", jsonBindingsNodesColOrd.get(0).path("MasterDate").path("value").asText()); assertEquals( 1, jsonBindingsNodesColOrd.get(0).path("DateCount").path("value").asInt()); - assertEquals( "null", jsonBindingsNodesColOrd.get(7).path("color").path("type").asText()); - assertEquals( "blue", jsonBindingsNodesColOrd.get(6).path("color").path("value").asText()); + // Rows with a null MasterDate have no guaranteed relative order, so locate rows by content instead of by index. + assertNotNull(findRow(jsonBindingsNodesColOrd, "color", "value", "blue"), "Expected a row for the blue color group"); + assertNotNull(findRow(jsonBindingsNodesColOrd, "color", "type", "null"), "Expected a row with a null color"); } + private static JsonNode findRow(JsonNode rows, String column, String field, String expectedValue) { + for (JsonNode row : rows) { + if (expectedValue.equals(row.path(column).path(field).asText())) { + return row; + } + } + return null; + } + // Same as testgroupsByUnion with facade. @Test public void testgroupByArrays() { diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/datamovement/SetMaxBatchesTest.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/datamovement/SetMaxBatchesTest.java index f9218baf66..64b139a46e 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/datamovement/SetMaxBatchesTest.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/datamovement/SetMaxBatchesTest.java @@ -6,40 +6,16 @@ import com.marklogic.client.datamovement.DataMovementManager; import com.marklogic.client.datamovement.QueryBatcher; import com.marklogic.client.fastfunctest.AbstractFunctionalTest; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.assertEquals; public class SetMaxBatchesTest extends AbstractFunctionalTest { - @Disabled("Getting this test in place to verify that setMaxBatches does not work when used with a query.") - @Test - void testWithQuery() { - writeJsonDocs(50, "max-batches-test"); - - DataMovementManager dmm = client.newDataMovementManager(); - AtomicInteger uriCount = new AtomicInteger(); - QueryBatcher queryBatcher = dmm - .newQueryBatcher(client.newQueryManager().newStructuredQueryBuilder().collection("max-batches-test")) - .withThreadCount(1) - .withBatchSize(10) - .onUrisReady(batch -> uriCount.addAndGet(batch.getItems().length)); - queryBatcher.setMaxBatches(2); - dmm.startJob(queryBatcher); - queryBatcher.awaitCompletion(); - dmm.stopJob(queryBatcher); - - assertEquals(20, uriCount.get(), "Because the batch size is 10 and we asked for 2 batches back, we only " + - "expect 20 URIs back. But through 6.2.2 (and probably going back much further), all URIs are returned. " + - "Modifying the thread count and batch size do not appear to affect this at all."); - } - /** * This verifies that setMaxBatches works with an iterator. The feature appears to have been introduced in 5.1.0, * so since then, it's only ever worked for an iterator. It does not work when an actual query is involved. diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/dataservices/BulkIOInputCallerTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/dataservices/BulkIOInputCallerTest.java index 6329de8d6c..d8e1a7bbf1 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/dataservices/BulkIOInputCallerTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/dataservices/BulkIOInputCallerTest.java @@ -12,19 +12,23 @@ import com.marklogic.client.io.InputStreamHandle; import com.marklogic.client.io.JacksonHandle; import com.marklogic.client.io.StringHandle; +import com.marklogic.client.test.AbstractClientTest; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.*; import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.stream.Stream; -public class BulkIOInputCallerTest { +public class BulkIOInputCallerTest extends AbstractClientTest { static ObjectNode[] apiObj = new ObjectNode[2]; static String[] apiNames = new String[]{"bulkIOInputCaller.api", "bulkIOAnyDocumentInputCaller.api"}; @@ -84,58 +88,52 @@ public void bulkInputEndpointTestWithMultipleCallContexts() { map.get("bulkInputTest_2") >= 1); } - @Disabled("Fails intermittently on Jenkins, never locally. Will revisit if we make enhancements to Bulk DS.") @Test public void bulkInputEndpointTestWithAnyDocument() { - - String endpointState = "{\"next\":"+1+"}"; - String endpointConstants = "{\"max\":6,\"collection\":\"bulkInputTest_1\"}"; - - String endpointState1 = "{\"next\":"+1+"}"; - String endpointConstants1 = "{\"max\":6,\"collection\":\"bulkInputTest_2\"}"; + final String collection1 = "bulkInputAnyDocTest_1"; + final String collection2 = "bulkInputAnyDocTest_2"; + final String endpointState = "{\"next\":1}"; InputCaller loadEndpt = InputCaller.onHandles(IOTestUtil.db, new JacksonHandle(apiObj[1]), new StringHandle()); - IOEndpoint.CallContext[] callContextArray = {loadEndpt.newCallContext() + IOEndpoint.CallContext[] callContextArray = { + loadEndpt.newCallContext() .withEndpointStateAs(endpointState) - .withEndpointConstantsAs(endpointConstants), loadEndpt.newCallContext() - .withEndpointStateAs(endpointState1) - .withEndpointConstantsAs(endpointConstants1)}; + .withEndpointConstantsAs("{\"max\":6,\"collection\":\"" + collection1 + "\"}"), + loadEndpt.newCallContext() + .withEndpointStateAs(endpointState) + .withEndpointConstantsAs("{\"max\":6,\"collection\":\"" + collection2 + "\"}") + }; InputCaller.BulkInputCaller loader = loadEndpt.bulkCaller(callContextArray); - Set input = IOTestUtil.setOf( // Set.of( - "{\"docNum\":1, \"docName\":\"doc1\"}", - "{\"docNum\":2, \"docName\":\"doc2\"}", - "{\"docNum\":3, \"docName\":\"doc3\"}", - "{\"docNum\":4, \"docName\":\"doc4\"}", - "{\"docNum\":5, \"docName\":\"doc5\"}", - "{\"docNum\":6, \"docName\":\"doc6\"}", - "{\"docNum\":7, \"docName\":\"doc7\"}", - "{\"docNum\":8, \"docName\":\"doc8\"}" - ); - - input.stream().forEach(value -> { - //System.out.println("adding "+value); - loader.accept(new StringHandle(value).withFormat(Format.JSON)); - }); + Set expectedDocNums = new TreeSet<>(); + for (int i = 1; i <= 8; i++) { + expectedDocNums.add(i); + loader.accept(new StringHandle("{\"docNum\":" + i + ", \"docName\":\"doc" + i + "\"}").withFormat(Format.JSON)); + } loader.awaitCompletion(); - checkDocuments("bulkInputTest_1"); - checkDocuments("bulkInputTest_2"); - assertTrue( counter == 4); - assertTrue( - map.get("bulkInputTest_1") >= 1); - assertTrue( - map.get("bulkInputTest_2") >= 1); + + // How the batches are spread across the two call contexts depends on thread timing, so only assert on the + // union of the inputs that were written, not on which collection or URI received them. + List actualDocNums = new ArrayList<>(); + for (String collection : new String[]{collection1, collection2}) { + for (int next = 2; next <= 6; next++) { + String uri = "/marklogic/ds/test/bulkInputCaller/" + collection + "/" + next + ".json"; + if (docMgr.exists(uri) != null) { + JsonNode doc = docMgr.read(uri, new JacksonHandle()).get(); + assertEquals(next, doc.get("state").get("next").asInt()); + assertEquals(6, doc.get("work").get("max").asInt()); + doc.get("inputs").forEach(input -> actualDocNums.add(input.get("docNum").asInt())); + } + } + } + Collections.sort(actualDocNums); + assertEquals(new ArrayList<>(expectedDocNums), actualDocNums, + "Every input should have been written exactly once, regardless of how batches were distributed"); } @AfterAll public static void cleanup(){ IOTestUtil.modMgr.delete(scriptPath, apiPath); - for(int i=2; i<5; i++) { - String uri = "/marklogic/ds/test/bulkInputCaller/bulkInputTest_1/" +i+".json"; - docMgr.delete(uri); - uri = "/marklogic/ds/test/bulkInputCaller/bulkInputTest_2/" +i+".json"; - docMgr.delete(uri); - } } private void checkDocuments(String collection) { diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java index a2d5087ebb..028d6ce57d 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java @@ -4,7 +4,6 @@ package com.marklogic.client.test.io; import com.marklogic.client.Transaction; -import com.marklogic.client.document.BinaryDocumentManager; import com.marklogic.client.document.DocumentManager.Metadata; import com.marklogic.client.document.XMLDocumentManager; import com.marklogic.client.impl.XmlFactories; @@ -14,23 +13,19 @@ import com.marklogic.client.io.InputStreamHandle; import com.marklogic.client.io.StringHandle; import com.marklogic.client.test.Common; -import com.marklogic.client.test.junit5.RequiresML11; import org.custommonkey.xmlunit.exceptions.XpathException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; -import javax.xml.namespace.QName; import javax.xml.parsers.ParserConfigurationException; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -177,46 +172,6 @@ public void testCapabilityEnum() { assertEquals(Capability.NODE_UPDATE, Capability.getValueOf("NODE_UPDATE")); } - @Test - // Requires MarkLogic 11 or higher now that we're using Docker; the INSTALL_CONVERTERS flag does not work for MarkLogic 10. - @ExtendWith(RequiresML11.class) - @Disabled("This is consistently failing in Jenkins with an error of: " + - "Process run error: fork: Cannot allocate memory. It runs fine locally and is ultimately just a test of a " + - "v1/documents feature and not of the Java Client.") - public void testMetadataPropertiesExtraction() { - String docId = "/test.bin"; - // Make a document manager to work with binary files - BinaryDocumentManager docMgr = Common.client.newBinaryDocumentManager(); - DocumentMetadataHandle metadataHandle = new DocumentMetadataHandle(); - docMgr.setMetadataExtraction(BinaryDocumentManager.MetadataExtraction.PROPERTIES); - // read binary from resources - ClassLoader classLoader = getClass().getClassLoader(); - File file = null; - FileInputStream docStream = null; - try { - file = new File(classLoader.getResource("test.bin").getFile()); - docStream = new FileInputStream(file); - } catch (Exception e) { - assertTrue(false); - } - - // Create a handle to hold string content. - InputStreamHandle handle = new InputStreamHandle(docStream); - docMgr.write(docId, metadataHandle, handle); - - DocumentMetadataHandle readMetadataHandle = new DocumentMetadataHandle(); - //read only the metadata into a handle - docMgr.readMetadata(docId, readMetadataHandle); - QName qn = new QName("", "testId"); - readMetadataHandle.getProperties().put(qn, "123456"); - docMgr.writeMetadata(docId, readMetadataHandle); - - readMetadataHandle = new DocumentMetadataHandle(); - - docMgr.readMetadata(docId, readMetadataHandle); - assertEquals("123456", readMetadataHandle.getProperties().get(qn)); - } - // testing https://github.com/marklogic/java-client-api/issues/783 // @Test diff --git a/marklogic-client-api/src/test/resources/test.bin b/marklogic-client-api/src/test/resources/test.bin deleted file mode 100644 index 6da2d8e2ab..0000000000 --- a/marklogic-client-api/src/test/resources/test.bin +++ /dev/null @@ -1 +0,0 @@ -my test binary document \ No newline at end of file