RAT-532: Bump tika.version from 3.3.2 to 4.0.0 and migrate charset detection

apache-rat-core/src/it/resources/ReportTest/RAT_81/verify.groovy

@@ -32,7 +32,8 @@ NodeList nodeList = XmlUtils.getNodeList(doc, xPath, "/rat-report/resource[@name
 assertEquals(1, nodeList.getLength())
 node = nodeList.item(0)
 attributes = node.getAttributes()
-assertEquals("IBM500", attributes.getNamedItem("encoding").getNodeValue())
+// pre-Tika4: recognized as IBM500 instead of IBM1047
+assertEquals("IBM1047", attributes.getNamedItem("encoding").getNodeValue())
 assertEquals("text/plain", attributes.getNamedItem("mediaType").getNodeValue())
 assertEquals("STANDARD", attributes.getNamedItem("type").getNodeValue())
 nodeList = XmlUtils.getNodeList(node, xPath, "license")
@@ -45,7 +46,7 @@ nodeList = XmlUtils.getNodeList(doc, xPath, "/rat-report/resource[@name='/UTF8.t
 assertEquals(1, nodeList.getLength())
 node = nodeList.item(0)
 attributes = node.getAttributes()
-assertEquals("ISO-8859-1", attributes.getNamedItem("encoding").getNodeValue())
+assertEquals("windows-1252", attributes.getNamedItem("encoding").getNodeValue())
 assertEquals("text/plain", attributes.getNamedItem("mediaType").getNodeValue())
 assertEquals("STANDARD", attributes.getNamedItem("type").getNodeValue())
 nodeList = XmlUtils.getNodeList(node, xPath, "license")

apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java

@@ -24,6 +24,7 @@
 import java.nio.charset.Charset;
 import java.nio.charset.UnsupportedCharsetException;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.rat.api.Document;
@@ -32,19 +33,29 @@
 import org.apache.rat.document.guesser.NoteGuesser;
 import org.apache.rat.utils.DefaultLog;
 import org.apache.tika.Tika;
+import org.apache.tika.detect.DefaultEncodingDetector;
+import org.apache.tika.detect.EncodingDetector;
+import org.apache.tika.detect.EncodingResult;
+import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.TikaCoreProperties;
 import org.apache.tika.mime.MediaType;
-import org.apache.tika.parser.txt.CharsetDetector;
-import org.apache.tika.parser.txt.CharsetMatch;
+import org.apache.tika.parser.ParseContext;
 
 /**
  * A wrapping around the Tika processor.
  */
 public final class TikaProcessor {
 
-    /** The Tika parser */
+    /** The Tika parser. */
     private static final Tika TIKA = new Tika();
+
+    /** The Tika encoding detector. */
+    private static final EncodingDetector ENCODING_DETECTOR = new DefaultEncodingDetector();
+
+    /** Due to performance reasons we do not read the whole file for charset detection (RAT-494). */
+    private static final int BYTES_FOR_CHARSET_DETECTION = 256;
+
     /** A map of mime type string to non-BINARY types.
      * "text" types are already handled somewhere else
      * BINARY unless listed here
@@ -165,21 +176,31 @@ public static String process(final Document document) throws RatDocumentAnalysis
      * @throws IOException on IO error.
      * @throws UnsupportedCharsetException on unsupported charset.
      */
-    private static Charset detectCharset(final InputStream stream, final DocumentName documentName) throws IOException, UnsupportedCharsetException {
-        final int bytesForCharsetDetection = 256;
-        CharsetDetector encodingDetector = new CharsetDetector(bytesForCharsetDetection);
-        encodingDetector.setText(stream);
-        CharsetMatch charsetMatch = encodingDetector.detect();
-        if (charsetMatch != null) {
-            try {
-                return Charset.forName(charsetMatch.getName());
-            } catch (UnsupportedCharsetException e) {
-                DefaultLog.getInstance().warn(String.format("Unsupported character set '%s' in file '%s'",
-                        charsetMatch.getName(), documentName));
-                throw e;
+    static Charset detectCharset(final InputStream stream, final DocumentName documentName) throws IOException, UnsupportedCharsetException {
+        stream.mark(BYTES_FOR_CHARSET_DETECTION);
+        try {
+            byte[] sample = stream.readNBytes(BYTES_FOR_CHARSET_DETECTION);
+            if (sample.length == 0) {
+              DefaultLog.getInstance().debug(String.format("No contents in file '%s'", documentName));
+              return null;
+            }
+
+            Metadata metadata = new Metadata();
+            ParseContext parseContext = new ParseContext();
+
+            try (TikaInputStream tis = TikaInputStream.get(sample, metadata)) {
+                List<EncodingResult> results = ENCODING_DETECTOR.detect(tis, metadata, parseContext);
+
+                if (results.isEmpty()) {
+                    DefaultLog.getInstance().warn(String.format("No encoding found for file '%s'", documentName));
+                    return null;
+                }
+
+                return results.get(0).getCharset();
             }
+        } finally {
+          stream.reset();
         }
-        return null;
     }
 
     /**

apache-rat-core/src/main/java/org/apache/rat/api/Document.java

@@ -20,13 +20,14 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.Reader;
+import java.nio.charset.Charset;
 import java.util.SortedSet;
 
 import org.apache.rat.analysis.TikaProcessor;
 import org.apache.rat.document.DocumentName;
 import org.apache.rat.document.DocumentNameMatcher;
-import org.apache.tika.parser.txt.CharsetDetector;
 
 /**
  * The representation of a document being scanned.
@@ -104,19 +105,24 @@ public boolean equals(final Object obj) {
     }
 
     /**
-     * Reads the contents of this document.
+     * Reads the contents of this document and
+     * relies on the charset detection of the underlying Tika processor.
+     *
      * @return <code>Reader</code> not null
      * @throws IOException if this document cannot be read.
      */
     public Reader reader() throws IOException {
-        final int bytesForCharsetDetection = 256;
-        CharsetDetector charsetDetector = new CharsetDetector(bytesForCharsetDetection);
-        // RAT-494: Tika's CharsetDetector.getReader() may return null if the read can not be constructed due to I/O or encoding errors
-        Reader result = charsetDetector.getReader(TikaProcessor.markSupportedInputStream(inputStream()), getMetaData().getCharset().name());
-        if (result == null) {
-            throw new IOException(String.format("Can not read document `%s`", getName()));
+        final Charset charset = getMetaData().getCharset();
+        if (charset == null) {
+            throw new IOException(
+                    String.format(
+                            "No charset detected for document `%s`",
+                            getName()));
         }
-        return result;
+
+        return new InputStreamReader(
+                TikaProcessor.markSupportedInputStream(inputStream()),
+                charset);
     }
 
     /**

apache-rat-core/src/test/java/org/apache/rat/ReporterOptionsProvider.java

@@ -866,7 +866,7 @@ private void styleSheetTest(final Option option) {
                         TextUtils.assertContainsExactly(1, "?????: 1 ", actualText);
                         break;
                     case XML:
-                        TextUtils.assertContainsExactly(1, "<resource encoding=\"ISO-8859-1\" mediaType=\"text/plain\" name=\"/stylesheet\" type=\"STANDARD\">", actualText);
+                        TextUtils.assertContainsExactly(1, "<resource encoding=\"windows-1252\" mediaType=\"text/plain\" name=\"/stylesheet\" type=\"STANDARD\">", actualText);
                         break;
                     case UNAPPROVED_LICENSES:
                         TextUtils.assertContainsExactly(1, "Files with unapproved licenses:" + System.lineSeparator() + "  /stylesheet", actualText);
@@ -928,7 +928,7 @@ protected void xmlTest() {
             assertThat(output.getStatistic().getCounter(ClaimStatistic.Counter.UNAPPROVED)).isEqualTo(1);
             output.format(config);
             String actualText = baos.toString(StandardCharsets.UTF_8);
-            TextUtils.assertContainsExactly(1, "<resource encoding=\"ISO-8859-1\" mediaType=\"text/plain\" name=\"/stylesheet\" type=\"STANDARD\">", actualText);
+            TextUtils.assertContainsExactly(1, "<resource encoding=\"windows-1252\" mediaType=\"text/plain\" name=\"/stylesheet\" type=\"STANDARD\">", actualText);
 
             try (InputStream expected = StyleSheets.getStyleSheet("xml").ioSupplier().get();
                  InputStream actual = config.getStyleSheet().get()) {

apache-rat-core/src/test/java/org/apache/rat/ReporterTest.java

@@ -244,30 +244,30 @@ void testXMLOutput() throws Exception {
         Map<String, Map<String, String>> expected = new HashMap<>();
         expected.put("/.hiddenDirectory", mapOf("isDirectory", "true", "mediaType", "application/octet-stream",
                 "type", "IGNORED"));
-        expected.put("/ILoggerFactory.java", mapOf("encoding", "ISO-8859-1", "mediaType", "text/x-java-source",
+        expected.put("/ILoggerFactory.java", mapOf("encoding", "windows-1252", "mediaType", "text/x-java-source",
                 "type", "STANDARD"));
         expected.put("/Image.png", mapOf("mediaType", "image/png", "type", "BINARY"));
-        expected.put("/LICENSE", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain", "type", "NOTICE"));
-        expected.put("/NOTICE", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain", "type", "NOTICE"));
-        expected.put("/Source.java", mapOf("encoding", "ISO-8859-1", "mediaType", "text/x-java-source",
+        expected.put("/LICENSE", mapOf("encoding", "windows-1252", "mediaType", "text/plain", "type", "NOTICE"));
+        expected.put("/NOTICE", mapOf("encoding", "windows-1252", "mediaType", "text/plain", "type", "NOTICE"));
+        expected.put("/Source.java", mapOf("encoding", "windows-1252", "mediaType", "text/x-java-source",
                 "type", "STANDARD"));
-        expected.put("/Text.txt", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain",
+        expected.put("/Text.txt", mapOf("encoding", "windows-1252", "mediaType", "text/plain",
                 "type", "STANDARD"));
-        expected.put("/TextHttps.txt", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain",
+        expected.put("/TextHttps.txt", mapOf("encoding", "windows-1252", "mediaType", "text/plain",
                 "type", "STANDARD"));
-        expected.put("/Xml.xml", mapOf("encoding", "ISO-8859-1", "mediaType", "application/xml",
+        expected.put("/Xml.xml", mapOf("encoding", "windows-1252", "mediaType", "application/xml",
                 "type", "STANDARD"));
-        expected.put("/buildr.rb", mapOf("encoding", "ISO-8859-1", "mediaType", "text/x-ruby",
+        expected.put("/buildr.rb", mapOf("encoding", "windows-1252", "mediaType", "text/x-ruby",
                 "type", "STANDARD"));
         expected.put("/dummy.jar", mapOf("mediaType", "application/java-archive",
                 "type", "ARCHIVE"));
-        expected.put("/generated.txt", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain",
+        expected.put("/generated.txt", mapOf("encoding", "windows-1252", "mediaType", "text/plain",
                 "type", "IGNORED"));
         expected.put("/plain.json", mapOf("mediaType", "application/json",
                 "type", "BINARY"));
-        expected.put("/sub/Empty.txt", mapOf("encoding", "UTF-8", "mediaType", "text/plain",
+        expected.put("/sub/Empty.txt", mapOf("encoding", "windows-1252", "mediaType", "text/plain",
                 "type", "STANDARD"));
-        expected.put("/tri.txt", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain",
+        expected.put("/tri.txt", mapOf("encoding", "windows-1252", "mediaType", "text/plain",
                 "type", "STANDARD"));
 
         File output = testPath.resolve(".rat/testXMLOutput").toFile();

apache-rat-core/src/test/java/org/apache/rat/analysis/TikaProcessorTest.java

@@ -27,11 +27,13 @@
 import org.apache.rat.document.DocumentName;
 import org.junit.jupiter.api.Test;
 
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
+import java.nio.charset.Charset;
 import java.nio.charset.MalformedInputException;
 import java.nio.charset.StandardCharsets;
 import java.util.Collections;
@@ -40,6 +42,7 @@
 import java.util.Objects;
 import java.util.SortedSet;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
@@ -52,7 +55,7 @@ public class TikaProcessorTest {
      * @see <a href="https://issues.apache.org/jira/browse/RAT-81">RAT-81</a>
      */
     @Test
-    public void RAT81() {
+    void RAT81() {
         // create a document that throws a MalformedInputException
         Document doc = mkDocument(new InputStream() {
             @Override
@@ -64,7 +67,7 @@ public int read() throws IOException {
     }
 
     @Test
-    public void UTF16_input() throws Exception {
+    void UTF16_input() throws Exception {
         Document doc = mkDocument(Resources.getResourceStream("/binaries/UTF16_with_signature.xml"),
                 DocumentNameMatcher.MATCHES_ALL);
         TikaProcessor.process(doc);
@@ -80,48 +83,48 @@ private FileDocument mkDocument(String fileName) throws IOException {
     }
 
     @Test
-    public void UTF8_input() throws Exception {
+    void UTF8_input() throws Exception {
         FileDocument doc = mkDocument("/binaries/UTF8_with_signature.xml");
         TikaProcessor.process(doc);
         assertEquals(Document.Type.STANDARD, doc.getMetaData().getDocumentType());
     }
 
     @Test
-    public void RAT178Test() {
+    void RAT178Test() {
         FileDocument doc = new FileDocument(new File("/not_a_real_file"), DocumentNameMatcher.MATCHES_ALL);
         assertThrows(RatDocumentAnalysisException.class, () ->TikaProcessor.process(doc));
     }
 
     @Test
-    public void missNamedBinaryTest() throws Exception {
+    void missNamedBinaryTest() throws Exception {
         FileDocument doc = mkDocument("/binaries/Image-png.not");
         TikaProcessor.process(doc);
         assertEquals(Document.Type.BINARY, doc.getMetaData().getDocumentType());
     }
 
     @Test
-    public void plainTextTest() throws Exception {
+    void plainTextTest() throws Exception {
         FileDocument doc = mkDocument(Resources.getExampleResource("exampleData/Text.txt"));
         TikaProcessor.process(doc);
         assertEquals(Document.Type.STANDARD, doc.getMetaData().getDocumentType());
     }
 
     @Test
-    public void emptyFileTest() throws Exception {
+    void emptyFileTest() throws Exception {
         FileDocument doc = mkDocument(Resources.getExampleResource("exampleData/sub/Empty.txt"));
         TikaProcessor.process(doc);
         assertEquals(Document.Type.STANDARD, doc.getMetaData().getDocumentType());
     }
 
     @Test
-    public void javaFileWithChineseCharacters_RAT301() throws Exception {
+    void javaFileWithChineseCharacters_RAT301() throws Exception {
         FileDocument doc = mkDocument("/tikaFiles/standard/ChineseCommentsJava.java");
         TikaProcessor.process(doc);
         assertEquals(Document.Type.STANDARD, doc.getMetaData().getDocumentType());
     }
 
     @Test
-    public void testTikaFiles() throws RatDocumentAnalysisException {
+    void testTikaFiles() throws RatDocumentAnalysisException {
         File dir = new File("src/test/resources/tikaFiles");
         Map<String, Document.Type> unseenMime = TikaProcessor.getDocumentTypeMap();
         ClaimStatistic statistic = new ClaimStatistic();
@@ -144,6 +147,22 @@ public void testTikaFiles() throws RatDocumentAnalysisException {
         }
     }
 
+    @Test
+    void testDetectionOfInvalidData() throws IOException {
+        byte[] invalidData = new byte[] {
+                0x00, (byte) 0xFF, 0x00, (byte) 0xFE,
+                0x01, (byte) 0x80, 0x00, 0x7F
+        };
+        // as Tika works with a probabilistic encoding detection it does not return NO encoding
+        assertThat(TikaProcessor.detectCharset(new ByteArrayInputStream(invalidData), null)).isEqualTo(Charset.forName("Windows-1258"));
+    }
+
+    @Test
+    void testEmptyFileEncoding() throws IOException {
+        byte[] empty = {};
+        assertThat(TikaProcessor.detectCharset(new ByteArrayInputStream(empty), null)).isNull();
+    }
+
     /**
      * Build a document with the specific input stream
      * @return a document with the specific input stream

apache-rat-core/src/test/java/org/apache/rat/document/DocumentNameTest.java

@@ -172,7 +172,6 @@ void resolveWithMultipleRootsTest() {
         String resolveName2 = fsInfo.roots()[0] + fsInfo.mkPath("dir", fsInfo.toString(), "thing");
         assertThat(base.resolve(resolveName2).getName())
                 .isEqualTo(resolveName2);
-
     }
 
     void testNoRootSpecified() {
@@ -258,7 +257,6 @@ void asPathTest() throws IOException {
         }
     }
 
-
     @ParameterizedTest(name = "{index} {0} {1}")
     @MethodSource("archiveEntryTestData")
     void archiveEntryNameTest(String os, String testName, DocumentName archiveName, String root, String separator, String baseName,
@@ -300,4 +298,5 @@ static List<Arguments> archiveEntryTestData() {
         }
         return lst;
     }
+
 }