From 9d1a3290f7d0109ae4fa00d7053dec10ef264727 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 2 Aug 2026 00:02:04 +0300 Subject: [PATCH] gh-75008: Detect the lineterminator in csv.Sniffer.sniff() It is guessed by a majority vote among the line endings of the sample, instead of always being '\r\n'. --- Doc/library/csv.rst | 7 +++- Doc/whatsnew/3.16.rst | 5 +++ Lib/csv.py | 20 ++++++++++- Lib/test/test_csv.py | 33 +++++++++++++++++++ ...6-08-01-23-50-47.gh-issue-75008.0Boa3r.rst | 2 ++ 5 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-01-23-50-47.gh-issue-75008.0Boa3r.rst diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 81949261c563d0..ffa1e41326f307 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -328,10 +328,15 @@ The :mod:`!csv` module defines the following classes: are preferred, in this order, no matter how many times each of them occurs. + The *lineterminator* parameter is deduced separately, + by a majority vote among the line endings of the sample. + A tie is broken in the order ``'\r\n'``, ``'\n'``, ``'\r'``, + so a sample without a complete line gives ``'\r\n'``. + .. versionchanged:: next The dialect is now deduced by trial parsing and the results may differ from those of earlier Python versions. - The *escapechar* parameter can now be detected, + The *escapechar* and *lineterminator* parameters can now be detected, and the requested *delimiters* are not restricted to ASCII. diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 6e69737768d5e1..698bc28d844423 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -120,6 +120,11 @@ csv The results may differ from those of earlier Python versions. (Contributed by Serhiy Storchaka in :gh:`83273`.) +* :meth:`csv.Sniffer.sniff` now detects the *lineterminator* parameter + by a majority vote among the line endings of the sample, + instead of always returning ``'\r\n'``. + (Contributed by Serhiy Storchaka in :gh:`75008`.) + curses ------ diff --git a/Lib/csv.py b/Lib/csv.py index c66717dc1ee59e..a47713a859d655 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -373,9 +373,9 @@ def sniff(self, sample, delimiters=None): class dialect(Dialect): _name = "sniffed" - lineterminator = '\r\n' quoting = QUOTE_MINIMAL + dialect.lineterminator = self._detect_lineterminator(lines) dialect.delimiter = delimiter # _csv.reader won't accept a quotechar of '' dialect.quotechar = quotechar or '"' @@ -598,6 +598,24 @@ def _detect_skipinitialspace(self, lines, delimiter, quotechar, for kept_row, skipped_row in zip(*results)] return all(first) or not any(first) + def _detect_lineterminator(self, lines): + """ + Detect the line terminator by majority vote among the line + endings. A line break inside a quoted field is counted too, + but it takes more of them than of the real ones to win the + vote. A tie is broken in the order '\\r\\n', '\\n', '\\r', + so a sample without a complete line gives '\\r\\n'. + """ + counts = dict.fromkeys(('\r\n', '\n', '\r'), 0) + for line in lines: + for lineterminator in counts: + if line.endswith(lineterminator): + counts[lineterminator] += 1 + break + # max() returns the first of equal candidates, and dict + # preserves the insertion order. + return max(counts, key=counts.get) + def has_header(self, sample): # Creates a dictionary of types of data in each column. If any # column is of a single type (say, integers), *except* for the first diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 91170cc16b3ac9..766e28b0bab1ba 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1644,6 +1644,39 @@ def test_sniff_crlf_lineterminator(self): dialect = sniffer.sniff(sample) self.assertEqual(dialect.delimiter, ',') self.assertEqual(dialect.quotechar, '"') + self.assertEqual(dialect.lineterminator, '\r\n') + + def test_sniff_lineterminator(self): + sniffer = csv.Sniffer() + for lineterminator in '\r\n', '\n', '\r': + with self.subTest(lineterminator=lineterminator): + sample = lineterminator.join(['a,b,c', 'd,e,f', 'g,h,i', '']) + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.lineterminator, lineterminator) + self.assertEqual(dialect.delimiter, ',') + # The majority wins. + sample = 'a,b,c\nd,e,f\r\ng,h,i\n' + self.assertEqual(sniffer.sniff(sample).lineterminator, '\n') + sample = 'a,b,c\r\nd,e,f\ng,h,i\r\n' + self.assertEqual(sniffer.sniff(sample).lineterminator, '\r\n') + # A line break inside a quoted field is counted too, but it is + # outvoted by the real ones. + sample = 'a,"x\ny",c\r\nd,e,f\r\ng,h,i\r\n' + self.assertEqual(sniffer.sniff(sample).lineterminator, '\r\n') + + def test_sniff_lineterminator_tie(self): + # A tie is broken in the order '\r\n', '\n', '\r'. + sniffer = csv.Sniffer() + for sample, lineterminator in ( + ('a,b,c\nd,e,f\r\n', '\r\n'), + ('a,b,c\r\nd,e,f\ng,h,i\rj,k,l', '\r\n'), + ('a,b,c\nd,e,f\rg,h,i', '\n'), + # A sample without a complete line is a tie of zeros. + ('a,b,c', '\r\n'), + ): + with self.subTest(sample=sample): + self.assertEqual(sniffer.sniff(sample).lineterminator, + lineterminator) def test_sniff_excel_tab_with_quotes(self): # gh-62029: tab-delimited data with a quoted field containing diff --git a/Misc/NEWS.d/next/Library/2026-08-01-23-50-47.gh-issue-75008.0Boa3r.rst b/Misc/NEWS.d/next/Library/2026-08-01-23-50-47.gh-issue-75008.0Boa3r.rst new file mode 100644 index 00000000000000..19545f041093bd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-01-23-50-47.gh-issue-75008.0Boa3r.rst @@ -0,0 +1,2 @@ +:meth:`csv.Sniffer.sniff` now detects the *lineterminator* parameter by a +majority vote among the line endings of the sample.