diff --git a/lib/csv/parser.rb b/lib/csv/parser.rb index 9b8a009b..c0826dc2 100644 --- a/lib/csv/parser.rb +++ b/lib/csv/parser.rb @@ -942,7 +942,11 @@ def skip_line?(line) def validate_field_size(field) return unless @max_field_size return if field.size <= @max_field_size - ignore_broken_line + if block_given? + yield + else + ignore_broken_line + end message = "Field size exceeded: #{field.size} > #{@max_field_size}" raise MalformedCSVError.new(message, @lineno) end @@ -1055,7 +1059,7 @@ def parse_quotable_robust(&block) value = parse_column_value if value @scanner.scan_all(@strip_value) if @strip_value - validate_field_size(value) + validate_field_size(value) {ignore_broken_record} end if parse_column_end row << value @@ -1264,6 +1268,20 @@ def ignore_broken_line @lineno += 1 end + def ignore_broken_record + lineno = @lineno + begin + while parse_column_end + @scanner.scan_all(@strip_value) if @strip_value + parse_column_value + @scanner.scan_all(@strip_value) if @strip_value + end + ignore_broken_line unless parse_row_end + rescue MalformedCSVError + end + @lineno = lineno + 1 + end + def start_row if @last_line @last_line = nil diff --git a/test/csv/parse/test_general.rb b/test/csv/parse/test_general.rb index 520eb8f0..ebad90b2 100644 --- a/test/csv/parse/test_general.rb +++ b/test/csv/parse/test_general.rb @@ -329,6 +329,83 @@ def test_max_field_size_in_extended_column_exceeding assert_parse_errors_out(data, max_field_size: 4) end + def test_max_field_size_multiline_record_liberal_parsing + csv = CSV.new(%Q{abcde,"x\ny"\nc,d\n}, + max_field_size: 4, liberal_parsing: true) + error = assert_raise(CSV::MalformedCSVError) do + csv.shift + end + assert_equal("Field size exceeded: 5 > 4 in line 1.", + error.message) + assert_equal(%Q{abcde,"x\ny"\n}, csv.line) + assert_equal(["c", "d"], csv.shift) + assert_equal(2, csv.lineno) + end + + def test_max_field_size_three_line_record_liberal_parsing + csv = CSV.new(%Q{abcde,"x\ny\nz"\nc,d\n}, + max_field_size: 4, liberal_parsing: true) + error = assert_raise(CSV::MalformedCSVError) do + csv.shift + end + assert_equal("Field size exceeded: 5 > 4 in line 1.", + error.message) + assert_equal(%Q{abcde,"x\ny\nz"\n}, csv.line) + assert_equal(["c", "d"], csv.shift) + assert_equal(2, csv.lineno) + end + + def test_max_field_size_multiline_record_after_multiline_row + csv = CSV.new(%Q{"a\nb",c\nabcde,"x\ny"\nc,d\n}, max_field_size: 4) + assert_equal(["a\nb", "c"], csv.shift) + error = assert_raise(CSV::MalformedCSVError) do + csv.shift + end + assert_equal("Field size exceeded: 5 > 4 in line 2.", + error.message) + assert_equal(%Q{abcde,"x\ny"\n}, csv.line) + assert_equal(["c", "d"], csv.shift) + assert_equal(3, csv.lineno) + end + + def test_max_field_size_multiline_record_strip + csv = CSV.new(%Q{abcde , "x\ny" \nc,d\n}, + max_field_size: 4, strip: true) + error = assert_raise(CSV::MalformedCSVError) do + csv.shift + end + assert_equal("Field size exceeded: 5 > 4 in line 1.", + error.message) + assert_equal(%Q{abcde , "x\ny" \n}, csv.line) + assert_equal(["c", "d"], csv.shift) + assert_equal(2, csv.lineno) + end + + def test_max_field_size_value_after_quoted_field + csv = CSV.new(%Q{"abcde"junk,b\nc,d\n}, max_field_size: 4) + error = assert_raise(CSV::MalformedCSVError) do + csv.shift + end + assert_equal("Field size exceeded: 5 > 4 in line 1.", + error.message) + assert_equal(%Q{"abcde"junk,b\n}, csv.line) + assert_equal(["c", "d"], csv.shift) + assert_equal(2, csv.lineno) + end + + def test_max_field_size_unclosed_quote_liberal_parsing + csv = CSV.new(%Q{abcde,"x\nc,d\n}, + max_field_size: 4, liberal_parsing: true) + error = assert_raise(CSV::MalformedCSVError) do + csv.shift + end + assert_equal("Field size exceeded: 5 > 4 in line 1.", + error.message) + assert_equal(%Q{abcde,"x\nc,d\n}, csv.line) + assert_nil(csv.shift) + assert_equal(1, csv.lineno) + end + def test_row_sep_auto_cr assert_equal([["a"]], CSV.parse("a\r")) end