diff --git a/lib/feed_reader/parser.ex b/lib/feed_reader/parser.ex index b4a69fc..fe60664 100644 --- a/lib/feed_reader/parser.ex +++ b/lib/feed_reader/parser.ex @@ -1,7 +1,7 @@ defmodule FeedReader.Parser do @moduledoc "One-pass local feed parsing used by `FeedReader.stream_file/2`." - alias FeedReader.{FormatRegistry, Session} + alias FeedReader.{FormatRegistry, Session, XMLRecovery} @default_chunk_size 65_536 @@ -27,11 +27,9 @@ defmodule FeedReader.Parser do end end - @spec detected_input(Path.t(), keyword()) :: - {:ok, module(), Enumerable.t()} | {:error, :unknown} defp detected_input(path, opts) do max_bytes = - case Keyword.get(opts, :max_detection_bytes, 65_536) do + case Keyword.get(opts, :max_detection_bytes, @default_chunk_size) do value when is_integer(value) and value > 0 -> value _invalid -> 0 end @@ -47,11 +45,10 @@ defmodule FeedReader.Parser do format -> input = File.stream!(path, Keyword.get(opts, :chunk_size, @default_chunk_size), [:raw]) - {:ok, format, normalize_xml_input(input, declared_encoding(prefix))} + {:ok, format, XMLRecovery.normalize(input, XMLRecovery.declared_encoding(prefix))} end end - @spec format_module(atom(), keyword()) :: module() | nil defp format_module(:rss, _opts), do: FeedReader.Formats.RSS defp format_module(:atom, _opts), do: FeedReader.Formats.Atom defp format_module(:rdf, _opts), do: FeedReader.Formats.RDF @@ -67,369 +64,6 @@ defmodule FeedReader.Parser do defp format_module(_format, _opts), do: nil - defp declared_encoding(prefix) do - case Regex.run(~r/<\?xml\b[^>]*\bencoding\s*=\s*["']([^"']+)["']/is, prefix, - capture: :all_but_first - ) do - [encoding] -> normalize_encoding(encoding) - _none -> :utf8 - end - end - - defp normalize_encoding(encoding) do - case String.downcase(encoding) do - value when value in ["iso-8859-1", "latin1", "latin-1"] -> :latin1 - _utf8 -> :utf8 - end - end - - defp normalize_xml_input(input, encoding) do - input - |> normalize_xml_prefix() - |> transcode_input(encoding) - |> normalize_encoding_declaration(encoding) - |> normalize_entities() - |> normalize_attribute_quotes() - end - - defp normalize_xml_prefix(input) do - Stream.transform( - input, - fn -> {:leading, <<>>} end, - &normalize_prefix_chunk/2, - &flush_prefix/1 - ) - end - - defp normalize_prefix_chunk(chunk, {:leading, prefix}) do - data = prefix <> chunk - - if bom_prefix?(data) do - {[], {:leading, data}} - else - output = - if starts_with_bom?(data), - do: binary_part(data, 3, byte_size(data) - 3), - else: data - - output = trim_leading_xml_whitespace(output) - - if output == <<>> do - {[], {:leading, <<>>}} - else - {[output], :ready} - end - end - end - - defp normalize_prefix_chunk(chunk, :ready), do: {[chunk], :ready} - - defp bom_prefix?(data) when byte_size(data) < 3, - do: data == binary_part(<<0xEF, 0xBB, 0xBF>>, 0, byte_size(data)) - - defp bom_prefix?(_data), do: false - - defp starts_with_bom?(<<0xEF, 0xBB, 0xBF, _rest::binary>>), do: true - defp starts_with_bom?(_data), do: false - - defp trim_leading_xml_whitespace(<>) when char in [9, 10, 13, 32], - do: trim_leading_xml_whitespace(rest) - - defp trim_leading_xml_whitespace(data), do: data - - defp flush_prefix({:leading, <<>>}), do: [] - defp flush_prefix({:leading, prefix}), do: [prefix] - defp flush_prefix(:ready), do: [] - - defp transcode_input(input, :utf8), do: input - - defp transcode_input(input, :latin1), - do: Stream.map(input, fn chunk -> :unicode.characters_to_binary(chunk, :latin1, :utf8) end) - - defp normalize_encoding_declaration(input, :utf8), do: input - - defp normalize_encoding_declaration(input, :latin1) do - Stream.transform( - input, - fn -> {:prolog, <<>>} end, - &rewrite_declaration_chunk/2, - &flush_declaration/1 - ) - end - - defp rewrite_declaration_chunk(chunk, {:prolog, prefix}) do - data = prefix <> chunk - - case :binary.match(data, "?>") do - {position, 2} -> - {declaration, rest} = :erlang.split_binary(data, position + 2) - {[rewrite_declaration(declaration) <> rest], :ready} - - :nomatch -> - {[], {:prolog, data}} - end - end - - defp rewrite_declaration_chunk(chunk, :ready), do: {[chunk], :ready} - - defp rewrite_declaration(declaration) do - Regex.replace( - ~r/(encoding\s*=\s*["'])[^"']+(["'])/i, - declaration, - "\\1UTF-8\\2", - global: false - ) - end - - defp flush_declaration({:prolog, prefix}), do: [prefix] - defp flush_declaration(:ready), do: [] - - defp normalize_entities(input) do - Stream.transform( - input, - fn -> {:normal, <<>>} end, - &normalize_entity_chunk/2, - &flush_entities/1 - ) - end - - defp normalize_entity_chunk(chunk, {mode, pending}) do - {output, mode, pending} = scan_entities(pending <> chunk, mode, []) - emit_entity_chunk(output, {mode, pending}) - end - - defp scan_entities(data, :normal, output) do - case next_markup(data) do - :none -> - case partial_markup_prefix(data) do - :none -> - {append_output(output, data), :normal, <<>>} - - length -> - split = byte_size(data) - length - - {append_output(output, binary_part(data, 0, split)), :normal, - binary_part(data, split, length)} - end - - {position, :cdata} -> - {prefix, rest} = :erlang.split_binary(data, position) - - scan_entities(binary_part(rest, 9, byte_size(rest) - 9), :cdata, [ - " - {prefix, rest} = :erlang.split_binary(data, position) - - scan_entities(binary_part(rest, 4, byte_size(rest) - 4), :comment, [ - "") do - :nomatch -> - keep = min(byte_size(data), 2) - split = byte_size(data) - keep - - {append_output(output, binary_part(data, 0, split)), :comment, - binary_part(data, split, keep)} - - {position, 3} -> - {prefix, rest} = :erlang.split_binary(data, position + 3) - scan_entities(rest, :normal, [prefix | output]) - end - end - - defp next_markup(data) do - [ - {:cdata, :binary.match(data, " Enum.reduce(:none, &earliest_markup/2) - end - - defp partial_markup_prefix(<<>>), do: :none - - defp partial_markup_prefix(data) do - max_length = min(byte_size(data), 8) - - Enum.find_value(max_length..1//-1, :none, fn length -> - suffix = binary_part(data, byte_size(data) - length, length) - - if suffix in [binary_part(" - entity = if valid_entity?(entity), do: entity, else: escape_entity(entity) - scan_entities(rest, :normal, [entity | output]) - - {:invalid, entity, rest} -> - scan_entities(rest, :normal, [escape_entity(entity) | output]) - - :incomplete -> - {output, :normal, "&" <> data} - end - end - - defp consume_entity(<<>>, _entity), do: :incomplete - defp consume_entity(<<";", rest::binary>>, entity), do: {:complete, entity <> ";", rest} - - defp consume_entity(<>, entity) - when char in [9, 10, 13, 32, 34, 38, 39, 60], - do: {:invalid, entity, <>} - - defp consume_entity(<>, entity), - do: consume_entity(rest, entity <> <>) - - defp valid_entity?(entity), - do: Regex.match?(~r/^&(amp|lt|gt|quot|apos|#\d+|#x[0-9a-f]+);$/i, entity) - - defp escape_entity(entity), do: "&" <> binary_part(entity, 1, byte_size(entity) - 1) - - defp append_output(output, <<>>), do: output - defp append_output(output, chunk), do: [chunk | output] - - defp emit_entity_chunk(output, state) do - output = output |> Enum.reverse() |> IO.iodata_to_binary() - if output == <<>>, do: {[], state}, else: {[output], state} - end - - defp flush_entities({_mode, <<>>}), do: [] - defp flush_entities({_mode, pending}), do: [pending] - - defp normalize_attribute_quotes(input) do - Stream.transform( - input, - fn -> <<>> end, - fn chunk, pending -> - {output, pending} = scan_attribute_markup(pending <> chunk, []) - {if(output == [], do: [], else: [IO.iodata_to_binary(Enum.reverse(output))]), pending} - end, - fn - <<>> -> [] - pending -> [pending] - end - ) - end - - defp scan_attribute_markup(<<>>, output), do: {output, <<>>} - - defp scan_attribute_markup(data, output) do - case :binary.match(data, "<") do - :nomatch -> - {[data | output], <<>>} - - {position, 1} when position > 0 -> - {prefix, rest} = :erlang.split_binary(data, position) - scan_attribute_markup(rest, [prefix | output]) - - {0, 1} -> - case markup_end(data) do - :incomplete -> {output, data} - {markup, rest} -> scan_attribute_markup(rest, [repair_markup_quotes(markup) | output]) - end - end - end - - defp markup_end(<<"> = data), do: special_markup_end(data, "]]>") - defp markup_end(<<"") - - defp markup_end(data) do - case :binary.match(data, ">") do - :nomatch -> :incomplete - {position, 1} -> :erlang.split_binary(data, position + 1) - end - end - - defp special_markup_end(data, closing) do - case :binary.match(data, closing) do - :nomatch -> :incomplete - {position, length} -> :erlang.split_binary(data, position + length) - end - end - - defp repair_markup_quotes(<<"> = markup), do: markup - defp repair_markup_quotes(<<"