From 849ef90d19bb94136b5ac603fa1ef2b946ad6b75 Mon Sep 17 00:00:00 2001 From: Moises Macia Date: Mon, 21 Sep 2026 23:24:59 +0000 Subject: [PATCH] use Recovery modules --- lib/feed_reader/parser.ex | 4 +- lib/feed_reader/recovery/recovery.ex | 40 +++ lib/feed_reader/xml_recovery.ex | 375 --------------------------- 3 files changed, 42 insertions(+), 377 deletions(-) create mode 100644 lib/feed_reader/recovery/recovery.ex delete mode 100644 lib/feed_reader/xml_recovery.ex diff --git a/lib/feed_reader/parser.ex b/lib/feed_reader/parser.ex index fe60664..3093953 100644 --- a/lib/feed_reader/parser.ex +++ b/lib/feed_reader/parser.ex @@ -1,7 +1,7 @@ defmodule FeedReader.Parser do @moduledoc "One-pass local feed parsing used by `FeedReader.stream_file/2`." - alias FeedReader.{FormatRegistry, Session, XMLRecovery} + alias FeedReader.{FormatRegistry, Recovery, Session} @default_chunk_size 65_536 @@ -45,7 +45,7 @@ defmodule FeedReader.Parser do format -> input = File.stream!(path, Keyword.get(opts, :chunk_size, @default_chunk_size), [:raw]) - {:ok, format, XMLRecovery.normalize(input, XMLRecovery.declared_encoding(prefix))} + {:ok, format, Recovery.normalize(input, Recovery.declared_encoding(prefix))} end end diff --git a/lib/feed_reader/recovery/recovery.ex b/lib/feed_reader/recovery/recovery.ex new file mode 100644 index 0000000..1b6c85e --- /dev/null +++ b/lib/feed_reader/recovery/recovery.ex @@ -0,0 +1,40 @@ +defmodule FeedReader.Recovery do + alias FeedReader.Recovery.{Attributes, Encoding, Entities, Prefix} + + @moduledoc """ + Public facade for streaming XML recovery. + + It detects declared encodings and applies prefix, encoding, entity, and + malformed-attribute recovery in that order. Each stage preserves streaming + behavior and carries incomplete markup across chunk boundaries. + """ + + @doc "Returns the encoding declared by an XML prefix, defaulting to UTF-8." + @spec declared_encoding(binary()) :: :utf8 | :latin1 + def declared_encoding(prefix) do + case Regex.run(~r/<\?xml\b[^>]*\bencoding\s*=\s*["']([^"']+)["']/is, prefix, + capture: :all_but_first + ) do + [encoding] -> normalize_encoding(encoding) + _none -> :utf8 + end + end + + @doc "Normalizes an enumerable of XML chunks without eagerly consuming it." + @spec normalize(Enumerable.t(), :utf8 | :latin1) :: Enumerable.t() + def normalize(input, encoding) do + input + |> Prefix.normalize() + |> Encoding.transcode(encoding) + |> Encoding.normalize_declaration(encoding) + |> Entities.normalize() + |> Attributes.normalize() + end + + defp normalize_encoding(encoding) do + case String.downcase(encoding) do + value when value in ["iso-8859-1", "latin1", "latin-1"] -> :latin1 + _utf8 -> :utf8 + end + end +end diff --git a/lib/feed_reader/xml_recovery.ex b/lib/feed_reader/xml_recovery.ex deleted file mode 100644 index 537e86d..0000000 --- a/lib/feed_reader/xml_recovery.ex +++ /dev/null @@ -1,375 +0,0 @@ -defmodule FeedReader.XMLRecovery do - @moduledoc """ - Streaming recovery and normalization for imperfect XML feeds. - - The recovery pipeline applies these compatibility repairs before the XML - parser receives the input: - - * Detects XML declarations that use ISO-8859-1, Latin-1, or Latin1. - * Removes a UTF-8 byte-order mark and whitespace before the XML document. - * Transcodes Latin-1 input to UTF-8. - * Rewrites a Latin-1 XML declaration to `encoding="UTF-8"` after transcoding. - * Preserves valid XML entities and escapes invalid or raw ampersand - references. - * Leaves CDATA sections and comments untouched while normalizing entities. - * Repairs literal quotes inside malformed quoted attributes, without - allowing a repair to cross XML element boundaries. - * Handles entities, markup, CDATA, comments, and attributes split across - arbitrary input chunks. - - Recovery is intentionally compatibility-oriented. It does not attempt to - make arbitrary malformed XML valid; it only repairs the feed irregularities - covered by the streaming normalizers above. - """ - - @doc "Returns the encoding declared by an XML prefix, defaulting to UTF-8." - @spec declared_encoding(binary()) :: :utf8 | :latin1 - def declared_encoding(prefix) do - case Regex.run(~r/<\?xml\b[^>]*\bencoding\s*=\s*["']([^"']+)["']/is, prefix, - capture: :all_but_first - ) do - [encoding] -> normalize_encoding(encoding) - _none -> :utf8 - end - end - - @doc "Normalizes an enumerable of XML chunks without eagerly consuming it." - @spec normalize(Enumerable.t(), :utf8 | :latin1) :: Enumerable.t() - def normalize(input, encoding) do - input - |> normalize_xml_prefix() - |> transcode_input(encoding) - |> normalize_encoding_declaration(encoding) - |> normalize_entities() - |> normalize_attribute_quotes() - end - - defp normalize_encoding(encoding) do - case String.downcase(encoding) do - value when value in ["iso-8859-1", "latin1", "latin-1"] -> :latin1 - _utf8 -> :utf8 - end - end - - defp normalize_xml_prefix(input) do - Stream.transform( - input, - fn -> {:leading, <<>>} end, - &normalize_prefix_chunk/2, - &flush_prefix/1 - ) - end - - defp normalize_prefix_chunk(chunk, {:leading, prefix}) do - data = prefix <> chunk - - if bom_prefix?(data) do - {[], {:leading, data}} - else - output = - if starts_with_bom?(data), do: binary_part(data, 3, byte_size(data) - 3), else: data - - output = trim_leading_xml_whitespace(output) - - if output == <<>>, do: {[], {:leading, <<>>}}, else: {[output], :ready} - end - end - - defp normalize_prefix_chunk(chunk, :ready), do: {[chunk], :ready} - - defp bom_prefix?(data) when byte_size(data) < 3, - do: data == binary_part(<<0xEF, 0xBB, 0xBF>>, 0, byte_size(data)) - - defp bom_prefix?(_data), do: false - defp starts_with_bom?(<<0xEF, 0xBB, 0xBF, _rest::binary>>), do: true - defp starts_with_bom?(_data), do: false - - defp trim_leading_xml_whitespace(<>) when char in [9, 10, 13, 32], - do: trim_leading_xml_whitespace(rest) - - defp trim_leading_xml_whitespace(data), do: data - defp flush_prefix({:leading, <<>>}), do: [] - defp flush_prefix({:leading, prefix}), do: [prefix] - defp flush_prefix(:ready), do: [] - - defp transcode_input(input, :utf8), do: input - - defp transcode_input(input, :latin1), - do: Stream.map(input, fn chunk -> :unicode.characters_to_binary(chunk, :latin1, :utf8) end) - - defp normalize_encoding_declaration(input, :utf8), do: input - - defp normalize_encoding_declaration(input, :latin1) do - Stream.transform( - input, - fn -> {:prolog, <<>>} end, - &rewrite_declaration_chunk/2, - &flush_declaration/1 - ) - end - - defp rewrite_declaration_chunk(chunk, {:prolog, prefix}) do - data = prefix <> chunk - - case :binary.match(data, "?>") do - {position, 2} -> - {declaration, rest} = :erlang.split_binary(data, position + 2) - {[rewrite_declaration(declaration) <> rest], :ready} - - :nomatch -> - {[], {:prolog, data}} - end - end - - defp rewrite_declaration_chunk(chunk, :ready), do: {[chunk], :ready} - - defp rewrite_declaration(declaration), - do: - Regex.replace(~r/(encoding\s*=\s*["'])[^"']+(["'])/i, declaration, "\\1UTF-8\\2", - global: false - ) - - defp flush_declaration({:prolog, prefix}), do: [prefix] - defp flush_declaration(:ready), do: [] - - defp normalize_entities(input) do - Stream.transform( - input, - fn -> {:normal, <<>>} end, - &normalize_entity_chunk/2, - &flush_entities/1 - ) - end - - defp normalize_entity_chunk(chunk, {mode, pending}) do - {output, mode, pending} = scan_entities(pending <> chunk, mode, []) - emit_entity_chunk(output, {mode, pending}) - end - - defp scan_entities(data, :normal, output) do - case next_markup(data) do - :none -> - case partial_markup_prefix(data) do - :none -> - {append_output(output, data), :normal, <<>>} - - length -> - split = byte_size(data) - length - - {append_output(output, binary_part(data, 0, split)), :normal, - binary_part(data, split, length)} - end - - {position, :cdata} -> - {prefix, rest} = :erlang.split_binary(data, position) - - scan_entities(binary_part(rest, 9, byte_size(rest) - 9), :cdata, [ - " - {prefix, rest} = :erlang.split_binary(data, position) - - scan_entities(binary_part(rest, 4, byte_size(rest) - 4), :comment, [ - "") do - :nomatch -> - keep = min(byte_size(data), 2) - split = byte_size(data) - keep - - {append_output(output, binary_part(data, 0, split)), :comment, - binary_part(data, split, keep)} - - {position, 3} -> - {prefix, rest} = :erlang.split_binary(data, position + 3) - scan_entities(rest, :normal, [prefix | output]) - end - end - - defp next_markup(data) do - [ - {:cdata, :binary.match(data, " Enum.reduce(:none, &earliest_markup/2) - end - - defp partial_markup_prefix(<<>>), do: :none - - defp partial_markup_prefix(data) do - max_length = min(byte_size(data), 8) - - Enum.find_value(max_length..1//-1, :none, fn length -> - suffix = binary_part(data, byte_size(data) - length, length) - - if suffix in [binary_part(" - entity = if valid_entity?(entity), do: entity, else: escape_entity(entity) - scan_entities(rest, :normal, [entity | output]) - - {:invalid, entity, rest} -> - scan_entities(rest, :normal, [escape_entity(entity) | output]) - - :incomplete -> - {output, :normal, "&" <> data} - end - end - - defp consume_entity(<<>>, _entity), do: :incomplete - defp consume_entity(<<";", rest::binary>>, entity), do: {:complete, entity <> ";", rest} - - defp consume_entity(<>, entity) - when char in [9, 10, 13, 32, 34, 38, 39, 60], - do: {:invalid, entity, <>} - - defp consume_entity(<>, entity), - do: consume_entity(rest, entity <> <>) - - defp valid_entity?(entity), - do: Regex.match?(~r/^&(amp|lt|gt|quot|apos|#\d+|#x[0-9a-f]+);$/i, entity) - - defp escape_entity(entity), do: "&" <> binary_part(entity, 1, byte_size(entity) - 1) - defp append_output(output, <<>>), do: output - defp append_output(output, chunk), do: [chunk | output] - - defp emit_entity_chunk(output, state) do - output = output |> Enum.reverse() |> IO.iodata_to_binary() - if output == <<>>, do: {[], state}, else: {[output], state} - end - - defp flush_entities({_mode, <<>>}), do: [] - defp flush_entities({_mode, pending}), do: [pending] - - defp normalize_attribute_quotes(input) do - Stream.transform( - input, - fn -> <<>> end, - fn chunk, pending -> - {output, pending} = scan_attribute_markup(pending <> chunk, []) - {if(output == [], do: [], else: [IO.iodata_to_binary(Enum.reverse(output))]), pending} - end, - fn - <<>> -> [] - pending -> [pending] - end - ) - end - - defp scan_attribute_markup(<<>>, output), do: {output, <<>>} - - defp scan_attribute_markup(data, output) do - case :binary.match(data, "<") do - :nomatch -> - {[data | output], <<>>} - - {position, 1} when position > 0 -> - {prefix, rest} = :erlang.split_binary(data, position) - scan_attribute_markup(rest, [prefix | output]) - - {0, 1} -> - case markup_end(data) do - :incomplete -> {output, data} - {markup, rest} -> scan_attribute_markup(rest, [repair_markup_quotes(markup) | output]) - end - end - end - - defp markup_end(<<"> = data), do: special_markup_end(data, "]]>") - defp markup_end(<<"") - - defp markup_end(data) do - case :binary.match(data, ">") do - :nomatch -> :incomplete - {position, 1} -> :erlang.split_binary(data, position + 1) - end - end - - defp special_markup_end(data, closing) do - case :binary.match(data, closing) do - :nomatch -> :incomplete - {position, length} -> :erlang.split_binary(data, position + length) - end - end - - defp repair_markup_quotes(<<"> = markup), do: markup - defp repair_markup_quotes(<<"