diff --git a/lib/feed_reader/recovery/prefix.ex b/lib/feed_reader/recovery/prefix.ex new file mode 100644 index 0000000..2c204ab --- /dev/null +++ b/lib/feed_reader/recovery/prefix.ex @@ -0,0 +1,45 @@ +defmodule FeedReader.Recovery.Prefix do + @moduledoc """ + Recovers the beginning of an XML document before SAX parsing. + + Feeds commonly contain a UTF-8 BOM or whitespace before the XML declaration, + while Saxy expects the document to begin at the XML prolog or root element. + This module removes those prefixes incrementally and retains incomplete BOM + bytes across chunks, so it is safe for arbitrarily small stream chunks. + """ + + def normalize(input) do + Stream.transform(input, fn -> {:leading, <<>>} end, &normalize_chunk/2, &flush/1) + end + + defp normalize_chunk(chunk, {:leading, prefix}) do + data = prefix <> chunk + + if bom_prefix?(data) do + {[], {:leading, data}} + else + output = + if starts_with_bom?(data), do: binary_part(data, 3, byte_size(data) - 3), else: data + + output = trim_whitespace(output) + if output == <<>>, do: {[], {:leading, <<>>}}, else: {[output], :ready} + end + end + + defp normalize_chunk(chunk, :ready), do: {[chunk], :ready} + + defp bom_prefix?(data) when byte_size(data) < 3, + do: data == binary_part(<<0xEF, 0xBB, 0xBF>>, 0, byte_size(data)) + + defp bom_prefix?(_data), do: false + defp starts_with_bom?(<<0xEF, 0xBB, 0xBF, _rest::binary>>), do: true + defp starts_with_bom?(_data), do: false + + defp trim_whitespace(<>) when char in [9, 10, 13, 32], + do: trim_whitespace(rest) + + defp trim_whitespace(data), do: data + defp flush({:leading, <<>>}), do: [] + defp flush({:leading, prefix}), do: [prefix] + defp flush(:ready), do: [] +end diff --git a/test/feed_reader/recovery/prefix_test.exs b/test/feed_reader/recovery/prefix_test.exs new file mode 100644 index 0000000..ba6e505 --- /dev/null +++ b/test/feed_reader/recovery/prefix_test.exs @@ -0,0 +1,28 @@ +defmodule FeedReader.Recovery.PrefixTest do + use ExUnit.Case, async: true + + alias FeedReader.Recovery.Prefix + + test "removes a BOM and leading XML whitespace" do + assert normalize([<<0xEF, 0xBB, 0xBF, 13, 10, 9, 32>>, ""]) == "" + end + + test "removes a BOM split across chunks" do + assert normalize([<<0xEF>>, <<0xBB>>, <<0xBF>>, " \n", ""]) == "" + end + + test "removes leading whitespace split across chunks" do + assert normalize(["\r", "\n", "\t", " ", ""]) == "" + end + + test "preserves content that does not have a removable prefix" do + assert normalize(["", ""]) == "" + end + + test "returns an empty stream for empty or whitespace-only input" do + assert normalize([]) == "" + assert normalize([" ", "\n", "\r\t"]) == "" + end + + defp normalize(chunks), do: chunks |> Prefix.normalize() |> Enum.join() +end