extend the leading normalization

This commit is contained in:
mmacia committed 2026-08-07 21:28:07 +00:00
1 parent 873b2ef061
commit 03e0f7c9d4
1 file changed
+35 -14
+35 -14
View File
@@ -47,7 +47,7 @@ defmodule FeedReader.Parser do
format -> format ->
input = File.stream!(path, Keyword.get(opts, :chunk_size, @default_chunk_size), [:raw]) input = File.stream!(path, Keyword.get(opts, :chunk_size, @default_chunk_size), [:raw])
{:ok, format, strip_utf8_bom(input)} {:ok, format, normalize_xml_prefix(input)}
end end
end end
@@ -67,33 +67,54 @@ defmodule FeedReader.Parser do
defp format_module(_format, _opts), do: nil defp format_module(_format, _opts), do: nil
defp strip_utf8_bom(input) do defp normalize_xml_prefix(input) do
Stream.transform(input, fn -> {:pending, <<>>} end, &strip_bom_chunk/2, &flush_bom/1) Stream.transform(
input,
fn -> {:leading, <<>>} end,
&normalize_prefix_chunk/2,
&flush_prefix/1
)
end end
defp strip_bom_chunk(chunk, {:pending, prefix}) do defp normalize_prefix_chunk(chunk, {:leading, prefix}) do
data = prefix <> chunk data = prefix <> chunk
if byte_size(data) < 3 do if bom_prefix?(data) do
{[], {:pending, data}} {[], {:leading, data}}
else else
output = output =
if binary_part(data, 0, 3) == <<0xEF, 0xBB, 0xBF>>, if starts_with_bom?(data),
do: binary_part(data, 3, byte_size(data) - 3), do: binary_part(data, 3, byte_size(data) - 3),
else: data else: data
emit_bom_chunk(output) output = trim_leading_xml_whitespace(output)
if output == <<>> do
{[], {:leading, <<>>}}
else
{[output], :ready}
end
end end
end end
defp strip_bom_chunk(chunk, :ready), do: {[chunk], :ready} defp normalize_prefix_chunk(chunk, :ready), do: {[chunk], :ready}
defp emit_bom_chunk(<<>>), do: {[], :ready} defp bom_prefix?(data) when byte_size(data) < 3,
defp emit_bom_chunk(chunk), do: {[chunk], :ready} do: data == binary_part(<<0xEF, 0xBB, 0xBF>>, 0, byte_size(data))
defp flush_bom({:pending, <<>>}), do: [] defp bom_prefix?(_data), do: false
defp flush_bom({:pending, prefix}), do: [prefix]
defp flush_bom(:ready), do: [] defp starts_with_bom?(<<0xEF, 0xBB, 0xBF, _rest::binary>>), do: true
defp starts_with_bom?(_data), do: false
defp trim_leading_xml_whitespace(<<char, rest::binary>>) when char in [9, 10, 13, 32],
do: trim_leading_xml_whitespace(rest)
defp trim_leading_xml_whitespace(data), do: data
defp flush_prefix({:leading, <<>>}), do: []
defp flush_prefix({:leading, prefix}), do: [prefix]
defp flush_prefix(:ready), do: []
defp event_stream(input, format, opts, chunk_size) do defp event_stream(input, format, opts, chunk_size) do
Stream.resource( Stream.resource(