extend the leading normalization
This commit is contained in:
1 parent
873b2ef061
commit
03e0f7c9d4
1 file changed
+35
-14
+35
-14
@@ -47,7 +47,7 @@ defmodule FeedReader.Parser do
|
|||||||
|
|
||||||
format ->
|
format ->
|
||||||
input = File.stream!(path, Keyword.get(opts, :chunk_size, @default_chunk_size), [:raw])
|
input = File.stream!(path, Keyword.get(opts, :chunk_size, @default_chunk_size), [:raw])
|
||||||
{:ok, format, strip_utf8_bom(input)}
|
{:ok, format, normalize_xml_prefix(input)}
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
@@ -67,33 +67,54 @@ defmodule FeedReader.Parser do
|
|||||||
|
|
||||||
defp format_module(_format, _opts), do: nil
|
defp format_module(_format, _opts), do: nil
|
||||||
|
|
||||||
defp strip_utf8_bom(input) do
|
defp normalize_xml_prefix(input) do
|
||||||
Stream.transform(input, fn -> {:pending, <<>>} end, &strip_bom_chunk/2, &flush_bom/1)
|
Stream.transform(
|
||||||
|
input,
|
||||||
|
fn -> {:leading, <<>>} end,
|
||||||
|
&normalize_prefix_chunk/2,
|
||||||
|
&flush_prefix/1
|
||||||
|
)
|
||||||
end
|
end
|
||||||
|
|
||||||
defp strip_bom_chunk(chunk, {:pending, prefix}) do
|
defp normalize_prefix_chunk(chunk, {:leading, prefix}) do
|
||||||
data = prefix <> chunk
|
data = prefix <> chunk
|
||||||
|
|
||||||
if byte_size(data) < 3 do
|
if bom_prefix?(data) do
|
||||||
{[], {:pending, data}}
|
{[], {:leading, data}}
|
||||||
else
|
else
|
||||||
output =
|
output =
|
||||||
if binary_part(data, 0, 3) == <<0xEF, 0xBB, 0xBF>>,
|
if starts_with_bom?(data),
|
||||||
do: binary_part(data, 3, byte_size(data) - 3),
|
do: binary_part(data, 3, byte_size(data) - 3),
|
||||||
else: data
|
else: data
|
||||||
|
|
||||||
emit_bom_chunk(output)
|
output = trim_leading_xml_whitespace(output)
|
||||||
|
|
||||||
|
if output == <<>> do
|
||||||
|
{[], {:leading, <<>>}}
|
||||||
|
else
|
||||||
|
{[output], :ready}
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
defp strip_bom_chunk(chunk, :ready), do: {[chunk], :ready}
|
defp normalize_prefix_chunk(chunk, :ready), do: {[chunk], :ready}
|
||||||
|
|
||||||
defp emit_bom_chunk(<<>>), do: {[], :ready}
|
defp bom_prefix?(data) when byte_size(data) < 3,
|
||||||
defp emit_bom_chunk(chunk), do: {[chunk], :ready}
|
do: data == binary_part(<<0xEF, 0xBB, 0xBF>>, 0, byte_size(data))
|
||||||
|
|
||||||
defp flush_bom({:pending, <<>>}), do: []
|
defp bom_prefix?(_data), do: false
|
||||||
defp flush_bom({:pending, prefix}), do: [prefix]
|
|
||||||
defp flush_bom(:ready), do: []
|
defp starts_with_bom?(<<0xEF, 0xBB, 0xBF, _rest::binary>>), do: true
|
||||||
|
defp starts_with_bom?(_data), do: false
|
||||||
|
|
||||||
|
defp trim_leading_xml_whitespace(<<char, rest::binary>>) when char in [9, 10, 13, 32],
|
||||||
|
do: trim_leading_xml_whitespace(rest)
|
||||||
|
|
||||||
|
defp trim_leading_xml_whitespace(data), do: data
|
||||||
|
|
||||||
|
defp flush_prefix({:leading, <<>>}), do: []
|
||||||
|
defp flush_prefix({:leading, prefix}), do: [prefix]
|
||||||
|
defp flush_prefix(:ready), do: []
|
||||||
|
|
||||||
defp event_stream(input, format, opts, chunk_size) do
|
defp event_stream(input, format, opts, chunk_size) do
|
||||||
Stream.resource(
|
Stream.resource(
|
||||||
|
|||||||
Reference in new issue
Block a user