diff --git a/lib/feed_reader/recovery/encoding.ex b/lib/feed_reader/recovery/encoding.ex new file mode 100644 index 0000000..58a42c1 --- /dev/null +++ b/lib/feed_reader/recovery/encoding.ex @@ -0,0 +1,45 @@ +defmodule FeedReader.Recovery.Encoding do + @moduledoc """ + Recovers character encoding mismatches in XML feeds. + + Some feeds declare ISO-8859-1 or Latin-1 while the XML parser pipeline is + UTF-8 based. `transcode/2` converts those chunks without eagerly consuming + the stream. `normalize_declaration/2` then rewrites the original declaration + to UTF-8, including declarations split across input chunks. + """ + + def transcode(input, :utf8), do: input + + def transcode(input, :latin1), + do: Stream.map(input, &:unicode.characters_to_binary(&1, :latin1, :utf8)) + + def normalize_declaration(input, :utf8), do: input + + def normalize_declaration(input, :latin1) do + Stream.transform(input, fn -> {:prolog, <<>>} end, &rewrite_chunk/2, &flush/1) + end + + defp rewrite_chunk(chunk, {:prolog, prefix}) do + data = prefix <> chunk + + case :binary.match(data, "?>") do + {position, 2} -> + {declaration, rest} = :erlang.split_binary(data, position + 2) + {[rewrite(declaration) <> rest], :ready} + + :nomatch -> + {[], {:prolog, data}} + end + end + + defp rewrite_chunk(chunk, :ready), do: {[chunk], :ready} + + defp rewrite(declaration), + do: + Regex.replace(~r/(encoding\s*=\s*["'])[^"']+(["'])/i, declaration, "\\1UTF-8\\2", + global: false + ) + + defp flush({:prolog, prefix}), do: [prefix] + defp flush(:ready), do: [] +end diff --git a/test/feed_reader/recovery/encoding_test.exs b/test/feed_reader/recovery/encoding_test.exs new file mode 100644 index 0000000..e6573ee --- /dev/null +++ b/test/feed_reader/recovery/encoding_test.exs @@ -0,0 +1,32 @@ +defmodule FeedReader.Recovery.EncodingTest do + use ExUnit.Case, async: true + + alias FeedReader.Recovery.Encoding + + test "passes UTF-8 input through unchanged" do + chunks = ["