add Recovery.Encoding module

lib/feed_reader/recovery/encoding.ex

@@ -0,0 +1,45 @@
+defmodule FeedReader.Recovery.Encoding do
+  @moduledoc """
+  Recovers character encoding mismatches in XML feeds.
+
+  Some feeds declare ISO-8859-1 or Latin-1 while the XML parser pipeline is
+  UTF-8 based. `transcode/2` converts those chunks without eagerly consuming
+  the stream. `normalize_declaration/2` then rewrites the original declaration
+  to UTF-8, including declarations split across input chunks.
+  """
+
+  def transcode(input, :utf8), do: input
+
+  def transcode(input, :latin1),
+    do: Stream.map(input, &:unicode.characters_to_binary(&1, :latin1, :utf8))
+
+  def normalize_declaration(input, :utf8), do: input
+
+  def normalize_declaration(input, :latin1) do
+    Stream.transform(input, fn -> {:prolog, <<>>} end, &rewrite_chunk/2, &flush/1)
+  end
+
+  defp rewrite_chunk(chunk, {:prolog, prefix}) do
+    data = prefix <> chunk
+
+    case :binary.match(data, "?>") do
+      {position, 2} ->
+        {declaration, rest} = :erlang.split_binary(data, position + 2)
+        {[rewrite(declaration) <> rest], :ready}
+
+      :nomatch ->
+        {[], {:prolog, data}}
+    end
+  end
+
+  defp rewrite_chunk(chunk, :ready), do: {[chunk], :ready}
+
+  defp rewrite(declaration),
+    do:
+      Regex.replace(~r/(encoding\s*=\s*["'])[^"']+(["'])/i, declaration, "\\1UTF-8\\2",
+        global: false
+      )
+
+  defp flush({:prolog, prefix}), do: [prefix]
+  defp flush(:ready), do: []
+end

test/feed_reader/recovery/encoding_test.exs

@@ -0,0 +1,32 @@
+defmodule FeedReader.Recovery.EncodingTest do
+  use ExUnit.Case, async: true
+
+  alias FeedReader.Recovery.Encoding
+
+  test "passes UTF-8 input through unchanged" do
+    chunks = ["<title>", "café", "</title>"]
+
+    assert Encoding.transcode(chunks, :utf8) == chunks
+    assert Encoding.normalize_declaration(chunks, :utf8) == chunks
+  end
+
+  test "transcodes Latin-1 chunks to UTF-8" do
+    chunks = ["<title>caf", <<0xE9>>, "</title>"]
+
+    assert chunks |> Encoding.transcode(:latin1) |> Enum.join() == "<title>café</title>"
+  end
+
+  test "rewrites a Latin-1 declaration to UTF-8" do
+    chunks = ["<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>", "<rss/>"]
+
+    assert chunks |> Encoding.normalize_declaration(:latin1) |> Enum.join() ==
+             "<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss/>"
+  end
+
+  test "rewrites declarations split across chunks" do
+    chunks = ["<?xml version=\"1.0\" enc", "oding=\"latin1\"?>", "<rss/>"]
+
+    assert chunks |> Encoding.normalize_declaration(:latin1) |> Enum.join() ==
+             "<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss/>"
+  end
+end