From d030408ab8475340f11bb69ece2ffcb575fb3bb4 Mon Sep 17 00:00:00 2001 From: Moises Macia Date: Fri, 25 Sep 2026 23:33:36 +0000 Subject: [PATCH] recovery us-ascii encoding --- lib/feed_reader/recovery/encoding.ex | 4 ++-- lib/feed_reader/recovery/recovery.ex | 3 ++- test/feed_reader/recovery/encoding_test.exs | 12 ++++++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/feed_reader/recovery/encoding.ex b/lib/feed_reader/recovery/encoding.ex index 58a42c1..cf57217 100644 --- a/lib/feed_reader/recovery/encoding.ex +++ b/lib/feed_reader/recovery/encoding.ex @@ -8,14 +8,14 @@ defmodule FeedReader.Recovery.Encoding do to UTF-8, including declarations split across input chunks. """ - def transcode(input, :utf8), do: input + def transcode(input, encoding) when encoding in [:utf8, :ascii], do: input def transcode(input, :latin1), do: Stream.map(input, &:unicode.characters_to_binary(&1, :latin1, :utf8)) def normalize_declaration(input, :utf8), do: input - def normalize_declaration(input, :latin1) do + def normalize_declaration(input, encoding) when encoding in [:latin1, :ascii] do Stream.transform(input, fn -> {:prolog, <<>>} end, &rewrite_chunk/2, &flush/1) end diff --git a/lib/feed_reader/recovery/recovery.ex b/lib/feed_reader/recovery/recovery.ex index 1b6c85e..2bb450d 100644 --- a/lib/feed_reader/recovery/recovery.ex +++ b/lib/feed_reader/recovery/recovery.ex @@ -10,7 +10,7 @@ defmodule FeedReader.Recovery do """ @doc "Returns the encoding declared by an XML prefix, defaulting to UTF-8." - @spec declared_encoding(binary()) :: :utf8 | :latin1 + @spec declared_encoding(binary()) :: :utf8 | :latin1 | :ascii def declared_encoding(prefix) do case Regex.run(~r/<\?xml\b[^>]*\bencoding\s*=\s*["']([^"']+)["']/is, prefix, capture: :all_but_first @@ -34,6 +34,7 @@ defmodule FeedReader.Recovery do defp normalize_encoding(encoding) do case String.downcase(encoding) do value when value in ["iso-8859-1", "latin1", "latin-1"] -> :latin1 + "us-ascii" -> :ascii _utf8 -> :utf8 end end diff --git a/test/feed_reader/recovery/encoding_test.exs b/test/feed_reader/recovery/encoding_test.exs index e6573ee..d159942 100644 --- a/test/feed_reader/recovery/encoding_test.exs +++ b/test/feed_reader/recovery/encoding_test.exs @@ -2,6 +2,7 @@ defmodule FeedReader.Recovery.EncodingTest do use ExUnit.Case, async: true alias FeedReader.Recovery.Encoding + alias FeedReader.Recovery test "passes UTF-8 input through unchanged" do chunks = ["", "café", ""] @@ -29,4 +30,15 @@ defmodule FeedReader.Recovery.EncodingTest do assert chunks |> Encoding.normalize_declaration(:latin1) |> Enum.join() == "" end + + test "rewrites a US-ASCII declaration to UTF-8" do + chunks = ["", ""] + + assert chunks |> Encoding.normalize_declaration(:ascii) |> Enum.join() == + "" + end + + test "recognizes US-ASCII declarations" do + assert Recovery.declared_encoding("") == :ascii + end end