From 1c3b0a8e2b98fed26b48f38fb962395ef7d69945 Mon Sep 17 00:00:00 2001 From: Moises Macia Date: Tue, 23 Jun 2026 20:58:37 +0000 Subject: [PATCH] resolve relative item urls with the absolute feed_url --- lib/feed_reader/formats/atom.ex | 14 ++++++++------ lib/feed_reader/formats/rdf.ex | 14 +++++++------- lib/feed_reader/formats/rss.ex | 11 ++--------- lib/feed_reader/url.ex | 32 ++++++++++++++++++++++++++++++++ test/feed_reader/atom_test.exs | 14 ++++++++++++++ test/feed_reader/rdf_test.exs | 14 ++++++++++++++ test/feed_reader/rss_test.exs | 18 +++++++++++++++++- 7 files changed, 94 insertions(+), 23 deletions(-) create mode 100644 lib/feed_reader/url.ex diff --git a/lib/feed_reader/formats/atom.ex b/lib/feed_reader/formats/atom.ex index d3eba54..395b506 100644 --- a/lib/feed_reader/formats/atom.ex +++ b/lib/feed_reader/formats/atom.ex @@ -1,7 +1,7 @@ defmodule FeedReader.Formats.Atom do @moduledoc "Atom feed detection and normalization." @behaviour FeedReader.Format - alias FeedReader.{Feed, Item, XMLResourceLimits} + alias FeedReader.{Feed, Item, URL, XMLResourceLimits} @impl true def matches_prefix?(prefix), do: Regex.match?(~r/ Map.get("title") |> text() |> clean() content = raw |> Map.get("content") |> content() summary = raw |> Map.get("summary") |> content() || if(is_nil(content), do: title) - canonical = raw |> Map.get("link") |> List.wrap() |> find_link() |> correct_url(feed.html_url) + + canonical = + raw + |> Map.get("link") + |> List.wrap() + |> find_link() + |> URL.resolve_root_relative([feed.html_url, feed.feed_url]) author = get_in(raw, ["author", "name"]) |> text() @@ -88,10 +94,6 @@ defmodule FeedReader.Formats.Atom do defp valid_url?(url), do: is_binary(url) and url |> URI.parse() |> then(&(!is_nil(&1.scheme) and !is_nil(&1.host))) - defp correct_url("/" <> _ = url, base) when is_binary(base), - do: URI.merge(base, url) |> URI.to_string() - - defp correct_url(url, _base), do: url defp clean(nil), do: nil defp clean(value), do: value |> HtmlSanitizeEx.strip_tags() |> blank_to_nil() defp blank_to_nil(value), do: if(String.trim(value) == "", do: nil, else: String.trim(value)) diff --git a/lib/feed_reader/formats/rdf.ex b/lib/feed_reader/formats/rdf.ex index 9fafe57..6c24d2f 100644 --- a/lib/feed_reader/formats/rdf.ex +++ b/lib/feed_reader/formats/rdf.ex @@ -2,7 +2,7 @@ defmodule FeedReader.Formats.RDF do @moduledoc "RDF Site Summary format detection and normalization." @behaviour FeedReader.Format - alias FeedReader.{Feed, Item, XMLResourceLimits} + alias FeedReader.{Feed, Item, URL, XMLResourceLimits} @impl true def matches_prefix?(prefix), do: Regex.match?(~r/ Map.get("title") |> text() |> clean() description = text(raw["description"]) content = text(raw["content:encoded"]) - canonical = raw |> Map.get("link") |> text() |> correct_url(feed.html_url) + + canonical = + raw + |> Map.get("link") + |> text() + |> URL.resolve_root_relative([feed.html_url, feed.feed_url]) {:ok, %Item{ @@ -76,11 +81,6 @@ defmodule FeedReader.Formats.RDF do defp clean(nil), do: nil defp clean(value), do: value |> HtmlSanitizeEx.strip_tags() |> text() - defp correct_url("/" <> _ = url, base) when is_binary(base), - do: URI.merge(base, url) |> URI.to_string() - - defp correct_url(url, _base), do: url - defp published_at(nil), do: DateTime.utc_now(:second) defp published_at(value) do diff --git a/lib/feed_reader/formats/rss.ex b/lib/feed_reader/formats/rss.ex index 6ea0909..33d1b1f 100644 --- a/lib/feed_reader/formats/rss.ex +++ b/lib/feed_reader/formats/rss.ex @@ -3,7 +3,7 @@ defmodule FeedReader.Formats.RSS do @behaviour FeedReader.Format - alias FeedReader.{Feed, Item, XMLResourceLimits} + alias FeedReader.{Feed, Item, URL, XMLResourceLimits} @impl true def matches_prefix?(prefix) do @@ -58,7 +58,7 @@ defmodule FeedReader.Formats.RSS do content = value(item, "content:encoded") link = value(item, "link") - canonical_url = correct_url(link, feed.html_url) + canonical_url = URL.resolve_root_relative(link, [feed.html_url, feed.feed_url]) uuid = value(item, "guid") || canonical_url summary = description || if(is_nil(content), do: title) language_text = Enum.join(Enum.reject([title, content, description], &is_nil/1), "\n") @@ -91,13 +91,6 @@ defmodule FeedReader.Formats.RSS do defp clean_title(nil), do: nil defp clean_title(value), do: value |> HtmlSanitizeEx.strip_tags() |> blank_to_nil() - defp correct_url(nil, _base), do: nil - - defp correct_url("/" <> _ = url, base) when is_binary(base), - do: URI.merge(base, url) |> URI.to_string() - - defp correct_url(url, _base), do: url - defp published_at(nil), do: nil defp published_at(value) do diff --git a/lib/feed_reader/url.ex b/lib/feed_reader/url.ex new file mode 100644 index 0000000..9049e3c --- /dev/null +++ b/lib/feed_reader/url.ex @@ -0,0 +1,32 @@ +defmodule FeedReader.URL do + @moduledoc false + + @doc "Resolves root-relative URLs against the first valid absolute base." + @spec resolve_root_relative(String.t() | nil, [String.t() | nil]) :: String.t() | nil + def resolve_root_relative(nil, _bases), do: nil + + def resolve_root_relative("/" <> _ = url, bases) when is_list(bases) do + case Enum.find(bases, &absolute?/1) do + nil -> url + base -> merge(base, url) + end + end + + def resolve_root_relative(url, _bases), do: url + + defp absolute?(url) when is_binary(url) do + uri = URI.parse(url) + is_binary(uri.scheme) and is_binary(uri.host) + rescue + URI.Error -> false + end + + defp absolute?(_url), do: false + + defp merge(base, url) do + URI.merge(base, url) |> URI.to_string() + rescue + URI.Error -> url + ArgumentError -> url + end +end diff --git a/test/feed_reader/atom_test.exs b/test/feed_reader/atom_test.exs index 4ea007f..e4e4f32 100644 --- a/test/feed_reader/atom_test.exs +++ b/test/feed_reader/atom_test.exs @@ -20,6 +20,20 @@ defmodule FeedReader.AtomTest do assert_receive {:item, %Item{title: "12345", content_summary: "12345"}} end + test "preserves relative links when no absolute feed base is available" do + raw = %{"title" => "Story", "link" => [%{"href" => "/story", "rel" => "alternate"}]} + + assert {:ok, %Item{canonical_url: "/story"}} = + Atom.normalize_item(raw, %Feed{html_url: "/"}, []) + + assert {:ok, %Item{canonical_url: "https://example.com/story"}} = + Atom.normalize_item( + raw, + %Feed{html_url: "/", feed_url: "https://example.com/feed.xml"}, + [] + ) + end + test "accounts for parser failures and closes empty feeds" do handler = Atom.handler() sink = fn event -> send(self(), event) end diff --git a/test/feed_reader/rdf_test.exs b/test/feed_reader/rdf_test.exs index 17f3d44..4ec1674 100644 --- a/test/feed_reader/rdf_test.exs +++ b/test/feed_reader/rdf_test.exs @@ -34,6 +34,20 @@ defmodule FeedReader.RDFTest do assert_receive {:item, %Item{title: "Title only", content_full: nil, content_summary: nil}} end + test "preserves relative links when no absolute feed base is available" do + raw = %{"title" => "Story", "link" => "/story"} + + assert {:ok, %Item{canonical_url: "/story"}} = + RDF.normalize_item(raw, %Feed{html_url: "/"}, []) + + assert {:ok, %Item{canonical_url: "https://example.com/story"}} = + RDF.normalize_item( + raw, + %Feed{html_url: "/", feed_url: "https://example.com/feed.xml"}, + [] + ) + end + test "enforces shared SAX limits and reports malformed XML" do sink = fn _event -> :ok end diff --git a/test/feed_reader/rss_test.exs b/test/feed_reader/rss_test.exs index 9e1ecf0..87151e0 100644 --- a/test/feed_reader/rss_test.exs +++ b/test/feed_reader/rss_test.exs @@ -1,7 +1,7 @@ defmodule FeedReader.RSSTest do use ExUnit.Case, async: true - alias FeedReader.Formats.RSS + alias FeedReader.{Feed, Formats.RSS, Item} @xml ~s(ExampleLatest newshttps://example.com/NewsNewsStorystory-1/storyWriterSummaryFull text

]]>Mon, 24 Feb 2025 17:00:05 GMT https://example.com/secondnot a date ) @@ -13,6 +13,22 @@ defmodule FeedReader.RSSTest do assert {:ok, _} = parse("", sink) end + test "preserves relative links when no absolute feed base is available" do + assert {:ok, %Item{canonical_url: "/story"}} = + RSS.normalize_item( + %{"title" => "Story", "link" => "/story"}, + %Feed{html_url: "/"}, + [] + ) + + assert {:ok, %Item{canonical_url: "https://example.com/story"}} = + RSS.normalize_item( + %{"title" => "Story", "link" => "/story"}, + %Feed{html_url: "/", feed_url: "https://example.com/feed.xml"}, + [] + ) + end + test "stops malformed and over-budget input and rejects orphan SAX events" do sink = fn _event -> :ok end assert {:error, _} = parse("", sink)