diff --git a/lib/feed_reader/format.ex b/lib/feed_reader/format.ex index 0d4a051..50ec48e 100644 --- a/lib/feed_reader/format.ex +++ b/lib/feed_reader/format.ex @@ -5,18 +5,73 @@ defmodule FeedReader.Format do Detection examines only the bounded, replayable prefix provided by the caller. XML formats provide their dedicated Saxy handler and create parser state with an event sink; normalization returns the standalone feed and item value types. + + Callback options are keyword lists. The shared parser limits are: + + - `:max_xml_events` — non-negative integer; default `1_000_000`. + - `:max_xml_depth` — non-negative integer; default `256`. + - `:max_attributes_per_element` — non-negative integer; default `128`. + - `:max_attribute_bytes` — non-negative integer; default `65_536`. + - `:max_feed_metadata_bytes` — non-negative integer; default `1_048_576`. + - `:max_item_bytes` — non-negative integer; default `4_194_304`. + + Zero is a valid limit. Missing or invalid limit values use the defaults. A + format may define additional options; their meaning and accepted values are + documented by that format. Parser limits are consumed while building parser + state; the keyword list is also passed to the normalization callbacks. RSS + recognizes `:feed_url` as an optional URL string. """ + @typedoc "Keyword options accepted by format callbacks." + @type options :: keyword() + + @doc "Returns the stable atom identifying this format; no options are accepted." @callback id() :: atom() + + @doc """ + Tests a bounded prefix for this format. + + The prefix is at most `:max_detection_bytes` bytes. That option belongs to the + caller of format detection, not this callback; it must be a positive integer + and defaults to `65_536`. This callback accepts no options and must not read + beyond the supplied prefix. + """ @callback matches_prefix?(prefix :: binary()) :: boolean() + + @doc "Returns the module implementing `Saxy.Handler`; no options are accepted." @callback handler() :: module() - @callback new_parser_state(event_sink :: function(), opts :: keyword()) :: term() - @callback normalize_feed(state :: term(), opts :: keyword()) :: FeedReader.Feed.t() + @doc """ + Creates the initial state for `handler/0` and its event sink. + The arity-one `event_sink` receives `{:feed, feed}` and `{:item, item}` events + as values are normalized. `opts` accepts the shared parser limits documented + in this module's options section, plus options defined by the format. RSS + accepts `:feed_url` as an optional `String.t()` and forwards it to + `normalize_feed/2`. + """ + @callback new_parser_state(event_sink :: (term() -> term()), opts :: options()) :: term() + + @doc """ + Converts completed format-specific parser state into normalized feed metadata. + + `opts` accepts the shared parser limits documented in this module's options + section and format-specific options. RSS recognizes `:feed_url` as an optional + `String.t()`; when omitted, the resulting feed URL is `nil`. + """ + @callback normalize_feed(state :: term(), opts :: options()) :: FeedReader.Feed.t() + + @doc """ + Converts one raw format-specific item using normalized feed metadata. + + Return `{:ok, item}` to deliver an item, `:skip` to omit it, or + `{:error, reason}` to terminate parsing. `opts` is a keyword list containing + the shared parser limits and any format-specific options. The RSS implementation + currently uses no item-normalization options. + """ @callback normalize_item( raw_item :: map(), feed :: FeedReader.Feed.t(), - opts :: keyword() + opts :: options() ) :: {:ok, FeedReader.Item.t()} | :skip | {:error, term()} end diff --git a/lib/feed_reader/formats/rss.ex b/lib/feed_reader/formats/rss.ex index d3291d7..6ea0909 100644 --- a/lib/feed_reader/formats/rss.ex +++ b/lib/feed_reader/formats/rss.ex @@ -1,9 +1,116 @@ defmodule FeedReader.Formats.RSS do - @moduledoc false + @moduledoc "RSS 2.0 format detection and normalization." + @behaviour FeedReader.Format + + alias FeedReader.{Feed, Item, XMLResourceLimits} + + @impl true def matches_prefix?(prefix) do Regex.match?(~r/]*\bversion\s*=\s*(["'])2\.0\1/is, prefix) end + @impl true def id, do: :rss + + @impl true + def handler, do: FeedReader.Formats.RSS.Handler + + @impl true + def new_parser_state(event_sink, opts) do + %{ + event_sink: event_sink, + opts: opts, + limits: XMLResourceLimits.new(opts), + frames: [], + channel: nil, + error: nil + } + end + + @impl true + def normalize_feed(%{channel: channel}, opts) do + channel = if is_map(channel), do: channel, else: %{} + title = value(channel, "title") + description = value(channel, "description") + link = value(channel, "link") + + %Feed{ + title: clean_title(title) || description || link, + description: description, + feed_url: Keyword.get(opts, :feed_url), + html_url: link, + type: :rss, + categories: + channel + |> Map.get("category", []) + |> List.wrap() + |> Enum.map(&text/1) + |> Enum.reject(&is_nil/1) + |> Enum.uniq() + } + end + + @impl true + def normalize_item(item, feed, _opts) do + title = clean_title(value(item, "title")) + description = value(item, "description") + content = value(item, "content:encoded") + link = value(item, "link") + + canonical_url = correct_url(link, feed.html_url) + uuid = value(item, "guid") || canonical_url + summary = description || if(is_nil(content), do: title) + language_text = Enum.join(Enum.reject([title, content, description], &is_nil/1), "\n") + + {:ok, + %Item{ + uuid: uuid, + title: title || clean_title(description) || canonical_url, + content_full: content, + content_summary: summary, + canonical_url: canonical_url, + author: value(item, "dc:creator") || value(item, "author"), + published_at: published_at(value(item, "pubDate") || value(item, "dc:date")), + language: language(language_text) + }} + end + + defp value(map, key), do: map |> Map.get(key) |> text() + defp text(%{"_text" => value}), do: text(value) + defp text(value) when is_binary(value), do: blank_to_nil(value) + defp text(_value), do: nil + + defp blank_to_nil(value) do + case String.trim(value) do + "" -> nil + trimmed -> trimmed + end + end + + defp clean_title(nil), do: nil + defp clean_title(value), do: value |> HtmlSanitizeEx.strip_tags() |> blank_to_nil() + + defp correct_url(nil, _base), do: nil + + defp correct_url("/" <> _ = url, base) when is_binary(base), + do: URI.merge(base, url) |> URI.to_string() + + defp correct_url(url, _base), do: url + + defp published_at(nil), do: nil + + defp published_at(value) do + case DateTimeParser.parse(value, to_utc: true) do + {:ok, %DateTime{} = date} -> date + _invalid -> nil + end + end + + defp language(text) do + case Tongue.detect(HtmlSanitizeEx.strip_tags(text)) do + [{language, _score} | _rest] -> language + _none -> :en + end + end end diff --git a/lib/feed_reader/formats/rss/handler.ex b/lib/feed_reader/formats/rss/handler.ex new file mode 100644 index 0000000..2cbdfb7 --- /dev/null +++ b/lib/feed_reader/formats/rss/handler.ex @@ -0,0 +1,103 @@ +defmodule FeedReader.Formats.RSS.Handler do + @moduledoc false + @behaviour Saxy.Handler + + alias FeedReader.Formats.RSS + alias FeedReader.XMLResourceLimits + + @impl true + def handle_event(event, data, state) when event in [:start_document, :end_document], + do: account(state, event, data) + + def handle_event(:start_element, {name, attrs}, state) do + state = if name == "item", do: emit_feed_before_item(state), else: state + limits = scope(state.limits, name, :start_element) + + case XMLResourceLimits.account_event(limits, :start_element, {name, attrs}) do + {:ok, limits} -> + frame = %{name: name, attrs: Map.new(attrs), children: %{}, text: ""} + {:ok, %{state | limits: limits, frames: [frame | state.frames]}} + + {:error, reason} -> + {:stop, %{state | limits: limits, error: reason}} + end + end + + def handle_event(:characters, data, state) do + with {:ok, limits} <- XMLResourceLimits.account_event(state.limits, :characters, data), + [frame | rest] <- state.frames do + {:ok, %{state | limits: limits, frames: [%{frame | text: frame.text <> data} | rest]}} + else + {:error, reason} -> {:stop, %{state | error: reason}} + [] -> {:stop, %{state | error: :invalid_sax_event}} + end + end + + def handle_event(:end_element, name, state) do + with {:ok, limits} <- XMLResourceLimits.account_event(state.limits, :end_element, name), + [frame | rest] <- state.frames do + value = element_value(frame) + state = %{state | limits: scope(limits, name, :end_element), frames: rest} + close_element(frame.name, value, state) + else + {:error, reason} -> {:stop, %{state | error: reason}} + [] -> {:stop, %{state | error: :invalid_sax_event}} + end + end + + defp account(state, event, data) do + case XMLResourceLimits.account_event(state.limits, event, data) do + {:ok, limits} -> {:ok, %{state | limits: limits}} + {:error, reason} -> {:stop, %{state | error: reason}} + end + end + + defp scope(limits, "channel", :start_element), do: XMLResourceLimits.set_scope(limits, :feed) + defp scope(limits, "item", :start_element), do: XMLResourceLimits.set_scope(limits, :item) + defp scope(limits, "item", :end_element), do: XMLResourceLimits.set_scope(limits, :feed) + defp scope(limits, "channel", :end_element), do: XMLResourceLimits.set_scope(limits, :none) + defp scope(limits, _name, _event), do: limits + + defp emit_feed_before_item(%{channel: nil, frames: [%{children: children} | _]} = state) do + state = %{state | channel: children} + state.event_sink.({:feed, RSS.normalize_feed(state, state.opts)}) + state + end + + defp emit_feed_before_item(state), do: state + + defp element_value(%{attrs: attrs, children: children, text: text}) do + text = String.trim(text) + + if map_size(attrs) == 0 and map_size(children) == 0, + do: text, + else: put_text(Map.merge(attrs, children), text) + end + + defp put_text(value, ""), do: value + defp put_text(value, text), do: Map.put(value, "_text", text) + + defp close_element("item", item, state) do + state.event_sink.( + {:item, + RSS.normalize_item(item, RSS.normalize_feed(state, state.opts), state.opts) |> elem(1)} + ) + + {:ok, state} + end + + defp close_element("channel", channel, %{channel: nil} = state) do + state = %{state | channel: channel} + state.event_sink.({:feed, RSS.normalize_feed(state, state.opts)}) + {:ok, state} + end + + defp close_element("channel", _channel, state), do: {:ok, state} + + defp close_element(name, value, %{frames: [parent | rest]} = state) do + children = Map.update(parent.children, name, value, &(List.wrap(&1) ++ [value])) + {:ok, %{state | frames: [%{parent | children: children} | rest]}} + end + + defp close_element(_name, _value, state), do: {:ok, state} +end