RSS parser

lib/feed_reader/format.ex

@@ -5,18 +5,73 @@ defmodule FeedReader.Format do
   Detection examines only the bounded, replayable prefix provided by the caller.
   XML formats provide their dedicated Saxy handler and create parser state with
   an event sink; normalization returns the standalone feed and item value types.
+
+  Callback options are keyword lists. The shared parser limits are:
+
+  - `:max_xml_events` — non-negative integer; default `1_000_000`.
+  - `:max_xml_depth` — non-negative integer; default `256`.
+  - `:max_attributes_per_element` — non-negative integer; default `128`.
+  - `:max_attribute_bytes` — non-negative integer; default `65_536`.
+  - `:max_feed_metadata_bytes` — non-negative integer; default `1_048_576`.
+  - `:max_item_bytes` — non-negative integer; default `4_194_304`.
+
+  Zero is a valid limit. Missing or invalid limit values use the defaults. A
+  format may define additional options; their meaning and accepted values are
+  documented by that format. Parser limits are consumed while building parser
+  state; the keyword list is also passed to the normalization callbacks. RSS
+  recognizes `:feed_url` as an optional URL string.
   """
 
+  @typedoc "Keyword options accepted by format callbacks."
+  @type options :: keyword()
+
+  @doc "Returns the stable atom identifying this format; no options are accepted."
   @callback id() :: atom()
+
+  @doc """
+  Tests a bounded prefix for this format.
+
+  The prefix is at most `:max_detection_bytes` bytes. That option belongs to the
+  caller of format detection, not this callback; it must be a positive integer
+  and defaults to `65_536`. This callback accepts no options and must not read
+  beyond the supplied prefix.
+  """
   @callback matches_prefix?(prefix :: binary()) :: boolean()
+
+  @doc "Returns the module implementing `Saxy.Handler`; no options are accepted."
   @callback handler() :: module()
-  @callback new_parser_state(event_sink :: function(), opts :: keyword()) :: term()
 
-  @callback normalize_feed(state :: term(), opts :: keyword()) :: FeedReader.Feed.t()
+  @doc """
+  Creates the initial state for `handler/0` and its event sink.
 
+  The arity-one `event_sink` receives `{:feed, feed}` and `{:item, item}` events
+  as values are normalized. `opts` accepts the shared parser limits documented
+  in this module's options section, plus options defined by the format. RSS
+  accepts `:feed_url` as an optional `String.t()` and forwards it to
+  `normalize_feed/2`.
+  """
+  @callback new_parser_state(event_sink :: (term() -> term()), opts :: options()) :: term()
+
+  @doc """
+  Converts completed format-specific parser state into normalized feed metadata.
+
+  `opts` accepts the shared parser limits documented in this module's options
+  section and format-specific options. RSS recognizes `:feed_url` as an optional
+  `String.t()`; when omitted, the resulting feed URL is `nil`.
+  """
+  @callback normalize_feed(state :: term(), opts :: options()) :: FeedReader.Feed.t()
+
+  @doc """
+  Converts one raw format-specific item using normalized feed metadata.
+
+  Return `{:ok, item}` to deliver an item, `:skip` to omit it, or
+  `{:error, reason}` to terminate parsing. `opts` is a keyword list containing
+  the shared parser limits and any format-specific options. The RSS implementation
+  currently uses no item-normalization options.
+  """
   @callback normalize_item(
               raw_item :: map(),
               feed :: FeedReader.Feed.t(),
-              opts :: keyword()
+              opts :: options()
             ) :: {:ok, FeedReader.Item.t()} | :skip | {:error, term()}
 end

lib/feed_reader/formats/rss.ex

@@ -1,9 +1,116 @@
 defmodule FeedReader.Formats.RSS do
-  @moduledoc false
+  @moduledoc "RSS 2.0 format detection and normalization."
 
+  @behaviour FeedReader.Format
+
+  alias FeedReader.{Feed, Item, XMLResourceLimits}
+
+  @impl true
   def matches_prefix?(prefix) do
     Regex.match?(~r/<rss\b[^>]*\bversion\s*=\s*(["'])2\.0\1/is, prefix)
   end
 
+  @impl true
   def id, do: :rss
+
+  @impl true
+  def handler, do: FeedReader.Formats.RSS.Handler
+
+  @impl true
+  def new_parser_state(event_sink, opts) do
+    %{
+      event_sink: event_sink,
+      opts: opts,
+      limits: XMLResourceLimits.new(opts),
+      frames: [],
+      channel: nil,
+      error: nil
+    }
+  end
+
+  @impl true
+  def normalize_feed(%{channel: channel}, opts) do
+    channel = if is_map(channel), do: channel, else: %{}
+    title = value(channel, "title")
+    description = value(channel, "description")
+    link = value(channel, "link")
+
+    %Feed{
+      title: clean_title(title) || description || link,
+      description: description,
+      feed_url: Keyword.get(opts, :feed_url),
+      html_url: link,
+      type: :rss,
+      categories:
+        channel
+        |> Map.get("category", [])
+        |> List.wrap()
+        |> Enum.map(&text/1)
+        |> Enum.reject(&is_nil/1)
+        |> Enum.uniq()
+    }
+  end
+
+  @impl true
+  def normalize_item(item, feed, _opts) do
+    title = clean_title(value(item, "title"))
+    description = value(item, "description")
+    content = value(item, "content:encoded")
+    link = value(item, "link")
+
+    canonical_url = correct_url(link, feed.html_url)
+    uuid = value(item, "guid") || canonical_url
+    summary = description || if(is_nil(content), do: title)
+    language_text = Enum.join(Enum.reject([title, content, description], &is_nil/1), "\n")
+
+    {:ok,
+     %Item{
+       uuid: uuid,
+       title: title || clean_title(description) || canonical_url,
+       content_full: content,
+       content_summary: summary,
+       canonical_url: canonical_url,
+       author: value(item, "dc:creator") || value(item, "author"),
+       published_at: published_at(value(item, "pubDate") || value(item, "dc:date")),
+       language: language(language_text)
+     }}
+  end
+
+  defp value(map, key), do: map |> Map.get(key) |> text()
+  defp text(%{"_text" => value}), do: text(value)
+  defp text(value) when is_binary(value), do: blank_to_nil(value)
+  defp text(_value), do: nil
+
+  defp blank_to_nil(value) do
+    case String.trim(value) do
+      "" -> nil
+      trimmed -> trimmed
+    end
+  end
+
+  defp clean_title(nil), do: nil
+  defp clean_title(value), do: value |> HtmlSanitizeEx.strip_tags() |> blank_to_nil()
+
+  defp correct_url(nil, _base), do: nil
+
+  defp correct_url("/" <> _ = url, base) when is_binary(base),
+    do: URI.merge(base, url) |> URI.to_string()
+
+  defp correct_url(url, _base), do: url
+
+  defp published_at(nil), do: nil
+
+  defp published_at(value) do
+    case DateTimeParser.parse(value, to_utc: true) do
+      {:ok, %DateTime{} = date} -> date
+      _invalid -> nil
+    end
+  end
+
+  defp language(text) do
+    case Tongue.detect(HtmlSanitizeEx.strip_tags(text)) do
+      [{language, _score} | _rest] -> language
+      _none -> :en
+    end
+  end
 end

lib/feed_reader/formats/rss/handler.ex

@@ -0,0 +1,103 @@
+defmodule FeedReader.Formats.RSS.Handler do
+  @moduledoc false
+  @behaviour Saxy.Handler
+
+  alias FeedReader.Formats.RSS
+  alias FeedReader.XMLResourceLimits
+
+  @impl true
+  def handle_event(event, data, state) when event in [:start_document, :end_document],
+    do: account(state, event, data)
+
+  def handle_event(:start_element, {name, attrs}, state) do
+    state = if name == "item", do: emit_feed_before_item(state), else: state
+    limits = scope(state.limits, name, :start_element)
+
+    case XMLResourceLimits.account_event(limits, :start_element, {name, attrs}) do
+      {:ok, limits} ->
+        frame = %{name: name, attrs: Map.new(attrs), children: %{}, text: ""}
+        {:ok, %{state | limits: limits, frames: [frame | state.frames]}}
+
+      {:error, reason} ->
+        {:stop, %{state | limits: limits, error: reason}}
+    end
+  end
+
+  def handle_event(:characters, data, state) do
+    with {:ok, limits} <- XMLResourceLimits.account_event(state.limits, :characters, data),
+         [frame | rest] <- state.frames do
+      {:ok, %{state | limits: limits, frames: [%{frame | text: frame.text <> data} | rest]}}
+    else
+      {:error, reason} -> {:stop, %{state | error: reason}}
+      [] -> {:stop, %{state | error: :invalid_sax_event}}
+    end
+  end
+
+  def handle_event(:end_element, name, state) do
+    with {:ok, limits} <- XMLResourceLimits.account_event(state.limits, :end_element, name),
+         [frame | rest] <- state.frames do
+      value = element_value(frame)
+      state = %{state | limits: scope(limits, name, :end_element), frames: rest}
+      close_element(frame.name, value, state)
+    else
+      {:error, reason} -> {:stop, %{state | error: reason}}
+      [] -> {:stop, %{state | error: :invalid_sax_event}}
+    end
+  end
+
+  defp account(state, event, data) do
+    case XMLResourceLimits.account_event(state.limits, event, data) do
+      {:ok, limits} -> {:ok, %{state | limits: limits}}
+      {:error, reason} -> {:stop, %{state | error: reason}}
+    end
+  end
+
+  defp scope(limits, "channel", :start_element), do: XMLResourceLimits.set_scope(limits, :feed)
+  defp scope(limits, "item", :start_element), do: XMLResourceLimits.set_scope(limits, :item)
+  defp scope(limits, "item", :end_element), do: XMLResourceLimits.set_scope(limits, :feed)
+  defp scope(limits, "channel", :end_element), do: XMLResourceLimits.set_scope(limits, :none)
+  defp scope(limits, _name, _event), do: limits
+
+  defp emit_feed_before_item(%{channel: nil, frames: [%{children: children} | _]} = state) do
+    state = %{state | channel: children}
+    state.event_sink.({:feed, RSS.normalize_feed(state, state.opts)})
+    state
+  end
+
+  defp emit_feed_before_item(state), do: state
+
+  defp element_value(%{attrs: attrs, children: children, text: text}) do
+    text = String.trim(text)
+
+    if map_size(attrs) == 0 and map_size(children) == 0,
+      do: text,
+      else: put_text(Map.merge(attrs, children), text)
+  end
+
+  defp put_text(value, ""), do: value
+  defp put_text(value, text), do: Map.put(value, "_text", text)
+
+  defp close_element("item", item, state) do
+    state.event_sink.(
+      {:item,
+       RSS.normalize_item(item, RSS.normalize_feed(state, state.opts), state.opts) |> elem(1)}
+    )
+
+    {:ok, state}
+  end
+
+  defp close_element("channel", channel, %{channel: nil} = state) do
+    state = %{state | channel: channel}
+    state.event_sink.({:feed, RSS.normalize_feed(state, state.opts)})
+    {:ok, state}
+  end
+
+  defp close_element("channel", _channel, state), do: {:ok, state}
+
+  defp close_element(name, value, %{frames: [parent | rest]} = state) do
+    children = Map.update(parent.children, name, value, &(List.wrap(&1) ++ [value]))
+    {:ok, %{state | frames: [%{parent | children: children} | rest]}}
+  end
+
+  defp close_element(_name, _value, state), do: {:ok, state}
+end