RDF parser

lib/feed_reader/formats/rdf.ex

@@ -1,7 +1,99 @@
 defmodule FeedReader.Formats.RDF do
-  @moduledoc false
+  @moduledoc "RDF Site Summary format detection and normalization."
+  @behaviour FeedReader.Format
 
+  alias FeedReader.{Feed, Item, XMLResourceLimits}
+
+  @impl true
   def matches_prefix?(prefix), do: Regex.match?(~r/<rdf:rdf\b/is, prefix)
 
+  @impl true
   def id, do: :rdf
+
+  @impl true
+  def handler, do: FeedReader.Formats.RDF.Handler
+
+  @impl true
+  def new_parser_state(event_sink, opts),
+    do: %{
+      event_sink: event_sink,
+      opts: opts,
+      limits: XMLResourceLimits.new(opts),
+      frames: [],
+      channel: nil,
+      error: nil
+    }
+
+  @impl true
+  def normalize_feed(%{channel: channel}, opts) do
+    channel = if is_map(channel), do: channel, else: %{}
+    title = text(channel["title"])
+    description = text(channel["description"])
+    link = text(channel["link"])
+
+    %Feed{
+      title: title || description || link,
+      description: description,
+      feed_url: Keyword.get(opts, :feed_url),
+      html_url: link,
+      type: :rdf,
+      categories:
+        channel
+        |> Map.get("dc:subject", [])
+        |> List.wrap()
+        |> Enum.map(&text/1)
+        |> Enum.reject(&is_nil/1)
+        |> Enum.uniq()
+    }
+  end
+
+  @impl true
+  def normalize_item(raw, feed, _opts) do
+    title = raw |> Map.get("title") |> text() |> clean()
+    description = text(raw["description"])
+    content = text(raw["content:encoded"])
+    canonical = raw |> Map.get("link") |> text() |> correct_url(feed.html_url)
+
+    {:ok,
+     %Item{
+       uuid: text(raw["guid"]) || text(raw["id"]) || canonical,
+       title: title,
+       content_full: content,
+       content_summary: description,
+       canonical_url: canonical,
+       author: text(raw["dc:creator"]),
+       published_at: published_at(text(raw["dc:date"])),
+       language: language(Enum.join(Enum.reject([title, content, description], &is_nil/1), "\n"))
+     }}
+  end
+
+  defp text(%{"_text" => value}), do: text(value)
+
+  defp text(value) when is_binary(value),
+    do: if(String.trim(value) == "", do: nil, else: String.trim(value))
+
+  defp text(_value), do: nil
+  defp clean(nil), do: nil
+  defp clean(value), do: value |> HtmlSanitizeEx.strip_tags() |> text()
+
+  defp correct_url("/" <> _ = url, base) when is_binary(base),
+    do: URI.merge(base, url) |> URI.to_string()
+
+  defp correct_url(url, _base), do: url
+
+  defp published_at(nil), do: DateTime.utc_now(:second)
+
+  defp published_at(value) do
+    case DateTimeParser.parse(value, to_utc: true) do
+      {:ok, %DateTime{} = date} -> date
+      _invalid -> DateTime.utc_now(:second)
+    end
+  end
+
+  defp language(text) do
+    case Tongue.detect(HtmlSanitizeEx.strip_tags(text)) do
+      [{language, _score} | _rest] -> language
+      _none -> :en
+    end
+  end
 end

lib/feed_reader/formats/rdf/handler.ex

@@ -0,0 +1,100 @@
+defmodule FeedReader.Formats.RDF.Handler do
+  @moduledoc false
+  @behaviour Saxy.Handler
+
+  alias FeedReader.Formats.RDF
+  alias FeedReader.XMLResourceLimits
+
+  @impl true
+  def handle_event(event, data, state) when event in [:start_document, :end_document],
+    do: account(state, event, data)
+
+  def handle_event(:start_element, {name, attrs}, state) do
+    state = if name == "item", do: emit_feed_before_item(state), else: state
+    limits = scope(state.limits, name, :start_element)
+
+    case XMLResourceLimits.account_event(limits, :start_element, {name, attrs}) do
+      {:ok, limits} ->
+        frame = %{name: name, attrs: Map.new(attrs), children: %{}, text: ""}
+        {:ok, %{state | limits: limits, frames: [frame | state.frames]}}
+
+      {:error, reason} ->
+        {:stop, %{state | limits: limits, error: reason}}
+    end
+  end
+
+  def handle_event(event, data, state) when event in [:characters, :cdata] do
+    with {:ok, limits} <- XMLResourceLimits.account_event(state.limits, event, data),
+         [frame | rest] <- state.frames do
+      {:ok, %{state | limits: limits, frames: [%{frame | text: frame.text <> data} | rest]}}
+    else
+      {:error, reason} -> {:stop, %{state | error: reason}}
+      [] -> {:stop, %{state | error: :invalid_sax_event}}
+    end
+  end
+
+  def handle_event(:end_element, name, state) do
+    with {:ok, limits} <- XMLResourceLimits.account_event(state.limits, :end_element, name),
+         [frame | rest] <- state.frames do
+      state = %{state | limits: scope(limits, name, :end_element), frames: rest}
+      close_element(frame, element_value(frame), state)
+    else
+      {:error, reason} -> {:stop, %{state | error: reason}}
+      [] -> {:stop, %{state | error: :invalid_sax_event}}
+    end
+  end
+
+  defp account(state, event, data) do
+    case XMLResourceLimits.account_event(state.limits, event, data) do
+      {:ok, limits} -> {:ok, %{state | limits: limits}}
+      {:error, reason} -> {:stop, %{state | error: reason}}
+    end
+  end
+
+  defp scope(limits, "channel", :start_element), do: XMLResourceLimits.set_scope(limits, :feed)
+  defp scope(limits, "item", :start_element), do: XMLResourceLimits.set_scope(limits, :item)
+  defp scope(limits, "item", :end_element), do: XMLResourceLimits.set_scope(limits, :feed)
+  defp scope(limits, "channel", :end_element), do: XMLResourceLimits.set_scope(limits, :none)
+  defp scope(limits, _name, _event), do: limits
+
+  defp emit_feed_before_item(%{channel: nil, frames: [root | _]} = state) do
+    channel = Map.get(root.children, "channel", %{})
+    state = %{state | channel: channel}
+    state.event_sink.({:feed, RDF.normalize_feed(state, state.opts)})
+    state
+  end
+
+  defp emit_feed_before_item(state), do: state
+
+  defp element_value(%{attrs: attrs, children: children, text: text}) do
+    text = String.trim(text)
+
+    if map_size(attrs) == 0 and map_size(children) == 0,
+      do: text,
+      else: put_text(Map.merge(attrs, children), text)
+  end
+
+  defp put_text(value, ""), do: value
+  defp put_text(value, text), do: Map.put(value, "_text", text)
+
+  defp close_element(%{name: "item"}, item, state) do
+    {:ok, normalized} =
+      RDF.normalize_item(item, RDF.normalize_feed(state, state.opts), state.opts)
+
+    state.event_sink.({:item, normalized})
+    {:ok, state}
+  end
+
+  defp close_element(%{name: "channel"}, channel, %{channel: nil} = state) do
+    state = %{state | channel: channel}
+    state.event_sink.({:feed, RDF.normalize_feed(state, state.opts)})
+    {:ok, state}
+  end
+
+  defp close_element(frame, value, %{frames: [parent | rest]} = state) do
+    children = Map.update(parent.children, frame.name, value, &(List.wrap(&1) ++ [value]))
+    {:ok, %{state | frames: [%{parent | children: children} | rest]}}
+  end
+
+  defp close_element(_frame, _value, state), do: {:ok, state}
+end