diff --git a/lib/feed_reader/formats/rdf.ex b/lib/feed_reader/formats/rdf.ex index 724286b..9fafe57 100644 --- a/lib/feed_reader/formats/rdf.ex +++ b/lib/feed_reader/formats/rdf.ex @@ -1,7 +1,99 @@ defmodule FeedReader.Formats.RDF do - @moduledoc false + @moduledoc "RDF Site Summary format detection and normalization." + @behaviour FeedReader.Format + alias FeedReader.{Feed, Item, XMLResourceLimits} + + @impl true def matches_prefix?(prefix), do: Regex.match?(~r/ Map.get("dc:subject", []) + |> List.wrap() + |> Enum.map(&text/1) + |> Enum.reject(&is_nil/1) + |> Enum.uniq() + } + end + + @impl true + def normalize_item(raw, feed, _opts) do + title = raw |> Map.get("title") |> text() |> clean() + description = text(raw["description"]) + content = text(raw["content:encoded"]) + canonical = raw |> Map.get("link") |> text() |> correct_url(feed.html_url) + + {:ok, + %Item{ + uuid: text(raw["guid"]) || text(raw["id"]) || canonical, + title: title, + content_full: content, + content_summary: description, + canonical_url: canonical, + author: text(raw["dc:creator"]), + published_at: published_at(text(raw["dc:date"])), + language: language(Enum.join(Enum.reject([title, content, description], &is_nil/1), "\n")) + }} + end + + defp text(%{"_text" => value}), do: text(value) + + defp text(value) when is_binary(value), + do: if(String.trim(value) == "", do: nil, else: String.trim(value)) + + defp text(_value), do: nil + defp clean(nil), do: nil + defp clean(value), do: value |> HtmlSanitizeEx.strip_tags() |> text() + + defp correct_url("/" <> _ = url, base) when is_binary(base), + do: URI.merge(base, url) |> URI.to_string() + + defp correct_url(url, _base), do: url + + defp published_at(nil), do: DateTime.utc_now(:second) + + defp published_at(value) do + case DateTimeParser.parse(value, to_utc: true) do + {:ok, %DateTime{} = date} -> date + _invalid -> DateTime.utc_now(:second) + end + end + + defp language(text) do + case Tongue.detect(HtmlSanitizeEx.strip_tags(text)) do + [{language, _score} | _rest] -> language + _none -> :en + end + end end diff --git a/lib/feed_reader/formats/rdf/handler.ex b/lib/feed_reader/formats/rdf/handler.ex new file mode 100644 index 0000000..2c727d4 --- /dev/null +++ b/lib/feed_reader/formats/rdf/handler.ex @@ -0,0 +1,100 @@ +defmodule FeedReader.Formats.RDF.Handler do + @moduledoc false + @behaviour Saxy.Handler + + alias FeedReader.Formats.RDF + alias FeedReader.XMLResourceLimits + + @impl true + def handle_event(event, data, state) when event in [:start_document, :end_document], + do: account(state, event, data) + + def handle_event(:start_element, {name, attrs}, state) do + state = if name == "item", do: emit_feed_before_item(state), else: state + limits = scope(state.limits, name, :start_element) + + case XMLResourceLimits.account_event(limits, :start_element, {name, attrs}) do + {:ok, limits} -> + frame = %{name: name, attrs: Map.new(attrs), children: %{}, text: ""} + {:ok, %{state | limits: limits, frames: [frame | state.frames]}} + + {:error, reason} -> + {:stop, %{state | limits: limits, error: reason}} + end + end + + def handle_event(event, data, state) when event in [:characters, :cdata] do + with {:ok, limits} <- XMLResourceLimits.account_event(state.limits, event, data), + [frame | rest] <- state.frames do + {:ok, %{state | limits: limits, frames: [%{frame | text: frame.text <> data} | rest]}} + else + {:error, reason} -> {:stop, %{state | error: reason}} + [] -> {:stop, %{state | error: :invalid_sax_event}} + end + end + + def handle_event(:end_element, name, state) do + with {:ok, limits} <- XMLResourceLimits.account_event(state.limits, :end_element, name), + [frame | rest] <- state.frames do + state = %{state | limits: scope(limits, name, :end_element), frames: rest} + close_element(frame, element_value(frame), state) + else + {:error, reason} -> {:stop, %{state | error: reason}} + [] -> {:stop, %{state | error: :invalid_sax_event}} + end + end + + defp account(state, event, data) do + case XMLResourceLimits.account_event(state.limits, event, data) do + {:ok, limits} -> {:ok, %{state | limits: limits}} + {:error, reason} -> {:stop, %{state | error: reason}} + end + end + + defp scope(limits, "channel", :start_element), do: XMLResourceLimits.set_scope(limits, :feed) + defp scope(limits, "item", :start_element), do: XMLResourceLimits.set_scope(limits, :item) + defp scope(limits, "item", :end_element), do: XMLResourceLimits.set_scope(limits, :feed) + defp scope(limits, "channel", :end_element), do: XMLResourceLimits.set_scope(limits, :none) + defp scope(limits, _name, _event), do: limits + + defp emit_feed_before_item(%{channel: nil, frames: [root | _]} = state) do + channel = Map.get(root.children, "channel", %{}) + state = %{state | channel: channel} + state.event_sink.({:feed, RDF.normalize_feed(state, state.opts)}) + state + end + + defp emit_feed_before_item(state), do: state + + defp element_value(%{attrs: attrs, children: children, text: text}) do + text = String.trim(text) + + if map_size(attrs) == 0 and map_size(children) == 0, + do: text, + else: put_text(Map.merge(attrs, children), text) + end + + defp put_text(value, ""), do: value + defp put_text(value, text), do: Map.put(value, "_text", text) + + defp close_element(%{name: "item"}, item, state) do + {:ok, normalized} = + RDF.normalize_item(item, RDF.normalize_feed(state, state.opts), state.opts) + + state.event_sink.({:item, normalized}) + {:ok, state} + end + + defp close_element(%{name: "channel"}, channel, %{channel: nil} = state) do + state = %{state | channel: channel} + state.event_sink.({:feed, RDF.normalize_feed(state, state.opts)}) + {:ok, state} + end + + defp close_element(frame, value, %{frames: [parent | rest]} = state) do + children = Map.update(parent.children, frame.name, value, &(List.wrap(&1) ++ [value])) + {:ok, %{state | frames: [%{parent | children: children} | rest]}} + end + + defp close_element(_frame, _value, state), do: {:ok, state} +end