Atom parser

lib/feed_reader/formats/atom.ex

@@ -1,7 +1,107 @@
 defmodule FeedReader.Formats.Atom do
-  @moduledoc false
+  @moduledoc "Atom feed detection and normalization."
+  @behaviour FeedReader.Format
+  alias FeedReader.{Feed, Item, XMLResourceLimits}
 
+  @impl true
   def matches_prefix?(prefix), do: Regex.match?(~r/<feed\b/is, prefix)
-
+  @impl true
   def id, do: :atom
+  @impl true
+  def handler, do: FeedReader.Formats.Atom.Handler
+  @impl true
+  def new_parser_state(sink, opts),
+    do: %{
+      event_sink: sink,
+      opts: opts,
+      limits: XMLResourceLimits.new(opts),
+      frames: [],
+      feed: nil,
+      error: nil
+    }
+
+  @impl true
+  def normalize_feed(%{feed: data}, opts) do
+    data = if is_map(data), do: data, else: %{}
+    title = text(data["title"])
+    description = text(data["subtitle"])
+    links = List.wrap(data["link"])
+    link = find_link(links)
+    id = text(data["id"])
+
+    %Feed{
+      title: title || description || link || id,
+      description: description,
+      feed_url: Keyword.get(opts, :feed_url),
+      html_url: link || if(valid_url?(id), do: id),
+      type: :atom,
+      categories: categories(data)
+    }
+  end
+
+  @impl true
+  def normalize_item(raw, feed, _opts) do
+    title = raw |> Map.get("title") |> text() |> clean()
+    content = raw |> Map.get("content") |> content()
+    summary = raw |> Map.get("summary") |> content() || if(is_nil(content), do: title)
+    canonical = raw |> Map.get("link") |> List.wrap() |> find_link() |> correct_url(feed.html_url)
+
+    author = get_in(raw, ["author", "name"]) |> text()
+
+    date = text(raw["published"]) || text(raw["updated"])
+
+    {:ok,
+     %Item{
+       uuid: text(raw["id"]) || canonical,
+       title: title || canonical,
+       content_full: content,
+       content_summary: summary,
+       canonical_url: canonical,
+       author: author,
+       published_at: published_at(date),
+       language: language(Enum.join(Enum.reject([title, content, summary], &is_nil/1), "\n"))
+     }}
+  end
+
+  defp categories(data),
+    do:
+      for(%{"term" => value} <- List.wrap(data["category"]), value = text(value), do: value)
+      |> Enum.uniq()
+
+  defp text(%{"_text" => value}), do: text(value)
+  defp text(value) when is_binary(value), do: blank_to_nil(value)
+  defp text(_value), do: nil
+  defp content(nil), do: nil
+
+  defp content(%{"type" => "xhtml", "_markup" => markup}),
+    do: markup |> IO.iodata_to_binary() |> blank_to_nil()
+
+  defp content(value), do: text(value)
+
+  defp find_link(links) do
+    case Enum.find(links, &((is_map(&1) and &1["href"]) && &1["rel"] in [nil, "alternate"])) do
+      %{"href" => href} -> clean(href)
+      _ -> nil
+    end
+  end
+
+  defp valid_url?(url),
+    do: is_binary(url) and url |> URI.parse() |> then(&(!is_nil(&1.scheme) and !is_nil(&1.host)))
+
+  defp correct_url("/" <> _ = url, base) when is_binary(base),
+    do: URI.merge(base, url) |> URI.to_string()
+
+  defp correct_url(url, _base), do: url
+  defp clean(nil), do: nil
+  defp clean(value), do: value |> HtmlSanitizeEx.strip_tags() |> blank_to_nil()
+  defp blank_to_nil(value), do: if(String.trim(value) == "", do: nil, else: String.trim(value))
+  defp published_at(nil), do: nil
+
+  defp published_at(value), do: DateTimeParser.parse(value, to_utc: true) |> parsed_date()
+  defp parsed_date({:ok, %DateTime{} = date}), do: date
+  defp parsed_date(_result), do: nil
+
+  defp language(text), do: Tongue.detect(HtmlSanitizeEx.strip_tags(text)) |> detected_language()
+  defp detected_language([{language, _} | _]), do: language
+  defp detected_language(_languages), do: :en
 end

lib/feed_reader/formats/atom/handler.ex

@@ -0,0 +1,112 @@
+defmodule FeedReader.Formats.Atom.Handler do
+  @moduledoc false
+  @behaviour Saxy.Handler
+
+  alias FeedReader.Formats.Atom
+  alias FeedReader.XMLResourceLimits
+
+  @impl true
+  def handle_event(event, data, state) when event in [:start_document, :end_document],
+    do: account(state, event, data)
+
+  def handle_event(:start_element, {name, attrs}, state) do
+    limits = start_scope(state.limits, name)
+
+    case XMLResourceLimits.account_event(limits, :start_element, {name, attrs}) do
+      {:ok, limits} ->
+        state = %{state | limits: limits}
+        state = if name == "entry", do: emit_feed(state), else: state
+        frame = %{name: name, attrs: Map.new(attrs), children: %{}, text: "", markup: []}
+        {:ok, %{state | frames: [frame | state.frames]}}
+
+      {:error, reason} ->
+        {:stop, %{state | limits: limits, error: reason}}
+    end
+  end
+
+  def handle_event(:characters, data, state) do
+    with {:ok, limits} <- XMLResourceLimits.account_event(state.limits, :characters, data),
+         [frame | rest] <- state.frames do
+      {:ok,
+       %{
+         state
+         | limits: limits,
+           frames: [%{frame | text: frame.text <> data, markup: [frame.markup, data]} | rest]
+       }}
+    else
+      {:error, reason} -> {:stop, %{state | error: reason}}
+      [] -> {:stop, %{state | error: :invalid_sax_event}}
+    end
+  end
+
+  def handle_event(:end_element, name, state) do
+    with {:ok, limits} <- XMLResourceLimits.account_event(state.limits, :end_element, name),
+         [frame | rest] <- state.frames do
+      value = element_value(frame)
+      limits = if name == "entry", do: XMLResourceLimits.set_scope(limits, :feed), else: limits
+      state = %{state | limits: limits, frames: rest}
+      close_element(frame, value, state)
+    else
+      {:error, reason} -> {:stop, %{state | error: reason}}
+      [] -> {:stop, %{state | error: :invalid_sax_event}}
+    end
+  end
+
+  defp account(state, event, data) do
+    case XMLResourceLimits.account_event(state.limits, event, data) do
+      {:ok, limits} -> {:ok, %{state | limits: limits}}
+      {:error, reason} -> {:stop, %{state | limits: state.limits, error: reason}}
+    end
+  end
+
+  defp start_scope(limits, "feed"), do: XMLResourceLimits.set_scope(limits, :feed)
+  defp start_scope(limits, "entry"), do: XMLResourceLimits.set_scope(limits, :item)
+  defp start_scope(limits, _name), do: limits
+
+  defp emit_feed(%{feed: nil, frames: frames} = state) do
+    data = frames |> List.last() |> then(&if(is_map(&1), do: &1.children, else: %{}))
+    state = %{state | feed: data}
+    state.event_sink.({:feed, Atom.normalize_feed(state, state.opts)})
+    state
+  end
+
+  defp emit_feed(state), do: state
+
+  defp element_value(%{attrs: attrs, children: children, text: text, markup: markup}) do
+    value = Map.merge(attrs, children)
+    value = if text == "", do: value, else: Map.put(value, "_text", String.trim(text))
+    value = Map.put(value, "_markup", markup)
+    if map_size(value) == 1, do: String.trim(text), else: value
+  end
+
+  defp close_element(%{name: "entry"}, item, state) do
+    state.event_sink.(
+      {:item,
+       elem(Atom.normalize_item(item, Atom.normalize_feed(state, state.opts), state.opts), 1)}
+    )
+
+    {:ok, state}
+  end
+
+  defp close_element(%{name: "feed"}, feed, %{feed: nil} = state) do
+    state = %{state | feed: feed}
+    state.event_sink.({:feed, Atom.normalize_feed(state, state.opts)})
+    {:ok, state}
+  end
+
+  defp close_element(%{name: "feed"}, _feed, state), do: {:ok, state}
+
+  defp close_element(frame, value, %{frames: [parent | rest]} = state) do
+    children = Map.update(parent.children, frame.name, value, &(List.wrap(&1) ++ [value]))
+    markup = [parent.markup, render(frame)]
+    {:ok, %{state | frames: [%{parent | children: children, markup: markup} | rest]}}
+  end
+
+  defp close_element(_frame, _value, state), do: {:ok, state}
+
+  defp render(frame) do
+    attrs = Enum.map_join(frame.attrs, " ", fn {key, value} -> ~s(#{key}="#{value}") end)
+    opening = if attrs == "", do: "<#{frame.name}>", else: "<#{frame.name} #{attrs}>"
+    [opening, frame.markup, "</#{frame.name}>"]
+  end
+end