diff --git a/lib/feed_reader/formats/atom.ex b/lib/feed_reader/formats/atom.ex index 5eab409..d3eba54 100644 --- a/lib/feed_reader/formats/atom.ex +++ b/lib/feed_reader/formats/atom.ex @@ -1,7 +1,107 @@ defmodule FeedReader.Formats.Atom do - @moduledoc false + @moduledoc "Atom feed detection and normalization." + @behaviour FeedReader.Format + alias FeedReader.{Feed, Item, XMLResourceLimits} + @impl true def matches_prefix?(prefix), do: Regex.match?(~r/ Map.get("title") |> text() |> clean() + content = raw |> Map.get("content") |> content() + summary = raw |> Map.get("summary") |> content() || if(is_nil(content), do: title) + canonical = raw |> Map.get("link") |> List.wrap() |> find_link() |> correct_url(feed.html_url) + + author = get_in(raw, ["author", "name"]) |> text() + + date = text(raw["published"]) || text(raw["updated"]) + + {:ok, + %Item{ + uuid: text(raw["id"]) || canonical, + title: title || canonical, + content_full: content, + content_summary: summary, + canonical_url: canonical, + author: author, + published_at: published_at(date), + language: language(Enum.join(Enum.reject([title, content, summary], &is_nil/1), "\n")) + }} + end + + defp categories(data), + do: + for(%{"term" => value} <- List.wrap(data["category"]), value = text(value), do: value) + |> Enum.uniq() + + defp text(%{"_text" => value}), do: text(value) + defp text(value) when is_binary(value), do: blank_to_nil(value) + defp text(_value), do: nil + defp content(nil), do: nil + + defp content(%{"type" => "xhtml", "_markup" => markup}), + do: markup |> IO.iodata_to_binary() |> blank_to_nil() + + defp content(value), do: text(value) + + defp find_link(links) do + case Enum.find(links, &((is_map(&1) and &1["href"]) && &1["rel"] in [nil, "alternate"])) do + %{"href" => href} -> clean(href) + _ -> nil + end + end + + defp valid_url?(url), + do: is_binary(url) and url |> URI.parse() |> then(&(!is_nil(&1.scheme) and !is_nil(&1.host))) + + defp correct_url("/" <> _ = url, base) when is_binary(base), + do: URI.merge(base, url) |> URI.to_string() + + defp correct_url(url, _base), do: url + defp clean(nil), do: nil + defp clean(value), do: value |> HtmlSanitizeEx.strip_tags() |> blank_to_nil() + defp blank_to_nil(value), do: if(String.trim(value) == "", do: nil, else: String.trim(value)) + defp published_at(nil), do: nil + + defp published_at(value), do: DateTimeParser.parse(value, to_utc: true) |> parsed_date() + defp parsed_date({:ok, %DateTime{} = date}), do: date + defp parsed_date(_result), do: nil + + defp language(text), do: Tongue.detect(HtmlSanitizeEx.strip_tags(text)) |> detected_language() + defp detected_language([{language, _} | _]), do: language + defp detected_language(_languages), do: :en end diff --git a/lib/feed_reader/formats/atom/handler.ex b/lib/feed_reader/formats/atom/handler.ex new file mode 100644 index 0000000..2f25bb3 --- /dev/null +++ b/lib/feed_reader/formats/atom/handler.ex @@ -0,0 +1,112 @@ +defmodule FeedReader.Formats.Atom.Handler do + @moduledoc false + @behaviour Saxy.Handler + + alias FeedReader.Formats.Atom + alias FeedReader.XMLResourceLimits + + @impl true + def handle_event(event, data, state) when event in [:start_document, :end_document], + do: account(state, event, data) + + def handle_event(:start_element, {name, attrs}, state) do + limits = start_scope(state.limits, name) + + case XMLResourceLimits.account_event(limits, :start_element, {name, attrs}) do + {:ok, limits} -> + state = %{state | limits: limits} + state = if name == "entry", do: emit_feed(state), else: state + frame = %{name: name, attrs: Map.new(attrs), children: %{}, text: "", markup: []} + {:ok, %{state | frames: [frame | state.frames]}} + + {:error, reason} -> + {:stop, %{state | limits: limits, error: reason}} + end + end + + def handle_event(:characters, data, state) do + with {:ok, limits} <- XMLResourceLimits.account_event(state.limits, :characters, data), + [frame | rest] <- state.frames do + {:ok, + %{ + state + | limits: limits, + frames: [%{frame | text: frame.text <> data, markup: [frame.markup, data]} | rest] + }} + else + {:error, reason} -> {:stop, %{state | error: reason}} + [] -> {:stop, %{state | error: :invalid_sax_event}} + end + end + + def handle_event(:end_element, name, state) do + with {:ok, limits} <- XMLResourceLimits.account_event(state.limits, :end_element, name), + [frame | rest] <- state.frames do + value = element_value(frame) + limits = if name == "entry", do: XMLResourceLimits.set_scope(limits, :feed), else: limits + state = %{state | limits: limits, frames: rest} + close_element(frame, value, state) + else + {:error, reason} -> {:stop, %{state | error: reason}} + [] -> {:stop, %{state | error: :invalid_sax_event}} + end + end + + defp account(state, event, data) do + case XMLResourceLimits.account_event(state.limits, event, data) do + {:ok, limits} -> {:ok, %{state | limits: limits}} + {:error, reason} -> {:stop, %{state | limits: state.limits, error: reason}} + end + end + + defp start_scope(limits, "feed"), do: XMLResourceLimits.set_scope(limits, :feed) + defp start_scope(limits, "entry"), do: XMLResourceLimits.set_scope(limits, :item) + defp start_scope(limits, _name), do: limits + + defp emit_feed(%{feed: nil, frames: frames} = state) do + data = frames |> List.last() |> then(&if(is_map(&1), do: &1.children, else: %{})) + state = %{state | feed: data} + state.event_sink.({:feed, Atom.normalize_feed(state, state.opts)}) + state + end + + defp emit_feed(state), do: state + + defp element_value(%{attrs: attrs, children: children, text: text, markup: markup}) do + value = Map.merge(attrs, children) + value = if text == "", do: value, else: Map.put(value, "_text", String.trim(text)) + value = Map.put(value, "_markup", markup) + if map_size(value) == 1, do: String.trim(text), else: value + end + + defp close_element(%{name: "entry"}, item, state) do + state.event_sink.( + {:item, + elem(Atom.normalize_item(item, Atom.normalize_feed(state, state.opts), state.opts), 1)} + ) + + {:ok, state} + end + + defp close_element(%{name: "feed"}, feed, %{feed: nil} = state) do + state = %{state | feed: feed} + state.event_sink.({:feed, Atom.normalize_feed(state, state.opts)}) + {:ok, state} + end + + defp close_element(%{name: "feed"}, _feed, state), do: {:ok, state} + + defp close_element(frame, value, %{frames: [parent | rest]} = state) do + children = Map.update(parent.children, frame.name, value, &(List.wrap(&1) ++ [value])) + markup = [parent.markup, render(frame)] + {:ok, %{state | frames: [%{parent | children: children, markup: markup} | rest]}} + end + + defp close_element(_frame, _value, state), do: {:ok, state} + + defp render(frame) do + attrs = Enum.map_join(frame.attrs, " ", fn {key, value} -> ~s(#{key}="#{value}") end) + opening = if attrs == "", do: "<#{frame.name}>", else: "<#{frame.name} #{attrs}>" + [opening, frame.markup, ""] + end +end