Language detector module

lib/feed_reader/formats/atom.ex

@@ -1,7 +1,7 @@
 defmodule FeedReader.Formats.Atom do
   @moduledoc "Atom feed detection and normalization."
   @behaviour FeedReader.Format
-  alias FeedReader.{Feed, Item, URL, XMLResourceLimits}
+  alias FeedReader.{Feed, Item, Language, URL, XMLResourceLimits}
 
   @impl true
   def matches_prefix?(prefix), do: Regex.match?(~r/<feed\b/is, prefix)
@@ -65,7 +65,7 @@ defmodule FeedReader.Formats.Atom do
        canonical_url: canonical,
        author: author,
        published_at: published_at(date),
-       language: language(Enum.join(Enum.reject([title, content, summary], &is_nil/1), "\n"))
+       language: Language.detect([title, content, summary])
      }}
   end
 
@@ -102,8 +102,4 @@ defmodule FeedReader.Formats.Atom do
   defp published_at(value), do: DateTimeParser.parse(value, to_utc: true) |> parsed_date()
   defp parsed_date({:ok, %DateTime{} = date}), do: date
   defp parsed_date(_result), do: nil
-
-  defp language(text), do: Tongue.detect(HtmlSanitizeEx.strip_tags(text)) |> detected_language()
-  defp detected_language([{language, _} | _]), do: language
-  defp detected_language(_languages), do: :en
 end

lib/feed_reader/formats/rdf.ex

@@ -2,7 +2,7 @@ defmodule FeedReader.Formats.RDF do
   @moduledoc "RDF Site Summary format detection and normalization."
   @behaviour FeedReader.Format
 
-  alias FeedReader.{Feed, Item, URL, XMLResourceLimits}
+  alias FeedReader.{Feed, Item, Language, URL, XMLResourceLimits}
 
   @impl true
   def matches_prefix?(prefix), do: Regex.match?(~r/<rdf:rdf\b/is, prefix)
@@ -68,7 +68,7 @@ defmodule FeedReader.Formats.RDF do
        canonical_url: canonical,
        author: text(raw["dc:creator"]),
        published_at: published_at(text(raw["dc:date"])),
-       language: language(Enum.join(Enum.reject([title, content, description], &is_nil/1), "\n"))
+       language: Language.detect([title, content, description])
      }}
   end
 
@@ -89,11 +89,4 @@ defmodule FeedReader.Formats.RDF do
       _invalid -> DateTime.utc_now(:second)
     end
   end
-
-  defp language(text) do
-    case Tongue.detect(HtmlSanitizeEx.strip_tags(text)) do
-      [{language, _score} | _rest] -> language
-      _none -> :en
-    end
-  end
 end

lib/feed_reader/formats/rss.ex

@@ -3,7 +3,7 @@ defmodule FeedReader.Formats.RSS do
 
   @behaviour FeedReader.Format
 
-  alias FeedReader.{Feed, Item, URL, XMLResourceLimits}
+  alias FeedReader.{Feed, Item, Language, URL, XMLResourceLimits}
 
   @impl true
   def matches_prefix?(prefix) do
@@ -61,7 +61,6 @@ defmodule FeedReader.Formats.RSS do
     canonical_url = URL.resolve_root_relative(link, [feed.html_url, feed.feed_url])
     uuid = value(item, "guid") || canonical_url
     summary = description || if(is_nil(content), do: title)
-    language_text = Enum.join(Enum.reject([title, content, description], &is_nil/1), "\n")
 
     {:ok,
      %Item{
@@ -72,7 +71,7 @@ defmodule FeedReader.Formats.RSS do
        canonical_url: canonical_url,
        author: value(item, "dc:creator") || value(item, "author"),
        published_at: published_at(value(item, "pubDate") || value(item, "dc:date")),
-       language: language(language_text)
+       language: Language.detect([title, content, description])
      }}
   end
 
@@ -99,11 +98,4 @@ defmodule FeedReader.Formats.RSS do
       _invalid -> nil
     end
   end
-
-  defp language(text) do
-    case Tongue.detect(HtmlSanitizeEx.strip_tags(text)) do
-      [{language, _score} | _rest] -> language
-      _none -> :en
-    end
-  end
 end