From 34dea7a2cadbd491ba51401eb47c5b67a68f03aa Mon Sep 17 00:00:00 2001
From: Moises Macia <mmacia@gmail.com>
Date: Mon, 29 Jun 2026 21:23:38 +0000
Subject: [PATCH] Language detector module

---
 lib/feed_reader/formats/atom.ex |  8 ++------
 lib/feed_reader/formats/rdf.ex  | 11 ++---------
 lib/feed_reader/formats/rss.ex  | 12 ++----------
 3 files changed, 6 insertions(+), 25 deletions(-)

diff --git a/lib/feed_reader/formats/atom.ex b/lib/feed_reader/formats/atom.ex
index 395b506..ec916cc 100644
--- a/lib/feed_reader/formats/atom.ex
+++ b/lib/feed_reader/formats/atom.ex
@@ -1,7 +1,7 @@
 defmodule FeedReader.Formats.Atom do
   @moduledoc "Atom feed detection and normalization."
   @behaviour FeedReader.Format
-  alias FeedReader.{Feed, Item, URL, XMLResourceLimits}
+  alias FeedReader.{Feed, Item, Language, URL, XMLResourceLimits}
 
   @impl true
   def matches_prefix?(prefix), do: Regex.match?(~r/<feed\b/is, prefix)
@@ -65,7 +65,7 @@ defmodule FeedReader.Formats.Atom do
        canonical_url: canonical,
        author: author,
        published_at: published_at(date),
-       language: language(Enum.join(Enum.reject([title, content, summary], &is_nil/1), "\n"))
+       language: Language.detect([title, content, summary])
      }}
   end
 
@@ -102,8 +102,4 @@ defmodule FeedReader.Formats.Atom do
   defp published_at(value), do: DateTimeParser.parse(value, to_utc: true) |> parsed_date()
   defp parsed_date({:ok, %DateTime{} = date}), do: date
   defp parsed_date(_result), do: nil
-
-  defp language(text), do: Tongue.detect(HtmlSanitizeEx.strip_tags(text)) |> detected_language()
-  defp detected_language([{language, _} | _]), do: language
-  defp detected_language(_languages), do: :en
 end
diff --git a/lib/feed_reader/formats/rdf.ex b/lib/feed_reader/formats/rdf.ex
index 6c24d2f..9091ab8 100644
--- a/lib/feed_reader/formats/rdf.ex
+++ b/lib/feed_reader/formats/rdf.ex
@@ -2,7 +2,7 @@ defmodule FeedReader.Formats.RDF do
   @moduledoc "RDF Site Summary format detection and normalization."
   @behaviour FeedReader.Format
 
-  alias FeedReader.{Feed, Item, URL, XMLResourceLimits}
+  alias FeedReader.{Feed, Item, Language, URL, XMLResourceLimits}
 
   @impl true
   def matches_prefix?(prefix), do: Regex.match?(~r/<rdf:rdf\b/is, prefix)
@@ -68,7 +68,7 @@ defmodule FeedReader.Formats.RDF do
        canonical_url: canonical,
        author: text(raw["dc:creator"]),
        published_at: published_at(text(raw["dc:date"])),
-       language: language(Enum.join(Enum.reject([title, content, description], &is_nil/1), "\n"))
+       language: Language.detect([title, content, description])
      }}
   end
 
@@ -89,11 +89,4 @@ defmodule FeedReader.Formats.RDF do
       _invalid -> DateTime.utc_now(:second)
     end
   end
-
-  defp language(text) do
-    case Tongue.detect(HtmlSanitizeEx.strip_tags(text)) do
-      [{language, _score} | _rest] -> language
-      _none -> :en
-    end
-  end
 end
diff --git a/lib/feed_reader/formats/rss.ex b/lib/feed_reader/formats/rss.ex
index 33d1b1f..71ff457 100644
--- a/lib/feed_reader/formats/rss.ex
+++ b/lib/feed_reader/formats/rss.ex
@@ -3,7 +3,7 @@ defmodule FeedReader.Formats.RSS do
 
   @behaviour FeedReader.Format
 
-  alias FeedReader.{Feed, Item, URL, XMLResourceLimits}
+  alias FeedReader.{Feed, Item, Language, URL, XMLResourceLimits}
 
   @impl true
   def matches_prefix?(prefix) do
@@ -61,7 +61,6 @@ defmodule FeedReader.Formats.RSS do
     canonical_url = URL.resolve_root_relative(link, [feed.html_url, feed.feed_url])
     uuid = value(item, "guid") || canonical_url
     summary = description || if(is_nil(content), do: title)
-    language_text = Enum.join(Enum.reject([title, content, description], &is_nil/1), "\n")
 
     {:ok,
      %Item{
@@ -72,7 +71,7 @@ defmodule FeedReader.Formats.RSS do
        canonical_url: canonical_url,
        author: value(item, "dc:creator") || value(item, "author"),
        published_at: published_at(value(item, "pubDate") || value(item, "dc:date")),
-       language: language(language_text)
+       language: Language.detect([title, content, description])
      }}
   end
 
@@ -99,11 +98,4 @@ defmodule FeedReader.Formats.RSS do
       _invalid -> nil
     end
   end
-
-  defp language(text) do
-    case Tongue.detect(HtmlSanitizeEx.strip_tags(text)) do
-      [{language, _score} | _rest] -> language
-      _none -> :en
-    end
-  end
 end