From 43466163104c4976814112b596e8356b590c7dbc Mon Sep 17 00:00:00 2001
From: Moises Macia <mmacia@gmail.com>
Date: Thu, 25 Jun 2026 21:12:47 +0000
Subject: [PATCH] detect language

---
 lib/feed_reader/language.ex        | 52 ++++++++++++++++++++++++++++++
 test/feed_reader/language_test.exs | 13 ++++++++
 2 files changed, 65 insertions(+)
 create mode 100644 lib/feed_reader/language.ex
 create mode 100644 test/feed_reader/language_test.exs

diff --git a/lib/feed_reader/language.ex b/lib/feed_reader/language.ex
new file mode 100644
index 0000000..2eee323
--- /dev/null
+++ b/lib/feed_reader/language.ex
@@ -0,0 +1,52 @@
+defmodule FeedReader.Language do
+  @moduledoc false
+
+  @max_detection_chars 20_000
+
+  @doc "Detects language from bounded text and falls back to English on failure."
+  @spec detect(binary() | [binary() | nil]) :: atom()
+  def detect(text) when is_binary(text), do: text |> sample() |> detect_sample()
+
+  def detect(texts) when is_list(texts), do: texts |> sample_texts() |> detect_sample()
+  def detect(_text), do: :en
+
+  defp sample_texts(texts) do
+    Enum.reduce_while(texts, "", fn
+      text, acc when is_binary(text) ->
+        append_sample(acc, text)
+
+      _text, acc ->
+        {:cont, acc}
+    end)
+  end
+
+  defp append_sample(acc, text) do
+    if String.length(acc) >= @max_detection_chars do
+      {:halt, acc}
+    else
+      separator = if acc == "", do: "", else: "\n"
+      remaining = @max_detection_chars - String.length(acc) - String.length(separator)
+      sample = String.slice(text, 0, max(remaining, 0))
+      {:cont, acc <> separator <> sample}
+    end
+  end
+
+  defp sample(text), do: String.slice(text, 0, @max_detection_chars)
+
+  defp detect_sample(text) do
+    text
+    |> HtmlSanitizeEx.strip_tags()
+    |> detect_with_tongue()
+  rescue
+    _error -> :en
+  catch
+    :exit, _reason -> :en
+  end
+
+  defp detect_with_tongue(text) do
+    case Tongue.detect(text) do
+      [{language, _score} | _rest] -> language
+      _none -> :en
+    end
+  end
+end
diff --git a/test/feed_reader/language_test.exs b/test/feed_reader/language_test.exs
new file mode 100644
index 0000000..a9aa2a2
--- /dev/null
+++ b/test/feed_reader/language_test.exs
@@ -0,0 +1,13 @@
+defmodule FeedReader.LanguageTest do
+  use ExUnit.Case, async: true
+
+  test "bounds long language detection input and returns a language" do
+    text = String.duplicate("This is a long English sentence. ", 100_000)
+
+    assert is_atom(FeedReader.Language.detect(text))
+  end
+
+  test "ignores non-text values when building a detection sample" do
+    assert FeedReader.Language.detect([nil, 123]) == :en
+  end
+end