116 lines
3.5 KiB
Elixir
116 lines
3.5 KiB
Elixir
defmodule FeedReader.ReqTransport do
|
|
@moduledoc """
|
|
Req implementation of `FeedReader.Transport`.
|
|
|
|
GET responses are returned as raw, lazy chunks. Automatic Req decompression
|
|
is disabled so the later download pipeline can enforce decoded-byte limits.
|
|
Status 400 and 500 responses and transport failures are retried; redirect
|
|
failures and TLS alerts are terminal. The default is five retries after the
|
|
initial request and five redirects.
|
|
"""
|
|
|
|
@behaviour FeedReader.Transport
|
|
|
|
@default_headers [
|
|
{"user-agent", "feedex/1"},
|
|
{"accept-encoding", "gzip, deflate, identity"}
|
|
]
|
|
|
|
@impl true
|
|
@spec request(String.t(), keyword()) ::
|
|
{:ok, FeedReader.Transport.response()} | {:error, term()}
|
|
def request(url, opts \\ [])
|
|
|
|
def request(url, opts) when is_binary(url) do
|
|
http_options = Keyword.get(opts, :http_options, [])
|
|
|
|
headers = Keyword.get(http_options, :headers, @default_headers)
|
|
etag = Keyword.get(opts, :previous_etag)
|
|
|
|
headers =
|
|
if is_binary(etag), do: conditional_headers(headers, etag), else: headers
|
|
|
|
request_options =
|
|
http_options
|
|
|> Keyword.put(:headers, headers)
|
|
|> Keyword.put(:into, :self)
|
|
|> Keyword.put(:raw, true)
|
|
|> Keyword.put(:decode_body, false)
|
|
|> Keyword.put(:compressed, false)
|
|
|> Keyword.put(:max_redirects, 5)
|
|
|> Keyword.put(:max_retries, retry_attempts(opts))
|
|
|> Keyword.put(:retry, &retry?/2)
|
|
|> Keyword.put(:retry_delay, retry_delay(opts))
|
|
|
|
case Req.run([url: url], request_options) do
|
|
{request, %Req.Response{} = response} ->
|
|
handle_response(request, response)
|
|
|
|
{_request, error} ->
|
|
{:error, error_reason(error)}
|
|
end
|
|
end
|
|
|
|
def request(_url, _opts), do: {:error, :invalid_url}
|
|
|
|
defp handle_response(request, response) do
|
|
case response.status do
|
|
status when status in 200..299 or status == 304 ->
|
|
{:ok,
|
|
%{
|
|
status: status,
|
|
headers: Req.get_headers_list(response),
|
|
final_url: URI.to_string(request.url),
|
|
body: if(status == 304, do: [], else: response.body)
|
|
}}
|
|
|
|
403 ->
|
|
{:error, :forbidden}
|
|
|
|
404 ->
|
|
{:error, :not_found}
|
|
|
|
429 ->
|
|
{:error, :too_many_requests}
|
|
|
|
500 ->
|
|
{:error, :internal_server_error}
|
|
|
|
status ->
|
|
{:error, {:http_status, status}}
|
|
end
|
|
end
|
|
|
|
defp conditional_headers(headers, etag) when is_binary(etag) do
|
|
headers
|
|
|> Enum.reject(fn {name, _value} -> String.downcase(name) == "if-none-match" end)
|
|
|> Kernel.++([{"if-none-match", etag}])
|
|
end
|
|
|
|
defp retry_attempts(opts) do
|
|
case Keyword.get(opts, :http_retry_attempts, 5) do
|
|
attempts when is_integer(attempts) and attempts >= 0 -> attempts
|
|
_ -> 5
|
|
end
|
|
end
|
|
|
|
defp retry_delay(opts) do
|
|
case Keyword.get(opts, :retry_delay, 0) do
|
|
delay when is_integer(delay) and delay >= 0 -> fn _ -> delay end
|
|
delay when is_function(delay, 1) -> delay
|
|
_ -> fn _ -> 0 end
|
|
end
|
|
end
|
|
|
|
defp retry?(_request, %Req.Response{status: status}), do: status in [400, 500]
|
|
defp retry?(_request, %Req.TransportError{reason: {:tls_alert, _}}), do: false
|
|
defp retry?(_request, %Req.TransportError{}), do: true
|
|
defp retry?(_request, %Req.TooManyRedirectsError{}), do: false
|
|
defp retry?(_request, _error), do: false
|
|
|
|
defp error_reason(%Req.TransportError{reason: {:tls_alert, {reason, _message}}}), do: reason
|
|
defp error_reason(%Req.TransportError{reason: reason}), do: reason
|
|
defp error_reason(%Req.TooManyRedirectsError{} = error), do: error
|
|
defp error_reason(error), do: error
|
|
end
|