Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ jobs:
elixir-version: 1.16
- otp-version: 27
elixir-version: 1.17
- otp-version: 27
elixir-version: 1.18
check-formatted: true
report-coverage: true
steps:
Expand Down
8 changes: 7 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ Add `logger_json` to your list of dependencies in `mix.exs`:
def deps do
[
# ...
{:logger_json, "~> 6.1"}
{:logger_json, "~> 7.0"}
# ...
]
end
Expand Down Expand Up @@ -76,6 +76,12 @@ formatter = LoggerJSON.Formatters.Basic.new(%{metadata: {:all_except, [:conn]}})
:logger.update_handler_config(:default, :formatter, formatter)
```

By default, `LoggerJSON` is using `Jason` as the JSON encoder. If you use Elixir 1.18 or later, you can
use the built-in `JSON` module as the encoder. To do this, you need to set the `:encoder` option in your
`config.exs` file. This setting is only available at compile-time:

config :logger_json, encoder: JSON

## Docs

The docs can be found at [https://hexdocs.pm/logger_json](https://hexdocs.pm/logger_json).
Expand Down
9 changes: 9 additions & 0 deletions config/test.exs
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
import Config

encoder =
if Version.compare(System.version(), "1.18.0") == :lt do
Jason
else
JSON
end

config :logger_json, encoder: encoder

config :logger,
handle_otp_reports: true,
handle_sasl_reports: false
22 changes: 16 additions & 6 deletions lib/logger_json.ex
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ defmodule LoggerJSON do
def deps do
[
# ...
{:logger_json, "~> 6.1"}
{:logger_json, "~> 7.0"}
# ...
]
end
Expand All @@ -43,7 +43,9 @@ defmodule LoggerJSON do

## Configuration

Configuration can be set using 2nd element of the tuple of the `:formatter` option in `Logger` configuration.
Configuration can be set using `new/1` helper of the formatter module,
or by setting the 2nd element of the `:formatter` option tuple in `Logger` configuration.

For example in `config.exs`:

config :logger, :default_handler,
Expand All @@ -54,18 +56,26 @@ defmodule LoggerJSON do
formatter = LoggerJSON.Formatters.Basic.new(metadata: {:all_except, [:conn]})
:logger.update_handler_config(:default, :formatter, formatter)

By default, `LoggerJSON` is using `Jason` as the JSON encoder. If you use Elixir 1.18 or later, you can
use the built-in `JSON` module as the encoder. To do this, you need to set the `:encoder` option in your
`config.exs` file. This setting is only available at compile-time:

config :logger_json, encoder: JSON

### Shared Options

Some formatters require additional configuration options. Here are the options that are common for each formatter:

* `:encoder_opts` - options to be passed directly to the JSON encoder. This allows you to customize the behavior of the JSON
encoder. See the [documentation for Jason](https://hexdocs.pm/jason/Jason.html#encode/2-options) for available options. By
default, no options are passed to the encoder.
* `:encoder_opts` - options to be passed directly to the JSON encoder. This allows you to customize the behavior
of the JSON encoder. If the encoder is `JSON`, it defaults to `JSON.protocol_encode/2`. Otherwise, defaults to
empty keywords. See the [documentation for Jason](https://hexdocs.pm/jason/Jason.html#encode/2-options) for
available options for `Jason` encoder.

* `:metadata` - a list of metadata keys to include in the log entry. By default, no metadata is included.
If `:all`is given, all metadata is included. If `{:all_except, keys}` is given, all metadata except
the specified keys is included. If `{:from_application_env, {app, module}, path}` is given, the metadata is fetched from
the application environment (eg. `{:from_application_env, {:logger, :default_formatter}, [:metadata]}`) on each logged message.
the application environment (eg. `{:from_application_env, {:logger, :default_formatter}, [:metadata]}`) during the
configuration initialization.

* `:redactors` - a list of tuples, where first element is the module that implements the `LoggerJSON.Redactor` behaviour,
and the second element is the options to pass to the redactor module. By default, no redactors are used.
Expand Down
5 changes: 3 additions & 2 deletions lib/logger_json/ecto.ex
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@ if Code.ensure_loaded?(Ecto) and Code.ensure_loaded?(:telemetry) do
@moduledoc """
A telemetry handler that logs Ecto query metrics in JSON format.

This module is not recommended to be used in production, as it can be
costly to log every single database query.
Please keep in mind that logging all database operations will have a performance impact
on your application, it's not recommended to use this module in high-throughput production
environments.
"""
require Logger

Expand Down
31 changes: 28 additions & 3 deletions lib/logger_json/formatter.ex
Original file line number Diff line number Diff line change
@@ -1,11 +1,36 @@
defmodule LoggerJSON.Formatter do
@type opts :: [
{:encoder_opts, [Jason.encode_opt()]}
{:encoder_opts, encoder_opts()}
| {:metadata, :all | {:all_except, [atom()]} | [atom()]}
| {:redactors, [{module(), term()}]}
| {atom(), term()}
]

@callback new(opts) :: {module, :logger.formatter_config()}
@callback format(:logger.log_event(), :logger.formatter_config()) :: iodata()
@type encoder_opts :: JSON.encoder() | [Jason.encode_opt()] | term()

@doc """
Creates a new configuration for the formatter.
"""
@callback new(opts) :: {module, term()}

@doc """
Formats a log event.
"""
@callback format(event :: :logger.log_event(), opts :: opts()) :: iodata()

@encoder Application.compile_env(:logger_json, :encoder, Jason)
@encoder_protocol Application.compile_env(:logger_json, :encoder_protocol) || Module.concat(@encoder, "Encoder")
@default_encoder_opts if(@encoder == JSON, do: &JSON.protocol_encode/2, else: [])

@doc false
@spec default_encoder_opts :: encoder_opts()
def default_encoder_opts, do: @default_encoder_opts

@doc false
@spec encoder :: module()
def encoder, do: @encoder

@doc false
@spec encoder_protocol :: module()
def encoder_protocol, do: @encoder_protocol
end
12 changes: 9 additions & 3 deletions lib/logger_json/formatter/redactor_encoder.ex
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
defmodule LoggerJSON.Formatter.RedactorEncoder do
@doc """
Takes a term and makes sure that it can be encoded by Jason.encode!/1 without errors
Takes a term and makes sure that it can be encoded by the encoder without errors
and without leaking sensitive information.

## Encoding rules
Expand All @@ -16,12 +16,14 @@ defmodule LoggerJSON.Formatter.RedactorEncoder do
`atom()` | unchanged | unchanged
`struct()` | converted to map | values are redacted
`keyword()` | converted to map | values are redacted
`%Jason.Fragment{}` | unchanged | unchanged
`%Jason.Fragment{}` | unchanged | unchanged if encoder is `Jason`
everything else | using `inspect/2` | unchanged
"""

@type redactor :: {redactor :: module(), redactor_opts :: term()}

@encoder_protocol LoggerJSON.Formatter.encoder_protocol()

@spec encode(term(), redactors :: [redactor()]) :: term()
def encode(nil, _redactors), do: nil
def encode(true, _redactors), do: true
Expand All @@ -31,7 +33,11 @@ defmodule LoggerJSON.Formatter.RedactorEncoder do
def encode(number, _redactors) when is_number(number), do: number
def encode("[REDACTED]", _redactors), do: "[REDACTED]"
def encode(binary, _redactors) when is_binary(binary), do: encode_binary(binary)
def encode(%Jason.Fragment{} = fragment, _redactors), do: fragment

if @encoder_protocol == Jason.Encoder do
def encode(fragment, _redactors) when is_struct(fragment, Jason.Fragment), do: fragment
end

def encode(%NaiveDateTime{} = naive_datetime, _redactors), do: naive_datetime
def encode(%DateTime{} = datetime, _redactors), do: datetime
def encode(%Date{} = date, _redactors), do: date
Expand Down
40 changes: 20 additions & 20 deletions lib/logger_json/formatters/basic.ex
Original file line number Diff line number Diff line change
Expand Up @@ -15,25 +15,27 @@ defmodule LoggerJSON.Formatters.Basic do
}
"""
import LoggerJSON.Formatter.{MapBuilder, DateTime, Message, Metadata, RedactorEncoder}
require Jason.Helpers
require LoggerJSON.Formatter, as: Formatter

@behaviour LoggerJSON.Formatter
@behaviour Formatter

@encoder Formatter.encoder()

@processed_metadata_keys ~w[otel_span_id span_id
otel_trace_id trace_id
conn]a

@impl true
@impl Formatter
def new(opts) do
opts = Keyword.new(opts)
encoder_opts = Keyword.get(opts, :encoder_opts, [])
encoder_opts = Keyword.get_lazy(opts, :encoder_opts, &Formatter.default_encoder_opts/0)
metadata_keys_or_selector = Keyword.get(opts, :metadata, [])
metadata_selector = update_metadata_selector(metadata_keys_or_selector, @processed_metadata_keys)
redactors = Keyword.get(opts, :redactors, [])
{__MODULE__, %{encoder_opts: encoder_opts, metadata: metadata_selector, redactors: redactors}}
end

@impl true
@impl Formatter
def format(%{level: level, meta: meta, msg: msg}, config) do
%{
encoder_opts: encoder_opts,
Expand All @@ -58,7 +60,7 @@ defmodule LoggerJSON.Formatters.Basic do
|> maybe_put(:request, format_http_request(meta))
|> maybe_put(:span, format_span(meta))
|> maybe_put(:trace, format_trace(meta))
|> Jason.encode_to_iodata!(encoder_opts)
|> @encoder.encode_to_iodata!(encoder_opts)

[line, "\n"]
end
Expand All @@ -84,20 +86,18 @@ defmodule LoggerJSON.Formatters.Basic do

if Code.ensure_loaded?(Plug.Conn) do
defp format_http_request(%{conn: %Plug.Conn{} = conn}) do
Jason.Helpers.json_map(
connection:
Jason.Helpers.json_map(
protocol: Plug.Conn.get_http_protocol(conn),
method: conn.method,
path: conn.request_path,
status: conn.status
),
client:
Jason.Helpers.json_map(
user_agent: LoggerJSON.Formatter.Plug.get_header(conn, "user-agent"),
ip: LoggerJSON.Formatter.Plug.remote_ip(conn)
)
)
%{
connection: %{
protocol: Plug.Conn.get_http_protocol(conn),
method: conn.method,
path: conn.request_path,
status: conn.status
},
client: %{
user_agent: Formatter.Plug.get_header(conn, "user-agent"),
ip: Formatter.Plug.remote_ip(conn)
}
}
end
end

Expand Down
75 changes: 42 additions & 33 deletions lib/logger_json/formatters/datadog.ex
Original file line number Diff line number Diff line change
Expand Up @@ -42,24 +42,26 @@ defmodule LoggerJSON.Formatters.Datadog do
}
"""
import LoggerJSON.Formatter.{MapBuilder, DateTime, Message, Metadata, Code, RedactorEncoder}
require Jason.Helpers
require LoggerJSON.Formatter, as: Formatter

@behaviour LoggerJSON.Formatter
@behaviour Formatter

@encoder Formatter.encoder()

@processed_metadata_keys ~w[pid file line mfa conn]a

@impl true
@impl Formatter
def new(opts) do
opts = Keyword.new(opts)
encoder_opts = Keyword.get(opts, :encoder_opts, [])
encoder_opts = Keyword.get_lazy(opts, :encoder_opts, &Formatter.default_encoder_opts/0)
redactors = Keyword.get(opts, :redactors, [])
hostname = Keyword.get(opts, :hostname, :system)
metadata_keys_or_selector = Keyword.get(opts, :metadata, [])
metadata_selector = update_metadata_selector(metadata_keys_or_selector, @processed_metadata_keys)
{__MODULE__, %{encoder_opts: encoder_opts, metadata: metadata_selector, redactors: redactors, hostname: hostname}}
end

@impl true
@impl Formatter
def format(%{level: level, meta: meta, msg: msg}, config) do
%{
encoder_opts: encoder_opts,
Expand Down Expand Up @@ -88,7 +90,7 @@ defmodule LoggerJSON.Formatters.Datadog do
|> maybe_merge(format_http_request(meta))
|> maybe_merge(encode(metadata, redactors))
|> maybe_merge(encode(message, redactors))
|> Jason.encode_to_iodata!(encoder_opts)
|> @encoder.encode_to_iodata!(encoder_opts)

[line, "\n"]
end
Expand Down Expand Up @@ -208,31 +210,8 @@ defmodule LoggerJSON.Formatters.Datadog do

if Code.ensure_loaded?(Plug.Conn) do
defp format_http_request(%{conn: %Plug.Conn{} = conn, duration_us: duration_us} = meta) do
request_url = Plug.Conn.request_url(conn)
user_agent = LoggerJSON.Formatter.Plug.get_header(conn, "user-agent")
remote_ip = LoggerJSON.Formatter.Plug.remote_ip(conn)
referer = LoggerJSON.Formatter.Plug.get_header(conn, "referer")

%{
http:
Jason.Helpers.json_map(
url: request_url,
status_code: conn.status,
method: conn.method,
referer: referer,
request_id: meta[:request_id],
useragent: user_agent,
url_details:
Jason.Helpers.json_map(
host: conn.host,
port: conn.port,
path: conn.request_path,
queryString: conn.query_string,
scheme: conn.scheme
)
),
network: Jason.Helpers.json_map(client: Jason.Helpers.json_map(ip: remote_ip))
}
conn
|> build_http_request_data(meta[:request_id])
|> maybe_put(:duration, to_nanosecs(duration_us))
end

Expand All @@ -241,6 +220,36 @@ defmodule LoggerJSON.Formatters.Datadog do

defp format_http_request(_meta), do: nil

defp to_nanosecs(duration_us) when is_number(duration_us), do: duration_us * 1000
defp to_nanosecs(_), do: nil
if Code.ensure_loaded?(Plug.Conn) do
defp build_http_request_data(%Plug.Conn{} = conn, request_id) do
request_url = Plug.Conn.request_url(conn)
user_agent = Formatter.Plug.get_header(conn, "user-agent")
remote_ip = Formatter.Plug.remote_ip(conn)
referer = Formatter.Plug.get_header(conn, "referer")

%{
http: %{
url: request_url,
status_code: conn.status,
method: conn.method,
referer: referer,
request_id: request_id,
useragent: user_agent,
url_details: %{
host: conn.host,
port: conn.port,
path: conn.request_path,
queryString: conn.query_string,
scheme: conn.scheme
}
},
network: %{client: %{ip: remote_ip}}
}
end
end

if Code.ensure_loaded?(Plug.Conn) do
defp to_nanosecs(duration_us) when is_number(duration_us), do: duration_us * 1000
defp to_nanosecs(_), do: nil
end
end
Loading