inconsistent formatting of parenthesized cast expression

[matthargett]

sample code below, some supporting screenshots:

stylua reformats, but still exceeds the line limit -- unless its somehow not counting the indentation tabs as 4 spaces?

function AssertionError.new(options: AssertionErrorOptions): AssertionError
	if message ~= nil then
		self = (setmetatable(Error.new(tostring(message)), AssertionError) :: any) :: AssertionError
	else
		if operator == "deepStrictEqual" or operator == "strictEqual" then
			-- StyLua does NOT break this up correctly
			self = (setmetatable(Error.new(createErrDiff(actual, expected, operator)), AssertionError) :: any) :: AssertionError
		elseif operator == "notDeepStrictEqual" or operator == "notStrictEqual" then
			if #res == 1 then
				-- StyLua DOES break this up correctly
				self = (
					setmetatable(
						Error.new(("%s%s%s"):format(base, if string.len(res[1]) > 5 then "\n\n" else " ", res[1])),
						AssertionError
					) :: any
				) :: AssertionError
			else
				-- StyLua DOES break this up correctly
				self = (
					setmetatable(Error.new(("%s\n\n%s\n"):format(base, Array.join(res, "\n"))), AssertionError) :: any
				) :: AssertionError
			end
		end
	end
end

[matthargett]

when I try to manually format the line that doesn't format correctly, it undoes my good formatting: