-- lx-bundle:true

local convert = {}

--[[
Encoding conversion helpers for document I/O.

Scope:
- Convert UTF-16LE/UTF-16BE byte streams into UTF-8 text for in-memory editing.
- Convert UTF-8 text back into UTF-16LE/UTF-16BE byte streams for persistence.
- Preserve BOM behavior by honoring an explicit 'with_bom' flag at encode time.
- Provide a no-BOM UTF-16 endianness heuristic for best-effort detection.

Design notes:
- The implementation is dependency-free pure Lua (no 'iconv', no FFI).
- Runtime target is Lua 5.3+ with native integer and bitwise operators.
- Invalid sequences are decoded/encoded with U+FFFD replacement semantics.
- Conversion runs through chunked byte buffers to reduce allocator churn and
  avoid per-codepoint string creation overhead.
- 'string.char(table.unpack(...))' is emitted in fixed slices to avoid stack
  pressure from large argument lists across Lua variants.

Behavioral guarantees:
- UTF-16 surrogate pairs are decoded and re-encoded correctly.
- Lone/invalid surrogates are replaced with U+FFFD.
- UTF-8 validation rejects overlong forms and surrogate code points.
- Low-level 'encode_utf16(..., with_bom)' only writes a BOM when 'with_bom' is true.
- Public 'utf8_to_utf16le/be' wrappers default to BOM-on unless caller passes 'false'.
  This is intentional as a BOM-on default for editor file-I/O workflows.
]]

local REPLACEMENT_CODEPOINT = 0xFFFD
local CHUNK_BYTES = 8192
local PACK_SLICE = 256
local table_unpack = table.unpack
local str_byte = string.byte
local str_char = string.char
local table_concat = table.concat
local math_min = math.min

-- Flushes staged byte values into one or more string chunks
local function flush_bytes(parts, bytes, n)
	if n > 0 then
		local i = 1
		while i <= n do
			local j = math_min(i + PACK_SLICE - 1, n)
			parts[#parts + 1] = str_char(table_unpack(bytes, i, j))
			i = j + 1
		end
		return 0
	end
	return n
end

-- Appends one byte to the staged buffer and auto-flushes when full
local function push_byte(parts, bytes, n, b)
	n = n + 1
	bytes[n] = b
	if n >= CHUNK_BYTES then
		n = flush_bytes(parts, bytes, n)
	end
	return n
end

-- Encodes one Unicode code point as UTF-8 into the staged byte buffer
local function append_utf8(parts, bytes, n, cp)
	if cp < 0 or cp > 0x10FFFF or (cp >= 0xD800 and cp <= 0xDFFF) then
		cp = REPLACEMENT_CODEPOINT
	end
	if cp <= 0x7F then
		n = push_byte(parts, bytes, n, cp)
	elseif cp <= 0x7FF then
		n = push_byte(parts, bytes, n, 0xC0 + (cp >> 6))
		n = push_byte(parts, bytes, n, 0x80 + (cp & 0x3F))
	elseif cp <= 0xFFFF then
		n = push_byte(parts, bytes, n, 0xE0 + (cp >> 12))
		n = push_byte(parts, bytes, n, 0x80 + ((cp >> 6) & 0x3F))
		n = push_byte(parts, bytes, n, 0x80 + (cp & 0x3F))
	else
		n = push_byte(parts, bytes, n, 0xF0 + (cp >> 18))
		n = push_byte(parts, bytes, n, 0x80 + ((cp >> 12) & 0x3F))
		n = push_byte(parts, bytes, n, 0x80 + ((cp >> 6) & 0x3F))
		n = push_byte(parts, bytes, n, 0x80 + (cp & 0x3F))
	end
	return n
end

-- Appends one UTF-16LE code unit to the staged byte buffer
local function push_u16le(parts, bytes, n, u)
	n = push_byte(parts, bytes, n, u % 0x100)
	n = push_byte(parts, bytes, n, u >> 8)
	return n
end

-- Appends one UTF-16BE code unit to the staged byte buffer
local function push_u16be(parts, bytes, n, u)
	n = push_byte(parts, bytes, n, u >> 8)
	n = push_byte(parts, bytes, n, u % 0x100)
	return n
end

-- Decodes UTF-16LE bytes into UTF-8 text
local function decode_utf16le(data)
	local parts = {}
	local bytes = {}
	local nout = 0
	local i = 1
	local n = #data
	local had_errors = false

	while i <= n - 1 do
		local b1, b2 = str_byte(data, i, i + 1)
		local u = b1 + (b2 << 8)
		i = i + 2

		if u >= 0xD800 and u <= 0xDBFF then
			if i <= n - 1 then
				local c1, c2 = str_byte(data, i, i + 1)
				local u2 = c1 + (c2 << 8)
				if u2 >= 0xDC00 and u2 <= 0xDFFF then
					local cp = 0x10000 + ((u - 0xD800) * 0x400) + (u2 - 0xDC00)
					nout = append_utf8(parts, bytes, nout, cp)
					i = i + 2
				else
					-- Chosen repair strategy: emit one U+FFFD for bad high-surrogate pair
					-- and continue at the same stream position for the following code unit
					had_errors = true
					nout = append_utf8(parts, bytes, nout, REPLACEMENT_CODEPOINT)
				end
			else
				had_errors = true
				nout = append_utf8(parts, bytes, nout, REPLACEMENT_CODEPOINT)
			end
		elseif u >= 0xDC00 and u <= 0xDFFF then
			had_errors = true
			nout = append_utf8(parts, bytes, nout, REPLACEMENT_CODEPOINT)
		else
			nout = append_utf8(parts, bytes, nout, u)
		end
	end

	if i <= n then
		had_errors = true
		nout = append_utf8(parts, bytes, nout, REPLACEMENT_CODEPOINT)
	end

	nout = flush_bytes(parts, bytes, nout)
	return table_concat(parts), had_errors
end

-- Decodes UTF-16BE bytes into UTF-8 text
local function decode_utf16be(data)
	local parts = {}
	local bytes = {}
	local nout = 0
	local i = 1
	local n = #data
	local had_errors = false

	while i <= n - 1 do
		local b1, b2 = str_byte(data, i, i + 1)
		local u = b2 + (b1 << 8)
		i = i + 2

		if u >= 0xD800 and u <= 0xDBFF then
			if i <= n - 1 then
				local c1, c2 = str_byte(data, i, i + 1)
				local u2 = c2 + (c1 << 8)
				if u2 >= 0xDC00 and u2 <= 0xDFFF then
					local cp = 0x10000 + ((u - 0xD800) * 0x400) + (u2 - 0xDC00)
					nout = append_utf8(parts, bytes, nout, cp)
					i = i + 2
				else
					-- Chosen repair strategy: emit one U+FFFD for bad high-surrogate pair
					-- and continue at the same stream position for the following code unit
					had_errors = true
					nout = append_utf8(parts, bytes, nout, REPLACEMENT_CODEPOINT)
				end
			else
				had_errors = true
				nout = append_utf8(parts, bytes, nout, REPLACEMENT_CODEPOINT)
			end
		elseif u >= 0xDC00 and u <= 0xDFFF then
			had_errors = true
			nout = append_utf8(parts, bytes, nout, REPLACEMENT_CODEPOINT)
		else
			nout = append_utf8(parts, bytes, nout, u)
		end
	end

	if i <= n then
		had_errors = true
		nout = append_utf8(parts, bytes, nout, REPLACEMENT_CODEPOINT)
	end

	nout = flush_bytes(parts, bytes, nout)
	return table_concat(parts), had_errors
end

-- Decodes one UTF-8 sequence and returns code point, byte count, and validity
local function decode_one_utf8(text, i)
	local b1 = str_byte(text, i)
	if not b1 then
		return nil, 0, true
	end
	if b1 <= 0x7F then
		return b1, 1, false
	end
	local b2 = str_byte(text, i + 1)
	if b1 >= 0xC2 and b1 <= 0xDF and b2 and b2 >= 0x80 and b2 <= 0xBF then
		local cp = (b1 - 0xC0) * 0x40 + (b2 - 0x80)
		return cp, 2, false
	end
	local b3 = str_byte(text, i + 2)
	if b1 >= 0xE0 and b1 <= 0xEF and b2 and b3 and b2 >= 0x80 and b2 <= 0xBF and b3 >= 0x80 and b3 <= 0xBF then
		local cp = (b1 - 0xE0) * 0x1000 + (b2 - 0x80) * 0x40 + (b3 - 0x80)
		if cp >= 0x0800 and not (cp >= 0xD800 and cp <= 0xDFFF) then
			return cp, 3, false
		end
	end
	local b4 = str_byte(text, i + 3)
	if
		b1 >= 0xF0
		and b1 <= 0xF4
		and b2
		and b3
		and b4
		and b2 >= 0x80
		and b2 <= 0xBF
		and b3 >= 0x80
		and b3 <= 0xBF
		and b4 >= 0x80
		and b4 <= 0xBF
	then
		local cp = (b1 - 0xF0) * 0x40000 + (b2 - 0x80) * 0x1000 + (b3 - 0x80) * 0x40 + (b4 - 0x80)
		if cp >= 0x10000 and cp <= 0x10FFFF then
			return cp, 4, false
		end
	end
	return REPLACEMENT_CODEPOINT, 1, true
end

-- Encodes UTF-8 text into UTF-16 bytes (LE/BE selected by flag), optional BOM
local function encode_utf16(text, little_endian, with_bom)
	local parts = {}
	local bytes = {}
	local nout = 0
	local had_errors = false
	local push_u16 = little_endian and push_u16le or push_u16be
	local n = #text

	if with_bom then
		if little_endian then
			nout = push_byte(parts, bytes, nout, 0xFF)
			nout = push_byte(parts, bytes, nout, 0xFE)
		else
			nout = push_byte(parts, bytes, nout, 0xFE)
			nout = push_byte(parts, bytes, nout, 0xFF)
		end
	end

	local i = 1
	while i <= n do
		local cp, consumed, bad = decode_one_utf8(text, i)
		if bad then
			had_errors = true
			cp = REPLACEMENT_CODEPOINT
		end
		i = i + consumed

		if cp <= 0xFFFF then
			nout = push_u16(parts, bytes, nout, cp)
		else
			cp = cp - 0x10000
			local hi = 0xD800 + (cp >> 10)
			local lo = 0xDC00 + (cp & 0x3FF)
			nout = push_u16(parts, bytes, nout, hi)
			nout = push_u16(parts, bytes, nout, lo)
		end
	end

	nout = flush_bytes(parts, bytes, nout)
	return table_concat(parts), had_errors
end

-- Decodes UTF-16LE bytes to UTF-8 text
function convert.utf16le_to_utf8(data)
	data = data or ""
	if data:sub(1, 2) == "\255\254" then
		data = data:sub(3)
	end
	return decode_utf16le(data)
end

-- Decodes UTF-16BE bytes to UTF-8 text
function convert.utf16be_to_utf8(data)
	data = data or ""
	if data:sub(1, 2) == "\254\255" then
		data = data:sub(3)
	end
	return decode_utf16be(data)
end

-- Encodes UTF-8 text to UTF-16LE bytes
-- NOTE: Default is BOM-on unless 'with_bom == false', by design for file I/O
function convert.utf8_to_utf16le(text, with_bom)
	return encode_utf16(text or "", true, with_bom ~= false)
end

-- Encodes UTF-8 text to UTF-16BE bytes
-- NOTE: Default is BOM-on unless 'with_bom == false', by design for file I/O
function convert.utf8_to_utf16be(text, with_bom)
	return encode_utf16(text or "", false, with_bom ~= false)
end

-- Encodes UTF-8 text to UTF-16 using a target encoding id
function convert.utf8_to_utf16(text, encoding, with_bom)
	encoding = tostring(encoding or ""):lower()
	if encoding == "utf16be" or encoding == "be" then
		return convert.utf8_to_utf16be(text, with_bom)
	end
	return convert.utf8_to_utf16le(text, with_bom)
end

-- Decodes UTF-16 bytes to UTF-8 using a source encoding id
function convert.utf16_to_utf8(data, encoding)
	encoding = tostring(encoding or ""):lower()
	if encoding == "utf16be" or encoding == "be" then
		return convert.utf16be_to_utf8(data)
	end
	return convert.utf16le_to_utf8(data)
end

-- Best-effort UTF-16 endianness hint when no BOM exists
function convert.detect_utf16_endianness(data)
	local sample = (data or "")
	if sample:sub(1, 2) == "\255\254" then
		return "le"
	end
	if sample:sub(1, 2) == "\254\255" then
		return "be"
	end
	sample = sample:sub(1, 4096)
	local sample_n = #sample
	local pairs = 0
	local odd_nul = 0
	local even_nul = 0
	for i = 1, sample_n - 1, 2 do
		local b1, b2 = str_byte(sample, i, i + 1)
		if not b2 then
			break
		end
		pairs = pairs + 1
		if b1 == 0 then
			even_nul = even_nul + 1
		end
		if b2 == 0 then
			odd_nul = odd_nul + 1
		end
	end
	if pairs < 8 then
		return nil
	end
	local odd_ratio = odd_nul / pairs
	local even_ratio = even_nul / pairs
	if odd_ratio > 0.30 and even_ratio < 0.05 then
		return "le"
	end
	if even_ratio > 0.30 and odd_ratio < 0.05 then
		return "be"
	end
	return nil
end

return convert
