lua-scripts-w-api/contrib/darktable-api/db.lua

-- Darktable SQLite catalog access (read-only for image queries; read-write for
-- module-parameter edits via the history table).
-- Schema inspected from an actual darktable library (recent version):
--
--  images(id, group_id, film_id, filename, width, height, aspect_ratio,
--         maker_id, model_id, lens_id, camera_id,
--         exposure, aperture, iso, focal_length, focus_distance,
--         datetime_taken INTEGER (Unix epoch),
--         flags, orientation, longitude, latitude, altitude,
--         version, max_version, write_timestamp, change_timestamp, ...)
--  makers(id, name)   models(id, name)   lens(id, name)
--  film_rolls(id, access_timestamp, folder)
--  tagged_images(imgid, tagid, position)
--  color_labels(imgid, color)   -- color: 0=red 1=yellow 2=green 3=blue 4=purple
--  data.tags(id, name, ...)     -- in data.db
--  meta_data(id, key, value)    -- key references data.meta_data.key
--
--  images.flags & 0x07  → rating (0=unrated, 1-5=stars, 6=rejected)
--  images.flags & 64    → DT_IMAGE_RAW
--  images.flags & 128   → DT_IMAGE_HDR
--  images.flags & 32    → DT_IMAGE_LDR
local sqlite3 = require "lsqlite3"
local bit     = require "bit"     -- LuaJIT built-in; Lua 5.1 bitwise
local config  = require "config"

local band  = bit.band
local unpack = unpack or table.unpack  -- Lua 5.1 vs 5.2+ compatibility

local M = {}

-- ── Connections ───────────────────────────────────────────────────────────────

local _db     -- read-only, lazy
local _db_rw  -- read-write, opened on first write

local function get_db()
  if _db then return _db end
  _db = sqlite3.open(config.library_db, sqlite3.OPEN_READONLY)
  if not _db then
    error("Cannot open darktable library: " .. config.library_db)
  end
  local data_db = config.data_db:gsub("'", "''")
  _db:exec(string.format("ATTACH DATABASE '%s' AS data", data_db))
  return _db
end

local function get_db_rw()
  if _db_rw then return _db_rw end
  _db_rw = sqlite3.open(config.library_db)   -- read-write
  if not _db_rw then
    error("Cannot open darktable library for writing: " .. config.library_db)
  end
  -- WAL mode: allow concurrent reads from the darktable process.
  _db_rw:exec("PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000;")
  return _db_rw
end

-- ── Query helpers ─────────────────────────────────────────────────────────────

local function query(sql, params)
  local d    = get_db()
  local stmt = d:prepare(sql)
  if not stmt then
    error("SQL error: " .. (d:errmsg() or "?") .. "\nSQL: " .. sql)
  end
  if params and #params > 0 then
    stmt:bind_values(unpack(params))
  end
  local rows = {}
  for row in stmt:nrows() do
    rows[#rows+1] = row
  end
  stmt:finalize()
  return rows
end

local function query_one(sql, params)
  return query(sql, params)[1]
end

-- ── Schema helpers ────────────────────────────────────────────────────────────

-- Rating is packed into the low 3 bits of flags: 0=unrated, 1-5=stars, 6=rejected
local function decode_rating(flags)
  local r = band(flags or 0, 0x07)
  return r == 6 and -1 or r
end

-- datetime_taken is stored as GLib microseconds since 0001-01-01 00:00:00 UTC.
-- Unix epoch (1970-01-01) = 62135596800 seconds after 0001-01-01.
local GLIB_EPOCH_OFFSET = 62135596800  -- seconds

local function glib_to_unix(glib_us)
  if not glib_us or glib_us == 0 then return nil end
  return math.floor(glib_us / 1000000) - GLIB_EPOCH_OFFSET
end

local function fmt_datetime(glib_us)
  local unix_ts = glib_to_unix(glib_us)
  if not unix_ts then return nil end
  return os.date("!%Y-%m-%dT%H:%M:%SZ", unix_ts)
end

local LABEL_NAMES = { [0]="red", [1]="yellow", [2]="green", [3]="blue", [4]="purple" }

local function row_to_image(row)
  if not row then return nil end
  return {
    id               = row.id,
    film_id          = row.film_id,
    group_id         = row.group_id,
    filename         = row.filename,
    path             = row.folder,
    width            = row.width,
    height           = row.height,
    aspect_ratio     = row.aspect_ratio,
    maker            = row.maker,
    model            = row.model,
    lens             = row.lens,
    exposure         = row.exposure,
    aperture         = row.aperture,
    iso              = row.iso,
    focal_length     = row.focal_length,
    datetime_taken   = fmt_datetime(row.datetime_taken),
    rating           = decode_rating(row.flags),
    is_raw           = band(row.flags or 0,  64) ~= 0,
    is_hdr           = band(row.flags or 0, 128) ~= 0,
    is_ldr           = band(row.flags or 0,  32) ~= 0,
    latitude         = row.latitude,
    longitude        = row.longitude,
    altitude         = row.altitude,
    version          = row.version,
    write_timestamp  = row.write_timestamp,
    change_timestamp = row.change_timestamp,
  }
end

-- Standard SELECT columns + JOINs used by both list_images and get_image.
local IMAGE_SELECT = [[
  SELECT i.id, i.group_id, i.film_id, i.filename,
         i.width, i.height, i.aspect_ratio,
         mk.name AS maker, mo.name AS model, ln.name AS lens,
         i.exposure, i.aperture, i.iso, i.focal_length,
         i.datetime_taken, i.flags, i.version,
         i.longitude, i.latitude, i.altitude,
         i.write_timestamp, i.change_timestamp,
         f.folder
  FROM images i
  JOIN film_rolls f ON f.id = i.film_id
  LEFT JOIN makers mk ON mk.id = i.maker_id
  LEFT JOIN models mo ON mo.id = i.model_id
  LEFT JOIN lens   ln ON ln.id = i.lens_id
]]

-- ── Filter builder ────────────────────────────────────────────────────────────

local COLOR_MAP  = { red=0, yellow=1, green=2, blue=3, purple=4 }
local VALID_SORT = {
  id=true, filename=true, datetime_taken=true, rating=true,
  focal_length=true, iso=true,
}

-- Convert a YYYY-MM-DD string to a GLib microsecond timestamp for DB comparison.
local function date_to_glib_us(date_str, end_of_day)
  local y, m, d = date_str:match("(%d%d%d%d)[%-:](%d%d)[%-:](%d%d)")
  if not y then return nil end
  local unix_ts = os.time({
    year=tonumber(y), month=tonumber(m), day=tonumber(d),
    hour = end_of_day and 23 or 0,
    min  = end_of_day and 59 or 0,
    sec  = end_of_day and 59 or 0,
  })
  return (unix_ts + GLIB_EPOCH_OFFSET) * 1000000
end

local function build_filter(filters)
  local conds  = {}
  local params = {}

  -- Star rating (packed in flags & 0x07; 6 = rejected)
  if filters.rating_min then
    local r = tonumber(filters.rating_min) or 0
    table.insert(conds,  "(i.flags & 7) >= ?")
    table.insert(params, r == -1 and 6 or r)
  end
  if filters.rating_max then
    local r = tonumber(filters.rating_max) or 5
    table.insert(conds,  "(i.flags & 7) <= ?")
    table.insert(params, r == -1 and 6 or r)
  end
  if filters.rating then
    local r = tonumber(filters.rating)
    if r then
      table.insert(conds,  "(i.flags & 7) = ?")
      table.insert(params, r == -1 and 6 or r)
    end
  end

  -- Film roll / folder
  if filters.film_id then
    table.insert(conds,  "i.film_id = ?")
    table.insert(params, tonumber(filters.film_id))
  end

  -- Date range (datetime_taken is GLib microseconds since 0001-01-01)
  if filters.from then
    local glib_us = date_to_glib_us(filters.from, false)
    if glib_us then
      table.insert(conds,  "i.datetime_taken >= ?")
      table.insert(params, glib_us)
    end
  end
  if filters.to then
    local glib_us = date_to_glib_us(filters.to, true)
    if glib_us then
      table.insert(conds,  "i.datetime_taken <= ?")
      table.insert(params, glib_us)
    end
  end

  -- Color label
  if filters.color_label then
    local cid = COLOR_MAP[filters.color_label:lower()]
    if cid ~= nil then
      table.insert(conds, "EXISTS (SELECT 1 FROM color_labels cl WHERE cl.imgid=i.id AND cl.color=?)")
      table.insert(params, cid)
    end
  end

  -- RAW / HDR flags (SQLite supports & bitwise)
  if filters.is_raw == "true"  then table.insert(conds, "(i.flags & 64) != 0") end
  if filters.is_raw == "false" then table.insert(conds, "(i.flags & 64) = 0")  end
  if filters.is_hdr == "true"  then table.insert(conds, "(i.flags & 128) != 0") end

  -- Free-text search across maker/model/lens/filename
  if filters.search and filters.search ~= "" then
    table.insert(conds, "(i.filename LIKE ? OR mk.name LIKE ? OR mo.name LIKE ? OR ln.name LIKE ?)")
    local pat = "%" .. filters.search .. "%"
    for _ = 1, 4 do table.insert(params, pat) end
  end

  -- Tag filter (correlated EXISTS; works for both data.tags and main.tags layouts)
  if filters.tag and filters.tag ~= "" then
    table.insert(conds, [[EXISTS (
      SELECT 1 FROM tagged_images ti2, data.tags t2
      WHERE ti2.imgid = i.id AND t2.id = ti2.tagid AND t2.name = ?)]])
    table.insert(params, filters.tag)
  end

  local where = #conds > 0 and ("WHERE " .. table.concat(conds, " AND ")) or ""
  return where, params
end

-- ── Public API ────────────────────────────────────────────────────────────────

function M.list_images(filters)
  filters = filters or {}
  local where, params = build_filter(filters)

  local limit   = math.max(1, math.min(tonumber(filters.limit)  or 50, 500))
  local offset  = math.max(0,           tonumber(filters.offset) or 0)
  local sort_by = VALID_SORT[filters.sort_by] and ("i." .. filters.sort_by) or "i.datetime_taken"
  local sort_dir = (filters.sort_dir == "asc") and "ASC" or "DESC"

  -- Total count (same filter, no pagination)
  local count_params = {}
  for _, v in ipairs(params) do count_params[#count_params+1] = v end

  local count_sql = string.format(
    "SELECT COUNT(*) AS total FROM images i " ..
    "JOIN film_rolls f ON f.id = i.film_id " ..
    "LEFT JOIN makers mk ON mk.id = i.maker_id " ..
    "LEFT JOIN models mo ON mo.id = i.model_id " ..
    "LEFT JOIN lens   ln ON ln.id = i.lens_id %s",
    where)
  local count_row = query_one(count_sql, count_params)
  local total     = count_row and count_row.total or 0

  -- Data rows
  local data_params = {}
  for _, v in ipairs(params) do data_params[#data_params+1] = v end
  data_params[#data_params+1] = limit
  data_params[#data_params+1] = offset

  local data_sql = IMAGE_SELECT ..
    where ..
    string.format(" ORDER BY %s %s LIMIT ? OFFSET ?", sort_by, sort_dir)

  local images = {}
  for _, row in ipairs(query(data_sql, data_params)) do
    images[#images+1] = row_to_image(row)
  end

  return { total = total, limit = limit, offset = offset, items = images }
end

function M.get_image(id)
  local row = query_one(IMAGE_SELECT .. " WHERE i.id = ?", { tonumber(id) })
  if not row then return nil end

  local img = row_to_image(row)

  -- Tags
  local tag_rows = query([[
    SELECT t.name FROM tagged_images ti
    JOIN data.tags t ON t.id = ti.tagid
    WHERE ti.imgid = ? ORDER BY t.name
  ]], { img.id })
  img.tags = {}
  for _, tr in ipairs(tag_rows) do img.tags[#img.tags+1] = tr.name end

  -- Color labels
  local cl_rows = query("SELECT color FROM color_labels WHERE imgid=?", { img.id })
  img.color_labels = {}
  for _, cl in ipairs(cl_rows) do
    if LABEL_NAMES[cl.color] then
      img.color_labels[#img.color_labels+1] = LABEL_NAMES[cl.color]
    end
  end

  -- Metadata (caption, description, creator, etc. via data.meta_data key map)
  local ok_meta, meta_rows = pcall(query, [[
    SELECT dm.tagname, m.value
    FROM meta_data m JOIN data.meta_data dm ON dm.key = m.key
    WHERE m.id = ?
  ]], { img.id })
  img.metadata = {}
  if ok_meta then
    for _, mr in ipairs(meta_rows) do
      local subkey = mr.tagname:match("%.([^%.]+)$") or mr.tagname
      img.metadata[subkey] = mr.value
    end
  end

  return img
end

-- ── Tags ──────────────────────────────────────────────────────────────────────

function M.list_tags(filters)
  filters = filters or {}
  local conds, params = {}, {}
  if filters.search and filters.search ~= "" then
    table.insert(conds, "t.name LIKE ?")
    table.insert(params, "%" .. filters.search .. "%")
  end
  local where  = #conds > 0 and ("WHERE " .. table.concat(conds, " AND ")) or ""
  local limit  = math.min(tonumber(filters.limit) or 200, 2000)
  local offset = tonumber(filters.offset) or 0
  params[#params+1] = limit
  params[#params+1] = offset

  local rows = query(string.format([[
    SELECT t.id, t.name,
           (SELECT COUNT(*) FROM tagged_images ti WHERE ti.tagid=t.id) AS image_count
    FROM data.tags t %s ORDER BY t.name LIMIT ? OFFSET ?
  ]], where), params)

  local tags = {}
  for _, r in ipairs(rows) do
    tags[#tags+1] = { id=r.id, name=r.name, image_count=r.image_count }
  end
  return tags
end

-- ── Film rolls ────────────────────────────────────────────────────────────────

function M.list_films(filters)
  filters = filters or {}
  local conds, params = {}, {}
  if filters.search and filters.search ~= "" then
    table.insert(conds, "f.folder LIKE ?")
    table.insert(params, "%" .. filters.search .. "%")
  end
  local where  = #conds > 0 and ("WHERE " .. table.concat(conds, " AND ")) or ""
  local limit  = math.min(tonumber(filters.limit) or 200, 2000)
  local offset = tonumber(filters.offset) or 0
  params[#params+1] = limit
  params[#params+1] = offset

  local rows = query(string.format([[
    SELECT f.id, f.folder, f.access_timestamp,
           (SELECT COUNT(*) FROM images i WHERE i.film_id=f.id) AS image_count
    FROM film_rolls f %s ORDER BY f.folder LIMIT ? OFFSET ?
  ]], where), params)

  local films = {}
  for _, r in ipairs(rows) do
    films[#films+1] = {
      id               = r.id,
      folder           = r.folder,
      access_timestamp = r.access_timestamp,
      image_count      = r.image_count,
    }
  end
  return films
end

-- ── Module history ────────────────────────────────────────────────────────────
-- The history table stores one or more entries per (imgid, operation).
-- Only the entry with the highest `num` is the active one.
-- `module` column = the params-struct version number for that row.

local modules_mod  -- lazy-loaded to avoid circular requires

local function get_modules_mod()
  if not modules_mod then modules_mod = require "modules" end
  return modules_mod
end

local function history_row_to_module(row)
  local mods = get_modules_mod()
  local blob  = row.op_params or ""
  local ver   = row.module or 0
  local params = mods.decode(row.operation, ver, blob)
  return {
    operation      = row.operation,
    enabled        = row.enabled ~= 0,
    module_version = ver,
    multi_name     = row.multi_name or "",
    multi_priority = row.multi_priority or 0,
    history_num    = row.num,
    params         = params,                                -- nil for unknown modules
    params_b64     = params and nil or require("base64").encode(blob),
  }
end

-- Return the active (highest-num) history entry for every operation on imgid.
function M.list_modules(imgid)
  local rows = query([[
    SELECT h.operation, h.module, h.enabled, h.op_params,
           h.multi_name, h.multi_priority, h.num
    FROM history h
    INNER JOIN (
      SELECT operation, MAX(num) AS max_num
      FROM history WHERE imgid = ?
      GROUP BY operation
    ) latest ON h.operation = latest.operation AND h.num = latest.max_num
    WHERE h.imgid = ?
    ORDER BY h.operation
  ]], { tonumber(imgid), tonumber(imgid) })

  local result = {}
  for _, row in ipairs(rows) do
    result[#result+1] = history_row_to_module(row)
  end
  return result
end

-- Return the raw op_params binary blob for one operation (latest history entry).
function M.get_module_blob(imgid, operation)
  local row = query_one([[
    SELECT op_params FROM history
    WHERE imgid = ? AND operation = ?
    ORDER BY num DESC LIMIT 1
  ]], { tonumber(imgid), operation })
  return row and row.op_params or nil
end

-- Return the single active history entry for one operation.
function M.get_module(imgid, operation)
  local row = query_one([[
    SELECT h.operation, h.module, h.enabled, h.op_params,
           h.multi_name, h.multi_priority, h.num
    FROM history h
    WHERE h.imgid = ? AND h.operation = ?
    ORDER BY h.num DESC LIMIT 1
  ]], { tonumber(imgid), operation })
  if not row then return nil end
  return history_row_to_module(row)
end

-- Update the active history entry for an operation.
-- `new_blob`   (string|nil) – new binary op_params; nil to leave unchanged
-- `new_enabled` (bool|nil)  – new enabled flag; nil to leave unchanged
-- Returns true on success, or false + error message.
function M.set_module_params(imgid, operation, new_enabled, new_blob)
  -- Find the current latest entry and its num.
  local row = query_one([[
    SELECT num, module, op_params, enabled, blendop_params, blendop_version,
           multi_priority, multi_name, multi_name_hand_edited
    FROM history
    WHERE imgid = ? AND operation = ?
    ORDER BY num DESC LIMIT 1
  ]], { tonumber(imgid), operation })

  if not row then
    return false, "module '" .. operation .. "' not found in history for image " .. imgid
  end

  local d = get_db_rw()

  -- Find the global max num for this image so we can insert after it.
  local max_row = query_one("SELECT MAX(num) AS m FROM history WHERE imgid=?", {tonumber(imgid)})
  local new_num = (max_row and max_row.m or 0) + 1

  local enabled_val = (new_enabled == nil) and row.enabled or (new_enabled and 1 or 0)
  local blob_val    = new_blob or row.op_params

  -- Insert a new history entry (preserves darktable's undo history).
  local sql = [[
    INSERT INTO history
      (imgid, num, module, operation, op_params, enabled,
       blendop_params, blendop_version, multi_priority, multi_name, multi_name_hand_edited)
    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  ]]
  local stmt = d:prepare(sql)
  if not stmt then
    return false, "SQL prepare failed: " .. (d:errmsg() or "?")
  end
  stmt:bind_values(
    tonumber(imgid), new_num, row.module, operation, blob_val, enabled_val,
    row.blendop_params, row.blendop_version,
    row.multi_priority, row.multi_name, row.multi_name_hand_edited
  )
  local rc = stmt:step()
  stmt:finalize()

  if rc ~= sqlite3.DONE then
    return false, "INSERT failed (rc=" .. tostring(rc) .. "): " .. (d:errmsg() or "?")
  end
  return true
end

return M