lua-scripts-w-api/contrib/darktable-api/darktable-api-bridge.lua

--[[
  darktable-api-bridge.lua
  Companion plugin for the darktable REST API server.

  Runs inside a live darktable instance and watches two queue files written by
  the standalone LuaJIT server.

    Refresh queue  (DT_REFRESH_QUEUE, default /tmp/dt-api-refresh.queue)
      One image ID per line.  The bridge calls image:drop_cache() +
      image:generate_cache(true, …) so that darktable re-renders thumbnails
      through its full processing pipeline, reflecting tone curves, exposure,
      colour grading, and all other development settings.

    Export queue  (DT_EXPORT_QUEUE, default /tmp/dt-api-export.queue)
      One image ID per line.  The bridge exports a full-resolution JPEG via
      darktable's rendering pipeline to DT_EXPORT_DIR/<id>.jpg.

  Install:  add the following line to ~/.config/darktable/luarc
              require "contrib/darktable-api/darktable-api-bridge"

  Or let the startup script (start-with-api.sh) do it automatically.
--]]

local dt = require "darktable"
local du = require "lib/dtutils"

du.check_min_api_version("7.0.0", "darktable-api-bridge")

local MODULE = "darktable-api-bridge"

-- ── Configuration ─────────────────────────────────────────────────────────────

-- Must match config.refresh_queue / export_queue / export_dir in config.lua.
local QUEUE_FILE   = os.getenv("DT_REFRESH_QUEUE") or "/tmp/dt-api-refresh.queue"
local EXPORT_QUEUE = os.getenv("DT_EXPORT_QUEUE")  or "/tmp/dt-api-export.queue"
local EXPORT_DIR   = os.getenv("DT_EXPORT_DIR")    or "/tmp/dt-api-exports"
local POLL_MS      = 1000   -- poll interval in milliseconds
-- Mipmap range to regenerate: 0 = smallest thumbnail, 4 ≈ 1080 px.
local MIP_MIN      = 0
local MIP_MAX      = 4
-- JPEG quality used for full-resolution exports.
local EXPORT_QUALITY = tonumber(os.getenv("DT_EXPORT_QUALITY")) or 95

-- ── Logging ───────────────────────────────────────────────────────────────────

local function log(msg)
  dt.print_log("[" .. MODULE .. "] " .. msg)
end

local function notify(msg)
  dt.print("[" .. MODULE .. "] " .. msg)
  log(msg)
end

-- ── Image lookup ──────────────────────────────────────────────────────────────

-- Build a fast id→image map from dt.database.
-- dt.database is a table-like object; iterating it is O(n) but fast enough
-- for typical library sizes (thousands of images).
local function build_id_map()
  local map = {}
  for _, img in ipairs(dt.database) do
    map[img.id] = img
  end
  return map
end

-- ── Queue file handling ───────────────────────────────────────────────────────

-- Atomically claim the queue file by renaming it, then read all IDs from it.
-- Returns a set table { [id] = true } of unique image IDs, or {} if none.
local function drain_queue()
  local processing = QUEUE_FILE .. ".processing"

  -- Rename is atomic on POSIX filesystems: the server's append-only writes
  -- will start a fresh QUEUE_FILE after we claim this one.
  local ok = os.rename(QUEUE_FILE, processing)
  if not ok then
    return {}   -- queue file doesn't exist yet
  end

  local ids = {}
  local f = io.open(processing, "r")
  if f then
    for line in f:lines() do
      local id = tonumber(line)
      if id then ids[id] = true end
    end
    f:close()
    os.remove(processing)
  end
  return ids
end

-- ── Mipmap directory ─────────────────────────────────────────────────────────

-- Darktable's mipmap_cache->cachedir is not directly accessible from Lua, so
-- we replicate the naming: SHA1 hex of realpath(library.db) + ".d".
-- We compute it once and cache it here.
local _mipmap_cachedir


local function mipmap_cachedir()
  if _mipmap_cachedir then return _mipmap_cachedir end
  local cache_dir = os.getenv("DT_CACHE_DIR")
  if not cache_dir then
    local home = os.getenv("HOME") or "~"
    cache_dir = home .. "/.cache/darktable"
  end
  -- There is exactly one mipmaps-<sha1>.d directory per library.
  -- Find it by globbing rather than recomputing the SHA1.
  local h = io.popen(string.format("ls -d %q/mipmaps-*.d 2>/dev/null | head -1", cache_dir))
  local found = h and h:read("*l"); if h then h:close() end
  if found and #found > 0 then
    -- Strip trailing ".d" to get the base prefix used in file paths.
    _mipmap_cachedir = found:match("^(.+)%.d$") or found
    log("mipmap cachedir: " .. _mipmap_cachedir .. ".d")
    return _mipmap_cachedir
  end
  log("WARNING: could not find mipmap cache directory under " .. cache_dir)
  return nil
end

-- Delete all mipmap disk files for an image so generate_cache regenerates them.
local function delete_mipmap_files(imgid)
  local base = mipmap_cachedir()
  if not base then return end
  for level = 0, 8 do
    local path = string.format("%s.d/%d/%d.jpg", base, level, imgid)
    os.remove(path)
  end
end

-- ── Cache regeneration ────────────────────────────────────────────────────────

local function refresh_image(img)
  local label = string.format("id=%d %s/%s", img.id, img.path, img.filename)
  log("refreshing " .. label)

  -- Remove disk mipmap files first so generate_cache doesn't skip any level.
  -- drop_cache() only clears the in-memory entry; existing disk files would
  -- cause generate_cache to skip those levels (it guards with dt_util_test_image_file).
  delete_mipmap_files(img.id)
  img:drop_cache()

  -- create_dirs=true ensures the per-level directories exist.
  img:generate_cache(true, MIP_MIN, MIP_MAX)

  log("done " .. label)
end

local function export_image(img, dest_path)
  local label = string.format("id=%d → %s", img.id, dest_path)
  log("exporting " .. label)

  local fmt = dt.new_format("jpeg")
  fmt.quality = EXPORT_QUALITY
  -- max_width/max_height = 0 means no scaling: full resolution.
  fmt.max_width  = 0
  fmt.max_height = 0

  local export_err = fmt:write_image(img, dest_path)
  if export_err then
    error("export failed: " .. tostring(export_err))
  end

  log("export done " .. label)
end

-- Atomically claim the export queue and return a set of unique image IDs.
local function drain_export_queue()
  local processing = EXPORT_QUEUE .. ".processing"
  local ok = os.rename(EXPORT_QUEUE, processing)
  if not ok then return {} end

  local ids = {}
  local f = io.open(processing, "r")
  if f then
    for line in f:lines() do
      local id = tonumber(line)
      if id then ids[id] = true end
    end
    f:close()
    os.remove(processing)
  end
  return ids
end

-- Process all pending refresh IDs in one pass.
local function process_queue()
  local ids = drain_queue()
  local count = 0
  for id, _ in pairs(ids) do count = count + 1 end
  if count == 0 then return end

  log(string.format("processing %d queued refresh request(s)", count))
  local id_map = build_id_map()

  for id, _ in pairs(ids) do
    local img = id_map[id]
    if img then
      local ok, err = pcall(refresh_image, img)
      if not ok then
        log("error refreshing id=" .. id .. ": " .. tostring(err))
      end
    else
      log("image id=" .. id .. " not found in database (may have been removed)")
    end
  end

  -- Nudge the lighttable to redraw so updated thumbnails appear immediately.
  dt.gui.views.lighttable.is_culling_mode = dt.gui.views.lighttable.is_culling_mode
end

-- Process all pending export IDs in one pass.
local function process_export_queue()
  local ids = drain_export_queue()
  local count = 0
  for id, _ in pairs(ids) do count = count + 1 end
  if count == 0 then return end

  -- Ensure export directory exists.
  os.execute('mkdir -p "' .. EXPORT_DIR .. '"')

  log(string.format("processing %d queued export request(s)", count))
  local id_map = build_id_map()

  for id, _ in pairs(ids) do
    local img = id_map[id]
    if img then
      local dest = EXPORT_DIR .. "/" .. tostring(id) .. ".jpg"
      local ok, err = pcall(export_image, img, dest)
      if not ok then
        log("error exporting id=" .. id .. ": " .. tostring(err))
      end
    else
      log("export: image id=" .. id .. " not found in database")
    end
  end
end

-- ── Background polling loop ───────────────────────────────────────────────────

-- dt.control.dispatch runs the function asynchronously in darktable's worker
-- thread pool, leaving the UI thread free.  We loop inside with sleep() so
-- one dispatch call drives the whole bridge.
local running = true

local function poll_loop()
  notify(string.format(
    "started; polling queues every %d ms\n  refresh: %s\n  export:  %s → %s",
    POLL_MS, QUEUE_FILE, EXPORT_QUEUE, EXPORT_DIR))

  while running do
    local ok, err = pcall(process_queue)
    if not ok then log("process_queue error: " .. tostring(err)) end

    local ok2, err2 = pcall(process_export_queue)
    if not ok2 then log("process_export_queue error: " .. tostring(err2)) end

    dt.control.sleep(POLL_MS)
  end

  notify("stopped")
end

-- ── Plugin lifecycle ──────────────────────────────────────────────────────────

local script_data = {}

script_data.metadata = {
  name    = "darktable API bridge",
  purpose = "Watch the REST API refresh queue and regenerate image thumbnails",
  author  = "darktable-api",
  help    = "https://github.com/darktable-org/lua-scripts",
}

local function destroy()
  running = false
  -- Give the poll loop one final chance to exit cleanly.
  dt.control.sleep(POLL_MS + 200)
  dt.destroy_event(MODULE, "shortcut")
end
script_data.destroy = destroy

-- Register a manual-trigger shortcut so the user can force an immediate poll
-- without waiting for the next timer tick.
dt.register_event(MODULE, "shortcut",
  function(event, shortcut)
    log("manual poll triggered via shortcut")
    local ok, err = pcall(process_queue)
    if not ok then notify("error during manual poll: " .. tostring(err)) end
    local ok2, err2 = pcall(process_export_queue)
    if not ok2 then notify("export error during manual poll: " .. tostring(err2)) end
  end,
  "poll refresh queue now"
)

-- Start the background loop.
dt.control.dispatch(poll_loop)

return script_data