Merge pull request #412 from wpferguson/add_substitution_to_string_lib

Add substitution to string lib

lib/dtutils/string.lua

@@ -1,6 +1,12 @@
 local dtutils_string = {}
 
 local dt = require "darktable"
+local du = require "lib/dtutils"
+local log = require "lib/dtutils.log"
+
+local DEFAULT_LOG_LEVEL = log.error
+
+dtutils_string.log_level = DEFAULT_LOG_LEVEL
 
 dtutils_string.libdoc = {
   Name = [[dtutils.string]],
@@ -48,6 +54,8 @@ dtutils_string.libdoc.functions["strip_accents"] = {
 }
 
 function dtutils_string.strip_accents( str )
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
   local tableAccents = {}
     tableAccents["à"] = "a"
     tableAccents["á"] = "a"
@@ -111,6 +119,7 @@ function dtutils_string.strip_accents( str )
     end
   end
         
+  log.log_level(old_log_level)
   return normalizedString
  
 end
@@ -136,6 +145,8 @@ dtutils_string.libdoc.functions["escape_xml_characters"] = {
 
 -- Keep & first, otherwise it will double escape other characters
 function dtutils_string.escape_xml_characters( str )
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
 
   str = string.gsub(str,"&", "&")
   str = string.gsub(str,"\"", """)
@@ -143,6 +154,7 @@ function dtutils_string.escape_xml_characters( str )
   str = string.gsub(str,"<", "&lt;")
   str = string.gsub(str,">", "&gt;")
 
+  log.log_level(old_log_level)
   return str
 end
 
@@ -165,11 +177,14 @@ dtutils_string.libdoc.functions["urlencode"] = {
 }
 
 function dtutils_string.urlencode(str)
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
   if (str) then
     str = string.gsub (str, "\n", "\r\n")
     str = string.gsub (str, "([^%w ])", function (c) return string.format ("%%%02X", string.byte(c)) end)
     str = string.gsub (str, " ", "+")
   end
+  log.log_level(old_log_level)
   return str
 end
 
@@ -192,38 +207,52 @@ dtutils_string.libdoc.functions["is_not_sanitized"] = {
 }
 
 local function _is_not_sanitized_posix(str)
-   -- A sanitized string must be quoted.
-   if not string.match(str, "^'.*'$") then
-       return true
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
+  -- A sanitized string must be quoted.
+  if not string.match(str, "^'.*'$") then
+    log.log_level(old_log_level)
+    return true
    -- A quoted string containing no quote characters within is sanitized.
-   elseif string.match(str, "^'[^']*'$") then
-       return false
-   end
+  elseif string.match(str, "^'[^']*'$") then
+    log.log_level(old_log_level)
+    return false
+  end
    
-   -- Any quote characters within a sanitized string must be properly
-   -- escaped.
-   local quotesStripped = string.sub(str, 2, -2)
-   local escapedQuotesRemoved = string.gsub(quotesStripped, "'\\''", "")
-   if string.find(escapedQuotesRemoved, "'") then
-       return true
-   else
-       return false
-   end
+  -- Any quote characters within a sanitized string must be properly
+  -- escaped.
+  local quotesStripped = string.sub(str, 2, -2)
+  local escapedQuotesRemoved = string.gsub(quotesStripped, "'\\''", "")
+  if string.find(escapedQuotesRemoved, "'") then
+    log.log_level(old_log_level)
+    return true
+  else
+    log.log_level(old_log_level)
+    return false
+  end
 end
 
 local function _is_not_sanitized_windows(str)
-   if not string.match(str, "^\".*\"$") then
-      return true
-   else
-      return false
-   end
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
+  if not string.match(str, "^\".*\"$") then
+    log.log_level(old_log_level)
+    return true
+  else
+    log.log_level(old_log_level)
+    return false
+  end
 end
 
 function dtutils_string.is_not_sanitized(str)
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
   if dt.configuration.running_os == "windows" then
-      return _is_not_sanitized_windows(str)
+    log.log_level(old_log_level)
+    return _is_not_sanitized_windows(str)
   else
-      return _is_not_sanitized_posix(str)
+    log.log_level(old_log_level)
+    return _is_not_sanitized_posix(str)
   end
 end
 
@@ -246,26 +275,38 @@ dtutils_string.libdoc.functions["sanitize"] = {
 }
 
 local function _sanitize_posix(str)
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
   if _is_not_sanitized_posix(str) then
-      return "'" .. string.gsub(str, "'", "'\\''") .. "'"
+    log.log_level(old_log_level)
+    return "'" .. string.gsub(str, "'", "'\\''") .. "'"
   else
-       return str
+    log.log_level(old_log_level)
+    return str
   end
 end
 
 local function _sanitize_windows(str)
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
   if _is_not_sanitized_windows(str) then
-      return "\"" .. string.gsub(str, "\"", "\"^\"\"") .. "\""
+    log.log_level(old_log_level)
+    return "\"" .. string.gsub(str, "\"", "\"^\"\"") .. "\""
   else
-      return str
+    log.log_level(old_log_level)
+    return str
   end
 end
 
 function dtutils_string.sanitize(str)
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
   if dt.configuration.running_os == "windows" then
-      return _sanitize_windows(str)
+    log.log_level(old_log_level)
+    return _sanitize_windows(str)
   else
-      return _sanitize_posix(str)
+    log.log_level(old_log_level)
+    return _sanitize_posix(str)
   end
 end
 
@@ -288,9 +329,16 @@ dtutils_string.libdoc.functions["sanitize_lua"] = {
 }
 
 function dtutils_string.sanitize_lua(str)
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
+  str = string.gsub(str, "%%", "%%%%")
   str = string.gsub(str, "%-", "%%-")
   str = string.gsub(str, "%(", "%%(")
   str = string.gsub(str, "%)", "%%)")
+  str = string.gsub(str, "%[", "%%[")
+  str = string.gsub(str, "%]", "%%]")
+  str = string.gsub(str, "+", "%%+")
+  log.log_level(old_log_level)
   return str
 end
 
@@ -313,7 +361,9 @@ dtutils_string.libdoc.functions["split_filepath"] = {
 }
 
 function dtutils_string.split_filepath(str)
-  -- strip out single quotes from quoted pathnames
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
+ -- strip out single quotes from quoted pathnames
   str = string.gsub(str, "'", "")
   str = string.gsub(str, '"', '')
   local result = {}
@@ -323,6 +373,7 @@ function dtutils_string.split_filepath(str)
     result["basename"] = result["filetype"]
     result["filetype"] = ""
   end
+  log.log_level(old_log_level)
   return result
 end
 
@@ -344,7 +395,10 @@ dtutils_string.libdoc.functions["get_path"] = {
 }
 
 function dtutils_string.get_path(str)
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
   local parts = dtutils_string.split_filepath(str)
+  log.log_level(old_log_level)
   return parts["path"]
 end
 
@@ -366,7 +420,10 @@ dtutils_string.libdoc.functions["get_filename"] = {
 }
 
 function dtutils_string.get_filename(str)
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
   local parts = dtutils_string.split_filepath(str)
+  log.log_level(old_log_level)
   return parts["filename"]
 end
 
@@ -389,7 +446,10 @@ dtutils_string.libdoc.functions["get_basename"] = {
 }
 
 function dtutils_string.get_basename(str)
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
   local parts = dtutils_string.split_filepath(str)
+  log.log_level(old_log_level)
   return parts["basename"]
 end
 
@@ -411,10 +471,601 @@ dtutils_string.libdoc.functions["get_filetype"] = {
 }
 
 function dtutils_string.get_filetype(str)
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
   local parts = dtutils_string.split_filepath(str)
+  log.log_level(old_log_level)
   return parts["filetype"]
 end
 
+dtutils_string.libdoc.functions["build_substitute_list"] = {
+  Name = [[build_substitute_list]],
+  Synopsis = [[build a list of variable substitutions]],
+  Usage = [[local ds = require "lib/dtutils.string"
+    ds.build_substitute_list(image, sequence, variable_string, [username], [pic_folder], [home], [desktop])
+      image - dt_lua_image_t - the image being processed
+      sequence - integer - the sequence number of the image being processed (exported)
+      variable_string - string - the substitution variable string
+      [username] - string - optional - user name.  Will be determined if not supplied
+      [pic_folder] - string - optional - pictures folder name.  Will be determined if not supplied
+      [home] - string - optional - home directory.  Will be determined if not supplied
+      [desktop] - string - optional - desktop directory.  Will be determined if not supplied]],
+  Description = [[build_substitute_list populates variables with values from the arguments
+    and determined from the system and darktable.]],
+  Return_Value = [[]],
+  Limitations = [[If the value for a variable can not be determined, or if it is not supported,
+    then an empty string is used for the value.]],
+  Example = [[]],
+  See_Also = [[https://docs.darktable.org/usermanual/4.6/en/special-topics/variables/]],
+  Reference = [[]],
+  License = [[]],
+  Copyright = [[]],
+}
+
+local substitutes = {}
+local category_substitutes = {}
+
+-- - - - - - - - - - - - - - - - - - - - - - - -
+-- C O N S T A N T S
+-- - - - - - - - - - - - - - - - - - - - - - - -
+
+local PLACEHOLDERS = {"ROLL.NAME", 
+                      "FILE.FOLDER", 
+                      "FILE.NAME",
+                      "FILE.EXTENSION", 
+                      "ID", 
+                      "VERSION", 
+                      "VERSION.IF.MULTI",
+                      "VERSION.NAME",
+                      "DARKTABLE.VERSION",
+                      "DARKTABLE.NAME",           -- Not Implemented
+                      "SEQUENCE",
+                      "WIDTH.SENSOR",
+                      "HEIGHT.SENSOR",
+                      "WIDTH.RAW",
+                      "HEIGHT.RAW",
+                      "WIDTH.CROP",
+                      "HEIGHT.CROP",
+                      "WIDTH.EXPORT",
+                      "HEIGHT.EXPORT",
+                      "WIDTH.MAX",                -- Not Implemented
+                      "HEIGHT.MAX",               -- Not Implemented
+                      "YEAR", 
+                      "YEAR.SHORT",
+                      "MONTH", 
+                      "MONTH.LONG",
+                      "MONTH.SHORT",
+                      "DAY", 
+                      "HOUR", 
+                      "MINUTE", 
+                      "SECOND", 
+                      "MSEC",
+                      "EXIF.YEAR", 
+                      "EXIF.YEAR.SHORT",
+                      "EXIF.MONTH", 
+                      "EXIF.MONTH.LONG",
+                      "EXIF.MONTH.SHORT",
+                      "EXIF.DAY",
+                      "EXIF.HOUR", 
+                      "EXIF.MINUTE", 
+                      "EXIF.SECOND", 
+                      "EXIF.MSEC",
+                      "EXIF.DATE.REGIONAL",       -- Not Implemented
+                      "EXIF.TIME.REGIONAL",       -- Not Implemented
+                      "EXIF.ISO",
+                      "EXIF.EXPOSURE",
+                      "EXIF.EXPOSURE.BIAS",
+                      "EXIF.APERTURE",
+                      "EXIF.CROP.FACTOR",
+                      "EXIF.FOCAL.LENGTH",
+                      "EXIF.FOCAL.LENGTH.EQUIV",  -- Not Implemented
+                      "EXIF.FOCUS.DISTANCE",
+                      "IMAGE.EXIF",               -- Not Implemented
+                      "LONGITUDE",
+                      "LATITUDE",
+                      "ELEVATION",
+                      "GPS.LOCATION",             -- Not Implemented
+                      "STARS", 
+                      "RATING.ICONS",             -- Not Implemented
+                      "LABELS",
+                      "LABELS.ICONS",             -- Not Implemented 
+                      "MAKER", 
+                      "MODEL", 
+                      "LENS",
+                      "TITLE",
+                      "DESCRIPTION",
+                      "CREATOR", 
+                      "PUBLISHER", 
+                      "RIGHTS", 
+                      "TAGS",                     -- Not Implemented
+                      "SIDECAR.TXT",              -- Not Implemented
+                      "FOLDER.PICTURES",
+                      "FOLDER.HOME", 
+                      "FOLDER.DESKTOP",
+                      "OPENCL.ACTIVATED",         -- Not Implemented
+                      "USERNAME",
+                      "NL",                       -- Not Implemented
+                      "JOBCODE"                   -- Not Implemented
+}
+
+local PS        = dt.configuration.running_os == "windows" and  "\\"  or  "/"
+local USER      = os.getenv("USERNAME")
+local HOME      =  dt.configuration.running_os == "windows" and  os.getenv("HOMEPATH") or os.getenv("HOME")
+local PICTURES  = HOME .. PS .. (dt.configuration.running_os == "windows" and "My Pictures" or "Pictures")
+local DESKTOP   = HOME .. PS .. "Desktop"
+
+local function get_colorlabels(image)
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
+
+  local colorlabels = {}
+
+  if image.red then table.insert(colorlabels, "red") end
+  if image.yellow then table.insert(colorlabels, "yellow") end
+  if image.green then table.insert(colorlabels, "green") end
+  if image.blue then table.insert(colorlabels, "blue") end
+  if image.purple then table.insert(colorlabels, "purple") end
+
+  local labels = #colorlabels == 1 and colorlabels[1] or du.join(colorlabels, "_")
+
+  log.log_level(old_log_level)
+
+  return labels 
+end
+
+-- find the $CATEGORYn and $CATEGORY[n,m] requests and add them to the substitute list
+
+local function build_category_substitution_list(image, variable_string)
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
+
+  for match in string.gmatch(variable_string, "%$%(.-%)?%)") do -- grab each complete variable 
+    log.msg(log.info, "match is " .. match)
+
+    local var = string.match(match, "%$%((.-%)?)%)")  -- strip of the leading $( and trailing )
+    log.msg(log.info, "var is " .. var)
+
+    if string.match(var, "CATEGORY%d") or string.match(var, "CATEGORY%[") then
+      local element
+      local tag
+
+      if string.match(var, "CATEGORY%d") then
+        element, tag = string.match(var, "CATEGORY(%d)%((.-)%)")   -- get the element number and the tag to match
+      else
+        element, tag = string.match(var, "%[(%d),(.-)%]") -- new syntax
+      end
+
+      element = element + 1  -- add one to element since lua arrays are 1 based
+      log.msg(log.debug, "element is " .. element .. " and tag is " .. tag)
+
+      local tags = image:get_tags()
+      log.msg(log.debug, "got " .. #tags .. " from image " .. image.filename)
+
+      for _, image_tag in ipairs(tags) do
+        log.msg(log.debug, "checking tag " .. image_tag.name)
+
+        if string.match(image_tag.name, tag) then
+          fields = du.split(image_tag.name, "|")
+
+          if element <= #fields then
+            substitutes[var] = fields[element]
+          else
+            substitutes[var] = ""
+            log.msg(log.warn, "requested field for tag " .. tag .. " doesn't exist")
+          end
+
+          log.msg(log.info, "set substitute for " .. var .. " to " .. fields[element])
+
+        end
+      end
+    end
+  end
+  log.log_level(old_log_level)
+end
+
+-- convert image.exif_datetime_taken to system time
+
+local function exiftime2systime(exiftime)
+  local yr,mo,dy,h,m,s = string.match(exiftime, "(%d-):(%d-):(%d-) (%d-):(%d-):(%d+)")
+  return(os.time{year=yr, month=mo, day=dy, hour=h, min=m, sec=s})
+end
+
+-- build the argument substitution list from each image
+
+function dtutils_string.build_substition_list(image, sequence, variable_string, username, pic_folder, home, desktop)
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
+
+  -- is time millisecond aware?  Implemented in API 9.1.0
+
+  local is_api_9_1 = true
+  if dt.configuration.api_version_string < "9.1.0" then
+    is_api_9_1 = false 
+  end
+
+  local datetime = os.date("*t")
+  local long_month = os.date("%B")
+  local short_month = os.date("%b")
+  local user_name = username or USER
+  local pictures_folder = pic_folder or PICTURES
+  local home_folder = home or HOME
+  local desktop_folder = desktop or DESKTOP
+
+  local labels = get_colorlabels(image)
+
+  local eyear, emon, eday, ehour, emin, esec, emsec
+  if dt.preferences.read("darktable", "lighttable/ui/milliseconds", "bool") and is_api_9_1 then
+    eyear, emon, eday, ehour, emin, esec, emsec = 
+      string.match(image.exif_datetime_taken, "(%d+):(%d+):(%d+) (%d+):(%d+):(%d+)%.(%d+)$")
+  else
+    emsec = "0"
+    eyear, emon, eday, ehour, emin, esec = 
+      string.match(image.exif_datetime_taken, "(%d+):(%d+):(%d+) (%d+):(%d+):(%d+)$")
+  end
+
+  local version_multi = #image:get_group_members() > 1 and image.version or ""
+
+  local replacements = {image.film.path,                       -- ROLL.NAME
+                        image.path,                            -- FILE.FOLDER
+                        image.filename,                        -- FILE.NAME
+                        dtutils_string.get_filetype(image.filename),-- FILE.EXTENSION
+                        image.id,                              -- ID
+                        image.duplicate_index,                 -- VERSION
+                        version_multi,                         -- VERSION.IF_MULTI
+                        image.version_name,                    -- VERSION.NAME
+                        dt.configuration.version,              -- DARKTABLE.VERSION
+                        "",                                    -- DARKTABLE.NAME
+                        string.format("%04d", sequence),       -- SEQUENCE
+                        image.width,                           -- WIDTH.SENSOR
+                        image.height,                          -- HEIGHT.SENSOR
+                        is_api_9_1 and image.p_width or "",    -- WIDTH.RAW
+                        is_api_9_1 and image.p_height or "",   -- HEIGHT.RAW
+                        is_api_9_1 and image.final_width or "",  -- WIDTH.CROP
+                        is_api_9_1 and image.final_height or "", -- HEIGHT.CROP
+                        is_api_9_1 and image.final_width or "",  -- WIDTH.EXPORT
+                        is_api_9_1 and image.final_height or "", -- HEIGHT.EXPORT
+                        "",                                    -- WIDTH.MAX   -- from export module
+                        "",                                    -- HEIGHT.MAX  -- from export module
+                        string.format("%4d", datetime.year),   -- YEAR
+                        string.sub(datetime.year, 3),          -- YEAR.SHORT
+                        string.format("%02d", datetime.month), -- MONTH
+                        long_month,                            -- MONTH.LONG
+                        short_month,                           -- MONTH.SHORT
+                        string.format("%02d", datetime.day),   -- DAY
+                        string.format("%02d", datetime.hour),  -- HOUR
+                        string.format("%02d", datetime.min),   -- MINUTE
+                        string.format("%02d", datetime.sec),   -- SECOND
+                        0,                                     -- MSEC
+                        eyear,                                 -- EXIF.YEAR
+                        string.sub(eyear, 3),                  -- EXIF.YEAR.SHORT
+                        emon,                                  -- EXIF.MONTH
+                        os.date("%B", exiftime2systime(image.exif_datetime_taken)), -- EXIF.MONTH.LONG
+                        os.date("%b", exiftime2systime(image.exif_datetime_taken)), -- EXIF.MONTH.SHORT
+                        eday,                                  -- EXIF.DAY
+                        ehour,                                 -- EXIF.HOUR
+                        emin,                                  -- EXIF.MINUTE
+                        esec,                                  -- EXIF.SECOND
+                        emsec,                                 -- EXIF.MSEC
+                        "",                                    -- EXIF.DATE.REGIONAL - wont be implemented
+                        "",                                    -- EXIF.TIME.REGIONAL - wont be implemented
+                        string.format("%d", image.exif_iso),   -- EXIF.ISO
+                        string.format("%.0f", 1./image.exif_exposure), -- EXIF.EXPOSURE
+                        image.exif_exposure_bias,              -- EXIF.EXPOSURE.BIAS
+                        string.format("%.01f", image.exif_aperture), -- EXIF.APERTURE
+                        string.format("%.01f", image.exif_crop),-- EXIF.CROP_FACTOR
+                        string.format("%.0f", image.exif_focal_length), -- EXIF.FOCAL.LENGTH
+                        string.format("%.0f", image.exif_focal_length * image.exif_crop), -- EXIF.FOCAL.LENGTH.EQUIV
+                        image.exif_focus_distance,             -- EXIF.FOCUS.DISTANCE
+                        "",                                    -- IMAGE.EXIF
+                        image.longitude or "",                 -- LONGITUDE
+                        image.latitude or "",                  -- LATITUDE
+                        image.elevation or "",                 -- ELEVATION
+                        "",                                    -- GPS.LOCATION - wont be implemented
+                        image.rating,                          -- STARS
+                        "",                                    -- RATING.ICONS - wont be implemented
+                        labels,                                -- LABELS
+                        "",                                    -- LABELS.ICONS - wont be implemented
+                        image.exif_maker,                      -- MAKER
+                        image.exif_model,                      -- MODEL
+                        image.exif_lens,                       -- LENS
+                        image.title,                           -- TITLE
+                        image.description,                     -- DESCRIPTION
+                        image.creator,                         -- CREATOR
+                        image.publisher,                       -- PUBLISHER
+                        image.rights,                          -- RIGHTS
+                        "",                                    -- TAGS - wont be implemented
+                        "",                                    -- SIDECAR.TXT - wont be implemented
+                        pictures_folder,                       -- FOLDER.PICTURES
+                        home_folder,                           -- FOLDER.HOME
+                        desktop_folder,                        -- FOLDER.DESKTOP
+                        "",                                    -- OPENCL.ACTIVATED - wont be implemented
+                        user_name,                             -- USERNAME
+                        "",                                    -- NL - wont be implemented
+                        ""                                     -- JOBCODE - wont be implemented
+  }
+
+  -- populate the substitution list
+
+  for i = 1, #PLACEHOLDERS, 1 do 
+    substitutes[PLACEHOLDERS[i]] = replacements[i] 
+    log.msg(log.info, "setting " .. PLACEHOLDERS[i] .. " to " .. tostring(replacements[i]))
+  end
+
+  -- do category substitutions separately
+
+  build_category_substitution_list(image, variable_string)
+
+  log.log_level(old_log_level)
+end
+
+-- handle different versions of names
+
+local function check_legacy_vars(var_name)
+  local var = var_name
+
+  if string.match(var, "_") then
+    var = string.gsub(var, "_", ".")
+  end
+
+  if string.match(var, "^HOME$") then var = "FOLDER.HOME" end
+  if string.match(var, "^PICTURES.FOLDER$") then var = "FOLDER.PICTURES" end
+  if string.match(var, "^DESKTOP$") then var = "FOLDER.DESKTOP" end
+
+  return var 
+end
+
+-- get the substitution and do any string manipulations requested
+
+local function treat(var_string)
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
+
+  local ret_val = ""
+
+  -- remove the var from the string
+  local var = string.match(var_string, "[%a%._]+")
+
+  var = check_legacy_vars(var)
+  log.msg(log.info, "var_string is " .. tostring(var_string) .. " and var is " .. tostring(var))
+
+  if string.match(var_string, "CATEGORY%d") or string.match(var_string, "CATEGORY%[") then
+    log.msg(log.info, "substituting for " .. var_string)
+    ret_val = substitutes[var_string]
+    log.msg(log.info, "ret_val is " .. ret_val)
+
+  elseif string.match(var_string, "SEQUENCE%[") then
+    local start, width = string.match(var_string, "(%d+),(%d)")
+    local seq_val = tonumber(substitutes[var])
+    local pat = "%0" .. width .. "d"
+    substitutes[var_string] = string.format(pat, start + (seq_val - 1))
+    ret_val = substitutes[var_string]
+
+  else
+    ret_val = substitutes[var]
+  end
+
+  local valid_var = false 
+
+  if ret_val then
+    valid_var = true 
+  end
+
+  if not valid_var then
+    log.msg(log.error, "variable " .. var .. " is not an allowed variable, returning empty value")
+    log.log_level(old_log_level)
+    return ""
+  end
+
+  -- string modifications
+
+  local args = string.gsub(var_string, var, "")
+  log.msg(log.info, "args is " .. tostring(args))
+
+  if string.len(args) > 0 then
+
+    if string.match(args, '^%^%^') then
+      ret_val = string.upper(ret_val)
+
+    elseif string.match(args, "^%^") then
+      ret_val = string.gsub(ret_val, "^%a", string.upper, 1)
+
+    elseif string.match(args, "^,,") then
+      ret_val = string.lower(ret_val)
+
+    elseif string.match(args, "^,") then
+      ret_val = string.gsub(ret_val, "^%a", string.lower, 1)
+
+    elseif string.match(args, "^:%-?%d+:%-?%d+") then
+
+      local soffset, slen = string.match(args, ":(%-?%d+):(%-?%d+)")
+      log.msg(log.info, "soffset is " .. soffset .. " and slen is " .. slen)
+
+      if tonumber(soffset) >= 0 then
+        soffset = soffset + 1
+      end
+      log.msg(log.info, "soffset is " .. soffset .. " and slen is " .. slen)
+
+      if tonumber(soffset) < 0 and tonumber(slen) < 0 then
+        local temp = soffset
+        soffset = slen 
+        slen = temp 
+      end
+      log.msg(log.info, "soffset is " .. soffset .. " and slen is " .. slen)
+
+      ret_val = string.sub(ret_val, soffset, slen)
+      log.msg(log.info, "ret_val is " .. ret_val)
+
+    elseif string.match(args, "^:%-?%d+") then
+
+      local soffset= string.match(args, ":(%-?%d+)")
+      if tonumber(soffset) >= 0 then
+        soffset = soffset + 1
+      end
+      ret_val = string.sub(ret_val, soffset, -1)
+
+    elseif string.match(args, "^-%$%(.-%)") then
+
+      local replacement = string.match(args, "-%$%(([%a%._]+)%)")
+      replacement = check_legacy_vars(replacement)
+      if string.len(ret_val) == 0 then
+        ret_val = substitutes[replacement]
+      end
+
+    elseif string.match(args, "^-.+$") then
+
+      local replacement = string.match(args, "-(.+)$")
+      if string.len(ret_val) == 0 then
+        ret_val = replacement
+      end
+
+    elseif string.match(args, "^+.+") then
+
+      local replacement = string.match(args, "+(.+)")
+      if string.len(ret_val) > 0 then
+        ret_val = replacement
+      end
+
+    elseif string.match(args, "^#.+") then
+
+      local pattern = string.match(args, "#(.+)")
+      log.msg(log.info, "pattern to remove is " .. tostring(pattern))
+      ret_val = string.gsub(ret_val, "^" .. dtutils_string.sanitize_lua(pattern), "")
+
+    elseif string.match(args, "^%%.+") then
+
+      local pattern = string.match(args, "%%(.+)")
+      ret_val = string.gsub(ret_val, pattern .. "$", "")
+
+    elseif string.match(args, "^//.-/.+") then
+
+      local pattern, replacement = string.match(args, "//(.-)/(.+)")
+      ret_val = string.gsub(ret_val, pattern, replacement)
+
+     elseif string.match(args, "^/#.+/.+") then
+
+      local pattern, replacement = string.match(args, "/#(.+)/(.+)")
+      ret_val = string.gsub(ret_val, "^" .. pattern, replacement, 1)
+
+    elseif string.match(args, "^/%%.-/.+") then
+
+      local pattern, replacement = string.match(args, "/%%(.-)/(.+)")
+      ret_val = string.gsub(ret_val, pattern .. "$", replacement)
+
+    elseif string.match(args, "^/.-/.+") then
+
+      log.msg(log.info, "took replacement branch")
+      local pattern, replacement = string.match(args, "/(.-)/(.+)")
+      ret_val = string.gsub(ret_val, pattern, replacement, 1)
+    end
+
+  end
+  log.log_level(old_log_level)
+  return ret_val
+end
+
+dtutils_string.libdoc.functions["substitute_list"] = {
+  Name = [[substitute_list]],
+  Synopsis = [[Replace variables in a string with their computed values]],
+  Usage = [[local ds = require "lib/dtutils.string"
+    local result = ds.substitute_list(str)
+      str - string - the string containing the variables to be substituted for]],
+  Description = [[substitute_list replaces the variables in the supplied string with
+    values computed in build_substitution_list().]],
+  Return_Value = [[result - string - the input string with values substituted for the variables]],
+  Limitations = [[]],
+  Example = [[]],
+  See_Also = [[https://docs.darktable.org/usermanual/4.6/en/special-topics/variables/]],
+  Reference = [[]],
+  License = [[]],
+  Copyright = [[]],
+}
+
+function dtutils_string.substitute_list(str)
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
+
+  -- replace the substitution variables in a string
+  for match in string.gmatch(str, "%$%(.-%)?%)") do
+
+    local var = string.match(match, "%$%((.-%)?)%)")
+
+    local treated_var = treat(var)
+    log.msg(log.info, "var is " .. var .. " and treated var is " .. tostring(treated_var))
+
+    str = string.gsub(str, "%$%(".. dtutils_string.sanitize_lua(var) .."%)", tostring(treated_var))
+    log.msg(log.info, "str after replacement is " .. str)
+
+  end
+
+  log.log_level(old_log_level)
+
+  return str
+end
+
+dtutils_string.libdoc.functions["clear_substitute_list"] = {
+  Name = [[clear_substitute_list]],
+  Synopsis = [[Clear the computed list of variable substitution values]],
+  Usage = [[local ds = require "lib/dtutils.string"
+    ds.clear_substitute_list()]],
+  Description = [[clear_substitute_list resets the list of variable replacement values]],
+  Return_Value = [[]],
+  Limitations = [[]],
+  Example = [[]],
+  See_Also = [[]],
+  Reference = [[]],
+  License = [[]],
+  Copyright = [[]],
+}
+
+function dtutils_string.clear_substitute_list()
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
+
+  substitutes = {}
+  category_substitutes = {}
+
+  log.log_level(old_log_level)
+end
+
+dtutils_string.libdoc.functions["substitute"] = {
+  Name = [[substitute]],
+  Synopsis = [[Check if a string has been sanitized]],
+  Usage = [[local ds = require "lib/dtutils.string"
+    ds.substitute(image, sequence, variable_string, [username], [pic_folder], [home], [desktop])
+      image - dt_lua_image_t - the image being processed
+      sequence - integer - the number of the image being processed (exported)
+      variable_string - string - the substitution variable string
+      [username] - string - optional - user name.  Will be determined if not supplied
+      [pic_folder] - string - optional - pictures folder name.  Will be determined if not supplied
+      [home] - string - optional - home directory.  Will be determined if not supplied
+      [desktop] - string - optional - desktop directory.  Will be determined if not supplied]],
+  Description = [[substitute initializes the substitution list by calling clear_substitute_list(),
+    then builds the substitutions by calling build_substitution_list() and finally does the 
+    substitution by calling substitute_list(), then returns the result string.]],
+  Return_Value = [[result - string - the input string with values substituted for the variables]],
+  Limitations = [[]],
+  Example = [[]],
+  See_Also = [[https://docs.darktable.org/usermanual/4.6/en/special-topics/variables/]],
+  Reference = [[]],
+  License = [[]],
+  Copyright = [[]],
+}
+
+function dtutils_string.substitute(image, sequence, variable_string, username, pic_folder, home, desktop)
+  local old_log_level = log.log_level()
+  log.log_level(dtutils_string.log_level)
+
+  dtutils_string.clear_substitute_list()
+
+  dtutils_string.build_substitute_list(image, sequence, variable_string, username, pic_folder, home, desktop)  
+
+  local str = dtutils_string.substitute_list(variable_string)
+
+  log.log_level(old_log_level)
+
+  return str
+end
+
 
 
 return dtutils_string