kind/src/kind.ag

#include <dirent.h>
#include <sys/stat.h>
#include <cstring>
#include <unistd.h>

#include <iostream>
#include <fstream>

#include <string>
#include <vector>
#include <set>

#include <algorithm>

#include "stringtools.h"
#include "Exception.h"

#include "DateTime.h"
#include "Strings.h"
#include "FileName.h"
#include "Image.h"

#include "KindConfig.h"
#include "filetools.h"
#include "Lexer.h"
#include "rulecomp.h"

#include "kind.h"

#include "expiretools.h"
#include "excludetools.h"

/*AppGen
  %%  Beschreibung des Programmes:
  prog: archiving backup
  %% Beschreibung Parameter
  % symbolischerName, Art, Typ,   Variablenname, Erklärung, Default-Wert
  para: vault_or_group, required, string, vault, Vault to backup
  %% Beschreibung der Optionen
  % kurz-Option, lang-Option, Typ, Variablenname, Erklärung, Default-Wert
  opt: c, masterconfig, string, masterConfig, Master config file, ""
  opt2: if not given or empty kind looks for
  opt2:   /etc/kind/master.conf
  opt2:   /ffp/etc/kind/master.conf
  opt: f, full, void, fullImage, Force full image == initial backup, false
  opt: B, backup, void, doBackup, Backup, false
  opt: E, expire, void, doExpire, Expire, false
  opt: C, listconfig, void, listConfig, Show configuration, false
  opt: I, listimages, void, listImages, List data of images, false
  opt2:   if none of backup, expire, listconfig and listimages is specified,
  opt2:   backup and expire is assumed.
  opt2:   listconfig and listimages cannot be combined with other actions
  opt: D, dryrun, Void, dryRun, Dry run (no real backup), false
  opt: F, forcebackup, string, forcedBackupSet, Create image for specified backup set, ""
  opt: v, verbose, Void, verbose,  Verbose,  false
  opt: d, debug, Void, debug, Debug output of many data, false
  opt: q, quiet, Void, quiet, Be quiet - no messages, false
  opt: h, help, usage, ignored , This help
AppGen*/

using namespace std;

/*AppGen:Global*/

Strings banks;

string findVault(const string& v);

typedef pair<long int, long int> Sizes;
map<string, Sizes> sizes;

void readSizes(const string& logSizeFile)
{
  if (!logSizeFile.empty() && fileExists(logSizeFile))
    {
      vector<string> ss;
      file2Strings(logSizeFile, ss);
      for (const string& s : ss)
        {
          unsigned int i = 0;
          string vault = getWord(s, i);
          long int s1 = getLongInt(s, i);
          long int s2 = getLongInt(s, i);
          try
            {
              findVault(vault);
              sizes[vault] = Sizes(s1, s2);
            }
          catch (...)
            {
              // ignore missing vaults
            }
        }
    }
}

void writeSizes(const string logSizeFile)
{
  if (!logSizeFile.empty())
    {
      Strings st;
      for (auto s : sizes)
        {
          string h = s.first + " " + to_string(s.second.first) + " " + to_string(s.second.second);
          st.push_back(h);
        }
      strings2File(st, logSizeFile);
    }
}

void verbosePrint(const string& text)
{
  if (verbose)
    cout << "  " << text << endl;
}

void debugPrint(const string& text)
{
  if (debug)
    cout << "    " << text << endl;
}

void readMasterConfig1(const string& fn, KindConfig& conf)
{
  verbosePrint("reading master config " + fn);
  conf.addFile(fn);
}

void readMasterConfig(const string& fn, KindConfig& conf)
{
  if (!fn.empty())  // master config given by user on commandline
    readMasterConfig1(fn, conf);
  else if (fileExists("/etc/kind/master.conf"))
    readMasterConfig1("/etc/kind/master.conf", conf);
  else if (fileExists("/ffp/etc/kind/master.conf"))
    readMasterConfig1("/ffp/etc/kind/master.conf", conf);
  else
    throw Exception("MasterConfig", "no file");
}

string findVault(const string& v)
{
  bool found = false;
  FileName fn;
  fn.setName(v);
  for (unsigned int i = 0; !found && i < banks.size(); ++i)
    {
      fn.setPath(banks[i]);
      if (dirExists(fn.getFileName()))
        found = true;
    }
  if (!found)
    throw Exception("find vault", v + " not found");
  verbosePrint("using vault " + fn.getFileName());
  return fn.getFileName();
}

void readVaultConfig(const string& vault, KindConfig& conf)
{
  string vaultpath = findVault(vault);
  const string& vaultConfigName = vaultpath + '/' + conf.getString("vaultConfigName");
  verbosePrint("reading vault config:");
  verbosePrint("  " + vaultConfigName);
  conf.addFile(vaultConfigName);
}

string getImageName(const KindConfig& conf,
                    const string& vaultPath,
                    const DateTime& imageTime)
{
  bool nonPortable = false;
  string imageName = conf.getString("imageName");
  for (unsigned int i = 0; !nonPortable && i < imageName.size(); ++i)
    {
      char c = imageName[i];
      if (!isalnum(c) && c != '.' && c != '_')
        nonPortable = true;
    }
  if (nonPortable)
    throw Exception("getImageName", "Invalid character in image name " + imageName);

  if (!imageName.empty())
    imageName += '-';

  string imageFullName =  vaultPath + "/" + imageName ;

  if (conf.getBool("longImageName"))
    imageFullName += imageTime.getString('m');
  else
    imageFullName += imageTime.getString('s');

  return imageFullName;
}

Images findImages(const string& vaultpath, const KindConfig& conf, bool all)
{
  Strings dirs;
  debugPrint("searching images in " + vaultpath);
  dirList(vaultpath, dirs);

  Images imageList;
  for (string dir : dirs)
    {
      FileName fn(dir);
      string imgname = conf.getString("imageName");
      if (startsWith(fn.getName(), imgname))
        {
          debugPrint("Checking " + dir);
          Image image(dir);

          if (all || image.valid)
            imageList.push_back(image);
        }
    }
  if (imageList.size() > 1)
    sort(imageList.begin(), imageList.end());
  return imageList;
}

void listImageInfo(const string& vault,
                   KindConfig conf /*Copy!*/ ,
                   const DateTime& imageTime,
                   const string& backupSet)
{
  readVaultConfig(vault, conf);
  string vaultPath = findVault(vault);
  Images imageList = findImages(vaultPath, conf, true);
  cout << "== " << vault << " ==" << endl;
  for (auto img : imageList)
    {
      if (img.series == backupSet || backupSet.empty())
        {
          img.printInfo();
          cout << "---" << endl;
        }
    }
}

void doBackup(const string& vault,
              const string& imageFullName,
              const string& referenceImage,
              const KindConfig& conf)
{
  // create image path

  bool shellMode = true;

  // create source descriptor
  string host;
  if (conf.hasKey("host"))
    host = conf.getString("host");

  string server;
  if (conf.hasKey("server"))
    {
      server = conf.getString("server");
      shellMode = false;
    }

  if (!host.empty() && !server.empty())
    throw Exception("backupVault", "Cannot have host and server");

  if (host.empty() && server.empty())
    throw Exception("backupVault", "No host or server specified");

  // ping host / server
  // ping -c 1 -W 5 -q $HOST
  string pingCommand = conf.getString("ping");
  debugPrint("PingCommand: " + pingCommand);
  if (!pingCommand.empty())
    {
      if (!host.empty())
        replacePlaceHolder(pingCommand, "%host", host);
      else
        replacePlaceHolder(pingCommand, "%host", server);
      int rc = 0;
      Strings pingResult = localExec(pingCommand, rc, debug);
      if (rc != 0)
        throw Exception("Host not available", pingCommand);
    }

  string path = conf.getString("path");
  if (path.empty())
    throw Exception("rsync", "empty source path");
  if (path.back() != '/')
    path += '/';

  string rsyncCmd = "rsync -vrltH --delete --stats -D --numeric-ids ";
  if (!conf.getBool("ignorePermission"))
    rsyncCmd += "-pgo";
  vector<string> rso = conf.getStrings("rsyncOption");
  for (const string& opt : rso)
    rsyncCmd += opt + " ";

  // excludes
  Strings excluded = getExclusions(conf, shellMode);

  // create image path
  if (!dryRun)
    if (mkdir(imageFullName.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0)
      throw Exception("Create image", "failed to create " + imageFullName);

  // error message
  // we write an generic error message to mark backup as unsuccessful
  // will be deleted at successful end of rsync
  string errorfile = imageFullName + "/error";
  if (!dryRun)
    {
      ofstream error(errorfile);
      error << "failed" << endl;
      error.close();
    }

  if (shellMode)  // shell mode
    {
      // cout << "USING SHELLMODE '" << host << "'" << endl;
      string remoteShell = conf.getString("remoteShell");
      string userAtHost = conf.getString("user") + "@" + conf.getString("host");
      string rshCommand = remoteShell;
      if (remoteShell.empty())
        rshCommand = "ssh";

      rshCommand += " " + userAtHost;

      if (!dryRun)
        strings2File(excluded, imageFullName + "/exclude");

      // rsync image

      if (!remoteShell.empty())
        rsyncCmd += " -e \'" + remoteShell + "\' ";

      rsyncCmd += "--exclude-from=" + imageFullName + "/exclude ";
      if (!referenceImage.empty())
        rsyncCmd += "--link-dest=" + referenceImage + "/tree ";
      rsyncCmd += userAtHost + ":" + path + " ";
      rsyncCmd += imageFullName + "/tree";
    } // shell mode
  else
    {
      // cout << "USING SERVERMODE" << endl;
      // we cannot use find without shell access
      // and do not read an exclude file on client side

      if (!dryRun)
        strings2File(excluded, imageFullName + "/exclude");

      rsyncCmd += "--exclude-from=" + imageFullName + "/exclude ";
      if (!referenceImage.empty())
        rsyncCmd += "--link-dest=" + referenceImage + "/tree ";
      rsyncCmd += conf.getString("server") + "::" + path + " ";
      rsyncCmd += imageFullName + "/tree";
    }

  debugPrint("Action: " + rsyncCmd);

  vector<string> backupResult;
  if (!dryRun)
    {
      verbosePrint("syncing (" + rsyncCmd + ")");
      int rc;
      backupResult = localExec(rsyncCmd, rc, debug, imageFullName + "/rsync-log");
      if (rc == 0 ||
          rc == 24 || // "no error" or "vanished source files" (ignored)
          rc == 6144) // workaround for wrong exit code ??!!
        {
          unlink(errorfile.c_str());
          long int st = 0;
          long int sc = 0;
          for (auto bl : backupResult)
            {
              if (startsWith(bl, "Total file size"))
                st = getNumber(bl);
              else if (startsWith(bl, "Total transferred file size"))
                sc = getNumber(bl);
            }
          // sizes[vault] = pair<long int, long int>(st, sc);
          sizes[vault] = Sizes(st, sc);
          //  cout << vault << " " << st << " || " << sc << endl;
        }
      else
        throw Exception("Backup", "Failed to execute rsync (result: " + to_string(rc) + ")");
    }
  else
    cout << "Not executing " << rsyncCmd << endl;
}

bool backupVault(const string& vault,
                 KindConfig conf /*Copy!*/ ,
                 const DateTime& imageTime,
                 bool fullImage,
                 const string& forcedBackupSet)
{
  if (!quiet)
    cout << DateTime::now().getString('h') << ": Backup of vault " << vault << endl;
  try
    {
      readVaultConfig(vault, conf);

      // where to store
      string vaultPath = findVault(vault);

      // image path
      string imageFullName = getImageName(conf, vaultPath, imageTime);

      bool backupNow = true;

      // existing images
      Images validImageList = findImages(vaultPath, conf, false);
      string currentSet = "expire"; // we are not using backupSets

      // check if we are using backup sets

      map<string, int> setIdx;
      vector<SetRule> backupSetRule;
      int setRuleIdx = -1;

      if (conf.hasKey("setRule"))
        {
          readSetRules(conf, setIdx, backupSetRule);
          if (!setIdx.empty())
            {
              if (forcedBackupSet.empty())
                {
                  backupNow = false;

                  // find time for nextBackup for every backupSet
                  // defaults to now == imageTime;
                  vector<DateTime> nextBackup(backupSetRule.size(), imageTime);

                  // find time for next backup

                  for (const Image& image : validImageList)
                    {
                      if (image.series != "expire")
                        {
                          string s = image.series;
                          if (setIdx.count(s) > 0) // rule for set exists?
                            {
                              int rIdx = setIdx[s];
                              // image is valid for this and "lower level" backupSets
                              for (unsigned int i = rIdx; i < backupSetRule.size(); ++i)
                                if (nextBackup[i] < image.time + backupSetRule[i].distance)
                                  nextBackup[i] =  image.time + backupSetRule[i].distance;
                            }
                        }
                    }
                  if (debug)
                    for (unsigned int i = 0; i < backupSetRule.size(); ++i)
                      cout << "       Next backup for " << backupSetRule[i].name << " at " <<  nextBackup[i].getString('h') << endl;

                  // find backupSet that
                  //    - needs backup
                  //    - has longest time to keep
                  // because of ordered list backupSetRule this is the first set, that need

                  currentSet = "";
                  for (unsigned int i = 0; i < backupSetRule.size() && currentSet.empty(); ++i)
                    {
                      string name = backupSetRule[i].name;
                      if (nextBackup[i] <= imageTime + 5) // small offset of 5s for "jitter"
                        {
                          backupNow = true;
                          currentSet = name;
                          setRuleIdx = i;
                        }
                    }
                }
              else
                {
                  if (setIdx.count(forcedBackupSet) > 0)
                    {
                      currentSet = forcedBackupSet;
                      setRuleIdx = setIdx[forcedBackupSet];
                    }
                  else
                    throw Exception("force backup of set " + forcedBackupSet, " set not exists");
                }
            } // if (!setIdx.empty())
        } // (conf.hasKey("setRule"))

      if (backupNow)
        {
          verbosePrint("backup to \"" + imageFullName + "\"");
          if (setRuleIdx >= 0 && !quiet)
            cout << "  backup set is \"" << currentSet << "\"" << endl;
        }
      else if (!quiet)
        cout << "  no backup set needs update" << endl;

      if (backupNow)
        {
          // find reference image
          string referenceImage;
          if (!fullImage)
            {
              if (validImageList.empty())
                throw Exception("backupVault", "no reference image found");
              // last image is newest image
              referenceImage = validImageList.back().name;
            }

          doBackup(vault, imageFullName, referenceImage, conf);

          if (!dryRun)
            {
              string lastPath = vaultPath + "/last";
              struct stat fstat;

              // remove last (dir or symlink)
              if (lstat(lastPath.c_str(), &fstat) == 0) // last exists
                {
                  if (S_ISDIR(fstat.st_mode))
                    removeDir(lastPath);
                  else
                    unlink(lastPath.c_str());
                }

              string linkType = conf.getString("lastLink");
              if (linkType == "hardLink")
                {
                  int rc;
                  string hardLinkCommand = "cp -al " + imageFullName + " " + lastPath;
                  Strings res = localExec(hardLinkCommand, rc, debug);
                }
              else if (linkType == "symLink")
                {
                  // set symlink to last image
                  symlink(imageFullName.c_str(), lastPath.c_str());
                }
              else if (linkType != "null")
                cerr << "invalid Value in \"lastLink\"" << endl;

              // write expire date to file
              DateTime expireTime;
              string rule;
              if (setRuleIdx < 0) // not backup set based
                expireTime = getExpireDate(imageTime, conf, rule);
              else
                {
                  expireTime = imageTime + backupSetRule[setRuleIdx].keep;
                  rule =  backupSetRule[setRuleIdx].rule;
                }

              ofstream expireFile(imageFullName + "/expires");
              expireFile << currentSet << "-" << expireTime.getString('m') << endl;
              expireFile << rule << endl;
            }
        }
      return backupNow;
    }
  catch (Exception ex)
    {
      cerr << "Exception in vault " << vault << ": " << ex.what() << endl;
      return false;
    }
}

void expireVault(const string& vault, KindConfig conf, DateTime now)
{
  if (!quiet)
    cout << DateTime::now().getString('h') << ": Expiring images in vault " << vault << endl;

  readVaultConfig(vault, conf);

  string vaultpath = findVault(vault);

  Images imagelist = findImages(vaultpath, conf, true);

  string lastValidImage;
  for (Image image : imagelist)
    {
      if (image.valid)
        lastValidImage = image.name;
    }

  for (Image image : imagelist)
    {
      if (debug)
        image.printInfo();

      DateTime imageTime = image.time;

      if (imageTime != now &&          // ignore just created image
          image.name != lastValidImage // ignore last valid image
         )
        {
          DateTime expireTime;
          string expireRule;
          if (!image.valid) // invalid image?
            {
              time_t expPeriod = stot(conf.getString("expireFailedImage"));
              if (expPeriod < 0)
                throw Exception("expireFailedImage", "Time period must be positive");
              expireTime = imageTime + stot(conf.getString("expireFailedImage"));
              expireRule = "invalid image: " + conf.getString("expireFailedImage");
              debugPrint("- invalid image");
            }
          else
            {
              debugPrint("- valid image");
              expireTime = image.expire;
              expireRule = image.expireRule;
            }

          if (expireTime < now)
            {
              if (!quiet)
                cout << "  removing image " << image.name << endl;
              try
                {
                  if (removeDir(image.name) != 0)
                    cout << "Error removing " <<  image.name << endl;
                }
              catch (Exception ex)
                {
                  cerr << "Exception: " << ex.what() << endl;
                }
            }
        }
      else
        debugPrint("- current image - ignored");
    }
}

/*AppGen:Main*/
int main(int argc, char* argv[])
{
  /*AppGen:MainEnd*/

  int exitCode = 0;
  string lockFile;
  try
    {
      // handling of parameters and switches
      if (debug)        // debug implies verbose
        verbose = true;

      if (!doBackup && !doExpire && !listConfig && !listImages)
        {
          doBackup = true;
          doExpire = true;
        }

      KindConfig conf;

      // default-values
      conf.add("imageName", "image");
      conf.add("vaultConfigName", "kind/vault.conf");
      conf.add("expireFailedImage", "3 days");
      conf.add("expireRule", "* * * * 1 month");
      conf.add("ping", "ping -c 1 -W 5 %host");
      conf.add("rsyncOption", ""); // no additional rsync option
      conf.add("remoteShell", "");
      conf.add("lockfile", "/var/lock/kind");
      conf.add("userExcludeFile", "nobackup.list");
      conf.add("userExcludeCommand",
               "find %path -type f -iname '*nobackup' -printf '%P\\\\n'");
      conf.add("logSize", "");
      conf.add("lastLink", "symLink");

      if (listConfig)
        {
          cout << "builtin config" << endl;
          conf.print(".   ");
        }

      readMasterConfig(masterConfig, conf);

      banks = conf.getStrings("bank");
      if (banks.empty())
        throw Exception("read master configuration", "no banks defined");

      vector<string> vaults;
      string groupname = "group_" + vault;
      if (conf.hasKey(groupname))
        {
          vaults = conf.getStrings(groupname);
          vault.clear(); // no single vault but group
        }
      else
        vaults.push_back(vault);

      if (listConfig)
        {
          cout << "global config:" << endl;
          conf.print(".   ");
          if (!vault.empty())
            {
              readVaultConfig(vault, conf);
              cout << "vault config:" << endl;
              conf.print(".   ");
            }
          else
            cout << "specify single vault (not group) to see vault config" << endl;
          exit(0);
        }

      DateTime imageTime = DateTime::now();

      if (listImages)
        {
          for (string vault : vaults)
            listImageInfo(vault, conf, imageTime, forcedBackupSet);
          exit(0);
        }

      // previous actions do not need locking
      lockFile = conf.getString("lockfile");
      createLock(lockFile);

      string logSizeFile = conf.getString("logSize");
      readSizes(logSizeFile);

      for (string vault : vaults)
        {
          if (doBackup)
            if (backupVault(vault, conf, imageTime, fullImage, forcedBackupSet))
              writeSizes(logSizeFile);
          if (doExpire)
            expireVault(vault, conf, imageTime);
        }

      if (!quiet)
        cout << DateTime::now().getString('h') << ": finished" << endl;

    }
  catch (const Exception& ex)
    {
      cerr << "Exception: " << ex.what() << endl;
      exitCode = 1;
    }
  catch (const char* msg)
    {
      cerr << "Exception(char*): " << msg << endl;
      exitCode = 1;
    }
  catch (const string& msg)
    {
      cerr << "Exception(string): " << msg << endl;
      exitCode = 1;
    }
  removeLock(lockFile);
  return exitCode;
}