""" Class for the LCG File Catalog Client

"""

from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
__RCSID__ = "$Id"

from past.builtins import long
import six
from stat import *
import os
import re
import time

import DIRAC
from DIRAC import S_OK, S_ERROR, gLogger
from DIRAC.Resources.Catalog.Utilities import checkCatalogArguments
from DIRAC.Core.Utilities.Time import fromEpoch
from DIRAC.Core.Utilities.List import breakListIntoChunks
from DIRAC.Core.Security.ProxyInfo import getProxyInfo, formatProxyInfoAsString
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername, getVOMSAttributeForGroup, \
    getVOForGroup, getVOOption
from DIRAC.Resources.Catalog.FileCatalogClientBase import FileCatalogClientBase

lfc = None
importedLFC = None

####################################################################
#
# These are some functions used by all methods in the class
#


def setLfnReplicas(lfn, replicas, successful, failed):
  if not lfn:
    return
  if replicas:
    successful[lfn] = replicas.copy()
    replicas.clear()
  elif lfn not in failed:
    failed[lfn] = 'No active replica'


def getClientCertInfo():
  res = getProxyInfo(False, False)
  if not res['OK']:
    gLogger.error("getClientCertInfo: Failed to get client proxy information.",
                  res['Message'])
    return res
  proxyInfo = res['Value']
  gLogger.debug(formatProxyInfoAsString(proxyInfo))
  if 'group' not in proxyInfo:
    errStr = "getClientCertInfo: Proxy information does not contain the group."
    gLogger.error(errStr)
    return S_ERROR(errStr)
  if 'VOMS' not in proxyInfo:
    proxyInfo['VOMS'] = getVOMSAttributeForGroup(proxyInfo['group'])
    errStr = "getClientCertInfo: Proxy information does not contain the VOMs information."
    gLogger.warn(errStr)
  res = getDNForUsername(proxyInfo['username'])
  if not res['OK']:
    errStr = "getClientCertInfo: Error getting known proxies for user."
    gLogger.error(errStr, res['Message'])
    return S_ERROR(errStr)
  diracGroup = proxyInfo.get('group', 'Unknown')
  vo = getVOForGroup(diracGroup)
  vomsVO = getVOOption(vo, 'VOMSVO', '')
  resDict = {'DN': proxyInfo['identity'],
             'Role': proxyInfo['VOMS'],
             'User': proxyInfo['username'],
             'AllDNs': res['Value'],
             'Group': diracGroup,
             'VO': vo,
             'VOMSVO': vomsVO
             }
  return S_OK(resDict)


def existsGuid(guid):
  """ Check if the guid exists
  """
  fstat = lfc.lfc_filestatg()
  error = lfc.lfc_statg('', guid, fstat)
  return returnCode(error and lfc.cvar.serrno != 2, not error)


def getDNFromUID(userID):
  buff = " " * (lfc.CA_MAXNAMELEN + 1)
  res = lfc.lfc_getusrbyuid(userID, buff)
  if res == 0:
    dn = buff[:buff.find('\x00')]
    gLogger.debug("LcgFileCatalogClient.getDNFromUID: UID %s maps to %s." % (userID, dn))
    return S_OK(dn)
  else:
    errStr = "LcgFileCatalogClient.getDNFromUID: Failed to get DN from UID"
    gLogger.error(errStr, "%s %s" % (userID, lfc.sstrerror(lfc.cvar.serrno)))
    return S_ERROR(errStr)


def getRoleFromGID(groupID, path=None):
  buff = " " * (lfc.CA_MAXNAMELEN + 1)
  res = lfc.lfc_getgrpbygid(groupID, buff)
  if res == 0:
    role = buff[:buff.find('\x00')]
    if role == 'lhcb':
      role = 'lhcb/Role=user'
    gLogger.debug("LcgFileCatalogClient.getRoleFromGID: GID %s maps to %s." % (groupID, role))
    return S_OK(role)
  else:
    errStr = "LcgFileCatalogClient:getRoleFromGID: Failed to get role from GID"
    gLogger.error(errStr, "%s %s%s" % (groupID, ('(%s) ' % path) if path else '', lfc.sstrerror(lfc.cvar.serrno)))
    return S_ERROR()


def addReplica(guid, pfn, se, master):
  fid = lfc.lfc_fileid()
  status = 'U'
  f_type = 'D'
  poolname = ''
  fs = ''
  error = lfc.lfc_addreplica(guid, fid, se, pfn, status, f_type, poolname, fs)
  # If replica exists, re-register it as one may have changed some parameters (status, se)
  if lfc.sstrerror(lfc.cvar.serrno) == "File exists":
    retCode = removeReplica(pfn)
    if not retCode['OK']:
      return retCode
    error = lfc.lfc_addreplica(guid, fid, se, pfn, status, f_type, poolname, fs)
  return returnCode(error)


def removeReplica(pfn):
  fid = lfc.lfc_fileid()
  error = lfc.lfc_delreplica('', fid, pfn)
  return returnCode(error and error != 2)


def setReplicaStatus(pfn, status):
  return returnCode(lfc.lfc_setrstatus(pfn, status))


def modReplica(pfn, newse):
  return returnCode(lfc.lfc_modreplica(pfn, '', '', newse))


def closeDirectory(oDirectory):
  return returnCode(lfc.lfc_closedir(oDirectory))


def getDNUserID(dn):
  error, users = lfc.lfc_getusrmap()
  userid = None
  for userMap in users if not error else []:
    if userMap.username == dn:
      userid = userMap.userid
      break
  return returnCode(userid is None, userid, errMsg="DN does not exist" if not error else '')


def addUserDN(userID, dn):
  error = lfc.lfc_enterusrmap(userID, dn)
  # 17 is if dn already exists, then OK
  return returnCode(error and lfc.cvar.serrno != 17)


def returnCode(error, value='', errMsg=''):
  if not error:
    return S_OK(value)
  elif errMsg:
    return S_ERROR(errMsg)
  else:
    return S_ERROR(lfc.sstrerror(lfc.cvar.serrno))

#####################################################
#
# LFC catalog client class
#
#####################################################


class LcgFileCatalogClient(FileCatalogClientBase):

  READ_METHODS = FileCatalogClientBase.READ_METHODS + ['isLink',
                                                       'readLink',
                                                       'isFile',
                                                       'getFileMetadata',
                                                       'getReplicas',
                                                       'getReplicaStatus',
                                                       'getFileSize',
                                                       'isDirectory',
                                                       'getDirectoryReplicas',
                                                       'listDirectory',
                                                       'getDirectoryMetadata',
                                                       'getDirectorySize',
                                                       'getDirectoryContents',
                                                       'resolveDataset',
                                                       'getLFNForPFN',
                                                       'getUserDirectory']

  WRITE_METHODS = FileCatalogClientBase.WRITE_METHODS + ['createLink',
                                                         'removeLink',
                                                         'addFile',
                                                         'addReplica',
                                                         'removeReplica',
                                                         'removeFile',
                                                         'setReplicaStatus',
                                                         'setReplicaHost',
                                                         'createDirectory',
                                                         'removeDirectory',
                                                         'removeDataset',
                                                         'removeFileFromDataset',
                                                         'createDataset',
                                                         'changePathOwner',
                                                         'changePathMode']

  NO_LFN_METHODS = FileCatalogClientBase.NO_LFN_METHODS + ['getUserDirectory', 'createUserDirectory',
                                                           'createUserMapping', 'removeUserDirectory']

  ADMIN_METHODS = FileCatalogClientBase.ADMIN_METHODS + ['getUserDirectory', 'createUserDirectory',
                                                         'createUserMapping',
                                                         'removeUserDirectory']

  def __init__(self, **options):
    global lfc, importedLFC

    if importedLFC is None:
      try:
        import lfcthr as lfc
        # This is necessary to make the LFC client thread safe.
        lfc.init()
        importedLFC = True
        gLogger.debug("LcgFileCatalogClient.__init__: Successfully imported lfc module.")

      except ImportError:
        importedLFC = False
        gLogger.exception("LcgFileCatalogClient.__init__: Failed to import lfc module.")

    self.valid = importedLFC

    infosys = options.get('LcgGfalInfosys')
    if not infosys and 'LCG_GFAL_INFOSYS' in os.environ:
      # if not in CS take from environ
      infosys = os.environ['LCG_GFAL_INFOSYS']

    host = options.get('MasterHost')
    if not host and 'LFC_HOST' in os.environ:
      # if not in CS take from environ
      host = os.environ['LFC_HOST']

    self.host = host

    if host:
      os.environ['LFC_HOST'] = host
    if infosys:
      os.environ['LCG_GFAL_INFOSYS'] = infosys

    if 'LFC_CONRETRYINT' not in os.environ:
      os.environ['LFC_CONRETRYINT'] = '5'
    if 'LFC_CONNTIMEOUT' not in os.environ:
      os.environ['LFC_CONNTIMEOUT'] = '5'
    if 'LFC_CONRETRY' not in os.environ:
      os.environ['LFC_CONRETRY'] = '5'

    self.prefix = '/grid'
    self.session = False
    self.transaction = False

  def isOK(self):
    return self.valid

  def getName(self):
    return S_OK(self.name)

  ####################################################################
  #
  # These are the methods for session/transaction manipulation
  #

  def __openSession(self):
    """Open the LFC client/server session"""
    if self.session:
      # Another thread is holding a session, can't create it
      return 0
    else:
      sessionName = 'DIRAC_%s.%s at %s at time %s' % (DIRAC.majorVersion,
                                                      DIRAC.minorVersion,
                                                      DIRAC.siteName(),
                                                      time.time())
      rc = lfc.lfc_startsess(self.host, sessionName)
      self.session = (rc == 0)
      # if there was an error, return -1, to be tested just after the call...
      return 1 if self.session else -1

  def __closeSession(self):
    """Close the LFC client/server session"""
    if self.session:
      lfc.lfc_endsess()
      self.session = False

  def __startTransaction(self):
    """ Begin transaction for one time commit """
    if not self.transaction:
      transactionName = 'Transaction: DIRAC_%s.%s at %s at time %s' % (DIRAC.majorVersion,
                                                                       DIRAC.minorVersion,
                                                                       DIRAC.siteName(),
                                                                       time.time())
      lfc.lfc_starttrans(self.host, transactionName)
      self.transaction = True

  def __abortTransaction(self):
    """ Abort transaction """
    if self.transaction:
      lfc.lfc_aborttrans()
      self.transaction = False

  def __endTransaction(self):
    """ End transaction gracefully """
    if self.transaction:
      lfc.lfc_endtrans()
      self.transaction = False

  def setAuthorizationId(self, dn):
    """ Set authorization id for the proxy-less LFC communication """
    lfc.lfc_client_setAuthorizationId(0, 0, 'GSI', dn)

  ####################################################################
  #
  # The following are read methods for paths
  #

  @checkCatalogArguments
  def exists(self, lfns):
    """ Check if the path exists
    """
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for lfn, guid in lfns.items():
      res = self.__existsLfn(lfn)
      if not res['OK']:
        failed[lfn] = res['Message']
      elif res['Value']:
        successful[lfn] = lfn
      elif not isinstance(guid, six.string_types):
        successful[lfn] = False
      else:
        res = existsGuid(guid)
        if not res['OK']:
          failed[lfn] = res['Message']
        elif not res['Value']:
          successful[lfn] = False
        else:
          successful[lfn] = self.__getLfnForGUID(guid)['Value']
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def __getPathAccess(self, path):
    """ Determine the permissions using the lfc function lfc_access """
    permDict = {'Read': 1,
                'Write': 2,
                'Execute': 4}
    resDict = {}
    for p in permDict:

      code = permDict[p]
      value = lfc.lfc_access(self.__fullLfn(path), code)
      if value == 0:
        resDict[p] = True
      else:
        resDict[p] = False
    return S_OK(resDict)

  @checkCatalogArguments
  def hasAccess(self, lfns, opType):

    if opType in LcgFileCatalogClient.READ_METHODS:
      opType = 'Read'
    elif opType in LcgFileCatalogClient.WRITE_METHODS:
      opType = 'Write'

    res = self.getPathPermissions(lfns)

    if not res['OK']:
      return res

    perms = res['Value']
    failed = perms['Failed']
    successful = dict((path, perms['Successful'][path].get(opType, False)) for path in perms['Successful'])

    return S_OK({'Successful': successful, 'Failed': failed})

  @checkCatalogArguments
  def getPathPermissions(self, lfns):
    """ Determine the VOMs based ACL information for a supplied path
    """
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for path in lfns:
      res = self.__getBasePath(path)
      if not res['OK']:
        failed[path] = res['Message']
      else:
        basePath = res['Value']
        res = self.__getPathAccess(basePath)
        if not res['OK']:
          failed[path] = res['Message']
        else:
          lfcPerm = res['Value']
          res = self.__getACLInformation(basePath)
          if not res['OK']:
            failed[path] = res['Message']
          else:
            # Evaluate access rights
            val = res['Value']
            try:
              lfcPerm['user'] = val['user']
              lfcPerm['group'] = val['group']
              lfcPerm['world'] = val['world']
              lfcPerm['DN'] = val['DN']
              lfcPerm['Role'] = val['Role']
            except KeyError:
              print('key not found: __getACLInformation returned incomplete dictionary', KeyError)
              failed[path] = lfcPerm
              continue
          # ACLs are just an additional information, therefore here it is successful
          successful[path] = lfcPerm

    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  ####################################################################
  #
  # The following are read methods for files
  #

  @checkCatalogArguments
  def isFile(self, lfns):

    # If we have less than three lfns to query a session doesn't make sense
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for lfn in lfns:
      res = self.__getPathStat(lfn)
      if not res['OK']:
        failed[lfn] = res['Message']
      elif S_ISREG(res['Value'].filemode):
        successful[lfn] = True
      else:
        successful[lfn] = False
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def getFileMetadata(self, lfns, ownership=False):
    """ Returns the file metadata associated to a supplied LFN
    """
    # If we have less than three lfns to query a session doesn't make sense
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for lfn in lfns:
      res = self.__getPathStat(lfn)
      if not res['OK']:
        failed[lfn] = res['Message']
      else:
        fstat = res['Value']
        successful[lfn] = {}
        successful[lfn]['Size'] = fstat.filesize
        successful[lfn]['ChecksumType'] = fstat.csumtype
        successful[lfn]['Checksum'] = fstat.csumvalue
        successful[lfn]['GUID'] = fstat.guid
        successful[lfn]['Status'] = fstat.status
        successful[lfn]['CreationDate'] = fromEpoch(fstat.ctime)
        successful[lfn]['ModificationDate'] = fromEpoch(fstat.mtime)
        successful[lfn]['NumberOfLinks'] = fstat.nlink
        successful[lfn]['Mode'] = S_IMODE(fstat.filemode)
        if ownership:
          res = getDNFromUID(fstat.uid)
          if res['OK']:
            successful[lfn]['OwnerDN'] = res['Value']
          else:
            successful[lfn]['OwnerDN'] = None
          res = getRoleFromGID(fstat.gid, path=lfn)
          if res['OK']:
            successful[lfn]['OwnerRole'] = res['Value']
          else:
            successful[lfn]['OwnerRole'] = None
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def getFileSize(self, lfns):
    """ Get the size of a supplied file
    """
    # If we have less than three lfns to query a session doesn't make sense
    created = False
    if len(lfns) > 2:
      created = self.__openSession()
      if created < 0:
        return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for lfn in lfns:
      res = self.__getPathStat(lfn)
      if not res['OK']:
        failed[lfn] = res['Message']
      else:
        successful[lfn] = res['Value'].filesize
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def getReplicas(self, lfns, allStatus=False):
    """ Returns replicas for an LFN or list of LFNs
    """
    lfnChunks = breakListIntoChunks(lfns, 1000)
    # If we have less than three groups to query a session doesn't make sense
    created = False
    if len(lfnChunks) > 2:
      created = self.__openSession()
      if created < 0:
        return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for lfnList in lfnChunks:
      fullLfnList = [self.__fullLfn(lfn) for lfn in lfnList if lfn]
      value, replicaList = lfc.lfc_getreplicasl(fullLfnList, '')
      if value != 0:
        for lfn in lfnList:
          reason = lfc.sstrerror(lfc.cvar.serrno)
          if 'Could not secure the connection' in reason:
            # This is a fatal error
            return S_ERROR('Could not secure the connection')
          elif 'Bad credentials' in reason:
            return S_ERROR("Bad Credentials")
        continue
      guid = ''
      it = iter(lfnList)
      replicas = {}
      # This is useless but makes pylinit happy as lfn is defined in the loop when the guid changes
      lfn = None
      for oReplica in replicaList:
        if oReplica.errcode != 0:
          if (oReplica.guid == '') or (oReplica.guid != guid):
            setLfnReplicas(lfn, replicas, successful, failed)
            lfn = it.next()
            failed[lfn] = lfc.sstrerror(oReplica.errcode)
            guid = oReplica.guid
        elif oReplica.sfn == '':
          setLfnReplicas(lfn, replicas, successful, failed)
          lfn = it.next()
          failed[lfn] = 'File has zero replicas'
          guid = oReplica.guid
        else:
          # This is where we change lfn for good!
          if oReplica.guid != guid:
            setLfnReplicas(lfn, replicas, successful, failed)
            lfn = it.next()
            guid = oReplica.guid
          if (oReplica.status != 'P') or allStatus:
            se = oReplica.host
            pfn = oReplica.sfn  # .strip()
            replicas[se] = pfn
      # This is for the last file in the list
      setLfnReplicas(lfn, replicas, successful, failed)
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def getReplicaStatus(self, lfns):
    # If we have less than three lfns to query a session doesn't make sense
    created = False
    if len(lfns) > 2:
      created = self.__openSession()
      if created < 0:
        return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for lfn, se in lfns.items():
      res = self.__getFileReplicaStatus(lfn, se)
      if not res['OK']:
        failed[lfn] = res['Message']
      else:
        successful[lfn] = res['Value']
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def getLFNForPFN(self, pfns):
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for pfn in pfns:
      res = self.__getLFNForPFN(pfn)
      if not res['OK']:
        failed[pfn] = res['Message']
      else:
        successful[pfn] = res['Value']
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  ####################################################################
  #
  # The following a read methods for directories
  #

  @checkCatalogArguments
  def isDirectory(self, lfns):
    """ Determine whether the path is a directory
    """
    # If we have less than three lfns to query a session doesn't make sense
    created = False
    if len(lfns) > 2:
      created = self.__openSession()
      if created < 0:
        return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for lfn in lfns:
      res = self.__getPathStat(lfn)
      if not res['OK']:
        failed[lfn] = res['Message']
      elif S_ISDIR(res['Value'].filemode):
        successful[lfn] = True
      else:
        successful[lfn] = False
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def getDirectoryMetadata(self, lfns):
    # If we have less than three lfns to query a session doesn't make sense
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for lfn in lfns:
      res = self.__getPathStat(lfn)
      if not res['OK']:
        failed[lfn] = res['Message']
      else:
        fstat = res['Value']
        successful[lfn] = {}
        successful[lfn]['Size'] = fstat.filesize
        successful[lfn]['ChecksumType'] = fstat.csumtype
        successful[lfn]['Checksum'] = fstat.csumvalue
        successful[lfn]['GUID'] = fstat.guid
        successful[lfn]['Status'] = fstat.status
        successful[lfn]['CreationDate'] = fromEpoch(fstat.ctime)
        successful[lfn]['ModificationDate'] = fromEpoch(fstat.mtime)
        successful[lfn]['NumberOfSubPaths'] = fstat.nlink
        res = getDNFromUID(fstat.uid)
        if res['OK']:
          successful[lfn]['OwnerDN'] = res['Value']
        else:
          successful[lfn]['OwnerDN'] = None
        res = getRoleFromGID(fstat.gid, path=lfn)
        if res['OK']:
          successful[lfn]['OwnerRole'] = res['Value']
        else:
          successful[lfn]['OwnerRole'] = None
        successful[lfn]['Mode'] = S_IMODE(fstat.filemode)
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def getDirectoryReplicas(self, lfns, allStatus=False):
    """ This method gets all of the pfns in the directory
    """
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for path in lfns:
      res = self.__getDirectoryContents(path)
      if not res['OK']:
        failed[path] = res['Message']
      else:
        pathReplicas = {}
        files = res['Value']['Files']
        for lfn, fileDict in files.items():
          pathReplicas[lfn] = {}
          for se, seDict in fileDict['Replicas'].items():
            pfn = seDict['PFN']
            status = seDict['Status']
            if (status != 'P') or allStatus:
              pathReplicas[lfn][se] = pfn
        successful[path] = pathReplicas
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def listDirectory(self, lfns, verbose=False):
    """ Returns the result of __getDirectoryContents for multiple supplied paths
    """
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for path in lfns:
      res = self.__getDirectoryContents(path, verbose)
      if res['OK']:
        successful[path] = res['Value']
      else:
        failed[path] = res['Message']
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def getDirectorySize(self, lfns, longOutput=False, rawFiles=False):
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for path in lfns.keys():
      res = self.__getDirectorySize(path, longOutput=longOutput)
      if res['OK']:
        successful[path] = res['Value']
      else:
        failed[path] = res['Message']
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  ####################################################################
  #
  # The following are read methods for links
  #

  @checkCatalogArguments
  def isLink(self, links):
    # If we have less than three lfns to query a session doesn't make sense
    failed = {}
    successful = {}
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    for link in links:
      res = self.__getLinkStat(link)
      if not res['OK']:
        failed[link] = res['Message']
      elif S_ISLNK(res['Value'].filemode):
        successful[link] = True
      else:
        successful[link] = False
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def readLink(self, links):
    # If we have less than three lfns to query a session doesn't make sense
    created = False
    if len(links) > 2:
      created = self.__openSession()
      if created < 0:
        return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for link in links:
      res = self.__readLink(link)
      if res['OK']:
        successful[link] = res['Value']
      else:
        failed[link] = res['Message']
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  ####################################################################
  #
  # The following are read methods for datasets
  #

  @checkCatalogArguments
  def resolveDataset(self, datasets, allStatus=False):
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    successful = {}
    failed = {}
    for datasetName in datasets:
      res = self.__getDirectoryContents(datasetName)
      if not res['OK']:
        failed[datasetName] = res['Message']
      else:
        # linkDict = res['Value']['Links']
        linkDict = res['Value']['Files']
        datasetFiles = {}
        for link, fileMetadata in linkDict.items():
          # target = fileMetadata[link]['MetaData']['Target']
          target = link
          datasetFiles[target] = fileMetadata['Replicas']
        successful[datasetName] = datasetFiles
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  ####################################################################
  #
  # The following a write methods for files
  #

  @checkCatalogArguments
  def addFile(self, lfns):
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    baseDirs = []
    for lfn in lfns:
      baseDir = os.path.dirname(lfn)
      if baseDir in baseDirs:
        continue
      baseDirs.append(baseDir)
      res = self.__executeOperation(baseDir, 'exists')
      # If we failed to find out whether the directory exists
      if not res['OK']:
        continue
      # If the directory exists
      if res['Value']:
        continue
      # Make the directories recursively if needed
      res = self.__makeDirs(baseDir)
      # If we failed to make the directory for the file
      if not res['OK']:
        continue
    lfc.lfc_umask(0000)
    for lfnList in breakListIntoChunks(lfns, 1000):
      fileChunk = []
      for lfn in list(lfnList):
        lfnInfo = lfns[lfn]
        pfn = lfnInfo['PFN']
        size = lfnInfo['Size']
        se = lfnInfo['SE']
        guid = lfnInfo['GUID']
        checksum = lfnInfo['Checksum']
        res = self.__checkAddFile(lfn, pfn, size, se, guid, checksum)
        if not res['OK']:
          # Error
          failed[lfn] = res['Message']
          lfnList.remove(lfn)
        elif not res['Value']:
          # File already exists adn is consistent
          successful[lfn] = True
          lfnList.remove(lfn)
        else:
          # File doesn't exist, create it
          oFile = lfc.lfc_filereg()
          oFile.lfn = self.__fullLfn(lfn)
          oFile.sfn = pfn
          oFile.size = size
          oFile.mode = 0o664
          oFile.server = se
          oFile.guid = guid
          oFile.csumtype = 'AD'
          oFile.status = 'U'
          oFile.csumvalue = lfnInfo['Checksum']
          fileChunk.append(oFile)
      if not lfnList:
        continue
      error, errCodes = lfc.lfc_registerfiles(fileChunk)
      if error or (len(errCodes) != len(lfnList)):
        for lfn in lfnList:
          failed[lfn] = lfc.sstrerror(lfc.cvar.serrno)
        continue
      for index in range(len(errCodes)):
        lfn = lfnList[index]
        errCode = errCodes[index]
        if errCode == 0:
          successful[lfn] = True
        elif errCode == 17:
          failed[lfn] = "The supplied GUID is already used"
          res = self.__getLfnForGUID(guid)
          if res['OK']:
            failed[lfn] = "The supplied GUID is already used by %s" % res['Value']
        else:
          failed[lfn] = lfc.sstrerror(errCode)
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def addReplica(self, lfns):
    """ This adds a replica to the catalogue.
    """
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for lfn, info in lfns.items():
      pfn = info['PFN']
      se = info['SE']
      if 'Master' not in info:
        master = False
      else:
        master = info['Master']
      res = self.__getLFNGuid(lfn)
      if not res['OK']:
        failed[lfn] = res['Message']
      else:
        guid = res['Value']
        res = addReplica(guid, pfn, se, master)
        if res['OK']:
          successful[lfn] = True
        else:
          failed[lfn] = res['Message']
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def removeFile(self, lfns):
    """ Remove the supplied path
    """
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    res = self.exists(lfns)
    if not res['OK']:
      if created:
        self.__closeSession()
      return res
    failed = res['Value']['Failed']
    successful = {}
    for lfn, exists in res['Value']['Successful'].items():
      if not exists:
        successful[lfn] = True
      else:
        res = self.__unlinkPath(lfn)
        if res['OK']:
          successful[lfn] = True
        else:
          failed[lfn] = res['Message']
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def removeReplica(self, lfns):
    created = False
    if len(lfns) > 2:
      created = self.__openSession()
      if created < 0:
        return S_ERROR("Error opening LFC session")
    res = self.getReplicas(lfns)  # We need the PFNs of the input lfn (list)
    if not res['OK']:
      return res
    for lfn, lfnrep in res['Value']['Successful'].items():
      if "PFN" not in lfns[lfn]:  # Update only if the PFN was not supplied
        lfns[lfn]["PFN"] = lfnrep[lfns[lfn]['SE']]
    failed = {}
    for lfn, message in res['Value']['Failed'].items():
      if "PFN" not in lfns[lfn]:  # Change only if PFN is not there
        failed[lfn] = message  # The replicas are not available, mark the lfn as failed
        lfns.pop(lfn)  # and remove them
    successful = {}
    for lfn, info in lfns.items():
      if ('PFN' not in info) or ('SE' not in info):
        failed[lfn] = "Required parameters not supplied"
      else:
        pfn = info['PFN']
        se = info['SE']
        res = removeReplica(pfn)
        if res['OK']:
          successful[lfn] = True
        else:
          if res['Message'] == 'No such file or directory':
            # The PFN didn't exist, but maybe it wsa changed...
            res1 = self.getReplicas(lfn)
            if res1['OK']:
              pfn1 = res1['Value']['Successful'].get(lfn, {}).get(se)
              if pfn1 and pfn1 != pfn:
                res = removeReplica(pfn1)
          if res['OK']:
            successful[lfn] = True
          else:
            failed[lfn] = res['Message']
    lfnRemoved = list(successful)
    if len(lfnRemoved) > 0:
      res = self.getReplicas(lfnRemoved, True)
      zeroReplicaFiles = []
      if not res['OK']:
        if created:
          self.__closeSession()
        return res
      else:
        for lfn, repDict in res['Value']['Successful'].items():
          if len(repDict) == 0:
            zeroReplicaFiles.append(lfn)
      if len(zeroReplicaFiles) > 0:
        res = self.removeFile(zeroReplicaFiles)
        if not res['OK']:
          if created:
            self.__closeSession()
          return res
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def setReplicaStatus(self, lfns):
    created = False
    if len(lfns) > 2:
      created = self.__openSession()
      if created < 0:
        return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for lfn, info in lfns.items():
      pfn = info['PFN']
      status = info['Status']
      res = setReplicaStatus(pfn, status[0])
      if res['OK']:
        successful[lfn] = True
      else:
        failed[lfn] = res['Message']
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def setReplicaProblematic(self, lfns, revert=False):
    """
      Set replicas to problematic.

      :param lfn lfns: has to be formated this way :
                  { lfn : { se1 : pfn1, se2 : pfn2, ...}, ...}
      :param revert: If True, remove the problematic flag

      :return: { Successful : { lfn : [ ses ] }, Failed : { lfn : { se : msg } } }
    """

    # This method does a batch treatment because the setReplicaStatus can only take one replica per lfn at once
    #
    # Illustration :
    #
    # lfns {'L2': {'S1': 'P3'}, 'L3': {'S3': 'P5', 'S2': 'P4', 'S4': 'P6'}, 'L1': {'S2': 'P2', 'S1': 'P1'}}
    #
    # loop1: lfnSEs {'L2': ['S1'], 'L3': ['S3', 'S2', 'S4'], 'L1': ['S2', 'S1']}
    # loop1 : batch {'L2': {'Status': 'P', 'SE': 'S1', 'PFN': 'P3'}, 'L3': {'Status': 'P', 'SE': 'S4', 'PFN': 'P6'},
    #                'L1': {'Status': 'P', 'SE': 'S1', 'PFN': 'P1'}}
    #
    # loop2: lfnSEs {'L2': [], 'L3': ['S3', 'S2'], 'L1': ['S2']}
    # loop2 : batch {'L3': {'Status': 'P', 'SE': 'S2', 'PFN': 'P4'}, 'L1': {'Status': 'P', 'SE': 'S2', 'PFN': 'P2'}}
    #
    # loop3: lfnSEs {'L3': ['S3'], 'L1': []}
    # loop3 : batch {'L3': {'Status': 'P', 'SE': 'S3', 'PFN': 'P5'}}
    #
    # loop4: lfnSEs {'L3': []}
    # loop4 : batch {}

    successful = {}
    failed = {}

    status = '-' if revert else 'P'

    # { lfn : [ se1, se2, ...], ...}
    lfnsSEs = dict((lfn, [se for se in lfns[lfn]]) for lfn in lfns)

    while lfnsSEs:

      # { lfn : { 'SE' : se1, 'PFN' : pfn1, 'Status' : status }, ... }
      batch = {}

      for lfn in list(lfnsSEs):
        # If there are still some Replicas (SE) for the given LFN, we put it in the next batch
        # else we remove the entry from the lfnsSEs dict
        if lfnsSEs[lfn]:
          se = lfnsSEs[lfn].pop()
          batch[lfn] = {'SE': se, 'PFN': lfns[lfn][se], 'Status': status}
        else:
          del lfnsSEs[lfn]

      # Happens when there is nothing to treat anymore
      if not batch:
        break

      res = self.setReplicaStatus(batch)
      if not res['OK']:
        for lfn in batch:
          failed.setdefault(lfn, {})[batch[lfn]['SE']] = res['Message']
        continue

      for lfn in res['Value']['Failed']:
        failed.setdefault(lfn, {})[batch[lfn]['SE']] = res['Value']['Failed'][lfn]

      for lfn in res['Value']['Successful']:
        successful.setdefault(lfn, []).append(batch[lfn]['SE'])

    return S_OK({'Successful': successful, 'Failed': failed})

  @checkCatalogArguments
  def setReplicaHost(self, lfns):
    """ This modifies the replica metadata for the SE.
    """
    created = False
    if len(lfns) > 2:
      created = self.__openSession()
      if created < 0:
        return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for lfn, info in lfns.items():
      pfn = info['PFN']
      newse = info['NewSE']
      res = modReplica(pfn, newse)
      if res['OK']:
        successful[lfn] = True
      else:
        failed[lfn] = res['Message']
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  ####################################################################
  #
  # The following are write methods for directories
  #

  @checkCatalogArguments
  def removeDirectory(self, lfns, recursive=False):
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    res = self.exists(lfns)
    if not res['OK']:
      if created:
        self.__closeSession()
      return res
    failed = res['Value']['Failed']
    successful = {}
    for lfn, exists in res['Value']['Successful'].items():
      if not exists:
        successful[lfn] = True
        continue
      if recursive:
        res = self.__removeDirs(lfn)
      else:
        res = self.__removeDirectory(lfn)
      if res['OK']:
        successful[lfn] = True
      else:
        failed[lfn] = res['Message']
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def createDirectory(self, lfns):
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for path in lfns:
      res = self.__makeDirs(path)
      if res['OK']:
        successful[path] = True
      else:
        failed[path] = res['Message']
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  ####################################################################
  #
  # The following are write methods for links
  #

  @checkCatalogArguments
  def createLink(self, links):
    # If we have less than three lfns to query a session doesn't make sense
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for link, target in links.items():
      res = self.__makeDirs(os.path.dirname(link))
      if not res['OK']:
        failed[link] = res['Message']
      else:
        res = self.__makeLink(link, target)
        if not res['OK']:
          failed[link] = res['Message']
        else:
          successful[link] = target
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def removeLink(self, links):
    # If we have less than three lfns to query a session doesn't make sense
    created = False
    if len(links) > 2:
      created = self.__openSession()
      if created < 0:
        return S_ERROR("Error opening LFC session")
    failed = {}
    successful = {}
    for link in links:
      res = self.__unlinkPath(link)
      if not res['OK']:
        failed[link] = res['Message']
      else:
        successful[link] = True
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  ####################################################################
  #
  # The following are write methods for datasets
  #

  @checkCatalogArguments
  def createDataset(self, datasets):
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    successful = {}
    failed = {}
    for datasetName, lfns in datasets.items():
      res = self.__executeOperation(datasetName, 'exists')
      if not res['OK']:
        return res
      elif res['Value']:
        return S_ERROR("LcgFileCatalogClient.createDataset: This dataset already exists.")
      res = self.__createDataset(datasetName, lfns)
      if res['OK']:
        successful[datasetName] = True
      else:
        self.__executeOperation(datasetName, 'removeDataset')
        failed[datasetName] = res['Message']
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def removeDataset(self, datasets):
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    successful = {}
    failed = {}
    for datasetName in datasets:
      res = self.__removeDataset(datasetName)
      if not res['OK']:
        failed[datasetName] = res['Message']
      else:
        successful[datasetName] = True
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def removeFileFromDataset(self, datasets):
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    successful = {}
    failed = {}
    for datasetName, lfns in datasets.items():
      res = self.__removeFilesFromDataset(datasetName, lfns)
      if not res['OK']:
        failed[datasetName] = res['Message']
      else:
        successful[datasetName] = True
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  ####################################################################
  #
  # These are the internal methods to be used by all methods
  #

  def __executeOperation(self, path, method):
    """ Executes the requested functionality with the supplied path
    """
    fcn = None
    if hasattr(self, method) and callable(getattr(self, method)):
      fcn = getattr(self, method)
    if not fcn:
      return S_ERROR("Unable to invoke %s, it isn't a member function of LcgFileCatalogClient" % method)
    res = fcn(path)
    if isinstance(path, dict):
      path = list(path)[0]
    if not res['OK']:
      return res
    elif path not in res['Value']['Successful']:
      return S_ERROR(res['Value']['Failed'][path])
    return S_OK(res['Value']['Successful'][path])

  def __getLFNForPFN(self, pfn):
    fstat = lfc.lfc_filestatg()
    error = lfc.lfc_statr(pfn, fstat)
    return returnCode(error, self.__getLfnForGUID(fstat.guid) if not error else None)

  def __existsLfn(self, lfn):
    """ Check whether the supplied LFN exists
    """
    error = lfc.lfc_access(self.__fullLfn(lfn), 0)
    return returnCode(error and lfc.cvar.serrno != 2, error == 0)

  @checkCatalogArguments
  def getLFNForGUID(self, guids):
    guidLFN = {}
    failed = {}

    for guid in guids:
      # I somehow have the feeling that lfnlist[0] of __getLfnForGUID
      # could throw an exception if the guid does not exist in the DB
      # but... not touching this black magic
      try:
        guidLFN[guid] = self.__getLfnForGUID(guid)['Value']
      except Exception as _e:
        failed[guid] = "GUID does not exist"
    return S_OK({"Successful": guidLFN, "Failed": failed})

  def __getLfnForGUID(self, guid):
    """ Resolve the LFN for a supplied GUID
    """
    if not guid:
      return S_OK()
    linkList = lfc.lfc_list()
    lfnlist = []
    listlinks = lfc.lfc_listlinks('', guid, lfc.CNS_LIST_BEGIN, linkList)
    while listlinks:
      ll = listlinks.path
      if re.search('^' + self.prefix, ll):
        ll = listlinks.path.replace(self.prefix, "", 1)
      lfnlist.append(ll)
      listlinks = lfc.lfc_listlinks('', guid, lfc.CNS_LIST_CONTINUE, linkList)
    else:
      lfc.lfc_listlinks('', guid, lfc.CNS_LIST_END, linkList)
    return S_OK(lfnlist[0])

  def __getBasePath(self, path):
    exists = False
    while not exists:
      res = self.__executeOperation(path, 'exists')
      if not res['OK']:
        return res
      else:
        exists = res['Value']
        if not exists:
          path = os.path.dirname(path)
    return S_OK(path)

  def __getACLInformation(self, path):
    results, objects = lfc.lfc_getacl(self.__fullLfn(path), 256)  # lfc.CNS_ACL_GROUP_OBJ)
    if results == -1:
      errStr = "LcgFileCatalogClient.__getACLInformation: Failed to obtain all path ACLs."
      gLogger.error(errStr, "%s %s" % (path, lfc.sstrerror(lfc.cvar.serrno)))
      return S_ERROR(errStr)
    permissionsDict = {}
    for obj in objects:
      if obj.a_type == lfc.CNS_ACL_USER_OBJ:
        res = getDNFromUID(obj.a_id)
        if not res['OK']:
          return res
        permissionsDict['DN'] = res['Value']
        permissionsDict['user'] = obj.a_perm
      elif obj.a_type == lfc.CNS_ACL_GROUP_OBJ:
        res = getRoleFromGID(obj.a_id, path=path)
        if not res['OK']:
          return res
        role = res['Value']
        permissionsDict['Role'] = role
        permissionsDict['group'] = obj.a_perm
      elif obj.a_type == lfc.CNS_ACL_OTHER:
        permissionsDict['world'] = obj.a_perm
      else:
        errStr = "LcgFileCatalogClient.__getACLInformation: ACL type not considered."
        gLogger.debug(errStr, obj.a_type)
    gLogger.verbose("LcgFileCatalogClient.__getACLInformation: %s owned by %s:%s." % (path,
                                                                                      permissionsDict['DN'],
                                                                                      permissionsDict['Role']))
    return S_OK(permissionsDict)

  def __getPathStat(self, path='', guid=''):
    if path:
      path = self.__fullLfn(path)
    fstat = lfc.lfc_filestatg()
    error = lfc.lfc_statg(path, guid, fstat)
    return returnCode(error, fstat)

  def __getFileReplicas(self, lfn, allStatus):
    error, replicaObjects = lfc.lfc_getreplica(self.__fullLfn(lfn), '', '')
    return returnCode(error or not replicaObjects,
                      dict([(replica.host,
                             replica.sfn) for replica in replicaObjects if allStatus or replica.status != 'P'])
                      if not error else None,
                      errMsg='File has zero replicas' if not error else '')

  def __getFileReplicaStatus(self, lfn, se):
    error, replicaObjects = lfc.lfc_getreplica(self.__fullLfn(lfn), '', '')
    status = None
    for replica in replicaObjects if not error else []:
      if se == replica.host:
        status = replica.status
        break
    return returnCode(status is None, status, errMsg="No replica at supplied site" if not error else '')

  def __checkAddFile(self, lfn, pfn, size, se, guid, checksum):
    res = self.__getPathStat(lfn)
    if not res['OK']:
      if res['Message'] != 'No such file or directory':
        return S_ERROR("Failed to find pre-existance of LFN")
    else:
      # File exists, check if consistent with supplied parameters
      fstat = res['Value']
      errStr = ''
      if fstat.guid != guid:
        errStr = "This LFN %s is already registered with another GUID" % lfn
      elif fstat.filesize != size:
        errStr = "This LFN %s is already registered with another size" % lfn
      elif fstat.csumvalue.upper() != checksum.upper():
        errStr = "This LFN %s is already registered with another adler32" % lfn
      if errStr:
        return S_ERROR(errStr)
      res = self.__getFileReplicas(lfn, True)
      if not res['OK']:
        return S_ERROR("Failed to obtain replicas for existing LFN %s" % lfn)
      replicas = res['Value']
      if replicas.get(se) != pfn:
        return S_ERROR("This LFN %s is already registered with another SE/PFN" % lfn)
      return S_OK(False)
    # We reach here only if the file doesn't exist, which is what we look for!!
    # Now we check the arguments
    try:
      errStr = ''
      size = long(size)
    except Exception:
      errStr = "The size of the file must be an 'int','long' or 'string'"
    if not se:
      errStr = "The SE for the file was not supplied."
    elif not pfn:
      errStr = "The PFN for the file was not supplied."
    elif not lfn:
      errStr = "The LFN for the file was not supplied."
    elif not guid:
      errStr = "The GUID for the file was not supplied."
    elif not checksum:
      errStr = "The adler32 for the file was not supplied."
    if errStr:
      return S_ERROR(errStr)
    return S_OK(True)

  def __unlinkPath(self, lfn):
    return returnCode(lfc.lfc_unlink(self.__fullLfn(lfn)))

  def __removeDirectory(self, path):
    return returnCode(lfc.lfc_rmdir(self.__fullLfn(path)))

  def __removeDirs(self, path):
    """ Black magic contained within...
    """
    res = self.__getDirectoryContents(path)
    if not res['OK']:
      return res
    subDirs = res['Value']['SubDirs']
    files = res['Value']['Files']
    for subDir in subDirs:
      res = self.__removeDirs(subDir)
      if not res['OK']:
        return res
    if files:
      return S_ERROR("Directory not empty")
    return self.__removeDirectory(path)

  def __makeDirs(self, path, mode=0o775):
    """  Black magic contained within....
    """
    dirName = os.path.dirname(path)
    res = self.__executeOperation(path, 'exists')
    if not res['OK']:
      return res
    if res['Value']:
      return S_OK()
    res = self.__executeOperation(dirName, 'exists')
    if not res['OK']:
      return res
    if res['Value']:
      res = self.__makeDirectory(path, mode)
    else:
      res = self.__makeDirs(dirName, mode)
      res = self.__makeDirectory(path, mode)
    return res

  def __makeDirectory(self, path, mode):
    lfc.lfc_umask(0000)
    return returnCode(lfc.lfc_mkdir(self.__fullLfn(path), mode))

  def __openDirectory(self, path):
    value = lfc.lfc_opendirg(self.__fullLfn(path), '')
    return returnCode(not value, value)

  def __getDirectoryContents(self, path, verbose=False):
    """ Returns a dictionary containing all of the contents of a directory.
        This includes the metadata associated to files (replicas, size, guid, status) and the subdirectories found.
    """
    # First check that the directory exists
    res = self.__executeOperation(path, 'exists')
    if not res['OK']:
      return res
    if not res['Value']:
      return S_ERROR('No such file or directory')

    res = self.__getPathStat(path)
    if not res['OK']:
      return res
    nbfiles = res['Value'].nlink
    res = self.__openDirectory(path)
    if not res['OK']:
      return res
    oDirectory = res['Value']
    subDirs = {}
    links = {}
    files = {}
    loop = list(range(nbfiles + 1))
    while loop.pop():
      result = lfc.lfc_readdirxr(oDirectory, "")
      if not result:
        # In some rare cases we reach the end of oDirectory, before nbfiles iterations (!!!)
        break
      entry, fileInfo = result
      pathMetadata = {}
      pathMetadata['Mode'] = S_IMODE(entry.filemode)
      subPath = '%s/%s' % (path, entry.d_name)
      if verbose:
        statRes = self.__getPathStat(subPath)
        if statRes['OK']:
          oPath = statRes['Value']
          pathMetadata['Size'] = oPath.filesize
          pathMetadata['ChecksumType'] = oPath.csumtype
          pathMetadata['Checksum'] = oPath.csumvalue
          pathMetadata['GUID'] = oPath.guid
          pathMetadata['Status'] = oPath.status
          pathMetadata['CreationDate'] = fromEpoch(oPath.ctime)
          pathMetadata['ModificationDate'] = fromEpoch(oPath.mtime)
          pathMetadata['NumberOfLinks'] = oPath.nlink
          pathMetadata['LastAccess'] = oPath.atime
          res = getDNFromUID(oPath.uid)
          if res['OK']:
            pathMetadata['OwnerDN'] = res['Value']
          else:
            pathMetadata['OwnerDN'] = None
          res = getRoleFromGID(oPath.gid, path=subPath)
          if res['OK']:
            pathMetadata['OwnerRole'] = res['Value']
          else:
            pathMetadata['OwnerRole'] = None
      if S_ISDIR(entry.filemode):
        subDirs[subPath] = pathMetadata
      else:
        replicaDict = {}
        if fileInfo:
          for replica in fileInfo:
            replicaDict[replica.host] = {'PFN': replica.sfn, 'Status': replica.status}
        pathMetadata['Size'] = entry.filesize
        pathMetadata['GUID'] = entry.guid
        if S_ISLNK(entry.filemode):
          res = self.__executeOperation(subPath, 'readLink')
          if res['OK']:
            pathMetadata['Target'] = res['Value']
          links[subPath] = {}
          links[subPath]['MetaData'] = pathMetadata
          links[subPath]['Replicas'] = replicaDict
        elif S_ISREG(entry.filemode):
          files[subPath] = {}
          files[subPath]['Replicas'] = replicaDict
          files[subPath]['MetaData'] = pathMetadata
    pathDict = {}
    res = closeDirectory(oDirectory)
    pathDict = {'Files': files, 'SubDirs': subDirs, 'Links': links}
    return S_OK(pathDict)

  def __getDirectorySize(self, path, longOutput=False):
    res = self.__executeOperation(path, 'exists')
    if not res['OK']:
      return res
    if not res['Value']:
      return S_ERROR('No such file or directory')

    res = self.__getPathStat(path)
    if not res['OK']:
      return res
    nbfiles = res['Value'].nlink
    res = self.__openDirectory(path)
    if not res['OK']:
      return res
    oDirectory = res['Value']
    pathDict = {'SubDirs': {}, 'ClosedDirs': [], 'Files': 0, 'TotalSize': 0, 'SiteUsage': {}}
    loop = list(range(nbfiles + 1))
    while loop.pop():
      entry, fileInfo = lfc.lfc_readdirxr(oDirectory, "")
      if S_ISDIR(entry.filemode):
        subDir = '%s/%s' % (path, entry.d_name)
        permissions = S_IMODE(entry.filemode)
        if (not permissions & S_IWUSR) and (not permissions & S_IWGRP) and (not permissions & S_IWOTH):
          pathDict['ClosedDirs'].append(subDir)
        modTime = time.ctime()
        statRes = self.__getPathStat(subDir)
        if statRes['OK']:
          modTime = fromEpoch(statRes['Value'].mtime)
        pathDict['SubDirs'][subDir] = modTime
      else:
        fileSize = entry.filesize
        pathDict['TotalSize'] += fileSize
        pathDict['Files'] += 1
        if not fileInfo:
          gLogger.error("LcgFileCatalogClient.__getDirectorySize: File found with no replicas",
                        "%s/%s" % (path, entry.d_name))
        else:
          for replica in fileInfo:
            if replica.host not in pathDict['SiteUsage']:
              pathDict['SiteUsage'][replica.host] = {'Files': 0, 'Size': 0}
            pathDict['SiteUsage'][replica.host]['Size'] += fileSize
            pathDict['SiteUsage'][replica.host]['Files'] += 1
    res = closeDirectory(oDirectory)
    return S_OK(pathDict)

  def __getLinkStat(self, link):
    lstat = lfc.lfc_filestat()
    return returnCode(lfc.lfc_lstat(self.__fullLfn(link), lstat), lstat)

  def __readLink(self, link):
    buff = " " * (lfc.CA_MAXPATHLEN + 1)
    chars = lfc.lfc_readlink(self.__fullLfn(link), buff, lfc.CA_MAXPATHLEN)
    if chars > 0:
      error = 0
      chars = buff[:chars].replace(self.prefix, '', 1).replace('\x00', '')
    else:
      error = 1
    return returnCode(error, chars)

  def __makeLink(self, source, target):
    return returnCode(lfc.lfc_symlink(self.__fullLfn(target), self.__fullLfn(source)))

  def __getLFNGuid(self, lfn):
    """Get the GUID for the given lfn"""
    fstat = lfc.lfc_filestatg()
    return returnCode(lfc.lfc_statg(self.__fullLfn(lfn), '', fstat), fstat.guid)

  def __createDataset(self, datasetName, lfns):
    res = self.__makeDirs(datasetName)
    if not res['OK']:
      return res
    links = {}
    for lfn in lfns:
      res = self.__getLFNGuid(lfn)
      if not res['OK']:
        return res
      else:
        link = "%s/%s" % (datasetName, res['Value'])
        links[link] = lfn
    res = self.createLink(links)
    if len(res['Value']['Successful']) == len(links):
      return S_OK()
    totalError = ""
    for link, error in res['Value']['Failed'].items():
      gLogger.error("LcgFileCatalogClient.__createDataset: Failed to create link",
                    "for %s: %s" % (link, error))
      totalError = "%s\n %s : %s" % (totalError, link, error)
    return S_ERROR(totalError)

  def __removeDataset(self, datasetName):
    res = self.__getDirectoryContents(datasetName)
    if not res['OK']:
      return res
    links = list(res['Value']['Files'])
    res = self.removeLink(links)
    if not res['OK']:
      return res
    elif len(res['Value']['Failed']):
      return S_ERROR("Failed to remove all links")
    else:
      res = self.__executeOperation(datasetName, 'removeDirectory')
      return res

  def __removeFilesFromDataset(self, datasetName, lfns):
    links = []
    for lfn in lfns:
      res = self.__getLFNGuid(lfn)
      if not res['OK']:
        return res
      guid = res['Value']
      linkPath = "%s/%s" % (datasetName, guid)
      links.append(linkPath)
    res = self.removeLink(links)
    if not res['OK']:
      return res
    if len(res['Value']['Successful']) == len(links):
      return S_OK()
    totalError = ""
    for link, error in res['Value']['Failed'].items():
      gLogger.error("LcgFileCatalogClient.__removeFilesFromDataset: Failed to remove link",
                    "%s: %s" % (link, error))
      totalError = "%s %s : %s" % (totalError, link, error)
    return S_ERROR(totalError)

  ####################################################################
  #
  # These are the methods required for the admin interface
  #

  def getUserDirectory(self, usernames):
    """ Takes a list of users and determines whether their directories already exist
    """
    result = getClientCertInfo()
    if not result['OK']:
      return result
    vo = result['Value']['VO']
    usernameDict = {}
    for username in usernames:
      userDirectory = "/%s/user/%s/%s" % (vo, username[0], username)
      usernameDict[userDirectory] = username
    res = self.exists(list(usernameDict))
    if not res['OK']:
      return res
    failed = {}
    for directory, reason in res['Value']['Failed'].items():
      failed[usernameDict[directory]] = reason
    successful = {}
    for directory, exists in res['Value']['Successful'].items():
      successful[usernameDict[directory]] = exists
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def createUserDirectory(self, usernames):
    """ Creates the user directory
    """
    result = getClientCertInfo()
    if not result['OK']:
      return result
    vo = result['Value']['VO']
    successful = {}
    failed = {}
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    for username in usernames:
      userDirectory = "/%s/user/%s/%s" % (vo, username[0], username)
      res = self.__makeDirs(userDirectory, 0o755)
      if res['OK']:
        successful[username] = userDirectory
      else:
        failed[username] = res['Message']
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def removeUserDirectory(self, username):
    """ Remove the user directory and remove the user mapping
    """
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    res = self.getUserDirectory(username)
    if not res['OK']:
      return res
    failed = {}
    for username, error in res['Value']['Failed'].items():
      failed[username] = error
    directoriesToRemove = {}
    successful = {}
    for username, directory in res['Value']['Successful'].items():
      if not directory:
        successful[username] = True
      else:
        directoriesToRemove[directory] = username
    res = self.removeDirectory(list(directoriesToRemove))
    if not res['OK']:
      return res
    for directory, error in res['Value']['Failed'].items():
      failed[directoriesToRemove[directory]] = error
    for directory in res['Value']['Successful']:
      successful[directoriesToRemove[directory]] = True
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def changePathOwner(self, paths):
    """ Change the ownership of the directory to the user associated to the supplied DN
    """
    successful = {}
    failed = {}
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    for dirPath, dn in paths.items():
      res = getDNUserID(dn)
      if not res['OK']:
        failed[dirPath] = res['Message']
      else:
        userID = res['Value']
        res = self.__changeOwner(dirPath, userID)
        if not res['OK']:
          failed[dirPath] = res['Message']
        else:
          successful[dirPath] = True
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  @checkCatalogArguments
  def changePathMode(self, paths):
    """ Change the ownership of the directory to the user associated to the supplied DN
    """
    successful = {}
    failed = {}
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    for dirPath, mode in paths.items():
      res = self.__changeMode(dirPath, mode)
      if not res['OK']:
        failed[dirPath] = res['Message']
      else:
        successful[dirPath] = True
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def createUserMapping(self, userDNs):
    """ Create a user with the supplied DN and return the userID
    """
    successful = {}
    failed = {}
    created = self.__openSession()
    if created < 0:
      return S_ERROR("Error opening LFC session")
    for userDN, uid in userDNs.items():
      if not uid:
        uid = -1
      res = addUserDN(uid, userDN)
      if not res['OK']:
        failed[userDN] = res['Message']
      else:
        res = getDNUserID(userDN)
        if not res['OK']:
          failed[userDN] = res['Message']
        else:
          successful[userDN] = res['Value']
    if created:
      self.__closeSession()
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  ####################################################################
  #
  # These are the internal methods used for the admin interface
  #

  def __changeOwner(self, lfn, userID):
    return returnCode(lfc.lfc_chown(self.__fullLfn(lfn), userID, -1))

  def __changeMod(self, lfn, mode):
    return returnCode(lfc.lfc_chmod(self.__fullLfn(lfn), mode))

  def __fullLfn(self, lfn):
    return str(self.prefix + lfn)

  # THIS IS NOT YET WORKING