Commit 057a93e7 authored by Manuela Kuhn's avatar Manuela Kuhn
Browse files

File only get touched if send or stored

parent 71dbd216
......@@ -222,10 +222,10 @@ class DataDispatcher():
targets = []
# get metadata of the file
# get metadata and paths of the file
try:
self.log.debug("Getting file metadata")
sourceFile, targetFile, metadata = self.dataFetcher.getMetadata(self.log, workload, self.chunkSize, self.localTarget)
self.log.debug("Getting file paths and metadata")
sourceFile, targetFile, metadata = self.dataFetcher.getMetadata(self.log, self.dataFetcherProp, targets, workload, self.chunkSize, self.localTarget)
except:
self.log.error("Building of metadata dictionary failed for workload: " + str(workload) + ".", exc_info=True)
......
......@@ -28,7 +28,7 @@ def setup (log, prop):
return True
def getMetadata (log, metadata, chunkSize, localTarget = None):
def getMetadata (log, prop, targets, metadata, chunkSize, localTarget = None):
#extract fileEvent metadata
try:
......@@ -56,51 +56,54 @@ def getMetadata (log, metadata, chunkSize, localTarget = None):
else:
targetFile = None
try:
# For quick testing set filesize of file as chunksize
log.debug("get filesize for '" + str(sourceFile) + "'...")
filesize = os.path.getsize(sourceFile)
fileModTime = os.stat(sourceFile).st_mtime
fileCreateTime = os.stat(sourceFile).st_ctime
chunksize = filesize #can be used later on to split multipart message
log.debug("filesize(%s) = %s" % (sourceFile, str(filesize)))
log.debug("fileModTime(%s) = %s" % (sourceFile, str(fileModTime)))
if targets:
try:
# For quick testing set filesize of file as chunksize
log.debug("get filesize for '" + str(sourceFile) + "'...")
filesize = os.path.getsize(sourceFile)
fileModTime = os.stat(sourceFile).st_mtime
fileCreateTime = os.stat(sourceFile).st_ctime
chunksize = filesize #can be used later on to split multipart message
log.debug("filesize(%s) = %s" % (sourceFile, str(filesize)))
log.debug("fileModTime(%s) = %s" % (sourceFile, str(fileModTime)))
except:
log.error("Unable to create metadata dictionary.")
raise
except:
log.error("Unable to create metadata dictionary.")
raise
try:
log.debug("create metadata for source file...")
#metadata = {
# "filename" : filename,
# "sourcePath" : sourcePath,
# "relativePath" : relativePath,
# "filesize" : filesize,
# "fileModTime" : fileModTime,
# "chunkSize" : self.zmqMessageChunkSize
# }
metadata[ "filesize" ] = filesize
metadata[ "fileModTime" ] = fileModTime
metadata[ "fileCreateTime"] = fileCreateTime
metadata[ "chunkSize" ] = chunkSize
log.debug("metadata = " + str(metadata))
except:
log.error("Unable to assemble multi-part message.")
raise
try:
log.debug("create metadata for source file...")
#metadata = {
# "filename" : filename,
# "sourcePath" : sourcePath,
# "relativePath" : relativePath,
# "filesize" : filesize,
# "fileModTime" : fileModTime,
# "chunkSize" : self.zmqMessageChunkSize
# }
metadata[ "filesize" ] = filesize
metadata[ "fileModTime" ] = fileModTime
metadata[ "fileCreateTime"] = fileCreateTime
metadata[ "chunkSize" ] = chunkSize
log.debug("metadata = " + str(metadata))
except:
log.error("Unable to assemble multi-part message.")
raise
return sourceFile, targetFile, metadata
return sourceFile, targetFile, metadata
else:
return sourceFile, targetFile, dict()
def sendData (log, targets, sourceFile, targetFile, metadata, openConnections, context, prop):
targets_data = [i for i in targets if i[2] == "data"]
if not targets_data:
if not targets:
prop["removeFlag"] = True
return
targets_data = [i for i in targets if i[2] == "data"]
prop["removeFlag"] = False
chunkSize = metadata[ "chunkSize" ]
......
......@@ -34,8 +34,7 @@ def setup (log, prop):
return True
def getMetadata (log, metadata, chunkSize, localTarget = None):
def getMetadata (log, prop, targets, metadata, chunkSize, localTarget = None):
#extract fileEvent metadata
try:
......
......@@ -35,7 +35,7 @@ def setup (log, prop):
return True
def getMetadata (log, metadata, chunkSize, localTarget = None):
def getMetadata (log, prop, targets, metadata, chunkSize, localTarget = None):
#extract fileEvent metadata
try:
......
......@@ -379,7 +379,8 @@ class EventDetector():
is_ofEventType = (self.monEventType in parts_array)
# self.log.debug(path + " " + event.name + " " + parts)
# if not is_dir:
# self.log.debug(path + " " + event.name + " " + parts)
# self.log.debug(event.name)
# self.log.debug("is_dir: " + str(is_dir))
# self.log.debug("is_created: " + str(is_created))
......@@ -527,7 +528,10 @@ if __name__ == '__main__':
"monSubdirs" : ["commissioning", "current", "local"],
"monSuffixes" : [".tif", ".cbf"],
"timeout" : 0.1,
"historySize" : 0
"historySize" : 0,
"useCleanUp" : False,
"cleanUpTime" : 5,
"actionTime" : 120
}
eventDetector = EventDetector(config, logQueue)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment