Some clean up

conf/receiver.conf

-# where incoming data will be stored to
+# Where incoming data will be stored to
 targetDir         = /space/projects/live-viewer/data/target
 
-# local ip to connect dataStream to
+# Local IP to connect dataStream to
 dataStreamIp      = 131.169.185.121     # zitpcx19282.desy.de
-# tcp port of data pipe"
+# TCP port of data pipe
 dataStreamPort    = 6061
 
-# path where logfile will be created
+# Path where logfile will be created
 logfilePath       = /space/projects/live-viewer/logs
-# filename used for logging
+# Filename used for logging
 logfileName       = zmq_receiver.log

conf/receiverLiveViewer.conf

-# where incoming data will be stored to"
+# Where incoming data will be stored to"
 targetDir             = /space/projects/live-viewer/data/zmq_target
 
-# local ip to connect dataStream to
+# Local ip to connect dataStream to
 dataStreamIp          = 127.0.0.1
-# tcp port of data pipe"
+# TCP port of data pipe"
 dataStreamPort        = 6070
-# local ip to bind LiveViewer to
+# Local ip to bind LiveViewer to
 liveViewerIp          = 127.0.0.1
-# tcp port of live viewer"
+# TCP port of live viewer
 liveViewerPort        = 6071
 
-# port number of dataStream-socket to send signals back to the sender
+# Port number of dataStream-socket to send signals back to the sender
 senderComPort         = 6080
-# time to wait for the sender to give a confirmation of the signal
+# Time to wait for the sender to give a confirmation of the signal
 senderResponseTimeout = 1000
 
-# path where logfile will be created
+# Path where logfile will be created
 logfilePath           = /space/projects/live-viewer/logs
-# filename used for logging
+# Filename used for logging
 logfileName           = zmq_receiver_LiveViewer.log
-# size of the ring buffer for the live viewer
+# Size of the ring buffer for the live viewer
 maxRingBufferSize     = 10
 

conf/sender.conf

-# folder you want to monitor for changes
-# inside this folder only the subdirectories "commissioning", "current" and "local" are monitored
+# Folder you want to monitor for changes
+# Inside this folder only the subdirectories "commissioning", "current" and "local" are monitored
 watchFolder         = /space/projects/live-viewer/data/source/
 # Target to move the files into
 cleanerTargetPath   = /space/projects/live-viewer/data/target/
 
-# subfolders of watchFolders to be monitored
+# Subfolders of watchFolders to be monitored
 monitoredSubfolders = ["commissioning", "current", "local"]
-# the formats to be monitored, files in an other format will be be neglected
+# The formats to be monitored, files in an other format will be be neglected
 monitoredFormats    = [".tif", ".cbf"]
 
-# number of parallel data streams
+# Number of parallel data streams
 parallelDataStreams = 1
 
-# list of hosts allowed to connect to the sender
+# List of hosts allowed to connect to the sender
 receiverWhiteList   = ["zitpcx19282", "zitpcx22614", "lsdma-lab04" , "haspp11eval01" , "it-hpc-cxi04", "it-hpc-cxi03" ]
 
+# Enable ZMQ pipe into storage system (if set to false: the file is moved into the cleanerTargetPath)
 useDataStream       = True
 
-# zmq endpoint (IP-address) to send file events to
+# ZMQ endpoint (IP-address) to send file events to
 fileEventIp         = 127.0.0.1
-# zmq endpoint (port) to send file events to
+# ZMQ endpoint (port) to send file events to
 fileEventPort       = 6060
-# ip of dataStream-socket to push new files to
+# IP of dataStream-socket to push new files to
 dataStreamIp        = zitpcx19282.desy.de
-# port number of dataStream-socket to push new files to
+# Port number of dataStream-socket to push new files to
 dataStreamPort      = 6061
-# zmq-pull-socket ip which deletes/moves given files
+# ZMQ-pull-socket IP which deletes/moves given files
 cleanerIp           = 127.0.0.1
-# zmq-pull-socket port which deletes/moves given files
+# ZMQ-pull-socket port which deletes/moves given files
 cleanerPort         = 6062
-# ip receive signals from the receiver
+# IP receive signals from the receiver
 receiverComIp       = 127.0.0.1
-# port number to receive signals from the receiver
+# Port number to receive signals from the receiver
 receiverComPort     = 6080
-# ip of liveViewer-socket to send new files to
+# IP of liveViewer-socket to send new files to
 liveViewerIp        = 127.0.0.1
-# port number of liveViewer-socket to send data to
+# Port number of liveViewer-socket to send data to
 liveViewerPort      = 6070
-# ports and ips to communicate with onda/realtime analysis
-# there needs to be one entry for each workerProcess (meaning streams)
+# Ports and ips to communicate with onda/realtime analysis
+# There needs to be one entry for each workerProcess (meaning streams)
 ondaIps             = ["127.0.0.1"]
 ondaPorts           = ["6081"]
 
-# chunk size of file-parts getting send via zmq
+# Chunk size of file-parts getting send via zmq
 chunkSize           = 10485760 ; # = 1024*1024*10
 #chunkSize           = 1073741824 ; # = 1024*1024*1024
 
-# path where the logfile will be created
+# Path where the logfile will be created
 logfilePath         = /space/projects/live-viewer/logs
 
-# filename used for logging
+# Filename used for logging
 logfileName         = zmq_sender.log
 

src/receiver/FileReceiver.py

@@ -42,7 +42,6 @@ class FileReceiver:
         # create pull socket
         self.zmqDataStreamSocket = self.zmqContext.socket(zmq.PULL)
         connectionStr            = "tcp://{ip}:{port}".format(ip=self.zmqDataStreamIp, port=self.zmqDataStreamPort)
-        print "connectionStr", connectionStr
         self.zmqDataStreamSocket.bind(connectionStr)
         self.log.debug("zmqDataStreamSocket started (bind) for '" + connectionStr + "'")
 

src/sender/FileMover.py

@@ -87,7 +87,6 @@ class FileMover():
         # create zmq socket for incoming file events
         self.fileEventSocket = self.zmqContext.socket(zmq.PULL)
         connectionStr        = "tcp://{ip}:{port}".format(ip=self.fileEventIp, port=self.fileEventPort)
-        print connectionStr
         self.fileEventSocket.bind(connectionStr)
         self.log.debug("fileEventSocket started (bind) for '" + connectionStr + "'")
 

src/sender/InotifyDetector.py

@@ -180,7 +180,7 @@ class InotifyDetector():
 #            is_closed  = ("IN_CLOSE" in parts_array or "IN_CLOSE_WRITE" in parts_array)
             is_created = ("IN_CREATE" in parts_array)
 
-            # if a new directory is created inside the monitored one,
+            # if a new directory is created or a directory is renamed inside the monitored one,
             # this one has to be monitored as well
             if is_dir and (is_created or is_moved_to):
                 dirname = os.path.join(path, event.name)
@@ -193,8 +193,8 @@ class InotifyDetector():
                     self.log.info("Added new directory to watch:" + str(dirname))
                 continue
 
+            # if a directory is renamed the old watch has to be removed
             if is_dir and is_moved_from:
-                print "---moved from", event.name
                 dirname = os.path.join(path, event.name)
                 for watch, watchPath in self.wd_to_path.iteritems():
                     if watchPath == dirname:
@@ -209,19 +209,21 @@ class InotifyDetector():
                 del self.wd_to_path[foundWatch]
                 continue
 
-            # TODO check if still necessary
-            # checks if one of the suffixes to monitore is contained in the event.name
-            resultSuffix = filter(lambda x: x in event.name, self.monitoredSuffixes)
-
-            if not resultSuffix:
-            #if not event.name.endswith(self.monitoredSuffixes):
-                self.log.debug("File ending not in monitored Suffixes: " + str(event.name))
-                self.log.debug("detected events were: " + str(parts))
-                continue
-
 
             # only closed files are send
             if not is_dir and is_closed:
+
+                # TODO check if still necessary
+                # checks if one of the suffixes to monitore is contained in the event.name
+                resultSuffix = filter(lambda x: x in event.name, self.monitoredSuffixes)
+
+                # only files with end with a suffix specified in monitoredSuffixed are monitored
+                if not resultSuffix:
+                #if not event.name.endswith(self.monitoredSuffixes):
+                    self.log.debug("File ending not in monitored Suffixes: " + str(event.name))
+                    self.log.debug("detected events were: " + str(parts))
+                    continue
+
 #                if self.previousEventPath != path or self.previousEventName != event.name:
                 if True:
 #            if (is_moved and not is_dir) or (is_closed and not is_dir):
@@ -242,7 +244,7 @@ class InotifyDetector():
                     while splitPath:
                         if parentDir not in self.paths:
                             (parentDir,relDir) = os.path.split(parentDir)
-                            print "debug1:", parentDir, relDir
+#                            print "debug1:", parentDir, relDir
                             # the os.sep is needed at the beginning because the relative path is built up from the right
                             # e.g.
                             # self.paths = ["/tmp/test/source"]
@@ -250,7 +252,7 @@ class InotifyDetector():
                             # first iteration:  parentDir = /tmp/test/source/local, relDir = /testfolder
                             # second iteration: parentDir = /tmp/test/source,       relDir = /local/testfolder
                             relativePath = os.sep + relDir + relativePath
-                            print "debug11:", relativePath
+#                            print "debug11:", relativePath
                         else:
                             # the event for a file /tmp/test/source/local/file1.tif is of the form:
                             # {