Commit d78bd5f1 authored by Manuela Kuhn's avatar Manuela Kuhn
Browse files

Merge branch 'release-2.3'

parents 28e31449 71d3c0b9
# API to communicate with a data transfer unit
__version__ = '2.2.0'
__version__ = '2.3.0'
import zmq
import socket
......@@ -147,26 +147,7 @@ class dataTransfer():
self.stop()
raise ConnectionFailed("No host to send signal to specified." )
self.targets = []
# [host, port, prio]
if len(targets) == 3 and type(targets[0]) != list and type(targets[1]) != list and type(targets[2]) != list:
host, port, prio = targets
self.targets = [[host + ":" + port, prio]]
# [[host, port, prio], ...]
else:
for t in targets:
if type(t) == list and len(t) == 3:
host, port, prio = t
self.targets.append([host + ":" + port, prio])
else:
self.stop()
self.log.debug("targets=" + str(targets))
raise FormatError("Argument 'targets' is of wrong format.")
# if type(dataPort) == list:
# self.dataHost = str([socket.gethostname() for i in dataPort])
# else:
# self.dataHost = socket.gethostname()
self.__setTargets (targets)
message = self.__sendSignal(signal)
......@@ -215,6 +196,33 @@ class dataTransfer():
self.poller.register(self.signalSocket, zmq.POLLIN)
def __setTargets (self, targets):
self.targets = []
# [host, port, prio]
if len(targets) == 3 and type(targets[0]) != list and type(targets[1]) != list and type(targets[2]) != list:
host, port, prio = targets
self.targets = [[host + ":" + port, prio, [""]]]
# [host, port, prio, suffixes]
elif len(targets) == 4 and type(targets[0]) != list and type(targets[1]) != list and type(targets[2]) != list and type(targets[3]) == list:
host, port, prio, suffixes = targets
self.targets = [[host + ":" + port, prio, suffixes]]
# [[host, port, prio], ...] or [[host, port, prio, suffixes], ...]
else:
for t in targets:
if type(t) == list and len(t) == 3:
host, port, prio = t
self.targets.append([host + ":" + port, prio, [""]])
elif type(t) == list and len(t) == 4 and type(t[3]):
host, port, prio, suffixes = t
self.targets.append([host + ":" + port, prio, suffixes])
else:
self.stop()
self.log.debug("targets=" + str(targets))
raise FormatError("Argument 'targets' is of wrong format.")
def __sendSignal (self, signal):
......@@ -628,6 +636,65 @@ class dataTransfer():
self.log.error("Closing ZMQ context...failed.", exc_info=True)
def forceStop (self, targets):
if type(targets) != list:
self.stop()
raise FormatError("Argument 'targets' must be list.")
if not self.context:
self.context = zmq.Context()
self.extContext = False
signal = None
# Signal exchange
if self.connectionType == "stream":
signalPort = self.signalPort
signal = "STOP_STREAM"
elif self.connectionType == "streamMetadata":
signalPort = self.signalPort
signal = "STOP_STREAM_METADATA"
elif self.connectionType == "queryNext":
signalPort = self.signalPort
signal = "STOP_QUERY_NEXT"
elif self.connectionType == "queryMetadata":
signalPort = self.signalPort
signal = "STOP_QUERY_METADATA"
self.log.debug("Create socket for signal exchange...")
if self.signalHost and not self.signalSocket:
self.__createSignalSocket(signalPort)
elif not self.signalHost:
self.stop()
raise ConnectionFailed("No host to send signal to specified." )
self.__setTargets (targets)
message = self.__sendSignal(signal)
if message and message == "VERSION_CONFLICT":
self.stop()
raise VersionError("Versions are conflicting.")
elif message and message == "NO_VALID_HOST":
self.stop()
raise AuthenticationFailed("Host is not allowed to connect.")
elif message and message == "CONNECTION_ALREADY_OPEN":
self.stop()
raise CommunicationFailed("Connection is already open.")
elif message and message == "NO_VALID_SIGNAL":
self.stop()
raise CommunicationFailed("Connection type is not supported for this kind of sender.")
# if there was no response or the response was of the wrong format, the receiver should be shut down
elif message and message.startswith(signal):
self.log.info("Received confirmation ...")
def __exit__ (self):
self.stop()
......
......@@ -3,10 +3,9 @@
#########################################
# Path where the logfile will be created
#logfilePath = /space/projects/zeromq-data-transfer/logs
logfilePath = /home/kuhnm/Arbeit/zeromq-data-transfer/logs
logfilePath = /space/projects/zeromq-data-transfer/logs
#logfilePath = /home/kuhnm/Arbeit/zeromq-data-transfer/logs
#logfilePath = /home/p11user/zeromq-data-transfer/logs
#logfilePath = /home/p11user/live-viewer/logs
# Filename used for logging
logfileName = dataManager.log
......@@ -57,17 +56,14 @@ fixSubdirs = ["commissioning", "current", "local"]
# Directory to be monitor for changes
# Inside this directory only the subdirectories "commissioning", "current" and "local" are monitored
# (needed if eventDetector is InotifyxDetector or WatchdogDetector)
#monitoredDir = /space/projects/zeromq-data-transfer/data/source
monitoredDir = /home/kuhnm/Arbeit/zeromq-data-transfer/data/source
monitoredDir = /space/projects/zeromq-data-transfer/data/source
#monitoredDir = /home/kuhnm/Arbeit/zeromq-data-transfer/data/source
#monitoredDir = /rd
# Event type of files to be monitored (options are: IN_CLOSE_WRITE, IN_MOVED_TO, ...)
# Event type of files (options are: IN_CLOSE_WRITE, IN_MOVED_TO, ...) and
# the formats to be monitored, files in an other format will be be neglected
# (needed if eventDetector is InotifyxDetector or WatchdogDetector)
monitoredEventType = IN_CLOSE_WRITE
#monitoredEventType = IN_MOVED_TO
# The formats to be monitored, files in an other format will be be neglected
# (needed if eventDetector is InotifyxDetector or WatchdogDetector)
monitoredFormats = [".tif", ".cbf"]
monitoredEvents = {"IN_CLOSE_WRITE" : [".tif", ".cbf", ".nxs"], "IN_MOVED_TO" : [".log"]}
# Number of events stored to look for doubles
# (needed if eventDetector is InotifyxDetector or HttpDetector)
......@@ -78,8 +74,12 @@ historySize = 0
# (needed if eventDetector is InotifyxDetector)
useCleanUp = False
# Intervall time (in seconds) used for clean up resp. checking of events
# (only needed if eventDetectorType is InotifyxDetector or WatchdogDetector)
actionTime = 150
# Time (in seconds) since last modification after which a file will be seen as closed
# (needed if eventDetector is WatchdogDetector)
# (needed if eventDetector is InotifyxDetector (for clean up) or WatchdogDetector)
timeTillClosed = 2
# ZMQ port to get events from
......@@ -110,7 +110,6 @@ dataFetcherType = getFromFile
dataFetcherPort = 50010
# Number of parallel data streams
# if this number is modifified, the port numbers also have to be adjusted
numberOfStreams = 1
# Enable ZMQ pipe into storage system (uses the fixedStreamHost and fixedStreamPort)
......@@ -129,8 +128,8 @@ chunkSize = 10485760 ; # = 1024*1024*10
routerPort = 50004
# Target to move the files into
#localTarget = /space/projects/zeromq-data-transfer/data/target
localTarget = /home/kuhnm/Arbeit/zeromq-data-transfer/data/target
localTarget = /space/projects/zeromq-data-transfer/data/target
#localTarget = /home/kuhnm/Arbeit/zeromq-data-transfer/data/target
#localTarget = /gpfs
# Flag describing if the data should be stored in localTarget
......@@ -138,5 +137,5 @@ localTarget = /home/kuhnm/Arbeit/zeromq-data-transfer/data/target
storeData = True
# Flag describing if the files should be removed from the source
# (needed if dataFetcherType is getFromHttp)
# (needed if dataFetcherType is getFromFile or getFromHttp)
removeData = True
#Ignore everything in this directory
*
# Except this file
!.gitignore
#Ignore everything in this directory
*
# Except this file
!.gitignore
#Ignore everything in this directory
*
# Except this file
!.gitignore
#Ignore everything in this directory
*
# Except this file
!.gitignore
Zeromq Data Transfer 2.2.0
- Fixed problems that the dataTransferAPI could not received data
Zeromq Data Transfer 2.3.0
- Added method to dataTransferAPI to manually stop streams/queries
- Added option to specify which file formats to be send via zeromq
- Added option look for multiple event types in parallel (combined with file suffixes)
- DataManager can now be controlled via tango
- Added systemd service script
- Added cleanup arguments into config file
- Files get accessed only if data or metadata is send via zeromq
- Fixed DataReceiver (no shell)
- Added command line argument error handling
- Removed ringbuffer remains of old architecture
Zeromq Data Transfer 2.2.1
- Fixed data receiving problems with dataTransferAPI due to ZAP
Zeromq Data Transfer 2.2.0
......
#!/usr/bin/env python
import socket
import sys
port = 51000
msgs = [
'set localTarget /root/zeromq-data-transfer/data/target',
# 'set localTarget /space/projects/zeromq-data-transfer/data/target',
'get localTarget',
'set detectorDevice haspp06:10000/p06/eigerdectris/exp.01',
'set filewriterDevice haspp06:10000/p06/eigerfilewriter/exp.01',
'set historySize 0',
'set storeData True',
'set removeData True',
'set whitelist ["localhost","zitpcx19282"]',
'do start',
'do status',
'do stop',
# 'exit'
'bye'
]
#host = socket.gethostname()
host = "asap3-bl-prx07"
sckt = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sckt.connect((host, port))
except Exception, e:
print "connect() failed", e
sckt.close()
sys.exit()
try:
for msg in msgs:
sckt.send(msg)
print "sent (len %2d): %s" % (len(msg), msg)
reply = sckt.recv(1024)
print "recv (len %2d): %s " % (len( reply), reply)
finally:
sckt.close()
#!/usr/bin/env python
#
import threading
import os
import sys
import socket
import subprocess
import logging
from multiprocessing import Queue
try:
BASE_PATH = os.path.dirname ( os.path.dirname ( os.path.dirname ( os.path.realpath ( __file__ ) )))
except:
BASE_PATH = os.path.dirname ( os.path.dirname ( os.path.dirname ( os.path.abspath ( sys.argv[0] ) )))
SHARED_PATH = BASE_PATH + os.sep + "src" + os.sep + "shared"
CONFIG_PATH = BASE_PATH + os.sep + "conf"
if not SHARED_PATH in sys.path:
sys.path.append ( SHARED_PATH )
del SHARED_PATH
del CONFIG_PATH
import helpers
from logutils.queue import QueueHandler
PORT = 51000
BASEDIR = "/root/zeromq-data-transfer"
#BASEDIR = "/space/projects/zeromq-data-transfer"
CONFIGPATH = "/root/zeromq-data-transfer/conf"
#CONFIGPATH = "/space/projects/zeromq-data-transfer/conf"
LOGPATH = "/root/zeromq-data-transfer/logs"
#LOGPATH = "/space/projects/zeromq-data-transfer/logs"
#
# assume that the server listening to 7651 serves p09
#
port2BL = {
"51000": "p00",
"51001": "p01",
"51002": "p02",
"51003": "p03",
"51004": "p04",
"51005": "p05",
"51006": "p06",
"51007": "p07",
"51008": "p08",
"51009": "p09",
"51010": "p10",
"51011": "p11"
}
class ZmqDT():
'''
this class holds getter/setter for all parameters
and function members that control the operation.
'''
def __init__ (self, beamline):
# Beamline is read-only, determined by portNo
self.beamline = beamline
self.procname = "zeromq-data-transfer_" + self.beamline
# TangoDevices to talk with
self.detectorDevice = None
self.filewriterDevice = None
# TODO replace TangoDevices with the following
# IP of the EIGER Detector
# self.eigerIp = None
# Number of events stored to look for doubles
self.historySize = None
# Target to move the files into
# e.g. /beamline/p11/current/raw
self.localTarget = None
# Flag describing if the data should be stored in localTarget
self.storeData = None
# Flag describing if the files should be removed from the source
self.removeData = None
# List of hosts allowed to connect to the data distribution
self.whitelist = None
def execMsg (self, msg):
'''
set filedir /gpfs/current/raw
returns DONE
get filedir
returns /gpfs/current/raw
do reset
return DONE
'''
tokens = msg.split(' ', 2)
if len(tokens) == 0:
return "ERROR"
if tokens[0].lower() == 'set':
if len( tokens) < 3:
return "ERROR"
return self.set(tokens[1], tokens[2])
elif tokens[0].lower() == 'get':
if len( tokens) != 2:
return "ERROR"
return self.get(tokens[1])
elif tokens[0].lower() == 'do':
if len( tokens) != 2:
return "ERROR"
return self.do(tokens[1])
else:
return "ERROR"
def set (self, param, value):
'''
set a parameter, e.g.: set filedir /gpfs/current/raw/
'''
key = param.lower()
if key == "detectordevice":
self.detectorDevice = value
return "DONE"
elif key == "filewriterdevice":
self.filewriterDevice = value
return "DONE"
# TODO replace detectordevice and filewriterDevice with eigerIP
# elif key == "eigerIp":
# self.eigerIP = value
# return "DONE"
elif key == "historysize":
self.historySize = value
return "DONE"
elif key == "localtarget":
self.localTarget = value
return "DONE"
elif key == "storedata":
self.storeData = value
return "DONE"
elif key == "removedata":
self.removeData = value
return "DONE"
elif key == "whitelist":
self.whitelist = value
return "DONE"
else:
return "ERROR"
def get (self, param):
'''
return the value of a parameter, e.g.: get localtarget
'''
key = param.lower()
if key == "detectordevice":
self.detectorDevice = value
return "DONE"
elif key == "filewriterdevice":
self.filewriterDevice = value
return "DONE"
# TODO replace detectordevice and filewriterDevice with eigerIP
# elif key == "eigerIP":
# return self.eigerIp
elif key == "historysize":
return self.historySize
elif key == "localtarget":
return self.localTarget
elif key == "storedata":
return self.storeData
elif key == "removedata":
return self.removeData
elif key == "whitelist":
return self.whitelist
else:
return "ERROR"
def do (self, cmd):
'''
executes commands
'''
key = cmd.lower()
if key == "start":
return self.start()
elif key == "stop":
return self.stop()
elif key == "restart":
return self.restart()
elif key == "status":
return self.status()
else:
return "ERROR"
def start (self):
'''
start ...
'''
#
# see, if all required params are there.
#
if (self.detectorDevice
and self.filewriterDevice
# TODO replace TangoDevices with the following
#and self.eigerIp
and self.historySize
and self.localTarget
and self.storeData
and self.removeData
and self.whitelist ):
#
# execute the start action ...
#
# write configfile
# /etc/zeromq-data-transfer/P01.conf
configFile = CONFIGPATH + os.sep + self.beamline + ".conf"
with open(configFile, 'w') as f:
f.write("logfilePath = " + LOGPATH + "\n")
f.write("logfileName = dataManager.log" + "\n")
f.write("logfileSize = 10485760" + "\n")
f.write("procname = " + self.procname + "\n")
f.write("comPort = 50000" + "\n")
f.write("requestPort = 50001" + "\n")
# f.write("eventDetectorType = HttpDetector" + "\n")
f.write("eventDetectorType = InotifyxDetector" + "\n")
f.write('fixSubdirs = ["commissioning", "current", "local"]' + "\n")
f.write("monitoredDir = " + BASEDIR + "/data/source" + "\n")
f.write("monitoredEventType = IN_CLOSE_WRITE" + "\n")
f.write('monitoredFormats = [".tif", ".cbf"]' + "\n")
f.write("useCleanUp = False" + "\n")
f.write("actionTime = 150" + "\n")
f.write("timeTillClosed = 2" + "\n")
# f.write("dataFetcherType = getFromHttp" + "\n")
f.write("dataFetcherType = getFromFile" + "\n")
f.write("numberOfStreams = 1" + "\n")
f.write("useDataStream = False" + "\n")
f.write("chunkSize = 10485760" + "\n")
f.write("detectorDevice = " + str(self.detectorDevice) + "\n")
f.write("filewriterDevice = " + str(self.filewriterDevice) + "\n")
# TODO replace TangoDevices with the following
#f.write("eigerIp = " + str(self.eigerIp) + "\n")
f.write("historySize = " + str(self.historySize) + "\n")
f.write("localTarget = " + str(self.localTarget) + "\n")
f.write("storeData = " + str(self.storeData) + "\n")
f.write("removeData = " + str(self.removeData) + "\n")
f.write("whitelist = " + str(self.whitelist) + "\n")
# check if service is running
if self.status() == "RUNNING":
return "ERROR"
# start service
p = subprocess.call(["systemctl", "start", "zeromq-data-transfer@" + self.beamline + ".service"])
if p == 0:
return "DONE"
else:
return "ERROR"
<