Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
H
hidra
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container Registry
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
steinbac
hidra
Commits
41daad8d
Commit
41daad8d
authored
8 years ago
by
Manuela Kuhn
Browse files
Options
Downloads
Patches
Plain Diff
Renamed remove and store parameters in config
parent
938ccfb9
No related branches found
Branches containing commit
No related tags found
Tags containing commit
No related merge requests found
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
conf/dataManager.conf
+2
-2
2 additions, 2 deletions
conf/dataManager.conf
src/sender/DataManager.py
+11
-11
11 additions, 11 deletions
src/sender/DataManager.py
src/sender/dataFetchers/getFromHttp.py
+12
-12
12 additions, 12 deletions
src/sender/dataFetchers/getFromHttp.py
with
25 additions
and
25 deletions
conf/dataManager.conf
+
2
−
2
View file @
41daad8d
...
...
@@ -125,8 +125,8 @@ localTarget = /space/projects/zeromq-data-transfer/data/target
# Flag describing if the data should be stored in localTarget
# (needed if dataFetcherType is getFromFile or getFromHttp)
store
Flag
=
True
store
Data
=
True
# Flag describing if the files should be removed from the source
# (needed if dataFetcherType is getFromHttp)
remove
Flag
=
True
remove
Data
=
True
This diff is collapsed.
Click to expand it.
src/sender/DataManager.py
+
11
−
11
View file @
41daad8d
...
...
@@ -178,8 +178,8 @@ def argumentParsing():
localTarget
=
config
.
get
(
'
asection
'
,
'
localTarget
'
)
store
Flag
=
config
.
getboolean
(
'
asection
'
,
'
store
Flag
'
)
remove
Flag
=
config
.
getboolean
(
'
asection
'
,
'
remove
Flag
'
)
store
Data
=
config
.
getboolean
(
'
asection
'
,
'
store
Data
'
)
remove
Data
=
config
.
getboolean
(
'
asection
'
,
'
remove
Data
'
)
parser
.
add_argument
(
"
--dataFetcherType
"
,
type
=
str
,
...
...
@@ -218,14 +218,14 @@ def argumentParsing():
help
=
"
Target to move the files into (default=
"
+
str
(
localTarget
)
+
"
)
"
,
default
=
localTarget
)
parser
.
add_argument
(
"
--store
Flag
"
,
type
=
bool
,
parser
.
add_argument
(
"
--store
Data
"
,
type
=
bool
,
help
=
"
Flag describing if the data should be stored in localTarget
\
(needed if dataFetcherType is getFromFile or getFromHttp; default=
"
+
str
(
store
Flag
)
+
"
)
"
,
default
=
store
Flag
)
parser
.
add_argument
(
"
--remove
Flag
"
,
type
=
bool
,
(needed if dataFetcherType is getFromFile or getFromHttp; default=
"
+
str
(
store
Data
)
+
"
)
"
,
default
=
store
Data
)
parser
.
add_argument
(
"
--remove
Data
"
,
type
=
bool
,
help
=
"
Flag describing if the files should be removed from the source
\
(needed if dataFetcherType is getFromHttp; default=
"
+
str
(
remove
Flag
)
+
"
)
"
,
default
=
remove
Flag
)
(needed if dataFetcherType is getFromHttp; default=
"
+
str
(
remove
Data
)
+
"
)
"
,
default
=
remove
Data
)
arguments
=
parser
.
parse_args
()
...
...
@@ -372,7 +372,7 @@ class DataManager():
self
.
dataFetcherProp
=
{
"
type
"
:
arguments
.
dataFetcherType
,
"
fixSubdirs
"
:
arguments
.
fixSubdirs
,
"
store
Flag
"
:
arguments
.
store
Flag
,
"
store
Data
"
:
arguments
.
store
Data
,
"
removeFlag
"
:
False
}
elif
arguments
.
dataFetcherType
==
"
getFromZmq
"
:
...
...
@@ -386,8 +386,8 @@ class DataManager():
self
.
dataFetcherProp
=
{
"
type
"
:
arguments
.
dataFetcherType
,
"
session
"
:
None
,
"
store
Flag
"
:
arguments
.
store
Flag
,
"
remove
Flag
"
:
arguments
.
remove
Flag
"
store
Data
"
:
arguments
.
store
Data
,
"
remove
Data
"
:
arguments
.
remove
Data
}
...
...
This diff is collapsed.
Click to expand it.
src/sender/dataFetchers/getFromHttp.py
+
12
−
12
View file @
41daad8d
...
...
@@ -18,8 +18,8 @@ from send_helpers import __sendToTargets
def
setup
(
log
,
prop
):
if
(
not
prop
.
has_key
(
"
session
"
)
or
not
prop
.
has_key
(
"
store
Flag
"
)
or
not
prop
.
has_key
(
"
remove
Flag
"
)
):
not
prop
.
has_key
(
"
store
Data
"
)
or
not
prop
.
has_key
(
"
remove
Data
"
)
):
log
.
error
(
"
Configuration of wrong format
"
)
log
.
debug
(
"
dataFetcherProp=
"
+
str
(
prop
))
...
...
@@ -96,7 +96,7 @@ def sendData (log, targets, sourceFile, targetFile, metadata, openConnections,
except
:
log
.
error
(
"
Unable to get chunkSize
"
,
exc_info
=
True
)
if
prop
[
"
store
Flag
"
]:
if
prop
[
"
store
Data
"
]:
try
:
log
.
debug
(
"
Opening
'"
+
str
(
targetFile
)
+
"'
...
"
)
fileDescriptor
=
open
(
str
(
targetFile
),
"
wb
"
)
...
...
@@ -109,11 +109,11 @@ def sendData (log, targets, sourceFile, targetFile, metadata, openConnections,
if
metadata
[
"
relativePath
"
]
in
prop
[
"
fixSubdirs
"
]:
log
.
error
(
"
Unable to move file
'"
+
sourceFile
+
"'
to
'"
+
targetFile
+
"
: Directory
"
+
metadata
[
"
relativePath
"
]
+
"
is not available
"
,
exc_info
=
True
)
prop
[
"
remove
Flag
"
]
=
False
prop
[
"
remove
Data
"
]
=
False
elif
subdir
in
prop
[
"
fixSubdirs
"
]
:
log
.
error
(
"
Unable to move file
'"
+
sourceFile
+
"'
to
'"
+
targetFile
+
"
: Directory
"
+
subdir
+
"
is not available
"
,
exc_info
=
True
)
prop
[
"
remove
Flag
"
]
=
False
prop
[
"
remove
Data
"
]
=
False
else
:
try
:
targetPath
,
filename
=
os
.
path
.
split
(
targetFile
)
...
...
@@ -126,11 +126,11 @@ def sendData (log, targets, sourceFile, targetFile, metadata, openConnections,
raise
else
:
log
.
error
(
"
Unable to open target file
'"
+
targetFile
+
"'
.
"
,
exc_info
=
True
)
prop
[
"
remove
Flag
"
]
=
False
prop
[
"
remove
Data
"
]
=
False
except
:
log
.
error
(
"
Unable to open target file
'"
+
targetFile
+
"'
.
"
,
exc_info
=
True
)
log
.
debug
(
"
e.errno =
"
+
str
(
e
.
errno
)
+
"
errno.EEXIST==
"
+
str
(
errno
.
EEXIST
))
prop
[
"
remove
Flag
"
]
=
False
prop
[
"
remove
Data
"
]
=
False
targets_data
=
[
i
for
i
in
targets
if
i
[
2
]
==
"
data
"
]
targets_metadata
=
[
i
for
i
in
targets
if
i
[
2
]
==
"
metadata
"
]
...
...
@@ -152,7 +152,7 @@ def sendData (log, targets, sourceFile, targetFile, metadata, openConnections,
except
:
log
.
error
(
"
Unable to pack multipart-message for file
"
+
str
(
sourceFile
),
exc_info
=
True
)
if
prop
[
"
store
Flag
"
]:
if
prop
[
"
store
Data
"
]:
fileDescriptor
.
write
(
data
)
#send message to data targets
...
...
@@ -165,7 +165,7 @@ def sendData (log, targets, sourceFile, targetFile, metadata, openConnections,
chunkNumber
+=
1
if
prop
[
"
store
Flag
"
]:
if
prop
[
"
store
Data
"
]:
try
:
log
.
debug
(
"
Closing
'"
+
str
(
targetFile
)
+
"'
...
"
)
fileDescriptor
.
close
()
...
...
@@ -185,7 +185,7 @@ def sendData (log, targets, sourceFile, targetFile, metadata, openConnections,
def
finishDataHandling
(
log
,
targets
,
sourceFile
,
targetFile
,
metadata
,
openConnections
,
context
,
prop
):
if
prop
[
"
remove
Flag
"
]:
if
prop
[
"
remove
Data
"
]:
#TODO delete file from detector after sending
responce
=
requests
.
delete
(
sourceFile
)
...
...
@@ -274,8 +274,8 @@ if __name__ == '__main__':
dataFetcherProp
=
{
"
type
"
:
"
getFromHttp
"
,
"
session
"
:
None
,
"
store
Flag
"
:
True
,
"
remove
Flag
"
:
False
"
store
Data
"
:
True
,
"
remove
Data
"
:
False
}
setup
(
logging
,
dataFetcherProp
)
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment