initial uncomplete batch script maker for tomograms

processing/batchProcFromMDOC.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Wed May 15 09:22:40 2019
+
+@author: adrian
+"""
+
+import configparser
+import os, sys
+from pprint import pprint
+import re
+import argparse
+
+# clusterBaseDir = '/projects/project-nieverge/'
+procDir = 'tomoProc'
+rawDir = 'raw'
+correctedDir = 'corrected'
+alignedDir = 'aligned'
+tiltStackDir = 'tiltStacks'
+subframesDir = 'subframes'
+correctAlignSLURM = "/projects/project-nieverge/scripts/cluster/correctAndAlign.slurm"
+
+
+parser = argparse.ArgumentParser(description = "Generates batch processing scripts for the MPI cluster to process tilt series subframes.")
+
+# parser.add_argument("-f", "--file", help="create overlay for a single file",
+parser.add_argument('-i', '--input', nargs='?', default='.', help='Input MDOC file to process.')
+parser.add_argument('-d', '--outDir', nargs='?', default='.', help='Optional output directory to write scripts to.')
+parser.add_argument('-dp', '--dataPath', nargs='?', default='.', help='Optional override to use if the data is mounted in a different network base path.')
+parser.add_argument('-cp', '--clusterPath', nargs='?', default='/projects/project-nieverge/', help='Destination path of the HPC filesystem data directory.')
+
+# parser.add_argument('-c', '--cutoff', nargs='?', type=float, default='1', help='The threshold of how many standard deviations above or below the a line must deviate before it is corrected')
+# parser.add_argument('-g', '--graph', action='store_true', help='Store a graph of the corrected and uncorrected line intensities for debug purposes.')
+
+args = parser.parse_args()
+
+def FakeSecHead(fp): yield '[general]\n'; yield from fp
+
+if args.input == '.':
+    print("Missing input file");
+    print(parser.print_help())
+    sys.exit()
+
+parser = configparser.ConfigParser()
+
+with open(args.input) as mdocFile:
+    parser.readfp(FakeSecHead(mdocFile))
+
+# getFilename = re.compile()
+
+stackParams = []
+
+for section in parser.sections():
+    if not section.startswith("ZValue"):
+        continue
+    
+    tempDict = {}
+    for option in parser.options(section):
+        tempDict[option] = parser.get(section, option)
+        
+    tempDict['filename'] = re.search("\\\\([\w.-]+)$", tempDict['subframepath']).group(1)
+    for imageParam in ['countsperelectron', 'defocus', 'exposuredose', 'exposuretime', 'intensity', 'pixelspacing', 'priorrecorddose', 'rotationangle', 'stagez', 'targetdefocus', 'tiltangle']:
+        tempDict[imageParam] = float(tempDict[imageParam])
+    for imageParam in ['binning', 'cameraindex', 'dividedby2', 'magindex', 'magnification', 'numsubframes', 'spotsize']:
+        tempDict[imageParam] = int(tempDict[imageParam])
+    
+    stackParams.append(tempDict)
+
+# Data transfer to cluster
+inPath = os.path.splitext(args.input)
+
+stackParams = sorted(stackParams, key = lambda i: i['tiltangle']) 
+
+with open(inPath[0] + "_0_transferToCluster.sh", 'w+') as transferFile:
+    transferFile.write('#!/bin/bash\n')
+    transferFile.write("# Execute on transfer server (e.g mack)\n")
+    transferFile.write("cd \"$(dirname \"$0\")\" # go to script location \n")
+    for imageParams in stackParams:
+        # print(imageParams)
+        transferFile.write("cp -v %s %s \n" % (os.path.join(args.dataPath, subframesDir, imageParams['filename']), os.path.join(args.clusterPath, procDir, rawDir)))
+
+with open(inPath[0] + "_1_correctAndAlign.sh", 'w+') as transferFile:
+    transferFile.write('#!/bin/bash\n')
+    transferFile.write("# Execute on any computer with ssh acceess to cluster (falcon)\n")
+    transferFile.write("cd \"$(dirname \"$0\")\" # go to script location \n")
+    for imageParams in stackParams:
+        transferFile.write("ssh falcon sbatch %s %s %f\n" % (correctAlignSLURM, imageParams['filename'], imageParams['tiltangle']))
+
+# write stackup list to 
+with open(inPath[0] + "_aux0.stackList", 'w+') as transferFile:
+    transferFile.write('%i\n' % (len(stackParams)))
+    for imageParams in stackParams:
+        transferFile.write("%s\n0\n" % (imageParams['filename']))
+        
+# write corresponding tilt angles file
+with open(inPath[0] + "_aux1.tiltsList", 'w+') as transferFile:
+    for imageParams in stackParams:
+        transferFile.write("%f\n" % (imageParams['tiltangle']))
+
+metaDataFile = os.path.splitext(os.path.basename(args.input))
+
+with open(inPath[0] + "_2_makeStack.sh", 'w+') as transferFile:
+    transferFile.write('#!/bin/bash\n')
+    transferFile.write("# Execute on any computer with ssh acceess to cluster (falcon)\n")
+    transferFile.write("cd \"$(dirname \"$0\")\" # go to script location \n")
+    transferFile.write("scp %s mack:%s\n" % (metaDataFile[0] + "_aux0.stackList", os.path.join(args.clusterPath, procDir, alignedDir)))
+    transferFile.write("scp %s mack:%s\n" % (metaDataFile[0] + "_aux1.tiltsList", os.path.join(args.clusterPath, procDir, alignedDir)))
\ No newline at end of file