GridUtilityScripts/Merge.sh at master · ariedel-cern/GridUtilityScripts · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
#!/bin/bash
# File              : Merge.sh
# Author            : Anton Riedel <anton.riedel@tum.de>
# Date              : 24.03.2021
# Last Modified Date: 25.02.2022
# Last Modified By  : Anton Riedel <anton.riedel@tum.de>

# merge .root files run by run

[ ! -f config.json ] && echo "No config file!!!" && exit 1
LockFile="$(jq -r '.LockFile' config.json)"
StatusFile="$(jq -r '.StatusFile' config.json)"
[ ! -f $StatusFile ] && echo "No $StatusFile file!!!" && exit 1

echo "Merging $(jq -r '.task.GridOutputFile' config.json) run by run in $(jq -r '.task.GridOutputDir' config.json)"

MergedFile=""
OutputFile="$(jq -r '.task.GridOutputFile' config.json)"
Runs="$(jq -r 'keys[]' $StatusFile)"

Data=""
Status=""
FilesCopied=""
FilesChecked=""
FilesMerged=""

for Run in $Runs; do

	echo "Waiting for lock..."
	{
		flock 100
		Data="$(jq -r --arg Run "$Run" '.[$Run]' $StatusFile)"
	} 100>$LockFile

	echo "Trying to merge $Run"
	Status="$(jq -r '.Status' <<<$Data)"
	FilesCopied="$(jq -r '.FilesCopied' <<<$Data)"
	FilesChecked="$(jq -r '.FilesChecked' <<<$Data)"
	FilesMerged="$(jq -r '.Merged' <<<$Data)"

	[ "${Status:=RUNNING}" != "DONE" ] && continue
	echo "Run is DONE"
	[ "${FilesMerged:=0}" -ge "1" ] && continue
	echo "Run is not merged yet"
	[ "$((${FilesCopied:=10} - 2))" -gt "${FilesChecked:=0}" ] && continue
	echo "Files are copied and check, let's go..."

	RunDir="$(jq -r '.task.GridOutputDir' config.json)/${Run}"

	# go into subdirectory
	pushd $RunDir

	echo "Start merging run $Run in $PWD"

	# construct filename for merged file
	MergedFile="$(basename $Run)_Merged.root"
	FilesToMerge="$(find . -type f -name "$OutputFile")"

	# merge files using hadd in parallel!!!
	hadd -f -k -j $(nproc) $MergedFile $FilesToMerge || exit 1

	# create backup, just in case
	cp "${MergedFile}" "${MergedFile}.bak"

	# count number of merge files
	FilesMerged="$(wc -l <<<$FilesToMerge)"

	# delete smaller root files (safing space on the local disk)
	find . -type f -name "$OutputFile" -delete

	# delete all empty directories
	find . -empty -type d -delete

	# go back
	popd

	echo "Waiting for lock..."
	{
		flock 100
		jq --arg Run "$Run" --arg FilesMerged "${FilesMerged:=0}" 'setpath([$Run,"Merged"];$FilesMerged)' $StatusFile | sponge $StatusFile
	} 100>$LockFile

done

exit 0