-
Notifications
You must be signed in to change notification settings - Fork 6
Expand file tree
/
Copy pathdead_code.txt
More file actions
190 lines (166 loc) · 10.3 KB
/
dead_code.txt
File metadata and controls
190 lines (166 loc) · 10.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
##############################################################################################
# def CertVerify_v2(fname: str) -> RET:
# """Print certificate information (subject, issuer, notbefore, notafter)"""
# cert_bytes = None
# try:
# with open(fname, "rb") as f: cert_bytes = f.read()
# except Exception:
# return RET(2, "", f"File >>>{fname}<<< not found") # ENOENT /* No such file or directory */
#
# try:
# x509cert = x509.load_pem_x509_certificate(cert_bytes)
# except Exception:
# logging.debug(traceback.format_exc())
# return RET(5, "", f"Could not load certificate >>>{fname}<<<") # EIO /* I/O error */
#
# ca_pem_list = glob.glob(f'{CA_PATH}/*.pem')
# ca_list_bytes = bytearray()
# for ca in ca_pem_list:
# try:
# with open(ca, "rb") as f: ca_list_bytes += f.read()
# except Exception:
# pass
# ca_list_x509 = x509.load_pem_x509_certificates(bytes(ca_list_bytes))
#
# ca_store = x509.verification.Store(ca_list_x509)
# builder = x509.verification.PolicyBuilder().store(ca_store).time(datetime.datetime.now())
# verifier = builder.build_client_verifier()
# verified_client = verifier.verify(x509cert, ca_list_x509)
#
# from rich.pretty import pprint
# pprint(verified_client)
##############################################################################################
def _xrdcp_executor(wb, copyjob: CopyFile, xrd_cp_args: XrdCpArgs, printout: str = '') -> Optional[CopyFile]:
"""xrdcp standalone copy executor"""
if not HAS_XROOTD:
print_err("XRootD not found")
return None
# MANDATORY DEFAULTS, always used
makedir = bool(True) # create the parent directories when creating a file
posc = bool(True) # persist on successful close; Files are automatically deleted should they not be successfully closed.
sources = int(1) # max number of download sources; we (ALICE) do not rely on parallel multi-source downloads
# passed arguments
overwrite = xrd_cp_args.overwrite
batch = xrd_cp_args.batch
tpc = xrd_cp_args.tpc
timeout = xrd_cp_args.timeout
hashtype = xrd_cp_args.hashtype
rate = xrd_cp_args.rate
src = copyjob.src
dst = copyjob.dst
lfn = copyjob.lfn
isUpload = copyjob.isUpload
token_data = copyjob.token_request
cksum_mode = 'none'
cksum_type = cksum_preset = ''
if isUpload:
cksum_mode = 'source'
cksum_type = 'md5'
cksum_preset = token_data['md5']
else: # for downloads we already have the md5 value, lets use that
cksum_mode = 'target'
cksum_type, cksum_preset = get_hash_meta(src)
# If the remote file had no hash registered
if not cksum_type or not cksum_preset:
logging.error('COPY:: MD5 missing for %s', lfn)
# let's customize the environment of xrdcp command
xrdcp_env = os.environ.copy()
# Customize environment based on size and other information
# XRD_CPPARALLELCHUNKS, XRD_CPCHUNKSIZE, XRD_STREAMTIMEOUT, timeout
xrdcp_cmdline = ['xrdcp', '-f', '-N', '-P']
# -C | --cksum type[:value|print|source]
# obtains the checksum of type (i.e. adler32, crc32, md5 or zcrc32) from the source, computes the checksum at the destination, and veri‐
# fies that they are the same. If auto is chosen as the checksum type, xrdcp will try to automatically infer the right checksum type based
# on source/destination configuration, source file type (metalink, ZIP), and available checksum plug-ins. If a value is specified, it is
# used as the source checksum. When print is specified, the checksum at the destination is printed but is not verified.
# --rm-bad-cksum
# Remove the target file if checksum verification failed (enables also POSC semantics).
if not isUpload: # is download
xrdcp_cmdline.extend(['--cksum', f'{cksum_type}:{cksum_preset}', '--rm-bad-cksum'])
# --xrate-threshold rate
# If the transfer rate drops below given threshold force the client to use different source or if no more sources are available fail the transfer.
if rate:
xrdcp_cmdline.extend(['--xrate-threshold', rate])
# PRINTING OF beginning ! here or in manager .. there is a need of index/total information
if isUpload:
xrdcp_cmdline.extend([src, dst])
# status = subprocess.run(xrdcp_cmdline, encoding = 'utf-8', errors = 'replace', timeout = timeout, capture_output = True, env = xrdcp_env) # pylint: disable=subprocess-run-check # nosec
# do commit
# do end print
# return copyjob if fail else None
# process download
list_of_replicas = get_url_meta(src)
for replica in list_of_replicas:
new_xrdcp_cmdline = xrdcp_cmdline.copy()
new_xrdcp_cmdline.extend([replica, dst])
# status = subprocess.run(new_xrdcp_cmdline, encoding = 'utf-8', errors = 'replace', timeout = timeout, capture_output = True, env = xrdcp_env) # pylint: disable=subprocess-run-check # nosec
# if success:
# do end print
# break
# DOWNLOAD
# CopyFile(
# src='/home/adrian/tmp/f3c361b6-8677-5585-acfb-56ced24e36d5.meta4',
# dst='/home/adrian/work-ALICE/jalien_py/test_area/test.sh~',
# isUpload=False,
# token_request={},
# lfn='/alice/cern.ch/user/a/asevcenc/test.sh~'
# )
# UPLOAD
# CopyFile(
# src='/home/adrian/work-ALICE/jalien_py/test_area/test2.sh',
# dst='root://neos.nipne.ro:1094//02/10490/4d798722-9786-11ef-8ad0-8030e01e6668?xrd.wantprot=unix&authz=-----BEGIN SEALED CIPHER-----\nqgdhc9kDcjjKsxudKLnwvYg6NyJNAQqLkL-EREsd7dgunQcPe8LO7hxq3zzqMlY94Ur+xpv3iNQJ\nDyQBrVNUmvC3x5D0n+oSvR6XMEr5va1QkdwK8nsyhB6KiAonlFDSvXfPPSNI3sq2VNeORmE5LV9g\nl4M7B3R0S+yC4X-hT9E=\n-----END SEALED CIPHER-----\n-----BEGIN SEALED ENVELOPE-----\nAAAAgJdVSgHfLO9Pwrl4YVwKt5AaK1QvK3n0D5X+JRnTYXxldLuQDV7WdqrS3noP8FCBqNl6DCnF\nsyCrBJdXPPBnNKuWH8sOdCHLzR5G27KhVcMBfvajK+8sCM8JGoXjgqEjDC0TVMe66ufAe7mwVnBo\nfQYcau-2aLZFUWvVOWjsgwHENeloV-OtJzhscaBoZTe3GJggEEl9A3gd6q3IlLkYFaTtDMRSey-k\noknKl1JdK1CN9UDi+Nw4nuj6C4djgomqLJR87m0XWDTIc-0OSbrNMWblQlI0BteNVrJDuJLKfogu\n-lM95gJ7-MUxTI21FgciF6cgiUgn2ZZdGQwSWUGIFmZ+AMvhUUcibN1Rtj7sDR9GYU9x-3wdfFQV\nD0uggZd656ctGgv0Z8OS86xx3dXdwC4AKdfOrwSBKqgN7pUAe701c+Aawl8637kAixK0klkXPpYk\nYjfCHgKtcrjIHwRzjKu3kAAFKtbvjCuabQfBelmVyu75onhg8FOWqEhEUgeTO1fKlRL3yxgqGHZy\nvWrBT7+Kr+nRy5Jv52pecgNx4oCsfusWv1M335buizhASbN99IUJv6IXvdTsk5IINkmia0jLq7-X\nm5OWsZ0hw+bObVl6+QYW0cSBrYjq8qBN608KRX752RxxXWwa9UjtERBcktS320B60n5cTtPwsrCz\nuKWD+o1eIBuQOO7Mv6EZQB6Tk5rhSwGZhzI1xuy0ncsiwcDktOg3Cc+tjRB8mu9dB6iuc7wRK34w\n18rDMMbmoiXbeabidTWTEQoFgCyTfZ-1ef1JMSwyLmyV-YxgqT3oIO1rOBSZgMutd+UAu8Ib6AN3\nKS75++IRDJy-nvqQn9H1NvqtglN7euZo9k5maUVWuJ0bc-JXtmxFP5Bxnthe4smMJg==\n-----END SEALED ENVELOPE-----\n&scitag.flow=335&eos.app=cliUpload',
# isUpload=True,
# token_request={
# 'envelope': '-----BEGIN SEALED CIPHER-----\nqgdhc9kDcjjKsxudKLnwvYg6NyJNAQqLkL-EREsd7dgunQcPe8LO7hxq3zzqMlY94Ur+xpv3iNQJ\nDyQBrVNUmvC3x5D0n+oSvR6XMEr5va1QkdwK8nsyhB6KiAonlFDSvXfPPSNI3sq2VNeORmE5LV9g\nl4M7B3R0S+yC4X-hT9E=\n-----END SEALED CIPHER-----\n-----BEGIN SEALED ENVELOPE-----\nAAAAgJdVSgHfLO9Pwrl4YVwKt5AaK1QvK3n0D5X+JRnTYXxldLuQDV7WdqrS3noP8FCBqNl6DCnF\nsyCrBJdXPPBnNKuWH8sOdCHLzR5G27KhVcMBfvajK+8sCM8JGoXjgqEjDC0TVMe66ufAe7mwVnBo\nfQYcau-2aLZFUWvVOWjsgwHENeloV-OtJzhscaBoZTe3GJggEEl9A3gd6q3IlLkYFaTtDMRSey-k\noknKl1JdK1CN9UDi+Nw4nuj6C4djgomqLJR87m0XWDTIc-0OSbrNMWblQlI0BteNVrJDuJLKfogu\n-lM95gJ7-MUxTI21FgciF6cgiUgn2ZZdGQwSWUGIFmZ+AMvhUUcibN1Rtj7sDR9GYU9x-3wdfFQV\nD0uggZd656ctGgv0Z8OS86xx3dXdwC4AKdfOrwSBKqgN7pUAe701c+Aawl8637kAixK0klkXPpYk\nYjfCHgKtcrjIHwRzjKu3kAAFKtbvjCuabQfBelmVyu75onhg8FOWqEhEUgeTO1fKlRL3yxgqGHZy\nvWrBT7+Kr+nRy5Jv52pecgNx4oCsfusWv1M335buizhASbN99IUJv6IXvdTsk5IINkmia0jLq7-X\nm5OWsZ0hw+bObVl6+QYW0cSBrYjq8qBN608KRX752RxxXWwa9UjtERBcktS320B60n5cTtPwsrCz\nuKWD+o1eIBuQOO7Mv6EZQB6Tk5rhSwGZhzI1xuy0ncsiwcDktOg3Cc+tjRB8mu9dB6iuc7wRK34w\n18rDMMbmoiXbeabidTWTEQoFgCyTfZ-1ef1JMSwyLmyV-YxgqT3oIO1rOBSZgMutd+UAu8Ib6AN3\nKS75++IRDJy-nvqQn9H1NvqtglN7euZo9k5maUVWuJ0bc-JXtmxFP5Bxnthe4smMJg==\n-----END SEALED ENVELOPE-----\n&scitag.flow=335&eos.app=cliUpload',
# 'url': 'root://neos.nipne.ro:1094//02/10490/4d798722-9786-11ef-8ad0-8030e01e6668',
# 'guid': '4d798722-9786-11ef-8ad0-8030e01e6668',
# 'se': 'ALICE::NIHAM::EOS',
# 'tags': '[disk, legoinput]',
# 'nSEs': '4',
# 'md5': '72c23cf3ecdade4ffb79f66d725d14b4',
# 'size': '259',
# 'qos_specs': [],
# 'SElist_specs': [],
# 'SElist': ['ALICE::UPB::EOS', 'ALICE::BRATISLAVA::SE', 'ALICE::ISS::EOS', 'ALICE::NIHAM::EOS'],
# 'file': '/home/adrian/work-ALICE/jalien_py/test_area/test2.sh',
# 'lfn': '/alice/cern.ch/user/a/asevcenc/test2.sh'
# },
# lfn='/alice/cern.ch/user/a/asevcenc/test2.sh'
# )
# keep it commented until is needed - dead code for now
# def XrdCopy_xrdcp(job_list: list, xrd_cp_args: XrdCpArgs) -> list: # , printout: str = ''
# """XRootD copy command :: the actual XRootD copy process"""
# if not HAS_XROOTD:
# print_err("XRootD not found or lower version than 5.3.3")
# return []
# if not xrd_cp_args:
# print_err("cp arguments are not set, XrdCpArgs tuple missing")
# return []
# # overwrite = xrd_cp_args.overwrite
# # batch = xrd_cp_args.batch
# # makedir = xrd_cp_args.makedir
#
# # ctx = mp.get_context('forkserver')
# # q = ctx.JoinableQueue()
# # p = ctx.Process(target=_xrdcp_copyjob, args=(q,))
# # p.start()
# # print(q.get())
# # p.join()
# for copy_job in job_list:
# if DEBUG: logging.debug('\nadd copy job with\nsrc: %s\ndst: %s\n', copy_job.src, copy_job.dst)
# # xrdcp_cmd = f' {copy_job.src} {copy_job.dst}'
# if DEBUG: print_out(copy_job)
# return []
# def xrdfile_set_attr(uri: str = '', xattr_list: Optional[list] = None):
# """For a given URI (token included) set the xattrs"""
# if not HAS_XROOTD or not uri or not xattr_list: return None
# mode = OpenFlags.READ | OpenFlags.UPDATE | OpenFlags.WRITE
# with xrd_client.File() as f:
# status, response = f.open(uri, mode)
# print(f'{status}\n{response}')
#
# status, list_of_statuses = f.set_xattr(attrs = xattr_list)
# print(status)
# for s in list_of_statuses:
# print(s[0])
# print(s[1])