|
|
|
@ -202,10 +202,13 @@ def download_and_decompress(url, path='.'): |
|
|
|
|
local_rank = paddle.distributed.get_rank() |
|
|
|
|
fname = osp.split(url)[-1] |
|
|
|
|
fullname = osp.join(path, fname) |
|
|
|
|
pth_path = fullname + '.path' |
|
|
|
|
|
|
|
|
|
if nranks <= 1: |
|
|
|
|
dst_dir = url2dir(url, path) |
|
|
|
|
if dst_dir is not None: |
|
|
|
|
with open(pth_path, 'w') as f: |
|
|
|
|
f.write(dst_dir) |
|
|
|
|
fullname = dst_dir |
|
|
|
|
else: |
|
|
|
|
lock_path = fullname + '.lock' |
|
|
|
@ -215,9 +218,14 @@ def download_and_decompress(url, path='.'): |
|
|
|
|
if local_rank == 0: |
|
|
|
|
dst_dir = url2dir(url, path) |
|
|
|
|
if dst_dir is not None: |
|
|
|
|
with open(pth_path, 'w') as f: |
|
|
|
|
f.write(dst_dir) |
|
|
|
|
fullname = dst_dir |
|
|
|
|
os.remove(lock_path) |
|
|
|
|
else: |
|
|
|
|
while os.path.exists(lock_path): |
|
|
|
|
time.sleep(1) |
|
|
|
|
if os.path.exists(pth_path): |
|
|
|
|
with open(pth_path, 'r') as f: |
|
|
|
|
fullname = next(f) |
|
|
|
|
return fullname |
|
|
|
|