Dataset zip extraction fail when creating folders concurrently (with Exception [Errno 17] File exists warning)

from clearml.datasets import Dataset

DATASET_ID = "Must contains a lot of folders in it."

Dataset.get(DATASET_ID).get_local_copy()

import os
import tempfile
import zipfile
from pathlib import Path
from clearml.storage.manager import StorageManager
from concurrent.futures import ThreadPoolExecutor
import psutil


def main():
    # Create sample 100 zip files with 100 folders
    # zip_[0-99] with [0-99]/{zip_idx}.txt

    base_temp = tempfile.gettempdir()
    zip_dir = os.path.join(base_temp, "temp", "zip")
    os.makedirs(zip_dir, exist_ok=True)

    for zip_idx in range(100):
        zip_filename = f"zip_{zip_idx}.zip"
        zip_filepath = os.path.join(zip_dir, zip_filename)

        with zipfile.ZipFile(
            zip_filepath, "w", compression=zipfile.ZIP_DEFLATED
        ) as zipf:
            for idx in range(100):
                arcname = f"{idx}/{idx}.txt"
                zipf.writestr(arcname, str(idx))

    # Run zip extraction
    dataset_dir = os.path.join(base_temp, "dataset_id")
    os.makedirs(dataset_dir, exist_ok=True)

    def _extract_part(local_zip: str, dataset_id: str) -> None:
        StorageManager._extract_to_cache(
            cached_file=local_zip,
            name=dataset_id,
            cache_context="datasets",
            target_folder=dataset_dir,
            force=True,
        )

    with ThreadPoolExecutor(max_workers=psutil.cpu_count()) as pool:
        for zip_file in Path(zip_dir).glob("*.zip"):
            pool.submit(_extract_part, str(zip_file), "dataset_id")

if __name__ == "__main__":
    main()