Threading overhead in Dataset._verify_dataset_folder

from clearml import Dataset
ds = Dataset.get(dataset_id=my_dataset_id)

ds.get_local_copy() # ~75 seconds
ds.get_local_copy(max_workers=1) # ~55 seconds
ds.get_local_copy(max_workers=2) # ~46 seconds
ds.get_local_copy(max_workers=4) # ~46 seconds
ds.get_local_copy(max_workers=8) # ~48 seconds
ds.get_local_copy(max_workers=16) # ~60 seconds

class DS(Dataset):
    def _verify_dataset_folder(
        self,
        target_base_folder: Path,
        part: int,
        chunk_selection: dict,
        max_workers: int,
        files_of_interest: Optional[Set[str]] = None,
    ) -> bool:
        def __verify_file_or_link(
            target_base_folder: Path,
            file_entry: Union[FileEntry, LinkEntry],
            part: Optional[int] = None,
            chunk_selection: Optional[dict] = None,
        ) -> bool:
            # check if we need the file for the requested dataset part
            if part is not None:
                f_parts = chunk_selection.get(file_entry.parent_dataset_id, [])
                # file is not in requested dataset part, no need to check it.
                if self._get_chunk_idx_from_artifact_name(file_entry.artifact_name) not in f_parts:
                    return True

            # check if the local size and the stored size match (faster than comparing hash)
            if (target_base_folder / file_entry.relative_path).stat().st_size != file_entry.size:
                return False

            return True

        target_base_folder = Path(target_base_folder)
        # check dataset file size, if we have a full match no need for parent dataset download / merge
        verified = True
        # noinspection PyBroadException
        tp = None
        try:
            verified = True
            for f in self._dataset_file_entries.values():
                if files_of_interest and f.relative_path not in files_of_interest:
                    continue
                if not __verify_file_or_link(
                    target_base_folder,
                    f,
                    part,
                    chunk_selection,
                ):
                    verified = False
                    break

            if verified:
                for f in self._dataset_link_entries.values():
                    if files_of_interest and f.relative_path not in files_of_interest:
                        continue
                    # don't check whether link is in dataset part, hence None for part and chunk_selection
                    if not __verify_file_or_link(target_base_folder, f, None, None):
                        verified = False
                        break
        except Exception:
            verified = False

        return verified

ds = DS.get(dataset_id=my_dataset_id)

ds.get_local_copy() # ~22 seconds