Dataset._verify_dataset_folder 中的线程开销

class DS(Dataset): def _verify_dataset_folder( self, target_base_folder: Path, part: int, chunk_selection: dict, max_workers: int, files_of_interest: Optional[Set[str]] = None ) -> bool: def __verify_file_or_link( target_base_folder: Path, file_entry: Union[FileEntry, LinkEntry], part: Optional[int] = None, chunk_selection: Optional[dict] = None ) -> bool: # check if we need the file for the requested dataset part if part is not None: f_parts = chunk_selection.get( file_entry.parent_dataset_id, [] ) # file is not in requested dataset part, no need to check it. if self._get_chunk_idx_from_artifact_name( file_entry.artifact_name ) not in f_parts: return True # check if the local size and the stored size match (faster than comparing hash) if (target_base_folder / file_entry.relative_path).stat().st_size != file_entry.size: return False return True target_base_folder = Path( target_base_folder ) # check dataset file size, if we have a full match no need for parent dataset download / merge verified = True # noinspection PyBroadException tp = None try: verified = True for f in self._dataset_file_entries.values(): if files_of_interest and f.relative_path not in files_of_interest: continue if not __verify_file_or_link( target_base_folder, f, part, chunk_selection ): verified = False break if verified: for f in self._dataset_link_entries.values(): if files_of_interest and f.relative_path not in files_of_interest: continue # don't check whether link is in dataset part, hence None for part and chunk_selection if not __verify_file_or_link( target_base_folder, f, None, None ): verified = False break return verified