out of memory issue for big pdf even decompose that in sending 5 pages for processing

# If already initialized, skip
if _converter is not None:
    logging.info("[Marker] Models already initialized, skipping.")
    return

# 1) Determine model cache directory
if model_cache_dir:
    cache_dir = model_cache_dir
else:
    # Download from S3
    bucket = os.environ.get('sparse_model_bucket', DEFAULT_BUCKET)
    key = os.environ.get('sparse_model_key', DEFAULT_KEY)
    logging.info(f"[Marker] Downloading model zip from S3://{bucket}/{key}...")
    downloader = DownloadLoadModels()
    zip_path = downloader.download_model_from_s3(bucket, key)
    if not zip_path or not os.path.exists(zip_path):
        raise RuntimeError(f"Failed to download model zip (S3://{bucket}/{key}).")

    # 2) Unzip and detect model folder
    model_folder = downloader.unzip_file_sparse(zip_path, extract_to='.')
    if not model_folder or not os.path.isdir(model_folder):
        raise RuntimeError(f"Unzip did not produce a valid model folder: {model_folder!r}")
    logging.info(f"[Marker] Unzipped model to: {model_folder}")
    cache_dir = model_folder

# Ensure environment caches
os.makedirs(cache_dir, exist_ok=True)
os.environ['TORCH_HOME'] = cache_dir
os.environ['HF_HOME'] = cache_dir
os.environ['TRANSFORMERS_CACHE'] = cache_dir
logging.info(f"[Marker] Loading models from local directory: {cache_dir}")

# Create artifact dict and config parser
_artifact_dict = create_model_dict()
config = {
    "output_format": "markdown",
    "disable_image_extraction": True,
    "disable_ocr": True,
    # "processors": ["TableProcessor", "EquationProcessor"],
    "OCR_ENGINE": None, #[("surya" (default), "ocrmypdf", or "None" to disable OCR)]
    "TORCH_DEVICE": "cpu", # "cuda" or "cpu"
    # "langs": ["en"], # Comma-separated list of languages for OCR processing
    
    
}
_config_parser = ConfigParser(config)

# Initialize converter
_converter = PdfConverter(
    config=_config_parser.generate_config_dict(), # Output of generate_config_dict(), a properly formatted settings dictionary for Marker’s engine.
    artifact_dict=_artifact_dict, # artifact_dict: Output of create_model_dict(), specifying all elements you want detected/extracted.
    processor_list=_config_parser.get_processors(), # processor_list: Sequence of processors for text, structure, postprocessing, etc. (e.g., OCR, table fixer, header/footer remover), as defined by config.
    renderer=_config_parser.get_renderer(), # renderer: Specifies the renderer for the desired output format (here, JSON).
    # llm_service=_config_parser.get_llm_service(), # llm_service: The large language model service object for enhanced extraction/formatting. Only used if "use_llm": True or specific LLM integration options are set.
)
logging.info("[Marker] Local models and converter initialized.")

# processors = _config_parser.get_processors()
# print("Processors in pipeline:")
# for p in processors:
#     print(p.__class__.__name__)

:param paths: List of PDF file paths to parse
:param batch_size: Number of files to process concurrently (per batch)
:param model_cache_dir: Optional local directory for model files; if provided, skips download
:param pages_per_chunk: Number of pages per in-memory chunk
:return: Mapping of PDF base name to cleaned text
"""
global _converter

if _converter is None:
    initialize_marker_models(model_cache_dir)

results: Dict[str, str] = {}

# Helper to batch files
def file_batches(iterable, size):
    it = iter(iterable)
    while True:
        batch = list(islice(it, size))
        if not batch:
            break
        yield batch

for file_batch in file_batches(paths, batch_size):
    valid_paths = [p for p in file_batch if os.path.isfile(p)]
    if not valid_paths:
        continue
    logging.info(f"[Marker] Processing file batch of {len(valid_paths)} PDFs...")

    for pdf_path in valid_paths:
        stem = os.path.splitext(os.path.basename(pdf_path))[0]
        try:
            reader = PdfReader(pdf_path)
            num_pages = len(reader.pages)
            logging.info(f"[Marker] '{stem}' has {num_pages} pages, chunking into {pages_per_chunk}-page slices")
            text_parts: List[str] = []

            for start in range(0, num_pages, pages_per_chunk):
                end = min(start + pages_per_chunk, num_pages)
                writer = PdfWriter()
                for p in reader.pages[start:end]:
                    writer.add_page(p)

                buffer = io.BytesIO()
                writer.write(buffer)
                buffer.seek(0)

                rendered = _converter(buffer)
                text, _, _ = text_from_rendered(rendered)
                text_parts.append(clean_text(text))

                # Free memory
                del rendered, text, writer, buffer
                gc.collect()
                try:
                    import torch
                    if torch.cuda.is_available():
                        torch.cuda.empty_cache()
                except ImportError:
                    logging.warning("Torch not installed, skipping torch.cuda.empty_cache()")
                    pass

            results[stem] = "\n\n".join(text_parts)
            logging.info(f"[Marker] Completed '{stem}', {sum(len(p) for p in text_parts)} characters")

        except Exception as e:
            logging.error(f"[Marker] Error parsing {pdf_path}: {e}")

return results

# Initialize models from local directory (do this once at startup)
model_cache_dir = "models--nlpconnect--vit-gpt2-image-captioning/snapshots/a0316644754a675fa47af1c7a5a2096a741947d3"
initialize_marker_models(model_cache_dir)

# Now use the function with local models
# pdf_paths = [r"C:\Work\src\R2-1705815 38331-003rm.pdf"]
pdf_paths = [r"C:\Work\R2-1705815 38331-003rm.pdf"]

results = parse_pdf_via_marker(pdf_paths, batch_size=2)

print(f"Processed {len(results)} files")
for filename, content in results.items():
    print(f"{filename}: {len(content)} characters")