Add OCRPDFLoader for extracting text from scanned PDFs- Implements OC…

libs/community/langchain_community/document_loaders/__init__.py

@@ -19,519 +19,522 @@
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
     from langchain_community.document_loaders.acreom import (
         AcreomLoader,
     )
     from langchain_community.document_loaders.airbyte import (
         AirbyteCDKLoader,
         AirbyteGongLoader,
         AirbyteHubspotLoader,
         AirbyteSalesforceLoader,
         AirbyteShopifyLoader,
         AirbyteStripeLoader,
         AirbyteTypeformLoader,
         AirbyteZendeskSupportLoader,
     )
     from langchain_community.document_loaders.airbyte_json import (
         AirbyteJSONLoader,
     )
     from langchain_community.document_loaders.airtable import (
         AirtableLoader,
     )
     from langchain_community.document_loaders.apify_dataset import (
         ApifyDatasetLoader,
     )
     from langchain_community.document_loaders.arcgis_loader import (
         ArcGISLoader,
     )
     from langchain_community.document_loaders.arxiv import (
         ArxivLoader,
     )
     from langchain_community.document_loaders.assemblyai import (
         AssemblyAIAudioLoaderById,
         AssemblyAIAudioTranscriptLoader,
     )
     from langchain_community.document_loaders.astradb import (
         AstraDBLoader,
     )
     from langchain_community.document_loaders.async_html import (
         AsyncHtmlLoader,
     )
     from langchain_community.document_loaders.athena import (
         AthenaLoader,
     )
     from langchain_community.document_loaders.azlyrics import (
         AZLyricsLoader,
     )
     from langchain_community.document_loaders.azure_ai_data import (
         AzureAIDataLoader,
     )
     from langchain_community.document_loaders.azure_blob_storage_container import (
         AzureBlobStorageContainerLoader,
     )
     from langchain_community.document_loaders.azure_blob_storage_file import (
         AzureBlobStorageFileLoader,
     )
     from langchain_community.document_loaders.bibtex import (
         BibtexLoader,
     )
     from langchain_community.document_loaders.bigquery import (
         BigQueryLoader,
     )
     from langchain_community.document_loaders.bilibili import (
         BiliBiliLoader,
     )
     from langchain_community.document_loaders.blackboard import (
         BlackboardLoader,
     )
     from langchain_community.document_loaders.blob_loaders import (
         Blob,
         BlobLoader,
         CloudBlobLoader,
         FileSystemBlobLoader,
         YoutubeAudioLoader,
     )
     from langchain_community.document_loaders.blockchain import (
         BlockchainDocumentLoader,
     )
     from langchain_community.document_loaders.brave_search import (
         BraveSearchLoader,
     )
     from langchain_community.document_loaders.browserbase import (
         BrowserbaseLoader,
     )
     from langchain_community.document_loaders.browserless import (
         BrowserlessLoader,
     )
     from langchain_community.document_loaders.cassandra import (
         CassandraLoader,
     )
     from langchain_community.document_loaders.chatgpt import (
         ChatGPTLoader,
     )
     from langchain_community.document_loaders.chm import (
         UnstructuredCHMLoader,
     )
     from langchain_community.document_loaders.chromium import (
         AsyncChromiumLoader,
     )
     from langchain_community.document_loaders.college_confidential import (
         CollegeConfidentialLoader,
     )
     from langchain_community.document_loaders.concurrent import (
         ConcurrentLoader,
     )
     from langchain_community.document_loaders.confluence import (
         ConfluenceLoader,
     )
     from langchain_community.document_loaders.conllu import (
         CoNLLULoader,
     )
     from langchain_community.document_loaders.couchbase import (
         CouchbaseLoader,
     )
     from langchain_community.document_loaders.csv_loader import (
         CSVLoader,
         UnstructuredCSVLoader,
     )
     from langchain_community.document_loaders.cube_semantic import (
         CubeSemanticLoader,
     )
     from langchain_community.document_loaders.datadog_logs import (
         DatadogLogsLoader,
     )
     from langchain_community.document_loaders.dataframe import (
         DataFrameLoader,
     )
     from langchain_community.document_loaders.dedoc import (
         DedocAPIFileLoader,
         DedocFileLoader,
     )
     from langchain_community.document_loaders.diffbot import (
         DiffbotLoader,
     )
     from langchain_community.document_loaders.directory import (
         DirectoryLoader,
     )
     from langchain_community.document_loaders.discord import (
         DiscordChatLoader,
     )
     from langchain_community.document_loaders.doc_intelligence import (
         AzureAIDocumentIntelligenceLoader,
     )
     from langchain_community.document_loaders.docugami import (
         DocugamiLoader,
     )
     from langchain_community.document_loaders.docusaurus import (
         DocusaurusLoader,
     )
     from langchain_community.document_loaders.dropbox import (
         DropboxLoader,
     )
     from langchain_community.document_loaders.duckdb_loader import (
         DuckDBLoader,
     )
     from langchain_community.document_loaders.email import (
         OutlookMessageLoader,
         UnstructuredEmailLoader,
     )
     from langchain_community.document_loaders.epub import (
         UnstructuredEPubLoader,
     )
     from langchain_community.document_loaders.etherscan import (
         EtherscanLoader,
     )
     from langchain_community.document_loaders.evernote import (
         EverNoteLoader,
     )
     from langchain_community.document_loaders.excel import (
         UnstructuredExcelLoader,
     )
     from langchain_community.document_loaders.facebook_chat import (
         FacebookChatLoader,
     )
     from langchain_community.document_loaders.fauna import (
         FaunaLoader,
     )
     from langchain_community.document_loaders.figma import (
         FigmaFileLoader,
     )
     from langchain_community.document_loaders.firecrawl import (
         FireCrawlLoader,
     )
     from langchain_community.document_loaders.gcs_directory import (
         GCSDirectoryLoader,
     )
     from langchain_community.document_loaders.gcs_file import (
         GCSFileLoader,
     )
     from langchain_community.document_loaders.geodataframe import (
         GeoDataFrameLoader,
     )
     from langchain_community.document_loaders.git import (
         GitLoader,
     )
     from langchain_community.document_loaders.gitbook import (
         GitbookLoader,
     )
     from langchain_community.document_loaders.github import (
         GithubFileLoader,
         GitHubIssuesLoader,
     )
     from langchain_community.document_loaders.glue_catalog import (
         GlueCatalogLoader,
     )
     from langchain_community.document_loaders.google_speech_to_text import (
         GoogleSpeechToTextLoader,
     )
     from langchain_community.document_loaders.googledrive import (
         GoogleDriveLoader,
     )
     from langchain_community.document_loaders.gutenberg import (
         GutenbergLoader,
     )
     from langchain_community.document_loaders.hn import (
         HNLoader,
     )
     from langchain_community.document_loaders.html import (
         UnstructuredHTMLLoader,
     )
     from langchain_community.document_loaders.html_bs import (
         BSHTMLLoader,
     )
     from langchain_community.document_loaders.hugging_face_dataset import (
         HuggingFaceDatasetLoader,
     )
     from langchain_community.document_loaders.hugging_face_model import (
         HuggingFaceModelLoader,
     )
     from langchain_community.document_loaders.ifixit import (
         IFixitLoader,
     )
     from langchain_community.document_loaders.image import (
         UnstructuredImageLoader,
     )
     from langchain_community.document_loaders.image_captions import (
         ImageCaptionLoader,
     )
     from langchain_community.document_loaders.imsdb import (
         IMSDbLoader,
     )
     from langchain_community.document_loaders.iugu import (
         IuguLoader,
     )
     from langchain_community.document_loaders.joplin import (
         JoplinLoader,
     )
     from langchain_community.document_loaders.json_loader import (
         JSONLoader,
     )
     from langchain_community.document_loaders.kinetica_loader import KineticaLoader
     from langchain_community.document_loaders.lakefs import (
         LakeFSLoader,
     )
     from langchain_community.document_loaders.larksuite import (
         LarkSuiteDocLoader,
     )
     from langchain_community.document_loaders.llmsherpa import (
         LLMSherpaFileLoader,
     )
     from langchain_community.document_loaders.markdown import (
         UnstructuredMarkdownLoader,
     )
     from langchain_community.document_loaders.mastodon import (
         MastodonTootsLoader,
     )
     from langchain_community.document_loaders.max_compute import (
         MaxComputeLoader,
     )
     from langchain_community.document_loaders.mediawikidump import (
         MWDumpLoader,
     )
     from langchain_community.document_loaders.merge import (
         MergedDataLoader,
     )
     from langchain_community.document_loaders.mhtml import (
         MHTMLLoader,
     )
     from langchain_community.document_loaders.modern_treasury import (
         ModernTreasuryLoader,
     )
     from langchain_community.document_loaders.mongodb import (
         MongodbLoader,
     )
     from langchain_community.document_loaders.needle import (
         NeedleLoader,
     )
     from langchain_community.document_loaders.news import (
         NewsURLLoader,
     )
     from langchain_community.document_loaders.notebook import (
         NotebookLoader,
     )
     from langchain_community.document_loaders.notion import (
         NotionDirectoryLoader,
     )
     from langchain_community.document_loaders.notiondb import (
         NotionDBLoader,
     )
     from langchain_community.document_loaders.obs_directory import (
         OBSDirectoryLoader,
     )
     from langchain_community.document_loaders.obs_file import (
         OBSFileLoader,
     )
     from langchain_community.document_loaders.obsidian import (
         ObsidianLoader,
     )
     from langchain_community.document_loaders.odt import (
         UnstructuredODTLoader,
     )
     from langchain_community.document_loaders.onedrive import (
         OneDriveLoader,
     )
     from langchain_community.document_loaders.onedrive_file import (
         OneDriveFileLoader,
     )
     from langchain_community.document_loaders.open_city_data import (
         OpenCityDataLoader,
     )
     from langchain_community.document_loaders.oracleadb_loader import (
         OracleAutonomousDatabaseLoader,
     )
     from langchain_community.document_loaders.oracleai import (
         OracleDocLoader,
         OracleTextSplitter,
     )
     from langchain_community.document_loaders.org_mode import (
         UnstructuredOrgModeLoader,
     )
     from langchain_community.document_loaders.pdf import (
         AmazonTextractPDFLoader,
         DedocPDFLoader,
         MathpixPDFLoader,
         OnlinePDFLoader,
         PagedPDFSplitter,
         PDFMinerLoader,
         PDFMinerPDFasHTMLLoader,
         PDFPlumberLoader,
         PyMuPDFLoader,
         PyPDFDirectoryLoader,
         PyPDFium2Loader,
         PyPDFLoader,
         UnstructuredPDFLoader,
     )
     from langchain_community.document_loaders.pebblo import (
         PebbloSafeLoader,
         PebbloTextLoader,
     )
     from langchain_community.document_loaders.polars_dataframe import (
         PolarsDataFrameLoader,
     )
     from langchain_community.document_loaders.powerpoint import (
         UnstructuredPowerPointLoader,
     )
     from langchain_community.document_loaders.psychic import (
         PsychicLoader,
     )
     from langchain_community.document_loaders.pubmed import (
         PubMedLoader,
     )
     from langchain_community.document_loaders.pyspark_dataframe import (
         PySparkDataFrameLoader,
     )
     from langchain_community.document_loaders.python import (
         PythonLoader,
     )
     from langchain_community.document_loaders.readthedocs import (
         ReadTheDocsLoader,
     )
     from langchain_community.document_loaders.recursive_url_loader import (
         RecursiveUrlLoader,
     )
     from langchain_community.document_loaders.reddit import (
         RedditPostsLoader,
     )
     from langchain_community.document_loaders.roam import (
         RoamLoader,
     )
     from langchain_community.document_loaders.rocksetdb import (
         RocksetLoader,
     )
     from langchain_community.document_loaders.rss import (
         RSSFeedLoader,
     )
     from langchain_community.document_loaders.rst import (
         UnstructuredRSTLoader,
     )
     from langchain_community.document_loaders.rtf import (
         UnstructuredRTFLoader,
     )
     from langchain_community.document_loaders.s3_directory import (
         S3DirectoryLoader,
     )
     from langchain_community.document_loaders.s3_file import (
         S3FileLoader,
     )
     from langchain_community.document_loaders.scrapfly import (
         ScrapflyLoader,
     )
     from langchain_community.document_loaders.scrapingant import (
         ScrapingAntLoader,
     )
     from langchain_community.document_loaders.sharepoint import (
         SharePointLoader,
     )
     from langchain_community.document_loaders.sitemap import (
         SitemapLoader,
     )
     from langchain_community.document_loaders.slack_directory import (
         SlackDirectoryLoader,
     )
     from langchain_community.document_loaders.snowflake_loader import (
         SnowflakeLoader,
     )
     from langchain_community.document_loaders.spider import (
         SpiderLoader,
     )
     from langchain_community.document_loaders.spreedly import (
         SpreedlyLoader,
     )
     from langchain_community.document_loaders.sql_database import (
         SQLDatabaseLoader,
     )
     from langchain_community.document_loaders.srt import (
         SRTLoader,
     )
     from langchain_community.document_loaders.stripe import (
         StripeLoader,
     )
     from langchain_community.document_loaders.surrealdb import (
         SurrealDBLoader,
     )
     from langchain_community.document_loaders.telegram import (
         TelegramChatApiLoader,
         TelegramChatFileLoader,
         TelegramChatLoader,
     )
     from langchain_community.document_loaders.tencent_cos_directory import (
         TencentCOSDirectoryLoader,
     )
     from langchain_community.document_loaders.tencent_cos_file import (
         TencentCOSFileLoader,
     )
     from langchain_community.document_loaders.tensorflow_datasets import (
         TensorflowDatasetLoader,
     )
     from langchain_community.document_loaders.text import (
         TextLoader,
     )
     from langchain_community.document_loaders.tidb import (
         TiDBLoader,
     )
     from langchain_community.document_loaders.tomarkdown import (
         ToMarkdownLoader,
     )
     from langchain_community.document_loaders.toml import (
         TomlLoader,
     )
     from langchain_community.document_loaders.trello import (
         TrelloLoader,
     )
     from langchain_community.document_loaders.tsv import (
         UnstructuredTSVLoader,
     )
     from langchain_community.document_loaders.twitter import (
         TwitterTweetLoader,
     )
     from langchain_community.document_loaders.unstructured import (
         UnstructuredAPIFileIOLoader,
         UnstructuredAPIFileLoader,
         UnstructuredFileIOLoader,
         UnstructuredFileLoader,
     )
     from langchain_community.document_loaders.url import (
         UnstructuredURLLoader,
     )
     from langchain_community.document_loaders.url_playwright import (
         PlaywrightURLLoader,
     )
     from langchain_community.document_loaders.url_selenium import (
         SeleniumURLLoader,
     )
     from langchain_community.document_loaders.vsdx import (
         VsdxLoader,
     )
     from langchain_community.document_loaders.weather import (
         WeatherDataLoader,
     )
     from langchain_community.document_loaders.web_base import (
         WebBaseLoader,
     )
     from langchain_community.document_loaders.whatsapp_chat import (
         WhatsAppChatLoader,
     )
     from langchain_community.document_loaders.wikipedia import (
         WikipediaLoader,
     )
     from langchain_community.document_loaders.word_document import (
         Docx2txtLoader,
         UnstructuredWordDocumentLoader,
     )
     from langchain_community.document_loaders.xml import (
         UnstructuredXMLLoader,
     )
     from langchain_community.document_loaders.xorbits import (
         XorbitsLoader,
     )
     from langchain_community.document_loaders.youtube import (
         GoogleApiClient,
         GoogleApiYoutubeLoader,
         YoutubeLoader,
     )
     from langchain_community.document_loaders.yuque import (
         YuqueLoader,
     )
+    from langchain_community.document_loaders.ocr_pdf import (
+        OCRPDFLoader,
+    )
 
 
 _module_lookup = {
@@ -732,6 +735,7 @@
     "YoutubeAudioLoader": "langchain_community.document_loaders.blob_loaders",
     "YoutubeLoader": "langchain_community.document_loaders.youtube",
     "YuqueLoader": "langchain_community.document_loaders.yuque",
+    "OCRPDFLoader": "langchain_community.document_loaders.ocr_pdf",
 }
 
 
@@ -940,4 +944,5 @@
     "YoutubeAudioLoader",
     "YoutubeLoader",
     "YuqueLoader",
+    "OCRPDFLoader",
 ]

libs/community/langchain_community/document_loaders/ocr_pdf.py

@@ -0,0 +1,174 @@
+"""Loader for extracting text from scanned PDFs using OCR."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Iterator, List, Optional
+
+from langchain_core.documents import Document
+from langchain_community.document_loaders.base import BaseLoader
+
+logger = logging.getLogger(__name__)
+
+
+class OCRPDFLoader(BaseLoader):
+    """Load scanned PDF files using OCR (Optical Character Recognition).
+
+    This loader converts PDF pages to images and applies Tesseract OCR
+    to extract text from scanned documents.
+
+    Setup:
+        Install required packages:
+        ```bash
+        pip install pdf2image pytesseract
+        ```
+
+        Install system dependencies:
+        - **Linux**: `sudo apt-get install poppler-utils tesseract-ocr`
+        - **macOS**: `brew install poppler tesseract`
+        - **Windows**: Download and install Poppler and Tesseract, add to PATH
+
+    Example:
+        ```python
+        from langchain_community.document_loaders import OCRPDFLoader
+
+        loader = OCRPDFLoader("scanned_document.pdf")
+        documents = loader.load()
+
+        # Access extracted text and metadata
+        for doc in documents:
+            print(f"Page {doc.metadata['page']}: {doc.page_content[:100]}...")
+        ```
+    """
+
+    def __init__(
+        self,
+        file_path: str | Path,
+        *,
+        tesseract_config: str = "",
+        poppler_path: Optional[str] = None,
+        first_page: Optional[int] = None,
+        last_page: Optional[int] = None,
+        dpi: int = 200,
+        fmt: str = "JPEG",
+    ) -> None:
+        """Initialize the OCR PDF loader.
+
+        Args:
+            file_path: Path to the PDF file to load.
+            tesseract_config: Additional configuration options for Tesseract OCR.
+                Example: "--psm 6" for uniform text blocks.
+            poppler_path: Path to poppler installation (Windows only).
+            first_page: First page to process (1-indexed). If None, starts from page 1.
+            last_page: Last page to process (1-indexed). If None, processes all pages.
+            dpi: Resolution for PDF to image conversion. Higher values improve
+                OCR accuracy but increase processing time.
+            fmt: Image format for conversion ("JPEG", "PNG", etc.).
+
+        Raises:
+            FileNotFoundError: If the specified PDF file does not exist.
+            ImportError: If required dependencies are not installed.
+        """
+        try:
+            import pdf2image  # noqa: F401
+            import pytesseract  # noqa: F401
+        except ImportError as e:
+            raise ImportError(
+                "OCRPDFLoader requires pdf2image and pytesseract. "
+                "Install with: pip install pdf2image pytesseract"
+            ) from e
+
+        self.file_path = Path(file_path)
+        if not self.file_path.exists():
+            raise FileNotFoundError(f"PDF file not found: {self.file_path}")
+
+        self.tesseract_config = tesseract_config
+        self.poppler_path = poppler_path
+        self.first_page = first_page
+        self.last_page = last_page
+        self.dpi = dpi
+        self.fmt = fmt
+
+    def load(self) -> List[Document]:
+        """Load all pages and return as a list of Documents.
+
+        Returns:
+            List of Document objects, one per page with extracted text.
+        """
+        return list(self.lazy_load())
+
+    def lazy_load(self) -> Iterator[Document]:
+        """Lazy load pages one at a time.
+
+        Yields:
+            Document objects with extracted text and metadata.
+
+        Raises:
+            Exception: If PDF processing or OCR fails.
+        """
+        try:
+            from pdf2image import convert_from_path
+            import pytesseract
+        except ImportError as e:
+            raise ImportError(
+                "Required dependencies not found. "
+                "Install with: pip install pdf2image pytesseract"
+            ) from e
+
+        try:
+            # Convert PDF pages to images
+            conversion_kwargs = {
+                "pdf_path": self.file_path,
+                "dpi": self.dpi,
+                "fmt": self.fmt,
+            }
+
+            if self.poppler_path:
+                conversion_kwargs["poppler_path"] = self.poppler_path
+            if self.first_page:
+                conversion_kwargs["first_page"] = self.first_page
+            if self.last_page:
+                conversion_kwargs["last_page"] = self.last_page
+
+            pages = convert_from_path(**conversion_kwargs)
+            total_pages = len(pages)
+
+            logger.info(f"Processing {total_pages} pages from {self.file_path}")
+
+        except Exception as e:
+            raise RuntimeError(f"Failed to convert PDF to images: {e}") from e
+
+        # Process each page with OCR
+        start_page = self.first_page or 1
+
+        for i, page_image in enumerate(pages):
+            page_number = start_page + i
+
+            try:
+                # Extract text using Tesseract OCR
+                ocr_kwargs = {"image": page_image}
+                if self.tesseract_config:
+                    ocr_kwargs["config"] = self.tesseract_config
+
+                text = pytesseract.image_to_string(**ocr_kwargs)
+
+                # Only yield documents with non-empty text
+                if text.strip():
+                    yield Document(
+                        page_content=text.strip(),
+                        metadata={
+                            "source": str(self.file_path),
+                            "page": page_number,
+                            "total_pages": total_pages,
+                            "loader": "OCRPDFLoader",
+                            "ocr_engine": "tesseract",
+                        },
+                    )
+                else:
+                    logger.warning(f"No text extracted from page {page_number}")
+
+            except Exception as e:
+                logger.error(f"OCR failed for page {page_number}: {e}")
+                # Continue processing other pages even if one fails
+                continue