How to deep crawl a website which contains pdf urls also ?

[gauravmindzk]

Hello everyone,
I want to deep crawl a website.
The code that I've come up with is below :

import asyncio
import os
from pathlib import Path

from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
from crawl4ai import PruningContentFilter
from crawl4ai import DefaultMarkdownGenerator
from crawl4ai import BFSDeepCrawlStrategy, DomainFilter, FilterChain

from crawl4ai.async_configs import CrawlerRunConfig

__cur_dir__ = Path(__file__).parent

import os
from crawl4ai.deep_crawling.filters import ( FilterChain, DomainFilter )

async def demo_deep_crawl():
    """Deep crawling with BFS strategy"""
    print("\n=== Deep Crawling ===")

    # Ensure the 'markdowns' directory exists
    os.makedirs("DirectoryName", exist_ok=True)

    filter_chain = FilterChain([DomainFilter(allowed_domains=["website_to_crawl.org"])])

    config = CrawlerRunConfig(
        deep_crawl_strategy=BFSDeepCrawlStrategy(max_depth=10,max_pages=500, filter_chain=filter_chain),
        stream=True , # Enable streaming
        markdown_generator=DefaultMarkdownGenerator(
            content_filter=PruningContentFilter()
        ),
        page_timeout=120000,
        delay_before_return_html=60,
        excluded_tags = ['header', 'footer']
    )
    i=0

    async with AsyncWebCrawler() as crawler:
        # Returns an async iterator
        async for result in await crawler.arun("https://website_to_crawl.org", config=config):
            # Process each result as it becomes available
            i=i+1
            filename = os.path.join("DirectoryName", f"{i}.md")
            if result.markdown:
                with open(filename, "w", encoding="utf-8") as f:
                    f.write(result.markdown.fit_markdown)
                with open(os.path.join("DirectoryName", "DirectoryName_urls.md"), "a", encoding="utf-8") as f:
                    f.write(result.url + "\n")

async def main():
    """Run all demo functions sequentially"""
    print("=== Comprehensive Crawl4AI Demo ===")
    
    await demo_deep_crawl()
    
    print("\n=== Demo Complete ===")

if __name__ == "__main__":
    asyncio.run(main())

The webpage URLs are getting scrapped properly and I have the fit markdown of each of them but my current code is not able to scrape PDF urls of the website , I want to scrape both the webpages and the pdf urls that are hosted by the website.

How to do it ?