diff --git a/src/huggingface_hub/file_download.py b/src/huggingface_hub/file_download.py
index 363cf86f65..2d60f9b6c1 100644
--- a/src/huggingface_hub/file_download.py
+++ b/src/huggingface_hub/file_download.py
@@ -1388,7 +1388,8 @@ def _hf_hub_download_to_cache_dir(
             filename=filename,
             force_download=force_download,
         )
-        _create_symlink(blob_path, pointer_path, new_blob=True)
+        if not os.path.exists(pointer_path):
+            _create_symlink(blob_path, pointer_path, new_blob=True)
 
     return pointer_path