[v2] refactor and create docs for results objects (#3155)

* [v2] refactor and create docs for results objects

This is the last thing needed for a documentation of the API.

https://embeddings-benchmark.github.io/mteb/

- renamed `load_results` to `results`
- moved `ModelResult` to its own script
- restructured import of results to use `results` module
- added documentation for the results module
- converted a few function marked with TODOs to private and deleted legacy loaders/converters
- moved `load_results.py` out of `results`
- Added missing documentation for a few of the types

sidenote: I really feel like we are starting to resolve some of our circular import issues, I rarely run into them now and when I do they are typically very easy to fix

* avoid rename of namespace during import

* Merge branch 'v2.0.0' of https://github.com/embeddings-benchmark/mteb into refactor-results

* remove comment

Author: KennethEnevoldsen

docs/api/model.md

@@ -1,7 +1,5 @@
 # Models
 
-<!-- TODO: Encoder or model? Encoder is consistent with the code, but might be less used WDYT? We also use ModelMeta ... -->
-
 A model in `mteb` covers two concepts: metadata and implementation.
 - Metadata contains information about the model such as maximum input
 length, valid frameworks, license, and degree of openness.

docs/api/results.md

@@ -0,0 +1,27 @@
+# Results
+
+When a models is evaluated in MTEB it produces results. These results consist of:
+
+- `TaskResult`: Result for a single task
+- `ModelResult`: Result for a model on a set of tasks 
+- `BenchmarkResults`: Result for a set of models models on a set of tasks 
+
+![](../images/visualizations/result_objects.png)
+
+In normal use these come up when running a model:
+```python
+# ...
+models_results = mteb.evaluate(model, tasks) 
+type(models_results) # mteb.results.ModelResults
+
+task_result = models_results.task_results
+type(models_results) # mteb.results.TaskResult
+```
+
+## Result Objects
+
+:::mteb.results.TaskResult
+
+:::mteb.results.ModelResult
+
+:::mteb.results.BenchmarkResults

mkdocs.yml

@@ -80,12 +80,6 @@ nav:
         - Loading Results: usage/loading_results.md
         - Command Line Interface: usage/cli.md
         - Running the Leaderboard: usage/leaderboard.md
-  - API:
-      - api/index.md
-      - Benchmark: api/benchmark.md
-      - Task: api/task.md
-      - Model: api/model.md
-      - Additional Types: api/types.md
   - Overview: 
       - overview/index.md
       - Benchmarks:
@@ -99,6 +93,13 @@ nav:
   #     - Adding a Benchmark: adding_a_leaderboard_tab.md
   #     - Adding a Task: adding_a_dataset.md
   #     - Development Setup: CONTRIBUTING.md
+  - API:
+      - Overview: api/index.md
+      - Benchmark: api/benchmark.md
+      - Task: api/task.md
+      - Model: api/model.md
+      - Results: api/results.md
+      - Additional Types: api/types.md
   - Leaderboard: https://huggingface.co/spaces/mteb/leaderboard
 
 plugins:

mteb/MTEB.py

@@ -27,14 +27,14 @@
 
 import mteb
 from mteb.abstasks import AbsTask
-from mteb.load_results.task_results import TaskResult
 from mteb.models import (
     CrossEncoderWrapper,
     Encoder,
     ModelMeta,
     MTEBModels,
     SentenceTransformerEncoderWrapper,
 )
+from mteb.results import TaskResult
 
 if TYPE_CHECKING:
     from sentence_transformers import CrossEncoder, SentenceTransformer

mteb/__init__.py

@@ -5,12 +5,12 @@
 from mteb.abstasks import AbsTask
 from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.evaluate import evaluate
-from mteb.load_results import BenchmarkResults, load_results
-from mteb.load_results.task_results import TaskResult
+from mteb.load_results import load_results
 from mteb.models import Encoder, SentenceTransformerEncoderWrapper
 from mteb.models.get_model_meta import get_model, get_model_meta, get_model_metas
 from mteb.MTEB import MTEB
 from mteb.overview import TASKS_REGISTRY, get_task, get_tasks
+from mteb.results import BenchmarkResults, TaskResult
 
 from .benchmarks.benchmark import Benchmark
 from .benchmarks.get_benchmark import get_benchmark, get_benchmarks

mteb/abstasks/AbsTaskTextRegression.py

@@ -18,9 +18,8 @@
     calculate_score_statistics,
     calculate_text_statistics,
 )
-from mteb.load_results.task_results import ScoresDict
 from mteb.models import MTEBModels
-from mteb.types import HFSubset
+from mteb.types import HFSubset, ScoresDict
 from mteb.types.statistics import DescriptiveStatistics, ScoreStatistics, TextStatistics
 
 from .AbsTask import AbsTask

mteb/abstasks/aggregated_task.py

@@ -5,13 +5,14 @@
 
 import numpy as np
 
+from mteb.results.task_result import TaskResult
+
 from .AbsTask import AbsTask
 from .aggregate_task_metadata import AggregateTaskMetadata
 
 if TYPE_CHECKING:
     from datasets import Dataset, DatasetDict
 
-    from mteb.load_results.task_results import TaskResult
     from mteb.models.models_protocols import Encoder
     from mteb.types import HFSubset, ScoresDict
     from mteb.types.statistics import DescriptiveStatistics
@@ -49,7 +50,7 @@ def task_results_to_scores(
             for task_res in task_results:
                 for langs in eval_langs:
                     main_scores.append(
-                        task_res.get_score_fast(
+                        task_res._get_score_fast(
                             languages=[lang.split("-")[0] for lang in langs],
                             splits=self.metadata.eval_splits,
                             subsets=subsets,
@@ -68,10 +69,6 @@ def combine_task_results(self, task_results: list[TaskResult]) -> TaskResult:
         """Combined the task results for using `task_results_to_scores`. Do not redefine this function if you want to implement a custom aggregation.
         Instead redefin `task_results_to_scores`.
         """
-        from mteb.load_results.task_results import (
-            TaskResult,  # to prevent circular imports, # TODO: can potentially likely be out of function in in v2.0.0
-        )
-
         eval_times = [tr.evaluation_time for tr in task_results if tr.evaluation_time]
         if len(eval_times) != len(task_results):
             logger.info(

mteb/benchmarks/benchmark.py

@@ -4,12 +4,12 @@
 from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
-from mteb.load_results.load_results import load_results
+from mteb.load_results import load_results
+from mteb.results import BenchmarkResults
 from mteb.types import StrURL
 
 if TYPE_CHECKING:
-    from mteb.abstasks.AbsTask import AbsTask
-    from mteb.load_results.benchmark_results import BenchmarkResults
+    from mteb.abstasks import AbsTask
 
 
 @dataclass

mteb/cache.py

@@ -11,9 +11,8 @@
 from typing import cast
 
 from mteb.abstasks import AbsTask
-from mteb.load_results.benchmark_results import BenchmarkResults, ModelResult
-from mteb.load_results.task_results import TaskResult
 from mteb.models import ModelMeta
+from mteb.results import BenchmarkResults, ModelResult, TaskResult
 from mteb.types import ModelName, Revision
 
 logger = logging.getLogger(__name__)

mteb/evaluate.py

@@ -11,8 +11,6 @@
 from mteb.abstasks.AbsTask import AbsTask
 from mteb.abstasks.aggregated_task import AbsTaskAggregate
 from mteb.cache import ResultCache
-from mteb.load_results.benchmark_results import ModelResult
-from mteb.load_results.task_results import TaskResult
 from mteb.models.model_meta import ModelMeta
 from mteb.models.models_protocols import (
     CrossEncoderProtocol,
@@ -23,6 +21,7 @@
     CrossEncoderWrapper,
     SentenceTransformerEncoderWrapper,
 )
+from mteb.results import ModelResult, TaskResult
 from mteb.types import HFSubset, SplitName
 from mteb.types._metadata import ModelName, Revision