Disaggregated stats wrong total

src/formpack/schema/fields.py

@@ -214,8 +214,20 @@ def get_disaggregated_stats(self, metrics, top_splitters,
         not_provided = 0
         provided = 0
         for val, counter in metrics.items():
-            not_provided += counter.pop(None, 0)
-            provided += counter.pop('__submissions__', 0)
+            none_submissions = counter.pop(None, 0)
+            not_provided += none_submissions
+            # `counter[None]` corresponds to submissions with no values for
+            # current field (`self`),
+            # If `val` is None, some submissions do not have any values for
+            # the `splitted_by_field`.
+            # We should consider all these submissions as `not_provided`
+            if val is None:
+                not_provided += sum(counter.values())
+            else:
+                # `counter['__submissions__'] contains the count of all
+                # submissions, including those where no response was provided.
+                # We need to subtract all submissions with no response
+                provided += counter.pop('__submissions__', 0) - none_submissions
 
         return {
             'total_count': not_provided + provided,

tests/fixtures/auto_report/v1.json

@@ -68,4 +68,4 @@
             "howmany": 1
         }
     ]
-}
+}

tests/fixtures/auto_report_without_answers/__init__.py

@@ -0,0 +1,15 @@
+# coding: utf-8
+
+from __future__ import (unicode_literals, print_function,
+                        absolute_import, division)
+
+
+from ..load_fixture_json import load_fixture_json
+
+DATA = {
+    u'title': 'Auto report without answers',
+    u'id_string': 'auto_report_without_answers',
+    u'versions': [
+        load_fixture_json('auto_report_without_answers/v1')
+    ],
+}

tests/fixtures/auto_report_without_answers/v1.json

@@ -0,0 +1,80 @@
+{
+    "id_string": "auto_report",
+    "version": "atwav1",
+    "content": {
+        "survey": [
+            {
+                "type": "select_one favorite_coffee",
+                "name": "favorite_coffee",
+                "label": "Favorite coffee"
+            },
+            {
+                "type": "select one type_of_coffee",
+                "name": "type_of_coffee",
+                "label": "Type of coffee"
+            }
+        ],
+        "choices": [
+            {
+                "list_name": "favorite_coffee",
+                "name": "nespresso",
+                "label": "Nespresso"
+            },
+            {
+                "list_name": "favorite_coffee",
+                "name": "keurig",
+                "label": "Keurig"
+            },
+            {
+                "list_name": "favorite_coffee",
+                "name": "tim_hortons",
+                "label": "Tim Hortons"
+            },
+            {
+                "list_name": "type_of_coffee",
+                "name": "latte",
+                "label": "Latte"
+            },
+            {
+                "list_name": "type_of_coffee",
+                "name": "regular",
+                "label": "Regular"
+            },
+            {
+                "list_name": "type_of_coffee",
+                "name": "espresso",
+                "label": "Espresso"
+            }
+        ]
+    },
+    "submissions": [
+        {
+            "favorite_coffee": "tim_hortons",
+            "type_of_coffee": "latte"
+        },
+        {
+            "favorite_coffee": "nespresso",
+            "type_of_coffee": "espresso"
+        },
+        {
+            "favorite_coffee": "keurig",
+            "type_of_coffee": null
+        },
+        {
+            "favorite_coffee": null,
+            "type_of_coffee": "latte"
+        },
+        {
+            "favorite_coffee": null,
+            "type_of_coffee": "latte"
+        },
+        {
+            "favorite_coffee": null,
+            "type_of_coffee": "latte"
+        },
+        {
+            "favorite_coffee": null,
+            "type_of_coffee": "latte"
+        }
+    ]
+}

tests/test_autoreport.py

@@ -483,6 +483,22 @@ def test_disaggregate_extended_fields(self):
                 assert percentage_responses == frequency_responses
                 assert percentage_responses[-1] == "..."
 
+    def test_disaggregate_without_answers(self):
+
+        title, schemas, submissions = build_fixture('auto_report_without_answers')  # noqa
+        fp = FormPack(schemas, title)
+
+        report = fp.autoreport()
+        stats = report.get_stats(submissions, split_by="favorite_coffee")
+
+        stats = [
+            (unicode(repr(field)), field_name, stats_dict)
+            for field, field_name, stats_dict in stats
+        ]
+
+        for stat in stats:
+            assert len(submissions) == stat[2]['total_count']
+
     def test_stats_with_non_numeric_value_for_numeric_field(self):
         """
         A string response to an integer question, for example, should not cause