add function to add frequently confused intent to intent report

rasa/nlu/test.py

@@ -398,6 +398,33 @@ def evaluate_response_selections(
     }
 
 
+def add_confused_intents_to_report(
+    report: Dict, target_intents: Iterable[Any], predicted_intents: Iterable[Any]
+) -> Dict:
+
+    from sklearn.metrics import confusion_matrix
+    from sklearn.utils.multiclass import unique_labels
+
+    cnf_matrix = confusion_matrix(target_intents, predicted_intents)
+
+    indices = np.argsort(cnf_matrix, axis=1)
+    labels = unique_labels(target_intents, predicted_intents)
+    n_candidates = min(3, len(labels))
+
+    for label in labels:
+        if report.get(label):
+            report[label]["confused_with"] = {}
+    for i, label in enumerate(labels):
+        for j in range(n_candidates):
+            label_idx = indices[i, -j]
+            _label = labels[label_idx]
+            num_hits = int(cnf_matrix[i, label_idx])
+            if _label != label and num_hits > 0:
+                report[label]["confused_with"][_label] = num_hits
+
+    return report
+
+
 def evaluate_intents(
     intent_results: List[IntentEvaluationResult],
     output_directory: Optional[Text],
@@ -435,6 +462,9 @@ def evaluate_intents(
         report, precision, f1, accuracy = get_evaluation_metrics(
             target_intents, predicted_intents, output_dict=True
         )
+        report = add_confused_intents_to_report(
+            report, target_intents, predicted_intents
+        )
 
         report_filename = os.path.join(output_directory, "intent_report.json")
 
@@ -473,6 +503,7 @@ def evaluate_intents(
 
         cnf_matrix = confusion_matrix(target_intents, predicted_intents)
         labels = unique_labels(target_intents, predicted_intents)
+
         plot_confusion_matrix(
             cnf_matrix,
             classes=labels,

tests/nlu/base/test_evaluation.py

@@ -306,7 +306,13 @@ def test_intent_evaluation_report(tmpdir_factory):
 
     report = json.loads(rasa.utils.io.read_file(report_filename))
 
-    greet_results = {"precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 1}
+    greet_results = {
+        "precision": 1.0,
+        "recall": 1.0,
+        "f1-score": 1.0,
+        "support": 1,
+        "confused_with": {},
+    }
 
     prediction = {
         "text": "hello",