overview.json
Overview​
The overview.json file is generated and stored in the output directory when running tests.
It contains metadata and the results from a test-run.
Sample overview.json​
{
"description": "QuickStart TestSet 1: NLU with ASR",
"scored_results_tsv_path": "output/latest/results.nuanscored.tsv",
"tool_version": "2.1.2-alpha.1",
"credentials": {
"asr_url": "asr.api.nuance.com:443",
"nlu_url": "nlu.api.nuance.com:443",
"client_id": "appID:devrel-demoapp-sandbox-20220110:geo:us:clientName:democlient",
"language": "eng-USA",
"asr": {
"topic": "GEN",
"auto_punctuate": true,
"dlm_uri": "urn:nuance-mix:tag:model/V1/mix.asr?=language=eng-USA",
"dlm_weight": 0.7,
"wordset_path": null,
"retry_count": 2,
"retry_sleep_s": 5
},
"asr_pprint": "{'auto_punctuate': True,\n 'dlm_uri': 'urn:nuance-mix:tag:model/V1/mix.asr?=language=eng-USA',\n 'dlm_weight': 0.7,\n 'retry_count': 2,\n 'retry_sleep_s': 5,\n 'topic': 'GEN',\n 'wordset_path': None}",
"nlu": {
"use_asr_results": true,
"model_uri": "urn:nuance-mix:tag:model/V1/mix.nlu?=language=eng-USA",
"retry_count": 2,
"retry_sleep_s": 5
},
"nlu_pprint": "{'model_uri': 'urn:nuance-mix:tag:model/V1/mix.nlu?=language=eng-USA',\n 'retry_count': 2,\n 'retry_sleep_s': 5,\n 'use_asr_results': True}",
"copy_audio": null
},
"session_id": "7f6c7f7e-929a-42e4-bb95-19c1c1cdae0f",
"transactions": {
"count": 21,
"passed": 21,
"failed": 0,
"server_failed": 0,
"tool_failed": 0,
"error_rate": 0.0
},
"transcriptions": {
"count": 130,
"passed": 108,
"failed": 22,
"word_error_rate": 16.92,
"S": 20,
"C": 108,
"I": 0,
"D": 2
},
"nlu": {
"count": 21,
"passed": 18,
"failed": 3,
"error_rate": 14.29
},
"nlu_stat_models": {
"count": 14,
"passed": 11,
"failed": 3,
"error_rate": 21.43
},
"nlu_grammar": {
"count": 7,
"passed": 7,
"failed": 0,
"error_rate": 0.0
},
"intents": {
"count": 21,
"passed": 18,
"failed": 3,
"error_rate": 14.29,
"confusion_matrix": {
"iBookFlight": {
"transcriptions": {
"count": 66,
"passed": 58,
"failed": 8,
"word_error_rate": 12.12,
"S": 8,
"C": 58,
"I": 0,
"D": 0
},
"nlu": {
"count": 8,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 6,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 2,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 25.0,
"correct_acceptance_rate": 0,
"correct_confirmation_rate": 0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"intents": {
"count": 8,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 6,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 2,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 25.0,
"correct_acceptance_rate": 0.0,
"correct_confirmation_rate": 0.0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"slots": {
"count": 16,
"passed": 16,
"failed": 0,
"error_rate": 0.0
},
"intents_actual": {
"iBookFlight": 6,
"NO_INTENT": 2
}
},
"iFAQLostLuggage": {
"transcriptions": {
"count": 11,
"passed": 5,
"failed": 6,
"word_error_rate": 54.55,
"S": 5,
"C": 5,
"I": 0,
"D": 1
},
"nlu": {
"count": 2,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 2,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 0,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 0.0,
"correct_acceptance_rate": 0,
"correct_confirmation_rate": 0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"intents": {
"count": 2,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 2,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 0,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 0.0,
"correct_acceptance_rate": 0.0,
"correct_confirmation_rate": 0.0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"slots": {
"count": 0,
"passed": 0,
"failed": 0,
"error_rate": 0.0
},
"intents_actual": {
"iFAQLostLuggage": 2
}
},
"iFlightStatus": {
"transcriptions": {
"count": 19,
"passed": 15,
"failed": 4,
"word_error_rate": 21.05,
"S": 3,
"C": 15,
"I": 0,
"D": 1
},
"nlu": {
"count": 4,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 4,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 0,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 0.0,
"correct_acceptance_rate": 0,
"correct_confirmation_rate": 0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"intents": {
"count": 4,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 4,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 0,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 0.0,
"correct_acceptance_rate": 0.0,
"correct_confirmation_rate": 0.0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"slots": {
"count": 0,
"passed": 0,
"failed": 0,
"error_rate": 0.0
},
"intents_actual": {
"iFlightStatus": 4
}
},
"NO_INTENT": {
"transcriptions": {
"count": 34,
"passed": 30,
"failed": 4,
"word_error_rate": 11.76,
"S": 4,
"C": 30,
"I": 0,
"D": 0
},
"nlu": {
"count": 7,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 6,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 1,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 14.29,
"correct_acceptance_rate": 0,
"correct_confirmation_rate": 0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"intents": {
"count": 7,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 6,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 1,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 14.29,
"correct_acceptance_rate": 0.0,
"correct_confirmation_rate": 0.0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"slots": {
"count": 8,
"passed": 7,
"failed": 1,
"error_rate": 12.5
},
"intents_actual": {
"iBookFlight": 1,
"NO_INTENT": 6
}
}
},
"stat_model_confusion_matrix": {
"iBookFlight": {
"transcriptions": {
"count": 43,
"passed": 38,
"failed": 5,
"word_error_rate": 11.63,
"S": 5,
"C": 38,
"I": 0,
"D": 0
},
"nlu": {
"count": 5,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 3,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 2,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 40.0,
"correct_acceptance_rate": 0,
"correct_confirmation_rate": 0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"intents": {
"count": 5,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 3,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 2,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 40.0,
"correct_acceptance_rate": 0.0,
"correct_confirmation_rate": 0.0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"slots": {
"count": 11,
"passed": 11,
"failed": 0,
"error_rate": 0.0
},
"intents_actual": {
"iBookFlight": 3,
"NO_INTENT": 2
}
},
"iFAQLostLuggage": {
"transcriptions": {
"count": 8,
"passed": 3,
"failed": 5,
"word_error_rate": 62.5,
"S": 4,
"C": 3,
"I": 0,
"D": 1
},
"nlu": {
"count": 1,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 1,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 0,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 0.0,
"correct_acceptance_rate": 0,
"correct_confirmation_rate": 0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"intents": {
"count": 1,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 1,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 0,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 0.0,
"correct_acceptance_rate": 0.0,
"correct_confirmation_rate": 0.0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"slots": {
"count": 0,
"passed": 0,
"failed": 0,
"error_rate": 0.0
},
"intents_actual": {
"iFAQLostLuggage": 1
}
},
"iFlightStatus": {
"transcriptions": {
"count": 19,
"passed": 15,
"failed": 4,
"word_error_rate": 21.05,
"S": 3,
"C": 15,
"I": 0,
"D": 1
},
"nlu": {
"count": 4,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 4,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 0,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 0.0,
"correct_acceptance_rate": 0,
"correct_confirmation_rate": 0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"intents": {
"count": 4,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 4,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 0,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 0.0,
"correct_acceptance_rate": 0.0,
"correct_confirmation_rate": 0.0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"slots": {
"count": 0,
"passed": 0,
"failed": 0,
"error_rate": 0.0
},
"intents_actual": {
"iFlightStatus": 4
}
},
"NO_INTENT": {
"transcriptions": {
"count": 28,
"passed": 24,
"failed": 4,
"word_error_rate": 14.29,
"S": 4,
"C": 24,
"I": 0,
"D": 0
},
"nlu": {
"count": 4,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 3,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 1,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 25.0,
"correct_acceptance_rate": 0,
"correct_confirmation_rate": 0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"intents": {
"count": 4,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 3,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 1,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 25.0,
"correct_acceptance_rate": 0.0,
"correct_confirmation_rate": 0.0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"slots": {
"count": 5,
"passed": 4,
"failed": 1,
"error_rate": 20.0
},
"intents_actual": {
"iBookFlight": 1,
"NO_INTENT": 3
}
}
},
"grammar_confusion_matrix": {
"iBookFlight": {
"transcriptions": {
"count": 23,
"passed": 20,
"failed": 3,
"word_error_rate": 13.04,
"S": 3,
"C": 20,
"I": 0,
"D": 0
},
"nlu": {
"count": 3,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 3,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 0,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 0.0,
"correct_acceptance_rate": 0,
"correct_confirmation_rate": 0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"intents": {
"count": 3,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 3,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 0,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 0.0,
"correct_acceptance_rate": 0.0,
"correct_confirmation_rate": 0.0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"slots": {
"count": 5,
"passed": 5,
"failed": 0,
"error_rate": 0.0
},
"intents_actual": {
"iBookFlight": 3
}
},
"iFAQLostLuggage": {
"transcriptions": {
"count": 3,
"passed": 2,
"failed": 1,
"word_error_rate": 33.33,
"S": 1,
"C": 2,
"I": 0,
"D": 0
},
"nlu": {
"count": 1,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 1,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 0,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 0.0,
"correct_acceptance_rate": 0,
"correct_confirmation_rate": 0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"intents": {
"count": 1,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 1,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 0,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 0.0,
"correct_acceptance_rate": 0.0,
"correct_confirmation_rate": 0.0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"slots": {
"count": 0,
"passed": 0,
"failed": 0,
"error_rate": 0.0
},
"intents_actual": {
"iFAQLostLuggage": 1
}
},
"NO_INTENT": {
"transcriptions": {
"count": 6,
"passed": 6,
"failed": 0,
"word_error_rate": 0.0,
"S": 0,
"C": 6,
"I": 0,
"D": 0
},
"nlu": {
"count": 3,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 3,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 0,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 0.0,
"correct_acceptance_rate": 0,
"correct_confirmation_rate": 0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"intents": {
"count": 3,
"count_above_acceptance_threshold": 0,
"count_above_confirmation_threshold": 0,
"passed": 3,
"passed_above_acceptance_threshold": 0,
"passed_above_confirmation_threshold": 0,
"failed": 0,
"failed_above_acceptance_threshold": 0,
"failed_above_confirmation_threshold": 0,
"error_rate": 0.0,
"correct_acceptance_rate": 0.0,
"correct_confirmation_rate": 0.0,
"failed_acceptance_rate": 0,
"failed_confirmation_rate": 0
},
"slots": {
"count": 3,
"passed": 3,
"failed": 0,
"error_rate": 0.0
},
"intents_actual": {
"NO_INTENT": 3
}
}
}
},
"slots": {
"count": 24,
"passed": 23,
"failed": 1,
"error_rate": 4.17,
"slots": {
"eArrivalCity": {
"passed": 10,
"failed": 1,
"count": 11,
"error_rate": 9.09
},
"eDepartureDate": {
"passed": 6,
"failed": 0,
"count": 6,
"error_rate": 0.0
},
"eDepartureCity": {
"passed": 6,
"failed": 0,
"count": 6,
"error_rate": 0.0
},
"nuance_CALENDARX": {
"passed": 1,
"failed": 0,
"count": 1,
"error_rate": 0.0
}
}
},
"rows": [
{
"index": 0,
"perfect": false,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-1.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0001_asraas.log",
"transaction_log_nlu": "./xaas_logs/0001_nluaas.log",
"word_error_rate": 20,
"slot_error_rate": 0.0,
"transcription": {
"expected": "Book a flight to Atlanta",
"actual": "Book a flight to Atlanta.",
"expected_annotated": [
{
"type": "C",
"word": "Book"
},
{
"type": "C",
"word": "a"
},
{
"type": "C",
"word": "flight"
},
{
"type": "C",
"word": "to"
},
{
"type": "S",
"word": "Atlanta"
}
],
"actual_annotated": [
{
"type": "C",
"word": "Book"
},
{
"type": "C",
"word": "a"
},
{
"type": "C",
"word": "flight"
},
{
"type": "C",
"word": "to"
},
{
"type": "S",
"word": "Atlanta."
}
],
"score": [
[
{
"type": "S",
"index_expected": 4,
"index_actual": 4,
"word_expected": "Atlanta",
"word_actual": "Atlanta."
}
]
],
"alignment": [
[
0,
0,
"C",
0,
"Book",
"Book"
],
[
1,
1,
"C",
0,
"a",
"a"
],
[
2,
2,
"C",
0,
"flight",
"flight"
],
[
3,
3,
"C",
0,
"to",
"to"
],
[
4,
4,
"S",
4,
"Atlanta",
"Atlanta."
]
]
},
"intent": {
"expected": "iBookFlight",
"actual": "iBookFlight",
"score": true
},
"slots": {
"passed": 1,
"failed": 0,
"slots": [
{
"slot": "eArrivalCity",
"expected": "Atlanta",
"actual": "Atlanta",
"score": true
}
]
},
"origin": "GRAMMAR",
"confidence": {
"ASR": "0.974",
"ASR+NLU": "0.9999524"
},
"request_id": "5b72b294-6761-4f7f-b058-fc6e3b26594f"
},
{
"index": 1,
"perfect": false,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-2.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0002_asraas.log",
"transaction_log_nlu": "./xaas_logs/0002_nluaas.log",
"word_error_rate": 11,
"slot_error_rate": 0.0,
"transcription": {
"expected": "I'd like to go to Atlanta on May 4th",
"actual": "I'd like to go to Atlanta on may fourth",
"expected_annotated": [
{
"type": "C",
"word": "I'd"
},
{
"type": "C",
"word": "like"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "go"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "Atlanta"
},
{
"type": "C",
"word": "on"
},
{
"type": "C",
"word": "May"
},
{
"type": "S",
"word": "4th"
}
],
"actual_annotated": [
{
"type": "C",
"word": "I'd"
},
{
"type": "C",
"word": "like"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "go"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "Atlanta"
},
{
"type": "C",
"word": "on"
},
{
"type": "C",
"word": "may"
},
{
"type": "S",
"word": "fourth"
}
],
"score": [
[
{
"type": "C",
"index_expected": 7,
"index_actual": 7,
"word_expected": "May",
"word_actual": "may"
}
],
[
{
"type": "S",
"index_expected": 8,
"index_actual": 8,
"word_expected": "4th",
"word_actual": "fourth"
}
]
],
"alignment": [
[
0,
0,
"C",
0,
"I'd",
"I'd"
],
[
1,
1,
"C",
0,
"like",
"like"
],
[
2,
2,
"C",
0,
"to",
"to"
],
[
3,
3,
"C",
0,
"go",
"go"
],
[
4,
4,
"C",
0,
"to",
"to"
],
[
5,
5,
"C",
0,
"Atlanta",
"Atlanta"
],
[
6,
6,
"C",
0,
"on",
"on"
],
[
7,
7,
"C",
0,
"May",
"may"
],
[
8,
8,
"S",
4,
"4th",
"fourth"
]
]
},
"intent": {
"expected": "iBookFlight",
"actual": "iBookFlight",
"score": true
},
"slots": {
"passed": 2,
"failed": 0,
"slots": [
{
"slot": "eArrivalCity",
"expected": "Atlanta",
"actual": "Atlanta",
"score": true
},
{
"slot": "eDepartureDate",
"expected": "{'nuance_CALENDAR.nuance_DATE.nuance_DATE_ABS.nuance_DAY': 4 'nuance_CALENDAR.nuance_DATE.nuance_DATE_ABS.nuance_MONTH': 5}",
"actual": {
"nuance_CALENDAR.nuance_DATE.nuance_DATE_ABS.nuance_MONTH": 5,
"nuance_CALENDAR.nuance_DATE.nuance_DATE_ABS.nuance_DAY": 4
},
"score": true
}
]
},
"origin": "GRAMMAR",
"confidence": {
"ASR": "0.724",
"ASR+NLU": "0.4339038"
},
"request_id": "053d02b0-600d-4a0a-8405-0f2a77a9579a"
},
{
"index": 2,
"perfect": false,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-3.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0003_asraas.log",
"transaction_log_nlu": "./xaas_logs/0003_nluaas.log",
"word_error_rate": 11,
"slot_error_rate": 0.0,
"transcription": {
"expected": "Is there a flight from New York to Phoenix?",
"actual": "Is there a flight from New York to Phoenix",
"expected_annotated": [
{
"type": "C",
"word": "Is"
},
{
"type": "C",
"word": "there"
},
{
"type": "C",
"word": "a"
},
{
"type": "C",
"word": "flight"
},
{
"type": "C",
"word": "from"
},
{
"type": "C",
"word": "New"
},
{
"type": "C",
"word": "York"
},
{
"type": "C",
"word": "to"
},
{
"type": "S",
"word": "Phoenix?"
}
],
"actual_annotated": [
{
"type": "C",
"word": "Is"
},
{
"type": "C",
"word": "there"
},
{
"type": "C",
"word": "a"
},
{
"type": "C",
"word": "flight"
},
{
"type": "C",
"word": "from"
},
{
"type": "C",
"word": "New"
},
{
"type": "C",
"word": "York"
},
{
"type": "C",
"word": "to"
},
{
"type": "S",
"word": "Phoenix"
}
],
"score": [
[
{
"type": "S",
"index_expected": 8,
"index_actual": 8,
"word_expected": "Phoenix?",
"word_actual": "Phoenix"
}
]
],
"alignment": [
[
0,
0,
"C",
0,
"Is",
"Is"
],
[
1,
1,
"C",
0,
"there",
"there"
],
[
2,
2,
"C",
0,
"a",
"a"
],
[
3,
3,
"C",
0,
"flight",
"flight"
],
[
4,
4,
"C",
0,
"from",
"from"
],
[
5,
5,
"C",
0,
"New",
"New"
],
[
6,
6,
"C",
0,
"York",
"York"
],
[
7,
7,
"C",
0,
"to",
"to"
],
[
8,
8,
"S",
4,
"Phoenix?",
"Phoenix"
]
]
},
"intent": {
"expected": "iBookFlight",
"actual": "iBookFlight",
"score": true
},
"slots": {
"passed": 2,
"failed": 0,
"slots": [
{
"slot": "eDepartureCity",
"expected": "New York City",
"actual": "New York City",
"score": true
},
{
"slot": "eArrivalCity",
"expected": "Phoenix",
"actual": "Phoenix",
"score": true
}
]
},
"origin": "GRAMMAR",
"confidence": {
"ASR": "0.922",
"ASR+NLU": "0.9999766"
},
"request_id": "c9b62bd1-67c6-4f79-888b-8339fbf52b19"
},
{
"index": 3,
"perfect": false,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-4.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0004_asraas.log",
"transaction_log_nlu": "./xaas_logs/0004_nluaas.log",
"word_error_rate": 33,
"slot_error_rate": 0.0,
"transcription": {
"expected": "Can I fly to San Francisco?",
"actual": "Can I flight to San Francisco.",
"expected_annotated": [
{
"type": "C",
"word": "Can"
},
{
"type": "C",
"word": "I"
},
{
"type": "S",
"word": "fly"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "San"
},
{
"type": "S",
"word": "Francisco?"
}
],
"actual_annotated": [
{
"type": "C",
"word": "Can"
},
{
"type": "C",
"word": "I"
},
{
"type": "S",
"word": "flight"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "San"
},
{
"type": "S",
"word": "Francisco."
}
],
"score": [
[
{
"type": "S",
"index_expected": 2,
"index_actual": 2,
"word_expected": "fly",
"word_actual": "flight"
}
],
[
{
"type": "S",
"index_expected": 5,
"index_actual": 5,
"word_expected": "Francisco?",
"word_actual": "Francisco."
}
]
],
"alignment": [
[
0,
0,
"C",
0,
"Can",
"Can"
],
[
1,
1,
"C",
0,
"I",
"I"
],
[
2,
2,
"S",
4,
"fly",
"flight"
],
[
3,
3,
"C",
0,
"to",
"to"
],
[
4,
4,
"C",
0,
"San",
"San"
],
[
5,
5,
"S",
4,
"Francisco?",
"Francisco."
]
]
},
"intent": {
"expected": "iBookFlight",
"actual": "iBookFlight",
"score": true
},
"slots": {
"passed": 1,
"failed": 0,
"slots": [
{
"slot": "eArrivalCity",
"expected": "San Francisco",
"actual": "San Francisco",
"score": true
}
]
},
"origin": "STATISTICAL",
"confidence": {
"ASR": "0.77",
"ASR+NLU": "0.6989437"
},
"request_id": "944fa869-0734-4d01-9c10-5c07c0a62d62"
},
{
"index": 4,
"perfect": false,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-5.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0005_asraas.log",
"transaction_log_nlu": "./xaas_logs/0005_nluaas.log",
"word_error_rate": 17,
"slot_error_rate": 0.0,
"transcription": {
"expected": "I'm going to New York tomorrow",
"actual": "I'm going to New York tomorrow.",
"expected_annotated": [
{
"type": "C",
"word": "I'm"
},
{
"type": "C",
"word": "going"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "New"
},
{
"type": "C",
"word": "York"
},
{
"type": "S",
"word": "tomorrow"
}
],
"actual_annotated": [
{
"type": "C",
"word": "I'm"
},
{
"type": "C",
"word": "going"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "New"
},
{
"type": "C",
"word": "York"
},
{
"type": "S",
"word": "tomorrow."
}
],
"score": [
[
{
"type": "S",
"index_expected": 5,
"index_actual": 5,
"word_expected": "tomorrow",
"word_actual": "tomorrow."
}
]
],
"alignment": [
[
0,
0,
"C",
0,
"I'm",
"I'm"
],
[
1,
1,
"C",
0,
"going",
"going"
],
[
2,
2,
"C",
0,
"to",
"to"
],
[
3,
3,
"C",
0,
"New",
"New"
],
[
4,
4,
"C",
0,
"York",
"York"
],
[
5,
5,
"S",
4,
"tomorrow",
"tomorrow."
]
]
},
"intent": {
"expected": "iBookFlight",
"actual": "iBookFlight",
"score": true
},
"slots": {
"passed": 2,
"failed": 0,
"slots": [
{
"slot": "eArrivalCity",
"expected": "New York City",
"actual": "New York City",
"score": true
},
{
"slot": "eDepartureDate",
"expected": "{'nuance_CALENDAR.nuance_DATE.nuance_DATE_REL.nuance_STEP': 'day' 'nuance_CALENDAR.nuance_DATE.nuance_DATE_REL.nuance_INCREMENT': 1}",
"actual": {
"nuance_CALENDAR.nuance_DATE.nuance_DATE_REL.nuance_STEP": "day",
"nuance_CALENDAR.nuance_DATE.nuance_DATE_REL.nuance_INCREMENT": 1
},
"score": true
}
]
},
"origin": "STATISTICAL",
"confidence": {
"ASR": "0.967",
"ASR+NLU": "0.97967607"
},
"request_id": "05c15627-3bf3-4c5b-b247-5d1f95ea860a"
},
{
"index": 5,
"perfect": false,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-6.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0006_asraas.log",
"transaction_log_nlu": "./xaas_logs/0006_nluaas.log",
"word_error_rate": 62,
"slot_error_rate": 0.0,
"transcription": {
"expected": "Did I lose my luggage in New York?",
"actual": "Today was my luggage & New York",
"expected_annotated": [
{
"type": "D",
"word": "Did"
},
{
"type": "S",
"word": "I"
},
{
"type": "S",
"word": "lose"
},
{
"type": "C",
"word": "my"
},
{
"type": "C",
"word": "luggage"
},
{
"type": "S",
"word": "in"
},
{
"type": "C",
"word": "New"
},
{
"type": "S",
"word": "York?"
}
],
"actual_annotated": [
{
"type": "S",
"word": "Today"
},
{
"type": "S",
"word": "was"
},
{
"type": "C",
"word": "my"
},
{
"type": "C",
"word": "luggage"
},
{
"type": "S",
"word": "&"
},
{
"type": "C",
"word": "New"
},
{
"type": "S",
"word": "York"
}
],
"score": [
[
{
"type": "D",
"index_expected": 0,
"index_actual": 0,
"word_expected": "Did",
"word_actual": "*"
},
{
"type": "S",
"index_expected": 1,
"index_actual": 0,
"word_expected": "I",
"word_actual": "Today"
},
{
"type": "S",
"index_expected": 2,
"index_actual": 1,
"word_expected": "lose",
"word_actual": "was"
}
],
[
{
"type": "S",
"index_expected": 5,
"index_actual": 4,
"word_expected": "in",
"word_actual": "&"
}
],
[
{
"type": "S",
"index_expected": 7,
"index_actual": 6,
"word_expected": "York?",
"word_actual": "York"
}
]
],
"alignment": [
[
0,
0,
"D",
3,
"Did",
"*"
],
[
1,
0,
"S",
4,
"I",
"Today"
],
[
2,
1,
"S",
4,
"lose",
"was"
],
[
3,
2,
"C",
0,
"my",
"my"
],
[
4,
3,
"C",
0,
"luggage",
"luggage"
],
[
5,
4,
"S",
4,
"in",
"&"
],
[
6,
5,
"C",
0,
"New",
"New"
],
[
7,
6,
"S",
4,
"York?",
"York"
]
]
},
"intent": {
"expected": "iFAQLostLuggage",
"actual": "iFAQLostLuggage",
"score": true
},
"slots": {
"passed": 0,
"failed": 0,
"slots": []
},
"origin": "STATISTICAL",
"confidence": {
"ASR": "0.582",
"ASR+NLU": "0.95779264"
},
"request_id": "bd0ba0c9-8430-4bf9-9c9a-a1d5048fc9a2"
},
{
"index": 6,
"perfect": false,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-7.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0007_asraas.log",
"transaction_log_nlu": "./xaas_logs/0007_nluaas.log",
"word_error_rate": 33,
"slot_error_rate": 0.0,
"transcription": {
"expected": "Where's my luggage?",
"actual": "Where's my luggage",
"expected_annotated": [
{
"type": "C",
"word": "Where's"
},
{
"type": "C",
"word": "my"
},
{
"type": "S",
"word": "luggage?"
}
],
"actual_annotated": [
{
"type": "C",
"word": "Where's"
},
{
"type": "C",
"word": "my"
},
{
"type": "S",
"word": "luggage"
}
],
"score": [
[
{
"type": "S",
"index_expected": 2,
"index_actual": 2,
"word_expected": "luggage?",
"word_actual": "luggage"
}
]
],
"alignment": [
[
0,
0,
"C",
0,
"Where's",
"Where's"
],
[
1,
1,
"C",
0,
"my",
"my"
],
[
2,
2,
"S",
4,
"luggage?",
"luggage"
]
]
},
"intent": {
"expected": "iFAQLostLuggage",
"actual": "iFAQLostLuggage",
"score": true
},
"slots": {
"passed": 0,
"failed": 0,
"slots": []
},
"origin": "GRAMMAR",
"confidence": {
"ASR": "0.952",
"ASR+NLU": "0.999963"
},
"request_id": "49e209f1-e0cb-42eb-9d2e-f173c99eafd1"
},
{
"index": 7,
"perfect": false,
"perfect_nlu": false,
"error": null,
"error_type": "",
"audio": "./wav/1-8.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0008_asraas.log",
"transaction_log_nlu": "./xaas_logs/0008_nluaas.log",
"word_error_rate": 9,
"slot_error_rate": 0.0,
"transcription": {
"expected": "I'd like to travel from Phoenix to Atlanta on July 20th",
"actual": "I'd like to travel from phoenix to Atlanta on July 20.",
"expected_annotated": [
{
"type": "C",
"word": "I'd"
},
{
"type": "C",
"word": "like"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "travel"
},
{
"type": "C",
"word": "from"
},
{
"type": "C",
"word": "Phoenix"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "Atlanta"
},
{
"type": "C",
"word": "on"
},
{
"type": "C",
"word": "July"
},
{
"type": "S",
"word": "20th"
}
],
"actual_annotated": [
{
"type": "C",
"word": "I'd"
},
{
"type": "C",
"word": "like"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "travel"
},
{
"type": "C",
"word": "from"
},
{
"type": "C",
"word": "phoenix"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "Atlanta"
},
{
"type": "C",
"word": "on"
},
{
"type": "C",
"word": "July"
},
{
"type": "S",
"word": "20."
}
],
"score": [
[
{
"type": "C",
"index_expected": 5,
"index_actual": 5,
"word_expected": "Phoenix",
"word_actual": "phoenix"
}
],
[
{
"type": "S",
"index_expected": 10,
"index_actual": 10,
"word_expected": "20th",
"word_actual": "20."
}
]
],
"alignment": [
[
0,
0,
"C",
0,
"I'd",
"I'd"
],
[
1,
1,
"C",
0,
"like",
"like"
],
[
2,
2,
"C",
0,
"to",
"to"
],
[
3,
3,
"C",
0,
"travel",
"travel"
],
[
4,
4,
"C",
0,
"from",
"from"
],
[
5,
5,
"C",
0,
"Phoenix",
"phoenix"
],
[
6,
6,
"C",
0,
"to",
"to"
],
[
7,
7,
"C",
0,
"Atlanta",
"Atlanta"
],
[
8,
8,
"C",
0,
"on",
"on"
],
[
9,
9,
"C",
0,
"July",
"July"
],
[
10,
10,
"S",
4,
"20th",
"20."
]
]
},
"intent": {
"expected": "iBookFlight",
"actual": "NO_INTENT",
"score": false
},
"slots": {
"passed": 3,
"failed": 0,
"slots": [
{
"slot": "eArrivalCity",
"expected": "Atlanta",
"actual": "Atlanta",
"score": true
},
{
"slot": "eDepartureCity",
"expected": "Phoenix",
"actual": "Phoenix",
"score": true
},
{
"slot": "eDepartureDate",
"expected": "{'nuance_CALENDAR.nuance_DATE.nuance_DATE_ABS.nuance_MONTH': 7 'nuance_CALENDAR.nuance_DATE.nuance_DATE_ABS.nuance_DAY': 20}",
"actual": {
"nuance_CALENDAR.nuance_DATE.nuance_DATE_ABS.nuance_MONTH": 7,
"nuance_CALENDAR.nuance_DATE.nuance_DATE_ABS.nuance_DAY": 20
},
"score": true
}
]
},
"origin": "STATISTICAL",
"confidence": {
"ASR": "0.489",
"ASR+NLU": "0.9602964"
},
"request_id": "ef8746ab-dc2f-4151-8503-63f1b0160c97"
},
{
"index": 8,
"perfect": true,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-9.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0009_asraas.log",
"transaction_log_nlu": "./xaas_logs/0009_nluaas.log",
"word_error_rate": 0,
"slot_error_rate": 0.0,
"transcription": {
"expected": "Yeah is my flight on time?",
"actual": "Yeah is my flight on time?",
"expected_annotated": [
{
"type": "C",
"word": "Yeah"
},
{
"type": "C",
"word": ""
},
{
"type": "C",
"word": "is"
},
{
"type": "C",
"word": "my"
},
{
"type": "C",
"word": "flight"
},
{
"type": "C",
"word": "on"
},
{
"type": "C",
"word": "time?"
}
],
"actual_annotated": [
{
"type": "C",
"word": "Yeah"
},
{
"type": "C",
"word": "is"
},
{
"type": "C",
"word": "my"
},
{
"type": "C",
"word": "flight"
},
{
"type": "C",
"word": "on"
},
{
"type": "C",
"word": "time?"
}
],
"score": [],
"alignment": [
[
0,
0,
"C",
0,
"Yeah",
"Yeah"
],
[
1,
1,
"C",
0,
"is",
"is"
],
[
2,
2,
"C",
0,
"my",
"my"
],
[
3,
3,
"C",
0,
"flight",
"flight"
],
[
4,
4,
"C",
0,
"on",
"on"
],
[
5,
5,
"C",
0,
"time?",
"time?"
]
]
},
"intent": {
"expected": "iFlightStatus",
"actual": "iFlightStatus",
"score": true
},
"slots": {
"passed": 0,
"failed": 0,
"slots": []
},
"origin": "STATISTICAL",
"confidence": {
"ASR": "0.735",
"ASR+NLU": "0.9963636"
},
"request_id": "f962fc89-3ed8-47f0-ba84-64656c60c4db"
},
{
"index": 9,
"perfect": false,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-10.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0010_asraas.log",
"transaction_log_nlu": "./xaas_logs/0010_nluaas.log",
"word_error_rate": 25,
"slot_error_rate": 0.0,
"transcription": {
"expected": "When is my flight?",
"actual": "When is my flight",
"expected_annotated": [
{
"type": "C",
"word": "When"
},
{
"type": "C",
"word": "is"
},
{
"type": "C",
"word": "my"
},
{
"type": "S",
"word": "flight?"
}
],
"actual_annotated": [
{
"type": "C",
"word": "When"
},
{
"type": "C",
"word": "is"
},
{
"type": "C",
"word": "my"
},
{
"type": "S",
"word": "flight"
}
],
"score": [
[
{
"type": "S",
"index_expected": 3,
"index_actual": 3,
"word_expected": "flight?",
"word_actual": "flight"
}
]
],
"alignment": [
[
0,
0,
"C",
0,
"When",
"When"
],
[
1,
1,
"C",
0,
"is",
"is"
],
[
2,
2,
"C",
0,
"my",
"my"
],
[
3,
3,
"S",
4,
"flight?",
"flight"
]
]
},
"intent": {
"expected": "iFlightStatus",
"actual": "iFlightStatus",
"score": true
},
"slots": {
"passed": 0,
"failed": 0,
"slots": []
},
"origin": "STATISTICAL",
"confidence": {
"ASR": "0.903",
"ASR+NLU": "0.97848344"
},
"request_id": "1876eee1-bd95-4aaa-ad5f-0b267e5bb357"
},
{
"index": 10,
"perfect": true,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-11.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0011_asraas.log",
"transaction_log_nlu": "./xaas_logs/0011_nluaas.log",
"word_error_rate": 0,
"slot_error_rate": 0.0,
"transcription": {
"expected": "What's my itinerary?",
"actual": "What's my itinerary?",
"expected_annotated": [
{
"type": "C",
"word": "What's"
},
{
"type": "C",
"word": "my"
},
{
"type": "C",
"word": "itinerary?"
}
],
"actual_annotated": [
{
"type": "C",
"word": "What's"
},
{
"type": "C",
"word": "my"
},
{
"type": "C",
"word": "itinerary?"
}
],
"score": [],
"alignment": [
[
0,
0,
"C",
0,
"What's",
"What's"
],
[
1,
1,
"C",
0,
"my",
"my"
],
[
2,
2,
"C",
0,
"itinerary?",
"itinerary?"
]
]
},
"intent": {
"expected": "iFlightStatus",
"actual": "iFlightStatus",
"score": true
},
"slots": {
"passed": 0,
"failed": 0,
"slots": []
},
"origin": "STATISTICAL",
"confidence": {
"ASR": "0.628",
"ASR+NLU": "0.42121994"
},
"request_id": "bdf78922-a5b8-4b68-a867-8a34487b13dd"
},
{
"index": 11,
"perfect": false,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-12.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0012_asraas.log",
"transaction_log_nlu": "./xaas_logs/0012_nluaas.log",
"word_error_rate": 60,
"slot_error_rate": 0.0,
"transcription": {
"expected": "What is my flight status?",
"actual": "What's my flight status",
"expected_annotated": [
{
"type": "D",
"word": "What"
},
{
"type": "S",
"word": "is"
},
{
"type": "C",
"word": "my"
},
{
"type": "C",
"word": "flight"
},
{
"type": "S",
"word": "status?"
}
],
"actual_annotated": [
{
"type": "S",
"word": "What's"
},
{
"type": "C",
"word": "my"
},
{
"type": "C",
"word": "flight"
},
{
"type": "S",
"word": "status"
}
],
"score": [
[
{
"type": "D",
"index_expected": 0,
"index_actual": 0,
"word_expected": "What",
"word_actual": "*"
},
{
"type": "S",
"index_expected": 1,
"index_actual": 0,
"word_expected": "is",
"word_actual": "What's"
}
],
[
{
"type": "S",
"index_expected": 4,
"index_actual": 3,
"word_expected": "status?",
"word_actual": "status"
}
]
],
"alignment": [
[
0,
0,
"D",
3,
"What",
"*"
],
[
1,
0,
"S",
4,
"is",
"What's"
],
[
2,
1,
"C",
0,
"my",
"my"
],
[
3,
2,
"C",
0,
"flight",
"flight"
],
[
4,
3,
"S",
4,
"status?",
"status"
]
]
},
"intent": {
"expected": "iFlightStatus",
"actual": "iFlightStatus",
"score": true
},
"slots": {
"passed": 0,
"failed": 0,
"slots": []
},
"origin": "STATISTICAL",
"confidence": {
"ASR": "0.951",
"ASR+NLU": "0.99828494"
},
"request_id": "45062730-c30f-4842-82be-aa0b36efbc08"
},
{
"index": 12,
"perfect": false,
"perfect_nlu": false,
"error": null,
"error_type": "",
"audio": "./wav/1-13.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0013_asraas.log",
"transaction_log_nlu": "./xaas_logs/0013_nluaas.log",
"word_error_rate": 29,
"slot_error_rate": 100.0,
"transcription": {
"expected": "What's a good restaurant in New York?",
"actual": "What's a good restaurant to New York.",
"expected_annotated": [
{
"type": "C",
"word": "What's"
},
{
"type": "C",
"word": "a"
},
{
"type": "C",
"word": "good"
},
{
"type": "C",
"word": "restaurant"
},
{
"type": "S",
"word": "in"
},
{
"type": "C",
"word": "New"
},
{
"type": "S",
"word": "York?"
}
],
"actual_annotated": [
{
"type": "C",
"word": "What's"
},
{
"type": "C",
"word": "a"
},
{
"type": "C",
"word": "good"
},
{
"type": "C",
"word": "restaurant"
},
{
"type": "S",
"word": "to"
},
{
"type": "C",
"word": "New"
},
{
"type": "S",
"word": "York."
}
],
"score": [
[
{
"type": "S",
"index_expected": 4,
"index_actual": 4,
"word_expected": "in",
"word_actual": "to"
}
],
[
{
"type": "S",
"index_expected": 6,
"index_actual": 6,
"word_expected": "York?",
"word_actual": "York."
}
]
],
"alignment": [
[
0,
0,
"C",
0,
"What's",
"What's"
],
[
1,
1,
"C",
0,
"a",
"a"
],
[
2,
2,
"C",
0,
"good",
"good"
],
[
3,
3,
"C",
0,
"restaurant",
"restaurant"
],
[
4,
4,
"S",
4,
"in",
"to"
],
[
5,
5,
"C",
0,
"New",
"New"
],
[
6,
6,
"S",
4,
"York?",
"York."
]
]
},
"intent": {
"expected": "NO_INTENT",
"actual": "iBookFlight",
"score": false
},
"slots": {
"passed": 0,
"failed": 1,
"slots": [
{
"slot": "eArrivalCity",
"expected": "NULL",
"actual": "New York City",
"score": false
}
]
},
"origin": "STATISTICAL",
"confidence": {
"ASR": "0.467",
"ASR+NLU": "0.90563804"
},
"request_id": "6de1709e-fda2-4ab3-a9ec-ea8839b3809c"
},
{
"index": 13,
"perfect": true,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-14.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0014_asraas.log",
"transaction_log_nlu": "./xaas_logs/0014_nluaas.log",
"word_error_rate": 0,
"slot_error_rate": 0.0,
"transcription": {
"expected": "To New York",
"actual": "To New York",
"expected_annotated": [
{
"type": "C",
"word": "To"
},
{
"type": "C",
"word": "New"
},
{
"type": "C",
"word": "York"
}
],
"actual_annotated": [
{
"type": "C",
"word": "To"
},
{
"type": "C",
"word": "New"
},
{
"type": "C",
"word": "York"
}
],
"score": [],
"alignment": [
[
0,
0,
"C",
0,
"To",
"To"
],
[
1,
1,
"C",
0,
"New",
"New"
],
[
2,
2,
"C",
0,
"York",
"York"
]
]
},
"intent": {
"expected": "NO_INTENT",
"actual": "NO_INTENT",
"score": true
},
"slots": {
"passed": 1,
"failed": 0,
"slots": [
{
"slot": "eArrivalCity",
"expected": "New York City",
"actual": "New York City",
"score": true
}
]
},
"origin": "GRAMMAR",
"confidence": {
"ASR": "0.658",
"ASR+NLU": "0.7873426"
},
"request_id": "21003710-b4c9-4a28-9f6c-b1d3429836d7"
},
{
"index": 14,
"perfect": true,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-15.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0015_asraas.log",
"transaction_log_nlu": "./xaas_logs/0015_nluaas.log",
"word_error_rate": 0,
"slot_error_rate": 0.0,
"transcription": {
"expected": "From Phoenix",
"actual": "From phoenix",
"expected_annotated": [
{
"type": "C",
"word": "From"
},
{
"type": "C",
"word": "Phoenix"
}
],
"actual_annotated": [
{
"type": "C",
"word": "From"
},
{
"type": "C",
"word": "phoenix"
}
],
"score": [
[
{
"type": "C",
"index_expected": 1,
"index_actual": 1,
"word_expected": "Phoenix",
"word_actual": "phoenix"
}
]
],
"alignment": [
[
0,
0,
"C",
0,
"From",
"From"
],
[
1,
1,
"C",
0,
"Phoenix",
"phoenix"
]
]
},
"intent": {
"expected": "NO_INTENT",
"actual": "NO_INTENT",
"score": true
},
"slots": {
"passed": 1,
"failed": 0,
"slots": [
{
"slot": "eDepartureCity",
"expected": "Phoenix",
"actual": "Phoenix",
"score": true
}
]
},
"origin": "GRAMMAR",
"confidence": {
"ASR": "0.862",
"ASR+NLU": "0.5502123"
},
"request_id": "6f022285-9fd6-4be6-8cc4-77ee9d807740"
},
{
"index": 15,
"perfect": true,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-16.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0016_asraas.log",
"transaction_log_nlu": "./xaas_logs/0016_nluaas.log",
"word_error_rate": 0,
"slot_error_rate": 0.0,
"transcription": {
"expected": "Tomorrow",
"actual": "Tomorrow",
"expected_annotated": [
{
"type": "C",
"word": "Tomorrow"
}
],
"actual_annotated": [
{
"type": "C",
"word": "Tomorrow"
}
],
"score": [],
"alignment": [
[
0,
0,
"C",
0,
"Tomorrow",
"Tomorrow"
]
]
},
"intent": {
"expected": "NO_INTENT",
"actual": "NO_INTENT",
"score": true
},
"slots": {
"passed": 1,
"failed": 0,
"slots": [
{
"slot": "nuance_CALENDARX",
"expected": "{'nuance_CALENDAR.nuance_DATE.nuance_DATE_REL.nuance_STEP': 'day' 'nuance_CALENDAR.nuance_DATE.nuance_DATE_REL.nuance_INCREMENT': 1}",
"actual": {
"nuance_CALENDAR.nuance_DATE.nuance_DATE_REL.nuance_INCREMENT": 1,
"nuance_CALENDAR.nuance_DATE.nuance_DATE_REL.nuance_STEP": "day"
},
"score": true
}
]
},
"origin": "GRAMMAR",
"confidence": {
"ASR": "0.965",
"ASR+NLU": "0.19017832"
},
"request_id": "0e3e4f19-8f19-4d3e-a8ad-eca5af17455f"
},
{
"index": 16,
"perfect": false,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-17.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0017_asraas.log",
"transaction_log_nlu": "./xaas_logs/0017_nluaas.log",
"word_error_rate": 12,
"slot_error_rate": 0.0,
"transcription": {
"expected": "No actually I want to leave from Atlanta",
"actual": "No actually I want to be from Atlanta",
"expected_annotated": [
{
"type": "C",
"word": "No"
},
{
"type": "C",
"word": "actually"
},
{
"type": "C",
"word": "I"
},
{
"type": "C",
"word": "want"
},
{
"type": "C",
"word": "to"
},
{
"type": "S",
"word": "leave"
},
{
"type": "C",
"word": "from"
},
{
"type": "C",
"word": "Atlanta"
}
],
"actual_annotated": [
{
"type": "C",
"word": "No"
},
{
"type": "C",
"word": "actually"
},
{
"type": "C",
"word": "I"
},
{
"type": "C",
"word": "want"
},
{
"type": "C",
"word": "to"
},
{
"type": "S",
"word": "be"
},
{
"type": "C",
"word": "from"
},
{
"type": "C",
"word": "Atlanta"
}
],
"score": [
[
{
"type": "S",
"index_expected": 5,
"index_actual": 5,
"word_expected": "leave",
"word_actual": "be"
}
]
],
"alignment": [
[
0,
0,
"C",
0,
"No",
"No"
],
[
1,
1,
"C",
0,
"actually",
"actually"
],
[
2,
2,
"C",
0,
"I",
"I"
],
[
3,
3,
"C",
0,
"want",
"want"
],
[
4,
4,
"C",
0,
"to",
"to"
],
[
5,
5,
"S",
4,
"leave",
"be"
],
[
6,
6,
"C",
0,
"from",
"from"
],
[
7,
7,
"C",
0,
"Atlanta",
"Atlanta"
]
]
},
"intent": {
"expected": "NO_INTENT",
"actual": "NO_INTENT",
"score": true
},
"slots": {
"passed": 1,
"failed": 0,
"slots": [
{
"slot": "eDepartureCity",
"expected": "Atlanta",
"actual": "Atlanta",
"score": true
}
]
},
"origin": "STATISTICAL",
"confidence": {
"ASR": "0.619",
"ASR+NLU": "0.9812793"
},
"request_id": "680ee358-38fd-42df-b032-285439f73502"
},
{
"index": 17,
"perfect": true,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-18.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0018_asraas.log",
"transaction_log_nlu": "./xaas_logs/0018_nluaas.log",
"word_error_rate": 0,
"slot_error_rate": 0.0,
"transcription": {
"expected": "Change the destination from New York to Atlanta",
"actual": "Change the destination from New York to Atlanta",
"expected_annotated": [
{
"type": "C",
"word": "Change"
},
{
"type": "C",
"word": "the"
},
{
"type": "C",
"word": "destination"
},
{
"type": "C",
"word": "from"
},
{
"type": "C",
"word": "New"
},
{
"type": "C",
"word": "York"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "Atlanta"
}
],
"actual_annotated": [
{
"type": "C",
"word": "Change"
},
{
"type": "C",
"word": "the"
},
{
"type": "C",
"word": "destination"
},
{
"type": "C",
"word": "from"
},
{
"type": "C",
"word": "New"
},
{
"type": "C",
"word": "York"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "Atlanta"
}
],
"score": [],
"alignment": [
[
0,
0,
"C",
0,
"Change",
"Change"
],
[
1,
1,
"C",
0,
"the",
"the"
],
[
2,
2,
"C",
0,
"destination",
"destination"
],
[
3,
3,
"C",
0,
"from",
"from"
],
[
4,
4,
"C",
0,
"New",
"New"
],
[
5,
5,
"C",
0,
"York",
"York"
],
[
6,
6,
"C",
0,
"to",
"to"
],
[
7,
7,
"C",
0,
"Atlanta",
"Atlanta"
]
]
},
"intent": {
"expected": "NO_INTENT",
"actual": "NO_INTENT",
"score": true
},
"slots": {
"passed": 2,
"failed": 0,
"slots": [
{
"slot": "eDepartureCity",
"expected": "New York City",
"actual": "New York City",
"score": true
},
{
"slot": "eArrivalCity",
"expected": "Atlanta",
"actual": "Atlanta",
"score": true
}
]
},
"origin": "STATISTICAL",
"confidence": {
"ASR": "0.885",
"ASR+NLU": "0.51903933"
},
"request_id": "b5c4337e-100a-42ce-9260-eac90c4035a3"
},
{
"index": 18,
"perfect": false,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/1-19.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0019_asraas.log",
"transaction_log_nlu": "./xaas_logs/0019_nluaas.log",
"word_error_rate": 20,
"slot_error_rate": 0.0,
"transcription": {
"expected": "Make it on August 5th",
"actual": "Make it on August 5.",
"expected_annotated": [
{
"type": "C",
"word": "Make"
},
{
"type": "C",
"word": "it"
},
{
"type": "C",
"word": "on"
},
{
"type": "C",
"word": "August"
},
{
"type": "S",
"word": "5th"
}
],
"actual_annotated": [
{
"type": "C",
"word": "Make"
},
{
"type": "C",
"word": "it"
},
{
"type": "C",
"word": "on"
},
{
"type": "C",
"word": "August"
},
{
"type": "S",
"word": "5."
}
],
"score": [
[
{
"type": "S",
"index_expected": 4,
"index_actual": 4,
"word_expected": "5th",
"word_actual": "5."
}
]
],
"alignment": [
[
0,
0,
"C",
0,
"Make",
"Make"
],
[
1,
1,
"C",
0,
"it",
"it"
],
[
2,
2,
"C",
0,
"on",
"on"
],
[
3,
3,
"C",
0,
"August",
"August"
],
[
4,
4,
"S",
4,
"5th",
"5."
]
]
},
"intent": {
"expected": "NO_INTENT",
"actual": "NO_INTENT",
"score": true
},
"slots": {
"passed": 1,
"failed": 0,
"slots": [
{
"slot": "eDepartureDate",
"expected": "{'nuance_CALENDAR.nuance_DATE.nuance_DATE_ABS.nuance_MONTH': 8 'nuance_CALENDAR.nuance_DATE.nuance_DATE_ABS.nuance_DAY': 5}",
"actual": {
"nuance_CALENDAR.nuance_DATE.nuance_DATE_ABS.nuance_DAY": 5,
"nuance_CALENDAR.nuance_DATE.nuance_DATE_ABS.nuance_MONTH": 8
},
"score": true
}
]
},
"origin": "STATISTICAL",
"confidence": {
"ASR": "0.328",
"ASR+NLU": "0.35708505"
},
"request_id": "b86e0b92-c9c1-4845-b1ad-7b9313416bbe"
},
{
"index": 19,
"perfect": false,
"perfect_nlu": false,
"error": null,
"error_type": "",
"audio": "./wav/1-20.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0020_asraas.log",
"transaction_log_nlu": "./xaas_logs/0020_nluaas.log",
"word_error_rate": 10,
"slot_error_rate": 0.0,
"transcription": {
"expected": "Can I go from Montreal to Phoenix on May 1st?",
"actual": "Can I go from Montreal to Phoenix on May 1",
"expected_annotated": [
{
"type": "C",
"word": "Can"
},
{
"type": "C",
"word": "I"
},
{
"type": "C",
"word": "go"
},
{
"type": "C",
"word": "from"
},
{
"type": "C",
"word": "Montreal"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "Phoenix"
},
{
"type": "C",
"word": "on"
},
{
"type": "C",
"word": "May"
},
{
"type": "S",
"word": "1st?"
}
],
"actual_annotated": [
{
"type": "C",
"word": "Can"
},
{
"type": "C",
"word": "I"
},
{
"type": "C",
"word": "go"
},
{
"type": "C",
"word": "from"
},
{
"type": "C",
"word": "Montreal"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "Phoenix"
},
{
"type": "C",
"word": "on"
},
{
"type": "C",
"word": "May"
},
{
"type": "S",
"word": "1"
}
],
"score": [
[
{
"type": "S",
"index_expected": 9,
"index_actual": 9,
"word_expected": "1st?",
"word_actual": "1"
}
]
],
"alignment": [
[
0,
0,
"C",
0,
"Can",
"Can"
],
[
1,
1,
"C",
0,
"I",
"I"
],
[
2,
2,
"C",
0,
"go",
"go"
],
[
3,
3,
"C",
0,
"from",
"from"
],
[
4,
4,
"C",
0,
"Montreal",
"Montreal"
],
[
5,
5,
"C",
0,
"to",
"to"
],
[
6,
6,
"C",
0,
"Phoenix",
"Phoenix"
],
[
7,
7,
"C",
0,
"on",
"on"
],
[
8,
8,
"C",
0,
"May",
"May"
],
[
9,
9,
"S",
4,
"1st?",
"1"
]
]
},
"intent": {
"expected": "iBookFlight",
"actual": "NO_INTENT",
"score": false
},
"slots": {
"passed": 2,
"failed": 0,
"slots": [
{
"slot": "eArrivalCity",
"expected": "Phoenix",
"actual": "Phoenix",
"score": true
},
{
"slot": "eDepartureDate",
"expected": "{'nuance_CALENDAR.nuance_DATE.nuance_DATE_ABS.nuance_DAY': 1 'nuance_CALENDAR.nuance_DATE.nuance_DATE_ABS.nuance_MONTH': 5}",
"actual": {
"nuance_CALENDAR.nuance_DATE.nuance_DATE_ABS.nuance_MONTH": 5,
"nuance_CALENDAR.nuance_DATE.nuance_DATE_ABS.nuance_DAY": 1
},
"score": true
}
]
},
"origin": "STATISTICAL",
"confidence": {
"ASR": "0.483",
"ASR+NLU": "0.54881936"
},
"request_id": "42f6d8dc-a353-42da-966f-01b363da3423"
},
{
"index": 20,
"perfect": true,
"perfect_nlu": true,
"error": null,
"error_type": "",
"audio": "./wav/2-1.wav",
"audio_exists": false,
"transaction_log_asr": "./xaas_logs/0021_asraas.log",
"transaction_log_nlu": "./xaas_logs/0021_nluaas.log",
"word_error_rate": 0,
"slot_error_rate": 0.0,
"transcription": {
"expected": "Book a flight from New York to Atlanta for tomorrow",
"actual": "Book a flight from New York to Atlanta for tomorrow",
"expected_annotated": [
{
"type": "C",
"word": "Book"
},
{
"type": "C",
"word": "a"
},
{
"type": "C",
"word": "flight"
},
{
"type": "C",
"word": "from"
},
{
"type": "C",
"word": "New"
},
{
"type": "C",
"word": "York"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "Atlanta"
},
{
"type": "C",
"word": "for"
},
{
"type": "C",
"word": "tomorrow"
}
],
"actual_annotated": [
{
"type": "C",
"word": "Book"
},
{
"type": "C",
"word": "a"
},
{
"type": "C",
"word": "flight"
},
{
"type": "C",
"word": "from"
},
{
"type": "C",
"word": "New"
},
{
"type": "C",
"word": "York"
},
{
"type": "C",
"word": "to"
},
{
"type": "C",
"word": "Atlanta"
},
{
"type": "C",
"word": "for"
},
{
"type": "C",
"word": "tomorrow"
}
],
"score": [],
"alignment": [
[
0,
0,
"C",
0,
"Book",
"Book"
],
[
1,
1,
"C",
0,
"a",
"a"
],
[
2,
2,
"C",
0,
"flight",
"flight"
],
[
3,
3,
"C",
0,
"from",
"from"
],
[
4,
4,
"C",
0,
"New",
"New"
],
[
5,
5,
"C",
0,
"York",
"York"
],
[
6,
6,
"C",
0,
"to",
"to"
],
[
7,
7,
"C",
0,
"Atlanta",
"Atlanta"
],
[
8,
8,
"C",
0,
"for",
"for"
],
[
9,
9,
"C",
0,
"tomorrow",
"tomorrow"
]
]
},
"intent": {
"expected": "iBookFlight",
"actual": "iBookFlight",
"score": true
},
"slots": {
"passed": 3,
"failed": 0,
"slots": [
{
"slot": "eArrivalCity",
"expected": "Atlanta",
"actual": "Atlanta",
"score": true
},
{
"slot": "eDepartureCity",
"expected": "New York City",
"actual": "New York City",
"score": true
},
{
"slot": "eDepartureDate",
"expected": "{'nuance_CALENDAR.nuance_DATE.nuance_DATE_REL.nuance_STEP': 'day' 'nuance_CALENDAR.nuance_DATE.nuance_DATE_REL.nuance_INCREMENT': 1}",
"actual": {
"nuance_CALENDAR.nuance_DATE.nuance_DATE_REL.nuance_STEP": "day",
"nuance_CALENDAR.nuance_DATE.nuance_DATE_REL.nuance_INCREMENT": 1
},
"score": true
}
]
},
"origin": "STATISTICAL",
"confidence": {
"ASR": "0.724",
"ASR+NLU": "0.9985352"
},
"request_id": "a79c71f6-23ae-4de1-82e5-b69880dd3333"
}
]
}