{
  "scope": "Known mean q and phase; no estimated phase, normalization, physiology, natural-speech quality, equivalence, adoption or sequential extension; PML retained.",
  "truths": {
    "W": {
      "attempted_pairs": 100,
      "planned_pairs": 100,
      "generation_failures": 0,
      "complete_five_score_pairs": 46,
      "winner_counts_including_exact_ties": [
        {
          "models": [
            "A"
          ],
          "count": 12,
          "denominator": 46,
          "rate": 0.2608695652173913,
          "wilson95": [
            0.156008864456426,
            0.4025915352801559
          ]
        },
        {
          "models": [
            "J"
          ],
          "count": 8,
          "denominator": 46,
          "rate": 0.17391304347826086,
          "wilson95": [
            0.09085791702246311,
            0.30723353716378493
          ]
        },
        {
          "models": [
            "S"
          ],
          "count": 5,
          "denominator": 46,
          "rate": 0.10869565217391304,
          "wilson95": [
            0.04733145858098943,
            0.2303782864425083
          ]
        },
        {
          "models": [
            "V"
          ],
          "count": 13,
          "denominator": 46,
          "rate": 0.2826086956521739,
          "wilson95": [
            0.17324202381870632,
            0.425485612305459
          ]
        },
        {
          "models": [
            "W"
          ],
          "count": 1,
          "denominator": 46,
          "rate": 0.021739130434782608,
          "wilson95": [
            0.0038478542580502872,
            0.11335294521511355
          ]
        },
        {
          "models": [
            "W",
            "J"
          ],
          "count": 3,
          "denominator": 46,
          "rate": 0.06521739130434782,
          "wilson95": [
            0.022427861429288584,
            0.17502741081904216
          ]
        },
        {
          "models": [
            "W",
            "S"
          ],
          "count": 1,
          "denominator": 46,
          "rate": 0.021739130434782608,
          "wilson95": [
            0.0038478542580502872,
            0.11335294521511355
          ]
        },
        {
          "models": [
            "W",
            "S",
            "J"
          ],
          "count": 3,
          "denominator": 46,
          "rate": 0.06521739130434782,
          "wilson95": [
            0.022427861429288584,
            0.17502741081904216
          ]
        }
      ],
      "confusion_unique_winner": {
        "W": {
          "count": 1,
          "denominator": 46,
          "rate": 0.021739130434782608,
          "wilson95": [
            0.0038478542580502872,
            0.11335294521511355
          ]
        },
        "A": {
          "count": 12,
          "denominator": 46,
          "rate": 0.2608695652173913,
          "wilson95": [
            0.156008864456426,
            0.4025915352801559
          ]
        },
        "V": {
          "count": 13,
          "denominator": 46,
          "rate": 0.2826086956521739,
          "wilson95": [
            0.17324202381870632,
            0.425485612305459
          ]
        },
        "S": {
          "count": 5,
          "denominator": 46,
          "rate": 0.10869565217391304,
          "wilson95": [
            0.04733145858098943,
            0.2303782864425083
          ]
        },
        "J": {
          "count": 8,
          "denominator": 46,
          "rate": 0.17391304347826086,
          "wilson95": [
            0.09085791702246311,
            0.30723353716378493
          ]
        }
      },
      "ties": {
        "count": 7,
        "denominator": 46,
        "rate": 0.15217391304347827,
        "wilson95": [
          0.07572282227805464,
          0.28224139552060995
        ]
      },
      "model_status": {
        "W": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          }
        },
        "A": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          }
        },
        "V": {
          "fit_failure": {
            "count": 54,
            "denominator": 100,
            "rate": 0.54,
            "wilson95": [
              0.44264860323368227,
              0.6343919169097589
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 46,
            "rate": 0.0,
            "wilson95": [
              0.0,
              0.07707356308558039
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 46,
            "rate": 0.0,
            "wilson95": [
              0.0,
              0.07707356308558039
            ]
          }
        },
        "S": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 50,
          "boundary": {
            "count": 50,
            "denominator": 100,
            "rate": 0.5,
            "wilson95": [
              0.4038315303659956,
              0.5961684696340044
            ]
          }
        },
        "J": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 58,
          "boundary": {
            "count": 58,
            "denominator": 100,
            "rate": 0.58,
            "wilson95": [
              0.4820648670304296,
              0.6720161732564526
            ]
          }
        }
      },
      "paired_score_differences": {
        "A": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 100,
          "pair_ids": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16,
            17,
            18,
            19,
            20,
            21,
            22,
            23,
            24,
            25,
            26,
            27,
            28,
            29,
            30,
            31,
            32,
            33,
            34,
            35,
            36,
            37,
            38,
            39,
            40,
            41,
            42,
            43,
            44,
            45,
            46,
            47,
            48,
            49,
            50,
            51,
            52,
            53,
            54,
            55,
            56,
            57,
            58,
            59,
            60,
            61,
            62,
            63,
            64,
            65,
            66,
            67,
            68,
            69,
            70,
            71,
            72,
            73,
            74,
            75,
            76,
            77,
            78,
            79,
            80,
            81,
            82,
            83,
            84,
            85,
            86,
            87,
            88,
            89,
            90,
            91,
            92,
            93,
            94,
            95,
            96,
            97,
            98,
            99
          ],
          "differences": [
            -0.26914940491587913,
            0.007947059240905219,
            0.2774078842194285,
            -0.1271373146246333,
            1.9496194177245343,
            -0.34319594245062035,
            1.2659484197429265,
            0.021958442071991158,
            3.866214670082627,
            0.46090225010266295,
            0.2249391011246189,
            1.5231469953887427,
            1.6067219611650216,
            -0.6675000473169348,
            0.651237157639116,
            2.0114591626206675,
            0.5476046213043446,
            0.29269415406997723,
            0.0900168286934786,
            0.15706270969167235,
            0.9729568935781572,
            -0.7735592492153955,
            -0.6770256364070519,
            0.6188225023270206,
            2.4264878704452713,
            -0.8569557408809487,
            0.13932437490257144,
            0.1050492205122282,
            -0.11034063032457198,
            -0.04538035619043512,
            0.7097369757048,
            -0.003288605492343777,
            0.07761730056699889,
            0.047713692985780654,
            -0.021268502965540392,
            -0.6150604950762499,
            -0.31466388340413687,
            0.02640674516442232,
            0.10627929138718173,
            -0.07353500760291354,
            0.005217377591179684,
            -1.1158394106314518,
            0.21007142660710088,
            0.03585202412978106,
            -0.013320938454853604,
            3.570919520705502,
            0.0764620076133724,
            -1.1505268958644592,
            -0.27970317521067045,
            -0.1278158072309452,
            1.601962493490646,
            0.5259921948218107,
            -0.03866483285310096,
            -0.4423852122399694,
            0.15820438578157336,
            -0.10874167031943216,
            2.411840611643129,
            0.2823712539138796,
            -0.024052444774497417,
            -0.2393643624454853,
            0.1846150525525445,
            -0.6237953450017812,
            0.8909673175749049,
            1.244714984550228,
            -1.018009384855759,
            0.8395076955839613,
            1.6172581018199708,
            -0.3545700326394581,
            -1.445841488590304,
            0.038067687699367525,
            3.2256610111526243,
            0.013546663789384183,
            -0.5475383163047809,
            0.04604139221191872,
            0.2074049946040759,
            0.8966689842400228,
            0.01238360398565419,
            0.9884217595463269,
            -0.1957321769277769,
            0.23506157811789308,
            -0.1942627843200171,
            0.26574705240273033,
            1.9870478645316325,
            -0.2742147391909384,
            -0.18015493363964197,
            -0.15464084841551085,
            -0.35655122230309644,
            0.5605459760026861,
            0.08530656119182822,
            2.187440481107842,
            1.5217366396900616,
            -0.03788375136355171,
            0.8772150383738335,
            5.066654716578341,
            -0.0019912121788365766,
            -0.8415671204456885,
            1.828531463706895,
            0.10007633454915776,
            0.06059164732869249,
            0.7258798298062175
          ],
          "mean": 0.40106034508384253,
          "bootstrap_pointwise95": [
            0.20091846323704843,
            0.6221332771618535
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        },
        "V": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 46,
          "pair_ids": [
            4,
            7,
            10,
            13,
            15,
            16,
            19,
            21,
            23,
            24,
            26,
            28,
            31,
            32,
            33,
            34,
            37,
            40,
            44,
            45,
            47,
            48,
            50,
            53,
            54,
            57,
            58,
            59,
            60,
            65,
            66,
            67,
            71,
            72,
            79,
            83,
            84,
            86,
            89,
            90,
            91,
            92,
            94,
            97,
            98,
            99
          ],
          "differences": [
            -0.5693633704486274,
            0.5136767691637942,
            -2.2429005954218155,
            -0.16460930690845998,
            0.804821108451506,
            0.46742321019701194,
            0.20245235927905014,
            1.3002445035435812,
            1.5639867859317746,
            -0.4302709746025357,
            -0.3290792730804242,
            0.14396260451030685,
            0.03287916759836662,
            -0.218744859808794,
            1.3494901450521866,
            0.49586388785246527,
            0.7798606461547024,
            -0.2101994528020441,
            -0.783942553534871,
            0.1704713224007719,
            0.9603220884218899,
            5.877071430328215,
            -0.9526851356094994,
            0.1177286566926341,
            -0.5544528489645018,
            0.26143232517461,
            1.4489577860767895,
            2.0987189357238094,
            2.323075920015981,
            0.9399074123066384,
            1.203491099375242,
            -0.4677582515996619,
            0.04862195230271027,
            0.7283948937856621,
            0.2876274226127862,
            0.0051873250340577215,
            -0.5359826626536233,
            -0.13712731981104298,
            6.066658471367191,
            -0.7434382200517575,
            0.33507671973893594,
            2.1778358358751575,
            0.5609694202175888,
            -0.44157051738511655,
            -0.7308211788331391,
            -0.6807350875151315
          ],
          "mean": 0.5015767086120515,
          "bootstrap_pointwise95": [
            0.10352105080820939,
            0.9535867272198716
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        },
        "S": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 100,
          "pair_ids": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16,
            17,
            18,
            19,
            20,
            21,
            22,
            23,
            24,
            25,
            26,
            27,
            28,
            29,
            30,
            31,
            32,
            33,
            34,
            35,
            36,
            37,
            38,
            39,
            40,
            41,
            42,
            43,
            44,
            45,
            46,
            47,
            48,
            49,
            50,
            51,
            52,
            53,
            54,
            55,
            56,
            57,
            58,
            59,
            60,
            61,
            62,
            63,
            64,
            65,
            66,
            67,
            68,
            69,
            70,
            71,
            72,
            73,
            74,
            75,
            76,
            77,
            78,
            79,
            80,
            81,
            82,
            83,
            84,
            85,
            86,
            87,
            88,
            89,
            90,
            91,
            92,
            93,
            94,
            95,
            96,
            97,
            98,
            99
          ],
          "differences": [
            0.9423924036291282,
            0.0,
            0.3972703343133617,
            -0.804094304508908,
            0.0,
            0.5106862168813677,
            0.0,
            0.3860254017072293,
            0.0,
            0.0,
            0.0,
            0.0,
            0.7225945588852483,
            0.0,
            -0.027787018567323685,
            0.0,
            0.011776818644648301,
            3.8603513739053597,
            0.0,
            0.0,
            0.0,
            0.9805171792140754,
            0.0,
            0.0,
            0.0,
            0.0,
            -0.5123528980348055,
            0.0,
            0.14767954592389287,
            0.3559468854637089,
            0.890876981711699,
            0.4436700458500127,
            0.0,
            0.5707425905638956,
            -0.15042707990141935,
            -0.5730814192720572,
            1.8101633509941166,
            0.0,
            0.0,
            -0.09448150700700353,
            0.0,
            4.286784861178603,
            0.0,
            0.5714215613043052,
            0.8834731337155972,
            0.41990943882592546,
            0.0,
            0.18855227545282105,
            0.0,
            -0.048352543912187684,
            4.641071982850917,
            0.0,
            0.07373749697217136,
            0.11459608432960522,
            0.0,
            0.0,
            0.0,
            -0.20771456568218127,
            0.0,
            0.00015201691712718457,
            0.0,
            -0.12623391355009517,
            0.0,
            0.0,
            0.0,
            0.37791797492536716,
            -0.3823430078355159,
            0.9742293186736788,
            0.0,
            0.11740200050371641,
            0.5083855129560106,
            0.0,
            0.0,
            2.9149030232802033,
            0.0,
            0.05063610530123697,
            0.0,
            0.0,
            0.0,
            -0.26392036595279933,
            0.0,
            0.0,
            0.0,
            0.0,
            0.0,
            0.0,
            -0.10862144832026388,
            0.0,
            0.0,
            0.38651667936755985,
            2.3198884935882234,
            0.0,
            0.6298088720595842,
            -0.1344967596269271,
            0.4458442516333889,
            0.0,
            0.5943644041544758,
            0.38010718158693635,
            0.506856126277853,
            -0.04483234818872006
          ],
          "mean": 0.29938513303182845,
          "bootstrap_pointwise95": [
            0.1432495662547176,
            0.4798490906571437
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        },
        "J": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 100,
          "pair_ids": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16,
            17,
            18,
            19,
            20,
            21,
            22,
            23,
            24,
            25,
            26,
            27,
            28,
            29,
            30,
            31,
            32,
            33,
            34,
            35,
            36,
            37,
            38,
            39,
            40,
            41,
            42,
            43,
            44,
            45,
            46,
            47,
            48,
            49,
            50,
            51,
            52,
            53,
            54,
            55,
            56,
            57,
            58,
            59,
            60,
            61,
            62,
            63,
            64,
            65,
            66,
            67,
            68,
            69,
            70,
            71,
            72,
            73,
            74,
            75,
            76,
            77,
            78,
            79,
            80,
            81,
            82,
            83,
            84,
            85,
            86,
            87,
            88,
            89,
            90,
            91,
            92,
            93,
            94,
            95,
            96,
            97,
            98,
            99
          ],
          "differences": [
            0.0,
            0.09954061802091019,
            0.0,
            0.0,
            0.0,
            0.0,
            0.0,
            0.0,
            0.10741343716836127,
            0.6906935690149112,
            0.0,
            -0.3151899848362518,
            0.0,
            0.0,
            0.0,
            0.01616034143808065,
            -0.1731124568323139,
            0.0,
            0.0,
            -0.09373391838926182,
            0.34548150626687857,
            0.0,
            1.6229320395923423,
            -0.08023448227686458,
            0.0,
            0.0,
            0.0,
            0.0,
            0.26331039580145443,
            0.0,
            -0.4619699799204682,
            -0.05517152979700768,
            0.39600392987813393,
            -0.09341236911859596,
            0.0,
            0.0,
            1.333776661525917,
            0.0,
            -0.4419324367008812,
            0.0,
            0.0,
            0.0,
            0.0,
            -0.7044633017922024,
            0.0,
            0.0,
            -0.03367647036611743,
            0.49125714654837793,
            0.7281484604791331,
            0.6419982766146859,
            0.0,
            0.0,
            0.0,
            0.0,
            0.0,
            0.0,
            1.2598569280853553,
            0.0,
            0.6540102678009134,
            0.0,
            0.0,
            0.0,
            0.015479383333513397,
            0.0,
            1.3460586587407306,
            2.0045917469124106,
            0.0,
            -0.8741175732429838,
            0.0,
            0.6271616381945933,
            -0.16158589252336242,
            0.0,
            0.0,
            0.7824708441457915,
            0.0,
            0.0,
            0.11670422352835885,
            0.02965762757048651,
            0.0,
            2.0505930344534136,
            0.0,
            0.0,
            0.1998486969350779,
            -0.27876177888356324,
            0.0,
            0.0,
            0.30989160134413396,
            0.3180383820563293,
            0.0,
            -0.03084868984478817,
            0.12736542154016206,
            1.7386581276678044,
            0.0,
            0.0,
            0.0,
            0.0,
            0.0,
            0.0,
            2.000276109260085,
            0.0
          ],
          "mean": 0.16519168209393684,
          "bootstrap_pointwise95": [
            0.0707938933792875,
            0.27210379760792963
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        }
      }
    },
    "A": {
      "attempted_pairs": 100,
      "planned_pairs": 100,
      "generation_failures": 0,
      "complete_five_score_pairs": 38,
      "winner_counts_including_exact_ties": [
        {
          "models": [
            "A"
          ],
          "count": 38,
          "denominator": 38,
          "rate": 1.0,
          "wilson95": [
            0.908190131774416,
            0.9999999999999999
          ]
        }
      ],
      "confusion_unique_winner": {
        "W": {
          "count": 0,
          "denominator": 38,
          "rate": 0.0,
          "wilson95": [
            0.0,
            0.09180986822558396
          ]
        },
        "A": {
          "count": 38,
          "denominator": 38,
          "rate": 1.0,
          "wilson95": [
            0.908190131774416,
            0.9999999999999999
          ]
        },
        "V": {
          "count": 0,
          "denominator": 38,
          "rate": 0.0,
          "wilson95": [
            0.0,
            0.09180986822558396
          ]
        },
        "S": {
          "count": 0,
          "denominator": 38,
          "rate": 0.0,
          "wilson95": [
            0.0,
            0.09180986822558396
          ]
        },
        "J": {
          "count": 0,
          "denominator": 38,
          "rate": 0.0,
          "wilson95": [
            0.0,
            0.09180986822558396
          ]
        }
      },
      "ties": {
        "count": 0,
        "denominator": 38,
        "rate": 0.0,
        "wilson95": [
          0.0,
          0.09180986822558396
        ]
      },
      "model_status": {
        "W": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          }
        },
        "A": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          }
        },
        "V": {
          "fit_failure": {
            "count": 62,
            "denominator": 100,
            "rate": 0.62,
            "wilson95": [
              0.5220975529551107,
              0.7090240074752127
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 38,
            "rate": 0.0,
            "wilson95": [
              0.0,
              0.09180986822558396
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 38,
            "rate": 0.0,
            "wilson95": [
              0.0,
              0.09180986822558396
            ]
          }
        },
        "S": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          }
        },
        "J": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          }
        }
      },
      "paired_score_differences": {
        "W": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 100,
          "pair_ids": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16,
            17,
            18,
            19,
            20,
            21,
            22,
            23,
            24,
            25,
            26,
            27,
            28,
            29,
            30,
            31,
            32,
            33,
            34,
            35,
            36,
            37,
            38,
            39,
            40,
            41,
            42,
            43,
            44,
            45,
            46,
            47,
            48,
            49,
            50,
            51,
            52,
            53,
            54,
            55,
            56,
            57,
            58,
            59,
            60,
            61,
            62,
            63,
            64,
            65,
            66,
            67,
            68,
            69,
            70,
            71,
            72,
            73,
            74,
            75,
            76,
            77,
            78,
            79,
            80,
            81,
            82,
            83,
            84,
            85,
            86,
            87,
            88,
            89,
            90,
            91,
            92,
            93,
            94,
            95,
            96,
            97,
            98,
            99
          ],
          "differences": [
            997.4045383459961,
            921.6746006472349,
            974.8601392814398,
            850.068313785905,
            982.2640962271653,
            860.4502378934594,
            931.3667886209169,
            954.988398140993,
            872.7041326626877,
            989.2795484462204,
            876.5003171067347,
            963.8674892208219,
            874.0538067619054,
            926.5049167357665,
            923.7225429155096,
            881.3836956508912,
            941.622263994177,
            943.4906395309245,
            1045.1397446166293,
            979.808738672662,
            946.2058923668937,
            961.8391929639256,
            889.0432633315322,
            880.3391858687082,
            901.781777158425,
            999.9076191475488,
            877.9778999052141,
            907.6209958322106,
            886.1448832115093,
            959.8622689290187,
            952.9848893330709,
            967.0918598879816,
            866.934182875124,
            904.9526891815985,
            955.7892136956725,
            905.1052734486329,
            920.6712608120906,
            902.2895439953154,
            891.8626434090274,
            895.3179756657828,
            998.2763774487048,
            974.0493077245246,
            936.0670836681729,
            1014.6282433926135,
            989.8876837930202,
            892.715284871807,
            950.9891578136867,
            882.0131896347375,
            945.3614891207071,
            965.085266921431,
            963.4006392088504,
            913.3021222614352,
            992.3861797563604,
            959.3409652243736,
            925.1956621200443,
            865.0650656953421,
            896.1862090980921,
            919.4580736593871,
            973.1474168577188,
            1000.4096800639509,
            986.954720827529,
            882.7133862259343,
            935.2959792947113,
            881.1415486249061,
            946.0787542025882,
            885.0629548845573,
            907.2447863647321,
            905.538064807215,
            955.0070169226783,
            911.6619649399472,
            953.8236019251344,
            989.5585997850412,
            1024.8368271858162,
            894.4474870319536,
            899.3356697858217,
            813.7193955259409,
            920.8717141596517,
            863.491413979882,
            891.4227256029753,
            1002.2670182250495,
            979.2901909861666,
            915.7917262695992,
            869.9092987462955,
            900.9951181582892,
            838.7215680914669,
            889.6727265327318,
            928.2756040246641,
            901.1643527178858,
            897.1122798186061,
            972.978581148227,
            1007.2023838035129,
            910.3031957783569,
            935.7477390038748,
            864.1276903576654,
            986.8722305931879,
            950.519779897184,
            971.3146978463792,
            841.3190621250942,
            933.4502604689624,
            909.4320732530286
          ],
          "mean": 928.7651674853554,
          "bootstrap_pointwise95": [
            919.685620001229,
            937.9459582298869
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        },
        "V": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 38,
          "pair_ids": [
            1,
            2,
            8,
            9,
            16,
            18,
            19,
            21,
            22,
            23,
            24,
            26,
            33,
            34,
            36,
            38,
            40,
            42,
            44,
            45,
            47,
            54,
            55,
            58,
            59,
            63,
            65,
            66,
            70,
            72,
            82,
            83,
            84,
            88,
            91,
            93,
            94,
            97
          ],
          "differences": [
            921.3004448482843,
            976.0215887434933,
            873.4282096814168,
            994.392514184101,
            949.2418697803187,
            1044.9656040038699,
            979.383811969492,
            961.8760721242979,
            889.2986770266052,
            880.6254735517614,
            901.718656639081,
            877.6633200727847,
            906.6473771439396,
            956.1935463319824,
            920.6409068743997,
            891.9777251314081,
            1007.778773195465,
            937.264166170924,
            991.0660925208795,
            892.0775466516388,
            882.6630761836768,
            925.3308789026378,
            868.5594881314446,
            972.9262273135519,
            999.1682433348524,
            881.4171149423491,
            889.2159453029708,
            905.082627370819,
            954.4881952401702,
            1028.98316801912,
            870.5214650903545,
            898.981071999513,
            839.2829454862003,
            896.3979219783232,
            912.1083881682571,
            864.1842055846864,
            987.2771221377516,
            842.4384133191525
          ],
          "mean": 925.5944440829466,
          "bootstrap_pointwise95": [
            909.3743776307815,
            942.3861951005557
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        },
        "S": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 100,
          "pair_ids": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16,
            17,
            18,
            19,
            20,
            21,
            22,
            23,
            24,
            25,
            26,
            27,
            28,
            29,
            30,
            31,
            32,
            33,
            34,
            35,
            36,
            37,
            38,
            39,
            40,
            41,
            42,
            43,
            44,
            45,
            46,
            47,
            48,
            49,
            50,
            51,
            52,
            53,
            54,
            55,
            56,
            57,
            58,
            59,
            60,
            61,
            62,
            63,
            64,
            65,
            66,
            67,
            68,
            69,
            70,
            71,
            72,
            73,
            74,
            75,
            76,
            77,
            78,
            79,
            80,
            81,
            82,
            83,
            84,
            85,
            86,
            87,
            88,
            89,
            90,
            91,
            92,
            93,
            94,
            95,
            96,
            97,
            98,
            99
          ],
          "differences": [
            972.3706194327297,
            888.6009996765133,
            922.7526487523028,
            803.916620270933,
            918.3307813388619,
            821.9456024882111,
            847.7287334983484,
            905.057194427749,
            821.2570224704941,
            916.1758354544108,
            834.0609920797979,
            927.5820057397395,
            820.0347902679587,
            872.804259843846,
            876.6228810042812,
            822.1350759783581,
            898.7777097435319,
            879.4198434756709,
            996.245866410869,
            925.5624860784701,
            906.2372176544432,
            937.6681083170697,
            829.4056977603177,
            815.9259649953947,
            838.3559517634949,
            927.635482706457,
            856.8180916661167,
            857.183483574936,
            857.2495056105945,
            902.387939849832,
            903.1744647013147,
            916.1274865583746,
            808.3413531207116,
            863.7648971048475,
            893.446631018398,
            810.4994918236371,
            885.4478692649172,
            847.5951872638434,
            854.65478862499,
            828.125034187824,
            928.747367414946,
            911.9267530070065,
            850.7951240755083,
            958.4706130201557,
            943.1691719484261,
            862.8502797746605,
            889.7238028085085,
            827.8845425887575,
            878.4281997748785,
            896.5475003989231,
            922.299369323744,
            863.2457863754171,
            926.5403724303378,
            899.2006477803898,
            863.4443850538082,
            825.7986715322731,
            874.3339098417637,
            861.7607714848527,
            942.3806935464454,
            955.6986648231577,
            928.6578020859124,
            849.4678233504474,
            884.1321773322834,
            840.7429738537139,
            870.183635279087,
            862.4642065750832,
            845.4611796140762,
            844.7658742402582,
            912.2671248599272,
            862.0613355741261,
            909.9884331708363,
            920.8956636657167,
            981.9407961182806,
            859.8588406321396,
            865.2827603633159,
            743.4129356512094,
            870.8035830456538,
            774.7399212828113,
            846.2583526464023,
            945.9798378258365,
            935.6349380006468,
            885.4986397805878,
            838.9226933124064,
            842.0995262389097,
            800.5559781698958,
            874.8033511981994,
            854.874888145001,
            829.6455841507777,
            839.3898877668089,
            937.0428085111562,
            959.3914781201238,
            859.4281823935125,
            883.1000344042986,
            839.9060068388571,
            896.9406707769595,
            871.5782496431839,
            911.7996302538704,
            776.0367983617925,
            888.4312151470149,
            865.7632016765492
          ],
          "mean": 877.048822630622,
          "bootstrap_pointwise95": [
            867.4780199727064,
            886.4951343363061
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        },
        "J": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 100,
          "pair_ids": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16,
            17,
            18,
            19,
            20,
            21,
            22,
            23,
            24,
            25,
            26,
            27,
            28,
            29,
            30,
            31,
            32,
            33,
            34,
            35,
            36,
            37,
            38,
            39,
            40,
            41,
            42,
            43,
            44,
            45,
            46,
            47,
            48,
            49,
            50,
            51,
            52,
            53,
            54,
            55,
            56,
            57,
            58,
            59,
            60,
            61,
            62,
            63,
            64,
            65,
            66,
            67,
            68,
            69,
            70,
            71,
            72,
            73,
            74,
            75,
            76,
            77,
            78,
            79,
            80,
            81,
            82,
            83,
            84,
            85,
            86,
            87,
            88,
            89,
            90,
            91,
            92,
            93,
            94,
            95,
            96,
            97,
            98,
            99
          ],
          "differences": [
            938.9589955024676,
            882.0873997847029,
            909.7832535680009,
            775.7847505046466,
            934.3105814676928,
            805.2236751385299,
            901.2083708460159,
            898.0905221405756,
            802.0148210200477,
            930.1851284136992,
            821.9409379318076,
            899.3586504447958,
            837.3589457220878,
            862.559792922204,
            868.6863653576438,
            846.738470325543,
            921.7740052299996,
            902.4472880362046,
            1001.9911816440817,
            915.9248118542891,
            868.6320292928212,
            910.8636324749277,
            863.2505414948191,
            839.8223832206731,
            861.5101218254404,
            961.1369411034284,
            830.9102782446917,
            860.6863176073857,
            835.0049314392218,
            916.9189625800664,
            927.2623546700397,
            937.1516755769117,
            828.9967152462305,
            834.8961406801154,
            908.4953101806004,
            867.4888994924986,
            869.6875723975263,
            857.7619799962231,
            840.8934964036016,
            876.5860658631791,
            949.3464279005584,
            911.2300911843668,
            886.5907476870325,
            949.3802689079421,
            936.4615463858245,
            823.0456750570975,
            895.7204604428553,
            853.2708422720061,
            914.9198017463896,
            863.130513381675,
            907.1532077421671,
            848.253236564311,
            958.1669092522425,
            913.0141299463103,
            879.8178914065629,
            814.8796050140918,
            870.6687916897772,
            853.0390633959087,
            897.2202911751137,
            964.2373909328107,
            935.759942889079,
            850.9143328034679,
            879.7026121443541,
            832.0502821576774,
            896.6253490017662,
            856.2609977400243,
            867.3049973903089,
            846.3497152588388,
            884.5480170440951,
            857.2041495296835,
            874.2483655916076,
            900.7736805529003,
            960.7999847818537,
            852.6889505011277,
            820.7400656997524,
            789.6807518898167,
            896.1306522400737,
            837.6598820455456,
            851.1563950560012,
            912.059344280231,
            910.7488023171245,
            892.5144113587176,
            801.574919383731,
            870.9773938010985,
            796.1191959629887,
            854.9945529385022,
            897.9823702534068,
            839.0849003063686,
            880.7037849712433,
            891.6115929448097,
            940.0533451798838,
            856.287059347369,
            866.3081103807926,
            806.2330309854042,
            939.838366404625,
            938.6768862242898,
            922.0066245300677,
            797.2445023751097,
            878.6343679774127,
            871.5811584253988
          ],
          "mean": 878.9976503439703,
          "bootstrap_pointwise95": [
            870.062839955838,
            887.6556041902109
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        }
      }
    },
    "V": {
      "attempted_pairs": 100,
      "planned_pairs": 100,
      "generation_failures": 0,
      "complete_five_score_pairs": 7,
      "winner_counts_including_exact_ties": [
        {
          "models": [
            "V"
          ],
          "count": 7,
          "denominator": 7,
          "rate": 1.0,
          "wilson95": [
            0.6456695649333126,
            1.0
          ]
        }
      ],
      "confusion_unique_winner": {
        "W": {
          "count": 0,
          "denominator": 7,
          "rate": 0.0,
          "wilson95": [
            5.551115123125783e-17,
            0.35433043506668743
          ]
        },
        "A": {
          "count": 0,
          "denominator": 7,
          "rate": 0.0,
          "wilson95": [
            5.551115123125783e-17,
            0.35433043506668743
          ]
        },
        "V": {
          "count": 7,
          "denominator": 7,
          "rate": 1.0,
          "wilson95": [
            0.6456695649333126,
            1.0
          ]
        },
        "S": {
          "count": 0,
          "denominator": 7,
          "rate": 0.0,
          "wilson95": [
            5.551115123125783e-17,
            0.35433043506668743
          ]
        },
        "J": {
          "count": 0,
          "denominator": 7,
          "rate": 0.0,
          "wilson95": [
            5.551115123125783e-17,
            0.35433043506668743
          ]
        }
      },
      "ties": {
        "count": 0,
        "denominator": 7,
        "rate": 0.0,
        "wilson95": [
          5.551115123125783e-17,
          0.35433043506668743
        ]
      },
      "model_status": {
        "W": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          }
        },
        "A": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          }
        },
        "V": {
          "fit_failure": {
            "count": 93,
            "denominator": 100,
            "rate": 0.93,
            "wilson95": [
              0.8625048526092651,
              0.9656807389327273
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 7,
            "rate": 0.0,
            "wilson95": [
              5.551115123125783e-17,
              0.35433043506668743
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 7,
            "rate": 0.0,
            "wilson95": [
              5.551115123125783e-17,
              0.35433043506668743
            ]
          }
        },
        "S": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 100,
          "boundary": {
            "count": 100,
            "denominator": 100,
            "rate": 1.0,
            "wilson95": [
              0.9630065017930143,
              1.0
            ]
          }
        },
        "J": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 93,
          "boundary": {
            "count": 93,
            "denominator": 100,
            "rate": 0.93,
            "wilson95": [
              0.8625048526092651,
              0.9656807389327273
            ]
          }
        }
      },
      "paired_score_differences": {
        "W": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 7,
          "pair_ids": [
            12,
            27,
            29,
            41,
            49,
            79,
            96
          ],
          "differences": [
            356.0892403274538,
            334.7896314972677,
            314.45656037177105,
            294.10362116082615,
            311.27873528707823,
            302.8687712556093,
            347.0337457200112
          ],
          "mean": 322.94575794571676,
          "bootstrap_pointwise95": [
            307.3302507128205,
            339.4037705512128
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        },
        "A": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 7,
          "pair_ids": [
            12,
            27,
            29,
            41,
            49,
            79,
            96
          ],
          "differences": [
            356.62503675168955,
            334.08560305488936,
            315.24487925606445,
            294.2221821809708,
            311.274487470886,
            303.0693830985365,
            347.70120448563466
          ],
          "mean": 323.1746823283816,
          "bootstrap_pointwise95": [
            307.49677542859143,
            339.2117963749383
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        },
        "S": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 7,
          "pair_ids": [
            12,
            27,
            29,
            41,
            49,
            79,
            96
          ],
          "differences": [
            356.0892403274538,
            334.7896314972677,
            314.45656037177105,
            294.10362116082615,
            311.27873528707823,
            302.8687712556093,
            347.0337457200112
          ],
          "mean": 322.94575794571676,
          "bootstrap_pointwise95": [
            307.78138742699826,
            339.0168532239612
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        },
        "J": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 7,
          "pair_ids": [
            12,
            27,
            29,
            41,
            49,
            79,
            96
          ],
          "differences": [
            356.0892403274538,
            334.7896314972677,
            314.45656037177105,
            294.10362116082615,
            311.27873528707823,
            302.8687712556093,
            347.0337457200112
          ],
          "mean": 322.94575794571676,
          "bootstrap_pointwise95": [
            307.3274124148993,
            338.5656455523354
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        }
      }
    },
    "S": {
      "attempted_pairs": 100,
      "planned_pairs": 100,
      "generation_failures": 0,
      "complete_five_score_pairs": 10,
      "winner_counts_including_exact_ties": [
        {
          "models": [
            "S"
          ],
          "count": 10,
          "denominator": 10,
          "rate": 1.0,
          "wilson95": [
            0.7224672001371107,
            0.9999999999999999
          ]
        }
      ],
      "confusion_unique_winner": {
        "W": {
          "count": 0,
          "denominator": 10,
          "rate": 0.0,
          "wilson95": [
            0.0,
            0.2775327998628892
          ]
        },
        "A": {
          "count": 0,
          "denominator": 10,
          "rate": 0.0,
          "wilson95": [
            0.0,
            0.2775327998628892
          ]
        },
        "V": {
          "count": 0,
          "denominator": 10,
          "rate": 0.0,
          "wilson95": [
            0.0,
            0.2775327998628892
          ]
        },
        "S": {
          "count": 10,
          "denominator": 10,
          "rate": 1.0,
          "wilson95": [
            0.7224672001371107,
            0.9999999999999999
          ]
        },
        "J": {
          "count": 0,
          "denominator": 10,
          "rate": 0.0,
          "wilson95": [
            0.0,
            0.2775327998628892
          ]
        }
      },
      "ties": {
        "count": 0,
        "denominator": 10,
        "rate": 0.0,
        "wilson95": [
          0.0,
          0.2775327998628892
        ]
      },
      "model_status": {
        "W": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          }
        },
        "A": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          }
        },
        "V": {
          "fit_failure": {
            "count": 90,
            "denominator": 100,
            "rate": 0.9,
            "wilson95": [
              0.8256343384950865,
              0.9447708629393249
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 10,
            "rate": 0.0,
            "wilson95": [
              0.0,
              0.2775327998628892
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 10,
            "rate": 0.0,
            "wilson95": [
              0.0,
              0.2775327998628892
            ]
          }
        },
        "S": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          }
        },
        "J": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 100,
          "boundary": {
            "count": 100,
            "denominator": 100,
            "rate": 1.0,
            "wilson95": [
              0.9630065017930143,
              1.0
            ]
          }
        }
      },
      "paired_score_differences": {
        "W": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 100,
          "pair_ids": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16,
            17,
            18,
            19,
            20,
            21,
            22,
            23,
            24,
            25,
            26,
            27,
            28,
            29,
            30,
            31,
            32,
            33,
            34,
            35,
            36,
            37,
            38,
            39,
            40,
            41,
            42,
            43,
            44,
            45,
            46,
            47,
            48,
            49,
            50,
            51,
            52,
            53,
            54,
            55,
            56,
            57,
            58,
            59,
            60,
            61,
            62,
            63,
            64,
            65,
            66,
            67,
            68,
            69,
            70,
            71,
            72,
            73,
            74,
            75,
            76,
            77,
            78,
            79,
            80,
            81,
            82,
            83,
            84,
            85,
            86,
            87,
            88,
            89,
            90,
            91,
            92,
            93,
            94,
            95,
            96,
            97,
            98,
            99
          ],
          "differences": [
            1295.0440258355266,
            1304.962550959186,
            1556.2425116602353,
            1192.484195892037,
            1214.1069476791035,
            1077.1697721583969,
            1660.9246653574246,
            1196.2204950208034,
            1509.5835063749473,
            1301.760333377386,
            1415.1039914161975,
            1371.8933618938954,
            1028.6740757345942,
            1078.7953037566767,
            1646.8178466583558,
            1344.9871089183343,
            1493.0206834171795,
            1691.9240129215486,
            1478.2776225537564,
            1271.32233457471,
            1180.4808828728037,
            1062.0299801078036,
            1156.4010620881836,
            1299.0977758888457,
            1577.677892122545,
            1390.2363983305895,
            1209.8252851129237,
            1184.0483177183323,
            1121.177252635065,
            1452.7224822503886,
            1274.0960992016644,
            1396.212245559409,
            894.2139214894069,
            1254.0582063158854,
            1377.317859309278,
            1248.6694225803967,
            1519.1120275244557,
            1144.7298057672015,
            1719.11567690417,
            1386.3098612368758,
            1566.2386751553977,
            1158.9703201854263,
            1252.365478517262,
            1220.681807391893,
            1166.0111682435,
            1521.6244695775022,
            1433.6576535122622,
            1138.5326291238525,
            1153.3356608047307,
            1397.8810001300135,
            1251.5833433570842,
            1094.5951102122417,
            1560.9729308788665,
            1147.9199987581687,
            1340.7142881882992,
            1315.6398077742924,
            1774.8814336852101,
            1388.6778894981417,
            1240.1628742703233,
            1383.0154371585159,
            1250.4306886701634,
            1725.7070931731068,
            1170.7458571215157,
            1545.0797701936908,
            1260.9512801793971,
            1105.7548134818917,
            1202.0574932797317,
            1096.495406969827,
            1028.0497367386106,
            961.243432358182,
            1364.7603741286948,
            1274.292813573773,
            1328.2501682616712,
            1448.8393787516598,
            1451.5709819028707,
            1268.856416851575,
            1593.7881687606477,
            1323.3658938818662,
            1232.3375232398012,
            1181.2324739136056,
            1435.0866839289156,
            1303.8741251295887,
            1226.6109985812927,
            1147.4707650804849,
            1180.1560301475402,
            1060.3376467889575,
            1124.3297591131013,
            1288.2618084092137,
            1379.7791732010464,
            1239.9261763796949,
            1261.2257646578728,
            1373.2654382340852,
            1643.3328873296505,
            1229.4540886666,
            1292.5550355079886,
            1269.1109837991025,
            1039.4633306053056,
            1157.5882825716399,
            1179.6766727802915,
            1105.2157241768473
          ],
          "mean": 1302.34840918121,
          "bootstrap_pointwise95": [
            1267.1324619962948,
            1337.7170286277628
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        },
        "A": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 100,
          "pair_ids": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16,
            17,
            18,
            19,
            20,
            21,
            22,
            23,
            24,
            25,
            26,
            27,
            28,
            29,
            30,
            31,
            32,
            33,
            34,
            35,
            36,
            37,
            38,
            39,
            40,
            41,
            42,
            43,
            44,
            45,
            46,
            47,
            48,
            49,
            50,
            51,
            52,
            53,
            54,
            55,
            56,
            57,
            58,
            59,
            60,
            61,
            62,
            63,
            64,
            65,
            66,
            67,
            68,
            69,
            70,
            71,
            72,
            73,
            74,
            75,
            76,
            77,
            78,
            79,
            80,
            81,
            82,
            83,
            84,
            85,
            86,
            87,
            88,
            89,
            90,
            91,
            92,
            93,
            94,
            95,
            96,
            97,
            98,
            99
          ],
          "differences": [
            690.2677650247751,
            763.8023372123989,
            774.6646262699942,
            681.5779461742095,
            668.3452201448035,
            612.9194163802676,
            779.7746395571849,
            686.7049912008952,
            742.5563781291421,
            720.6785189741495,
            730.165581258123,
            746.958520785156,
            635.6890682480516,
            661.7845297327331,
            866.9513010353403,
            718.8607554629507,
            762.2648560835041,
            847.1644967558605,
            762.0138634826326,
            721.2683185284204,
            687.2678093127433,
            628.2422199500088,
            621.3089289055715,
            697.285499272064,
            822.5034529272325,
            732.8303381240239,
            669.0496504759249,
            666.9616556298461,
            634.0544168086744,
            716.8900198430438,
            679.482951700571,
            749.6037350466195,
            610.5071552273785,
            703.6042862140548,
            758.2968224221113,
            663.7078571011389,
            791.6118169633046,
            685.2540276523523,
            847.9902957367594,
            741.2453915106962,
            770.2352985866619,
            650.5027167134267,
            714.8171159952944,
            725.3489899789129,
            665.6730480335955,
            806.7860072762669,
            760.4457338189131,
            677.1708875179138,
            681.8120220834116,
            735.2804876390401,
            707.2031169659258,
            620.8724781397195,
            853.3614856163549,
            665.7780682553112,
            717.7892173428718,
            708.9436413891908,
            841.023071697231,
            749.6581371175962,
            684.8319015138968,
            721.3466705231513,
            689.9004997318352,
            871.2604565095862,
            678.6123237458542,
            806.9963974656439,
            674.1531900901646,
            647.106002098506,
            671.034940347774,
            643.1272458142994,
            627.5273941057221,
            594.5184100728238,
            746.1584516389939,
            678.2742505982023,
            725.0792222247783,
            760.5656024330438,
            786.8747095284307,
            688.9456262562944,
            814.1466064594915,
            686.8905573646716,
            701.1431056795664,
            672.4425754086806,
            739.2567538049061,
            738.4028809516112,
            705.795671556336,
            620.701403456047,
            680.4687071696662,
            676.8784485531669,
            641.6539440302422,
            679.3681405476018,
            710.5253586539802,
            705.1177889061546,
            718.0236308173244,
            734.2867537271923,
            796.2491735917756,
            687.9516288086033,
            725.3184941739128,
            706.9836195856042,
            591.4921123280237,
            668.4520232419109,
            661.3113426585278,
            621.4144340959556
          ],
          "mean": 711.1540143570231,
          "bootstrap_pointwise95": [
            699.0416589317757,
            723.7396731181154
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        },
        "V": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 10,
          "pair_ids": [
            1,
            5,
            6,
            17,
            29,
            42,
            84,
            88,
            89,
            92
          ],
          "differences": [
            1036.6908974592934,
            827.4888210731442,
            1256.3296417043493,
            1247.1075132801525,
            1096.2365344710997,
            972.9132512630313,
            922.1818278727806,
            1042.4787249154506,
            959.1139275828737,
            1232.0639855689406
          ],
          "mean": 1059.2605125191117,
          "bootstrap_pointwise95": [
            973.8050484536645,
            1144.5436363805595
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        },
        "J": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 100,
          "pair_ids": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16,
            17,
            18,
            19,
            20,
            21,
            22,
            23,
            24,
            25,
            26,
            27,
            28,
            29,
            30,
            31,
            32,
            33,
            34,
            35,
            36,
            37,
            38,
            39,
            40,
            41,
            42,
            43,
            44,
            45,
            46,
            47,
            48,
            49,
            50,
            51,
            52,
            53,
            54,
            55,
            56,
            57,
            58,
            59,
            60,
            61,
            62,
            63,
            64,
            65,
            66,
            67,
            68,
            69,
            70,
            71,
            72,
            73,
            74,
            75,
            76,
            77,
            78,
            79,
            80,
            81,
            82,
            83,
            84,
            85,
            86,
            87,
            88,
            89,
            90,
            91,
            92,
            93,
            94,
            95,
            96,
            97,
            98,
            99
          ],
          "differences": [
            1295.0440258355266,
            1304.962550959186,
            1556.2425116602353,
            1192.484195892037,
            1214.1069476791035,
            1077.1697721583969,
            1660.9246653574246,
            1196.2204950208034,
            1509.5835063749473,
            1301.760333377386,
            1415.1039914161975,
            1371.8933618938954,
            1028.6740757345942,
            1078.7953037566767,
            1646.8178466583558,
            1344.9871089183343,
            1493.0206834171795,
            1691.9240129215486,
            1478.2776225537564,
            1271.32233457471,
            1180.4808828728037,
            1062.0299801078036,
            1156.4010620881836,
            1299.0977758888457,
            1577.677892122545,
            1390.2363983305895,
            1209.8252851129237,
            1184.0483177183323,
            1121.177252635065,
            1452.7224822503886,
            1274.0960992016644,
            1396.212245559409,
            894.2139214894069,
            1254.0582063158854,
            1377.317859309278,
            1248.6694225803967,
            1519.1120275244557,
            1144.7298057672015,
            1719.11567690417,
            1386.3098612368758,
            1566.2386751553977,
            1158.9703201854263,
            1252.365478517262,
            1220.681807391893,
            1166.0111682435,
            1521.6244695775022,
            1433.6576535122622,
            1138.5326291238525,
            1153.3356608047307,
            1397.8810001300135,
            1251.5833433570842,
            1094.5951102122417,
            1560.9729308788665,
            1147.9199987581687,
            1340.7142881882992,
            1315.6398077742924,
            1774.8814336852101,
            1388.6778894981417,
            1240.1628742703233,
            1383.0154371585159,
            1250.4306886701634,
            1725.7070931731068,
            1170.7458571215157,
            1545.0797701936908,
            1260.9512801793971,
            1105.7548134818917,
            1202.0574932797317,
            1096.495406969827,
            1028.0497367386106,
            961.243432358182,
            1364.7603741286948,
            1274.292813573773,
            1328.2501682616712,
            1448.8393787516598,
            1451.5709819028707,
            1268.856416851575,
            1593.7881687606477,
            1323.3658938818662,
            1232.3375232398012,
            1181.2324739136056,
            1435.0866839289156,
            1303.8741251295887,
            1226.6109985812927,
            1147.4707650804849,
            1180.1560301475402,
            1060.3376467889575,
            1124.3297591131013,
            1288.2618084092137,
            1379.7791732010464,
            1239.9261763796949,
            1261.2257646578728,
            1373.2654382340852,
            1643.3328873296505,
            1229.4540886666,
            1292.5550355079886,
            1269.1109837991025,
            1039.4633306053056,
            1157.5882825716399,
            1179.6766727802915,
            1105.2157241768473
          ],
          "mean": 1302.34840918121,
          "bootstrap_pointwise95": [
            1267.5910970586478,
            1337.5799652497756
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        }
      }
    },
    "J": {
      "attempted_pairs": 100,
      "planned_pairs": 100,
      "generation_failures": 0,
      "complete_five_score_pairs": 11,
      "winner_counts_including_exact_ties": [
        {
          "models": [
            "J"
          ],
          "count": 11,
          "denominator": 11,
          "rate": 1.0,
          "wilson95": [
            0.7411670330319684,
            1.0
          ]
        }
      ],
      "confusion_unique_winner": {
        "W": {
          "count": 0,
          "denominator": 11,
          "rate": 0.0,
          "wilson95": [
            0.0,
            0.2588329669680317
          ]
        },
        "A": {
          "count": 0,
          "denominator": 11,
          "rate": 0.0,
          "wilson95": [
            0.0,
            0.2588329669680317
          ]
        },
        "V": {
          "count": 0,
          "denominator": 11,
          "rate": 0.0,
          "wilson95": [
            0.0,
            0.2588329669680317
          ]
        },
        "S": {
          "count": 0,
          "denominator": 11,
          "rate": 0.0,
          "wilson95": [
            0.0,
            0.2588329669680317
          ]
        },
        "J": {
          "count": 11,
          "denominator": 11,
          "rate": 1.0,
          "wilson95": [
            0.7411670330319684,
            1.0
          ]
        }
      },
      "ties": {
        "count": 0,
        "denominator": 11,
        "rate": 0.0,
        "wilson95": [
          0.0,
          0.2588329669680317
        ]
      },
      "model_status": {
        "W": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          }
        },
        "A": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          }
        },
        "V": {
          "fit_failure": {
            "count": 89,
            "denominator": 100,
            "rate": 0.89,
            "wilson95": [
              0.8136870349691969,
              0.9374580364293543
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 11,
            "rate": 0.0,
            "wilson95": [
              0.0,
              0.2588329669680317
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 11,
            "rate": 0.0,
            "wilson95": [
              0.0,
              0.2588329669680317
            ]
          }
        },
        "S": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 100,
          "boundary": {
            "count": 100,
            "denominator": 100,
            "rate": 1.0,
            "wilson95": [
              0.9630065017930143,
              1.0
            ]
          }
        },
        "J": {
          "fit_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "score_failure": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          },
          "shared_white_path_count": 0,
          "boundary": {
            "count": 0,
            "denominator": 100,
            "rate": 0.0,
            "wilson95": [
              3.469446951953614e-18,
              0.03699349820698568
            ]
          }
        }
      },
      "paired_score_differences": {
        "W": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 100,
          "pair_ids": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16,
            17,
            18,
            19,
            20,
            21,
            22,
            23,
            24,
            25,
            26,
            27,
            28,
            29,
            30,
            31,
            32,
            33,
            34,
            35,
            36,
            37,
            38,
            39,
            40,
            41,
            42,
            43,
            44,
            45,
            46,
            47,
            48,
            49,
            50,
            51,
            52,
            53,
            54,
            55,
            56,
            57,
            58,
            59,
            60,
            61,
            62,
            63,
            64,
            65,
            66,
            67,
            68,
            69,
            70,
            71,
            72,
            73,
            74,
            75,
            76,
            77,
            78,
            79,
            80,
            81,
            82,
            83,
            84,
            85,
            86,
            87,
            88,
            89,
            90,
            91,
            92,
            93,
            94,
            95,
            96,
            97,
            98,
            99
          ],
          "differences": [
            1264.3806910449111,
            1314.7459388368661,
            1379.5324366503373,
            1088.9607143401645,
            1381.223535011577,
            1154.6580901312118,
            1287.859420850018,
            1683.6768995988496,
            979.0854871033334,
            1200.441448673948,
            1490.6980369776056,
            1335.9750414150349,
            1114.3661493957516,
            1574.1907021164097,
            1380.7717301223802,
            1110.9893798152243,
            1256.7414477607454,
            1168.738531942803,
            1137.880205512738,
            1885.0232792964234,
            1368.2690132806692,
            911.8752692258004,
            1186.0806316895996,
            1249.0977020076625,
            1313.029154449854,
            1647.685425794849,
            936.0511022013834,
            1180.0549183106268,
            1316.785097045593,
            1566.1186258583184,
            1126.4635213843812,
            1241.087105325354,
            1204.6623530322668,
            1300.3187363874895,
            1261.5131791963904,
            1310.6217908695708,
            1123.515636061773,
            1468.69460333794,
            1221.1881111236235,
            1553.488523534752,
            1025.3180368769044,
            1385.7962198698333,
            1539.4475715995786,
            1088.7330506570815,
            1039.456707886302,
            1019.9972192142232,
            1394.6469488705516,
            1197.4168182160556,
            1341.5248603362452,
            1699.660979233906,
            823.0659574470665,
            1308.3320671915008,
            1204.0416538379286,
            1932.6092096015673,
            1205.2312660133211,
            1332.9204056358358,
            1127.3886799642423,
            941.068119429543,
            1327.8411279949178,
            1252.592738062889,
            1604.2457743804753,
            1802.3737532692612,
            1018.026258180249,
            1175.8566486278396,
            1337.7982381654365,
            1311.724460025809,
            1116.7922480076595,
            1106.9577471291414,
            1700.2774336433204,
            1321.6265023609812,
            1166.226531160759,
            1197.229868469567,
            1418.4504974095607,
            1352.7111520193757,
            1046.9541753432968,
            1096.2393002473837,
            1401.3149654055578,
            895.244333120856,
            1339.536854308093,
            1536.5439378531555,
            1191.3061630606735,
            934.6904550351792,
            1488.488589014005,
            1344.1105595101762,
            904.7994145851935,
            1045.9196617939851,
            1717.7159680982932,
            1322.0591174227484,
            1314.7883579423687,
            1139.2927644854826,
            1095.154551318281,
            1159.437335660854,
            1435.5441457213074,
            1207.2253250486847,
            989.9670879206751,
            1289.4850145126438,
            1249.5860461421325,
            1243.3966504709515,
            1384.1177877517894,
            1169.1254403421553
          ],
          "mean": 1274.359704196191,
          "bootstrap_pointwise95": [
            1231.9665782136062,
            1316.98412760513
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        },
        "A": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 100,
          "pair_ids": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16,
            17,
            18,
            19,
            20,
            21,
            22,
            23,
            24,
            25,
            26,
            27,
            28,
            29,
            30,
            31,
            32,
            33,
            34,
            35,
            36,
            37,
            38,
            39,
            40,
            41,
            42,
            43,
            44,
            45,
            46,
            47,
            48,
            49,
            50,
            51,
            52,
            53,
            54,
            55,
            56,
            57,
            58,
            59,
            60,
            61,
            62,
            63,
            64,
            65,
            66,
            67,
            68,
            69,
            70,
            71,
            72,
            73,
            74,
            75,
            76,
            77,
            78,
            79,
            80,
            81,
            82,
            83,
            84,
            85,
            86,
            87,
            88,
            89,
            90,
            91,
            92,
            93,
            94,
            95,
            96,
            97,
            98,
            99
          ],
          "differences": [
            669.5584196073978,
            720.8568693547895,
            740.1850862028514,
            651.0124716923165,
            714.2421048607375,
            645.5894907059082,
            699.6699448571835,
            831.2978713090324,
            616.1392004691988,
            669.0708883528005,
            749.1308719082772,
            722.4684590120887,
            626.2186833093983,
            798.5227260181564,
            727.9253553036415,
            633.3977839904001,
            740.4125806771372,
            652.7717049223502,
            660.7477537073391,
            901.4237185325528,
            751.7085512814247,
            579.0597548571732,
            660.8848476344301,
            724.8510005442604,
            731.2403758355758,
            809.4560182126534,
            598.9999231622678,
            662.4240058724008,
            746.7158541858535,
            771.8667058980409,
            664.9795985334458,
            695.3846871887072,
            686.8799577011923,
            713.6950666618286,
            712.110188251505,
            703.6786829943776,
            660.2859149990018,
            751.5171610105972,
            708.0355018038808,
            775.2599289705959,
            621.6627813559808,
            726.709548056082,
            741.6202821062288,
            644.961900506145,
            661.2267660554353,
            598.6968773928747,
            769.0264837813029,
            684.7946950844798,
            726.518431137667,
            890.1920308168192,
            555.3288020807613,
            738.1594495649952,
            721.0062686802321,
            914.4826815497108,
            685.8339162505381,
            716.0477979422976,
            663.9631031174849,
            650.3299452582578,
            736.8376270357367,
            729.8404973938395,
            833.1818235102492,
            874.5341936578316,
            591.6979775260716,
            651.2621212345766,
            713.0494229042506,
            715.5498455013039,
            660.7962042312465,
            643.5470690628044,
            847.3785368454246,
            736.7727655775598,
            720.6124943340619,
            649.0524837978337,
            750.1397107779212,
            753.8320293989873,
            625.5091017587147,
            676.2211021585099,
            773.1521934477405,
            541.6507355616304,
            760.9220433717546,
            777.3095373344022,
            655.4583337476197,
            575.8787086116554,
            800.7836796104257,
            717.3957695301451,
            587.0594349439725,
            612.364751101577,
            845.5615438831392,
            697.3818314385317,
            751.8893777147914,
            656.566037380142,
            657.6423212762002,
            642.5145542274713,
            738.9039269731311,
            674.0539709110672,
            610.1500790328828,
            733.3977728482168,
            669.2942159950617,
            692.4978733400585,
            764.4973152243456,
            704.4021082224735
          ],
          "mean": 705.4078059356543,
          "bootstrap_pointwise95": [
            690.9264173260651,
            720.1010061836282
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        },
        "V": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 11,
          "pair_ids": [
            16,
            25,
            30,
            47,
            50,
            52,
            58,
            72,
            75,
            81,
            97
          ],
          "differences": [
            1232.7769323327993,
            1599.2729981339426,
            1100.3242015662017,
            1167.8883145906202,
            815.060159683313,
            1175.7394140447668,
            1294.6208446223554,
            1388.4693810133595,
            1065.8781572460302,
            917.3835064642863,
            1222.1239855750118
          ],
          "mean": 1179.9579904793352,
          "bootstrap_pointwise95": [
            1061.5060400512905,
            1300.8014072446676
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        },
        "S": {
          "direction": "truth-model minus competitor; higher is better",
          "units": "nats per whole test record",
          "denominator": 100,
          "pair_ids": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16,
            17,
            18,
            19,
            20,
            21,
            22,
            23,
            24,
            25,
            26,
            27,
            28,
            29,
            30,
            31,
            32,
            33,
            34,
            35,
            36,
            37,
            38,
            39,
            40,
            41,
            42,
            43,
            44,
            45,
            46,
            47,
            48,
            49,
            50,
            51,
            52,
            53,
            54,
            55,
            56,
            57,
            58,
            59,
            60,
            61,
            62,
            63,
            64,
            65,
            66,
            67,
            68,
            69,
            70,
            71,
            72,
            73,
            74,
            75,
            76,
            77,
            78,
            79,
            80,
            81,
            82,
            83,
            84,
            85,
            86,
            87,
            88,
            89,
            90,
            91,
            92,
            93,
            94,
            95,
            96,
            97,
            98,
            99
          ],
          "differences": [
            1264.3806910449111,
            1314.7459388368661,
            1379.5324366503373,
            1088.9607143401645,
            1381.223535011577,
            1154.6580901312118,
            1287.859420850018,
            1683.6768995988496,
            979.0854871033334,
            1200.441448673948,
            1490.6980369776056,
            1335.9750414150349,
            1114.3661493957516,
            1574.1907021164097,
            1380.7717301223802,
            1110.9893798152243,
            1256.7414477607454,
            1168.738531942803,
            1137.880205512738,
            1885.0232792964234,
            1368.2690132806692,
            911.8752692258004,
            1186.0806316895996,
            1249.0977020076625,
            1313.029154449854,
            1647.685425794849,
            936.0511022013834,
            1180.0549183106268,
            1316.785097045593,
            1566.1186258583184,
            1126.4635213843812,
            1241.087105325354,
            1204.6623530322668,
            1300.3187363874895,
            1261.5131791963904,
            1310.6217908695708,
            1123.515636061773,
            1468.69460333794,
            1221.1881111236235,
            1553.488523534752,
            1025.3180368769044,
            1385.7962198698333,
            1539.4475715995786,
            1088.7330506570815,
            1039.456707886302,
            1019.9972192142232,
            1394.6469488705516,
            1197.4168182160556,
            1341.5248603362452,
            1699.660979233906,
            823.0659574470665,
            1308.3320671915008,
            1204.0416538379286,
            1932.6092096015673,
            1205.2312660133211,
            1332.9204056358358,
            1127.3886799642423,
            941.068119429543,
            1327.8411279949178,
            1252.592738062889,
            1604.2457743804753,
            1802.3737532692612,
            1018.026258180249,
            1175.8566486278396,
            1337.7982381654365,
            1311.724460025809,
            1116.7922480076595,
            1106.9577471291414,
            1700.2774336433204,
            1321.6265023609812,
            1166.226531160759,
            1197.229868469567,
            1418.4504974095607,
            1352.7111520193757,
            1046.9541753432968,
            1096.2393002473837,
            1401.3149654055578,
            895.244333120856,
            1339.536854308093,
            1536.5439378531555,
            1191.3061630606735,
            934.6904550351792,
            1488.488589014005,
            1344.1105595101762,
            904.7994145851935,
            1045.9196617939851,
            1717.7159680982932,
            1322.0591174227484,
            1314.7883579423687,
            1139.2927644854826,
            1095.154551318281,
            1159.437335660854,
            1435.5441457213074,
            1207.2253250486847,
            989.9670879206751,
            1289.4850145126438,
            1249.5860461421325,
            1243.3966504709515,
            1384.1177877517894,
            1169.1254403421553
          ],
          "mean": 1274.359704196191,
          "bootstrap_pointwise95": [
            1231.5917083198713,
            1318.2607712942736
          ],
          "interval_missing_reason": null,
          "interpretation": "conditional on both scores available; no multiplicity adjustment"
        }
      }
    }
  }
}