{ "plan": { "status": "exploratory follow-up; chapter 45 test results were already seen", "sizes": [ 20, 50, 100, 269 ], "subset_repeats": 20, "subset_seed": 202646, "bootstrap_repeats": 2000, "bootstrap_seed": 4601, "bootstrap_method": "paired percentile, fixed fitted models and temperatures", "ece_bins": [ 5, 10, 15, 30 ], "temperature_bounds": [ 0.05, 20.0 ], "primary_metric": "test mean NLL after minus before; report every training seed", "selection_rule": "no selection by test outcomes; retain every subset and bin count" }, "plan_sha256": "4bae256b9cc5d3070698064ee723485713e75066ad8a2b5da7139473ebf8dc05", "source_file_sha256": "cd140760a7408c351e25064ac65fb04774fbdc4676a939e3778658b9e7032681", "source_numeric_sha256": "133538814ead066cb15838b80e9c87c64da847bac3aba192c0969161a6138b5e", "script_sha256": "8c9e52141e70999e856f161636f06f92db46441a5caab4051e2dd76943046a09", "method_sha256": "5d3fa24114ec1b15ecbb2212f964b3d0b89a3340e543c05de2e7e10a9c6367dd", "environment": { "python": "3.12.12", "numpy": "2.5.4", "scipy": "1.18.1", "platform": "macOS-26.6.2-arm64-arm-64bit" }, "test_ids": [ 474, 843, 398, 1267, 736, 913, 174, 765, 734, 148, 1159, 676, 614, 1186, 1281, 575, 732, 1457, 371, 1466, 170, 680, 964, 344, 564, 848, 126, 1212, 1224, 1531, 755, 318, 163, 1365, 253, 298, 440, 1556, 1536, 1290, 998, 623, 32, 1486, 1767, 933, 513, 1198, 1407, 577, 63, 1610, 378, 1608, 1498, 992, 135, 1064, 205, 922, 881, 1435, 1501, 1479, 688, 1283, 353, 713, 1172, 101, 931, 1041, 150, 1082, 159, 1580, 951, 343, 231, 444, 1685, 830, 306, 982, 196, 1163, 871, 1561, 629, 1274, 213, 879, 1664, 1520, 366, 1409, 567, 229, 1124, 984, 1627, 346, 1398, 1236, 586, 1063, 136, 1488, 563, 143, 1226, 646, 1006, 1421, 1430, 1695, 928, 416, 527, 1705, 1339, 241, 1275, 26, 506, 1461, 1500, 1021, 531, 1768, 1694, 1582, 977, 1231, 1302, 1448, 520, 804, 1401, 626, 917, 251, 1429, 27, 498, 1131, 750, 554, 900, 166, 1074, 335, 592, 190, 1782, 1217, 476, 1635, 685, 1323, 939, 359, 1416, 5, 1145, 1038, 18, 1320, 1382, 1393, 397, 269, 1182, 505, 1662, 596, 1020, 569, 274, 1660, 1513, 610, 1751, 1194, 819, 884, 1227, 433, 1096, 364, 1043, 1672, 1772, 1632, 171, 443, 635, 537, 552, 1720, 86, 293, 1360, 199, 197, 141, 312, 1743, 538, 795, 862, 532, 204, 182, 1108, 1737, 749, 1037, 132, 72, 1079, 117, 548, 557, 999, 12, 463, 1315, 1160, 113, 752, 873, 147, 1578, 919, 875, 1288, 441, 846, 461, 657, 960, 1583, 624, 609, 1117, 845, 952, 1397, 1036, 1040, 1125, 110, 1415, 1197, 1723, 1328, 722, 647, 542, 29, 973, 1373, 1450, 1123, 243, 502, 1424, 953, 1384, 837, 507, 1260, 1603, 590, 347, 1657, 1211, 528, 214, 39, 524, 690, 403, 1370, 445, 1364, 584, 465, 1774, 585, 1589, 1165, 733, 772, 1422, 1206, 1225, 723, 1105, 549, 240, 1473, 957, 1033, 1234, 305, 1749, 897, 67, 1559, 545, 526, 663, 758, 715, 599, 811, 1571, 789, 641, 643, 1602, 217, 1481, 246, 1642, 111, 807, 140, 421, 139, 424, 100, 1605, 1026, 1263, 49, 257, 1477, 1492, 1166, 1489, 1527, 426, 304, 455, 277, 691, 121, 618, 1122, 916, 1193, 654, 475, 215, 959, 1399, 1463 ], "subset_orders": [ [ 103, 1336, 326, 611, 177, 495, 570, 1246, 303, 222, 858, 157, 97, 546, 1093, 606, 737, 1419, 1179, 583, 947, 1121, 183, 1372, 436, 1146, 131, 456, 460, 340, 1490, 697, 16, 273, 400, 324, 153, 381, 1251, 825, 1313, 865, 1171, 1067, 74, 1342, 108, 388, 1047, 1380, 743, 699, 1003, 300, 1425, 211, 122, 836, 905, 1285, 168, 1530, 775, 573, 1138, 488, 435, 1738, 41, 1048, 296, 14, 523, 92, 155, 694, 1586, 1644, 1792, 360, 571, 137, 1686, 81, 1470, 1362, 263, 1241, 1588, 161, 88, 235, 230, 745, 1716, 1703, 899, 501, 1250, 829, 574, 927, 1203, 1550, 1563, 969, 1778, 1289, 1357, 1651, 25, 689, 1367, 1790, 1624, 107, 601, 780, 338, 1741, 1533, 1404, 1652, 1111, 412, 995, 1688, 1745, 664, 1215, 1335, 206, 1377, 1478, 1216, 65, 423, 1045, 1673, 482, 1715, 1666, 1639, 1626, 225, 1235, 1394, 511, 1587, 1641, 1690, 541, 1233, 763, 1204, 1434, 558, 391, 1780, 729, 721, 1169, 946, 1562, 1598, 809, 1019, 1176, 125, 102, 978, 1248, 1296, 1597, 824, 854, 1541, 1097, 1268, 185, 782, 661, 770, 1726, 179, 264, 1517, 176, 921, 149, 808, 1282, 314, 882, 1228, 874, 802, 432, 1712, 631, 712, 178, 1630, 1791, 1444, 1459, 944, 1331, 490, 1187, 361, 218, 659, 1718, 284, 1022, 254, 1157, 986, 1130, 1649, 1073, 591, 509, 756, 535, 1456, 813, 805, 605, 1614, 739, 1363, 975, 1271, 660, 1374, 23, 34, 754, 1661, 1725, 1088, 1013, 1090, 1521, 1195, 1076, 480, 1262, 566, 252, 244, 1389, 788, 519, 1084, 1777, 675, 1438, 1787, 682, 1496, 1729, 1149, 1391, 1023, 1213, 1280 ], [ 1335, 969, 1296, 782, 391, 1233, 1362, 1780, 412, 179, 1176, 480, 273, 435, 1073, 490, 802, 314, 1048, 381, 235, 183, 326, 108, 1586, 944, 1738, 296, 809, 946, 1380, 1367, 300, 185, 1726, 1644, 1425, 1725, 1550, 1703, 1280, 882, 1342, 659, 1336, 535, 1076, 1216, 1741, 1203, 177, 1715, 905, 858, 1246, 263, 805, 97, 1282, 675, 25, 1090, 1084, 1313, 102, 1470, 1228, 682, 1444, 423, 168, 1419, 1377, 756, 697, 1649, 1157, 661, 155, 1639, 1215, 788, 1690, 1235, 1787, 978, 1652, 495, 1262, 92, 566, 921, 1490, 1562, 1121, 103, 340, 1651, 1716, 338, 519, 1587, 1673, 1251, 137, 1530, 825, 34, 611, 74, 284, 1588, 1521, 780, 1067, 763, 1563, 721, 1745, 1478, 1438, 1541, 161, 1019, 324, 854, 1459, 400, 591, 1093, 570, 1517, 1666, 694, 1597, 88, 1013, 14, 541, 1097, 1179, 1790, 361, 1729, 65, 1204, 1271, 1130, 660, 899, 824, 1688, 574, 252, 1241, 1138, 737, 432, 1630, 1285, 157, 460, 1331, 222, 1045, 1626, 1250, 573, 388, 808, 1003, 1171, 1533, 631, 927, 1456, 1149, 254, 1686, 1374, 1195, 986, 1598, 1778, 1712, 699, 583, 606, 1372, 1389, 218, 1047, 1268, 947, 813, 1394, 122, 436, 571, 501, 1363, 745, 1391, 1169, 488, 1777, 107, 829, 770, 1248, 125, 1661, 482, 1404, 605, 244, 511, 1146, 546, 1289, 1641, 1088, 178, 264, 176, 601, 1022, 689, 1791, 360, 1792, 211, 1111, 456, 712, 1357, 995, 131, 206, 303, 739, 509, 230, 1624, 775, 729, 975, 41, 1434, 1614, 153, 754, 16, 149, 1187, 225, 865, 874, 664, 523, 1496, 836, 558, 1718, 23, 81, 743, 1213, 1023 ], [ 739, 775, 802, 1459, 1023, 1374, 675, 1745, 1215, 1716, 1729, 1362, 1233, 780, 423, 1712, 1045, 225, 1241, 659, 1090, 1246, 566, 605, 921, 699, 546, 1363, 978, 490, 222, 1521, 1377, 1550, 361, 1121, 986, 1013, 14, 1562, 570, 1587, 1203, 1336, 1342, 611, 179, 1661, 1496, 899, 519, 412, 1718, 97, 1777, 1434, 273, 456, 1282, 1703, 1588, 558, 721, 177, 1130, 1313, 1726, 1725, 1644, 1357, 1019, 1022, 1111, 1251, 975, 1248, 34, 1235, 509, 1614, 23, 1478, 264, 1792, 161, 314, 1067, 149, 103, 1176, 244, 829, 1790, 1641, 400, 1688, 782, 743, 535, 1367, 381, 1380, 131, 697, 1530, 25, 805, 122, 1084, 41, 1791, 905, 137, 591, 1171, 230, 1296, 206, 541, 1517, 583, 495, 1787, 874, 432, 303, 107, 1419, 944, 1003, 235, 660, 825, 927, 324, 1268, 155, 824, 1157, 1626, 1639, 338, 1187, 523, 689, 1280, 745, 712, 1228, 754, 606, 682, 882, 1195, 809, 388, 436, 1666, 756, 947, 153, 88, 1391, 1047, 1738, 1271, 1250, 284, 737, 183, 252, 946, 488, 1149, 1216, 865, 1686, 1285, 1690, 1213, 391, 1289, 501, 1389, 326, 1073, 664, 1780, 601, 254, 813, 573, 1093, 482, 81, 157, 1444, 1179, 360, 1563, 1335, 176, 460, 1425, 263, 1541, 16, 1088, 854, 1715, 574, 1741, 1778, 1597, 480, 1138, 808, 74, 763, 1651, 102, 661, 858, 65, 1097, 631, 1048, 125, 296, 1262, 511, 218, 340, 694, 1630, 1470, 1649, 571, 770, 168, 1204, 1673, 1076, 729, 1652, 1598, 1438, 1331, 1169, 1146, 1586, 1490, 969, 836, 300, 788, 1533, 1624, 435, 211, 185, 1394, 92, 178, 995, 1372, 1404, 1456, 108 ], [ 157, 300, 519, 436, 185, 1179, 178, 273, 264, 1363, 1169, 1090, 495, 1729, 1313, 88, 660, 107, 1047, 81, 176, 1067, 263, 1121, 1377, 611, 254, 1003, 1404, 1246, 488, 1597, 1792, 1331, 1652, 340, 1686, 825, 1563, 1478, 1712, 161, 591, 1296, 1790, 865, 874, 1213, 1216, 222, 1470, 1490, 1271, 1716, 16, 1688, 1438, 1521, 1587, 858, 1073, 103, 125, 975, 729, 1285, 137, 802, 1673, 739, 284, 946, 1726, 230, 1434, 1171, 1703, 854, 1459, 1533, 1149, 1389, 1394, 882, 1335, 808, 697, 1690, 1787, 813, 1262, 1342, 1157, 432, 149, 605, 1419, 131, 400, 1138, 1357, 1780, 102, 1517, 659, 571, 1248, 480, 546, 381, 699, 1530, 97, 1022, 1111, 1598, 490, 1391, 206, 92, 314, 1282, 183, 1048, 1251, 770, 252, 511, 1614, 34, 324, 509, 501, 535, 218, 1233, 1639, 1362, 737, 303, 108, 391, 682, 583, 809, 1641, 1745, 423, 1280, 456, 1550, 947, 541, 360, 244, 1651, 782, 570, 1097, 574, 1215, 664, 969, 1586, 1146, 899, 836, 743, 177, 601, 1777, 65, 1228, 1195, 211, 689, 921, 986, 1624, 1076, 1644, 1250, 179, 1666, 1791, 1130, 1626, 338, 763, 1372, 944, 1013, 1204, 296, 573, 482, 153, 721, 1588, 1541, 1241, 225, 388, 1630, 775, 23, 25, 1289, 829, 523, 756, 1019, 978, 754, 1045, 558, 1176, 1088, 1718, 326, 1725, 661, 1741, 905, 1444, 235, 694, 1023, 155, 995, 412, 745, 1738, 1084, 1268, 1456, 1778, 361, 566, 712, 606, 122, 805, 168, 1374, 631, 927, 780, 1093, 1336, 675, 788, 824, 1649, 460, 1367, 435, 1187, 14, 74, 1203, 1715, 1380, 1562, 41, 1661, 1235, 1496, 1425 ], [ 1362, 1777, 1787, 1138, 1176, 1048, 1690, 1715, 81, 809, 1251, 1045, 511, 501, 65, 1289, 1703, 805, 1067, 1169, 689, 763, 782, 1438, 756, 1250, 836, 788, 1530, 381, 905, 1367, 1394, 1651, 1478, 1271, 854, 1171, 721, 570, 566, 244, 1673, 1496, 1630, 1644, 179, 235, 1228, 211, 1282, 1741, 661, 1791, 927, 1268, 161, 1718, 1521, 168, 824, 1614, 1517, 775, 92, 825, 1073, 1470, 254, 813, 1780, 601, 865, 1541, 222, 1377, 1649, 1213, 1726, 1296, 436, 697, 1389, 745, 583, 1738, 606, 729, 1434, 573, 1121, 1111, 1790, 1090, 432, 1586, 1624, 1241, 225, 1023, 770, 874, 183, 1093, 571, 995, 694, 1357, 780, 482, 1233, 107, 975, 303, 535, 178, 611, 1195, 986, 23, 495, 361, 456, 423, 480, 296, 1425, 185, 1745, 1404, 1331, 1661, 324, 155, 921, 1456, 858, 177, 1490, 206, 340, 16, 737, 25, 122, 882, 1084, 558, 435, 388, 102, 1729, 1666, 1626, 1374, 460, 808, 523, 969, 699, 1716, 1013, 1157, 314, 1179, 88, 326, 1203, 1380, 1076, 230, 137, 488, 131, 125, 675, 412, 605, 739, 338, 149, 546, 273, 1097, 1372, 1146, 1313, 252, 1444, 1712, 1342, 1778, 574, 14, 176, 1047, 490, 300, 153, 1003, 157, 1533, 391, 263, 946, 1652, 1562, 660, 1235, 1215, 1022, 1587, 1187, 1336, 591, 1130, 1262, 1285, 284, 899, 74, 1598, 1597, 947, 541, 978, 509, 659, 1019, 1459, 1363, 218, 754, 1641, 103, 829, 1204, 1419, 664, 802, 712, 1725, 1216, 400, 360, 1588, 1088, 1149, 264, 682, 631, 41, 1688, 1248, 34, 1391, 1246, 519, 1792, 944, 1550, 1686, 108, 743, 1280, 1335, 97, 1639, 1563 ], [ 1268, 1649, 511, 1363, 946, 1456, 770, 456, 1023, 739, 211, 802, 179, 1215, 1250, 1076, 1521, 1715, 41, 805, 865, 177, 176, 157, 400, 206, 675, 1496, 1588, 699, 92, 183, 185, 284, 1688, 1745, 606, 1045, 1777, 97, 1090, 1703, 1233, 25, 1587, 1533, 1213, 721, 1741, 361, 631, 1342, 574, 102, 460, 756, 1425, 874, 1434, 1088, 1187, 252, 131, 601, 661, 1271, 338, 1157, 1169, 1718, 1084, 391, 1630, 829, 921, 1121, 1019, 573, 432, 107, 1176, 1377, 360, 1419, 1146, 340, 535, 153, 995, 1729, 1216, 1517, 296, 1478, 218, 1787, 1790, 412, 1626, 1331, 1624, 65, 1003, 1335, 1374, 161, 1563, 808, 1661, 435, 1652, 825, 689, 1296, 541, 482, 882, 314, 1394, 583, 743, 1203, 1598, 1391, 809, 1130, 836, 225, 1586, 1067, 1336, 1097, 1111, 273, 558, 1459, 222, 1093, 1282, 230, 1262, 495, 1780, 1022, 436, 1644, 81, 1251, 1666, 1149, 490, 23, 813, 1725, 178, 1285, 1470, 1313, 1404, 1195, 1444, 74, 1367, 1246, 1597, 480, 501, 780, 122, 88, 381, 263, 1138, 546, 1438, 1204, 788, 16, 1280, 905, 1047, 745, 326, 34, 571, 694, 1639, 659, 1228, 1289, 899, 1716, 712, 570, 1362, 1690, 1530, 1235, 1562, 244, 1726, 927, 235, 324, 1614, 591, 108, 969, 264, 1792, 1490, 523, 1171, 149, 1248, 664, 1380, 303, 763, 1686, 1791, 611, 1550, 737, 754, 168, 1372, 660, 1357, 1241, 519, 254, 1048, 1673, 978, 986, 682, 14, 775, 300, 155, 1179, 947, 1641, 729, 697, 488, 137, 1073, 509, 858, 423, 824, 566, 605, 1778, 944, 975, 103, 1013, 1389, 125, 388, 1712, 1738, 1541, 782, 854, 1651 ], [ 788, 1331, 1651, 729, 523, 780, 81, 436, 1598, 218, 1176, 25, 168, 1673, 969, 535, 423, 211, 1013, 1003, 1639, 1715, 1251, 1204, 1336, 1718, 1394, 88, 1241, 412, 284, 664, 1363, 1541, 1652, 264, 519, 1233, 809, 1729, 1738, 836, 254, 361, 1195, 858, 65, 802, 16, 314, 1649, 1179, 1280, 1171, 137, 978, 1372, 1045, 1780, 161, 601, 108, 340, 899, 1741, 501, 1213, 1521, 1562, 511, 1203, 1380, 631, 509, 1630, 1067, 829, 183, 986, 400, 1216, 456, 41, 1404, 854, 1586, 1790, 1342, 14, 244, 1550, 605, 1367, 252, 23, 176, 1262, 975, 273, 1777, 1088, 157, 1530, 1157, 460, 1517, 1624, 1456, 153, 303, 435, 1470, 571, 763, 660, 263, 1019, 1097, 107, 1587, 591, 1271, 1084, 1282, 230, 1313, 1791, 1130, 1726, 1792, 1614, 103, 1644, 1246, 1377, 1725, 743, 1716, 824, 754, 1228, 1391, 1146, 1490, 206, 570, 697, 1690, 1076, 1362, 659, 1686, 338, 541, 770, 573, 122, 488, 1169, 1248, 1712, 1215, 546, 1268, 694, 1666, 813, 1093, 1496, 661, 179, 1641, 737, 235, 300, 97, 480, 927, 611, 782, 102, 381, 1597, 225, 756, 1703, 882, 675, 1389, 155, 360, 1425, 721, 995, 1289, 1149, 1688, 1357, 324, 1419, 149, 125, 74, 1459, 1073, 1745, 1434, 574, 874, 1563, 1235, 1438, 689, 1478, 391, 185, 946, 1374, 1022, 583, 495, 1444, 1047, 326, 1187, 921, 558, 905, 825, 808, 606, 482, 92, 1048, 775, 805, 739, 682, 34, 1533, 177, 1285, 296, 1296, 222, 1661, 745, 944, 1335, 566, 388, 1588, 699, 490, 432, 178, 1787, 1138, 865, 1250, 1121, 1778, 1090, 1023, 131, 947, 1626, 712, 1111 ], [ 456, 23, 296, 1157, 1271, 664, 739, 743, 1791, 1787, 605, 125, 1444, 1203, 1587, 1651, 1171, 591, 1213, 1023, 1282, 1331, 252, 571, 1456, 225, 1076, 825, 179, 263, 168, 611, 1438, 566, 1215, 1639, 176, 541, 460, 1530, 1251, 1517, 805, 1541, 1716, 780, 1313, 1248, 381, 1459, 921, 480, 488, 975, 1490, 1362, 770, 206, 137, 1790, 1084, 1268, 721, 1130, 1394, 423, 1703, 436, 1614, 546, 675, 41, 1195, 391, 131, 1673, 836, 1285, 1335, 712, 1598, 1372, 1048, 583, 222, 854, 1093, 254, 697, 1380, 1718, 802, 829, 185, 400, 788, 601, 1149, 1367, 1712, 97, 1073, 754, 1521, 689, 1741, 1624, 745, 1725, 1111, 1745, 986, 1289, 899, 1792, 25, 1097, 1729, 1250, 1434, 102, 1003, 782, 412, 495, 858, 682, 178, 149, 1661, 558, 230, 490, 1374, 1262, 995, 218, 969, 157, 1022, 535, 946, 1715, 1187, 1235, 1626, 1090, 361, 1690, 326, 1045, 482, 1666, 874, 153, 1204, 1778, 511, 107, 1688, 570, 501, 108, 661, 1228, 435, 824, 1241, 155, 659, 1597, 1641, 1496, 699, 1363, 660, 1357, 1389, 1586, 1649, 1296, 388, 775, 1146, 284, 1686, 865, 338, 1777, 1067, 509, 1404, 14, 756, 694, 1121, 1563, 1047, 300, 1216, 523, 103, 882, 808, 1630, 1391, 324, 1652, 1425, 235, 432, 1644, 1138, 264, 314, 1419, 1533, 947, 183, 122, 927, 729, 631, 1550, 1176, 813, 1246, 1588, 1377, 606, 74, 65, 1013, 340, 1342, 1280, 905, 1336, 1738, 1179, 1233, 1019, 88, 360, 1169, 303, 92, 1562, 34, 809, 737, 1470, 574, 177, 244, 1478, 161, 1780, 273, 763, 1088, 978, 573, 16, 81, 211, 1726, 519, 944 ], [ 1235, 1228, 1013, 1745, 1666, 1718, 495, 185, 1478, 1587, 460, 1738, 300, 660, 882, 412, 324, 103, 573, 1362, 263, 1067, 1187, 206, 1597, 1146, 1130, 1149, 65, 1389, 1716, 235, 1652, 1357, 222, 995, 1586, 108, 1490, 605, 1246, 809, 1778, 14, 1641, 1363, 1649, 1391, 1726, 541, 1372, 1195, 1380, 1204, 1563, 74, 813, 25, 606, 570, 1047, 921, 854, 1690, 1530, 153, 1282, 1336, 899, 252, 211, 509, 1216, 694, 865, 805, 583, 1444, 34, 511, 1241, 1342, 571, 1639, 836, 682, 381, 501, 1313, 432, 1169, 1176, 780, 1048, 388, 756, 858, 1271, 1517, 739, 131, 1630, 1729, 1374, 230, 1703, 754, 1076, 1777, 1285, 178, 88, 1296, 1550, 566, 435, 1541, 1045, 631, 1280, 986, 946, 1741, 1023, 254, 1093, 1335, 874, 360, 546, 303, 480, 947, 1588, 1073, 1434, 601, 1084, 149, 661, 1250, 1790, 1289, 782, 743, 1661, 1780, 1725, 574, 1419, 905, 1438, 1203, 1673, 361, 523, 1562, 535, 1248, 1533, 1377, 824, 1644, 1331, 490, 81, 1688, 829, 1088, 927, 1598, 697, 1626, 436, 264, 944, 273, 519, 808, 978, 1791, 1624, 1686, 745, 1715, 721, 137, 1233, 802, 558, 1614, 102, 456, 1459, 1171, 122, 1179, 284, 611, 183, 179, 92, 1121, 125, 97, 675, 157, 1792, 177, 1496, 1019, 1262, 1651, 699, 788, 770, 1090, 155, 391, 1213, 1521, 326, 225, 1022, 488, 314, 338, 400, 176, 1003, 218, 1157, 16, 41, 23, 1251, 482, 244, 1425, 161, 423, 1215, 969, 168, 737, 975, 1367, 296, 1394, 729, 1138, 107, 591, 1097, 775, 1111, 1456, 1787, 763, 1404, 712, 340, 825, 1268, 689, 1712, 659, 1470, 664 ], [ 1146, 694, 244, 1138, 1787, 675, 1686, 412, 788, 326, 225, 1791, 660, 391, 1630, 1084, 1013, 1652, 381, 1271, 712, 284, 1296, 1216, 423, 986, 436, 1587, 177, 975, 1121, 65, 1778, 1342, 1235, 1434, 273, 780, 1391, 978, 1019, 360, 1130, 1729, 108, 523, 432, 1777, 338, 829, 183, 573, 1215, 1289, 1624, 1093, 1067, 1563, 131, 1377, 874, 1362, 137, 153, 361, 570, 1090, 1268, 546, 1550, 808, 122, 16, 1176, 601, 155, 995, 969, 591, 882, 809, 102, 739, 1712, 1203, 1285, 1745, 490, 1425, 1597, 149, 1149, 511, 1651, 745, 775, 754, 211, 813, 1673, 1076, 157, 682, 1022, 519, 836, 743, 1233, 921, 1644, 74, 1282, 1541, 905, 1250, 230, 1335, 1444, 161, 944, 1780, 314, 611, 1336, 435, 729, 482, 41, 689, 697, 1363, 1023, 659, 1496, 303, 1367, 802, 400, 1517, 222, 1204, 14, 763, 1456, 388, 1438, 1357, 1598, 1389, 264, 1741, 252, 1470, 1241, 1738, 340, 1169, 92, 1533, 661, 1374, 1313, 865, 1179, 107, 1718, 1246, 254, 824, 1649, 1248, 858, 1213, 480, 1639, 558, 782, 899, 1530, 1490, 737, 927, 605, 1187, 125, 81, 296, 1380, 495, 805, 1047, 583, 1048, 770, 218, 1045, 263, 947, 631, 1097, 1404, 1171, 460, 1459, 574, 235, 1792, 1588, 1666, 664, 1716, 34, 1394, 1372, 179, 1688, 1195, 501, 1419, 854, 721, 1690, 535, 176, 168, 1478, 23, 1111, 88, 185, 566, 946, 1073, 1641, 756, 97, 1725, 1562, 103, 1521, 1726, 25, 456, 1790, 178, 1228, 1661, 541, 1586, 509, 1251, 324, 1703, 1626, 1715, 488, 1003, 571, 300, 1280, 699, 1088, 1262, 1614, 825, 606, 1331, 1157, 206 ], [ 264, 1013, 108, 361, 1076, 432, 1268, 858, 1661, 1022, 338, 874, 824, 1726, 340, 1780, 882, 969, 1289, 1791, 263, 1377, 1171, 1688, 488, 1778, 944, 1363, 157, 81, 1738, 230, 1626, 1391, 1588, 1434, 1517, 1718, 179, 946, 802, 546, 921, 1716, 660, 1048, 1187, 1587, 1146, 273, 571, 995, 813, 16, 284, 574, 519, 541, 1169, 122, 1093, 1745, 1521, 770, 1729, 808, 763, 391, 177, 88, 211, 1047, 601, 664, 1533, 1357, 1703, 1374, 1563, 1195, 153, 254, 583, 1111, 1073, 754, 435, 1003, 161, 225, 1666, 1067, 1023, 1715, 986, 978, 1787, 1456, 183, 436, 1097, 1280, 1262, 1235, 836, 805, 1470, 1394, 1204, 739, 480, 745, 689, 34, 168, 1459, 788, 1179, 699, 854, 523, 296, 573, 606, 737, 1216, 300, 495, 74, 947, 1203, 1652, 137, 1630, 1550, 1331, 1090, 1686, 1404, 107, 1335, 125, 535, 1130, 1251, 566, 185, 682, 611, 1649, 1336, 1712, 235, 1496, 303, 1241, 721, 218, 1045, 1233, 412, 780, 1438, 1478, 456, 41, 1530, 482, 829, 381, 1725, 131, 1777, 509, 1019, 324, 782, 605, 1213, 1444, 1121, 865, 1088, 756, 314, 1598, 92, 1673, 1362, 1246, 697, 729, 1490, 631, 1176, 1389, 1586, 1367, 570, 675, 825, 1313, 1296, 1562, 1250, 927, 905, 244, 14, 511, 591, 388, 1282, 252, 1157, 1138, 1741, 1372, 694, 661, 1614, 1624, 1084, 975, 222, 1271, 1690, 712, 1425, 1419, 1651, 1597, 400, 102, 1792, 1342, 360, 460, 1380, 1644, 1641, 23, 659, 1790, 1228, 1639, 423, 558, 490, 1248, 1149, 326, 1285, 743, 206, 65, 103, 155, 25, 178, 176, 775, 899, 501, 1215, 1541, 97, 149, 809 ], [ 675, 605, 1084, 927, 1362, 388, 155, 763, 1651, 1394, 1630, 303, 1530, 541, 546, 149, 1490, 1250, 682, 381, 694, 1550, 1444, 1157, 1626, 754, 432, 1019, 1271, 1726, 1067, 490, 583, 74, 235, 1716, 34, 743, 745, 206, 1169, 1342, 1262, 1216, 535, 456, 1363, 1391, 391, 177, 1111, 1280, 1203, 1587, 1187, 995, 1251, 1130, 659, 168, 874, 1235, 1712, 1285, 788, 1792, 523, 326, 273, 161, 1090, 1588, 218, 1171, 296, 254, 314, 360, 882, 1562, 1248, 92, 157, 712, 570, 1121, 482, 14, 412, 1470, 338, 975, 1703, 1195, 97, 179, 1213, 1644, 460, 1456, 824, 107, 1690, 1331, 631, 1438, 1614, 1780, 573, 103, 108, 1586, 1022, 501, 1641, 571, 1003, 1215, 921, 899, 1146, 1517, 1419, 1380, 436, 986, 65, 661, 770, 244, 81, 1661, 1787, 488, 721, 865, 825, 16, 511, 176, 1296, 1729, 1228, 1686, 1149, 1541, 737, 802, 1377, 284, 1496, 263, 23, 947, 729, 905, 978, 1404, 1076, 1179, 1374, 1639, 1023, 264, 946, 782, 1624, 480, 230, 1718, 1434, 1246, 611, 780, 1268, 1241, 1790, 400, 1459, 225, 300, 1533, 361, 1745, 1715, 944, 591, 775, 1233, 122, 1673, 574, 340, 805, 969, 1282, 102, 1336, 660, 25, 1738, 739, 1425, 1357, 125, 664, 1176, 756, 222, 1289, 1652, 435, 1597, 1013, 1389, 601, 1777, 1478, 1088, 131, 1093, 509, 519, 1741, 1073, 137, 252, 41, 558, 1367, 183, 211, 1372, 606, 1666, 1313, 1598, 566, 808, 324, 1688, 1047, 1649, 689, 88, 1791, 858, 178, 854, 423, 699, 836, 697, 1725, 1097, 153, 1204, 1138, 1521, 809, 813, 829, 1778, 495, 1335, 1045, 1048, 1563, 185 ], [ 1673, 1787, 1587, 1097, 74, 729, 1630, 41, 1335, 1171, 978, 273, 946, 675, 1741, 1729, 541, 743, 179, 1550, 1563, 1195, 81, 360, 137, 1149, 1088, 1111, 802, 254, 16, 1688, 1404, 1716, 1626, 1342, 606, 296, 1296, 244, 882, 1456, 813, 1614, 340, 1203, 435, 388, 1791, 432, 1690, 825, 157, 423, 782, 490, 235, 944, 1586, 495, 107, 1336, 770, 1176, 1639, 1562, 566, 511, 611, 558, 206, 756, 23, 1530, 995, 1496, 1067, 1073, 230, 1228, 1641, 168, 1490, 1271, 1777, 519, 1019, 927, 460, 947, 178, 1541, 535, 745, 338, 601, 809, 400, 754, 1533, 546, 763, 874, 775, 1725, 102, 131, 631, 854, 697, 1517, 264, 1285, 1726, 1363, 122, 1268, 1459, 300, 1718, 1715, 456, 1093, 661, 573, 1389, 1003, 1169, 436, 1282, 1084, 583, 1703, 836, 211, 391, 570, 1138, 1372, 1790, 488, 1780, 1179, 808, 153, 185, 1357, 712, 1391, 664, 682, 1367, 805, 225, 183, 125, 699, 25, 1187, 660, 865, 1597, 1213, 780, 858, 1745, 1013, 1394, 1652, 412, 97, 921, 1262, 149, 1778, 1419, 1738, 1598, 1649, 1313, 1470, 1712, 1045, 1235, 739, 986, 218, 1644, 605, 591, 1380, 721, 523, 1434, 34, 1251, 824, 1130, 263, 1521, 1090, 1246, 1250, 1651, 65, 103, 314, 1248, 1362, 108, 1661, 222, 1792, 326, 177, 480, 501, 1280, 1022, 88, 689, 1215, 509, 381, 284, 1331, 694, 574, 1377, 176, 975, 1146, 1121, 1374, 324, 1023, 1289, 1076, 1233, 161, 1425, 1047, 1588, 788, 1444, 905, 1438, 1666, 829, 571, 303, 659, 1624, 1686, 1216, 361, 155, 1157, 899, 1241, 252, 737, 1048, 1478, 1204, 14, 482, 969, 92 ], [ 927, 1331, 824, 1780, 947, 102, 81, 809, 1296, 1246, 946, 300, 721, 813, 1562, 185, 218, 388, 1073, 16, 1357, 1280, 1389, 659, 570, 225, 1597, 153, 1530, 1088, 836, 1335, 1587, 1097, 978, 1048, 944, 1313, 1641, 183, 1787, 874, 1235, 273, 1362, 763, 1138, 125, 1067, 682, 284, 88, 995, 754, 986, 1233, 435, 65, 697, 1626, 490, 571, 1241, 574, 1690, 1726, 338, 1541, 601, 1778, 1745, 558, 206, 482, 802, 161, 1639, 149, 1588, 825, 546, 1111, 1666, 631, 1686, 1712, 1271, 865, 775, 400, 1372, 1076, 1023, 1651, 1652, 1586, 661, 1673, 178, 1090, 1195, 1367, 157, 1456, 1268, 1517, 1661, 566, 495, 155, 1380, 92, 899, 168, 1282, 523, 858, 921, 107, 1391, 303, 264, 1459, 1013, 1262, 535, 808, 222, 1121, 1598, 230, 1419, 1250, 360, 675, 606, 34, 381, 456, 1146, 460, 1130, 975, 1718, 1169, 1022, 1725, 1791, 1716, 519, 501, 1084, 780, 254, 1216, 1251, 1490, 699, 1157, 689, 1093, 1630, 664, 882, 324, 969, 252, 1336, 1729, 756, 1377, 211, 770, 1521, 509, 737, 829, 137, 1688, 361, 1425, 1228, 176, 1792, 1703, 1047, 1045, 1404, 488, 782, 854, 1289, 712, 1374, 1738, 541, 1019, 1533, 314, 605, 177, 1434, 573, 1342, 1003, 1563, 480, 235, 1179, 905, 1496, 1204, 74, 244, 1624, 805, 340, 611, 1285, 1470, 1203, 41, 1644, 1777, 1176, 660, 1248, 391, 511, 739, 1550, 1394, 1215, 179, 412, 1614, 1478, 326, 1444, 583, 729, 14, 1149, 1715, 131, 97, 1438, 1363, 103, 743, 263, 1213, 1790, 25, 745, 694, 432, 788, 436, 122, 1649, 23, 296, 591, 1187, 1171, 423, 1741, 108 ], [ 570, 737, 721, 1372, 1624, 754, 1228, 1271, 1023, 1213, 1216, 161, 1335, 1614, 664, 157, 763, 326, 519, 501, 1792, 1157, 263, 206, 222, 975, 1791, 1419, 770, 1090, 273, 809, 1644, 921, 1389, 775, 423, 1246, 1716, 23, 824, 1289, 1588, 511, 1022, 183, 969, 660, 340, 92, 978, 400, 1204, 1203, 661, 460, 103, 1598, 1093, 1048, 854, 1478, 391, 1179, 836, 1434, 155, 1076, 435, 176, 1597, 689, 303, 1121, 1521, 631, 1729, 1715, 1195, 808, 264, 1138, 122, 1459, 1088, 284, 825, 490, 1533, 682, 1745, 743, 1313, 1187, 788, 482, 829, 1363, 81, 1013, 16, 436, 946, 137, 74, 659, 34, 102, 480, 1550, 1778, 456, 1470, 1241, 1342, 1073, 381, 1149, 230, 179, 1738, 905, 1562, 1391, 1262, 1626, 782, 558, 694, 1641, 1563, 1712, 1233, 1045, 1268, 601, 780, 211, 1790, 535, 488, 1718, 1357, 805, 41, 1282, 1651, 1111, 1067, 523, 185, 252, 1367, 882, 1725, 697, 1587, 1084, 756, 986, 412, 574, 606, 865, 131, 1703, 927, 1374, 1777, 509, 571, 1438, 605, 178, 591, 1169, 218, 1780, 573, 1686, 1331, 802, 1741, 108, 88, 314, 1496, 168, 541, 361, 338, 1097, 1146, 858, 944, 1047, 1690, 254, 97, 244, 546, 225, 296, 1248, 1235, 995, 177, 583, 1285, 1726, 1394, 1404, 745, 712, 107, 360, 739, 1673, 324, 1171, 1377, 813, 1296, 432, 1541, 388, 14, 1652, 1003, 149, 611, 1019, 729, 153, 1336, 1130, 65, 1649, 1444, 25, 1425, 1666, 235, 1639, 1380, 1456, 1280, 947, 1251, 675, 1530, 1661, 566, 1215, 1490, 300, 1517, 874, 1787, 899, 1362, 1688, 1250, 495, 699, 125, 1176, 1630, 1586 ], [ 1130, 739, 153, 122, 858, 1048, 1171, 788, 222, 743, 675, 1725, 1076, 1335, 1550, 1251, 400, 1651, 546, 23, 1362, 509, 1296, 338, 1562, 829, 1478, 14, 682, 1459, 1262, 721, 1425, 412, 92, 81, 125, 1438, 1176, 1204, 1652, 969, 1282, 808, 1792, 541, 1363, 927, 1444, 34, 1630, 712, 1530, 1639, 482, 805, 854, 1777, 1073, 571, 168, 423, 523, 157, 235, 211, 558, 1149, 825, 1215, 1285, 1342, 1394, 1780, 947, 591, 1391, 1626, 566, 946, 161, 1614, 1195, 1716, 1541, 1456, 495, 65, 501, 178, 1271, 1280, 606, 361, 511, 1703, 218, 975, 754, 1022, 775, 699, 1690, 1023, 865, 1729, 177, 102, 836, 1268, 1111, 296, 601, 763, 456, 1470, 300, 986, 1047, 995, 1404, 381, 303, 88, 1289, 1741, 729, 97, 388, 1377, 1533, 1233, 882, 391, 745, 183, 360, 149, 1246, 1649, 809, 1641, 1313, 1673, 225, 488, 1241, 230, 264, 107, 1666, 605, 1045, 1715, 1790, 1434, 103, 1084, 1686, 1090, 1357, 583, 1097, 460, 1419, 314, 480, 244, 694, 1003, 1228, 689, 185, 436, 155, 535, 1216, 1389, 1588, 1778, 1688, 1203, 1250, 263, 252, 1745, 1791, 1088, 756, 206, 1067, 1644, 1248, 813, 326, 340, 1169, 1586, 1235, 1490, 611, 944, 1013, 1336, 490, 41, 1179, 1093, 737, 660, 1146, 284, 273, 432, 176, 137, 664, 659, 574, 1712, 1587, 1138, 1019, 254, 1517, 1563, 631, 435, 324, 1597, 1367, 782, 1331, 921, 899, 802, 661, 874, 780, 1374, 1121, 1787, 1521, 1661, 1598, 25, 1187, 179, 1738, 824, 905, 1726, 573, 1496, 570, 131, 519, 1213, 1380, 16, 978, 74, 1372, 108, 1718, 697, 1157, 770, 1624 ], [ 1241, 858, 263, 1614, 1521, 697, 1130, 1374, 1686, 1149, 1438, 177, 1335, 1624, 591, 946, 1741, 721, 1367, 1673, 1690, 480, 1530, 1171, 1268, 1470, 1216, 1718, 1496, 1251, 1377, 1138, 1563, 1097, 865, 1047, 14, 1562, 1228, 570, 1203, 314, 1296, 235, 1404, 1651, 601, 699, 1215, 1023, 1777, 1342, 340, 1111, 1280, 1716, 501, 360, 131, 391, 1792, 944, 1336, 1726, 81, 1389, 1246, 1204, 1597, 566, 1490, 324, 183, 1179, 541, 659, 1045, 125, 1271, 254, 423, 756, 809, 1363, 176, 682, 1738, 1394, 155, 300, 519, 1550, 97, 252, 178, 338, 927, 712, 836, 1688, 488, 1146, 1289, 1090, 1725, 137, 1586, 1073, 1641, 460, 788, 1282, 185, 825, 511, 34, 573, 780, 432, 1195, 1712, 1088, 284, 230, 1666, 1084, 664, 1357, 1639, 435, 1380, 660, 1176, 1169, 805, 153, 854, 168, 326, 1076, 509, 222, 361, 107, 1372, 264, 1517, 244, 611, 1745, 1213, 1456, 102, 1729, 65, 605, 899, 631, 1233, 689, 495, 456, 1478, 1598, 92, 388, 296, 969, 986, 571, 743, 400, 1425, 1235, 694, 1250, 1703, 775, 546, 675, 661, 824, 829, 1187, 108, 1790, 1019, 273, 1533, 225, 947, 978, 802, 206, 606, 1778, 782, 303, 1588, 482, 1419, 1067, 808, 41, 523, 23, 739, 25, 1121, 1787, 490, 754, 1391, 574, 1644, 737, 1048, 122, 179, 1649, 770, 874, 995, 813, 1652, 1285, 1780, 74, 535, 905, 436, 1093, 1459, 1661, 16, 1003, 1541, 1791, 1715, 218, 1587, 975, 921, 1630, 745, 211, 558, 882, 1313, 1013, 1248, 1434, 1262, 412, 157, 161, 1157, 149, 1626, 88, 103, 1331, 1022, 583, 381, 763, 1362, 729, 1444 ], [ 41, 25, 1335, 1013, 1285, 34, 153, 865, 519, 480, 1084, 1187, 324, 660, 1745, 1673, 737, 591, 252, 1367, 754, 1649, 218, 1022, 1216, 131, 836, 1470, 1262, 1541, 1235, 770, 1313, 1703, 1787, 809, 824, 1741, 501, 1374, 606, 65, 490, 780, 183, 432, 161, 1047, 1097, 1459, 400, 1331, 874, 1067, 1686, 986, 14, 882, 802, 682, 1213, 1171, 1073, 1563, 74, 1280, 1419, 1296, 921, 1204, 1251, 1111, 178, 1176, 675, 975, 745, 16, 775, 125, 1644, 808, 235, 854, 1149, 176, 1517, 1377, 296, 694, 1088, 300, 1614, 107, 756, 1533, 858, 103, 944, 1490, 659, 1019, 222, 482, 1372, 1048, 1716, 1268, 1521, 436, 254, 829, 697, 435, 177, 721, 1641, 813, 412, 825, 179, 1146, 108, 571, 1550, 664, 947, 1791, 460, 1688, 1562, 805, 946, 1394, 1661, 264, 541, 1045, 1434, 763, 1425, 273, 574, 558, 1651, 1777, 689, 338, 1233, 495, 1157, 92, 1712, 969, 1456, 314, 1715, 1093, 1250, 97, 1241, 927, 137, 1725, 423, 1597, 1624, 1404, 566, 535, 1342, 340, 225, 611, 1444, 782, 1130, 661, 1587, 788, 511, 1246, 1271, 230, 699, 1778, 168, 303, 1792, 1729, 631, 1023, 391, 149, 1138, 244, 1228, 1090, 381, 23, 1169, 1780, 1391, 1666, 1588, 1215, 456, 1586, 583, 899, 326, 1336, 1639, 88, 739, 360, 729, 546, 1357, 1738, 605, 1380, 1438, 488, 1389, 211, 1121, 263, 905, 361, 1363, 743, 1003, 1076, 1203, 978, 1195, 122, 185, 1248, 1726, 81, 1530, 102, 1179, 1478, 1289, 155, 523, 1362, 509, 1282, 388, 1718, 570, 601, 1630, 1626, 206, 1652, 573, 995, 157, 1496, 712, 1790, 284, 1598, 1690 ], [ 775, 1586, 1179, 509, 436, 1391, 303, 432, 1389, 131, 1045, 206, 176, 1195, 756, 969, 805, 1235, 482, 1019, 149, 340, 523, 601, 921, 1521, 763, 573, 675, 1335, 423, 122, 324, 361, 824, 74, 1023, 501, 125, 1248, 1790, 1268, 1088, 809, 412, 108, 456, 211, 1380, 225, 1372, 1686, 802, 1251, 1048, 1725, 1624, 1138, 88, 230, 16, 254, 103, 178, 1204, 1470, 1729, 813, 23, 825, 1745, 905, 1651, 606, 519, 1639, 511, 1073, 263, 1215, 611, 1641, 244, 1404, 300, 218, 566, 338, 874, 155, 1250, 1246, 1121, 944, 745, 1792, 177, 1093, 1357, 381, 788, 583, 1444, 1666, 1690, 1157, 739, 1280, 1342, 927, 314, 1022, 107, 1649, 1216, 25, 41, 1419, 780, 1541, 1084, 1434, 1130, 1741, 882, 1614, 183, 1726, 1149, 535, 661, 65, 1780, 495, 1563, 1517, 1003, 558, 14, 1688, 986, 1626, 273, 1367, 1738, 1111, 1716, 697, 1777, 854, 488, 1712, 1630, 326, 659, 460, 571, 1187, 1490, 1313, 296, 1213, 1090, 1791, 1047, 1588, 1598, 1233, 168, 1331, 1718, 770, 391, 660, 102, 1478, 1459, 865, 1146, 1296, 978, 1241, 400, 631, 591, 34, 157, 541, 1228, 1597, 682, 235, 1169, 546, 179, 737, 1377, 574, 712, 699, 81, 435, 1715, 829, 1438, 1282, 975, 1425, 947, 1203, 1787, 694, 1652, 754, 97, 137, 570, 1076, 1394, 284, 1271, 899, 664, 1067, 689, 1097, 264, 1336, 1530, 1703, 782, 360, 222, 729, 1550, 1673, 836, 1362, 161, 858, 490, 1363, 1013, 946, 1496, 1285, 1778, 1533, 1562, 605, 1661, 721, 480, 808, 388, 995, 1289, 1176, 1374, 1456, 1587, 1262, 153, 743, 1644, 92, 252, 1171, 185 ], [ 1121, 1235, 1787, 423, 1661, 412, 264, 1745, 1394, 1550, 153, 1022, 1690, 125, 570, 808, 1686, 805, 737, 1703, 1490, 1673, 1377, 1725, 699, 1517, 1726, 1313, 1715, 1262, 161, 969, 1130, 1374, 177, 1587, 1342, 14, 1067, 178, 986, 1530, 488, 1777, 211, 1084, 1076, 788, 296, 1791, 179, 1666, 490, 218, 511, 1614, 591, 682, 103, 1438, 1738, 546, 1149, 206, 631, 97, 1741, 1296, 975, 1790, 400, 1336, 81, 324, 460, 157, 921, 254, 775, 694, 432, 605, 176, 829, 16, 340, 1268, 1644, 1171, 1639, 1215, 745, 391, 1023, 601, 480, 107, 149, 1250, 300, 88, 606, 978, 1282, 1597, 1652, 235, 360, 501, 435, 809, 1090, 65, 1203, 1419, 108, 1624, 743, 1048, 947, 1434, 802, 523, 661, 858, 899, 1335, 664, 92, 675, 754, 770, 1241, 825, 1157, 1459, 155, 1470, 519, 611, 284, 41, 836, 1496, 1362, 122, 944, 1251, 1729, 361, 1598, 573, 1285, 1233, 102, 1641, 1718, 729, 252, 739, 244, 1649, 1228, 1248, 225, 1391, 865, 1013, 314, 1169, 1712, 1093, 689, 574, 1521, 1716, 1179, 874, 1630, 763, 1216, 1331, 660, 854, 74, 1357, 1088, 1045, 1187, 185, 1176, 1444, 1213, 1047, 436, 1651, 558, 273, 1456, 780, 326, 824, 995, 1246, 509, 1688, 1792, 927, 303, 1073, 882, 583, 25, 1586, 1146, 34, 946, 697, 1588, 222, 1363, 23, 456, 1367, 1195, 482, 1563, 813, 338, 1138, 137, 782, 1478, 659, 1562, 495, 1271, 1111, 1533, 1204, 1778, 571, 168, 1404, 1626, 1425, 535, 721, 541, 1389, 756, 1780, 1019, 381, 1003, 388, 183, 905, 263, 1541, 1280, 230, 712, 1372, 1380, 566, 131, 1289, 1097 ] ], "bootstrap_indices_sha256": "80e8bc588ffbf216f5d9d419f360579450ba450e1988ffb0ef22094c7906e2e1", "runs": [ { "seed": 11, "size": 20, "repeat": 0, "calibration_ids": [ 103, 1336, 326, 611, 177, 495, 570, 1246, 303, 222, 858, 157, 97, 546, 1093, 606, 737, 1419, 1179, 583 ], "class_counts": [ 2, 4, 2, 3, 2, 1, 4, 2, 0, 0 ], "calibration_errors": 1, "temperature": 0.9077335812394969, "at_boundary": false, "fit_nll_before": 0.08468785713121271, "fit_nll_after": 0.08415438912737788, "test_nll": 0.1690158604634602, "test_ece": 0.02478658734445734, "test_accuracy": 0.9583333333333334, "delta_nll": 0.009306464403776388 }, { "seed": 11, "size": 20, "repeat": 1, "calibration_ids": [ 1335, 969, 1296, 782, 391, 1233, 1362, 1780, 412, 179, 1176, 480, 273, 435, 1073, 490, 802, 314, 1048, 381 ], "class_counts": [ 3, 0, 2, 0, 0, 1, 6, 3, 1, 4 ], "calibration_errors": 1, "temperature": 2.159320273244335, "at_boundary": false, "fit_nll_before": 0.39204068963631505, "fit_nll_after": 0.2547476159105434, "test_nll": 0.1747215869285188, "test_ece": 0.05736271552869865, "test_accuracy": 0.9583333333333334, "delta_nll": 0.015012190868834985 }, { "seed": 11, "size": 20, "repeat": 2, "calibration_ids": [ 739, 775, 802, 1459, 1023, 1374, 675, 1745, 1215, 1716, 1729, 1362, 1233, 780, 423, 1712, 1045, 225, 1241, 659 ], "class_counts": [ 2, 1, 2, 3, 4, 0, 3, 1, 2, 2 ], "calibration_errors": 2, "temperature": 1.2215276088736542, "at_boundary": false, "fit_nll_before": 0.2592328574491535, "fit_nll_after": 0.2545819905283183, "test_nll": 0.1470486944930506, "test_ece": 0.016691596054311444, "test_accuracy": 0.9583333333333334, "delta_nll": -0.012660701566633209 }, { "seed": 11, "size": 20, "repeat": 3, "calibration_ids": [ 157, 300, 519, 436, 185, 1179, 178, 273, 264, 1363, 1169, 1090, 495, 1729, 1313, 88, 660, 107, 1047, 81 ], "class_counts": [ 2, 1, 3, 3, 1, 1, 2, 4, 3, 0 ], "calibration_errors": 1, "temperature": 1.2420826466174961, "at_boundary": false, "fit_nll_before": 0.11476184364955533, "fit_nll_after": 0.11128652709707257, "test_nll": 0.1464160489558031, "test_ece": 0.016035332952932642, "test_accuracy": 0.9583333333333334, "delta_nll": -0.01329334710388072 }, { "seed": 11, "size": 20, "repeat": 4, "calibration_ids": [ 1362, 1777, 1787, 1138, 1176, 1048, 1690, 1715, 81, 809, 1251, 1045, 511, 501, 65, 1289, 1703, 805, 1067, 1169 ], "class_counts": [ 2, 0, 4, 1, 2, 2, 2, 2, 1, 4 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.012839497888153945, "fit_nll_after": 1.7328003963679371e-18, "test_nll": 2.60325235892821, "test_ece": 0.0402650500095188, "test_accuracy": 0.9583333333333334, "delta_nll": 2.443542962868526 }, { "seed": 11, "size": 20, "repeat": 5, "calibration_ids": [ 1268, 1649, 511, 1363, 946, 1456, 770, 456, 1023, 739, 211, 802, 179, 1215, 1250, 1076, 1521, 1715, 41, 805 ], "class_counts": [ 2, 4, 0, 0, 6, 0, 3, 2, 1, 2 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.008937598056967784, "fit_nll_after": 1.0911080549710869e-22, "test_nll": 2.60325235892821, "test_ece": 0.0402650500095188, "test_accuracy": 0.9583333333333334, "delta_nll": 2.443542962868526 }, { "seed": 11, "size": 20, "repeat": 6, "calibration_ids": [ 788, 1331, 1651, 729, 523, 780, 81, 436, 1598, 218, 1176, 25, 168, 1673, 969, 535, 423, 211, 1013, 1003 ], "class_counts": [ 1, 1, 2, 1, 2, 2, 2, 5, 1, 3 ], "calibration_errors": 1, "temperature": 1.2714880703090718, "at_boundary": false, "fit_nll_before": 0.10945180810315394, "fit_nll_after": 0.10515973614583869, "test_nll": 0.14564559613979292, "test_ece": 0.015458605884365715, "test_accuracy": 0.9583333333333334, "delta_nll": -0.014063799919890901 }, { "seed": 11, "size": 20, "repeat": 7, "calibration_ids": [ 456, 23, 296, 1157, 1271, 664, 739, 743, 1791, 1787, 605, 125, 1444, 1203, 1587, 1651, 1171, 591, 1213, 1023 ], "class_counts": [ 1, 3, 1, 2, 5, 2, 0, 1, 3, 2 ], "calibration_errors": 1, "temperature": 1.8998078435194785, "at_boundary": false, "fit_nll_before": 0.30855049146269053, "fit_nll_after": 0.21899107622562877, "test_nll": 0.15780626591945593, "test_ece": 0.03823674281572128, "test_accuracy": 0.9583333333333334, "delta_nll": -0.0019031301402278877 }, { "seed": 11, "size": 20, "repeat": 8, "calibration_ids": [ 1235, 1228, 1013, 1745, 1666, 1718, 495, 185, 1478, 1587, 460, 1738, 300, 660, 882, 412, 324, 103, 573, 1362 ], "class_counts": [ 3, 0, 3, 2, 2, 4, 2, 3, 1, 0 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.02254324030111625, "fit_nll_after": 1.362175417548919e-14, "test_nll": 2.60325235892821, "test_ece": 0.0402650500095188, "test_accuracy": 0.9583333333333334, "delta_nll": 2.443542962868526 }, { "seed": 11, "size": 20, "repeat": 9, "calibration_ids": [ 1146, 694, 244, 1138, 1787, 675, 1686, 412, 788, 326, 225, 1791, 660, 391, 1630, 1084, 1013, 1652, 381, 1271 ], "class_counts": [ 1, 1, 3, 1, 5, 2, 1, 1, 1, 4 ], "calibration_errors": 1, "temperature": 1.8824079463431787, "at_boundary": false, "fit_nll_before": 0.2866780179466567, "fit_nll_after": 0.2035302559601954, "test_nll": 0.15688067126507665, "test_ece": 0.03670557438071012, "test_accuracy": 0.9583333333333334, "delta_nll": -0.0028287247946071625 }, { "seed": 11, "size": 20, "repeat": 10, "calibration_ids": [ 264, 1013, 108, 361, 1076, 432, 1268, 858, 1661, 1022, 338, 874, 824, 1726, 340, 1780, 882, 969, 1289, 1791 ], "class_counts": [ 1, 1, 3, 2, 4, 1, 4, 2, 1, 1 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.006938579232905443, "fit_nll_after": 3.3477726847035673e-26, "test_nll": 2.60325235892821, "test_ece": 0.0402650500095188, "test_accuracy": 0.9583333333333334, "delta_nll": 2.443542962868526 }, { "seed": 11, "size": 20, "repeat": 11, "calibration_ids": [ 675, 605, 1084, 927, 1362, 388, 155, 763, 1651, 1394, 1630, 303, 1530, 541, 546, 149, 1490, 1250, 682, 381 ], "class_counts": [ 2, 3, 6, 2, 1, 3, 0, 0, 0, 3 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.030739207749348445, "fit_nll_after": 0.0007439797275835948, "test_nll": 2.60325235892821, "test_ece": 0.0402650500095188, "test_accuracy": 0.9583333333333334, "delta_nll": 2.443542962868526 }, { "seed": 11, "size": 20, "repeat": 12, "calibration_ids": [ 1673, 1787, 1587, 1097, 74, 729, 1630, 41, 1335, 1171, 978, 273, 946, 675, 1741, 1729, 541, 743, 179, 1550 ], "class_counts": [ 2, 1, 0, 3, 4, 5, 1, 2, 1, 1 ], "calibration_errors": 1, "temperature": 1.2389777327635503, "at_boundary": false, "fit_nll_before": 0.10726765826539278, "fit_nll_after": 0.1031404066333254, "test_nll": 0.14650655887062597, "test_ece": 0.01613570621859198, "test_accuracy": 0.9583333333333334, "delta_nll": -0.013202837189057848 }, { "seed": 11, "size": 20, "repeat": 13, "calibration_ids": [ 927, 1331, 824, 1780, 947, 102, 81, 809, 1296, 1246, 946, 300, 721, 813, 1562, 185, 218, 388, 1073, 16 ], "class_counts": [ 2, 2, 4, 1, 1, 2, 1, 4, 0, 3 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.015185678889250806, "fit_nll_after": 1.3287396163916566e-12, "test_nll": 2.60325235892821, "test_ece": 0.0402650500095188, "test_accuracy": 0.9583333333333334, "delta_nll": 2.443542962868526 }, { "seed": 11, "size": 20, "repeat": 14, "calibration_ids": [ 570, 737, 721, 1372, 1624, 754, 1228, 1271, 1023, 1213, 1216, 161, 1335, 1614, 664, 157, 763, 326, 519, 501 ], "class_counts": [ 1, 3, 3, 4, 1, 3, 0, 2, 2, 1 ], "calibration_errors": 1, "temperature": 1.9217939592261142, "at_boundary": false, "fit_nll_before": 0.28957362947413284, "fit_nll_after": 0.20501812261880895, "test_nll": 0.15901544228661552, "test_ece": 0.03811199656967788, "test_accuracy": 0.9583333333333334, "delta_nll": -0.000693953773068301 }, { "seed": 11, "size": 20, "repeat": 15, "calibration_ids": [ 1130, 739, 153, 122, 858, 1048, 1171, 788, 222, 743, 675, 1725, 1076, 1335, 1550, 1251, 400, 1651, 546, 23 ], "class_counts": [ 2, 2, 1, 3, 4, 1, 2, 2, 1, 2 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.0015514559768927088, "fit_nll_after": 7.408599054015548e-42, "test_nll": 2.60325235892821, "test_ece": 0.0402650500095188, "test_accuracy": 0.9583333333333334, "delta_nll": 2.443542962868526 }, { "seed": 11, "size": 20, "repeat": 16, "calibration_ids": [ 1241, 858, 263, 1614, 1521, 697, 1130, 1374, 1686, 1149, 1438, 177, 1335, 1624, 591, 946, 1741, 721, 1367, 1673 ], "class_counts": [ 1, 2, 4, 3, 2, 2, 3, 1, 1, 1 ], "calibration_errors": 1, "temperature": 2.301394974597253, "at_boundary": false, "fit_nll_before": 0.5271766416691895, "fit_nll_after": 0.35096657386151886, "test_nll": 0.18622392929306134, "test_ece": 0.06907370636849991, "test_accuracy": 0.9583333333333334, "delta_nll": 0.02651453323337752 }, { "seed": 11, "size": 20, "repeat": 17, "calibration_ids": [ 41, 25, 1335, 1013, 1285, 34, 153, 865, 519, 480, 1084, 1187, 324, 660, 1745, 1673, 737, 591, 252, 1367 ], "class_counts": [ 4, 1, 2, 4, 3, 1, 2, 2, 0, 1 ], "calibration_errors": 1, "temperature": 2.0983327314345157, "at_boundary": false, "fit_nll_before": 0.3842088116588496, "fit_nll_after": 0.25634991411459584, "test_nll": 0.17024825903071447, "test_ece": 0.0543999345275793, "test_accuracy": 0.9583333333333334, "delta_nll": 0.010538862971030655 }, { "seed": 11, "size": 20, "repeat": 18, "calibration_ids": [ 775, 1586, 1179, 509, 436, 1391, 303, 432, 1389, 131, 1045, 206, 176, 1195, 756, 969, 805, 1235, 482, 1019 ], "class_counts": [ 1, 2, 2, 0, 1, 3, 3, 2, 5, 1 ], "calibration_errors": 1, "temperature": 1.0021388357548107, "at_boundary": false, "fit_nll_before": 0.10813837799728594, "fit_nll_after": 0.10813802423526846, "test_nll": 0.15952681637035523, "test_ece": 0.021131705988904518, "test_accuracy": 0.9583333333333334, "delta_nll": -0.00018257968932858248 }, { "seed": 11, "size": 20, "repeat": 19, "calibration_ids": [ 1121, 1235, 1787, 423, 1661, 412, 264, 1745, 1394, 1550, 153, 1022, 1690, 125, 570, 808, 1686, 805, 737, 1703 ], "class_counts": [ 3, 1, 1, 3, 2, 3, 1, 1, 1, 4 ], "calibration_errors": 1, "temperature": 0.8497069164637774, "at_boundary": false, "fit_nll_before": 0.08953567511549124, "fit_nll_after": 0.08851307004161603, "test_nll": 0.17651983081750053, "test_ece": 0.026585435662736193, "test_accuracy": 0.9583333333333334, "delta_nll": 0.01681043475781671 }, { "seed": 11, "size": 50, "repeat": 0, "calibration_ids": [ 103, 1336, 326, 611, 177, 495, 570, 1246, 303, 222, 858, 157, 97, 546, 1093, 606, 737, 1419, 1179, 583, 947, 1121, 183, 1372, 436, 1146, 131, 456, 460, 340, 1490, 697, 16, 273, 400, 324, 153, 381, 1251, 825, 1313, 865, 1171, 1067, 74, 1342, 108, 388, 1047, 1380 ], "class_counts": [ 4, 9, 7, 5, 5, 3, 6, 6, 3, 2 ], "calibration_errors": 1, "temperature": 0.7476833045567588, "at_boundary": false, "fit_nll_before": 0.04023780246326243, "fit_nll_after": 0.03703675084654251, "test_nll": 0.19372035516297087, "test_ece": 0.029441027315559042, "test_accuracy": 0.9583333333333334, "delta_nll": 0.03401095910328705 }, { "seed": 11, "size": 50, "repeat": 1, "calibration_ids": [ 1335, 969, 1296, 782, 391, 1233, 1362, 1780, 412, 179, 1176, 480, 273, 435, 1073, 490, 802, 314, 1048, 381, 235, 183, 326, 108, 1586, 944, 1738, 296, 809, 946, 1380, 1367, 300, 185, 1726, 1644, 1425, 1725, 1550, 1703, 1280, 882, 1342, 659, 1336, 535, 1076, 1216, 1741, 1203 ], "class_counts": [ 8, 5, 2, 4, 1, 6, 8, 6, 4, 6 ], "calibration_errors": 1, "temperature": 1.7227012795788006, "at_boundary": false, "fit_nll_before": 0.15842536913529595, "fit_nll_after": 0.12040953482154595, "test_nll": 0.14974558323614912, "test_ece": 0.026289951145747927, "test_accuracy": 0.9583333333333334, "delta_nll": -0.009963812823534701 }, { "seed": 11, "size": 50, "repeat": 2, "calibration_ids": [ 739, 775, 802, 1459, 1023, 1374, 675, 1745, 1215, 1716, 1729, 1362, 1233, 780, 423, 1712, 1045, 225, 1241, 659, 1090, 1246, 566, 605, 921, 699, 546, 1363, 978, 490, 222, 1521, 1377, 1550, 361, 1121, 986, 1013, 14, 1562, 570, 1587, 1203, 1336, 1342, 611, 179, 1661, 1496, 899 ], "class_counts": [ 6, 2, 5, 6, 6, 2, 7, 6, 6, 4 ], "calibration_errors": 2, "temperature": 1.0211910885444833, "at_boundary": false, "fit_nll_before": 0.1319259677343164, "fit_nll_after": 0.13189824572136394, "test_nll": 0.1579604789708331, "test_ece": 0.02046050077108139, "test_accuracy": 0.9583333333333334, "delta_nll": -0.001748917088850721 }, { "seed": 11, "size": 50, "repeat": 3, "calibration_ids": [ 157, 300, 519, 436, 185, 1179, 178, 273, 264, 1363, 1169, 1090, 495, 1729, 1313, 88, 660, 107, 1047, 81, 176, 1067, 263, 1121, 1377, 611, 254, 1003, 1404, 1246, 488, 1597, 1792, 1331, 1652, 340, 1686, 825, 1563, 1478, 1712, 161, 591, 1296, 1790, 865, 874, 1213, 1216, 222 ], "class_counts": [ 4, 3, 6, 9, 2, 5, 3, 8, 5, 5 ], "calibration_errors": 2, "temperature": 1.0704897324620493, "at_boundary": false, "fit_nll_before": 0.070914945164912, "fit_nll_after": 0.07068658014255003, "test_nll": 0.15438125446354367, "test_ece": 0.020747193422759273, "test_accuracy": 0.9583333333333334, "delta_nll": -0.005328141596140146 }, { "seed": 11, "size": 50, "repeat": 4, "calibration_ids": [ 1362, 1777, 1787, 1138, 1176, 1048, 1690, 1715, 81, 809, 1251, 1045, 511, 501, 65, 1289, 1703, 805, 1067, 1169, 689, 763, 782, 1438, 756, 1250, 836, 788, 1530, 381, 905, 1367, 1394, 1651, 1478, 1271, 854, 1171, 721, 570, 566, 244, 1673, 1496, 1630, 1644, 179, 235, 1228, 211 ], "class_counts": [ 4, 4, 10, 6, 5, 4, 4, 5, 3, 5 ], "calibration_errors": 2, "temperature": 1.8518292643784184, "at_boundary": false, "fit_nll_before": 0.24044258259086587, "fit_nll_after": 0.17373795291721733, "test_nll": 0.1553223044611966, "test_ece": 0.034688271588754704, "test_accuracy": 0.9583333333333334, "delta_nll": -0.004387091598487214 }, { "seed": 11, "size": 50, "repeat": 5, "calibration_ids": [ 1268, 1649, 511, 1363, 946, 1456, 770, 456, 1023, 739, 211, 802, 179, 1215, 1250, 1076, 1521, 1715, 41, 805, 865, 177, 176, 157, 400, 206, 675, 1496, 1588, 699, 92, 183, 185, 284, 1688, 1745, 606, 1045, 1777, 97, 1090, 1703, 1233, 25, 1587, 1533, 1213, 721, 1741, 361 ], "class_counts": [ 5, 8, 1, 3, 9, 3, 4, 6, 6, 5 ], "calibration_errors": 2, "temperature": 0.9508365350120909, "at_boundary": false, "fit_nll_before": 0.07715785513409393, "fit_nll_after": 0.07698909536510849, "test_nll": 0.1643043570837989, "test_ece": 0.023368124319011595, "test_accuracy": 0.9583333333333334, "delta_nll": 0.0045949610241150796 }, { "seed": 11, "size": 50, "repeat": 6, "calibration_ids": [ 788, 1331, 1651, 729, 523, 780, 81, 436, 1598, 218, 1176, 25, 168, 1673, 969, 535, 423, 211, 1013, 1003, 1639, 1715, 1251, 1204, 1336, 1718, 1394, 88, 1241, 412, 284, 664, 1363, 1541, 1652, 264, 519, 1233, 809, 1729, 1738, 836, 254, 361, 1195, 858, 65, 802, 16, 314 ], "class_counts": [ 4, 3, 4, 5, 3, 4, 9, 6, 7, 5 ], "calibration_errors": 2, "temperature": 1.0122544607864752, "at_boundary": false, "fit_nll_before": 0.10604366694227343, "fit_nll_after": 0.10603351218351537, "test_nll": 0.1586818751454223, "test_ece": 0.020777092362383523, "test_accuracy": 0.9583333333333334, "delta_nll": -0.0010275209142615038 }, { "seed": 11, "size": 50, "repeat": 7, "calibration_ids": [ 456, 23, 296, 1157, 1271, 664, 739, 743, 1791, 1787, 605, 125, 1444, 1203, 1587, 1651, 1171, 591, 1213, 1023, 1282, 1331, 252, 571, 1456, 225, 1076, 825, 179, 263, 168, 611, 1438, 566, 1215, 1639, 176, 541, 460, 1530, 1251, 1517, 805, 1541, 1716, 780, 1313, 1248, 381, 1459 ], "class_counts": [ 7, 4, 3, 5, 8, 6, 3, 5, 4, 5 ], "calibration_errors": 2, "temperature": 1.7285961238099858, "at_boundary": false, "fit_nll_before": 0.16513549923289453, "fit_nll_after": 0.1286169028451278, "test_nll": 0.14996336040203206, "test_ece": 0.025391642691080615, "test_accuracy": 0.9583333333333334, "delta_nll": -0.009746035657651753 }, { "seed": 11, "size": 50, "repeat": 8, "calibration_ids": [ 1235, 1228, 1013, 1745, 1666, 1718, 495, 185, 1478, 1587, 460, 1738, 300, 660, 882, 412, 324, 103, 573, 1362, 263, 1067, 1187, 206, 1597, 1146, 1130, 1149, 65, 1389, 1716, 235, 1652, 1357, 222, 995, 1586, 108, 1490, 605, 1246, 809, 1778, 14, 1641, 1363, 1649, 1391, 1726, 541 ], "class_counts": [ 5, 2, 5, 6, 6, 7, 4, 9, 5, 1 ], "calibration_errors": 2, "temperature": 1.7822994063765558, "at_boundary": false, "fit_nll_before": 0.2650593418959691, "fit_nll_after": 0.21391938674735023, "test_nll": 0.15211250687339117, "test_ece": 0.028478014842798553, "test_accuracy": 0.9583333333333334, "delta_nll": -0.0075968891862926435 }, { "seed": 11, "size": 50, "repeat": 9, "calibration_ids": [ 1146, 694, 244, 1138, 1787, 675, 1686, 412, 788, 326, 225, 1791, 660, 391, 1630, 1084, 1013, 1652, 381, 1271, 712, 284, 1296, 1216, 423, 986, 436, 1587, 177, 975, 1121, 65, 1778, 1342, 1235, 1434, 273, 780, 1391, 978, 1019, 360, 1130, 1729, 108, 523, 432, 1777, 338, 829 ], "class_counts": [ 3, 2, 5, 4, 8, 3, 6, 7, 4, 8 ], "calibration_errors": 3, "temperature": 1.6195286470509058, "at_boundary": false, "fit_nll_before": 0.20282821498079362, "fit_nll_after": 0.17054004724861266, "test_nll": 0.14654182877082045, "test_ece": 0.020912699334072006, "test_accuracy": 0.9583333333333334, "delta_nll": -0.013167567288863369 }, { "seed": 11, "size": 50, "repeat": 10, "calibration_ids": [ 264, 1013, 108, 361, 1076, 432, 1268, 858, 1661, 1022, 338, 874, 824, 1726, 340, 1780, 882, 969, 1289, 1791, 263, 1377, 1171, 1688, 488, 1778, 944, 1363, 157, 81, 1738, 230, 1626, 1391, 1588, 1434, 1517, 1718, 179, 946, 802, 546, 921, 1716, 660, 1048, 1187, 1587, 1146, 273 ], "class_counts": [ 5, 4, 4, 3, 8, 5, 7, 7, 2, 5 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.006250009761431093, "fit_nll_after": 3.4947295692295954e-22, "test_nll": 2.60325235892821, "test_ece": 0.0402650500095188, "test_accuracy": 0.9583333333333334, "delta_nll": 2.443542962868526 }, { "seed": 11, "size": 50, "repeat": 11, "calibration_ids": [ 675, 605, 1084, 927, 1362, 388, 155, 763, 1651, 1394, 1630, 303, 1530, 541, 546, 149, 1490, 1250, 682, 381, 694, 1550, 1444, 1157, 1626, 754, 432, 1019, 1271, 1726, 1067, 490, 583, 74, 235, 1716, 34, 743, 745, 206, 1169, 1342, 1262, 1216, 535, 456, 1363, 1391, 391, 177 ], "class_counts": [ 6, 8, 7, 4, 2, 7, 4, 2, 4, 6 ], "calibration_errors": 2, "temperature": 1.4615467734637477, "at_boundary": false, "fit_nll_before": 0.16689476959640576, "fit_nll_after": 0.14815552145051428, "test_nll": 0.14405760769437154, "test_ece": 0.017966100151122242, "test_accuracy": 0.9583333333333334, "delta_nll": -0.01565178836531228 }, { "seed": 11, "size": 50, "repeat": 12, "calibration_ids": [ 1673, 1787, 1587, 1097, 74, 729, 1630, 41, 1335, 1171, 978, 273, 946, 675, 1741, 1729, 541, 743, 179, 1550, 1563, 1195, 81, 360, 137, 1149, 1088, 1111, 802, 254, 16, 1688, 1404, 1716, 1626, 1342, 606, 296, 1296, 244, 882, 1456, 813, 1614, 340, 1203, 435, 388, 1791, 432 ], "class_counts": [ 6, 4, 4, 3, 6, 9, 5, 5, 4, 4 ], "calibration_errors": 3, "temperature": 1.7518094922475218, "at_boundary": false, "fit_nll_before": 0.28781785556884654, "fit_nll_after": 0.23559402138005983, "test_nll": 0.15085610087422732, "test_ece": 0.026708293945405876, "test_accuracy": 0.9583333333333334, "delta_nll": -0.008853295185456495 }, { "seed": 11, "size": 50, "repeat": 13, "calibration_ids": [ 927, 1331, 824, 1780, 947, 102, 81, 809, 1296, 1246, 946, 300, 721, 813, 1562, 185, 218, 388, 1073, 16, 1357, 1280, 1389, 659, 570, 225, 1597, 153, 1530, 1088, 836, 1335, 1587, 1097, 978, 1048, 944, 1313, 1641, 183, 1787, 874, 1235, 273, 1362, 763, 1138, 125, 1067, 682 ], "class_counts": [ 5, 4, 8, 5, 4, 5, 2, 7, 4, 6 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.011164255799990674, "fit_nll_after": 5.31496539676821e-13, "test_nll": 2.60325235892821, "test_ece": 0.0402650500095188, "test_accuracy": 0.9583333333333334, "delta_nll": 2.443542962868526 }, { "seed": 11, "size": 50, "repeat": 14, "calibration_ids": [ 570, 737, 721, 1372, 1624, 754, 1228, 1271, 1023, 1213, 1216, 161, 1335, 1614, 664, 157, 763, 326, 519, 501, 1792, 1157, 263, 206, 222, 975, 1791, 1419, 770, 1090, 273, 809, 1644, 921, 1389, 775, 423, 1246, 1716, 23, 824, 1289, 1588, 511, 1022, 183, 969, 660, 340, 92 ], "class_counts": [ 4, 3, 5, 9, 6, 5, 2, 5, 5, 6 ], "calibration_errors": 2, "temperature": 1.5134145006068, "at_boundary": false, "fit_nll_before": 0.16294824492528648, "fit_nll_after": 0.14210604070215374, "test_nll": 0.14452748175460844, "test_ece": 0.01658044595630981, "test_accuracy": 0.9583333333333334, "delta_nll": -0.015181914305075372 }, { "seed": 11, "size": 50, "repeat": 15, "calibration_ids": [ 1130, 739, 153, 122, 858, 1048, 1171, 788, 222, 743, 675, 1725, 1076, 1335, 1550, 1251, 400, 1651, 546, 23, 1362, 509, 1296, 338, 1562, 829, 1478, 14, 682, 1459, 1262, 721, 1425, 412, 92, 81, 125, 1438, 1176, 1204, 1652, 969, 1282, 808, 1792, 541, 1363, 927, 1444, 34 ], "class_counts": [ 4, 3, 4, 5, 6, 3, 6, 4, 4, 11 ], "calibration_errors": 1, "temperature": 0.7909691187956637, "at_boundary": false, "fit_nll_before": 0.03592240789248061, "fit_nll_after": 0.03473868455498714, "test_nll": 0.18571509414007104, "test_ece": 0.028277348293337275, "test_accuracy": 0.9583333333333334, "delta_nll": 0.02600569808038722 }, { "seed": 11, "size": 50, "repeat": 16, "calibration_ids": [ 1241, 858, 263, 1614, 1521, 697, 1130, 1374, 1686, 1149, 1438, 177, 1335, 1624, 591, 946, 1741, 721, 1367, 1673, 1690, 480, 1530, 1171, 1268, 1470, 1216, 1718, 1496, 1251, 1377, 1138, 1563, 1097, 865, 1047, 14, 1562, 1228, 570, 1203, 314, 1296, 235, 1404, 1651, 601, 699, 1215, 1023 ], "class_counts": [ 3, 6, 7, 6, 8, 5, 5, 4, 3, 3 ], "calibration_errors": 2, "temperature": 2.0327843770789262, "at_boundary": false, "fit_nll_before": 0.35924015813425697, "fit_nll_after": 0.2441827128312575, "test_nll": 0.16577104971492404, "test_ece": 0.05208607833530843, "test_accuracy": 0.9583333333333334, "delta_nll": 0.006061653655240223 }, { "seed": 11, "size": 50, "repeat": 17, "calibration_ids": [ 41, 25, 1335, 1013, 1285, 34, 153, 865, 519, 480, 1084, 1187, 324, 660, 1745, 1673, 737, 591, 252, 1367, 754, 1649, 218, 1022, 1216, 131, 836, 1470, 1262, 1541, 1235, 770, 1313, 1703, 1787, 809, 824, 1741, 501, 1374, 606, 65, 490, 780, 183, 432, 161, 1047, 1097, 1459 ], "class_counts": [ 9, 5, 3, 6, 7, 5, 5, 5, 2, 3 ], "calibration_errors": 3, "temperature": 1.6540659548464345, "at_boundary": false, "fit_nll_before": 0.22561228031628236, "fit_nll_after": 0.18632126484356543, "test_nll": 0.14748346875660334, "test_ece": 0.02134593249443676, "test_accuracy": 0.9583333333333334, "delta_nll": -0.012225927303080475 }, { "seed": 11, "size": 50, "repeat": 18, "calibration_ids": [ 775, 1586, 1179, 509, 436, 1391, 303, 432, 1389, 131, 1045, 206, 176, 1195, 756, 969, 805, 1235, 482, 1019, 149, 340, 523, 601, 921, 1521, 763, 573, 675, 1335, 423, 122, 324, 361, 824, 74, 1023, 501, 125, 1248, 1790, 1268, 1088, 809, 412, 108, 456, 211, 1380, 225 ], "class_counts": [ 3, 5, 5, 1, 5, 6, 6, 6, 7, 6 ], "calibration_errors": 2, "temperature": 0.9238254422831563, "at_boundary": false, "fit_nll_before": 0.07413618665652448, "fit_nll_after": 0.07380987737296522, "test_nll": 0.16717695663755933, "test_ece": 0.02426525120287749, "test_accuracy": 0.9583333333333334, "delta_nll": 0.007467560577875509 }, { "seed": 11, "size": 50, "repeat": 19, "calibration_ids": [ 1121, 1235, 1787, 423, 1661, 412, 264, 1745, 1394, 1550, 153, 1022, 1690, 125, 570, 808, 1686, 805, 737, 1703, 1490, 1673, 1377, 1725, 699, 1517, 1726, 1313, 1715, 1262, 161, 969, 1130, 1374, 177, 1587, 1342, 14, 1067, 178, 986, 1530, 488, 1777, 211, 1084, 1076, 788, 296, 1791 ], "class_counts": [ 6, 4, 6, 5, 6, 5, 5, 3, 4, 6 ], "calibration_errors": 1, "temperature": 0.6519612734543396, "at_boundary": false, "fit_nll_before": 0.043316142016423476, "fit_nll_after": 0.038664634269952376, "test_nll": 0.2161609012152549, "test_ece": 0.03176194990829963, "test_accuracy": 0.9583333333333334, "delta_nll": 0.0564515051555711 }, { "seed": 11, "size": 100, "repeat": 0, "calibration_ids": [ 103, 1336, 326, 611, 177, 495, 570, 1246, 303, 222, 858, 157, 97, 546, 1093, 606, 737, 1419, 1179, 583, 947, 1121, 183, 1372, 436, 1146, 131, 456, 460, 340, 1490, 697, 16, 273, 400, 324, 153, 381, 1251, 825, 1313, 865, 1171, 1067, 74, 1342, 108, 388, 1047, 1380, 743, 699, 1003, 300, 1425, 211, 122, 836, 905, 1285, 168, 1530, 775, 573, 1138, 488, 435, 1738, 41, 1048, 296, 14, 523, 92, 155, 694, 1586, 1644, 1792, 360, 571, 137, 1686, 81, 1470, 1362, 263, 1241, 1588, 161, 88, 235, 230, 745, 1716, 1703, 899, 501, 1250, 829 ], "class_counts": [ 11, 12, 12, 8, 9, 8, 8, 13, 11, 8 ], "calibration_errors": 2, "temperature": 1.2552558428333456, "at_boundary": false, "fit_nll_before": 0.09016738430766055, "fit_nll_after": 0.0859078547937725, "test_nll": 0.14605167011562295, "test_ece": 0.016117614388096297, "test_accuracy": 0.9583333333333334, "delta_nll": -0.013657725944060867 }, { "seed": 11, "size": 100, "repeat": 1, "calibration_ids": [ 1335, 969, 1296, 782, 391, 1233, 1362, 1780, 412, 179, 1176, 480, 273, 435, 1073, 490, 802, 314, 1048, 381, 235, 183, 326, 108, 1586, 944, 1738, 296, 809, 946, 1380, 1367, 300, 185, 1726, 1644, 1425, 1725, 1550, 1703, 1280, 882, 1342, 659, 1336, 535, 1076, 1216, 1741, 1203, 177, 1715, 905, 858, 1246, 263, 805, 97, 1282, 675, 25, 1090, 1084, 1313, 102, 1470, 1228, 682, 1444, 423, 168, 1419, 1377, 756, 697, 1649, 1157, 661, 155, 1639, 1215, 788, 1690, 1235, 1787, 978, 1652, 495, 1262, 92, 566, 921, 1490, 1562, 1121, 103, 340, 1651, 1716, 338 ], "class_counts": [ 14, 7, 8, 9, 6, 13, 13, 9, 7, 14 ], "calibration_errors": 2, "temperature": 1.6177499952519987, "at_boundary": false, "fit_nll_before": 0.14850305568167912, "fit_nll_after": 0.12020233654855186, "test_nll": 0.14649701255798628, "test_ece": 0.021037295929194438, "test_accuracy": 0.9583333333333334, "delta_nll": -0.013212383501697539 }, { "seed": 11, "size": 100, "repeat": 2, "calibration_ids": [ 739, 775, 802, 1459, 1023, 1374, 675, 1745, 1215, 1716, 1729, 1362, 1233, 780, 423, 1712, 1045, 225, 1241, 659, 1090, 1246, 566, 605, 921, 699, 546, 1363, 978, 490, 222, 1521, 1377, 1550, 361, 1121, 986, 1013, 14, 1562, 570, 1587, 1203, 1336, 1342, 611, 179, 1661, 1496, 899, 519, 412, 1718, 97, 1777, 1434, 273, 456, 1282, 1703, 1588, 558, 721, 177, 1130, 1313, 1726, 1725, 1644, 1357, 1019, 1022, 1111, 1251, 975, 1248, 34, 1235, 509, 1614, 23, 1478, 264, 1792, 161, 314, 1067, 149, 103, 1176, 244, 829, 1790, 1641, 400, 1688, 782, 743, 535, 1367 ], "class_counts": [ 8, 8, 9, 15, 12, 3, 13, 9, 11, 12 ], "calibration_errors": 3, "temperature": 0.82133378279915, "at_boundary": false, "fit_nll_before": 0.08722686457827247, "fit_nll_after": 0.085282141105105, "test_nll": 0.18074160095595232, "test_ece": 0.027961929380890958, "test_accuracy": 0.9583333333333334, "delta_nll": 0.021032204896268503 }, { "seed": 11, "size": 100, "repeat": 3, "calibration_ids": [ 157, 300, 519, 436, 185, 1179, 178, 273, 264, 1363, 1169, 1090, 495, 1729, 1313, 88, 660, 107, 1047, 81, 176, 1067, 263, 1121, 1377, 611, 254, 1003, 1404, 1246, 488, 1597, 1792, 1331, 1652, 340, 1686, 825, 1563, 1478, 1712, 161, 591, 1296, 1790, 865, 874, 1213, 1216, 222, 1470, 1490, 1271, 1716, 16, 1688, 1438, 1521, 1587, 858, 1073, 103, 125, 975, 729, 1285, 137, 802, 1673, 739, 284, 946, 1726, 230, 1434, 1171, 1703, 854, 1459, 1533, 1149, 1389, 1394, 882, 1335, 808, 697, 1690, 1787, 813, 1262, 1342, 1157, 432, 149, 605, 1419, 131, 400, 1138 ], "class_counts": [ 11, 7, 8, 15, 7, 10, 9, 13, 8, 12 ], "calibration_errors": 5, "temperature": 1.6560035811552378, "at_boundary": false, "fit_nll_before": 0.2219420872375282, "fit_nll_after": 0.18564255352403225, "test_nll": 0.14754027446168402, "test_ece": 0.021447578516841113, "test_accuracy": 0.9583333333333334, "delta_nll": -0.012169121597999794 }, { "seed": 11, "size": 100, "repeat": 4, "calibration_ids": [ 1362, 1777, 1787, 1138, 1176, 1048, 1690, 1715, 81, 809, 1251, 1045, 511, 501, 65, 1289, 1703, 805, 1067, 1169, 689, 763, 782, 1438, 756, 1250, 836, 788, 1530, 381, 905, 1367, 1394, 1651, 1478, 1271, 854, 1171, 721, 570, 566, 244, 1673, 1496, 1630, 1644, 179, 235, 1228, 211, 1282, 1741, 661, 1791, 927, 1268, 161, 1718, 1521, 168, 824, 1614, 1517, 775, 92, 825, 1073, 1470, 254, 813, 1780, 601, 865, 1541, 222, 1377, 1649, 1213, 1726, 1296, 436, 697, 1389, 745, 583, 1738, 606, 729, 1434, 573, 1121, 1111, 1790, 1090, 432, 1586, 1624, 1241, 225, 1023 ], "class_counts": [ 8, 9, 18, 11, 9, 11, 6, 10, 6, 12 ], "calibration_errors": 4, "temperature": 1.530630761008389, "at_boundary": false, "fit_nll_before": 0.14925822745528233, "fit_nll_after": 0.12753071742909458, "test_nll": 0.14476047927927332, "test_ece": 0.015556424505717086, "test_accuracy": 0.9583333333333334, "delta_nll": -0.014948916780410493 }, { "seed": 11, "size": 100, "repeat": 5, "calibration_ids": [ 1268, 1649, 511, 1363, 946, 1456, 770, 456, 1023, 739, 211, 802, 179, 1215, 1250, 1076, 1521, 1715, 41, 805, 865, 177, 176, 157, 400, 206, 675, 1496, 1588, 699, 92, 183, 185, 284, 1688, 1745, 606, 1045, 1777, 97, 1090, 1703, 1233, 25, 1587, 1533, 1213, 721, 1741, 361, 631, 1342, 574, 102, 460, 756, 1425, 874, 1434, 1088, 1187, 252, 131, 601, 661, 1271, 338, 1157, 1169, 1718, 1084, 391, 1630, 829, 921, 1121, 1019, 573, 432, 107, 1176, 1377, 360, 1419, 1146, 340, 535, 153, 995, 1729, 1216, 1517, 296, 1478, 218, 1787, 1790, 412, 1626, 1331 ], "class_counts": [ 10, 14, 7, 9, 11, 10, 9, 11, 10, 9 ], "calibration_errors": 5, "temperature": 1.2993614053571405, "at_boundary": false, "fit_nll_before": 0.12776821219149037, "fit_nll_after": 0.12035201966283736, "test_nll": 0.14505552729903265, "test_ece": 0.014190703880475674, "test_accuracy": 0.9583333333333334, "delta_nll": -0.014653868760651168 }, { "seed": 11, "size": 100, "repeat": 6, "calibration_ids": [ 788, 1331, 1651, 729, 523, 780, 81, 436, 1598, 218, 1176, 25, 168, 1673, 969, 535, 423, 211, 1013, 1003, 1639, 1715, 1251, 1204, 1336, 1718, 1394, 88, 1241, 412, 284, 664, 1363, 1541, 1652, 264, 519, 1233, 809, 1729, 1738, 836, 254, 361, 1195, 858, 65, 802, 16, 314, 1649, 1179, 1280, 1171, 137, 978, 1372, 1045, 1780, 161, 601, 108, 340, 899, 1741, 501, 1213, 1521, 1562, 511, 1203, 1380, 631, 509, 1630, 1067, 829, 183, 986, 400, 1216, 456, 41, 1404, 854, 1586, 1790, 1342, 14, 244, 1550, 605, 1367, 252, 23, 176, 1262, 975, 273, 1777 ], "class_counts": [ 7, 9, 11, 9, 8, 9, 11, 11, 15, 10 ], "calibration_errors": 3, "temperature": 0.8643467734262748, "at_boundary": false, "fit_nll_before": 0.08172359064057234, "fit_nll_after": 0.08066358476179988, "test_nll": 0.17448983873048168, "test_ece": 0.026143547746684325, "test_accuracy": 0.9583333333333334, "delta_nll": 0.014780442670797866 }, { "seed": 11, "size": 100, "repeat": 7, "calibration_ids": [ 456, 23, 296, 1157, 1271, 664, 739, 743, 1791, 1787, 605, 125, 1444, 1203, 1587, 1651, 1171, 591, 1213, 1023, 1282, 1331, 252, 571, 1456, 225, 1076, 825, 179, 263, 168, 611, 1438, 566, 1215, 1639, 176, 541, 460, 1530, 1251, 1517, 805, 1541, 1716, 780, 1313, 1248, 381, 1459, 921, 480, 488, 975, 1490, 1362, 770, 206, 137, 1790, 1084, 1268, 721, 1130, 1394, 423, 1703, 436, 1614, 546, 675, 41, 1195, 391, 131, 1673, 836, 1285, 1335, 712, 1598, 1372, 1048, 583, 222, 854, 1093, 254, 697, 1380, 1718, 802, 829, 185, 400, 788, 601, 1149, 1367, 1712 ], "class_counts": [ 13, 10, 11, 8, 12, 9, 9, 8, 9, 11 ], "calibration_errors": 6, "temperature": 1.8928279448259697, "at_boundary": false, "fit_nll_before": 0.294007995449541, "fit_nll_after": 0.21863861843299898, "test_nll": 0.1574316165244565, "test_ece": 0.03556047369342113, "test_accuracy": 0.9583333333333334, "delta_nll": -0.0022777795352273145 }, { "seed": 11, "size": 100, "repeat": 8, "calibration_ids": [ 1235, 1228, 1013, 1745, 1666, 1718, 495, 185, 1478, 1587, 460, 1738, 300, 660, 882, 412, 324, 103, 573, 1362, 263, 1067, 1187, 206, 1597, 1146, 1130, 1149, 65, 1389, 1716, 235, 1652, 1357, 222, 995, 1586, 108, 1490, 605, 1246, 809, 1778, 14, 1641, 1363, 1649, 1391, 1726, 541, 1372, 1195, 1380, 1204, 1563, 74, 813, 25, 606, 570, 1047, 921, 854, 1690, 1530, 153, 1282, 1336, 899, 252, 211, 509, 1216, 694, 865, 805, 583, 1444, 34, 511, 1241, 1342, 571, 1639, 836, 682, 381, 501, 1313, 432, 1169, 1176, 780, 1048, 388, 756, 858, 1271, 1517, 739 ], "class_counts": [ 13, 7, 11, 12, 8, 11, 9, 10, 10, 9 ], "calibration_errors": 5, "temperature": 1.6298682201929422, "at_boundary": false, "fit_nll_before": 0.23356915477715837, "fit_nll_after": 0.1995322249911043, "test_nll": 0.14680953909590072, "test_ece": 0.0200925450189589, "test_accuracy": 0.9583333333333334, "delta_nll": -0.012899856963783096 }, { "seed": 11, "size": 100, "repeat": 9, "calibration_ids": [ 1146, 694, 244, 1138, 1787, 675, 1686, 412, 788, 326, 225, 1791, 660, 391, 1630, 1084, 1013, 1652, 381, 1271, 712, 284, 1296, 1216, 423, 986, 436, 1587, 177, 975, 1121, 65, 1778, 1342, 1235, 1434, 273, 780, 1391, 978, 1019, 360, 1130, 1729, 108, 523, 432, 1777, 338, 829, 183, 573, 1215, 1289, 1624, 1093, 1067, 1563, 131, 1377, 874, 1362, 137, 153, 361, 570, 1090, 1268, 546, 1550, 808, 122, 16, 1176, 601, 155, 995, 969, 591, 882, 809, 102, 739, 1712, 1203, 1285, 1745, 490, 1425, 1597, 149, 1149, 511, 1651, 745, 775, 754, 211, 813, 1673 ], "class_counts": [ 7, 7, 11, 9, 10, 9, 13, 11, 9, 14 ], "calibration_errors": 5, "temperature": 1.6619109530378255, "at_boundary": false, "fit_nll_before": 0.2225448749270051, "fit_nll_after": 0.18384697064039743, "test_nll": 0.14771603691096138, "test_ece": 0.021758643721646785, "test_accuracy": 0.9583333333333334, "delta_nll": -0.011993359148722432 }, { "seed": 11, "size": 100, "repeat": 10, "calibration_ids": [ 264, 1013, 108, 361, 1076, 432, 1268, 858, 1661, 1022, 338, 874, 824, 1726, 340, 1780, 882, 969, 1289, 1791, 263, 1377, 1171, 1688, 488, 1778, 944, 1363, 157, 81, 1738, 230, 1626, 1391, 1588, 1434, 1517, 1718, 179, 946, 802, 546, 921, 1716, 660, 1048, 1187, 1587, 1146, 273, 571, 995, 813, 16, 284, 574, 519, 541, 1169, 122, 1093, 1745, 1521, 770, 1729, 808, 763, 391, 177, 88, 211, 1047, 601, 664, 1533, 1357, 1703, 1374, 1563, 1195, 153, 254, 583, 1111, 1073, 754, 435, 1003, 161, 225, 1666, 1067, 1023, 1715, 986, 978, 1787, 1456, 183, 436 ], "class_counts": [ 11, 7, 8, 6, 13, 11, 13, 12, 11, 8 ], "calibration_errors": 2, "temperature": 0.7533815179084477, "at_boundary": false, "fit_nll_before": 0.051262367111041866, "fit_nll_after": 0.04759477507018233, "test_nll": 0.19259990957507744, "test_ece": 0.029291882168905718, "test_accuracy": 0.9583333333333334, "delta_nll": 0.03289051351539363 }, { "seed": 11, "size": 100, "repeat": 11, "calibration_ids": [ 675, 605, 1084, 927, 1362, 388, 155, 763, 1651, 1394, 1630, 303, 1530, 541, 546, 149, 1490, 1250, 682, 381, 694, 1550, 1444, 1157, 1626, 754, 432, 1019, 1271, 1726, 1067, 490, 583, 74, 235, 1716, 34, 743, 745, 206, 1169, 1342, 1262, 1216, 535, 456, 1363, 1391, 391, 177, 1111, 1280, 1203, 1587, 1187, 995, 1251, 1130, 659, 168, 874, 1235, 1712, 1285, 788, 1792, 523, 326, 273, 161, 1090, 1588, 218, 1171, 296, 254, 314, 360, 882, 1562, 1248, 92, 157, 712, 570, 1121, 482, 14, 412, 1470, 338, 975, 1703, 1195, 97, 179, 1213, 1644, 460, 1456 ], "class_counts": [ 11, 11, 10, 12, 6, 9, 10, 9, 9, 13 ], "calibration_errors": 2, "temperature": 1.2229765074287136, "at_boundary": false, "fit_nll_before": 0.09012311731177836, "fit_nll_after": 0.08677157650806068, "test_nll": 0.1470014945990263, "test_ece": 0.016645969886578807, "test_accuracy": 0.9583333333333334, "delta_nll": -0.012707901460657506 }, { "seed": 11, "size": 100, "repeat": 12, "calibration_ids": [ 1673, 1787, 1587, 1097, 74, 729, 1630, 41, 1335, 1171, 978, 273, 946, 675, 1741, 1729, 541, 743, 179, 1550, 1563, 1195, 81, 360, 137, 1149, 1088, 1111, 802, 254, 16, 1688, 1404, 1716, 1626, 1342, 606, 296, 1296, 244, 882, 1456, 813, 1614, 340, 1203, 435, 388, 1791, 432, 1690, 825, 157, 423, 782, 490, 235, 944, 1586, 495, 107, 1336, 770, 1176, 1639, 1562, 566, 511, 611, 558, 206, 756, 23, 1530, 995, 1496, 1067, 1073, 230, 1228, 1641, 168, 1490, 1271, 1777, 519, 1019, 927, 460, 947, 178, 1541, 535, 745, 338, 601, 809, 400, 754, 1533 ], "class_counts": [ 10, 10, 8, 7, 11, 14, 9, 13, 8, 10 ], "calibration_errors": 5, "temperature": 1.6308117497714054, "at_boundary": false, "fit_nll_before": 0.22617338519323787, "fit_nll_after": 0.1909620925838623, "test_nll": 0.14683457696957614, "test_ece": 0.020140861594865018, "test_accuracy": 0.9583333333333334, "delta_nll": -0.012874819090107681 }, { "seed": 11, "size": 100, "repeat": 13, "calibration_ids": [ 927, 1331, 824, 1780, 947, 102, 81, 809, 1296, 1246, 946, 300, 721, 813, 1562, 185, 218, 388, 1073, 16, 1357, 1280, 1389, 659, 570, 225, 1597, 153, 1530, 1088, 836, 1335, 1587, 1097, 978, 1048, 944, 1313, 1641, 183, 1787, 874, 1235, 273, 1362, 763, 1138, 125, 1067, 682, 284, 88, 995, 754, 986, 1233, 435, 65, 697, 1626, 490, 571, 1241, 574, 1690, 1726, 338, 1541, 601, 1778, 1745, 558, 206, 482, 802, 161, 1639, 149, 1588, 825, 546, 1111, 1666, 631, 1686, 1712, 1271, 865, 775, 400, 1372, 1076, 1023, 1651, 1652, 1586, 661, 1673, 178, 1090 ], "class_counts": [ 12, 9, 13, 12, 9, 6, 9, 10, 11, 9 ], "calibration_errors": 2, "temperature": 1.2622977776609812, "at_boundary": false, "fit_nll_before": 0.08859251755337696, "fit_nll_after": 0.08442178453679254, "test_nll": 0.14586974958653778, "test_ece": 0.015689850235920882, "test_accuracy": 0.9583333333333334, "delta_nll": -0.013839646473146033 }, { "seed": 11, "size": 100, "repeat": 14, "calibration_ids": [ 570, 737, 721, 1372, 1624, 754, 1228, 1271, 1023, 1213, 1216, 161, 1335, 1614, 664, 157, 763, 326, 519, 501, 1792, 1157, 263, 206, 222, 975, 1791, 1419, 770, 1090, 273, 809, 1644, 921, 1389, 775, 423, 1246, 1716, 23, 824, 1289, 1588, 511, 1022, 183, 969, 660, 340, 92, 978, 400, 1204, 1203, 661, 460, 103, 1598, 1093, 1048, 854, 1478, 391, 1179, 836, 1434, 155, 1076, 435, 176, 1597, 689, 303, 1121, 1521, 631, 1729, 1715, 1195, 808, 264, 1138, 122, 1459, 1088, 284, 825, 490, 1533, 682, 1745, 743, 1313, 1187, 788, 482, 829, 1363, 81, 1013 ], "class_counts": [ 12, 6, 9, 13, 9, 12, 6, 12, 13, 8 ], "calibration_errors": 4, "temperature": 1.2767366995622265, "at_boundary": false, "fit_nll_before": 0.12719984312674448, "fit_nll_after": 0.12185032979486747, "test_nll": 0.14552424403416012, "test_ece": 0.01502183953608391, "test_accuracy": 0.9583333333333334, "delta_nll": -0.0141851520255237 }, { "seed": 11, "size": 100, "repeat": 15, "calibration_ids": [ 1130, 739, 153, 122, 858, 1048, 1171, 788, 222, 743, 675, 1725, 1076, 1335, 1550, 1251, 400, 1651, 546, 23, 1362, 509, 1296, 338, 1562, 829, 1478, 14, 682, 1459, 1262, 721, 1425, 412, 92, 81, 125, 1438, 1176, 1204, 1652, 969, 1282, 808, 1792, 541, 1363, 927, 1444, 34, 1630, 712, 1530, 1639, 482, 805, 854, 1777, 1073, 571, 168, 423, 523, 157, 235, 211, 558, 1149, 825, 1215, 1285, 1342, 1394, 1780, 947, 591, 1391, 1626, 566, 946, 161, 1614, 1195, 1716, 1541, 1456, 495, 65, 501, 178, 1271, 1280, 606, 361, 511, 1703, 218, 975, 754, 1022 ], "class_counts": [ 12, 10, 9, 7, 10, 5, 10, 9, 10, 18 ], "calibration_errors": 4, "temperature": 1.6386727258120608, "at_boundary": false, "fit_nll_before": 0.19893364110758555, "fit_nll_after": 0.16461287832458527, "test_nll": 0.14704710972102397, "test_ece": 0.020545165002101785, "test_accuracy": 0.9583333333333334, "delta_nll": -0.012662286338659845 }, { "seed": 11, "size": 100, "repeat": 16, "calibration_ids": [ 1241, 858, 263, 1614, 1521, 697, 1130, 1374, 1686, 1149, 1438, 177, 1335, 1624, 591, 946, 1741, 721, 1367, 1673, 1690, 480, 1530, 1171, 1268, 1470, 1216, 1718, 1496, 1251, 1377, 1138, 1563, 1097, 865, 1047, 14, 1562, 1228, 570, 1203, 314, 1296, 235, 1404, 1651, 601, 699, 1215, 1023, 1777, 1342, 340, 1111, 1280, 1716, 501, 360, 131, 391, 1792, 944, 1336, 1726, 81, 1389, 1246, 1204, 1597, 566, 1490, 324, 183, 1179, 541, 659, 1045, 125, 1271, 254, 423, 756, 809, 1363, 176, 682, 1738, 1394, 155, 300, 519, 1550, 97, 252, 178, 338, 927, 712, 836, 1688 ], "class_counts": [ 9, 10, 15, 11, 12, 13, 9, 6, 7, 8 ], "calibration_errors": 3, "temperature": 1.8171377230443782, "at_boundary": false, "fit_nll_before": 0.24233541752997742, "fit_nll_after": 0.17940129270694954, "test_nll": 0.15366190448933875, "test_ece": 0.03245174630674752, "test_accuracy": 0.9583333333333334, "delta_nll": -0.006047491570345065 }, { "seed": 11, "size": 100, "repeat": 17, "calibration_ids": [ 41, 25, 1335, 1013, 1285, 34, 153, 865, 519, 480, 1084, 1187, 324, 660, 1745, 1673, 737, 591, 252, 1367, 754, 1649, 218, 1022, 1216, 131, 836, 1470, 1262, 1541, 1235, 770, 1313, 1703, 1787, 809, 824, 1741, 501, 1374, 606, 65, 490, 780, 183, 432, 161, 1047, 1097, 1459, 400, 1331, 874, 1067, 1686, 986, 14, 882, 802, 682, 1213, 1171, 1073, 1563, 74, 1280, 1419, 1296, 921, 1204, 1251, 1111, 178, 1176, 675, 975, 745, 16, 775, 125, 1644, 808, 235, 854, 1149, 176, 1517, 1377, 296, 694, 1088, 300, 1614, 107, 756, 1533, 858, 103, 944, 1490 ], "class_counts": [ 14, 11, 6, 9, 12, 10, 10, 11, 7, 10 ], "calibration_errors": 5, "temperature": 1.7195196470911296, "at_boundary": false, "fit_nll_before": 0.2329295937525104, "fit_nll_after": 0.18789458678933113, "test_nll": 0.14962955783237317, "test_ece": 0.026092910822041064, "test_accuracy": 0.9583333333333334, "delta_nll": -0.01007983822731065 }, { "seed": 11, "size": 100, "repeat": 18, "calibration_ids": [ 775, 1586, 1179, 509, 436, 1391, 303, 432, 1389, 131, 1045, 206, 176, 1195, 756, 969, 805, 1235, 482, 1019, 149, 340, 523, 601, 921, 1521, 763, 573, 675, 1335, 423, 122, 324, 361, 824, 74, 1023, 501, 125, 1248, 1790, 1268, 1088, 809, 412, 108, 456, 211, 1380, 225, 1372, 1686, 802, 1251, 1048, 1725, 1624, 1138, 88, 230, 16, 254, 103, 178, 1204, 1470, 1729, 813, 23, 825, 1745, 905, 1651, 606, 519, 1639, 511, 1073, 263, 1215, 611, 1641, 244, 1404, 300, 218, 566, 338, 874, 155, 1250, 1246, 1121, 944, 745, 1792, 177, 1093, 1357, 381 ], "class_counts": [ 7, 13, 7, 9, 8, 9, 14, 11, 8, 14 ], "calibration_errors": 5, "temperature": 1.3063653003980056, "at_boundary": false, "fit_nll_before": 0.14151955117196802, "fit_nll_after": 0.13431642409436784, "test_nll": 0.14492800492573626, "test_ece": 0.014099462904666949, "test_accuracy": 0.9583333333333334, "delta_nll": -0.014781391133947552 }, { "seed": 11, "size": 100, "repeat": 19, "calibration_ids": [ 1121, 1235, 1787, 423, 1661, 412, 264, 1745, 1394, 1550, 153, 1022, 1690, 125, 570, 808, 1686, 805, 737, 1703, 1490, 1673, 1377, 1725, 699, 1517, 1726, 1313, 1715, 1262, 161, 969, 1130, 1374, 177, 1587, 1342, 14, 1067, 178, 986, 1530, 488, 1777, 211, 1084, 1076, 788, 296, 1791, 179, 1666, 490, 218, 511, 1614, 591, 682, 103, 1438, 1738, 546, 1149, 206, 631, 97, 1741, 1296, 975, 1790, 400, 1336, 81, 324, 460, 157, 921, 254, 775, 694, 432, 605, 176, 829, 16, 340, 1268, 1644, 1171, 1639, 1215, 745, 391, 1023, 601, 480, 107, 149, 1250, 300 ], "class_counts": [ 11, 9, 9, 10, 12, 12, 9, 7, 10, 11 ], "calibration_errors": 5, "temperature": 1.681105796064509, "at_boundary": false, "fit_nll_before": 0.23603212252061528, "fit_nll_after": 0.1944943968925219, "test_nll": 0.14831370314115613, "test_ece": 0.023202352707678075, "test_accuracy": 0.9583333333333334, "delta_nll": -0.011395692918527683 }, { "seed": 11, "size": 269, "repeat": 0, "calibration_ids": [ 211, 1157, 185, 699, 264, 946, 1715, 1048, 829, 721, 102, 495, 300, 14, 854, 456, 1282, 969, 1521, 1045, 360, 314, 244, 535, 1374, 1686, 754, 1778, 1389, 825, 1235, 1204, 921, 326, 1121, 1179, 1187, 1377, 611, 1213, 978, 1614, 1470, 178, 694, 558, 1067, 1047, 1268, 41, 149, 1550, 1280, 1588, 432, 809, 1019, 1787, 858, 1171, 225, 601, 770, 1248, 1241, 631, 1688, 1111, 412, 230, 1285, 1262, 153, 108, 125, 435, 511, 1419, 338, 571, 436, 729, 1146, 218, 361, 1624, 697, 490, 573, 944, 1587, 1246, 460, 541, 1088, 874, 1649, 813, 664, 1130, 1169, 1176, 836, 523, 97, 1562, 1630, 296, 927, 168, 583, 1425, 1666, 1690, 1216, 1149, 1563, 1372, 1138, 1661, 1289, 263, 1362, 574, 1335, 1296, 605, 1741, 743, 391, 775, 1394, 183, 1712, 675, 824, 1233, 660, 1626, 1490, 1729, 1745, 1434, 1703, 1530, 137, 482, 161, 1517, 1597, 1777, 737, 1738, 986, 1444, 254, 1541, 1792, 23, 1725, 802, 177, 1342, 222, 206, 1331, 1251, 606, 1533, 303, 1336, 1644, 1084, 1023, 975, 1203, 1641, 1250, 1228, 739, 882, 284, 423, 88, 81, 103, 712, 235, 1459, 1013, 509, 782, 1790, 1003, 682, 1076, 176, 591, 122, 1718, 1215, 659, 1195, 788, 1639, 808, 1404, 1097, 131, 1363, 1380, 566, 1780, 107, 157, 905, 899, 273, 1090, 570, 1586, 519, 1496, 179, 1716, 1652, 661, 1478, 763, 155, 865, 1391, 745, 1726, 16, 501, 546, 1456, 381, 92, 1093, 65, 324, 1598, 340, 1271, 1651, 780, 488, 480, 34, 995, 1022, 1357, 1791, 1073, 1313, 25, 805, 252, 689, 1367, 756, 1438, 947, 1673, 388, 74, 400 ], "class_counts": [ 27, 27, 27, 27, 27, 27, 27, 27, 26, 27 ], "calibration_errors": 10, "temperature": 1.472669073385246, "at_boundary": false, "fit_nll_before": 0.1623781775298586, "fit_nll_after": 0.14443492807454364, "test_nll": 0.14412838152046395, "test_ece": 0.018249305063047363, "test_accuracy": 0.9583333333333334, "delta_nll": -0.015581014539219867 }, { "seed": 22, "size": 20, "repeat": 0, "calibration_ids": [ 103, 1336, 326, 611, 177, 495, 570, 1246, 303, 222, 858, 157, 97, 546, 1093, 606, 737, 1419, 1179, 583 ], "class_counts": [ 2, 4, 2, 3, 2, 1, 4, 2, 0, 0 ], "calibration_errors": 1, "temperature": 0.7096328456900876, "at_boundary": false, "fit_nll_before": 0.04985676705494822, "fit_nll_after": 0.04692639026182964, "test_nll": 0.1296491175140447, "test_ece": 0.01998806778618444, "test_accuracy": 0.9666666666666667, "delta_nll": 0.02155703366603516 }, { "seed": 22, "size": 20, "repeat": 1, "calibration_ids": [ 1335, 969, 1296, 782, 391, 1233, 1362, 1780, 412, 179, 1176, 480, 273, 435, 1073, 490, 802, 314, 1048, 381 ], "class_counts": [ 3, 0, 2, 0, 0, 1, 6, 3, 1, 4 ], "calibration_errors": 1, "temperature": 2.2444284016024114, "at_boundary": false, "fit_nll_before": 0.4269408785319171, "fit_nll_after": 0.26707020936522397, "test_nll": 0.1404900827863934, "test_ece": 0.054987684987893026, "test_accuracy": 0.9666666666666667, "delta_nll": 0.03239799893838387 }, { "seed": 22, "size": 20, "repeat": 2, "calibration_ids": [ 739, 775, 802, 1459, 1023, 1374, 675, 1745, 1215, 1716, 1729, 1362, 1233, 780, 423, 1712, 1045, 225, 1241, 659 ], "class_counts": [ 2, 1, 2, 3, 4, 0, 3, 1, 2, 2 ], "calibration_errors": 2, "temperature": 2.0028276844848945, "at_boundary": false, "fit_nll_before": 0.4742328240448366, "fit_nll_after": 0.33937678365178425, "test_nll": 0.12484967856659551, "test_ece": 0.040589786491538125, "test_accuracy": 0.9666666666666667, "delta_nll": 0.01675759471858597 }, { "seed": 22, "size": 20, "repeat": 3, "calibration_ids": [ 157, 300, 519, 436, 185, 1179, 178, 273, 264, 1363, 1169, 1090, 495, 1729, 1313, 88, 660, 107, 1047, 81 ], "class_counts": [ 2, 1, 3, 3, 1, 1, 2, 4, 3, 0 ], "calibration_errors": 1, "temperature": 2.342489143818184, "at_boundary": false, "fit_nll_before": 0.3852891428600412, "fit_nll_after": 0.23589699458187904, "test_nll": 0.14784656106739338, "test_ece": 0.06206212461861817, "test_accuracy": 0.9666666666666667, "delta_nll": 0.03975447721938384 }, { "seed": 22, "size": 20, "repeat": 4, "calibration_ids": [ 1362, 1777, 1787, 1138, 1176, 1048, 1690, 1715, 81, 809, 1251, 1045, 511, 501, 65, 1289, 1703, 805, 1067, 1169 ], "class_counts": [ 2, 0, 4, 1, 2, 2, 2, 2, 1, 4 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.008433359649687345, "fit_nll_after": 1.5881436748118961e-18, "test_nll": 1.538585570016098, "test_ece": 0.03246444570680628, "test_accuracy": 0.9666666666666667, "delta_nll": 1.4304934861680885 }, { "seed": 22, "size": 20, "repeat": 5, "calibration_ids": [ 1268, 1649, 511, 1363, 946, 1456, 770, 456, 1023, 739, 211, 802, 179, 1215, 1250, 1076, 1521, 1715, 41, 805 ], "class_counts": [ 2, 4, 0, 0, 6, 0, 3, 2, 1, 2 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.012869095895235407, "fit_nll_after": 1.5881437383128124e-18, "test_nll": 1.538585570016098, "test_ece": 0.03246444570680628, "test_accuracy": 0.9666666666666667, "delta_nll": 1.4304934861680885 }, { "seed": 22, "size": 20, "repeat": 6, "calibration_ids": [ 788, 1331, 1651, 729, 523, 780, 81, 436, 1598, 218, 1176, 25, 168, 1673, 969, 535, 423, 211, 1013, 1003 ], "class_counts": [ 1, 1, 2, 1, 2, 2, 2, 5, 1, 3 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.027113611471385086, "fit_nll_after": 2.8633680844288095e-08, "test_nll": 1.538585570016098, "test_ece": 0.03246444570680628, "test_accuracy": 0.9666666666666667, "delta_nll": 1.4304934861680885 }, { "seed": 22, "size": 20, "repeat": 7, "calibration_ids": [ 456, 23, 296, 1157, 1271, 664, 739, 743, 1791, 1787, 605, 125, 1444, 1203, 1587, 1651, 1171, 591, 1213, 1023 ], "class_counts": [ 1, 3, 1, 2, 5, 2, 0, 1, 3, 2 ], "calibration_errors": 1, "temperature": 1.9830197811713917, "at_boundary": false, "fit_nll_before": 0.2365006424443167, "fit_nll_after": 0.16625951061589903, "test_nll": 0.12373182707474747, "test_ece": 0.03950499632620069, "test_accuracy": 0.9666666666666667, "delta_nll": 0.01563974322673793 }, { "seed": 22, "size": 20, "repeat": 8, "calibration_ids": [ 1235, 1228, 1013, 1745, 1666, 1718, 495, 185, 1478, 1587, 460, 1738, 300, 660, 882, 412, 324, 103, 573, 1362 ], "class_counts": [ 3, 0, 3, 2, 2, 4, 2, 3, 1, 0 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.0029627359465087713, "fit_nll_after": 6.572357132014729e-33, "test_nll": 1.538585570016098, "test_ece": 0.03246444570680628, "test_accuracy": 0.9666666666666667, "delta_nll": 1.4304934861680885 }, { "seed": 22, "size": 20, "repeat": 9, "calibration_ids": [ 1146, 694, 244, 1138, 1787, 675, 1686, 412, 788, 326, 225, 1791, 660, 391, 1630, 1084, 1013, 1652, 381, 1271 ], "class_counts": [ 1, 1, 3, 1, 5, 2, 1, 1, 1, 4 ], "calibration_errors": 1, "temperature": 1.887869048375994, "at_boundary": false, "fit_nll_before": 0.23237015030872019, "fit_nll_after": 0.16731324305266004, "test_nll": 0.11872631728070991, "test_ece": 0.0353781793795208, "test_accuracy": 0.9666666666666667, "delta_nll": 0.010634233432700368 }, { "seed": 22, "size": 20, "repeat": 10, "calibration_ids": [ 264, 1013, 108, 361, 1076, 432, 1268, 858, 1661, 1022, 338, 874, 824, 1726, 340, 1780, 882, 969, 1289, 1791 ], "class_counts": [ 1, 1, 3, 2, 4, 1, 4, 2, 1, 1 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.0020100619527981833, "fit_nll_after": 4.264532142784423e-36, "test_nll": 1.538585570016098, "test_ece": 0.03246444570680628, "test_accuracy": 0.9666666666666667, "delta_nll": 1.4304934861680885 }, { "seed": 22, "size": 20, "repeat": 11, "calibration_ids": [ 675, 605, 1084, 927, 1362, 388, 155, 763, 1651, 1394, 1630, 303, 1530, 541, 546, 149, 1490, 1250, 682, 381 ], "class_counts": [ 2, 3, 6, 2, 1, 3, 0, 0, 0, 3 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.01519661029165815, "fit_nll_after": 9.796273900210604e-12, "test_nll": 1.538585570016098, "test_ece": 0.03246444570680628, "test_accuracy": 0.9666666666666667, "delta_nll": 1.4304934861680885 }, { "seed": 22, "size": 20, "repeat": 12, "calibration_ids": [ 1673, 1787, 1587, 1097, 74, 729, 1630, 41, 1335, 1171, 978, 273, 946, 675, 1741, 1729, 541, 743, 179, 1550 ], "class_counts": [ 2, 1, 0, 3, 4, 5, 1, 2, 1, 1 ], "calibration_errors": 1, "temperature": 2.3005351517471944, "at_boundary": false, "fit_nll_before": 0.36424558630077236, "fit_nll_after": 0.2105340995888733, "test_nll": 0.14463077737616545, "test_ece": 0.058994619681911095, "test_accuracy": 0.9666666666666667, "delta_nll": 0.036538693528155905 }, { "seed": 22, "size": 20, "repeat": 13, "calibration_ids": [ 927, 1331, 824, 1780, 947, 102, 81, 809, 1296, 1246, 946, 300, 721, 813, 1562, 185, 218, 388, 1073, 16 ], "class_counts": [ 2, 2, 4, 1, 1, 2, 1, 4, 0, 3 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.0033239671850753758, "fit_nll_after": 3.3088604899654525e-28, "test_nll": 1.538585570016098, "test_ece": 0.03246444570680628, "test_accuracy": 0.9666666666666667, "delta_nll": 1.4304934861680885 }, { "seed": 22, "size": 20, "repeat": 14, "calibration_ids": [ 570, 737, 721, 1372, 1624, 754, 1228, 1271, 1023, 1213, 1216, 161, 1335, 1614, 664, 157, 763, 326, 519, 501 ], "class_counts": [ 1, 3, 3, 4, 1, 3, 0, 2, 2, 1 ], "calibration_errors": 1, "temperature": 1.8927881564453435, "at_boundary": false, "fit_nll_before": 0.24295944824493088, "fit_nll_after": 0.17765729310674278, "test_nll": 0.1189700846554619, "test_ece": 0.03304618027373693, "test_accuracy": 0.9666666666666667, "delta_nll": 0.010878000807452354 }, { "seed": 22, "size": 20, "repeat": 15, "calibration_ids": [ 1130, 739, 153, 122, 858, 1048, 1171, 788, 222, 743, 675, 1725, 1076, 1335, 1550, 1251, 400, 1651, 546, 23 ], "class_counts": [ 2, 2, 1, 3, 4, 1, 2, 2, 1, 2 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.0020075035984497215, "fit_nll_after": 1.2909898749196527e-30, "test_nll": 1.538585570016098, "test_ece": 0.03246444570680628, "test_accuracy": 0.9666666666666667, "delta_nll": 1.4304934861680885 }, { "seed": 22, "size": 20, "repeat": 16, "calibration_ids": [ 1241, 858, 263, 1614, 1521, 697, 1130, 1374, 1686, 1149, 1438, 177, 1335, 1624, 591, 946, 1741, 721, 1367, 1673 ], "class_counts": [ 1, 2, 4, 3, 2, 2, 3, 1, 1, 1 ], "calibration_errors": 1, "temperature": 2.505710212646791, "at_boundary": false, "fit_nll_before": 0.5529883671432213, "fit_nll_after": 0.32856112436802387, "test_nll": 0.16129513581674068, "test_ece": 0.07455417447126254, "test_accuracy": 0.9666666666666667, "delta_nll": 0.05320305196873114 }, { "seed": 22, "size": 20, "repeat": 17, "calibration_ids": [ 41, 25, 1335, 1013, 1285, 34, 153, 865, 519, 480, 1084, 1187, 324, 660, 1745, 1673, 737, 591, 252, 1367 ], "class_counts": [ 4, 1, 2, 4, 3, 1, 2, 2, 0, 1 ], "calibration_errors": 1, "temperature": 2.2647661421119993, "at_boundary": false, "fit_nll_before": 0.4505318898258565, "fit_nll_after": 0.29307553960691773, "test_nll": 0.14196964256503214, "test_ece": 0.056427391976284116, "test_accuracy": 0.9666666666666667, "delta_nll": 0.033877558717022596 }, { "seed": 22, "size": 20, "repeat": 18, "calibration_ids": [ 775, 1586, 1179, 509, 436, 1391, 303, 432, 1389, 131, 1045, 206, 176, 1195, 756, 969, 805, 1235, 482, 1019 ], "class_counts": [ 1, 2, 2, 0, 1, 3, 3, 2, 5, 1 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.014310725172196176, "fit_nll_after": 7.993964162527205e-14, "test_nll": 1.538585570016098, "test_ece": 0.03246444570680628, "test_accuracy": 0.9666666666666667, "delta_nll": 1.4304934861680885 }, { "seed": 22, "size": 20, "repeat": 19, "calibration_ids": [ 1121, 1235, 1787, 423, 1661, 412, 264, 1745, 1394, 1550, 153, 1022, 1690, 125, 570, 808, 1686, 805, 737, 1703 ], "class_counts": [ 3, 1, 1, 3, 2, 3, 1, 1, 1, 4 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.03340516271662651, "fit_nll_after": 4.121363525742446e-06, "test_nll": 1.538585570016098, "test_ece": 0.03246444570680628, "test_accuracy": 0.9666666666666667, "delta_nll": 1.4304934861680885 }, { "seed": 22, "size": 50, "repeat": 0, "calibration_ids": [ 103, 1336, 326, 611, 177, 495, 570, 1246, 303, 222, 858, 157, 97, 546, 1093, 606, 737, 1419, 1179, 583, 947, 1121, 183, 1372, 436, 1146, 131, 456, 460, 340, 1490, 697, 16, 273, 400, 324, 153, 381, 1251, 825, 1313, 865, 1171, 1067, 74, 1342, 108, 388, 1047, 1380 ], "class_counts": [ 4, 9, 7, 5, 5, 3, 6, 6, 3, 2 ], "calibration_errors": 1, "temperature": 0.6805740699499074, "at_boundary": false, "fit_nll_before": 0.020961640830934512, "fit_nll_after": 0.018878896997191245, "test_nll": 0.1333771021808163, "test_ece": 0.021114405619805238, "test_accuracy": 0.9666666666666667, "delta_nll": 0.02528501833280676 }, { "seed": 22, "size": 50, "repeat": 1, "calibration_ids": [ 1335, 969, 1296, 782, 391, 1233, 1362, 1780, 412, 179, 1176, 480, 273, 435, 1073, 490, 802, 314, 1048, 381, 235, 183, 326, 108, 1586, 944, 1738, 296, 809, 946, 1380, 1367, 300, 185, 1726, 1644, 1425, 1725, 1550, 1703, 1280, 882, 1342, 659, 1336, 535, 1076, 1216, 1741, 1203 ], "class_counts": [ 8, 5, 2, 4, 1, 6, 8, 6, 4, 6 ], "calibration_errors": 1, "temperature": 1.8140524873148778, "at_boundary": false, "fit_nll_before": 0.17939596235131955, "fit_nll_after": 0.1325774876668687, "test_nll": 0.11526962384228884, "test_ece": 0.029673832894221413, "test_accuracy": 0.9666666666666667, "delta_nll": 0.0071775399942792945 }, { "seed": 22, "size": 50, "repeat": 2, "calibration_ids": [ 739, 775, 802, 1459, 1023, 1374, 675, 1745, 1215, 1716, 1729, 1362, 1233, 780, 423, 1712, 1045, 225, 1241, 659, 1090, 1246, 566, 605, 921, 699, 546, 1363, 978, 490, 222, 1521, 1377, 1550, 361, 1121, 986, 1013, 14, 1562, 570, 1587, 1203, 1336, 1342, 611, 179, 1661, 1496, 899 ], "class_counts": [ 6, 2, 5, 6, 6, 2, 7, 6, 6, 4 ], "calibration_errors": 2, "temperature": 1.6336766626590307, "at_boundary": false, "fit_nll_before": 0.20027020932558134, "fit_nll_after": 0.16621848756715024, "test_nll": 0.10847708919935682, "test_ece": 0.02670256507088217, "test_accuracy": 0.9666666666666667, "delta_nll": 0.0003850053513472784 }, { "seed": 22, "size": 50, "repeat": 3, "calibration_ids": [ 157, 300, 519, 436, 185, 1179, 178, 273, 264, 1363, 1169, 1090, 495, 1729, 1313, 88, 660, 107, 1047, 81, 176, 1067, 263, 1121, 1377, 611, 254, 1003, 1404, 1246, 488, 1597, 1792, 1331, 1652, 340, 1686, 825, 1563, 1478, 1712, 161, 591, 1296, 1790, 865, 874, 1213, 1216, 222 ], "class_counts": [ 4, 3, 6, 9, 2, 5, 3, 8, 5, 5 ], "calibration_errors": 1, "temperature": 1.7773344681355476, "at_boundary": false, "fit_nll_before": 0.1589053177207182, "fit_nll_after": 0.1205289559112369, "test_nll": 0.1136932044321237, "test_ece": 0.026415020814567394, "test_accuracy": 0.9666666666666667, "delta_nll": 0.005601120584114161 }, { "seed": 22, "size": 50, "repeat": 4, "calibration_ids": [ 1362, 1777, 1787, 1138, 1176, 1048, 1690, 1715, 81, 809, 1251, 1045, 511, 501, 65, 1289, 1703, 805, 1067, 1169, 689, 763, 782, 1438, 756, 1250, 836, 788, 1530, 381, 905, 1367, 1394, 1651, 1478, 1271, 854, 1171, 721, 570, 566, 244, 1673, 1496, 1630, 1644, 179, 235, 1228, 211 ], "class_counts": [ 4, 4, 10, 6, 5, 4, 4, 5, 3, 5 ], "calibration_errors": 2, "temperature": 1.787760332883194, "at_boundary": false, "fit_nll_before": 0.19087991623427356, "fit_nll_after": 0.14185799975684424, "test_nll": 0.1141310108744304, "test_ece": 0.02779779970045679, "test_accuracy": 0.9666666666666667, "delta_nll": 0.006038927026420851 }, { "seed": 22, "size": 50, "repeat": 5, "calibration_ids": [ 1268, 1649, 511, 1363, 946, 1456, 770, 456, 1023, 739, 211, 802, 179, 1215, 1250, 1076, 1521, 1715, 41, 805, 865, 177, 176, 157, 400, 206, 675, 1496, 1588, 699, 92, 183, 185, 284, 1688, 1745, 606, 1045, 1777, 97, 1090, 1703, 1233, 25, 1587, 1533, 1213, 721, 1741, 361 ], "class_counts": [ 5, 8, 1, 3, 9, 3, 4, 6, 6, 5 ], "calibration_errors": 1, "temperature": 0.4688117135803622, "at_boundary": false, "fit_nll_before": 0.0330579741767503, "fit_nll_after": 0.02157583270592892, "test_nll": 0.17821538582903831, "test_ece": 0.021498366355857352, "test_accuracy": 0.9666666666666667, "delta_nll": 0.07012330198102877 }, { "seed": 22, "size": 50, "repeat": 6, "calibration_ids": [ 788, 1331, 1651, 729, 523, 780, 81, 436, 1598, 218, 1176, 25, 168, 1673, 969, 535, 423, 211, 1013, 1003, 1639, 1715, 1251, 1204, 1336, 1718, 1394, 88, 1241, 412, 284, 664, 1363, 1541, 1652, 264, 519, 1233, 809, 1729, 1738, 836, 254, 361, 1195, 858, 65, 802, 16, 314 ], "class_counts": [ 4, 3, 4, 5, 3, 4, 9, 6, 7, 5 ], "calibration_errors": 1, "temperature": 1.5689239139479316, "at_boundary": false, "fit_nll_before": 0.16814414917874668, "fit_nll_after": 0.142163661308512, "test_nll": 0.10664426615707837, "test_ece": 0.01874032845973875, "test_accuracy": 0.9666666666666667, "delta_nll": -0.0014478176909311774 }, { "seed": 22, "size": 50, "repeat": 7, "calibration_ids": [ 456, 23, 296, 1157, 1271, 664, 739, 743, 1791, 1787, 605, 125, 1444, 1203, 1587, 1651, 1171, 591, 1213, 1023, 1282, 1331, 252, 571, 1456, 225, 1076, 825, 179, 263, 168, 611, 1438, 566, 1215, 1639, 176, 541, 460, 1530, 1251, 1517, 805, 1541, 1716, 780, 1313, 1248, 381, 1459 ], "class_counts": [ 7, 4, 3, 5, 8, 6, 3, 5, 4, 5 ], "calibration_errors": 2, "temperature": 1.7589389263166288, "at_boundary": false, "fit_nll_before": 0.13413942500661397, "fit_nll_after": 0.10277886514666783, "test_nll": 0.11293987130267717, "test_ece": 0.02811007674363077, "test_accuracy": 0.9666666666666667, "delta_nll": 0.004847787454667629 }, { "seed": 22, "size": 50, "repeat": 8, "calibration_ids": [ 1235, 1228, 1013, 1745, 1666, 1718, 495, 185, 1478, 1587, 460, 1738, 300, 660, 882, 412, 324, 103, 573, 1362, 263, 1067, 1187, 206, 1597, 1146, 1130, 1149, 65, 1389, 1716, 235, 1652, 1357, 222, 995, 1586, 108, 1490, 605, 1246, 809, 1778, 14, 1641, 1363, 1649, 1391, 1726, 541 ], "class_counts": [ 5, 2, 5, 6, 6, 7, 4, 9, 5, 1 ], "calibration_errors": 1, "temperature": 1.8515557660129198, "at_boundary": false, "fit_nll_before": 0.2330111117329562, "fit_nll_after": 0.17494031114824662, "test_nll": 0.11697837925526157, "test_ece": 0.031451284636128285, "test_accuracy": 0.9666666666666667, "delta_nll": 0.008886295407252026 }, { "seed": 22, "size": 50, "repeat": 9, "calibration_ids": [ 1146, 694, 244, 1138, 1787, 675, 1686, 412, 788, 326, 225, 1791, 660, 391, 1630, 1084, 1013, 1652, 381, 1271, 712, 284, 1296, 1216, 423, 986, 436, 1587, 177, 975, 1121, 65, 1778, 1342, 1235, 1434, 273, 780, 1391, 978, 1019, 360, 1130, 1729, 108, 523, 432, 1777, 338, 829 ], "class_counts": [ 3, 2, 5, 4, 8, 3, 6, 7, 4, 8 ], "calibration_errors": 2, "temperature": 1.9118086046329141, "at_boundary": false, "fit_nll_before": 0.24356270994564774, "fit_nll_after": 0.17298071079483163, "test_nll": 0.11992819610302445, "test_ece": 0.034056730114344624, "test_accuracy": 0.9666666666666667, "delta_nll": 0.011836112255014908 }, { "seed": 22, "size": 50, "repeat": 10, "calibration_ids": [ 264, 1013, 108, 361, 1076, 432, 1268, 858, 1661, 1022, 338, 874, 824, 1726, 340, 1780, 882, 969, 1289, 1791, 263, 1377, 1171, 1688, 488, 1778, 944, 1363, 157, 81, 1738, 230, 1626, 1391, 1588, 1434, 1517, 1718, 179, 946, 802, 546, 921, 1716, 660, 1048, 1187, 1587, 1146, 273 ], "class_counts": [ 5, 4, 4, 3, 8, 5, 7, 7, 2, 5 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.004927248555356359, "fit_nll_after": 8.158158790657561e-19, "test_nll": 1.538585570016098, "test_ece": 0.03246444570680628, "test_accuracy": 0.9666666666666667, "delta_nll": 1.4304934861680885 }, { "seed": 22, "size": 50, "repeat": 11, "calibration_ids": [ 675, 605, 1084, 927, 1362, 388, 155, 763, 1651, 1394, 1630, 303, 1530, 541, 546, 149, 1490, 1250, 682, 381, 694, 1550, 1444, 1157, 1626, 754, 432, 1019, 1271, 1726, 1067, 490, 583, 74, 235, 1716, 34, 743, 745, 206, 1169, 1342, 1262, 1216, 535, 456, 1363, 1391, 391, 177 ], "class_counts": [ 6, 8, 7, 4, 2, 7, 4, 2, 4, 6 ], "calibration_errors": 1, "temperature": 1.4032727123656612, "at_boundary": false, "fit_nll_before": 0.1102368919360296, "fit_nll_after": 0.10006392321408454, "test_nll": 0.10354875025733061, "test_ece": 0.018922620139252777, "test_accuracy": 0.9666666666666667, "delta_nll": -0.004543333590678933 }, { "seed": 22, "size": 50, "repeat": 12, "calibration_ids": [ 1673, 1787, 1587, 1097, 74, 729, 1630, 41, 1335, 1171, 978, 273, 946, 675, 1741, 1729, 541, 743, 179, 1550, 1563, 1195, 81, 360, 137, 1149, 1088, 1111, 802, 254, 16, 1688, 1404, 1716, 1626, 1342, 606, 296, 1296, 244, 882, 1456, 813, 1614, 340, 1203, 435, 388, 1791, 432 ], "class_counts": [ 6, 4, 4, 3, 6, 9, 5, 5, 4, 4 ], "calibration_errors": 3, "temperature": 2.1818377175025208, "at_boundary": false, "fit_nll_before": 0.3880104753789987, "fit_nll_after": 0.25629430630470756, "test_nll": 0.13609162835502334, "test_ece": 0.052979754781346006, "test_accuracy": 0.9666666666666667, "delta_nll": 0.027999544507013793 }, { "seed": 22, "size": 50, "repeat": 13, "calibration_ids": [ 927, 1331, 824, 1780, 947, 102, 81, 809, 1296, 1246, 946, 300, 721, 813, 1562, 185, 218, 388, 1073, 16, 1357, 1280, 1389, 659, 570, 225, 1597, 153, 1530, 1088, 836, 1335, 1587, 1097, 978, 1048, 944, 1313, 1641, 183, 1787, 874, 1235, 273, 1362, 763, 1138, 125, 1067, 682 ], "class_counts": [ 5, 4, 8, 5, 4, 5, 2, 7, 4, 6 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.0017674238596299282, "fit_nll_after": 1.323544195986181e-28, "test_nll": 1.538585570016098, "test_ece": 0.03246444570680628, "test_accuracy": 0.9666666666666667, "delta_nll": 1.4304934861680885 }, { "seed": 22, "size": 50, "repeat": 14, "calibration_ids": [ 570, 737, 721, 1372, 1624, 754, 1228, 1271, 1023, 1213, 1216, 161, 1335, 1614, 664, 157, 763, 326, 519, 501, 1792, 1157, 263, 206, 222, 975, 1791, 1419, 770, 1090, 273, 809, 1644, 921, 1389, 775, 423, 1246, 1716, 23, 824, 1289, 1588, 511, 1022, 183, 969, 660, 340, 92 ], "class_counts": [ 4, 3, 5, 9, 6, 5, 2, 5, 5, 6 ], "calibration_errors": 1, "temperature": 1.3848040798993257, "at_boundary": false, "fit_nll_before": 0.10891939667811255, "fit_nll_after": 0.099807206174912, "test_nll": 0.10335549962518059, "test_ece": 0.01830899847826955, "test_accuracy": 0.9666666666666667, "delta_nll": -0.004736584222828952 }, { "seed": 22, "size": 50, "repeat": 15, "calibration_ids": [ 1130, 739, 153, 122, 858, 1048, 1171, 788, 222, 743, 675, 1725, 1076, 1335, 1550, 1251, 400, 1651, 546, 23, 1362, 509, 1296, 338, 1562, 829, 1478, 14, 682, 1459, 1262, 721, 1425, 412, 92, 81, 125, 1438, 1176, 1204, 1652, 969, 1282, 808, 1792, 541, 1363, 927, 1444, 34 ], "class_counts": [ 4, 3, 4, 5, 6, 3, 6, 4, 4, 11 ], "calibration_errors": 1, "temperature": 1.2449549728750395, "at_boundary": false, "fit_nll_before": 0.050524462987508026, "fit_nll_after": 0.04846580486035183, "test_nll": 0.10299595700609751, "test_ece": 0.013519543876805885, "test_accuracy": 0.9666666666666667, "delta_nll": -0.005096126841912033 }, { "seed": 22, "size": 50, "repeat": 16, "calibration_ids": [ 1241, 858, 263, 1614, 1521, 697, 1130, 1374, 1686, 1149, 1438, 177, 1335, 1624, 591, 946, 1741, 721, 1367, 1673, 1690, 480, 1530, 1171, 1268, 1470, 1216, 1718, 1496, 1251, 1377, 1138, 1563, 1097, 865, 1047, 14, 1562, 1228, 570, 1203, 314, 1296, 235, 1404, 1651, 601, 699, 1215, 1023 ], "class_counts": [ 3, 6, 7, 6, 8, 5, 5, 4, 3, 3 ], "calibration_errors": 2, "temperature": 2.263990886980669, "at_boundary": false, "fit_nll_before": 0.39073127818928144, "fit_nll_after": 0.24177479216426018, "test_nll": 0.14191279537686977, "test_ece": 0.056372244941220645, "test_accuracy": 0.9666666666666667, "delta_nll": 0.03382071152886022 }, { "seed": 22, "size": 50, "repeat": 17, "calibration_ids": [ 41, 25, 1335, 1013, 1285, 34, 153, 865, 519, 480, 1084, 1187, 324, 660, 1745, 1673, 737, 591, 252, 1367, 754, 1649, 218, 1022, 1216, 131, 836, 1470, 1262, 1541, 1235, 770, 1313, 1703, 1787, 809, 824, 1741, 501, 1374, 606, 65, 490, 780, 183, 432, 161, 1047, 1097, 1459 ], "class_counts": [ 9, 5, 3, 6, 7, 5, 5, 5, 2, 3 ], "calibration_errors": 3, "temperature": 1.7741467654725458, "at_boundary": false, "fit_nll_before": 0.23830640497460512, "fit_nll_after": 0.18643358359467094, "test_nll": 0.11356090800493575, "test_ece": 0.026260027938532413, "test_accuracy": 0.9666666666666667, "delta_nll": 0.005468824156926205 }, { "seed": 22, "size": 50, "repeat": 18, "calibration_ids": [ 775, 1586, 1179, 509, 436, 1391, 303, 432, 1389, 131, 1045, 206, 176, 1195, 756, 969, 805, 1235, 482, 1019, 149, 340, 523, 601, 921, 1521, 763, 573, 675, 1335, 423, 122, 324, 361, 824, 74, 1023, 501, 125, 1248, 1790, 1268, 1088, 809, 412, 108, 456, 211, 1380, 225 ], "class_counts": [ 3, 5, 5, 1, 5, 6, 6, 6, 7, 6 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.012324132908491332, "fit_nll_after": 3.205089992190305e-14, "test_nll": 1.538585570016098, "test_ece": 0.03246444570680628, "test_accuracy": 0.9666666666666667, "delta_nll": 1.4304934861680885 }, { "seed": 22, "size": 50, "repeat": 19, "calibration_ids": [ 1121, 1235, 1787, 423, 1661, 412, 264, 1745, 1394, 1550, 153, 1022, 1690, 125, 570, 808, 1686, 805, 737, 1703, 1490, 1673, 1377, 1725, 699, 1517, 1726, 1313, 1715, 1262, 161, 969, 1130, 1374, 177, 1587, 1342, 14, 1067, 178, 986, 1530, 488, 1777, 211, 1084, 1076, 788, 296, 1791 ], "class_counts": [ 6, 4, 6, 5, 6, 5, 5, 3, 4, 6 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.016309954786393077, "fit_nll_after": 1.6485454102969856e-06, "test_nll": 1.538585570016098, "test_ece": 0.03246444570680628, "test_accuracy": 0.9666666666666667, "delta_nll": 1.4304934861680885 }, { "seed": 22, "size": 100, "repeat": 0, "calibration_ids": [ 103, 1336, 326, 611, 177, 495, 570, 1246, 303, 222, 858, 157, 97, 546, 1093, 606, 737, 1419, 1179, 583, 947, 1121, 183, 1372, 436, 1146, 131, 456, 460, 340, 1490, 697, 16, 273, 400, 324, 153, 381, 1251, 825, 1313, 865, 1171, 1067, 74, 1342, 108, 388, 1047, 1380, 743, 699, 1003, 300, 1425, 211, 122, 836, 905, 1285, 168, 1530, 775, 573, 1138, 488, 435, 1738, 41, 1048, 296, 14, 523, 92, 155, 694, 1586, 1644, 1792, 360, 571, 137, 1686, 81, 1470, 1362, 263, 1241, 1588, 161, 88, 235, 230, 745, 1716, 1703, 899, 501, 1250, 829 ], "class_counts": [ 11, 12, 12, 8, 9, 8, 8, 13, 11, 8 ], "calibration_errors": 2, "temperature": 1.3006932116452865, "at_boundary": false, "fit_nll_before": 0.06004420039893826, "fit_nll_after": 0.0556367086509808, "test_nll": 0.10289596530506522, "test_ece": 0.013109204083870746, "test_accuracy": 0.9666666666666667, "delta_nll": -0.005196118542944328 }, { "seed": 22, "size": 100, "repeat": 1, "calibration_ids": [ 1335, 969, 1296, 782, 391, 1233, 1362, 1780, 412, 179, 1176, 480, 273, 435, 1073, 490, 802, 314, 1048, 381, 235, 183, 326, 108, 1586, 944, 1738, 296, 809, 946, 1380, 1367, 300, 185, 1726, 1644, 1425, 1725, 1550, 1703, 1280, 882, 1342, 659, 1336, 535, 1076, 1216, 1741, 1203, 177, 1715, 905, 858, 1246, 263, 805, 97, 1282, 675, 25, 1090, 1084, 1313, 102, 1470, 1228, 682, 1444, 423, 168, 1419, 1377, 756, 697, 1649, 1157, 661, 155, 1639, 1215, 788, 1690, 1235, 1787, 978, 1652, 495, 1262, 92, 566, 921, 1490, 1562, 1121, 103, 340, 1651, 1716, 338 ], "class_counts": [ 14, 7, 8, 9, 6, 13, 13, 9, 7, 14 ], "calibration_errors": 2, "temperature": 1.6854265978758372, "at_boundary": false, "fit_nll_before": 0.1396235905791136, "fit_nll_after": 0.10851837569634781, "test_nll": 0.11017750361688311, "test_ece": 0.027831649837549776, "test_accuracy": 0.9666666666666667, "delta_nll": 0.002085419768873567 }, { "seed": 22, "size": 100, "repeat": 2, "calibration_ids": [ 739, 775, 802, 1459, 1023, 1374, 675, 1745, 1215, 1716, 1729, 1362, 1233, 780, 423, 1712, 1045, 225, 1241, 659, 1090, 1246, 566, 605, 921, 699, 546, 1363, 978, 490, 222, 1521, 1377, 1550, 361, 1121, 986, 1013, 14, 1562, 570, 1587, 1203, 1336, 1342, 611, 179, 1661, 1496, 899, 519, 412, 1718, 97, 1777, 1434, 273, 456, 1282, 1703, 1588, 558, 721, 177, 1130, 1313, 1726, 1725, 1644, 1357, 1019, 1022, 1111, 1251, 975, 1248, 34, 1235, 509, 1614, 23, 1478, 264, 1792, 161, 314, 1067, 149, 103, 1176, 244, 829, 1790, 1641, 400, 1688, 782, 743, 535, 1367 ], "class_counts": [ 8, 8, 9, 15, 12, 3, 13, 9, 11, 12 ], "calibration_errors": 2, "temperature": 1.2911625554781467, "at_boundary": false, "fit_nll_before": 0.11539624456702623, "fit_nll_after": 0.10935589597594035, "test_nll": 0.10288930598748335, "test_ece": 0.013935386558218516, "test_accuracy": 0.9666666666666667, "delta_nll": -0.005202777860526198 }, { "seed": 22, "size": 100, "repeat": 3, "calibration_ids": [ 157, 300, 519, 436, 185, 1179, 178, 273, 264, 1363, 1169, 1090, 495, 1729, 1313, 88, 660, 107, 1047, 81, 176, 1067, 263, 1121, 1377, 611, 254, 1003, 1404, 1246, 488, 1597, 1792, 1331, 1652, 340, 1686, 825, 1563, 1478, 1712, 161, 591, 1296, 1790, 865, 874, 1213, 1216, 222, 1470, 1490, 1271, 1716, 16, 1688, 1438, 1521, 1587, 858, 1073, 103, 125, 975, 729, 1285, 137, 802, 1673, 739, 284, 946, 1726, 230, 1434, 1171, 1703, 854, 1459, 1533, 1149, 1389, 1394, 882, 1335, 808, 697, 1690, 1787, 813, 1262, 1342, 1157, 432, 149, 605, 1419, 131, 400, 1138 ], "class_counts": [ 11, 7, 8, 15, 7, 10, 9, 13, 8, 12 ], "calibration_errors": 4, "temperature": 1.8922127820069707, "at_boundary": false, "fit_nll_before": 0.2620423626067714, "fit_nll_after": 0.19236446617877068, "test_nll": 0.11894148627628086, "test_ece": 0.03301583496547012, "test_accuracy": 0.9666666666666667, "delta_nll": 0.010849402428271315 }, { "seed": 22, "size": 100, "repeat": 4, "calibration_ids": [ 1362, 1777, 1787, 1138, 1176, 1048, 1690, 1715, 81, 809, 1251, 1045, 511, 501, 65, 1289, 1703, 805, 1067, 1169, 689, 763, 782, 1438, 756, 1250, 836, 788, 1530, 381, 905, 1367, 1394, 1651, 1478, 1271, 854, 1171, 721, 570, 566, 244, 1673, 1496, 1630, 1644, 179, 235, 1228, 211, 1282, 1741, 661, 1791, 927, 1268, 161, 1718, 1521, 168, 824, 1614, 1517, 775, 92, 825, 1073, 1470, 254, 813, 1780, 601, 865, 1541, 222, 1377, 1649, 1213, 1726, 1296, 436, 697, 1389, 745, 583, 1738, 606, 729, 1434, 573, 1121, 1111, 1790, 1090, 432, 1586, 1624, 1241, 225, 1023 ], "class_counts": [ 8, 9, 18, 11, 9, 11, 6, 10, 6, 12 ], "calibration_errors": 3, "temperature": 1.491562652242518, "at_boundary": false, "fit_nll_before": 0.10719686200226176, "fit_nll_after": 0.09220189703435169, "test_nll": 0.10490303854629417, "test_ece": 0.019453033000505195, "test_accuracy": 0.9666666666666667, "delta_nll": -0.0031890453017153764 }, { "seed": 22, "size": 100, "repeat": 5, "calibration_ids": [ 1268, 1649, 511, 1363, 946, 1456, 770, 456, 1023, 739, 211, 802, 179, 1215, 1250, 1076, 1521, 1715, 41, 805, 865, 177, 176, 157, 400, 206, 675, 1496, 1588, 699, 92, 183, 185, 284, 1688, 1745, 606, 1045, 1777, 97, 1090, 1703, 1233, 25, 1587, 1533, 1213, 721, 1741, 361, 631, 1342, 574, 102, 460, 756, 1425, 874, 1434, 1088, 1187, 252, 131, 601, 661, 1271, 338, 1157, 1169, 1718, 1084, 391, 1630, 829, 921, 1121, 1019, 573, 432, 107, 1176, 1377, 360, 1419, 1146, 340, 535, 153, 995, 1729, 1216, 1517, 296, 1478, 218, 1787, 1790, 412, 1626, 1331 ], "class_counts": [ 10, 14, 7, 9, 11, 10, 9, 11, 10, 9 ], "calibration_errors": 3, "temperature": 1.4911601231893092, "at_boundary": false, "fit_nll_before": 0.14118413739223792, "fit_nll_after": 0.12361004933590475, "test_nll": 0.10489530031453585, "test_ece": 0.01943845498549676, "test_accuracy": 0.9666666666666667, "delta_nll": -0.0031967835334736905 }, { "seed": 22, "size": 100, "repeat": 6, "calibration_ids": [ 788, 1331, 1651, 729, 523, 780, 81, 436, 1598, 218, 1176, 25, 168, 1673, 969, 535, 423, 211, 1013, 1003, 1639, 1715, 1251, 1204, 1336, 1718, 1394, 88, 1241, 412, 284, 664, 1363, 1541, 1652, 264, 519, 1233, 809, 1729, 1738, 836, 254, 361, 1195, 858, 65, 802, 16, 314, 1649, 1179, 1280, 1171, 137, 978, 1372, 1045, 1780, 161, 601, 108, 340, 899, 1741, 501, 1213, 1521, 1562, 511, 1203, 1380, 631, 509, 1630, 1067, 829, 183, 986, 400, 1216, 456, 41, 1404, 854, 1586, 1790, 1342, 14, 244, 1550, 605, 1367, 252, 23, 176, 1262, 975, 273, 1777 ], "class_counts": [ 7, 9, 11, 9, 8, 9, 11, 11, 15, 10 ], "calibration_errors": 1, "temperature": 1.2628770389323436, "at_boundary": false, "fit_nll_before": 0.0951978930055024, "fit_nll_after": 0.09105299864677624, "test_nll": 0.10292677773870602, "test_ece": 0.014155539741696184, "test_accuracy": 0.9666666666666667, "delta_nll": -0.005165306109303519 }, { "seed": 22, "size": 100, "repeat": 7, "calibration_ids": [ 456, 23, 296, 1157, 1271, 664, 739, 743, 1791, 1787, 605, 125, 1444, 1203, 1587, 1651, 1171, 591, 1213, 1023, 1282, 1331, 252, 571, 1456, 225, 1076, 825, 179, 263, 168, 611, 1438, 566, 1215, 1639, 176, 541, 460, 1530, 1251, 1517, 805, 1541, 1716, 780, 1313, 1248, 381, 1459, 921, 480, 488, 975, 1490, 1362, 770, 206, 137, 1790, 1084, 1268, 721, 1130, 1394, 423, 1703, 436, 1614, 546, 675, 41, 1195, 391, 131, 1673, 836, 1285, 1335, 712, 1598, 1372, 1048, 583, 222, 854, 1093, 254, 697, 1380, 1718, 802, 829, 185, 400, 788, 601, 1149, 1367, 1712 ], "class_counts": [ 13, 10, 11, 8, 12, 9, 9, 8, 9, 11 ], "calibration_errors": 4, "temperature": 1.9588327187068177, "at_boundary": false, "fit_nll_before": 0.2674745717664606, "fit_nll_after": 0.19064826098446017, "test_nll": 0.12240193442486237, "test_ece": 0.03820184671915543, "test_accuracy": 0.9666666666666667, "delta_nll": 0.014309850576852828 }, { "seed": 22, "size": 100, "repeat": 8, "calibration_ids": [ 1235, 1228, 1013, 1745, 1666, 1718, 495, 185, 1478, 1587, 460, 1738, 300, 660, 882, 412, 324, 103, 573, 1362, 263, 1067, 1187, 206, 1597, 1146, 1130, 1149, 65, 1389, 1716, 235, 1652, 1357, 222, 995, 1586, 108, 1490, 605, 1246, 809, 1778, 14, 1641, 1363, 1649, 1391, 1726, 541, 1372, 1195, 1380, 1204, 1563, 74, 813, 25, 606, 570, 1047, 921, 854, 1690, 1530, 153, 1282, 1336, 899, 252, 211, 509, 1216, 694, 865, 805, 583, 1444, 34, 511, 1241, 1342, 571, 1639, 836, 682, 381, 501, 1313, 432, 1169, 1176, 780, 1048, 388, 756, 858, 1271, 1517, 739 ], "class_counts": [ 13, 7, 11, 12, 8, 11, 9, 10, 10, 9 ], "calibration_errors": 3, "temperature": 1.6140660005794416, "at_boundary": false, "fit_nll_before": 0.1742924906390294, "fit_nll_after": 0.1449809123720566, "test_nll": 0.10788689893237774, "test_ece": 0.024072509769023934, "test_accuracy": 0.9666666666666667, "delta_nll": -0.00020518491563180363 }, { "seed": 22, "size": 100, "repeat": 9, "calibration_ids": [ 1146, 694, 244, 1138, 1787, 675, 1686, 412, 788, 326, 225, 1791, 660, 391, 1630, 1084, 1013, 1652, 381, 1271, 712, 284, 1296, 1216, 423, 986, 436, 1587, 177, 975, 1121, 65, 1778, 1342, 1235, 1434, 273, 780, 1391, 978, 1019, 360, 1130, 1729, 108, 523, 432, 1777, 338, 829, 183, 573, 1215, 1289, 1624, 1093, 1067, 1563, 131, 1377, 874, 1362, 137, 153, 361, 570, 1090, 1268, 546, 1550, 808, 122, 16, 1176, 601, 155, 995, 969, 591, 882, 809, 102, 739, 1712, 1203, 1285, 1745, 490, 1425, 1597, 149, 1149, 511, 1651, 745, 775, 754, 211, 813, 1673 ], "class_counts": [ 7, 7, 11, 9, 10, 9, 13, 11, 9, 14 ], "calibration_errors": 3, "temperature": 1.8689400898095019, "at_boundary": false, "fit_nll_before": 0.23874289573960994, "fit_nll_after": 0.17448068978239295, "test_nll": 0.1178037990039165, "test_ece": 0.03437420131086085, "test_accuracy": 0.9666666666666667, "delta_nll": 0.009711715155906964 }, { "seed": 22, "size": 100, "repeat": 10, "calibration_ids": [ 264, 1013, 108, 361, 1076, 432, 1268, 858, 1661, 1022, 338, 874, 824, 1726, 340, 1780, 882, 969, 1289, 1791, 263, 1377, 1171, 1688, 488, 1778, 944, 1363, 157, 81, 1738, 230, 1626, 1391, 1588, 1434, 1517, 1718, 179, 946, 802, 546, 921, 1716, 660, 1048, 1187, 1587, 1146, 273, 571, 995, 813, 16, 284, 574, 519, 541, 1169, 122, 1093, 1745, 1521, 770, 1729, 808, 763, 391, 177, 88, 211, 1047, 601, 664, 1533, 1357, 1703, 1374, 1563, 1195, 153, 254, 583, 1111, 1073, 754, 435, 1003, 161, 225, 1666, 1067, 1023, 1715, 986, 978, 1787, 1456, 183, 436 ], "class_counts": [ 11, 7, 8, 6, 13, 11, 13, 12, 11, 8 ], "calibration_errors": 1, "temperature": 1.3457825680039042, "at_boundary": false, "fit_nll_before": 0.08806841572014795, "fit_nll_after": 0.0807735704632794, "test_nll": 0.10305470541969694, "test_ece": 0.01574132780839242, "test_accuracy": 0.9666666666666667, "delta_nll": -0.005037378428312603 }, { "seed": 22, "size": 100, "repeat": 11, "calibration_ids": [ 675, 605, 1084, 927, 1362, 388, 155, 763, 1651, 1394, 1630, 303, 1530, 541, 546, 149, 1490, 1250, 682, 381, 694, 1550, 1444, 1157, 1626, 754, 432, 1019, 1271, 1726, 1067, 490, 583, 74, 235, 1716, 34, 743, 745, 206, 1169, 1342, 1262, 1216, 535, 456, 1363, 1391, 391, 177, 1111, 1280, 1203, 1587, 1187, 995, 1251, 1130, 659, 168, 874, 1235, 1712, 1285, 788, 1792, 523, 326, 273, 161, 1090, 1588, 218, 1171, 296, 254, 314, 360, 882, 1562, 1248, 92, 157, 712, 570, 1121, 482, 14, 412, 1470, 338, 975, 1703, 1195, 97, 179, 1213, 1644, 460, 1456 ], "class_counts": [ 11, 11, 10, 12, 6, 9, 10, 9, 9, 13 ], "calibration_errors": 1, "temperature": 1.2007482234622135, "at_boundary": false, "fit_nll_before": 0.057629342549861046, "fit_nll_after": 0.05569623498685479, "test_nll": 0.10332391773531112, "test_ece": 0.013998175110636054, "test_accuracy": 0.9666666666666667, "delta_nll": -0.004768166112698424 }, { "seed": 22, "size": 100, "repeat": 12, "calibration_ids": [ 1673, 1787, 1587, 1097, 74, 729, 1630, 41, 1335, 1171, 978, 273, 946, 675, 1741, 1729, 541, 743, 179, 1550, 1563, 1195, 81, 360, 137, 1149, 1088, 1111, 802, 254, 16, 1688, 1404, 1716, 1626, 1342, 606, 296, 1296, 244, 882, 1456, 813, 1614, 340, 1203, 435, 388, 1791, 432, 1690, 825, 157, 423, 782, 490, 235, 944, 1586, 495, 107, 1336, 770, 1176, 1639, 1562, 566, 511, 611, 558, 206, 756, 23, 1530, 995, 1496, 1067, 1073, 230, 1228, 1641, 168, 1490, 1271, 1777, 519, 1019, 927, 460, 947, 178, 1541, 535, 745, 338, 601, 809, 400, 754, 1533 ], "class_counts": [ 10, 10, 8, 7, 11, 14, 9, 13, 8, 10 ], "calibration_errors": 4, "temperature": 1.8551568503272775, "at_boundary": false, "fit_nll_before": 0.25232222722923614, "fit_nll_after": 0.19108326179401464, "test_nll": 0.11714763748407833, "test_ece": 0.03162503816712022, "test_accuracy": 0.9666666666666667, "delta_nll": 0.009055553636068789 }, { "seed": 22, "size": 100, "repeat": 13, "calibration_ids": [ 927, 1331, 824, 1780, 947, 102, 81, 809, 1296, 1246, 946, 300, 721, 813, 1562, 185, 218, 388, 1073, 16, 1357, 1280, 1389, 659, 570, 225, 1597, 153, 1530, 1088, 836, 1335, 1587, 1097, 978, 1048, 944, 1313, 1641, 183, 1787, 874, 1235, 273, 1362, 763, 1138, 125, 1067, 682, 284, 88, 995, 754, 986, 1233, 435, 65, 697, 1626, 490, 571, 1241, 574, 1690, 1726, 338, 1541, 601, 1778, 1745, 558, 206, 482, 802, 161, 1639, 149, 1588, 825, 546, 1111, 1666, 631, 1686, 1712, 1271, 865, 775, 400, 1372, 1076, 1023, 1651, 1652, 1586, 661, 1673, 178, 1090 ], "class_counts": [ 12, 9, 13, 12, 9, 6, 9, 10, 11, 9 ], "calibration_errors": 1, "temperature": 1.2800107615247256, "at_boundary": false, "fit_nll_before": 0.049291414310508916, "fit_nll_after": 0.04561185265279641, "test_nll": 0.1028937623174732, "test_ece": 0.01402557306574242, "test_accuracy": 0.9666666666666667, "delta_nll": -0.005198321530536343 }, { "seed": 22, "size": 100, "repeat": 14, "calibration_ids": [ 570, 737, 721, 1372, 1624, 754, 1228, 1271, 1023, 1213, 1216, 161, 1335, 1614, 664, 157, 763, 326, 519, 501, 1792, 1157, 263, 206, 222, 975, 1791, 1419, 770, 1090, 273, 809, 1644, 921, 1389, 775, 423, 1246, 1716, 23, 824, 1289, 1588, 511, 1022, 183, 969, 660, 340, 92, 978, 400, 1204, 1203, 661, 460, 103, 1598, 1093, 1048, 854, 1478, 391, 1179, 836, 1434, 155, 1076, 435, 176, 1597, 689, 303, 1121, 1521, 631, 1729, 1715, 1195, 808, 264, 1138, 122, 1459, 1088, 284, 825, 490, 1533, 682, 1745, 743, 1313, 1187, 788, 482, 829, 1363, 81, 1013 ], "class_counts": [ 12, 6, 9, 13, 9, 12, 6, 12, 13, 8 ], "calibration_errors": 3, "temperature": 1.5988979850778842, "at_boundary": false, "fit_nll_before": 0.15393913152741734, "fit_nll_after": 0.12828416081666627, "test_nll": 0.10745119910375892, "test_ece": 0.020525477758042827, "test_accuracy": 0.9666666666666667, "delta_nll": -0.0006408847442506277 }, { "seed": 22, "size": 100, "repeat": 15, "calibration_ids": [ 1130, 739, 153, 122, 858, 1048, 1171, 788, 222, 743, 675, 1725, 1076, 1335, 1550, 1251, 400, 1651, 546, 23, 1362, 509, 1296, 338, 1562, 829, 1478, 14, 682, 1459, 1262, 721, 1425, 412, 92, 81, 125, 1438, 1176, 1204, 1652, 969, 1282, 808, 1792, 541, 1363, 927, 1444, 34, 1630, 712, 1530, 1639, 482, 805, 854, 1777, 1073, 571, 168, 423, 523, 157, 235, 211, 558, 1149, 825, 1215, 1285, 1342, 1394, 1780, 947, 591, 1391, 1626, 566, 946, 161, 1614, 1195, 1716, 1541, 1456, 495, 65, 501, 178, 1271, 1280, 606, 361, 511, 1703, 218, 975, 754, 1022 ], "class_counts": [ 12, 10, 9, 7, 10, 5, 10, 9, 10, 18 ], "calibration_errors": 4, "temperature": 1.7593681002099724, "at_boundary": false, "fit_nll_before": 0.19018181270759288, "fit_nll_after": 0.1465220671847343, "test_nll": 0.11295716717837168, "test_ece": 0.02813121712141052, "test_accuracy": 0.9666666666666667, "delta_nll": 0.004865083330362133 }, { "seed": 22, "size": 100, "repeat": 16, "calibration_ids": [ 1241, 858, 263, 1614, 1521, 697, 1130, 1374, 1686, 1149, 1438, 177, 1335, 1624, 591, 946, 1741, 721, 1367, 1673, 1690, 480, 1530, 1171, 1268, 1470, 1216, 1718, 1496, 1251, 1377, 1138, 1563, 1097, 865, 1047, 14, 1562, 1228, 570, 1203, 314, 1296, 235, 1404, 1651, 601, 699, 1215, 1023, 1777, 1342, 340, 1111, 1280, 1716, 501, 360, 131, 391, 1792, 944, 1336, 1726, 81, 1389, 1246, 1204, 1597, 566, 1490, 324, 183, 1179, 541, 659, 1045, 125, 1271, 254, 423, 756, 809, 1363, 176, 682, 1738, 1394, 155, 300, 519, 1550, 97, 252, 178, 338, 927, 712, 836, 1688 ], "class_counts": [ 9, 10, 15, 11, 12, 13, 9, 6, 7, 8 ], "calibration_errors": 3, "temperature": 1.9697513412460728, "at_boundary": false, "fit_nll_before": 0.24569555165054763, "fit_nll_after": 0.17062637280643675, "test_nll": 0.12299748059930414, "test_ece": 0.03878719305188371, "test_accuracy": 0.9666666666666667, "delta_nll": 0.014905396751294597 }, { "seed": 22, "size": 100, "repeat": 17, "calibration_ids": [ 41, 25, 1335, 1013, 1285, 34, 153, 865, 519, 480, 1084, 1187, 324, 660, 1745, 1673, 737, 591, 252, 1367, 754, 1649, 218, 1022, 1216, 131, 836, 1470, 1262, 1541, 1235, 770, 1313, 1703, 1787, 809, 824, 1741, 501, 1374, 606, 65, 490, 780, 183, 432, 161, 1047, 1097, 1459, 400, 1331, 874, 1067, 1686, 986, 14, 882, 802, 682, 1213, 1171, 1073, 1563, 74, 1280, 1419, 1296, 921, 1204, 1251, 1111, 178, 1176, 675, 975, 745, 16, 775, 125, 1644, 808, 235, 854, 1149, 176, 1517, 1377, 296, 694, 1088, 300, 1614, 107, 756, 1533, 858, 103, 944, 1490 ], "class_counts": [ 14, 11, 6, 9, 12, 10, 10, 11, 7, 10 ], "calibration_errors": 4, "temperature": 1.8456148917435682, "at_boundary": false, "fit_nll_before": 0.23390258473694275, "fit_nll_after": 0.17453676149993086, "test_nll": 0.11670112163930704, "test_ece": 0.031165815221662545, "test_accuracy": 0.9666666666666667, "delta_nll": 0.0086090377912975 }, { "seed": 22, "size": 100, "repeat": 18, "calibration_ids": [ 775, 1586, 1179, 509, 436, 1391, 303, 432, 1389, 131, 1045, 206, 176, 1195, 756, 969, 805, 1235, 482, 1019, 149, 340, 523, 601, 921, 1521, 763, 573, 675, 1335, 423, 122, 324, 361, 824, 74, 1023, 501, 125, 1248, 1790, 1268, 1088, 809, 412, 108, 456, 211, 1380, 225, 1372, 1686, 802, 1251, 1048, 1725, 1624, 1138, 88, 230, 16, 254, 103, 178, 1204, 1470, 1729, 813, 23, 825, 1745, 905, 1651, 606, 519, 1639, 511, 1073, 263, 1215, 611, 1641, 244, 1404, 300, 218, 566, 338, 874, 155, 1250, 1246, 1121, 944, 745, 1792, 177, 1093, 1357, 381 ], "class_counts": [ 7, 13, 7, 9, 8, 9, 14, 11, 8, 14 ], "calibration_errors": 3, "temperature": 1.4797034523928367, "at_boundary": false, "fit_nll_before": 0.14496992602998568, "fit_nll_after": 0.12707703451718255, "test_nll": 0.10468093027714234, "test_ece": 0.019026129638668163, "test_accuracy": 0.9666666666666667, "delta_nll": -0.003411153570867201 }, { "seed": 22, "size": 100, "repeat": 19, "calibration_ids": [ 1121, 1235, 1787, 423, 1661, 412, 264, 1745, 1394, 1550, 153, 1022, 1690, 125, 570, 808, 1686, 805, 737, 1703, 1490, 1673, 1377, 1725, 699, 1517, 1726, 1313, 1715, 1262, 161, 969, 1130, 1374, 177, 1587, 1342, 14, 1067, 178, 986, 1530, 488, 1777, 211, 1084, 1076, 788, 296, 1791, 179, 1666, 490, 218, 511, 1614, 591, 682, 103, 1438, 1738, 546, 1149, 206, 631, 97, 1741, 1296, 975, 1790, 400, 1336, 81, 324, 460, 157, 921, 254, 775, 694, 432, 605, 176, 829, 16, 340, 1268, 1644, 1171, 1639, 1215, 745, 391, 1023, 601, 480, 107, 149, 1250, 300 ], "class_counts": [ 11, 9, 9, 10, 12, 12, 9, 7, 10, 11 ], "calibration_errors": 2, "temperature": 1.7567183217967917, "at_boundary": false, "fit_nll_before": 0.2112276106316779, "fit_nll_after": 0.1657984653481678, "test_nll": 0.11285059418239861, "test_ece": 0.028000805859618835, "test_accuracy": 0.9666666666666667, "delta_nll": 0.0047585103343890656 }, { "seed": 22, "size": 269, "repeat": 0, "calibration_ids": [ 211, 1157, 185, 699, 264, 946, 1715, 1048, 829, 721, 102, 495, 300, 14, 854, 456, 1282, 969, 1521, 1045, 360, 314, 244, 535, 1374, 1686, 754, 1778, 1389, 825, 1235, 1204, 921, 326, 1121, 1179, 1187, 1377, 611, 1213, 978, 1614, 1470, 178, 694, 558, 1067, 1047, 1268, 41, 149, 1550, 1280, 1588, 432, 809, 1019, 1787, 858, 1171, 225, 601, 770, 1248, 1241, 631, 1688, 1111, 412, 230, 1285, 1262, 153, 108, 125, 435, 511, 1419, 338, 571, 436, 729, 1146, 218, 361, 1624, 697, 490, 573, 944, 1587, 1246, 460, 541, 1088, 874, 1649, 813, 664, 1130, 1169, 1176, 836, 523, 97, 1562, 1630, 296, 927, 168, 583, 1425, 1666, 1690, 1216, 1149, 1563, 1372, 1138, 1661, 1289, 263, 1362, 574, 1335, 1296, 605, 1741, 743, 391, 775, 1394, 183, 1712, 675, 824, 1233, 660, 1626, 1490, 1729, 1745, 1434, 1703, 1530, 137, 482, 161, 1517, 1597, 1777, 737, 1738, 986, 1444, 254, 1541, 1792, 23, 1725, 802, 177, 1342, 222, 206, 1331, 1251, 606, 1533, 303, 1336, 1644, 1084, 1023, 975, 1203, 1641, 1250, 1228, 739, 882, 284, 423, 88, 81, 103, 712, 235, 1459, 1013, 509, 782, 1790, 1003, 682, 1076, 176, 591, 122, 1718, 1215, 659, 1195, 788, 1639, 808, 1404, 1097, 131, 1363, 1380, 566, 1780, 107, 157, 905, 899, 273, 1090, 570, 1586, 519, 1496, 179, 1716, 1652, 661, 1478, 763, 155, 865, 1391, 745, 1726, 16, 501, 546, 1456, 381, 92, 1093, 65, 324, 1598, 340, 1271, 1651, 780, 488, 480, 34, 995, 1022, 1357, 1791, 1073, 1313, 25, 805, 252, 689, 1367, 756, 1438, 947, 1673, 388, 74, 400 ], "class_counts": [ 27, 27, 27, 27, 27, 27, 27, 27, 26, 27 ], "calibration_errors": 7, "temperature": 1.6092275608405882, "at_boundary": false, "fit_nll_before": 0.15688863834184202, "fit_nll_after": 0.1295732366140301, "test_nll": 0.10774593556856667, "test_ece": 0.02388533000822119, "test_accuracy": 0.9666666666666667, "delta_nll": -0.00034614827944287607 }, { "seed": 33, "size": 20, "repeat": 0, "calibration_ids": [ 103, 1336, 326, 611, 177, 495, 570, 1246, 303, 222, 858, 157, 97, 546, 1093, 606, 737, 1419, 1179, 583 ], "class_counts": [ 2, 4, 2, 3, 2, 1, 4, 2, 0, 0 ], "calibration_errors": 1, "temperature": 0.5351349125730849, "at_boundary": false, "fit_nll_before": 0.047069273634118836, "fit_nll_after": 0.039784730593569774, "test_nll": 0.21874856703882675, "test_ece": 0.03327198552440991, "test_accuracy": 0.9611111111111111, "delta_nll": 0.07767166362020708 }, { "seed": 33, "size": 20, "repeat": 1, "calibration_ids": [ 1335, 969, 1296, 782, 391, 1233, 1362, 1780, 412, 179, 1176, 480, 273, 435, 1073, 490, 802, 314, 1048, 381 ], "class_counts": [ 3, 0, 2, 0, 0, 1, 6, 3, 1, 4 ], "calibration_errors": 1, "temperature": 2.040014837295968, "at_boundary": false, "fit_nll_before": 0.37056209918733624, "fit_nll_after": 0.25625206970807196, "test_nll": 0.14853813766128038, "test_ece": 0.03926348745477297, "test_accuracy": 0.9611111111111111, "delta_nll": 0.007461234242660708 }, { "seed": 33, "size": 20, "repeat": 2, "calibration_ids": [ 739, 775, 802, 1459, 1023, 1374, 675, 1745, 1215, 1716, 1729, 1362, 1233, 780, 423, 1712, 1045, 225, 1241, 659 ], "class_counts": [ 2, 1, 2, 3, 4, 0, 3, 1, 2, 2 ], "calibration_errors": 2, "temperature": 1.9649217684459594, "at_boundary": false, "fit_nll_before": 0.4736007997490675, "fit_nll_after": 0.35246543968784366, "test_nll": 0.1443427438573958, "test_ece": 0.03881206411446797, "test_accuracy": 0.9611111111111111, "delta_nll": 0.0032658404387761186 }, { "seed": 33, "size": 20, "repeat": 3, "calibration_ids": [ 157, 300, 519, 436, 185, 1179, 178, 273, 264, 1363, 1169, 1090, 495, 1729, 1313, 88, 660, 107, 1047, 81 ], "class_counts": [ 2, 1, 3, 3, 1, 1, 2, 4, 3, 0 ], "calibration_errors": 1, "temperature": 2.3138860316116414, "at_boundary": false, "fit_nll_before": 0.4017832262194364, "fit_nll_after": 0.24228076315469177, "test_nll": 0.16731023772564232, "test_ece": 0.06089657441145819, "test_accuracy": 0.9611111111111111, "delta_nll": 0.026233334307022654 }, { "seed": 33, "size": 20, "repeat": 4, "calibration_ids": [ 1362, 1777, 1787, 1138, 1176, 1048, 1690, 1715, 81, 809, 1251, 1045, 511, 501, 65, 1289, 1703, 805, 1067, 1169 ], "class_counts": [ 2, 0, 4, 1, 2, 2, 2, 2, 1, 4 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.005844718040721093, "fit_nll_after": 4.725964594247801e-25, "test_nll": 2.1782317635654924, "test_ece": 0.04003324704965055, "test_accuracy": 0.9611111111111111, "delta_nll": 2.037154860146873 }, { "seed": 33, "size": 20, "repeat": 5, "calibration_ids": [ 1268, 1649, 511, 1363, 946, 1456, 770, 456, 1023, 739, 211, 802, 179, 1215, 1250, 1076, 1521, 1715, 41, 805 ], "class_counts": [ 2, 4, 0, 0, 6, 0, 3, 2, 1, 2 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.02022523897253131, "fit_nll_after": 2.400382552497569e-12, "test_nll": 2.1782317635654924, "test_ece": 0.04003324704965055, "test_accuracy": 0.9611111111111111, "delta_nll": 2.037154860146873 }, { "seed": 33, "size": 20, "repeat": 6, "calibration_ids": [ 788, 1331, 1651, 729, 523, 780, 81, 436, 1598, 218, 1176, 25, 168, 1673, 969, 535, 423, 211, 1013, 1003 ], "class_counts": [ 1, 1, 2, 1, 2, 2, 2, 5, 1, 3 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.015336134063500987, "fit_nll_after": 1.7024434630322043e-17, "test_nll": 2.1782317635654924, "test_ece": 0.04003324704965055, "test_accuracy": 0.9611111111111111, "delta_nll": 2.037154860146873 }, { "seed": 33, "size": 20, "repeat": 7, "calibration_ids": [ 456, 23, 296, 1157, 1271, 664, 739, 743, 1791, 1787, 605, 125, 1444, 1203, 1587, 1651, 1171, 591, 1213, 1023 ], "class_counts": [ 1, 3, 1, 2, 5, 2, 0, 1, 3, 2 ], "calibration_errors": 1, "temperature": 1.9175183982139201, "at_boundary": false, "fit_nll_before": 0.22398800087103105, "fit_nll_after": 0.1565199024647413, "test_nll": 0.14192147694962454, "test_ece": 0.030915211470739064, "test_accuracy": 0.9611111111111111, "delta_nll": 0.0008445735310048663 }, { "seed": 33, "size": 20, "repeat": 8, "calibration_ids": [ 1235, 1228, 1013, 1745, 1666, 1718, 495, 185, 1478, 1587, 460, 1738, 300, 660, 882, 412, 324, 103, 573, 1362 ], "class_counts": [ 3, 0, 3, 2, 2, 4, 2, 3, 1, 0 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.014354869681155064, "fit_nll_after": 7.14858337730272e-15, "test_nll": 2.1782317635654924, "test_ece": 0.04003324704965055, "test_accuracy": 0.9611111111111111, "delta_nll": 2.037154860146873 }, { "seed": 33, "size": 20, "repeat": 9, "calibration_ids": [ 1146, 694, 244, 1138, 1787, 675, 1686, 412, 788, 326, 225, 1791, 660, 391, 1630, 1084, 1013, 1652, 381, 1271 ], "class_counts": [ 1, 1, 3, 1, 5, 2, 1, 1, 1, 4 ], "calibration_errors": 1, "temperature": 1.8455471733470818, "at_boundary": false, "fit_nll_before": 0.21927806575095204, "fit_nll_after": 0.15308529849237368, "test_nll": 0.13859438045015213, "test_ece": 0.03076623167647336, "test_accuracy": 0.9611111111111111, "delta_nll": -0.002482522968467543 }, { "seed": 33, "size": 20, "repeat": 10, "calibration_ids": [ 264, 1013, 108, 361, 1076, 432, 1268, 858, 1661, 1022, 338, 874, 824, 1726, 340, 1780, 882, 969, 1289, 1791 ], "class_counts": [ 1, 1, 3, 2, 4, 1, 4, 2, 1, 1 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.009465437244578892, "fit_nll_after": 2.1740602046187634e-18, "test_nll": 2.1782317635654924, "test_ece": 0.04003324704965055, "test_accuracy": 0.9611111111111111, "delta_nll": 2.037154860146873 }, { "seed": 33, "size": 20, "repeat": 11, "calibration_ids": [ 675, 605, 1084, 927, 1362, 388, 155, 763, 1651, 1394, 1630, 303, 1530, 541, 546, 149, 1490, 1250, 682, 381 ], "class_counts": [ 2, 3, 6, 2, 1, 3, 0, 0, 0, 3 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.005284852208254434, "fit_nll_after": 7.291688364072062e-23, "test_nll": 2.1782317635654924, "test_ece": 0.04003324704965055, "test_accuracy": 0.9611111111111111, "delta_nll": 2.037154860146873 }, { "seed": 33, "size": 20, "repeat": 12, "calibration_ids": [ 1673, 1787, 1587, 1097, 74, 729, 1630, 41, 1335, 1171, 978, 273, 946, 675, 1741, 1729, 541, 743, 179, 1550 ], "class_counts": [ 2, 1, 0, 3, 4, 5, 1, 2, 1, 1 ], "calibration_errors": 1, "temperature": 2.283586742600672, "at_boundary": false, "fit_nll_before": 0.3925594785197805, "fit_nll_after": 0.22555356386942144, "test_nll": 0.1649810054693859, "test_ece": 0.05853338417826437, "test_accuracy": 0.9611111111111111, "delta_nll": 0.023904102050766235 }, { "seed": 33, "size": 20, "repeat": 13, "calibration_ids": [ 927, 1331, 824, 1780, 947, 102, 81, 809, 1296, 1246, 946, 300, 721, 813, 1562, 185, 218, 388, 1073, 16 ], "class_counts": [ 2, 2, 4, 1, 1, 2, 1, 4, 0, 3 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.012994106163019154, "fit_nll_after": 1.0399327578177657e-13, "test_nll": 2.1782317635654924, "test_ece": 0.04003324704965055, "test_accuracy": 0.9611111111111111, "delta_nll": 2.037154860146873 }, { "seed": 33, "size": 20, "repeat": 14, "calibration_ids": [ 570, 737, 721, 1372, 1624, 754, 1228, 1271, 1023, 1213, 1216, 161, 1335, 1614, 664, 157, 763, 326, 519, 501 ], "class_counts": [ 1, 3, 3, 4, 1, 3, 0, 2, 2, 1 ], "calibration_errors": 1, "temperature": 1.8608517294174731, "at_boundary": false, "fit_nll_before": 0.22764120980032154, "fit_nll_after": 0.1642846893597126, "test_nll": 0.1392659104327523, "test_ece": 0.031540067924182034, "test_accuracy": 0.9611111111111111, "delta_nll": -0.0018109929858673712 }, { "seed": 33, "size": 20, "repeat": 15, "calibration_ids": [ 1130, 739, 153, 122, 858, 1048, 1171, 788, 222, 743, 675, 1725, 1076, 1335, 1550, 1251, 400, 1651, 546, 23 ], "class_counts": [ 2, 2, 1, 3, 4, 1, 2, 2, 1, 2 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.002682652902229838, "fit_nll_after": 2.876161688204008e-28, "test_nll": 2.1782317635654924, "test_ece": 0.04003324704965055, "test_accuracy": 0.9611111111111111, "delta_nll": 2.037154860146873 }, { "seed": 33, "size": 20, "repeat": 16, "calibration_ids": [ 1241, 858, 263, 1614, 1521, 697, 1130, 1374, 1686, 1149, 1438, 177, 1335, 1624, 591, 946, 1741, 721, 1367, 1673 ], "class_counts": [ 1, 2, 4, 3, 2, 2, 3, 1, 1, 1 ], "calibration_errors": 1, "temperature": 2.33383614661209, "at_boundary": false, "fit_nll_before": 0.5016661771527644, "fit_nll_after": 0.3187909685151765, "test_nll": 0.16887630128841807, "test_ece": 0.06247021500351165, "test_accuracy": 0.9611111111111111, "delta_nll": 0.027799397869798403 }, { "seed": 33, "size": 20, "repeat": 17, "calibration_ids": [ 41, 25, 1335, 1013, 1285, 34, 153, 865, 519, 480, 1084, 1187, 324, 660, 1745, 1673, 737, 591, 252, 1367 ], "class_counts": [ 4, 1, 2, 4, 3, 1, 2, 2, 0, 1 ], "calibration_errors": 1, "temperature": 2.06914576592321, "at_boundary": false, "fit_nll_before": 0.38312167522710405, "fit_nll_after": 0.2691014888049077, "test_nll": 0.15028102525008538, "test_ece": 0.043197689637293216, "test_accuracy": 0.9611111111111111, "delta_nll": 0.009204121831465711 }, { "seed": 33, "size": 20, "repeat": 18, "calibration_ids": [ 775, 1586, 1179, 509, 436, 1391, 303, 432, 1389, 131, 1045, 206, 176, 1195, 756, 969, 805, 1235, 482, 1019 ], "class_counts": [ 1, 2, 2, 0, 1, 3, 3, 2, 5, 1 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.03962406859543301, "fit_nll_after": 0.007628052097700411, "test_nll": 2.1782317635654924, "test_ece": 0.04003324704965055, "test_accuracy": 0.9611111111111111, "delta_nll": 2.037154860146873 }, { "seed": 33, "size": 20, "repeat": 19, "calibration_ids": [ 1121, 1235, 1787, 423, 1661, 412, 264, 1745, 1394, 1550, 153, 1022, 1690, 125, 570, 808, 1686, 805, 737, 1703 ], "class_counts": [ 3, 1, 1, 3, 2, 3, 1, 1, 1, 4 ], "calibration_errors": 1, "temperature": 0.9899377097115856, "at_boundary": false, "fit_nll_before": 0.08867830167542924, "fit_nll_after": 0.08867005090789273, "test_nll": 0.14177862350999354, "test_ece": 0.028536769024291846, "test_accuracy": 0.9611111111111111, "delta_nll": 0.0007017200913738697 }, { "seed": 33, "size": 50, "repeat": 0, "calibration_ids": [ 103, 1336, 326, 611, 177, 495, 570, 1246, 303, 222, 858, 157, 97, 546, 1093, 606, 737, 1419, 1179, 583, 947, 1121, 183, 1372, 436, 1146, 131, 456, 460, 340, 1490, 697, 16, 273, 400, 324, 153, 381, 1251, 825, 1313, 865, 1171, 1067, 74, 1342, 108, 388, 1047, 1380 ], "class_counts": [ 4, 9, 7, 5, 5, 3, 6, 6, 3, 2 ], "calibration_errors": 1, "temperature": 0.5224454698734423, "at_boundary": false, "fit_nll_before": 0.02061670302290176, "fit_nll_after": 0.015942221064784393, "test_nll": 0.22326015181666514, "test_ece": 0.033825062747533476, "test_accuracy": 0.9611111111111111, "delta_nll": 0.08218324839804547 }, { "seed": 33, "size": 50, "repeat": 1, "calibration_ids": [ 1335, 969, 1296, 782, 391, 1233, 1362, 1780, 412, 179, 1176, 480, 273, 435, 1073, 490, 802, 314, 1048, 381, 235, 183, 326, 108, 1586, 944, 1738, 296, 809, 946, 1380, 1367, 300, 185, 1726, 1644, 1425, 1725, 1550, 1703, 1280, 882, 1342, 659, 1336, 535, 1076, 1216, 1741, 1203 ], "class_counts": [ 8, 5, 2, 4, 1, 6, 8, 6, 4, 6 ], "calibration_errors": 1, "temperature": 1.642090090410328, "at_boundary": false, "fit_nll_before": 0.15086737826168709, "fit_nll_after": 0.12032971674180487, "test_nll": 0.13162535516660875, "test_ece": 0.027051716201361733, "test_accuracy": 0.9611111111111111, "delta_nll": -0.009451548252010916 }, { "seed": 33, "size": 50, "repeat": 2, "calibration_ids": [ 739, 775, 802, 1459, 1023, 1374, 675, 1745, 1215, 1716, 1729, 1362, 1233, 780, 423, 1712, 1045, 225, 1241, 659, 1090, 1246, 566, 605, 921, 699, 546, 1363, 978, 490, 222, 1521, 1377, 1550, 361, 1121, 986, 1013, 14, 1562, 570, 1587, 1203, 1336, 1342, 611, 179, 1661, 1496, 899 ], "class_counts": [ 6, 2, 5, 6, 6, 2, 7, 6, 6, 4 ], "calibration_errors": 2, "temperature": 1.581929911248967, "at_boundary": false, "fit_nll_before": 0.19796372979598792, "fit_nll_after": 0.16928040440863154, "test_nll": 0.1303090174191942, "test_ece": 0.027275181536475217, "test_accuracy": 0.9611111111111111, "delta_nll": -0.010767885999425464 }, { "seed": 33, "size": 50, "repeat": 3, "calibration_ids": [ 157, 300, 519, 436, 185, 1179, 178, 273, 264, 1363, 1169, 1090, 495, 1729, 1313, 88, 660, 107, 1047, 81, 176, 1067, 263, 1121, 1377, 611, 254, 1003, 1404, 1246, 488, 1597, 1792, 1331, 1652, 340, 1686, 825, 1563, 1478, 1712, 161, 591, 1296, 1790, 865, 874, 1213, 1216, 222 ], "class_counts": [ 4, 3, 6, 9, 2, 5, 3, 8, 5, 5 ], "calibration_errors": 1, "temperature": 1.7565257859873025, "at_boundary": false, "fit_nll_before": 0.17144730852193404, "fit_nll_after": 0.13216429946254843, "test_nll": 0.13508659551923435, "test_ece": 0.026206685439995405, "test_accuracy": 0.9611111111111111, "delta_nll": -0.0059903078993853165 }, { "seed": 33, "size": 50, "repeat": 4, "calibration_ids": [ 1362, 1777, 1787, 1138, 1176, 1048, 1690, 1715, 81, 809, 1251, 1045, 511, 501, 65, 1289, 1703, 805, 1067, 1169, 689, 763, 782, 1438, 756, 1250, 836, 788, 1530, 381, 905, 1367, 1394, 1651, 1478, 1271, 854, 1171, 721, 570, 566, 244, 1673, 1496, 1630, 1644, 179, 235, 1228, 211 ], "class_counts": [ 4, 4, 10, 6, 5, 4, 4, 5, 3, 5 ], "calibration_errors": 2, "temperature": 1.7960067980747199, "at_boundary": false, "fit_nll_before": 0.1918875938214604, "fit_nll_after": 0.13877577737656976, "test_nll": 0.13655726509477203, "test_ece": 0.02922793897250148, "test_accuracy": 0.9611111111111111, "delta_nll": -0.004519638323847636 }, { "seed": 33, "size": 50, "repeat": 5, "calibration_ids": [ 1268, 1649, 511, 1363, 946, 1456, 770, 456, 1023, 739, 211, 802, 179, 1215, 1250, 1076, 1521, 1715, 41, 805, 865, 177, 176, 157, 400, 206, 675, 1496, 1588, 699, 92, 183, 185, 284, 1688, 1745, 606, 1045, 1777, 97, 1090, 1703, 1233, 25, 1587, 1533, 1213, 721, 1741, 361 ], "class_counts": [ 5, 8, 1, 3, 9, 3, 4, 6, 6, 5 ], "calibration_errors": 1, "temperature": 0.26209208400033346, "at_boundary": false, "fit_nll_before": 0.04419466372893334, "fit_nll_after": 0.028415406865594593, "test_nll": 0.42184688430215134, "test_ece": 0.036652256571655134, "test_accuracy": 0.9611111111111111, "delta_nll": 0.28076998088353167 }, { "seed": 33, "size": 50, "repeat": 6, "calibration_ids": [ 788, 1331, 1651, 729, 523, 780, 81, 436, 1598, 218, 1176, 25, 168, 1673, 969, 535, 423, 211, 1013, 1003, 1639, 1715, 1251, 1204, 1336, 1718, 1394, 88, 1241, 412, 284, 664, 1363, 1541, 1652, 264, 519, 1233, 809, 1729, 1738, 836, 254, 361, 1195, 858, 65, 802, 16, 314 ], "class_counts": [ 4, 3, 4, 5, 3, 4, 9, 6, 7, 5 ], "calibration_errors": 1, "temperature": 1.517696449391817, "at_boundary": false, "fit_nll_before": 0.17254138286926396, "fit_nll_after": 0.14795991205520864, "test_nll": 0.12931123600927658, "test_ece": 0.022778838459510474, "test_accuracy": 0.9611111111111111, "delta_nll": -0.011765667409343095 }, { "seed": 33, "size": 50, "repeat": 7, "calibration_ids": [ 456, 23, 296, 1157, 1271, 664, 739, 743, 1791, 1787, 605, 125, 1444, 1203, 1587, 1651, 1171, 591, 1213, 1023, 1282, 1331, 252, 571, 1456, 225, 1076, 825, 179, 263, 168, 611, 1438, 566, 1215, 1639, 176, 541, 460, 1530, 1251, 1517, 805, 1541, 1716, 780, 1313, 1248, 381, 1459 ], "class_counts": [ 7, 4, 3, 5, 8, 6, 3, 5, 4, 5 ], "calibration_errors": 2, "temperature": 1.670571873348447, "at_boundary": false, "fit_nll_before": 0.11680718879265747, "fit_nll_after": 0.09415857698621681, "test_nll": 0.13237211438144517, "test_ece": 0.026382118014025263, "test_accuracy": 0.9611111111111111, "delta_nll": -0.008704789037174504 }, { "seed": 33, "size": 50, "repeat": 8, "calibration_ids": [ 1235, 1228, 1013, 1745, 1666, 1718, 495, 185, 1478, 1587, 460, 1738, 300, 660, 882, 412, 324, 103, 573, 1362, 263, 1067, 1187, 206, 1597, 1146, 1130, 1149, 65, 1389, 1716, 235, 1652, 1357, 222, 995, 1586, 108, 1490, 605, 1246, 809, 1778, 14, 1641, 1363, 1649, 1391, 1726, 541 ], "class_counts": [ 5, 2, 5, 6, 6, 7, 4, 9, 5, 1 ], "calibration_errors": 1, "temperature": 1.788013210641176, "at_boundary": false, "fit_nll_before": 0.223305202288822, "fit_nll_after": 0.17661392535039222, "test_nll": 0.13624842518354044, "test_ece": 0.028840983961705067, "test_accuracy": 0.9611111111111111, "delta_nll": -0.004828478235079231 }, { "seed": 33, "size": 50, "repeat": 9, "calibration_ids": [ 1146, 694, 244, 1138, 1787, 675, 1686, 412, 788, 326, 225, 1791, 660, 391, 1630, 1084, 1013, 1652, 381, 1271, 712, 284, 1296, 1216, 423, 986, 436, 1587, 177, 975, 1121, 65, 1778, 1342, 1235, 1434, 273, 780, 1391, 978, 1019, 360, 1130, 1729, 108, 523, 432, 1777, 338, 829 ], "class_counts": [ 3, 2, 5, 4, 8, 3, 6, 7, 4, 8 ], "calibration_errors": 2, "temperature": 1.8822258194469788, "at_boundary": false, "fit_nll_before": 0.25098842731412324, "fit_nll_after": 0.17745106562096055, "test_nll": 0.14023650669095405, "test_ece": 0.031130639427260562, "test_accuracy": 0.9611111111111111, "delta_nll": -0.0008403967276656155 }, { "seed": 33, "size": 50, "repeat": 10, "calibration_ids": [ 264, 1013, 108, 361, 1076, 432, 1268, 858, 1661, 1022, 338, 874, 824, 1726, 340, 1780, 882, 969, 1289, 1791, 263, 1377, 1171, 1688, 488, 1778, 944, 1363, 157, 81, 1738, 230, 1626, 1391, 1588, 1434, 1517, 1718, 179, 946, 802, 546, 921, 1716, 660, 1048, 1187, 1587, 1146, 273 ], "class_counts": [ 5, 4, 4, 3, 8, 5, 7, 7, 2, 5 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.010983230243725144, "fit_nll_after": 3.750001084029303e-15, "test_nll": 2.1782317635654924, "test_ece": 0.04003324704965055, "test_accuracy": 0.9611111111111111, "delta_nll": 2.037154860146873 }, { "seed": 33, "size": 50, "repeat": 11, "calibration_ids": [ 675, 605, 1084, 927, 1362, 388, 155, 763, 1651, 1394, 1630, 303, 1530, 541, 546, 149, 1490, 1250, 682, 381, 694, 1550, 1444, 1157, 1626, 754, 432, 1019, 1271, 1726, 1067, 490, 583, 74, 235, 1716, 34, 743, 745, 206, 1169, 1342, 1262, 1216, 535, 456, 1363, 1391, 391, 177 ], "class_counts": [ 6, 8, 7, 4, 2, 7, 4, 2, 4, 6 ], "calibration_errors": 1, "temperature": 1.398617661715386, "at_boundary": false, "fit_nll_before": 0.10664539680417498, "fit_nll_after": 0.09558148150878781, "test_nll": 0.1286609589796804, "test_ece": 0.019428475881255246, "test_accuracy": 0.9611111111111111, "delta_nll": -0.012415944438939258 }, { "seed": 33, "size": 50, "repeat": 12, "calibration_ids": [ 1673, 1787, 1587, 1097, 74, 729, 1630, 41, 1335, 1171, 978, 273, 946, 675, 1741, 1729, 541, 743, 179, 1550, 1563, 1195, 81, 360, 137, 1149, 1088, 1111, 802, 254, 16, 1688, 1404, 1716, 1626, 1342, 606, 296, 1296, 244, 882, 1456, 813, 1614, 340, 1203, 435, 388, 1791, 432 ], "class_counts": [ 6, 4, 4, 3, 6, 9, 5, 5, 4, 4 ], "calibration_errors": 3, "temperature": 2.098624800744334, "at_boundary": false, "fit_nll_before": 0.3789189834497263, "fit_nll_after": 0.26011481211359266, "test_nll": 0.15210875131392365, "test_ece": 0.045413112853439, "test_accuracy": 0.9611111111111111, "delta_nll": 0.011031847895303981 }, { "seed": 33, "size": 50, "repeat": 13, "calibration_ids": [ 927, 1331, 824, 1780, 947, 102, 81, 809, 1296, 1246, 946, 300, 721, 813, 1562, 185, 218, 388, 1073, 16, 1357, 1280, 1389, 659, 570, 225, 1597, 153, 1530, 1088, 836, 1335, 1587, 1097, 978, 1048, 944, 1313, 1641, 183, 1787, 874, 1235, 273, 1362, 763, 1138, 125, 1067, 682 ], "class_counts": [ 5, 4, 8, 5, 4, 5, 2, 7, 4, 6 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.00637478347927228, "fit_nll_after": 4.159731031271063e-14, "test_nll": 2.1782317635654924, "test_ece": 0.04003324704965055, "test_accuracy": 0.9611111111111111, "delta_nll": 2.037154860146873 }, { "seed": 33, "size": 50, "repeat": 14, "calibration_ids": [ 570, 737, 721, 1372, 1624, 754, 1228, 1271, 1023, 1213, 1216, 161, 1335, 1614, 664, 157, 763, 326, 519, 501, 1792, 1157, 263, 206, 222, 975, 1791, 1419, 770, 1090, 273, 809, 1644, 921, 1389, 775, 423, 1246, 1716, 23, 824, 1289, 1588, 511, 1022, 183, 969, 660, 340, 92 ], "class_counts": [ 4, 3, 5, 9, 6, 5, 2, 5, 5, 6 ], "calibration_errors": 1, "temperature": 1.3579257590911724, "at_boundary": false, "fit_nll_before": 0.11597123654670784, "fit_nll_after": 0.10773489152207322, "test_nll": 0.12882439502585005, "test_ece": 0.024612822019102584, "test_accuracy": 0.9611111111111111, "delta_nll": -0.012252508392769618 }, { "seed": 33, "size": 50, "repeat": 15, "calibration_ids": [ 1130, 739, 153, 122, 858, 1048, 1171, 788, 222, 743, 675, 1725, 1076, 1335, 1550, 1251, 400, 1651, 546, 23, 1362, 509, 1296, 338, 1562, 829, 1478, 14, 682, 1459, 1262, 721, 1425, 412, 92, 81, 125, 1438, 1176, 1204, 1652, 969, 1282, 808, 1792, 541, 1363, 927, 1444, 34 ], "class_counts": [ 4, 3, 4, 5, 6, 3, 6, 4, 4, 11 ], "calibration_errors": 2, "temperature": 1.2371428383805552, "at_boundary": false, "fit_nll_before": 0.05570932743422648, "fit_nll_after": 0.053680107740824566, "test_nll": 0.13060447293882432, "test_ece": 0.02291690696086538, "test_accuracy": 0.9611111111111111, "delta_nll": -0.010472430479795347 }, { "seed": 33, "size": 50, "repeat": 16, "calibration_ids": [ 1241, 858, 263, 1614, 1521, 697, 1130, 1374, 1686, 1149, 1438, 177, 1335, 1624, 591, 946, 1741, 721, 1367, 1673, 1690, 480, 1530, 1171, 1268, 1470, 1216, 1718, 1496, 1251, 1377, 1138, 1563, 1097, 865, 1047, 14, 1562, 1228, 570, 1203, 314, 1296, 235, 1404, 1651, 601, 699, 1215, 1023 ], "class_counts": [ 3, 6, 7, 6, 8, 5, 5, 4, 3, 3 ], "calibration_errors": 2, "temperature": 2.0954730494062703, "at_boundary": false, "fit_nll_before": 0.3472464191998281, "fit_nll_after": 0.2324816696824339, "test_nll": 0.1519102976237564, "test_ece": 0.045984652026734, "test_accuracy": 0.9611111111111111, "delta_nll": 0.010833394205136737 }, { "seed": 33, "size": 50, "repeat": 17, "calibration_ids": [ 41, 25, 1335, 1013, 1285, 34, 153, 865, 519, 480, 1084, 1187, 324, 660, 1745, 1673, 737, 591, 252, 1367, 754, 1649, 218, 1022, 1216, 131, 836, 1470, 1262, 1541, 1235, 770, 1313, 1703, 1787, 809, 824, 1741, 501, 1374, 606, 65, 490, 780, 183, 432, 161, 1047, 1097, 1459 ], "class_counts": [ 9, 5, 3, 6, 7, 5, 5, 5, 2, 3 ], "calibration_errors": 3, "temperature": 1.5505189691303827, "at_boundary": false, "fit_nll_before": 0.20523488933332815, "fit_nll_after": 0.1804847468310272, "test_nll": 0.1297671095429989, "test_ece": 0.022341021725213846, "test_accuracy": 0.9611111111111111, "delta_nll": -0.011309793875620783 }, { "seed": 33, "size": 50, "repeat": 18, "calibration_ids": [ 775, 1586, 1179, 509, 436, 1391, 303, 432, 1389, 131, 1045, 206, 176, 1195, 756, 969, 805, 1235, 482, 1019, 149, 340, 523, 601, 921, 1521, 763, 573, 675, 1335, 423, 122, 324, 361, 824, 74, 1023, 501, 125, 1248, 1790, 1268, 1088, 809, 412, 108, 456, 211, 1380, 225 ], "class_counts": [ 3, 5, 5, 1, 5, 6, 6, 6, 7, 6 ], "calibration_errors": 0, "temperature": 0.05, "at_boundary": true, "fit_nll_before": 0.02643261896749769, "fit_nll_after": 0.003051222676222426, "test_nll": 2.1782317635654924, "test_ece": 0.04003324704965055, "test_accuracy": 0.9611111111111111, "delta_nll": 2.037154860146873 }, { "seed": 33, "size": 50, "repeat": 19, "calibration_ids": [ 1121, 1235, 1787, 423, 1661, 412, 264, 1745, 1394, 1550, 153, 1022, 1690, 125, 570, 808, 1686, 805, 737, 1703, 1490, 1673, 1377, 1725, 699, 1517, 1726, 1313, 1715, 1262, 161, 969, 1130, 1374, 177, 1587, 1342, 14, 1067, 178, 986, 1530, 488, 1777, 211, 1084, 1076, 788, 296, 1791 ], "class_counts": [ 6, 4, 6, 5, 6, 5, 5, 3, 4, 6 ], "calibration_errors": 1, "temperature": 0.8761936396018407, "at_boundary": false, "fit_nll_before": 0.0388145453093855, "fit_nll_after": 0.038082043981484605, "test_nll": 0.15163957241541057, "test_ece": 0.027685927260445827, "test_accuracy": 0.9611111111111111, "delta_nll": 0.010562668996790897 }, { "seed": 33, "size": 100, "repeat": 0, "calibration_ids": [ 103, 1336, 326, 611, 177, 495, 570, 1246, 303, 222, 858, 157, 97, 546, 1093, 606, 737, 1419, 1179, 583, 947, 1121, 183, 1372, 436, 1146, 131, 456, 460, 340, 1490, 697, 16, 273, 400, 324, 153, 381, 1251, 825, 1313, 865, 1171, 1067, 74, 1342, 108, 388, 1047, 1380, 743, 699, 1003, 300, 1425, 211, 122, 836, 905, 1285, 168, 1530, 775, 573, 1138, 488, 435, 1738, 41, 1048, 296, 14, 523, 92, 155, 694, 1586, 1644, 1792, 360, 571, 137, 1686, 81, 1470, 1362, 263, 1241, 1588, 161, 88, 235, 230, 745, 1716, 1703, 899, 501, 1250, 829 ], "class_counts": [ 11, 12, 12, 8, 9, 8, 8, 13, 11, 8 ], "calibration_errors": 2, "temperature": 1.27405294185175, "at_boundary": false, "fit_nll_before": 0.06419620457253683, "fit_nll_after": 0.06018751737583648, "test_nll": 0.12984322442847263, "test_ece": 0.023101284132944752, "test_accuracy": 0.9611111111111111, "delta_nll": -0.011233678990147045 }, { "seed": 33, "size": 100, "repeat": 1, "calibration_ids": [ 1335, 969, 1296, 782, 391, 1233, 1362, 1780, 412, 179, 1176, 480, 273, 435, 1073, 490, 802, 314, 1048, 381, 235, 183, 326, 108, 1586, 944, 1738, 296, 809, 946, 1380, 1367, 300, 185, 1726, 1644, 1425, 1725, 1550, 1703, 1280, 882, 1342, 659, 1336, 535, 1076, 1216, 1741, 1203, 177, 1715, 905, 858, 1246, 263, 805, 97, 1282, 675, 25, 1090, 1084, 1313, 102, 1470, 1228, 682, 1444, 423, 168, 1419, 1377, 756, 697, 1649, 1157, 661, 155, 1639, 1215, 788, 1690, 1235, 1787, 978, 1652, 495, 1262, 92, 566, 921, 1490, 1562, 1121, 103, 340, 1651, 1716, 338 ], "class_counts": [ 14, 7, 8, 9, 6, 13, 13, 9, 7, 14 ], "calibration_errors": 2, "temperature": 1.5903106415687107, "at_boundary": false, "fit_nll_before": 0.12898008375974732, "fit_nll_after": 0.10465730795842813, "test_nll": 0.13047070000221828, "test_ece": 0.027498506489919652, "test_accuracy": 0.9611111111111111, "delta_nll": -0.010606203416401394 }, { "seed": 33, "size": 100, "repeat": 2, "calibration_ids": [ 739, 775, 802, 1459, 1023, 1374, 675, 1745, 1215, 1716, 1729, 1362, 1233, 780, 423, 1712, 1045, 225, 1241, 659, 1090, 1246, 566, 605, 921, 699, 546, 1363, 978, 490, 222, 1521, 1377, 1550, 361, 1121, 986, 1013, 14, 1562, 570, 1587, 1203, 1336, 1342, 611, 179, 1661, 1496, 899, 519, 412, 1718, 97, 1777, 1434, 273, 456, 1282, 1703, 1588, 558, 721, 177, 1130, 1313, 1726, 1725, 1644, 1357, 1019, 1022, 1111, 1251, 975, 1248, 34, 1235, 509, 1614, 23, 1478, 264, 1792, 161, 314, 1067, 149, 103, 1176, 244, 829, 1790, 1641, 400, 1688, 782, 743, 535, 1367 ], "class_counts": [ 8, 8, 9, 15, 12, 3, 13, 9, 11, 12 ], "calibration_errors": 2, "temperature": 1.2096928828601408, "at_boundary": false, "fit_nll_before": 0.11141565183475337, "fit_nll_after": 0.10796460898572598, "test_nll": 0.13130392247637832, "test_ece": 0.02258947034371921, "test_accuracy": 0.9611111111111111, "delta_nll": -0.009772980942241355 }, { "seed": 33, "size": 100, "repeat": 3, "calibration_ids": [ 157, 300, 519, 436, 185, 1179, 178, 273, 264, 1363, 1169, 1090, 495, 1729, 1313, 88, 660, 107, 1047, 81, 176, 1067, 263, 1121, 1377, 611, 254, 1003, 1404, 1246, 488, 1597, 1792, 1331, 1652, 340, 1686, 825, 1563, 1478, 1712, 161, 591, 1296, 1790, 865, 874, 1213, 1216, 222, 1470, 1490, 1271, 1716, 16, 1688, 1438, 1521, 1587, 858, 1073, 103, 125, 975, 729, 1285, 137, 802, 1673, 739, 284, 946, 1726, 230, 1434, 1171, 1703, 854, 1459, 1533, 1149, 1389, 1394, 882, 1335, 808, 697, 1690, 1787, 813, 1262, 1342, 1157, 432, 149, 605, 1419, 131, 400, 1138 ], "class_counts": [ 11, 7, 8, 15, 7, 10, 9, 13, 8, 12 ], "calibration_errors": 5, "temperature": 1.8292506921225693, "at_boundary": false, "fit_nll_before": 0.26088772105105806, "fit_nll_after": 0.19890436278797557, "test_nll": 0.13790107436005156, "test_ece": 0.03304452148337709, "test_accuracy": 0.9611111111111111, "delta_nll": -0.0031758290585681137 }, { "seed": 33, "size": 100, "repeat": 4, "calibration_ids": [ 1362, 1777, 1787, 1138, 1176, 1048, 1690, 1715, 81, 809, 1251, 1045, 511, 501, 65, 1289, 1703, 805, 1067, 1169, 689, 763, 782, 1438, 756, 1250, 836, 788, 1530, 381, 905, 1367, 1394, 1651, 1478, 1271, 854, 1171, 721, 570, 566, 244, 1673, 1496, 1630, 1644, 179, 235, 1228, 211, 1282, 1741, 661, 1791, 927, 1268, 161, 1718, 1521, 168, 824, 1614, 1517, 775, 92, 825, 1073, 1470, 254, 813, 1780, 601, 865, 1541, 222, 1377, 1649, 1213, 1726, 1296, 436, 697, 1389, 745, 583, 1738, 606, 729, 1434, 573, 1121, 1111, 1790, 1090, 432, 1586, 1624, 1241, 225, 1023 ], "class_counts": [ 8, 9, 18, 11, 9, 11, 6, 10, 6, 12 ], "calibration_errors": 3, "temperature": 1.487197945549238, "at_boundary": false, "fit_nll_before": 0.11178964081841691, "fit_nll_after": 0.0966656830242782, "test_nll": 0.12899168972100528, "test_ece": 0.01868307488569253, "test_accuracy": 0.9611111111111111, "delta_nll": -0.012085213697614394 }, { "seed": 33, "size": 100, "repeat": 5, "calibration_ids": [ 1268, 1649, 511, 1363, 946, 1456, 770, 456, 1023, 739, 211, 802, 179, 1215, 1250, 1076, 1521, 1715, 41, 805, 865, 177, 176, 157, 400, 206, 675, 1496, 1588, 699, 92, 183, 185, 284, 1688, 1745, 606, 1045, 1777, 97, 1090, 1703, 1233, 25, 1587, 1533, 1213, 721, 1741, 361, 631, 1342, 574, 102, 460, 756, 1425, 874, 1434, 1088, 1187, 252, 131, 601, 661, 1271, 338, 1157, 1169, 1718, 1084, 391, 1630, 829, 921, 1121, 1019, 573, 432, 107, 1176, 1377, 360, 1419, 1146, 340, 535, 153, 995, 1729, 1216, 1517, 296, 1478, 218, 1787, 1790, 412, 1626, 1331 ], "class_counts": [ 10, 14, 7, 9, 11, 10, 9, 11, 10, 9 ], "calibration_errors": 3, "temperature": 1.48487778205585, "at_boundary": false, "fit_nll_before": 0.15210530257746702, "fit_nll_after": 0.13414542196238297, "test_nll": 0.12897156819450975, "test_ece": 0.01799820384430095, "test_accuracy": 0.9611111111111111, "delta_nll": -0.01210533522410992 }, { "seed": 33, "size": 100, "repeat": 6, "calibration_ids": [ 788, 1331, 1651, 729, 523, 780, 81, 436, 1598, 218, 1176, 25, 168, 1673, 969, 535, 423, 211, 1013, 1003, 1639, 1715, 1251, 1204, 1336, 1718, 1394, 88, 1241, 412, 284, 664, 1363, 1541, 1652, 264, 519, 1233, 809, 1729, 1738, 836, 254, 361, 1195, 858, 65, 802, 16, 314, 1649, 1179, 1280, 1171, 137, 978, 1372, 1045, 1780, 161, 601, 108, 340, 899, 1741, 501, 1213, 1521, 1562, 511, 1203, 1380, 631, 509, 1630, 1067, 829, 183, 986, 400, 1216, 456, 41, 1404, 854, 1586, 1790, 1342, 14, 244, 1550, 605, 1367, 252, 23, 176, 1262, 975, 273, 1777 ], "class_counts": [ 7, 9, 11, 9, 8, 9, 11, 11, 15, 10 ], "calibration_errors": 1, "temperature": 1.2487715837672895, "at_boundary": false, "fit_nll_before": 0.09615488341536108, "fit_nll_after": 0.0918203114185075, "test_nll": 0.13034290071314397, "test_ece": 0.022745075300163932, "test_accuracy": 0.9611111111111111, "delta_nll": -0.010734002705475698 }, { "seed": 33, "size": 100, "repeat": 7, "calibration_ids": [ 456, 23, 296, 1157, 1271, 664, 739, 743, 1791, 1787, 605, 125, 1444, 1203, 1587, 1651, 1171, 591, 1213, 1023, 1282, 1331, 252, 571, 1456, 225, 1076, 825, 179, 263, 168, 611, 1438, 566, 1215, 1639, 176, 541, 460, 1530, 1251, 1517, 805, 1541, 1716, 780, 1313, 1248, 381, 1459, 921, 480, 488, 975, 1490, 1362, 770, 206, 137, 1790, 1084, 1268, 721, 1130, 1394, 423, 1703, 436, 1614, 546, 675, 41, 1195, 391, 131, 1673, 836, 1285, 1335, 712, 1598, 1372, 1048, 583, 222, 854, 1093, 254, 697, 1380, 1718, 802, 829, 185, 400, 788, 601, 1149, 1367, 1712 ], "class_counts": [ 13, 10, 11, 8, 12, 9, 9, 8, 9, 11 ], "calibration_errors": 4, "temperature": 1.846711447923948, "at_boundary": false, "fit_nll_before": 0.24546127338189336, "fit_nll_after": 0.18652817349281667, "test_nll": 0.138644773518911, "test_ece": 0.030824759197461228, "test_accuracy": 0.9611111111111111, "delta_nll": -0.0024321298997086738 }, { "seed": 33, "size": 100, "repeat": 8, "calibration_ids": [ 1235, 1228, 1013, 1745, 1666, 1718, 495, 185, 1478, 1587, 460, 1738, 300, 660, 882, 412, 324, 103, 573, 1362, 263, 1067, 1187, 206, 1597, 1146, 1130, 1149, 65, 1389, 1716, 235, 1652, 1357, 222, 995, 1586, 108, 1490, 605, 1246, 809, 1778, 14, 1641, 1363, 1649, 1391, 1726, 541, 1372, 1195, 1380, 1204, 1563, 74, 813, 25, 606, 570, 1047, 921, 854, 1690, 1530, 153, 1282, 1336, 899, 252, 211, 509, 1216, 694, 865, 805, 583, 1444, 34, 511, 1241, 1342, 571, 1639, 836, 682, 381, 501, 1313, 432, 1169, 1176, 780, 1048, 388, 756, 858, 1271, 1517, 739 ], "class_counts": [ 13, 7, 11, 12, 8, 11, 9, 10, 10, 9 ], "calibration_errors": 3, "temperature": 1.5308497768337457, "at_boundary": false, "fit_nll_before": 0.17211541636477637, "fit_nll_after": 0.15060653741746094, "test_nll": 0.1294801712200044, "test_ece": 0.02253252402545459, "test_accuracy": 0.9611111111111111, "delta_nll": -0.011596732198615262 }, { "seed": 33, "size": 100, "repeat": 9, "calibration_ids": [ 1146, 694, 244, 1138, 1787, 675, 1686, 412, 788, 326, 225, 1791, 660, 391, 1630, 1084, 1013, 1652, 381, 1271, 712, 284, 1296, 1216, 423, 986, 436, 1587, 177, 975, 1121, 65, 1778, 1342, 1235, 1434, 273, 780, 1391, 978, 1019, 360, 1130, 1729, 108, 523, 432, 1777, 338, 829, 183, 573, 1215, 1289, 1624, 1093, 1067, 1563, 131, 1377, 874, 1362, 137, 153, 361, 570, 1090, 1268, 546, 1550, 808, 122, 16, 1176, 601, 155, 995, 969, 591, 882, 809, 102, 739, 1712, 1203, 1285, 1745, 490, 1425, 1597, 149, 1149, 511, 1651, 745, 775, 754, 211, 813, 1673 ], "class_counts": [ 7, 7, 11, 9, 10, 9, 13, 11, 9, 14 ], "calibration_errors": 4, "temperature": 1.8151971846319852, "at_boundary": false, "fit_nll_before": 0.24425646167228257, "fit_nll_after": 0.1831144067051705, "test_nll": 0.137321375146832, "test_ece": 0.03237047130873997, "test_accuracy": 0.9611111111111111, "delta_nll": -0.0037555282717876837 }, { "seed": 33, "size": 100, "repeat": 10, "calibration_ids": [ 264, 1013, 108, 361, 1076, 432, 1268, 858, 1661, 1022, 338, 874, 824, 1726, 340, 1780, 882, 969, 1289, 1791, 263, 1377, 1171, 1688, 488, 1778, 944, 1363, 157, 81, 1738, 230, 1626, 1391, 1588, 1434, 1517, 1718, 179, 946, 802, 546, 921, 1716, 660, 1048, 1187, 1587, 1146, 273, 571, 995, 813, 16, 284, 574, 519, 541, 1169, 122, 1093, 1745, 1521, 770, 1729, 808, 763, 391, 177, 88, 211, 1047, 601, 664, 1533, 1357, 1703, 1374, 1563, 1195, 153, 254, 583, 1111, 1073, 754, 435, 1003, 161, 225, 1666, 1067, 1023, 1715, 986, 978, 1787, 1456, 183, 436 ], "class_counts": [ 11, 7, 8, 6, 13, 11, 13, 12, 11, 8 ], "calibration_errors": 2, "temperature": 1.3089705208475848, "at_boundary": false, "fit_nll_before": 0.11069890655168121, "fit_nll_after": 0.10385271714102734, "test_nll": 0.1293031060107102, "test_ece": 0.02258963049502132, "test_accuracy": 0.9611111111111111, "delta_nll": -0.011773797407909459 }, { "seed": 33, "size": 100, "repeat": 11, "calibration_ids": [ 675, 605, 1084, 927, 1362, 388, 155, 763, 1651, 1394, 1630, 303, 1530, 541, 546, 149, 1490, 1250, 682, 381, 694, 1550, 1444, 1157, 1626, 754, 432, 1019, 1271, 1726, 1067, 490, 583, 74, 235, 1716, 34, 743, 745, 206, 1169, 1342, 1262, 1216, 535, 456, 1363, 1391, 391, 177, 1111, 1280, 1203, 1587, 1187, 995, 1251, 1130, 659, 168, 874, 1235, 1712, 1285, 788, 1792, 523, 326, 273, 161, 1090, 1588, 218, 1171, 296, 254, 314, 360, 882, 1562, 1248, 92, 157, 712, 570, 1121, 482, 14, 412, 1470, 338, 975, 1703, 1195, 97, 179, 1213, 1644, 460, 1456 ], "class_counts": [ 11, 11, 10, 12, 6, 9, 10, 9, 9, 13 ], "calibration_errors": 1, "temperature": 1.1815092653201953, "at_boundary": false, "fit_nll_before": 0.057404888879420975, "fit_nll_after": 0.05565162784703181, "test_nll": 0.13214638768546447, "test_ece": 0.022980447821995053, "test_accuracy": 0.9611111111111111, "delta_nll": -0.008930515733155203 }, { "seed": 33, "size": 100, "repeat": 12, "calibration_ids": [ 1673, 1787, 1587, 1097, 74, 729, 1630, 41, 1335, 1171, 978, 273, 946, 675, 1741, 1729, 541, 743, 179, 1550, 1563, 1195, 81, 360, 137, 1149, 1088, 1111, 802, 254, 16, 1688, 1404, 1716, 1626, 1342, 606, 296, 1296, 244, 882, 1456, 813, 1614, 340, 1203, 435, 388, 1791, 432, 1690, 825, 157, 423, 782, 490, 235, 944, 1586, 495, 107, 1336, 770, 1176, 1639, 1562, 566, 511, 611, 558, 206, 756, 23, 1530, 995, 1496, 1067, 1073, 230, 1228, 1641, 168, 1490, 1271, 1777, 519, 1019, 927, 460, 947, 178, 1541, 535, 745, 338, 601, 809, 400, 754, 1533 ], "class_counts": [ 10, 10, 8, 7, 11, 14, 9, 13, 8, 10 ], "calibration_errors": 4, "temperature": 1.7943358740652826, "at_boundary": false, "fit_nll_before": 0.2468595161996821, "fit_nll_after": 0.19137186371694032, "test_nll": 0.1364922461597539, "test_ece": 0.029146833528092223, "test_accuracy": 0.9611111111111111, "delta_nll": -0.004584657258865782 }, { "seed": 33, "size": 100, "repeat": 13, "calibration_ids": [ 927, 1331, 824, 1780, 947, 102, 81, 809, 1296, 1246, 946, 300, 721, 813, 1562, 185, 218, 388, 1073, 16, 1357, 1280, 1389, 659, 570, 225, 1597, 153, 1530, 1088, 836, 1335, 1587, 1097, 978, 1048, 944, 1313, 1641, 183, 1787, 874, 1235, 273, 1362, 763, 1138, 125, 1067, 682, 284, 88, 995, 754, 986, 1233, 435, 65, 697, 1626, 490, 571, 1241, 574, 1690, 1726, 338, 1541, 601, 1778, 1745, 558, 206, 482, 802, 161, 1639, 149, 1588, 825, 546, 1111, 1666, 631, 1686, 1712, 1271, 865, 775, 400, 1372, 1076, 1023, 1651, 1652, 1586, 661, 1673, 178, 1090 ], "class_counts": [ 12, 9, 13, 12, 9, 6, 9, 10, 11, 9 ], "calibration_errors": 1, "temperature": 1.2054859864969305, "at_boundary": false, "fit_nll_before": 0.058216573204492555, "fit_nll_after": 0.05611555812228313, "test_nll": 0.13142153705390408, "test_ece": 0.02265012629379111, "test_accuracy": 0.9611111111111111, "delta_nll": -0.00965536636471559 }, { "seed": 33, "size": 100, "repeat": 14, "calibration_ids": [ 570, 737, 721, 1372, 1624, 754, 1228, 1271, 1023, 1213, 1216, 161, 1335, 1614, 664, 157, 763, 326, 519, 501, 1792, 1157, 263, 206, 222, 975, 1791, 1419, 770, 1090, 273, 809, 1644, 921, 1389, 775, 423, 1246, 1716, 23, 824, 1289, 1588, 511, 1022, 183, 969, 660, 340, 92, 978, 400, 1204, 1203, 661, 460, 103, 1598, 1093, 1048, 854, 1478, 391, 1179, 836, 1434, 155, 1076, 435, 176, 1597, 689, 303, 1121, 1521, 631, 1729, 1715, 1195, 808, 264, 1138, 122, 1459, 1088, 284, 825, 490, 1533, 682, 1745, 743, 1313, 1187, 788, 482, 829, 1363, 81, 1013 ], "class_counts": [ 12, 6, 9, 13, 9, 12, 6, 12, 13, 8 ], "calibration_errors": 4, "temperature": 1.5804477652276478, "at_boundary": false, "fit_nll_before": 0.16745012901185669, "fit_nll_after": 0.14190251645848156, "test_nll": 0.13028116668415077, "test_ece": 0.027235944312943353, "test_accuracy": 0.9611111111111111, "delta_nll": -0.010795736734468903 }, { "seed": 33, "size": 100, "repeat": 15, "calibration_ids": [ 1130, 739, 153, 122, 858, 1048, 1171, 788, 222, 743, 675, 1725, 1076, 1335, 1550, 1251, 400, 1651, 546, 23, 1362, 509, 1296, 338, 1562, 829, 1478, 14, 682, 1459, 1262, 721, 1425, 412, 92, 81, 125, 1438, 1176, 1204, 1652, 969, 1282, 808, 1792, 541, 1363, 927, 1444, 34, 1630, 712, 1530, 1639, 482, 805, 854, 1777, 1073, 571, 168, 423, 523, 157, 235, 211, 558, 1149, 825, 1215, 1285, 1342, 1394, 1780, 947, 591, 1391, 1626, 566, 946, 161, 1614, 1195, 1716, 1541, 1456, 495, 65, 501, 178, 1271, 1280, 606, 361, 511, 1703, 218, 975, 754, 1022 ], "class_counts": [ 12, 10, 9, 7, 10, 5, 10, 9, 10, 18 ], "calibration_errors": 5, "temperature": 1.680381642447294, "at_boundary": false, "fit_nll_before": 0.1822253322294316, "fit_nll_after": 0.14687407120431403, "test_nll": 0.13264723993386515, "test_ece": 0.026825539057820053, "test_accuracy": 0.9611111111111111, "delta_nll": -0.008429663484754518 }, { "seed": 33, "size": 100, "repeat": 16, "calibration_ids": [ 1241, 858, 263, 1614, 1521, 697, 1130, 1374, 1686, 1149, 1438, 177, 1335, 1624, 591, 946, 1741, 721, 1367, 1673, 1690, 480, 1530, 1171, 1268, 1470, 1216, 1718, 1496, 1251, 1377, 1138, 1563, 1097, 865, 1047, 14, 1562, 1228, 570, 1203, 314, 1296, 235, 1404, 1651, 601, 699, 1215, 1023, 1777, 1342, 340, 1111, 1280, 1716, 501, 360, 131, 391, 1792, 944, 1336, 1726, 81, 1389, 1246, 1204, 1597, 566, 1490, 324, 183, 1179, 541, 659, 1045, 125, 1271, 254, 423, 756, 809, 1363, 176, 682, 1738, 1394, 155, 300, 519, 1550, 97, 252, 178, 338, 927, 712, 836, 1688 ], "class_counts": [ 9, 10, 15, 11, 12, 13, 9, 6, 7, 8 ], "calibration_errors": 3, "temperature": 1.8435560019384676, "at_boundary": false, "fit_nll_before": 0.22103949203130327, "fit_nll_after": 0.1626325002312418, "test_nll": 0.13850846228058117, "test_ece": 0.031142029023008635, "test_accuracy": 0.9611111111111111, "delta_nll": -0.002568441138038502 }, { "seed": 33, "size": 100, "repeat": 17, "calibration_ids": [ 41, 25, 1335, 1013, 1285, 34, 153, 865, 519, 480, 1084, 1187, 324, 660, 1745, 1673, 737, 591, 252, 1367, 754, 1649, 218, 1022, 1216, 131, 836, 1470, 1262, 1541, 1235, 770, 1313, 1703, 1787, 809, 824, 1741, 501, 1374, 606, 65, 490, 780, 183, 432, 161, 1047, 1097, 1459, 400, 1331, 874, 1067, 1686, 986, 14, 882, 802, 682, 1213, 1171, 1073, 1563, 74, 1280, 1419, 1296, 921, 1204, 1251, 1111, 178, 1176, 675, 975, 745, 16, 775, 125, 1644, 808, 235, 854, 1149, 176, 1517, 1377, 296, 694, 1088, 300, 1614, 107, 756, 1533, 858, 103, 944, 1490 ], "class_counts": [ 14, 11, 6, 9, 12, 10, 10, 11, 7, 10 ], "calibration_errors": 5, "temperature": 1.6979462162822538, "at_boundary": false, "fit_nll_before": 0.21821089457177206, "fit_nll_after": 0.1778814967631236, "test_nll": 0.13316248791868668, "test_ece": 0.02763048930491866, "test_accuracy": 0.9611111111111111, "delta_nll": -0.007914415499932992 }, { "seed": 33, "size": 100, "repeat": 18, "calibration_ids": [ 775, 1586, 1179, 509, 436, 1391, 303, 432, 1389, 131, 1045, 206, 176, 1195, 756, 969, 805, 1235, 482, 1019, 149, 340, 523, 601, 921, 1521, 763, 573, 675, 1335, 423, 122, 324, 361, 824, 74, 1023, 501, 125, 1248, 1790, 1268, 1088, 809, 412, 108, 456, 211, 1380, 225, 1372, 1686, 802, 1251, 1048, 1725, 1624, 1138, 88, 230, 16, 254, 103, 178, 1204, 1470, 1729, 813, 23, 825, 1745, 905, 1651, 606, 519, 1639, 511, 1073, 263, 1215, 611, 1641, 244, 1404, 300, 218, 566, 338, 874, 155, 1250, 1246, 1121, 944, 745, 1792, 177, 1093, 1357, 381 ], "class_counts": [ 7, 13, 7, 9, 8, 9, 14, 11, 8, 14 ], "calibration_errors": 3, "temperature": 1.4785755244345398, "at_boundary": false, "fit_nll_before": 0.15758202265976617, "fit_nll_after": 0.13793411354074686, "test_nll": 0.1289199388340104, "test_ece": 0.01925467383691701, "test_accuracy": 0.9611111111111111, "delta_nll": -0.01215696458460927 }, { "seed": 33, "size": 100, "repeat": 19, "calibration_ids": [ 1121, 1235, 1787, 423, 1661, 412, 264, 1745, 1394, 1550, 153, 1022, 1690, 125, 570, 808, 1686, 805, 737, 1703, 1490, 1673, 1377, 1725, 699, 1517, 1726, 1313, 1715, 1262, 161, 969, 1130, 1374, 177, 1587, 1342, 14, 1067, 178, 986, 1530, 488, 1777, 211, 1084, 1076, 788, 296, 1791, 179, 1666, 490, 218, 511, 1614, 591, 682, 103, 1438, 1738, 546, 1149, 206, 631, 97, 1741, 1296, 975, 1790, 400, 1336, 81, 324, 460, 157, 921, 254, 775, 694, 432, 605, 176, 829, 16, 340, 1268, 1644, 1171, 1639, 1215, 745, 391, 1023, 601, 480, 107, 149, 1250, 300 ], "class_counts": [ 11, 9, 9, 10, 12, 12, 9, 7, 10, 11 ], "calibration_errors": 3, "temperature": 1.6655418332439886, "at_boundary": false, "fit_nll_before": 0.20641074569924267, "fit_nll_after": 0.17032053034810474, "test_nll": 0.13223458268918872, "test_ece": 0.02615645692438577, "test_accuracy": 0.9611111111111111, "delta_nll": -0.008842320729430947 }, { "seed": 33, "size": 269, "repeat": 0, "calibration_ids": [ 211, 1157, 185, 699, 264, 946, 1715, 1048, 829, 721, 102, 495, 300, 14, 854, 456, 1282, 969, 1521, 1045, 360, 314, 244, 535, 1374, 1686, 754, 1778, 1389, 825, 1235, 1204, 921, 326, 1121, 1179, 1187, 1377, 611, 1213, 978, 1614, 1470, 178, 694, 558, 1067, 1047, 1268, 41, 149, 1550, 1280, 1588, 432, 809, 1019, 1787, 858, 1171, 225, 601, 770, 1248, 1241, 631, 1688, 1111, 412, 230, 1285, 1262, 153, 108, 125, 435, 511, 1419, 338, 571, 436, 729, 1146, 218, 361, 1624, 697, 490, 573, 944, 1587, 1246, 460, 541, 1088, 874, 1649, 813, 664, 1130, 1169, 1176, 836, 523, 97, 1562, 1630, 296, 927, 168, 583, 1425, 1666, 1690, 1216, 1149, 1563, 1372, 1138, 1661, 1289, 263, 1362, 574, 1335, 1296, 605, 1741, 743, 391, 775, 1394, 183, 1712, 675, 824, 1233, 660, 1626, 1490, 1729, 1745, 1434, 1703, 1530, 137, 482, 161, 1517, 1597, 1777, 737, 1738, 986, 1444, 254, 1541, 1792, 23, 1725, 802, 177, 1342, 222, 206, 1331, 1251, 606, 1533, 303, 1336, 1644, 1084, 1023, 975, 1203, 1641, 1250, 1228, 739, 882, 284, 423, 88, 81, 103, 712, 235, 1459, 1013, 509, 782, 1790, 1003, 682, 1076, 176, 591, 122, 1718, 1215, 659, 1195, 788, 1639, 808, 1404, 1097, 131, 1363, 1380, 566, 1780, 107, 157, 905, 899, 273, 1090, 570, 1586, 519, 1496, 179, 1716, 1652, 661, 1478, 763, 155, 865, 1391, 745, 1726, 16, 501, 546, 1456, 381, 92, 1093, 65, 324, 1598, 340, 1271, 1651, 780, 488, 480, 34, 995, 1022, 1357, 1791, 1073, 1313, 25, 805, 252, 689, 1367, 756, 1438, 947, 1673, 388, 74, 400 ], "class_counts": [ 27, 27, 27, 27, 27, 27, 27, 27, 26, 27 ], "calibration_errors": 8, "temperature": 1.5367823253742512, "at_boundary": false, "fit_nll_before": 0.15586618342478, "fit_nll_after": 0.13378309519743573, "test_nll": 0.12956240966921712, "test_ece": 0.022418584576312864, "test_accuracy": 0.9611111111111111, "delta_nll": -0.011514493749402555 } ], "summaries": [ { "seed": 11, "size": 20, "count": 20, "temperature": { "median": 1.1118332223142324, "min": 0.05, "max": 2.301394974597253 }, "delta_nll": { "median": 0.01277552691993282, "min": -0.014063799919890901, "max": 2.443542962868526 }, "worse_count": 12, "boundary_count": 7, "zero_error_count": 7 }, { "seed": 11, "size": 50, "count": 20, "temperature": { "median": 1.2660182529628985, "min": 0.05, "max": 2.0327843770789262 }, "delta_nll": { "median": -0.0030680043436689675, "min": -0.01565178836531228, "max": 2.443542962868526 }, "worse_count": 8, "boundary_count": 2, "zero_error_count": 2 }, { "seed": 11, "size": 100, "count": 20, "temperature": { "median": 1.5741903781301938, "min": 0.7533815179084477, "max": 1.8928279448259697 }, "delta_nll": { "median": -0.012685093899658675, "min": -0.014948916780410493, "max": 0.03289051351539363 }, "worse_count": 3, "boundary_count": 0, "zero_error_count": 0 }, { "seed": 11, "size": 269, "count": 1, "temperature": { "median": 1.472669073385246, "min": 1.472669073385246, "max": 1.472669073385246 }, "delta_nll": { "median": -0.015581014539219867, "min": -0.015581014539219867, "max": -0.015581014539219867 }, "worse_count": 0, "boundary_count": 0, "zero_error_count": 0 }, { "seed": 22, "size": 20, "count": 20, "temperature": { "median": 0.3798164228450438, "min": 0.05, "max": 2.505710212646791 }, "delta_nll": { "median": 0.7418482690684098, "min": 0.010634233432700368, "max": 1.4304934861680885 }, "worse_count": 20, "boundary_count": 10, "zero_error_count": 10 }, { "seed": 22, "size": 50, "count": 20, "temperature": { "median": 1.6013002883034813, "min": 0.05, "max": 2.263990886980669 }, "delta_nll": { "median": 0.00803191770076566, "min": -0.005096126841912033, "max": 1.4304934861680885 }, "worse_count": 16, "boundary_count": 4, "zero_error_count": 4 }, { "seed": 22, "size": 100, "count": 20, "temperature": { "median": 1.6064819928286629, "min": 1.2007482234622135, "max": 1.9697513412460728 }, "delta_nll": { "median": -0.00042303482994121566, "min": -0.005202777860526198, "max": 0.014905396751294597 }, "worse_count": 9, "boundary_count": 0, "zero_error_count": 0 }, { "seed": 22, "size": 269, "count": 1, "temperature": { "median": 1.6092275608405882, "min": 1.6092275608405882, "max": 1.6092275608405882 }, "delta_nll": { "median": -0.00034614827944287607, "min": -0.00034614827944287607, "max": -0.00034614827944287607 }, "worse_count": 0, "boundary_count": 0, "zero_error_count": 0 }, { "seed": 33, "size": 20, "count": 20, "temperature": { "median": 0.7625363111423352, "min": 0.05, "max": 2.33383614661209 }, "delta_nll": { "median": 0.05273553074500274, "min": -0.002482522968467543, "max": 2.037154860146873 }, "worse_count": 18, "boundary_count": 9, "zero_error_count": 9 }, { "seed": 33, "size": 50, "count": 20, "temperature": { "median": 1.5341077092610997, "min": 0.05, "max": 2.098624800744334 }, "delta_nll": { "median": -0.004674058279463433, "min": -0.012415944438939258, "max": 2.037154860146873 }, "worse_count": 8, "boundary_count": 3, "zero_error_count": 3 }, { "seed": 33, "size": 100, "count": 20, "temperature": { "median": 1.5556487710306968, "min": 1.1815092653201953, "max": 1.846711447923948 }, "delta_nll": { "median": -0.009714173653478472, "min": -0.01215696458460927, "max": -0.0024321298997086738 }, "worse_count": 0, "boundary_count": 0, "zero_error_count": 0 }, { "seed": 33, "size": 269, "count": 1, "temperature": { "median": 1.5367823253742512, "min": 1.5367823253742512, "max": 1.5367823253742512 }, "delta_nll": { "median": -0.011514493749402555, "min": -0.011514493749402555, "max": -0.011514493749402555 }, "worse_count": 0, "boundary_count": 0, "zero_error_count": 0 } ], "full": [ { "seed": 11, "temperature": 1.472669073385246, "bootstrap": { "mean": -0.01558101453921988, "interval": [ -0.043045137913599214, 0.007012211937036684 ], "per_item_delta": [ 0.00203149693159673, 0.09870633513761422, 0.007665362494147333, 0.015619328352606049, 0.15509358279431373, 0.11336340353704388, 0.00035747265940646336, 0.016116361916082993, 0.001757144460318969, 0.0039014588173577024, 0.00020568198311406562, 0.0012672489671999956, 0.003162984048923627, -0.22395795337761593, 0.02903387142604779, 0.040920322780293034, 0.0003521665710761057, 0.0746285199645553, 0.00040084760320555, 0.00047187520908085595, -0.9657110225411896, 0.00016369070264740483, 0.03526931443449346, 0.0005187535129357011, 0.012870414172130378, 9.007791666372821e-05, 0.0008473238817806366, 0.005418808960745579, 0.00045102947990083574, 0.0005398466586851817, 0.0045105567718640716, 0.0009102969128705536, 0.004580255839279587, 0.00043903149326469427, 0.1353570409240688, 0.011254920210160644, 0.0004204793741691103, 0.04147557397490477, 0.0015585813322160434, 8.560946123820805e-05, 0.022229642221506234, 0.007452163935689184, 4.747470475572149e-06, 0.04234140742056029, 0.007082604946516002, 0.03902105706979851, 0.006873161120939412, 0.0039183559262996066, 8.91281209726941e-05, 0.0006548319428064652, 0.0006263833772216565, 0.0010718743366942365, 0.032966101681962234, 0.0005358971763997667, 0.000757163094538487, 0.004381299496098903, 0.0017986439252623853, 0.001319695153881853, 0.01598588870365246, 0.042604908468083943, 0.0032900145491555448, 0.01469615747659445, 0.00288071087587312, 0.0019749619220061935, 0.04529149302106361, 0.14018958715094382, 0.004181266342104339, 0.000845649099636498, 0.0019223721036774912, 0.02054865856112019, 0.0017441745874647845, 6.058770202192315e-05, 0.009697790853609806, 0.00018106745987232355, 0.0029577824838805384, 0.10827902161687966, 0.10896881230719774, 0.0021083271829098436, 0.0008994042466491612, 0.197774638751011, 0.022802694955788468, 0.004720904980888196, 0.0004372675768618005, 0.0026224121898280354, 0.0005160005065843852, 0.010236621973806162, 0.0012994113149583726, 0.004678981677381461, 0.0015387824316198983, -2.067710515177967, 0.00831352435094083, 0.016488402136119157, 0.09390066855215876, 0.02719337753180244, 0.01567116586251076, 0.048842298571653016, 0.016920165828380237, 3.757380624374069e-05, 0.008906907014040023, 0.0018303612726576534, 0.0009718647657027298, 0.012625261701061973, 0.06569014785384941, 0.01290779303864848, 0.0066814475816798875, 0.00045663139417041504, 0.0013528800743138145, 0.06563884740178874, 0.08648307417122889, 0.00036195552145097047, 0.00030341771391264655, 9.468152912772853e-05, 0.0012671047252490997, 0.0014488317908078173, 0.0018801126117325016, 0.03925322606542571, 0.011158409307029604, 0.00015346167215492655, 0.10562839772595585, 0.031914415791719754, 7.87439383434637e-05, 0.00686684922904552, 0.0002059835598480655, 0.00077937334991914, 0.07170248539444435, 0.006801497382962626, 0.1683295893003794, 0.01095796026732413, 0.0011521640302190183, 0.013034454696896578, 0.0003370595923122793, 0.11946163036478621, 3.650379796932366e-05, 0.09514820193304505, 0.0036122843625277265, 0.006202193393692742, 0.005487900684589716, 0.07687549072376926, 0.14483269623242206, 0.009061856753083509, 0.002118010996930918, 0.018753510211047427, 0.0023513346465544177, 0.1824531112548965, 0.14322623639807808, 0.003945514102782664, 0.0033691268164460332, 0.010425583728992192, 0.018105380520188882, 4.046941615218421e-05, 0.008687447963057036, 0.0045187695966002785, 0.021826810791045433, 0.03315644409412979, 7.726857562066719e-05, 0.00037326595414714173, 0.006355637583580146, 0.0005402435199045101, 0.03668933955695753, 0.0008791922940758262, 0.0013809817622289944, 0.0038566652765480206, 0.036415012118301275, -3.039030778422857, 0.00018910812562957267, 0.07801505839480645, 0.1525452353055302, 0.0026546500310236517, 0.013456909415133073, 0.001365946801867221, 0.012489613958269516, 0.00539028933232885, 0.025466468703379132, 0.02279500337651487, -0.8254587998674534, 0.10755286342997007, 0.002928060123542564, 0.09307754861044315, 0.007930964036352513, 0.21661904219495426, 0.002211584585658739, 0.00015612723956964599, 0.0001931322756173707, 0.013896151200192745, 0.00012838420254267333, 2.4706766116096615e-05, 0.027136615968447107, 0.011636191846277257, 0.00660788511119892, 0.00025938277267913, 0.014048866071642348, 7.724762204324356e-05, 0.008895683206393707, 0.11974992603894205, 0.07036061138966934, 0.005147835425879318, 0.0010786388581300165, 0.10271837908576605, 0.006861966702341985, 0.01957713336415894, 0.017718038924123124, 0.008421856982382962, 0.0022587431258271565, 0.002289896257787979, 0.00018457698967658563, 0.005653584202253892, 0.007752939174464054, 0.038340000975651434, 0.003392932687634093, 0.05302815642723495, 8.535647191383776e-05, 0.032776943303418675, 0.007765139704892921, 0.00033256708245531013, 0.0018995198419731662, 0.06079021570685206, 9.814949966852258e-05, 0.2816809876601038, 0.0007187172600656385, 0.00019622432919926924, 0.00041236176989015177, 0.00031452322936452524, 0.0002313244570910689, 0.012186836351478743, 0.01799379753788822, 0.029199693870987187, 0.00409896321261476, 0.01374843408218811, 0.0001537344907582567, 0.0003531752496662977, 0.0010665479906463735, 0.023224033496097217, 0.0005493739398111146, 0.005214134465577248, 0.0018503483691147521, 0.009677912532698523, -0.3167158345031622, 0.0002427375461554116, 0.02525742574976275, 0.0015156524485820926, 0.020194253811461487, 0.003051231833690634, 0.11170641238545714, 0.0002311153624337865, 0.01030170934995888, -0.24927651543716123, 0.04190720619187068, 0.012649892623730294, 0.06465624236095495, 0.0001532871327210819, 0.004128032266104764, 0.0860338097546658, 0.06598588232736571, 0.0036871051330662373, -1.1840508240280414, 0.028823151023972097, 0.08558977623425815, 0.014496744547685722, 0.014719792615047776, 0.00190040304640431, 0.03031241093651088, 0.012050675327434507, 8.818281699478612e-05, 0.0010646853620317346, 0.004495862312962376, 0.013727629254672148, 0.11288389915179763, 0.0165465993697489, 2.3823407144551223e-05, 0.10457185166516932, 9.156514259477036e-05, 0.06960208718859176, 0.0007881590660120152, -1.460320651805045, 0.01238044117740406, 0.00044453559547077336, 4.238338338282253e-05, 0.00038311704930884345, 0.015748376102277854, 0.0011410284209393427, 0.028779452127465383, 0.008311181737469142, 0.07332148880192337, 0.00760047334523773, 0.001317767747542524, 0.05873647702088164, 0.07792700080152395, 0.021634837678146942, 0.0009003119557704398, 0.025259895513931083, 0.021983555924028564, 0.0027651810590409265, -0.6942527655876867, 0.0026867800351140603, 0.00019341644864319363, 0.0033859717376514653, 0.004203846229857014, 0.0038899136729766545, 0.1343816449864506, 0.000290042648023137, 0.010243085923109019, 0.001527137402137479, 0.016094038753905554, 0.0006032704778102093, 0.027954184598556843, 0.001719438711161525, 0.0011745443971563471, 0.0003670016391348155, 0.03704017057895434, 0.0009057396983666617, 0.0159030435868479, 0.04582351191232893, 0.003360863473933355, 0.0014719857644791448, 0.0006689511599317675, 0.026838226985043408, 0.10286357366499739, 0.00765633341923869, -0.3346075173587879, 0.0006100438260900666, 0.006655972649054696, 0.05584646373476287, 0.1251621547938854, 0.06853084672496108, 0.0035552524980856006, 0.006057865557144459, 0.0003980354079987392, 0.0346619855198981, 0.01362345382774444, 9.538851588094009e-05, -1.6425727863137967, 0.0014984938789278617, 0.00419212557517263, 0.005364294733727385, -0.488589621256424, 0.04539328189618578, 0.0005389137012164818, 0.0007149344280048106, 0.026404615715865, 0.0007128926789882051, 0.00015702379213453343, 0.006066318845580753, 0.010170613388721333, 0.00046149229312876164, 0.010073580751674052, 0.0077485843238869105, 0.007208094790157986, 0.006693884722629169, 0.005010773682972446, 0.023252339756290576, 0.014178377298497885, 0.0022046626637285474, 0.024774991065146884, 0.0024893862547434547, 0.0027192387115237302, 0.0074892721291894395, 0.04282722559740105, 0.00027435424213360416, 0.0007816840614567347, 0.0004206995142879423 ], "resampled_means": [ -0.008757618745799152, -0.008277197897351934, -0.009302999408444305, -0.0020564941269244634, -0.03507666090335408, -0.011427002945326696, -0.00840643525723695, -0.03842431211553801, -0.0038508795355735154, -0.010152753545548012, -0.016540557637106105, -0.0018616181015930445, -0.01741627740406398, -0.0017588523186033201, -0.017416275272486983, -0.029889242767237246, 0.0008535712510339342, -0.005277525058219422, -0.004875591807478805, -0.00939789346510703, -0.011756020713551411, -0.024343252723237264, -0.03645996090860654, 0.0007921276609845911, -0.02118344628019167, -0.005523282058294324, -0.02351568127338006, -0.006150042324640291, -0.001675210843202068, -0.0012005114149386046, -0.028182914637093687, -0.012943691940655496, -0.0008236307116415481, -0.015723008183951177, -0.027486785107347117, -0.005782375575042382, -0.032628644331829666, -0.020902812261998156, -0.00574941480948965, -0.03228680980746279, -0.013050951810796659, -0.014581028483745628, -0.010952361233018355, -0.03118822468042161, -0.019265225848647177, -0.011849977630602518, -0.03135442316147552, -0.03218215926819328, -0.015470877499036388, -0.030996926071823118, -0.013414617024276886, -0.05704995040711839, -0.025645155732975336, -0.010504288515290762, -0.022960935300525984, 0.006503839679049308, -0.015248392752317486, 0.0024296818816040783, -0.0009005446142219826, -0.031107357743292773, -0.020064529309280717, -0.00994718517795224, -0.0005374527384538406, 0.003308996085176633, -0.005821873354995846, -0.0021400129563822422, -0.007390817920027453, -0.011678946525531742, -0.008808304729864694, -0.0009915121710651971, 0.0011287263824272063, -0.008701339862278315, -0.007253616294049292, -0.023398645907891966, -0.010710260502278341, -0.0035302936695490376, -0.010032814316582936, -0.03990111309554356, -0.020351012441311232, -0.012023195603981433, -0.032970787191234266, -0.04153423231802513, -0.019284293692457623, -0.012025282819631474, -0.027953238793933435, -0.008498413500438435, -0.027030837434749422, -0.03146621255477898, -0.0020080945778507275, -0.042281957975597695, 4.400047751637542e-05, 0.0049960597777223125, -0.006742402667250744, -0.014486807317844463, -0.012156841474229895, -0.004621845213958472, -0.01649820977078795, -0.020618417322473562, -0.018690551971963638, -0.0012650135720211392, -0.029671465293588915, -0.0009117918656541013, -0.0037525145017229446, -0.028452333747022655, -0.02137032305074915, -0.027466172321556223, -0.020595818128965073, 0.0028710583060057642, 0.012414616810075007, -0.019315162792734354, -0.026648302764332513, -0.000954111533087824, -0.02102665233383695, 0.0012802551950879975, -0.03430544835113735, -0.03517596629593722, -0.022948707027674216, -0.002336657728819691, -0.03156602433480822, -0.015927781465636323, -0.02634166046797148, -0.017651456300594136, -0.03167478941585354, -0.030220237554156062, -0.02120886903557502, -0.014302619881155998, -0.022598624852456284, -0.016310036896163237, -0.0018758141736622924, -0.009803162642321957, -0.016273670768779534, -0.0258932412790097, -0.014874168659185139, -0.02506168720296796, -0.030322021122242512, -0.017137455515887925, -0.004433397308785727, -0.0113105762862943, -0.019952253081458336, -0.013414233190191234, -0.018575635440407287, -0.0368779247531344, -0.010050273655834725, -0.027570785721919948, -0.018662604246809048, -0.034770779779648577, -0.003237514505764056, -0.008794145582887007, -0.016971006089011743, 0.008937301444800632, -0.0019794878041370133, -0.006381230525948583, 0.008986680483380858, -0.020417979491286212, -0.05048533839755037, -0.008666051552142257, -0.028658418520080976, -0.020930125938379084, -0.011986274514163167, 0.0009076947012934902, -0.003997774668034625, -0.006963535277477535, -0.010730467670734734, -0.022058049613802026, -0.006326580378802942, -0.012076761270568026, 0.0015511959241710977, -0.02754214743651278, -0.004282400154758585, 0.001369835557134234, -0.011728720998391536, 0.003373605391235148, -0.0409514833835827, -0.02136548936680176, 0.0015718305840545647, -0.007076132700306402, -0.014564404591074068, -0.015154831825271244, -0.020934461190722172, -0.003523547190299346, -0.03423797848692436, -0.01915339475546026, -0.0148625683447965, 0.006919620205448402, -0.005404679561043911, -0.03590967063670183, -0.03953815056652663, -0.014066923210439062, -0.021160452269056208, -7.103529023404375e-05, -0.04129769400039085, -0.031410725919867925, -0.013410324327075077, -0.03891589214724552, -0.023890269087085022, -0.013567727255450102, -0.021821244814121062, -0.014318801868819708, -0.018555579800556627, -0.01821210296085322, -0.0027203376430498575, -0.002646678408818545, -0.03128635182716662, -0.008124680702796643, -0.01740770187240124, -0.013129557364506815, 0.0030766694755853235, -0.033364160309277385, 0.001284423999417751, -0.03253914390640943, -0.012609470487777297, -0.018769743178515805, -0.02387148439224594, -0.006276556325184135, -0.018000772573428706, -0.020484725886705676, -0.034214078779618656, -0.010381972651634229, -0.03168357404671729, -0.00853147578632363, -0.011664484933748671, -0.0166378460621934, 0.010828609548043136, -0.021456373553207127, -0.005963653680641805, 0.006176004117291176, -0.0269052761960471, -0.024605790198620348, -0.0021882011856967478, -0.010929331075612714, -0.017857817104483517, -0.003214325888162611, -0.004401650286108327, -0.05200453009729516, -0.0065318802491880846, -0.03438708177460677, -0.017380151023440912, 0.0006633471557263268, -0.00105272958421995, -0.004088952197964157, -0.027982709266256536, -0.037219682031147935, -0.031368050578371325, 0.0027667021297649117, -0.004142123448219534, -0.004056689352444334, -0.013888730828785226, -0.010018900788866093, -0.0020289742839222358, -0.005450222050171031, -0.018307369016413305, -0.00578685412825277, -0.0155349824141179, -0.005249042891672232, -0.00927166225066559, -0.05302702221525585, -0.003003715748131294, -0.004518967035302194, -0.03122222740991879, -0.01008586343588469, -0.013438554839397897, 0.010159599442058798, 0.003333914979420792, -0.012599733736450982, 0.0006098943346171503, -0.014482377637650316, -0.008972981928727301, 0.002047531376551509, 0.007137870805632764, -0.00630309886852724, 0.0030487875976816545, -0.0004932463361050089, -0.03349003167088714, 0.001057633124689183, -0.02421717896014246, 0.011584087146531181, -0.029722449281876638, -0.012952323763890116, -0.0012237894258052082, -0.010847627761182194, -0.02710606749355356, -0.02575174423235112, -0.018140608894412994, -0.026316498907888525, -0.0377696566856506, -0.01145340313240413, -0.03213619919981451, -0.04532203200597306, -0.0002133973483840269, -0.02678320267140206, -0.015754183738524066, -0.023306105976554543, -0.0035547389717015634, -0.017672126150202486, -0.009387823289665932, -0.031665521796093485, -0.012692775214326435, -0.016103605564277326, -0.012514051609788532, -0.022354513683878977, -0.010741858422759981, -0.00037602502851347944, -0.009791909511934963, -0.005580216254411332, -0.030304692746189306, -0.01703497285447261, -0.0161305427830581, -0.015294538740915766, -0.020220042475501528, 0.0003256972938482847, -0.03277643481696192, -0.0279616389934451, 0.011107816967682146, -0.012715890852393623, -0.017633609257819913, -0.039518311729961626, -0.01807333482879957, -0.024185495361301247, 0.0013883024597460924, 0.012090503439183786, 0.015743576865720265, -0.02251433964557753, 0.0030920534449599356, -0.02080550580288041, -0.00017624687723227908, -0.005047747672969037, -0.013909021755241774, -0.026148054843450487, -0.023494518569279588, -0.022809738790855693, 0.006808690155762336, -0.007721955757934847, -0.019105195738205776, -0.02060950536293084, -0.008675737566938277, -0.016830954027130394, -0.015548087893497313, 0.00695831580604342, -0.019883184174772323, -0.014362909362931643, -0.00212694905273869, -0.021691077110036403, -0.026506490414316804, -0.0174148869031184, -0.033572569223626315, -0.03293505746758047, -0.01952571166015632, -0.019278406148677776, -0.02681493500512267, -0.038438255260849145, -0.022358011169983726, -0.003456221021605921, -0.01803216931973127, -0.00534950092414394, -0.009172485022072739, -0.004090980959821116, 0.006874326175346545, 0.005086566115113864, -0.01837704065424201, -0.015721699847899222, -0.014819539652267299, -0.005037688369072813, -0.009264792124013675, -0.012642439159919321, -0.04398206148195349, -0.02282874156694835, 0.006856690810911685, -0.008149620521483669, -0.005615711149268966, -0.010851795438718512, -0.013342368383191886, -0.028343078372668563, 0.008581172268293963, -0.02504455546914985, -0.006577927782436093, -0.02106631963567521, -0.0439541043850522, -0.023095137025838916, 0.0016502483467867926, 0.0017150419961073477, -0.02434227588219031, -0.016676194212494208, 0.00907817932658567, -0.01316561245441683, -0.056998036768067625, -0.024584481451168837, -0.026325949169175963, -0.034333482458629785, -0.018124442520783084, -0.01723460609607639, -0.017052591690834184, -0.034838319251914426, -0.041714294396659, -0.01704517890075101, -0.004528261693761254, -0.011097186710626275, -0.008460188223960366, -0.0037525878112664225, 0.0015328258268075589, -0.014211038092584132, -0.010329614743554911, 0.0025426697346485716, -0.02263458942857534, -0.04570079211481889, -0.0403427873294501, -0.04469448873779227, -0.031736188149406264, -0.020735371578162593, -0.0038401445717031117, -0.006268929924601546, -0.005448213589107386, -0.031947350208217776, -0.013825890589185121, 0.01070870153478282, -0.026335345111082542, -0.001893442514019994, -0.015316738882258624, -0.026918307954617835, -0.021887883662676674, -0.029913596415381, -0.011564494053753975, -0.031183566164412112, -0.024884311431458524, -0.02333535127913511, -0.008106755869410575, -0.013215326661842209, -0.029483946494118715, 8.557230825326902e-05, -0.010949684022273101, -0.02115821583391516, -0.05162437622121902, -0.011354989097933001, -0.012199766140099391, -0.03552598226938583, -0.026614255294348084, -0.0062963121928348445, -0.0050925810624330315, -0.04463853386633645, -0.03240177602711453, -0.002744826370444759, -0.0025180296630756245, -0.012978826975799912, -0.003033778582631344, -0.00995060552118872, -0.0018742227464749665, 0.000769652605601592, -0.019957745143402903, -0.0464806121662627, -0.025632490593188735, -0.01237076555962183, -0.015946417369316797, -0.019738219730284435, -0.05840411890325388, -0.0027814919495771145, -0.002441884264590825, -0.008140964550541738, -0.006314405798448022, -0.003907839888727584, -0.0016093607223348898, -0.004404846904271419, 0.00455532107965764, -0.018900685637746054, -0.012784559029136958, -0.031083974181704437, -0.028261681853485696, -0.024977235978195456, -0.016392328548404507, -0.006509070030679524, -0.007282053078716504, -0.03468886240056563, -0.005233990762794583, -0.02718605589857077, -0.038593753236028414, -0.0055284528351069884, -0.004689604687435827, -0.04295896755013592, -0.015248196447091224, -0.008767742179081016, -0.021277402751798496, -0.0031537522608466755, -0.028118357018243372, -0.03281994487078979, -0.007825472236369008, -0.018031182258449692, -0.02208313173115229, -0.002890261707840838, 0.002814198940424278, 0.0062668928782010705, -0.05657174272139935, 0.0017906734757145494, 0.0017345659200471584, 0.007915870024957056, 0.0013440372665749242, -0.001034251540721981, -0.021193352712992494, -0.02810369602417888, -0.006849645867850283, -0.01345242585773533, -0.003307489964374505, -0.001162832568320405, -0.028139508138343237, -0.007360152072454693, -0.004829713792926141, 0.005468252314837593, -0.020841109976314828, -0.016502908684766854, -0.010376270097072836, -0.02353114040309887, 0.007910223666385044, -0.03274903350446065, -0.007498933814507359, -0.02404945797576545, -0.024438196798567544, -0.004284446022811703, -0.025335458735229327, 0.009504395740448413, -0.01274378929956534, -0.07234428471434291, 0.0003961763378207076, -0.026531483001332498, -0.030863460558019598, -0.01874217509736045, -0.018795760037912043, -0.03620932736456631, -0.010066278981487724, -0.011781718850264439, -0.0056225233277586795, -0.03004344079823812, -0.03055874817207911, -0.011373905907887064, -0.006960306306268695, -0.03097734480114873, -0.03025330642024981, -0.008460436986351249, -0.0009797883766362548, -0.01860288934594834, 0.00594455716452166, -0.029656480132587478, -0.012753958298359564, -0.02302239363286053, -0.03322381954995793, -0.025041691965453597, -0.026841015904398594, -0.013824655496066105, 0.0015303867294074575, -0.005692922350479864, -0.0012980006657407398, -0.04680639382092862, -0.006274523388983486, -0.012083466243967587, -0.01219283340524254, -0.004216688742759774, -0.01816918405969086, 0.0042913916384939976, -0.043966560810811905, -0.004523334870497981, -0.026899370649115965, -0.0362065715224992, -0.01721667556781539, -0.024714307129489604, -0.0044108418333335805, -0.032775193377104286, -0.009416137388672412, -0.01052234909307241, -0.03576688249337469, 0.0092604040367015, -0.018663008931071305, -0.02245682067781263, -0.016130134647962695, -0.01598911849961392, -0.010842004213071648, -0.04435946034939766, -0.02119983030025369, -0.0029597397467790603, -0.002042790851166459, -0.015431133997000772, -0.03385033095764293, -0.014991572181519376, -0.008773727544286144, 0.005140720813291497, -0.02930573531851066, -0.009706926952909617, -0.034649647633260325, -0.004441649622959161, -0.025476846647323317, -0.0012031173007293523, -0.0429008942834653, -0.004178386646241004, 0.001622231640844261, -0.02559115220477314, -0.024110233736952088, -0.01243209965590247, 0.00278954894412584, -0.033735230806116394, -0.02194353923393917, -0.016411477788179245, -0.007371426629794712, -0.006374292569908977, -0.017323460998953957, -0.005344221555686696, -0.0017231380168482443, -0.027984645141568724, -0.011422107967696675, -0.011539049618522737, -0.02403315125727263, -0.01933936281236679, -0.025209124632911046, -0.006817473639514474, -0.04562139155038186, -0.012719193224093721, -0.02452099206627532, -0.012450967199175069, -0.013211147961802028, -0.03959097234043212, -0.0007566516670209269, -0.00790379916676345, -0.023838121979867413, -0.020508888529233715, -0.04195622936381748, -0.029772268591157086, -0.024091304754547006, -0.028538903156798787, -0.005617718665872309, -0.0023716595199855207, -0.013524474960843823, -0.0041692146539660776, -0.011765127976467251, -0.0025147051001114095, -0.010032947668523086, -0.03932211972718486, -0.018441610970876696, 0.004156852535588873, -0.011449920673993945, 0.01299750985896583, -0.0028862143644903643, -0.01336087908912244, 0.00023664424903955755, -0.02121820929594886, 0.00940189903722574, -0.010345125274542878, 0.0065014832010524045, -0.01191625077076151, -0.00785760531622408, -0.012634824287460942, -0.013114946151277175, -0.03773571612746865, -0.0005570198066542637, -0.052835196126952325, -0.0015671160254463893, -0.004890649504797354, -0.0011425391122775247, 0.005491057282003389, -0.020272281712524477, -0.004418842735930187, -0.023434136108472784, -0.016530973942008368, -0.009875766449793417, -0.01959090721011774, -0.04533414811925991, -0.036390382697489426, -0.01893313498388879, -0.01311041580379264, -0.006719574831933098, -0.019565897686747182, -0.0004637318124734175, -0.012868770853254838, -0.03230491736240223, -0.01361663617427633, 0.009572407453061368, -0.00030567753318984337, -0.01244565187801466, -0.026090202464057317, -0.020646959231757148, -0.02632039520327481, 0.014196158754795704, -0.008050095810173295, -0.018820586784456278, -0.04450414811574357, -0.0017840134903822117, -0.010284261759272191, -0.02073621667809739, -0.018664335378581934, -0.013876159839104155, -0.02745708995090498, -0.0602161013063851, -0.03409689873595217, -0.028758753900543964, -0.025342674595781326, -0.022788001991574974, -0.021574780082591782, 0.0032625169802345754, -0.007522010960963991, -0.012752860295208812, -0.0190001032060648, -0.02845203572818042, -0.012368844112755021, -0.013549814557500145, -0.017437659618951055, -0.031457638634797454, -0.035085245711099305, -0.010698584858568422, -0.02051398008036786, -0.001004550367140173, -0.009318311977585307, -0.002991769738531025, 0.003182063168963477, 9.194772058419072e-06, -0.02846411870217173, -0.025752652320892168, -0.01666757827219406, 0.0006961621711970259, -0.0062461993090624, -0.0217189540170542, -0.018382384928605063, -0.014696777889995741, -0.023383710262190504, -0.0077678783897108025, -0.002807238837004265, -0.026432673520517008, -0.011004683764438831, -0.015168872982441056, -0.007368405542513107, -0.014101046921741172, -0.04164481869036614, -0.025336472059661257, -0.012264621176761777, -0.027467334001346076, -0.015072020157996312, -0.004491232670673058, -0.0038346308929336093, -0.026323737703485568, -0.0424505857732688, -0.021793914157355185, -0.01864240103136476, -0.019337411515087933, 0.0006284858918845547, -0.029308554207344347, -0.017768652005052427, -0.01599063907957452, -0.014385400126883467, -0.029833264030531784, -0.019019199997794292, -0.016802974068559143, -0.004759306706803548, -0.02747383966946459, -0.037920542011575895, -0.019973062569441015, -0.019256914921533145, -0.020335154564993716, -0.01894240079403579, 0.0020870664295779555, -0.03005941573028712, -0.0042388451617443685, -0.009085675521852014, -0.004465047448748145, 0.007008989914765008, -0.030439928818888816, -0.027819719013220896, -0.026541912078927844, -0.021619085764278036, -0.025679549000657758, -0.010970376896567734, -0.02798216619284423, -0.0188395420097051, -0.04117983533273632, -0.013674934332928726, -0.01749423422667289, -0.013373913208650702, -0.01434133992729058, -0.024210287868346177, 0.0019805255461234216, -0.03248129073184346, -0.01977454783529012, -0.009171605616640608, -0.039613750439992275, -0.0335547625771416, -0.005782683107091817, -0.016062824898725417, 0.007518027752437341, -0.02106016497668276, -0.028503022695547525, -4.402627036012605e-05, -0.02856254743089135, -0.03721976034347083, -0.016699076484494, -0.017687882174800225, -0.016457784428585497, -0.01160567847558268, -0.015396278337467133, -0.010933052003818148, -0.021201014901328325, -0.014114449560974572, -0.0025383800116364963, -0.015435837334992267, -0.0037901199208102876, -0.03438660661272747, -0.01780260519709091, -0.015372277592301559, -0.013342306236463626, -0.025320359921559194, -0.021502216064301825, -0.026304507635267368, -0.018996992813895476, -0.030228937420375572, -0.014043815705787723, -0.003886163620751305, -0.012072877000748598, -0.007714268016612127, -0.021269927239939933, -0.007680886530807254, -0.02345442663684164, -0.019441224458605813, -0.014701286211283943, -0.02607394122246141, -0.027317967616335936, -0.01867499904082311, -0.007249646689720803, -0.02246232335310419, -0.022720139111845276, -0.012952156470131372, -0.007108217289252382, -0.026622632371424388, -0.018895454836404296, -0.022839646920319798, -0.010992392949622865, -0.0026095545310924236, -0.031586903778645294, -0.026286799662819885, -0.008792161328595493, -0.004065951923143205, -0.03557115973504366, -0.02268829749506445, -0.012249481002921432, -0.015731907374973472, -0.02528350937501472, -0.02676300956924241, -0.01644102506159573, -0.018498281051621327, -0.0159426807195676, 0.0058318098132288995, -0.029670851570996082, -0.012530773665701546, -0.04016715365795621, -0.010937459194311279, -0.011338634856927276, -0.04322882434924402, -0.036809347434570365, -0.0007129459252329005, -0.015078773436571992, -0.032461718290791676, -0.030792700391144793, -0.026817004431149538, -0.015494593910148529, -0.008205544060602084, -0.012913702282971336, -0.015046333797482722, -0.00614386928622829, -0.013833232064776485, -0.017886079690175524, -0.0007335037319423951, -0.004247505339336528, -0.011296088337003228, -0.0007132511824743023, -0.00929236264227918, -0.02430405490070423, -0.039033428457677544, -0.01730040566280321, -0.018887289316940243, -0.013765147870569998, -0.006825386589592138, -0.031124871821382967, -0.0232150980396548, -0.008750788069396772, -0.002903017946556876, -0.003174228298689335, -0.03600289251700677, -0.04200464468359539, -0.01132301165852635, -0.010195333309398468, -0.013450837071497487, -0.013553692287121836, -0.00362619371156399, -0.021348960231593703, -0.02000177009722755, -0.01243409547114451, 0.002063380127462408, -0.023744882718413966, -0.00701382171820263, -0.029425644072193214, -0.01707496233263531, -0.017652486720103167, -0.019240869587128766, -0.05196913376554334, -0.0066544206231157, -0.013114500040960821, -0.03941142640465998, 0.006126204240745459, -0.02234891328641583, -0.013122015178784145, -0.03981096178468582, -0.03126477898017075, -0.01975111879074129, -0.00600306979059986, -0.03335485021924413, -0.00555446154704056, 0.013570191578240909, -0.011146608281348816, -0.004329844752397844, -0.0165575353139862, -0.005703059088934967, -0.01763555943170723, -0.024796637546129662, -0.021420028278680406, -0.02910888989293702, -0.01165979535346851, -0.0007126891817984066, -0.0132754261125106, -0.026219777426247964, -0.006344842405671865, -0.013196781868227496, -0.010060524681549585, -0.02584543749095355, -0.00409462959716008, -0.03560774695989543, -0.022362573014229396, -0.012613782779174423, -0.017990338854353978, -0.0022495690215026285, 0.004091362497903887, -0.010758760587008661, -0.0072890786772613955, -0.009575514510829948, -0.021242263252796174, -0.00951741209106764, -0.02694249737598198, -0.03963596457018288, -0.019275002951188117, -0.016899305927666204, -0.0028437628903509064, -0.026360364842940923, -0.010727245478975444, -0.018422134819805477, -0.025229863782541637, -0.014078270015656546, -0.017438537691713652, -0.009196674975590772, -0.00935618345573651, -0.0013102961984250107, -0.0030013560505038005, -0.005125850082859626, -0.005466021238024365, -0.00318541321355188, -0.026760319574241815, 0.0033276129172028156, -0.02251075051293924, -0.02768432361256931, -0.016066755103742694, -0.026099072464001585, -0.017282219035266402, -0.01587596470555873, -0.02504469536682519, -0.014821025612139084, 0.002277315360682338, -0.007455342821865717, -0.013104067106934897, 0.005061347437195737, 0.0031356832758794016, -0.0045976494002554005, -0.020369195987739813, -0.027881070008238144, -0.011327759866243388, -0.018258294756205523, -0.015358055271383758, -0.027787914042273078, -0.0038019143885178637, -0.015565130756030304, -0.011101273562784723, -0.007102486657596738, -0.0078049737888161155, -0.009253197757851836, -0.028858389948464677, -0.023770603825861134, -0.024863637383550245, -0.023400846500155323, -0.03575234525206145, -0.01736972070688341, -0.016689715083686184, -0.00599862837404441, -0.028117921255769993, -0.009612825414609233, -0.014267750704530756, -0.019281658816982773, -0.024814173603746274, -0.043332753375117584, -0.00625135651425368, -0.022379988358707446, 0.0006412712755897844, 0.0015825270263598072, -0.02493312072274697, -0.013767390321880697, -0.008077914864303043, -0.028311849642042898, 0.0018738049699097894, -0.007350296127066659, -0.01306306962795391, -0.008753015753886265, -0.05444574658970477, -0.011051291618503198, -0.029188329710318435, -0.004134282093397014, -0.02224975653727695, 0.003833832446805986, -0.04667555308296385, -0.02602360070917809, -0.013569493402705043, -0.019345278552881272, -0.048099012057089656, -0.014671878362159453, -0.02118114336397927, -0.031022068033836538, -0.04659886528779338, -0.006460994457751556, -0.027469901883761264, 0.007852352018802076, -0.005237644367201228, -0.030648298350320763, -0.00953807569229823, -0.004337770677190059, -0.023718306918553287, -0.013574321871405257, -0.022584322714887473, -0.01970446097782985, -0.004084916619519258, -0.039170573630323835, -0.03654034690715974, -0.027111621560992456, -0.019805500842402554, -0.016844213291724582, 0.001894049345013048, -0.025109977217392516, -0.016090525851439164, -0.009310979634834116, -0.019692256578826197, -0.011184925525352532, -0.02211060142195495, -0.012827718580984712, -0.011840305141267509, -0.02344355538072157, -0.002252337593397955, -0.043802190633854474, -0.02580458886668813, -0.008295141135498707, -0.04371169723405793, -0.019232370890052856, -0.016185784713861215, -0.016294285708933706, 0.0011649308223271814, -0.013368816149824624, -0.014435094336286146, -0.02915799423540568, -0.0027024867974808635, -0.03457879960254324, -0.018593819705300503, -0.024775851925844587, -0.02457649593348564, -0.016101255379799926, -0.00845714489123774, -0.028405666935285585, -0.012005390624047013, -0.008766994030850489, -0.0198381012380196, -0.021386638543493333, -0.04423513029119048, -0.004625375455772707, -0.0016799691613033432, -0.016580369074274168, -0.02268403810930801, 0.0013502198824042245, 0.0024866442632796876, -0.018759950714909192, -0.007210010906848117, -0.0161493465599429, -0.00856105414505526, -0.022279718779528935, -0.009175536380020713, -0.0015587547932413353, -0.021383234501418835, -0.019058754672676186, -0.022663652886414934, -0.029615184885909702, 0.0014518292898382064, -0.02057600081267804, 0.0027762045540365553, 7.70316985972223e-05, -0.020991853758662478, -0.03398054854092252, -0.033115269922980334, -0.027501807680638868, -0.007136900956214192, -0.031044009259143277, 0.0066529302620071335, -0.022558550420371092, -0.01330859687183128, -0.016848659864979503, -0.006802894242689936, 0.0015367423866488475, -0.0075887080934553194, -0.01781477034129824, 0.006561531261460904, -0.015309635075071338, -0.026798776175611227, -0.004395017276281302, -0.011759593613099383, -0.017566105599484797, -0.01501651320827224, -0.026519154888032217, -0.004426770878932098, -0.014815164682800985, -0.020553560028737237, -0.018178918055684546, -0.020969413172612757, -0.000575942056812143, -0.007961029041911089, 0.013779473321787061, -0.01993901198894413, -0.007735718397432496, -0.01943923948467585, -0.01425279972391809, -0.029307546808744374, -0.005233952832635978, -0.02551870379813349, -0.0376169303070708, -0.016694478242648338, -0.013607425829254503, -0.007186336012889924, -0.021248019335672687, -0.032879045716960896, 0.009624233428159561, -0.02476215050306333, -0.0189495416697569, -0.016191794616397925, 0.0011571942258759504, -0.022213132854629305, -0.005505963521005561, 0.012437178361079947, -0.0017429011793107679, -0.00738448940665752, -0.01050225276830338, -0.0030435913773381148, -0.00017534356329733164, 0.0041671703510124105, -0.06255181718821633, -0.03398427673265158, -0.027572293399262596, 0.00466070265789153, -0.011822178532996977, -0.0016951421704685585, -0.013168793423939467, -0.0026638204838482266, -0.016421341386064773, -0.010261954182522794, -0.024125999631891926, -0.02627220203376508, -0.012071171221717058, 0.006172524717651202, -0.012772849951826896, -0.023225796385713524, -0.022733931294578062, -0.036522635458848283, -0.006922737589212922, -0.037039133496145696, 0.0005443730738409067, -0.0457765332153798, -0.016761061555087064, -0.021643999924717165, -0.014686550121419888, -0.0047019400260199205, -0.017922993471844405, 0.0041131050572030875, -0.005351194511511893, -0.010094549710734865, 0.002708782849723175, -0.008301294466826953, -0.02693766446155309, -0.020060691672009905, -0.015443796586762345, -0.008392700842137524, -0.017158469447346125, -0.02001901918146051, -0.012667230051564279, -0.009192998067511216, -0.028205265270821286, -0.018545615406557126, -0.030790331436877046, -0.016801514383339038, -0.01256027410840721, -0.01634757992913708, -0.02024181643197791, -0.021825213511776905, -0.013799581595141894, -0.02382017235342998, -0.01894740237986464, -0.029259392808568832, 0.01030408022361986, -0.006492901658170777, -0.023266913816144718, -0.020949789344355416, -0.016891391667439592, -0.017733973980659, -0.007562380342134476, -0.03683108796033798, -0.01838974744692079, -0.031591871611003235, -0.012973623631074075, 0.0032128421548922237, 0.007843999599272424, -0.008589713139713955, -0.02056180180218452, -0.0034579956185224383, -0.0023242516683412715, 0.002322165124355556, -0.01449785368691641, -0.02827676845251098, -0.033079577604745405, -0.01441929409322365, -0.01467359796743929, -0.007327998906356752, -0.025862700869627473, -0.012503244430814296, -0.0021856206126755692, -0.025434929181977116, -0.02938969722368653, -0.02259381136993092, -0.01797768262624296, -0.007191465948087984, -0.015829426647786488, -0.008736073928985968, -0.01309542177737684, -0.0012585930955360991, -0.0068611440016093685, -0.010150006398161923, 0.006625150274934831, -0.012905325033971748, -0.019337506681856815, -0.02127998885241965, -0.003771204221919951, -0.008378951080573205, -0.022564820379151706, -0.00871391928121163, -0.02759046618464996, -0.02710185369937498, -0.01763762768654633, -0.006222336200214824, -0.0318173017945014, -0.0052025643185820885, -0.006380899020752986, -0.017494257818750098, -0.0038739886383047165, -0.013163579800297854, -0.020609234499802526, -0.0006037165191186602, -0.03162476248264089, -0.0013576276585216811, -0.03882876493783175, -0.023305738584827415, -0.011944952114215486, -0.014468193090801827, -0.019832403913739267, -0.0021800692111937406, 0.0016255484933911835, -0.03533724128011833, -0.00903924648281748, -0.021137244894205466, -0.02835390274881644, -0.02622353834526138, -0.038060141129550175, -0.01819071310263385, -0.034440023855779975, -0.02206229861497139, -0.018199221782838466, -0.007900077042260869, -0.027146294576120597, 0.010338464039871258, -0.004637084733310854, -0.02529636462384874, -0.02277768730080276, -0.008400999372304156, 0.0005475978044035743, 0.002667579516450497, -0.033764294807848066, -0.012862796562979586, -0.026995204870220637, -0.03772439032860728, -0.03137632068651515, -0.01204047935120489, -0.011596469114845768, -0.01453098601589646, -0.005362540605576399, -0.03191330578615654, -0.028297691171522782, -0.012974845089386785, -0.03063393372701917, -0.0005755384092612923, -0.010120728923879158, 0.00046342302555339075, -0.015370760920959039, -0.012157717400423726, -0.01488026870342541, -0.010196958202242147, -0.030941832974343948, -0.022632284962929274, 0.01114644458265776, -0.02690240333380657, -0.008319119280458927, 0.0003675761842168964, -0.006993056931476964, -0.021088342597843864, -0.019552841152171565, -0.012187400865867887, 0.010564348335098285, -0.009318373372144655, -0.03883836414436685, -0.014954463010569654, -0.012033079229665566, -0.03509513253954893, -0.022011505208724285, -0.025297706344938935, -0.007771865280298276, 0.003873629569714805, -0.034587913242058475, -0.012788074795184828, -0.030614252190237792, -0.003217149424812043, -0.018890810041879376, -0.021103579221755396, -0.01646831686229217, -0.0010639178756706989, -0.016413007194493325, -0.01473838657625451, -0.022285343041247446, -0.011920737382261504, -0.008937081028121713, -0.011688028812405573, -0.012009391328583878, -0.004746735284547773, -0.036962806040228134, -0.009906086692437064, -0.015136072373933836, -0.019614144011895013, 0.0031224591440600358, -0.027999059134749062, -0.020059704612760174, -0.008557506546597756, -0.02519735893817591, -0.024325968131455177, -0.002230532061992828, -0.0038464912205534714, 0.008271664350577012, -0.010376174851439055, -0.018849776290095585, -0.009172796740645863, -0.04344717268262718, -0.012124113912543236, -0.03001284055280269, 0.010471361300683614, -0.01374145560714877, -0.03501112713181686, -0.012415902426367278, -0.0011915515416448406, -0.04038298679300357, -0.005756600239244706, -0.0030587282790348865, -0.015662813893173533, -0.005968690180652926, -0.020970501987934386, -0.01709521840106609, -0.012754067850403537, -0.011310616897160486, 0.004226986745583119, -0.012192155025423042, -0.03318504086902385, -0.0047856352758262125, -0.008602204831229157, 0.004597798953062825, -0.0071597699828211585, -0.03499219661162581, 0.0011990377666347665, -0.023797919094989427, -0.026469021321254123, 0.0012122336480534937, -0.025073886501556095, -0.01977164234017667, -0.03894985779319869, -0.004484896229515458, 0.0006896072671041432, -0.019734018809423246, -0.017760042041974552, -0.01902635317813496, -0.009120327930999286, 0.01178839357277546, -0.015093837348865016, -0.04204443101173837, 0.003500912050416087, -0.02656347174348986, -0.03807212666382482, -0.007953208723741435, -0.0020206712985015385, -0.014577437037506538, -0.014670532316468381, -0.022211353038370223, -0.024250818426051514, -0.035467047908761165, -0.012078830875253157, -0.008053831309230689, -0.02675809148023932, -0.007189255274100289, -0.005194211752762179, -0.03554965036383888, -0.005926562619719452, 0.0041683678373324136, -0.01750265863456625, -0.020061795921558566, -0.004336018883441997, -0.04268666936354072, -0.02466052287010886, -0.021401822514991772, -0.025238174848408266, -0.022454618275127396, -0.043495413084252525, -0.02516749051054621, -0.042243658035190645, -0.01981570549022427, -0.038946106622458135, -0.0400815713899236, -0.014421178033637898, -0.02568274076909969, 0.0007786326771020954, 0.008928960066875914, -0.03656916041034349, -0.018338846966155096, -0.008167020051825711, -0.005689861661427694, -0.0033041599564668112, -0.007596836184536282, -0.018565247636834002, -0.03818503837361472, -0.019428248984151686, -0.012471335095888806, -0.004052466971892114, -0.015002190319658366, -0.03133292798368766, -0.010090756660346607, -0.009384130762729948, -0.023517161169822023, -0.043040428004992935, -0.02400317544367512, -0.039695484713998294, -0.025941178895982877, -0.036118448190973367, -0.007281284989578211, -0.03179870045326201, -0.01917083045526929, -0.01942575149510306, -0.0014211076766774745, -0.02645986466686484, -0.003256968901570962, -0.04182391148538719, -0.02616136691027909, -0.0167668495422872, -0.00927815517246447, 0.002768204912440253, -0.030534866692550296, -0.007339204633996908, -0.011567861313789339, -0.025524740133584956, -0.01856495925651253, -0.023937552086846066, -0.01672757207177978, -0.013192791295301229, -0.012696784466627305, -0.008800268351228555, -0.018068325077220754, -0.027440564900322175, -0.03589791646749867, -0.01842507707523176, -0.010780640509274714, -0.01548776150583377, -0.03574537018448083, -0.003914276497511944, -0.016601183751819543, -0.0402285081582121, -0.01952355189582222, -0.0060431433964918645, -0.018494412459454268, -0.022897493445845837, -0.016896356571346176, -0.036179016204744126, -0.017131069735145537, -0.028232823045069435, -0.020712934311014196, -0.011210067714797758, -0.03597192373251668, -0.03839085062294102, -0.021012791006921383, -0.0056107537362956395, -0.012746984596881185, -0.005545092263166541, -0.019341599204690325, -0.04230545930729725, -0.031999044496814184, -0.02129981895607578, -0.004812252371771333, -0.01909188132991514, -0.01575196615207682, -0.000278745022618566, -0.025945115110328636, -0.019028956431942433, -0.0008483252436115493, -0.036533743239818284, -0.014635257989761782, -0.03473948130527113, -0.012426822718863936, -0.01496905813104837, -0.022263597472628373, 0.001031627953157256, -0.03667793336637144, -0.0028215639756223434, -0.018544294466928693, -0.023154876329792278, -0.014723677429538808, -0.024599824483952887, -0.0027205421684025964, -0.010857774816472798, -0.017227191120454614, -0.003905337215417907, 0.008056493950084474, -0.01251262976511523, -0.006195414085351484, -0.02219275621693123, -0.02066768493663581, -0.03457709299714435, -0.017629018502237526, -0.012212683865725937, -0.0006772349379873495, -0.041863885859516774, -0.020690982496977647, -0.029272441466237434, -0.012624634788733083, -0.014787849708438019, -0.018883816309121955, 0.003922140503359903, -0.02510124513891738, 0.002893482662291855, -0.05233857023648021, -0.011579559006699262, -0.028122627256489213, -0.019948993917214424, 0.006228710376213983, -0.002181999790088734, -0.02623194331955727, -0.004027809482615509, -0.03242967504503255, -0.03306446338817622, -0.016830739576817726, -0.02594935229080706, -0.02442448251022469, -0.005822961328153113, -0.006961227191820098, -0.014452059394075217, -0.018788522033426257, -0.040483705948670706, 0.0062921809072638375, -0.007938279756190044, -0.0013275387722271003, -0.02363469844261315, -0.017006912940015974, -0.016875097731621925, -0.025283184157102154, -0.02959502129622834, 0.019947164237896536, -0.01554630168742413, -0.017546806725056673, -0.030073425521386497, -0.033355956439592396, -0.033834744923861314, -0.013139801321268161, 0.0177572626961625, -0.03328117304954057, 0.0011880166151829038, 0.0015375554581775418, -0.011818466702698502, 0.008261534832605971, -0.029982402979153288, -0.026414182774011894, -0.028339284445802818, -0.0021515999832297386, 0.003549399059941676, -0.03141457770269622, -0.011522677407957775, -0.02102506655706855, -0.027057286630732928, -0.018192262724503604, -0.010249600326788212, -0.01770130658248879, -0.011291655971861594, -0.010562353990741395, -0.01031009588091572, 0.0012453099210606858, 0.0014632547990884885, -0.04206380045469928, -0.017642644497504717, -0.009853042264930204, -0.01925042866339796, -0.04185359611889375, -0.015432132489034004, 0.0006183563832054527, -0.008044774006995537, -0.01009407383749761, -0.014312333427999506, -0.028984360365415334, -0.001070940673883732, -0.010882632622389315, 0.002267849620635894, -0.040128294150716046, -0.01888850534888202, -0.006924303427879764, -0.02192562427767495, 0.003862135172037376, -0.010543190200073636, -0.03275587578666634, -0.03359503630934109, -0.025581429769746408, -0.035132803816128426, -0.053905235109939005, 0.005674685831911084, -0.03433367200315528, -0.0011265947138425578, -0.012190822635992835, -0.008328000216623728, -0.025800960105811913, -0.03042596197277662, 0.0076709122543455665, -0.022539700278368376, -0.020736234359213197, -0.009244409076973171, -0.023806137757040154, -0.020510117506205204, -0.024931300038219337, -0.020631560818615453, -0.01131218396533084, -0.019414504961122365, -0.021104693342926244, -0.005300825962644017, -0.021880897014108284, 0.005090261314913543, -0.005726101058571679, -0.007955750263510903, -0.022562594923711397, -0.03531169384109755, -0.0026839097287657, -0.019014814344001655, -0.029023783889138382, 0.006760773514363236, -0.004173318798638441, -0.02005489579891643, -0.008457183911705217, -0.01871580064641989, -0.010542096953474602, -0.02911886137991342, -0.00775791646757849, -0.022091972909577724, 0.0013395886908704703, -0.024393974931602527, -0.02107549071739784, -0.0103494947592134, -0.015092290690979789, -0.016796487116522426, -0.002667661166001863, -0.043274814401032397, -0.023790501043117442, -0.033331409395813756, 0.0038430361624825927, -0.01655018357717131, -0.01748223025066938, -0.01003672453769248, -0.04147182060339841, -0.0009064474646818977, -0.014720135193974759, -0.004396826167001385, 0.008291342448915434, -0.002833401518803151, -0.036562781951404916, -0.006848497689892751, -0.0060116755624188885, -0.009770210621220165, -0.020013657159335486, -0.01932804446025564, -0.021399434509203133, 0.011421026381429169, -0.0029786631696029093, -0.003384959715490058, -0.03563781317627189, -0.052573213252869504, -0.006871755406637774, -0.033272986492343073, -0.03211347403961956, -0.061029839381746, -0.020878379818966807, -0.017276426466488554, -0.03037871012470453, -0.009290187762094412, -0.008838913594492941, -0.019140648080188283, -0.010638076895193931, -0.03951958652573221, -0.0006761001383457856, -0.00950315943887581, -0.00595091075043964, -0.026520043196635548, -0.022090393847832783, -0.004137916569519162, -0.014379288065560198, -0.00839234306795413, -0.05820622081175687, -0.04959333690135688, -0.03550029859202954, -0.02009685395121192, -0.008684297396531449, -0.0014266655824889236, -0.027728510812526263, -0.03247680078706846, -0.03888411885565511, -0.01270546880558761, -0.03288418271796553, -0.03293784961001796, -0.009112117796924977, -0.008026124518094572, -0.009292754666557745, -0.015460330260634961, -0.018130894243477465, -0.025401378110706623, 0.0006681059193154173, 0.012779819389877304, -0.02532677924764757, -0.016229249641985154, -0.002418392814109939, -0.028069034300451545, -0.008124017830665397, -0.01551764748956353, -0.025691892191223524, 0.006496603373587879, -0.01800426841320894, -0.004962091250961457, -0.029927208821343112, 0.002156188923270878, -0.02034933338516111, -0.0289161396823021, -0.014150840466142957, -0.02304039142097662, -0.019391802676535102, -0.007523443471601178, -0.021002196508780064, -0.016741093338469588, -0.02541594916044619, -0.02895112122452555, -0.040813002989595455, -0.007004497782771872, -0.03142716552841229, -0.004413003162662288, 0.005873381132140038, 0.0038867409426456964, -0.0337356693132493, -0.000647033400455209, -0.011299120239768931, -0.023767461650644854, -0.00040272228780219977, -0.009665072698128913, -0.02675942611279694, -0.0199194559500061, 0.0022022377909772708, -0.011043583057968511, 0.005866960679329651, -0.015936769780129593, -0.018338326775584654, 0.004211938125200253, -0.012003439746344262, -0.004255898638111248, -0.009459848285446935, -0.017403967448099574, 0.00033106804873803564, 0.00015093025400743704, -0.01660761408304656, -0.011100426446797065, -0.001441816442400739, -0.01517211050586537, -0.03359621053882412, -0.0038644003606661524, -0.0060685950485541785, -0.019449947720517544, -0.00044880449412108494, -0.024949565318443293, 0.005519073782404361, -0.02375187430029887, 0.009241067606790564, -0.00615378196065266, -0.028776267290061223, -0.007492189412421446, -0.04677223070229365, -0.02150143366429159, -0.020239163661120028, -0.009923939777147171, -0.012677684197838369, -0.011871045423740659, -0.010055398125799603, -0.006669630372483474, -0.022467217296213537, -0.031053169830871863, -0.005768196482865613, -0.0035263681760136253, -0.013581441717876494, -0.015148277204814719, -0.021919052791806722, -0.014066132977496149, -0.024947893453831495, -0.010088557475126093, -0.03612116297086987, 0.0002332979285233609, 0.009713748917673222, -0.0022564213200160083, -0.005704962340417625, -0.03817086131955584, 0.011165746493865935, -0.021746816134678236, 0.004102359599341032, -0.026275602252821998, -0.031474912661696657, -0.014926245898423613, -0.00886872462328869, -0.005383633979790137, -0.031417149314959904, -0.03451990811143073, -0.01627676223297645, -0.007692767763390633, -0.01092640162114652, -0.009876945196722447, 0.0012873390189108538, -0.02939029948630904, -0.02018810954474271, -0.012690056416213564, -0.01748782739648664, -0.017909217537601035, -0.013772165190239095, -0.018783889039797486, -0.024390627020574356, -0.004951707931959069, -0.012870701872119836, -0.002589932012051006, -0.009389211956683486, -0.0006157444233313917, -0.005283928527406103, -0.04032444238272969, -0.013949985622198138, -0.0032006449919196327, -0.016190020292260156, 0.014312686295160137, -0.02288294036397815, -0.01566539437796098, -0.011171579279375591, -0.017861748017106997, 0.0028581300755679668, 0.0013551940155619102, -0.015476938930371355, -0.03718223410879699, -0.02962333571354388, 0.0027187061700476548, -0.018165466012108073, -0.018003141183822693, -0.01820594451466628, -0.03404657411442504, -0.01693748952520478, -0.0003715995129834905, -0.014353020560650506, -0.019021929128708346, -0.015332322274485428, -0.025006075375961955, -0.003449985296828022, -0.027289085129936275, -0.021103144718278927, -0.043804952478395144, -0.019826865755409023, -0.012207725035495672, -0.018435418104264543, -0.020682169034746123, -0.010064672182187042, -0.016027274435026224, -5.2611344097683e-05, -0.0076213277948736806, -0.01932182583384186, -0.03410471422203542, -0.033410163299997686, -0.031100284589392915, -0.003311220264830682, -0.016805346450016072, -0.030718223121067235, -0.043740725603085596, -0.030219441838307704, -0.024431247088964846, -0.0015207982113571415, -0.005343220984902895, -0.036090725932830414, -0.014682229171772842, -0.0033011698583005024, 0.002981291262966384, -0.008646388319490524, -0.010842435259601461, 0.006110747903594549, -0.004202959266855058, -0.02449743461245335, -0.006466996803414311, -0.013459842913913432, -0.033088239735095885, -0.03321385817083835, -0.023095537531326135, 0.002755602982422694, -0.00920182059657514, -0.018989782693987337, -0.04073521927256556, -0.03597340604183365, -0.029705135710467328, -0.02092023026000616, -0.0192117501093941, -0.012722411096420104, -0.03212493955833656, -0.007519810870449007, -0.02537343139593575, 0.011003989960473033, -0.02023632936524566, -0.022918345933840314, -0.01433405033113522, 0.0031981862926179446, -0.007458475211544371, -0.04001039227867516, -0.001839022406655217, -0.03430256970523394, -0.01799782250827925, -0.0035959417596885934, -0.004951694544020008, -0.015402469425862838, -0.004018596979681914, -0.005671497192080715, -0.008697523357836983, -0.02623037152966489, -0.04472568815299113, -0.009756891690629883, -0.006965549807052565, -0.03894771826159626, -0.010031169873327151, -0.007513861180377387, -4.327073019477151e-05, -0.02802206193587621, -0.026096076234564117, -0.00043144688190071125, -0.017649921631102534, -0.01970758602653581, -0.002925656124209149, -0.016929639892052508, -0.002618610005880981, -0.00040850897101897267, -0.026960088363747164, -0.02782053712737645, -0.008312811597195022, -0.011467176837160179, -0.0338734167870806, -0.01419541644491833, -0.029404572103292764, 0.009312924755277814, -0.02088286361024771, -0.02191554069764425, -0.006464582563932677, -0.017964749094464296, -0.011145850963401458, -0.015653510771500075, -0.021556987819004517, -0.016529662011207338, -0.00797871522455796, -0.004521472691759267, -0.038793260360748705, -0.01382323133472925, -0.014868017582592713, -0.008500915625710969, -0.022181539241060574, -0.03243469204011221, -0.03191899388159025, -0.017423994694734584, -0.034281540426524405, -0.040761003383873645, -0.027606185215417304, -0.04865220008876165, -0.011546403961322414, -0.03012547975044144 ] } }, { "seed": 22, "temperature": 1.6092275608405882, "bootstrap": { "mean": -0.00034614827944289337, "interval": [ -0.02784322236364146, 0.020387747101339324 ], "per_item_delta": [ 0.0019762459436821426, 0.12899602731556314, 0.005644700258024595, 0.002515014249449683, 0.07168204580016867, 0.023544225244722453, 0.001439426682217304, 0.03154308462340122, 0.00938005410018304, 0.00037208024149755373, 0.0002096258317101304, 0.0025269708397569037, 0.005834276902937773, 0.17021289018018027, 0.008150518568942385, 0.15137976384811566, 4.357289231253628e-05, 0.09136225568334652, 0.0004796051034773327, 0.000683785459813688, 0.10094666074626746, 0.0005145380781759547, 0.028929465844106536, 0.0008906054607500616, 0.0047634167541833455, 0.0004923311531406728, 0.002370942815438611, 0.005839805054416256, 0.0009321871262159932, 0.00040085634706019455, 0.0009683691553088854, 0.0008952252653540649, 0.0007340064204545611, 0.0007980988988421742, 0.08420793705335346, 0.00909492682875176, 0.0007588681493423147, 0.054971770434708675, 0.00017912083299047456, 9.219310084227144e-05, 0.008208314332468698, 0.0072863769468527306, 2.030119931496681e-06, 0.009461741379604828, 0.0019352923209120072, 0.02289748195749428, 0.00138497761900163, 0.0007138968749340555, 5.72426296344628e-05, 0.0006184642584293097, 0.0012803744112192643, 0.0015104352771880907, 0.23163240821850967, 0.0015389326587505131, 0.0015129511433719664, 0.01273793511678803, 0.0020141855503118065, 0.0010557533841784237, 0.0077934781635771635, 0.09392717063839928, 0.004819121462062526, 0.011514567071542612, 0.01918365684324315, 0.00020689262644615187, 0.13546804600505125, 0.18403478444737076, 0.0005703921710907627, 0.0032394605616942506, 0.000317424750576163, 0.029419296624002826, 0.004233024925392364, 2.3187739640708733e-05, 0.009606949808347205, 0.0004247200894906202, 0.002999482455169638, 0.16640652080739438, 0.04631235356914627, 0.0004888369395002035, 0.0015012320637726607, 0.06691744145444928, 0.01665575529782359, 0.005318773344169823, 0.0006430570237311697, 0.0017910582895068264, 0.003299872059208942, 0.014039281846397052, 0.00361933175942716, 0.006954897519605458, 0.00036853325257301604, -0.04106274824094269, 0.006656170475702363, 0.016589422322420135, 0.1045846847956123, 0.017647922875760655, 0.0031235860731595594, 0.020407347320385513, 0.014362260755989986, 0.0002646940752476278, 0.002755489554185661, 0.0030557602671638646, 0.0016754051551390908, 0.05125432235641376, 0.019756661341926135, 0.010470673871612934, 0.010675600766358751, 0.0007245767891586943, 0.007752776813918789, 0.029601056566697873, 0.14967659352716867, 0.0005257982652484795, 0.0004284532924811425, 0.0005491229878948758, 0.0017690628849008187, 0.005149481513356098, 0.00024197367825579195, 0.045000422964180664, 0.014255034974374517, 0.0011332446801480948, 0.13681010939923882, 0.003813966414012834, 9.123311461500351e-05, 0.003169606784688471, 4.511265550688217e-05, 0.003956594205030708, 0.023241509264534277, 0.0014339511151618956, 0.08470796493463228, 0.0014088109147752962, 0.0002736758156594641, 0.027096618397372034, 0.0002913840047270889, 0.15885604476216558, 5.770071132902105e-05, 0.02442329826159654, 0.0005481725033557995, 0.0008387674758559128, 0.000728183035088703, 0.027344064312859472, 0.055781174401223786, 0.00046278413429616495, 0.0006786504022133702, 0.012676471894775297, 0.0003669554283252439, 0.16841568033165638, 0.19532746002198662, 0.01120992273321274, 0.007476926494019462, 0.00911741133632834, 0.0020175038262323923, 0.0003047975127646416, 0.012195644572321649, 0.00958631127693465, 0.026515971868573775, 0.013388848942252132, 6.1307082459453e-05, 0.00039537294398175914, 0.005720040365490304, 0.003480298403946776, 0.09404524417499475, 0.00210409317398844, 0.0014791772536296838, 0.004201076293122839, 0.018345883228457198, -3.5255909809793904, 0.000704992628314913, 0.10813762469865507, 0.13775235617590328, 0.0005373595122147066, 0.015175049659779062, 0.0035143258002073435, 0.05408965003437527, 0.009449403777570182, 0.03379231493570764, 0.10644723608952057, -1.8192897805887767, 0.18473149380941123, 0.004861745536639265, 0.0885511507327486, 0.001793225077769897, -0.23700626549698, 0.002640026509644995, 0.00011290057582601937, 0.00016222307212459365, 0.01715247288880713, 7.543067020607041e-05, 7.883093290440148e-05, 0.05456959898851201, 0.018454561752065217, 0.008268630426717567, 0.0014798837150055697, 0.00909725597876885, 5.161840609528342e-05, 0.005330820627635243, -0.04937931205508028, 0.016111183159048102, 0.004577699543938857, 0.0009511200601770416, 0.16410437601219735, 0.003376829602895658, 0.005974631896532719, 0.023516552373895515, 0.019363402147852447, 0.0031504542179296045, 0.0026802408330714885, 0.001550648628368587, 0.006263937023702614, 0.03419296258170634, 0.021419570668922458, 0.0035975661485020422, 0.08994146565945436, 1.7339862344203076e-05, 0.028619189941411112, 0.001860035955151781, 0.00039984797021591655, 0.00044694447680642156, 0.13785478379124416, 9.252420709910331e-05, 0.310483294395862, 0.0002968455751100353, 0.0009273800722941518, 0.0030139970994355913, 0.00015925965679520134, 3.5061731392090726e-05, 0.0010809687502740433, 0.019361848961229115, 0.009007340438479786, 0.00045567857939855317, 0.004512494604287937, 0.0004716231018723902, 0.00014621269995058344, 0.004026002761232991, 0.0007902930886912525, 0.0020663092029347464, 0.0016184376216796246, 0.00017666599893799672, 0.0083276547336545, 0.2851553097337284, 0.0005125732734458411, 0.04070561072748001, 0.001289265556823998, 0.04169519363759142, 0.0063357119699509355, 0.10120441269434088, 0.0002470517162920018, 0.011847370756315556, 0.1920055558515314, 0.0379470478069521, 0.010955174304679101, 0.012741558448777623, 8.558503864759072e-05, 0.004764707368775972, 0.053513872605178515, 0.015380529458474445, 0.0048266198177935446, -0.8123378801104837, 0.0528823540598755, 0.029116010189773896, 0.03280513828263085, 0.025104557949019474, 0.0019285075457215128, 0.05149244308864229, 0.0020203852599554855, 0.00015184988532989155, 0.0001347385423274048, 0.002658296378838058, 0.011987718245870227, -0.012990320758794915, 0.009730906864509004, 5.7519400594095354e-05, 0.08750526291804459, 0.00015990796789269038, 0.008512208607015792, 0.002348226452558806, -0.12275726516393548, 0.003041865487076549, 0.0006065432802141109, 0.00021653355031908046, 0.00034556066657356893, 0.010797403217634725, 0.0003428682590001524, 0.022164673230710134, 0.1292289464551455, 0.030439133805135472, 0.011962735666101376, 0.0006666912907996415, 0.1326048218594161, 0.0757240876871958, 0.008836362068902606, 0.0032142804496385724, 0.0795277197671518, 0.03895350649713239, 0.0008216302470673435, -0.010711061898289964, 0.00012021541684270284, 0.0009339314695161224, 0.0080009564367032, 0.006900986794014161, 0.00022103960126956015, 0.14000685052929035, 0.0018186909773895447, 0.0025383353865250204, 0.0028149360979060445, 0.01713083120589185, 0.0009148407868070639, 0.019963947689625262, 0.0003872573822388647, 0.0024253846608778273, 0.002708470432036579, 0.0028965598568000115, 0.004897599660917879, 0.0037967988073122814, 0.014788767860886173, 0.0035901353005264773, 0.0007210614182479928, 0.0044165718659857, 0.01511312791181519, 0.15110682960942695, 0.0035095496318454343, -0.8916336126167055, 0.0003088790766095593, 0.009795628326775956, 0.024228557075359678, 0.22240577408409692, 0.020775400684942706, 0.011239067502063918, 0.003540205930515298, 0.003251861000273578, 0.013448406513002536, 0.012721312540733363, 0.000577071836419833, -1.3803612096244788, 0.0019613608933762602, 0.0010007638056739405, 0.0025337369244618295, 0.15935585859531265, 0.03831566968940653, 0.00093755315265644, 0.0016022961773066934, 0.1249151859243383, 0.001581065465673694, 0.0001230096892160337, 0.0047923510384374965, 0.0024252511578066248, 0.0007753296728153676, 0.005194128574404926, 0.010902335355320463, 0.004032661479814193, 0.03771190456281579, 0.01894494657270747, 0.006201149605665452, 0.006072147954353032, 0.003320451830464519, 0.09468717645484515, 0.0036584196515773645, 0.0010860328113250297, 0.02314579792407452, 0.14099977645638614, 0.00017562854681538227, 0.004684931098860238, 0.0007326215444612602 ], "resampled_means": [ -0.001787874584893793, 0.005787287250087345, 0.016422162527581836, 0.01071518414688168, -0.00041679339449816243, 0.009375410008431137, -0.0019011424034505386, -0.03827974031107617, -0.007404438652903814, 0.007227353567008827, -0.0036516793046645307, 0.009209402460332384, -0.004053572302587271, 0.00517307149451288, 0.002075656610325348, -0.017895996149837248, 0.01730700502554103, 0.01042162116764275, 0.004526416813867144, 0.004007065259064252, -0.004516323518333669, -0.017457592053797048, -0.010323395660377217, 0.011891133907283889, 0.005432684365552458, 0.008931207169457943, 0.0016628842853040767, 0.012150064431111489, 0.004026353211491993, 0.005062842765915133, -0.008364981303290808, 0.021477145007853516, 0.017185244858607014, 0.001968521309414721, -0.013164361762021235, 0.010457394156619669, -0.004345609326010936, -0.007883003459334554, 0.011618603039598946, -0.029063089325075103, -0.0038205435782050477, 0.00791277907403599, 0.007653069306886234, -0.010254312412329984, 0.0046508662803854415, 0.0030278974132424493, -0.01936522516781796, -0.01609966938425072, 0.007154504972634213, -0.021473969541841626, 0.001772665338899346, -0.018930985865003094, -0.022987070261338005, 0.015403134565854789, -0.018357871548929566, 0.015658766327386928, 0.00017328636657632713, 0.0033658157143584, 0.011872473534753349, -0.01256420916327326, 0.00398960871274339, 0.009430438448129156, 0.022773355108295812, 0.007066644441567408, 0.00033808381596538427, 0.018268892125528523, 0.00944596619651594, 0.006913243043966606, -0.0033036744745497805, 0.001307273679334286, 0.013719333663524565, -0.005189450291457073, 0.00582984045044769, 0.0008498212582877442, 0.00856672130599299, 0.01956511745003081, 0.005758608651485032, -0.01896345479447264, 0.008833135854040434, 0.007109545469429333, -0.0009350902152660126, -0.008926895410229539, -0.006839890497195775, 0.0018257538267872288, -0.007125923958125524, -0.003560566225516051, -0.010109913756254226, -0.006463256564633033, 0.01714079625912464, -0.0033149984857189577, 0.011736059187261358, 0.015557942889866077, -0.0013647237578850054, 0.004484723591774541, 0.007217607014661602, 0.013198825370586264, 0.004706948197616419, -0.01664786790734532, -0.0015371835617331872, -0.0005169403239683518, -0.029879400169654517, 0.006863045811414789, 0.022222446607757258, -0.015341773719180728, -0.0024310116788795193, -0.004604203495376, -0.0077963575805545604, 0.015487054786129403, 0.011791754693945878, -0.004526990076400783, -0.013711806540602655, 0.009308599843226144, -0.004782048369465981, 0.008265080579567362, -0.004295079339456702, -0.01676762566620205, 0.0054248672901562505, 0.003112733136493688, -0.020780485110262013, -0.006219064959561853, -0.008553462793103596, 0.005712502017394403, -0.0067781378002806725, -0.0074300348068624955, -0.00025996840371177073, 0.008619085403558456, 0.00254670566236273, -0.003861302975513232, 0.012454906925731462, 0.005720177565143671, -0.008236935221569142, 0.0014171199062721388, 0.00893293326914776, -0.023966798693055846, -0.0015973892515743992, -0.025236532432586826, -0.002080289904383354, -0.007545808738184588, -0.013175526331828327, -0.0031700440374331546, -0.001248512371600246, -0.00710429284884259, 0.0023813160527158947, -0.00027554274304102263, 0.006365428198597294, -0.029005124577680817, 0.006433381505284426, -0.004863617701189073, 0.005112481095800583, 0.013189842188712999, 0.008367505189213758, 0.007942152992608205, 0.011483057303889802, -0.004128935017773662, -0.034089548833873114, -0.002951521465595994, -0.004784651663530284, -0.008440535761245817, 0.0007059513489106674, 0.005644549216809031, 0.005600225142409984, -0.010198314636363554, 0.009528021293648593, 0.002073697245059502, 0.008793268004489624, 0.009692155769680123, 0.011970771093830527, -0.012330906944335966, 0.011229770741099019, 0.0066918733237738386, -0.0017727860503677386, 0.01557788089291692, -0.027618072875009538, -0.0017670361261863722, 0.003672685944811615, 0.000476184965240979, 0.02156772429626784, 0.0030577900803935077, -0.0016480614710879743, -0.007564587977436427, -0.010019832468293294, 0.011354416759636103, 0.009622263629995728, 0.01304085655027805, 0.007080748780129703, -0.004166089389373976, -0.025311598205361514, 0.015353588807020301, -0.0015269541063410892, 0.011491266768385966, -0.0161904750202624, -0.01724065900374303, -0.001696124485537561, -0.009845419880812476, -0.00032623263080067995, 0.007412844175088622, -0.00729508106520053, -0.0028336039572343726, -0.0027278374346025426, 0.0001002843350306315, 0.013576147366427361, 0.02283859551459961, 0.001044889737641699, 0.006529980873209721, 0.0046130320258802255, -0.010410754381461813, 0.0001287480730066601, -0.017851782430926045, 0.019980457469227747, 0.0018198235875984026, -0.004203551867917753, 0.001281277202207017, -0.0036602824625271537, 0.0005417609327561746, -0.007037729667884392, 0.0007424006793453505, -0.0269834961913999, 0.0012626785395688542, -0.013012124683337314, -0.005142595758982954, 0.004459715085399011, 0.009968519169999324, 0.013794320798752232, -0.007310098225265632, 0.01716502334532459, 0.015633149003276173, -0.019020940279163602, -0.020273609917397985, 0.016522468967538485, 0.002502213353947073, -0.004699877434070118, -0.006488413635123585, 0.009402522261152487, -0.04571515705958272, 0.0026522827913448227, 0.0018443239589109662, -0.010076960562271184, 0.002761602342431846, 0.01951561612590435, 0.009849538962623973, -0.008845084369212358, -0.020304178968458868, 0.002645878949609195, 0.011155963614083325, 0.010880759310992293, 0.02102570254965002, 0.0038160308106281256, -0.008912473822689317, 0.003654246094647484, 0.00021326297817556418, -0.006585822658561952, 0.010647456798271808, 0.00020259795258719188, 0.014160177417829125, 0.002170013778072599, -0.018496785935235644, 0.015011034264350688, 0.00011490524015615825, 0.0008733395621598206, 0.005610057478324262, -0.004174073176136656, 0.01590775418077273, 0.015747000467585995, 0.0001099951355647999, 0.008008175718292371, -0.006181785462279918, 0.0044972663268267555, -0.002462608200124767, 0.018550635379871956, 0.009346490169459131, 0.010088791621268264, 0.008740866582800499, -0.019433822304131126, 0.018984888221341055, -0.0042716756941932645, 0.02227305006997204, -0.012817610996897212, -0.021410441451574863, 0.012576467521417506, 0.019308326954074927, 0.0044765810962851225, -0.0223908264477522, -0.013124943510587113, -0.018453446329328774, -0.021902954529363472, -0.002143055118811926, -0.017405236507862652, -0.039921396426900416, 0.006324738998347941, -0.008769714346222247, 0.0026070822734067614, -0.010765910014449213, 0.004259726802907026, 0.0215605006217607, -0.010341403702596121, -0.013909078214078018, 0.003515736970426517, -0.008776825490277964, 0.003450966409790963, -0.013535772133388016, 0.007863635321268204, 0.018948525043299587, 0.003242444029481908, 0.0013852221259741493, -0.0057666363104765484, -0.017063362994885495, -0.007557650270477181, 0.00300564918131278, 0.0003402903316833658, 0.009786964446841712, -0.004282103022809929, -0.021092779710093233, 0.02204139134597925, 0.00974699726544227, -0.00620708301159054, -0.04707587585700356, -0.01043276542154097, -0.00708761600023106, 0.012655133434125686, 0.01948312447816742, 0.020637471421514675, -0.01728129462734633, 0.010555064864082276, -1.1569515956719487e-05, 0.008591081238672926, 0.003671427655216693, 0.002677419557931705, -0.004199273388920321, -0.008289187991208268, -0.0048285919995327865, 0.003970550836094105, -0.005269500858602719, -0.010912201321112052, 0.0003230254885348486, 0.0212501072495122, -0.010942265810068179, 0.003303334601863571, 0.003424029920341525, -0.015457937381017231, -0.019538697836296116, 0.012895218860584675, -0.011732685256812427, -0.008370457663386, -0.009213014703792908, -0.012867995474767484, -0.024554828557905115, -0.012823190710177911, 0.0023315157136390785, -0.02429303409464602, -0.03816489356415246, -0.007201897092253758, -0.002009686078556599, -0.005587223545980346, 0.0030728306068599966, -0.005834046073655531, 0.008155241260692593, 0.016152633596105786, 0.020641484238036738, -0.018081166097450238, -0.0037879111053372517, -0.004845483864547829, 0.008804035882529489, 0.00638546388629769, 0.003634134793995619, -0.018090809385859732, -0.011085183713981843, 0.02317629543178158, 0.0038154805262545235, 5.9026103589235016e-05, 0.002836284695679466, -0.008653695734865316, -0.0030940430307006654, 0.014202284083303445, -0.020903674015196178, 0.00958103395780462, 0.005336065134707711, -0.026352478865039584, -0.00346583716355957, 0.01998277962392236, 0.020872257834483435, -0.006578714942585232, 0.0057721682830716966, 0.01532346927081442, 0.0014321151736617793, -0.04235933539726758, 0.004735351899353328, -0.011397300058291591, -0.02401451141264252, 0.004272385233413085, -0.003155300413257462, -0.003099610573156088, -0.010677816166613015, -0.010297976525906659, -0.0044382756205778905, 0.007079788503911547, 0.006959046161951669, -0.005774106814707747, 0.015443720511896731, 0.016261562138356684, 0.004275190836581059, -0.002953582973783384, 0.012776277962123113, -0.003743666320767998, -0.02195703883667139, -0.02408106557882002, -0.009217428730626672, -0.017682961762006736, -0.022899521174955736, 0.0009007419967108669, 0.006896872982337289, 0.010388570357550347, -0.0022009727221409606, 0.008311979147287844, 0.012541198865244975, -0.01009997806206691, 0.01403884971056875, -0.003571810877863858, -0.016223427077110737, -0.019538408073313862, -0.005227610447199461, 0.012144834217314241, 0.0002647699754490973, 0.0028650087764666332, 0.0002454521738283841, 0.011565516107187238, 0.005009734922532061, -0.0024589790499566764, 0.007602702208575317, -0.014589464215612274, -0.010770443590738574, -0.019308510384819722, 0.008039930159000015, 0.002946000427921118, -0.003850238596557001, 0.006823271942893082, 0.00469316041163538, 0.0021410768584043424, -0.022212286230915895, -0.001944159911256122, 0.013658289597564632, 0.0015201800459508992, 0.02241098305739846, -0.0008911721602390257, -0.001669444209745193, 0.01274299176411927, 0.02710090935668458, -0.011799534800123069, -0.02914374783828477, -0.009051334735768175, -0.004517927645891332, -0.01175786446952411, -0.008150769686701186, -0.03657529853575561, 0.006004775938892514, 0.0033390042691615767, 0.006187556374831371, 0.009315507608786713, 0.002821172054640154, 0.008240817683275692, 0.016162622997169713, 0.014829697345680567, 0.004700740726763704, 0.010404568943525823, 9.047788752059767e-05, 7.280563770666017e-05, 0.004337913368500547, -0.005240362535118504, 0.0074375090695089595, 0.008599870026989111, -0.019499394120173257, 0.0002545135218910878, 0.004803492238441251, -0.01806359826501543, 0.01077535924388424, 0.00782957944469793, -0.023033206379776763, 0.013343421761093746, 0.00891307022904538, -0.006568534238829281, 0.00623145441001457, -0.01540647118070742, -0.029237160677561155, 0.013951818045812844, 0.0018688715570186044, 0.005472942223469842, 0.014491844496260686, 0.013409578745266142, -0.0024011837653410925, -0.042946159318809636, 0.012296086741383274, 0.008947112832674206, 0.016704402806071144, 0.022585440223272232, 0.010383576269076166, -0.009464495535399483, -0.024364464424962914, 0.0030277857892839887, -0.0008087460183301014, 0.011034318814241509, 0.014730752495939733, -0.0045425607802007896, 0.0015023039450613273, -0.010490988743175311, 0.008697718023888358, -0.011556239713594972, 0.005363707408187762, 0.00653548623602323, -0.004497575070615994, 0.021797846245895182, -0.009684316731174173, 0.003994194044415697, -0.015244569554725248, -0.012054976810137763, -0.005157476359022435, 0.0026512272533928015, 0.015210759550312019, -0.0018950575943741863, -0.03782893703665281, 0.005099575699287206, 0.0029858902601546298, -0.0009305434068878799, -0.006003495133246445, -0.009134371615520857, -0.007788889617439156, -0.0013432063377064151, 0.0026351435561656304, 0.0063322543734408016, -0.004824700612833547, -0.013144577922614954, -0.015738777126958137, 0.010055052597087062, -0.005292031191513504, -0.004057840360437135, -0.0010699174484990125, 0.015779284508346675, 0.008364062669011678, 0.017395134892330993, -0.0034454481658392606, 0.0018330567901805933, -0.006665014757253636, 0.0031135648883614427, 7.733808052316386e-05, -0.011269219277490437, 0.008890876797654447, 0.016401925758312372, 0.010449184883077904, 0.00660151034159892, -0.03273626818809881, 0.012965077110178292, 0.0075844305988711745, 0.018304791337830254, 0.02122359596290504, -0.0012958859105066446, 0.010201932481275626, -0.013989969048427676, 0.002965176446456773, -0.024435681869785543, -0.011292811706668194, -0.014753750707985293, -0.024044743432456384, 0.012791865363316317, -0.002109399433622536, -0.008512195414813907, 0.004232211590360948, -0.03334049551838887, 0.0191730251473265, -0.008828523740445505, -0.020797372272858848, 0.005625283176577237, -0.013716037176810518, 0.0028167717466685695, -0.023908224766363805, -0.0016248149859724424, 0.009314498062061504, 0.0057493796280874265, 0.0032852209389363637, -0.00022449614262458494, 0.0002816447677474132, 0.0010026998742825484, 0.012941166733098614, -0.021637191739896786, 0.013747657691312717, -0.012700373676455134, 0.0021515919797757527, -0.0023675234409211636, -0.002858523565508173, -0.02450239724928777, 0.01147379931196739, 0.016972141326489294, -0.014062944264665877, 0.00013352320426447407, 0.016037815108524662, 0.015523167892859995, 0.0007540352022723092, 0.0007908034931050856, -0.006957727837053802, 0.014759940836910297, 0.020857331336621723, -0.015693747080085956, 0.018767946471453387, 0.008664858884666912, -0.006980531972946773, -0.0034656688268852698, -0.003258322991558305, -0.00037605116904638255, -0.022938427587143154, -0.008759765445865018, 0.0066811516968479055, -0.012135825814035448, -0.011444512280725206, -0.019223486131389557, 0.0028770514683156834, -0.012924836822079724, -0.022403577068497056, 0.016571724009988804, 0.00223453785707739, -0.0068958369814248344, -0.01737201920313893, -0.005593198017170614, -0.011614536614503236, -0.01271833062868195, 0.0022636595466098775, 0.002714043179541355, 0.011802231939448493, -0.002181140526707466, 0.005690282082895301, 0.01518265517279341, 0.008845389341279857, 0.012704093107060842, -0.009791501603136899, -0.00789235045027765, 0.022372792055641008, -0.007304220091817072, 0.01705378858178019, 0.009401405202794599, 0.009041024923674139, 0.0030302395100446017, -0.011309467679892108, 0.020063256488336182, 0.0015414755095939485, 0.01466961995519062, -0.0008610929603503875, 0.009137879374351456, 8.250309647048513e-05, 0.0038647106112845607, -0.01089445886946795, 0.006077134129502857, -0.02843037099379917, 0.02456238041884996, 0.002793062355174479, -0.004622562822210895, 0.00844408293656701, -0.00602552143247379, 0.00601887502502201, -0.012632461809454968, 0.0021097853401411226, -0.0019013349048105181, -0.00011951468409982081, -0.014274032860200726, -0.02084200911207219, -0.008053444504749594, 0.013607600095610507, 0.010362611544029019, -0.001528702119144191, 0.007659336462749147, 0.00886498437656916, -0.010322769219956684, 0.0025977773384510943, 0.0037800535793503422, -0.0009490198924760774, 0.008665356835820693, 0.006290762229488124, -0.02548074472132585, 0.010357806153884866, 0.019320155869663604, 0.010565499706033581, -0.012806434684287469, -0.04222253479818386, 0.007250074797065218, -0.0009045912823640658, -0.009966970569872004, 0.004366566359439466, -0.005771077620235505, -0.007433696475962171, -0.042229004477775806, -0.015859104366317624, -0.008607806577946141, -0.013400193737454451, -0.01501353490503637, -0.0032757628701998165, 0.00829343320717706, 0.012006292373244434, 0.006725034958955186, -0.009744049670083077, -0.018672526023961213, 0.004629651610245363, 0.0007161467620166155, 0.0010883433567905338, -0.020267629828377758, -0.022491827599638435, 0.0035998026735837136, -0.000343555715802533, 0.014826009246061932, -3.572925597181574e-05, 0.01903886346090534, 0.010674917180337296, 0.009950490593673365, 0.0051221368906264105, -0.0064772706544408256, -0.00961078458249873, 0.008904782714195926, 0.0019410186088412045, -0.015490924762272994, 0.01228025011340844, 0.0026724221609369173, -0.001008952453551304, 0.012081629558687926, 0.010609870115864152, -0.015455489933875051, 0.0032833441503551286, 0.013818614344481504, 0.00563238496734046, -0.004391815448101236, 0.004699868413477962, -0.018732116551692306, 0.007307471456497451, -0.011817821043320884, 0.005381160398663917, 0.0006916128785006769, 0.0012403430839059493, 0.005655131965496258, -0.029108883046914447, 0.0013196850378282407, 0.018898512643611243, 0.00679233241466995, 0.02010154854377401, -0.005828003517057851, 0.006250190448421421, -0.007590127524828429, 0.009727233393873531, -0.012138693739128743, 0.010163298087083692, 0.011227413206678409, 0.00845040287979629, -0.010471307928050599, -0.026489484760047373, 3.9811016436813115e-05, -0.010222414760074109, -0.012722372055574156, -0.011929921347095, 0.020872008285364972, -0.013850958971756899, 0.005579893965342092, 0.00014455539409691614, 0.0028391997247387934, 0.013029105220249537, -0.011496956217147454, -0.01573937209408785, -0.020792464788382755, 0.021472425490432677, -0.006799975291080106, -0.009973449919456354, -0.002592098167621394, -0.000479882258490038, -0.010765199989563686, -0.00010367808390403829, 0.001294017906519508, 0.009638078302417533, -0.008564122965239656, -0.013433912046820078, 0.014146194223709234, -0.015767524260212672, 0.000103132932778583, -0.0011152600859527899, 0.0007244165462480498, -0.0037327539257963407, 0.004280887791759785, 0.004818147079225723, 0.021053736723995024, 0.002972730685714381, -0.0025655781825697343, 0.01425553910270022, -0.013125869106937277, -0.028910890884129216, 0.003348729858526711, 0.015850512504151315, -0.008998881725854653, 0.005418379886756223, -3.216557054948795e-05, 0.0034839477251083087, -0.008147656945183586, -0.004935767575700667, 0.009168376083973084, -0.010680981611329412, -0.00018808825594097874, -0.006916582542797105, -0.0018517749015520698, 0.00757582946022437, -0.0014888551904057763, 0.010847730918352153, -0.004405362937319298, -0.009516525634312453, -0.0011422343177116335, 0.001855165763149074, 0.008404739028802928, 0.010856159435851413, 0.012499612271397224, 0.015118397286131454, -0.0049975170663045364, 0.0024668029064938823, -0.015573009487178617, -0.011998020408842838, 0.01810935660445419, 0.003744047103309757, -0.007437876801881665, -0.003033229421919227, 0.001966243279932932, -0.007044048776253887, -0.0011869085313178506, 0.003865115865275284, -0.002297303186421519, -0.0026455577093008774, 0.0021562528837648642, 0.002858562444069248, 0.011253167330924201, 0.0037664014719470955, -0.01556442331531832, -0.0002519505431985499, 0.0030490675888700937, -0.002490166084032527, -0.0190945998624361, -0.007640269227912106, 0.02401029532763767, -0.008398854841350627, 0.002040655287625295, -0.009666499135553131, 0.0057130305452994575, -0.005972638637201462, -0.007905082824014838, 0.025922857364937782, -0.013519554357806152, -0.0007261513731304681, -0.01207102906059201, 0.005091783228431896, 0.008496868583345576, -0.013037200156333141, -0.02145022558622451, 0.011499642705896153, 0.0034779744305264006, -0.0007846658174161414, -0.033271986056691384, -0.005889032978101245, 0.020625230705096028, -0.010437166551910503, -0.006490465847132061, -0.00041451561643292127, -0.0011603046539446352, -0.0041182654357745235, -0.0016509169446793555, 0.016783256262007523, 0.013952763736975411, 0.00373722606655956, 0.004470041572719086, -0.010468322264323858, 0.0014886108824734314, -0.024613711956587448, -0.003093118895340639, -0.01006218581335114, 0.01267911816374006, -0.004688678696561694, -0.003821204095497781, 0.001056237879144243, -0.0014567046263720266, 0.0004361558400659561, 0.023828886099338593, -0.020039092071348515, -0.030082364576882497, 0.0026415642618790223, 0.010046305965416249, -0.00924901096681058, 0.001452953025906652, 0.00955392021146998, -0.005012560958529518, 0.003320070726337433, -0.0032465073053939086, 0.010242753011937345, 0.0035437004079083992, 0.0020887825675938504, -0.007793545987983594, -0.020438477653240885, -0.00187493923584758, -0.004459043441994024, -0.05263785029103395, 0.0002695200064854994, -0.002427265225055112, -0.023423397379015942, 0.020073053846333216, -0.002459277797640943, -0.0005308580190076295, -0.024442223240441648, -0.010849013723362421, 0.002028366316354592, -0.001430625730039829, -0.009375887532101636, 0.01721555353121151, 0.02498390191822256, 0.0015328531004425007, 0.008659274733921543, 0.012267787812152391, 0.006025214081668405, -0.003472878219340329, -0.0003556705301487526, -0.02340663868230676, -0.011789763953400239, 0.010093545755788864, 0.007460481908999037, -0.005068692421822762, -0.0046937206288121605, 0.011400741854760922, 0.006699576504518588, 0.012894682353307018, -0.013137582495091562, 0.0041842308043416685, -0.02933704545517351, 0.015074604661235282, -0.0010024632067130847, -0.012067880071379695, 0.011468485167610377, 0.021212258295948395, 0.020376138861706142, -0.0015774885239641925, 0.008912816458742392, -0.010929612672689944, 0.008340846804560804, -0.024518450574915918, -0.022382569403618272, -0.0068065257762714, -0.0009476416925354996, 0.0038265121153208004, 0.011963400768533031, 0.005550724650055064, -0.011634545478073107, -0.009576149401445286, 0.003404472898849787, -0.006723826462322779, 0.009669547408429225, -0.001481792661691424, 0.0031980741215080566, 0.010947057972640064, -0.006570315076806338, 0.008519242075848064, -0.00024152345739903127, -0.012243562005495631, 0.008571819827774064, 0.0035133502976081483, 0.00935994641878773, 0.015192155265300863, -0.011197933103073278, -0.01667394976576607, -0.002717485047975419, -0.013032520550810616, 0.008400976133290577, 0.014390118703096134, 0.00012191042897930555, 0.008613484751276544, 0.024599342269478595, 0.018625108618680164, 0.0013364864841247967, -0.005379308767350173, -0.007430366436101598, 0.0035849573748877378, 0.0062200561751119, 0.00035993252697110486, -0.01613616768790354, 0.015208516223030098, -0.015672000884706074, 0.0091355175639209, 0.025044929948519278, -0.004698725966577823, 0.003200392185984524, -0.0005520588547518961, -0.022947416367217366, -0.01998582780559472, -0.008274158097295169, -0.01287689328946366, -0.010170066236516966, -0.012216095051021673, 0.007103013844263968, -0.005681343693226894, -0.001985970408063125, 0.009748002526049225, -0.006811968139589873, -0.0039948976656140845, -0.010140526013644669, 0.0037561491503574347, -0.013668896246647196, 0.016676195143970594, 0.01023663844965556, 0.0020334230719425772, -0.0007813275253441109, 0.0021217840396998513, -0.021713879670328284, 0.013633790116653596, -0.0011631891380958596, -0.0013837949637270062, 0.018295714011655455, -0.020081770964942822, 0.004580363967292892, -0.0027328094498857984, 0.006216059543587428, -0.018505985773777073, 0.011295712267622897, -0.03433609942043869, -0.016109441853030506, -0.0031272842262463716, -0.013553278022792053, -0.03153135624928643, -0.011026656275502221, -0.018725318715260394, -0.0010274610112998117, -0.009050460407591146, -0.001263645534477175, -0.0056519534544024, 0.015607683190720959, -0.008298744530208636, -0.012357487315215434, 0.004491213982712434, 0.004572573686172467, 0.016147838040475582, -0.006583533378838506, 0.009366771984538285, 0.007673919567415795, 0.0013833353386224984, -0.027572533102064454, -0.025927530888819036, -0.012476529407124948, 0.004822712083621892, -0.00956033449531568, 0.008924122119950428, -0.029921597327717726, -0.003994949046577973, 0.01702437096208797, -0.012106020214116144, -0.004025694863000854, -0.0141840293933707, 0.002311551225301744, -0.007868105523829057, -0.007660098067081354, 0.003316143238253281, 0.0031436275698827286, -0.009446733614317396, -0.003704623306661582, -0.03642313236028289, 0.00013164062456584672, -0.00437667701050669, 0.003800884285966102, 0.0029997835453120106, 0.020913817971506963, 0.0008316048580617463, -0.02985298759714394, 0.0004980116927211268, -0.023698556092409284, 0.005031740509576578, -0.017934731408436762, -7.483402950037021e-06, 0.0031479946307408926, -0.006630795342599022, -0.0104783873304893, 0.0028544046115101212, -0.01142271745160005, 0.0021166509712417137, 0.009752892636157938, -0.024093403477657007, 5.9787034368374846e-06, 0.0010824353348572204, 0.0024200802088262894, -0.014139199616300759, 0.002105627779351033, 0.017907126426214098, -0.002287664971985066, 0.012831461539209993, 0.0075661197812739315, -0.01125264476959064, -0.010936442193444373, 0.00910781016574891, 0.0017311964400450093, -0.0007155512786354401, -0.002378582087493717, -0.0062208986334129955, -0.01852439065068393, 0.008963032551126645, -0.012796023736169266, 0.017440486366413346, 0.007552034101177949, -0.018833887455823373, -0.022746254221361974, -0.003804194985669662, 0.00014166654233902756, 0.0006040283124312898, -0.027071367464551275, 0.018698898035705902, -0.011159207722410875, 0.013371885633016029, -0.010089863571968713, -2.3052209360088475e-05, 0.016409162862478344, -0.0050839537757358146, 0.0005869577804621779, 0.0076178150968849055, 0.010038683172123612, -0.006190176688404176, 0.007357494276399686, -0.004461907872704306, -0.010578298962891251, 0.004194306229204046, -0.00827781779083084, 0.0007118344912309006, 0.006068710462498344, 0.0010618882635721422, 0.0036605742755662068, 0.010595359047904889, 0.02038165777816611, 0.00662073351062056, 0.011636365742253584, 0.0006656937297457496, 0.008319596972093687, 0.001776597392177315, -0.0008189284179002036, 0.002505438897995599, 0.011904369125693821, -0.015823585169850114, -0.03591660877373638, -0.015817717617457337, -0.00016479945154462703, 0.003780582568966283, -0.012989754125373853, -0.019332415746159664, 0.02119883455097252, -0.01907274461914107, -0.006482205643305284, 0.00627202823773033, 0.014034808794967355, -0.01684491809979015, -0.0033936280834697685, 0.013579673213867905, 0.011140744985749464, 0.001557082606502289, 0.007253946917347215, 0.014120234319074616, 0.015233342172100378, 0.018692731647129444, -0.02842634142731077, -0.018820408146047688, -0.0046886316265991775, 0.017647264569380233, 0.01454274170338969, 0.015274379067086983, 0.00015530879579663413, 0.010101067733045066, -0.009729776703839172, 0.01307780970539472, -0.000638542646968654, -0.006920533336116293, -0.005935101322433871, 0.005887303704085882, 0.0034286661394104007, -0.01042732266927301, 0.0003617424973758301, -0.01570344341363665, 0.009680412221534387, -0.01147242825367935, 0.018512798381611028, -0.004811617191672112, -0.005960734297403102, -0.01476366127174387, 0.008592662589460882, 0.003940622369857836, 0.008174941780253278, 0.013382958232547556, 0.006244413285404386, -0.010912593432749808, 0.007246116371779825, 0.016990910974051445, -0.01432469973956801, -0.006713955590176068, -0.008158256513088399, 0.01129534565175291, -0.005185958652972418, -0.005783877478596031, 0.003954383419532824, 0.0038640216450674847, -0.014660095141643131, -0.01139646204407748, -0.008750230305219457, 0.011325557395761665, 3.603855978648133e-05, 0.004390004776493306, -0.011656224101407138, -0.004852463900111673, 0.008828694222061005, -0.013156391447697733, -0.015211120861937783, -0.004395564662898529, 0.01599923523971889, 0.005726284167896922, 0.00044738885899377716, -0.008982335184712463, -0.015380078017870462, 0.006047898224521648, 0.005591447855088223, -0.014238545545049195, -0.004079922623145084, -0.012594290975321868, -0.0023934740715196253, 0.013793846075500252, 0.024376953725919068, 0.0014934605710121593, -0.019336966383793952, 0.00960120300743834, 0.00758057056400709, 0.016799955379045054, -0.0064656899169544535, 0.00021867451857211876, -0.014959985439611521, 0.016353526048475318, -0.0018546968617864288, 0.011226407805855845, 0.013491566408157425, -0.007017925746384204, 0.012705779109472733, -0.01627617949660987, -0.011300052823313181, 0.0015843935820408464, -0.004949333657761069, -0.00047941452776738806, -0.00670049142359968, -0.0003434146424728381, -0.0002170904990052421, 0.004731012549668041, 0.015807312035896297, -0.0027176533941834923, 0.004504599267942339, 0.0010365487053363525, -0.011654472490169956, -0.014762687273965926, 0.015241373671260949, 0.003836737194421169, 0.005534410365969704, -0.008267935530011549, -0.011404970935133183, 0.006811374580030903, 0.005475888487403201, 0.011337121993968304, -0.013681529175135968, 0.0139828120573622, 0.011091187909263476, -0.001146071519253179, 0.009852370743969874, -0.009101664693640214, -0.009326909516742223, 0.022117660579284153, 0.0015598835465552032, 0.00813281602300906, -0.013789601239159704, -0.014299397197437133, -0.008198979849952092, 0.009064936245823753, 0.0010503574051312452, 0.011602979237509544, 0.01496767302291828, -0.01541715284330003, -0.0028734814564356807, -0.004765289782444268, -0.004595652795613299, -0.005273414123537773, -0.02280214413959389, -0.0052729649513615934, -0.027875232562812987, -0.013350009218937845, -0.006471543918482017, 0.00683953988968879, -0.005848126262293451, 0.013123376124591423, 0.005961564936557411, -0.0020100338241341247, 0.005490325602522281, -0.001006618826242916, 0.0023793031600361256, 0.01595495059242834, -0.018862648446479724, 0.002238695469331816, -0.01608275598197118, 0.00043600086142323084, -0.012773562624058172, -0.0004276901763573969, -0.007600920639118124, -0.0005518629641094255, 0.004141446418276476, -0.018600075191464987, -0.01469452324765356, 0.0034711896720822275, -0.02599625287978836, 0.013186830169961248, -0.002427354346257471, 0.011119132767207785, -0.0077100942933540926, 0.0023182788977385363, 0.0012273146495717887, 0.008524901460803706, -0.008201721149525703, -0.014650255378738873, 0.022315783482269674, -0.015394368614470364, 0.009471355188051716, 0.010764784519981991, 0.0030132982270782145, 0.0018006534696170344, -0.011163247479879882, 0.01438252917946104, 0.01866587807935658, 0.013430131661712981, -0.023007117077230108, -0.006452190970619635, 0.010287903795751861, -0.020705968922379917, -0.005634569728034126, 0.0011755239500369755, 0.00590753479703183, 0.005920711454614338, -0.009688249547122767, 0.015490407605755813, -0.004156698748014021, 0.002559108911267226, -0.004712999507187162, -0.007102696047264528, -0.0072555384362524705, 0.023000246998609288, 0.012962928275593508, -0.0023088114249488205, -0.018331996053570044, 0.002176453858105216, 0.019516653366157172, 0.005639964946778401, -0.0013551133940413556, 0.007155398988473344, -0.013253214037859376, -0.004275900656503106, 0.007657835937956641, -0.0033479176016653115, 0.023365949508638004, -0.001214158639545562, -0.012483097077720617, 0.013460953531369632, -0.008273955603671418, -0.016657942476474875, 0.00702762543963616, -0.001327138072684966, 0.015424006507985302, -0.006082806799324837, -0.01232788993278093, 0.012308071264039195, -0.03503837119315632, 0.008807897757547156, -0.006212517106053866, 0.009700209239107999, 0.005567535768998342, -0.00043057680491632105, -0.007962633917528944, 0.00907410030783265, -0.02475895359866116, 0.02036350771538738, 0.005062928506325458, -0.011427375877711808, 0.008299024949311756, 0.0012236440816139399, -0.006944058083874294, -0.002517876600689011, 0.015443287747174983, 0.018969133483816934, 0.008565696785875848, -0.026662105211697963, 0.004682727778249117, 0.007067888349234774, 0.011499807865580622, 0.009412216969349506, -0.00306795706369113, -0.00028049062317737845, -0.007939922012837858, -0.005209291167898754, 0.009296494207541538, -0.03217619318132568, -0.0049597706179397945, -0.02422866484266127, 0.009878608187198597, 0.01388319186597836, 0.009807645774868059, 0.002501871455582561, 0.0071612944889355955, 0.010739480003055876, 0.01779230037838767, -0.005662936680238805, -0.026549392056316425, 0.015846079703745162, -0.007239055443223319, -0.013920135291409494, -0.0016610588166634792, 0.014969872856359536, 0.006668572125165937, 0.00040346927698659614, -0.007152277992235025, 3.789682361995765e-05, -0.02924818700614414, 0.001377449655999636, -0.002150656252206587, -0.016519725869213922, 0.006263829653352735, 0.0012409278088161791, -0.005908704258263363, 0.005971972493666132, 0.010378301722680639, -0.013494303580521127, 0.013583023177382206, 0.005100379432520137, -0.026124135257462072, -0.0103457857232488, -0.009138121747462738, 0.007522545653294658, -0.013023695140623065, -0.01518185689705555, -0.00844394468219267, -0.010282900940814568, -0.009437319350747339, -0.010453886027141543, -0.004185232137116663, -0.01398884196479784, -0.0162188686111932, 0.011658525356733881, 0.018433463974013634, -0.0242998996378146, -0.005665979410574424, 0.007711194912009994, 0.008502559929838045, -0.004329012691701396, 4.3350212511150675e-05, -0.007634651763344998, -0.003594024273935476, 0.009445458725734159, -0.0019282698736796543, 0.006362727036433202, 0.005039577720038112, -0.007043761640311133, 0.008983565039278364, 0.01251750021317724, 0.006911930394730542, -0.02922220630410712, -0.0076251452236617435, -0.013699414713498219, 0.015502847587250151, -0.018012729458229588, -0.008717950056376404, -0.00042347404565581514, -0.0002182605068392222, -0.012634946280220325, -0.0033935967747368042, -0.014140042955157108, -0.0070640523232399196, -0.013232450053808744, -0.0044829385564173745, -0.0031980376586632627, 0.009427783296798732, -0.0032489355944752686, -0.01861054635057208, 0.0016836778466436707, -0.015658315184215513, -0.022003127787363467, -0.005961841677450485, -0.006125992032628634, -0.009188901894273165, -0.0144821685268479, -0.0017833373274218276, 0.005289619114462436, 0.010769127375808065, -0.014911684440786131, -0.024112236817197215, -0.009210183495260867, -0.01633938966203411, -0.006360972242580227, -0.00900447826775457, 0.004770790904541363, 0.006171924370420487, -0.02311332723815538, -0.010934888155340557, 0.015439094886229505, 0.009390918506568275, -0.0016187585327076001, -0.001598357727591422, -0.003238034353684844, -0.006032796355440658, -0.015990365802424265, 0.004057492262917692, -0.0036797920434074195, -0.015389825508587585, -0.031130677339395122, -0.012030551024725413, 0.000438262313778683, 0.004307224637605467, 0.010198510108241163, -0.001262849631969607, -0.02784240158930373, -0.0085641581939102, 0.0013529956337258076, 0.004033226087991366, -0.014980035641887346, 0.006870770206151643, 0.01702006637652355, 0.0034523977982343352, 0.013173228335475852, 0.017921032435905374, -0.02368457758242229, 0.012946539851285356, -0.016905396710081077, 0.008303035572903156, -0.0029828924652813777, 0.010879958546099222, 0.0074527088263752145, -0.006152683819329482, 0.008526802226867698, 0.002261762631583844, -0.010260750284764836, -0.012417352909077955, -0.005110807699705019, 0.015768136493461954, -0.002862610756388672, 0.0014596229995303763, 0.000541814656326261, 0.022900121760039773, 0.018508716075046214, 0.01043980689321099, 0.003776005707602031, 0.00749590888175666, -0.028941444908166733, -0.006339752442984921, 0.006126210251250051, 0.0074674499694455596, -0.011165608114013688, -0.01406772943864981, 0.008011114953980355, -0.0009562944176326595, 0.0009646774795798028, 0.005112465006700537, 0.01005878869227575, -0.004731857574910927, 0.01476108115075279, -0.02442566000712976, 0.00887661483003846, -0.02891731700746874, 0.005394061887848371, 0.019830696943176452, 0.004477668210373715, -0.02509776322754468, 0.0029745831216818975, 0.0013151955062175333, -0.023818556404757753, -0.0026965202992737585, -0.015197932951315773, -0.0029871794786076114, -0.00709391782038766, 0.0023560710542518738, 0.00853892079049589, -0.008695910105748252, -0.028972871277392676, 0.017937670683828084, -0.006777974941359536, 0.00935112209500531, -0.017268810947698006, -0.014253513268067838, 0.012207971175921777, -0.018795996433872395, -0.02615441690199661, 0.018763878354722596, 0.018262645237174437, -0.0023268279713927814, 0.00012005040017180428, -0.006564686363624281, -0.027421247845509398, 0.001169527453832839, 0.018848391228385164, -0.01716919108998574, 0.008704816262394903, 0.0028938591890507698, 0.0002565299715566491, 0.013444161829227147, -0.03102479222187152, -0.00694381850653422, -0.008294299927418652, 0.006126181574183566, 0.008444524906748149, -0.01834681796530533, -0.009844367537012013, 0.0016079081981217147, -0.014580592372815766, -0.0017817997737021966, -0.013014815842530019, 0.0030175268377274148, 0.004154236540479571, 0.007092458125262167, 0.004436747465588273, 0.02109210392177326, 0.010859468223290423, -0.0074113068279403884, 0.011352713592273024, -0.01769227191321208, -0.021595947299611482, -0.017780539080586202, -0.007610975401872993, 0.014635651768593206, 0.006420495466013611, 0.002615655015698262, 0.007247324461998675, -0.0004805473143306433, 0.0029841301253612272, 0.0015199213480891593, 0.01597180374996558, -0.02742708842722764, -0.008014346268196334, -0.0018972764621304128, -0.004584332948764168, 0.01784661945086308, -0.005509246092783114, -0.0022522007937121587, -0.021444822606406535, -0.00032968035270288984, -0.02103045675409855, 0.0008387247920396838, 0.01984345581080658, -0.013762901766123773, 0.020923763846784706, 0.007508512517793998, 0.005138738500984654, -0.009629059978727072, -0.021994857034400083, 0.009615659418303396, 0.001872029724480388, -0.009085725608936883, -0.0020971576537314757, 0.005248278662815458, 0.008965812518875653, -0.009665162807186791, -0.002891756618971159, -0.009105180294739834, -0.0010494022085594493, 0.00818773139767942, 0.013142847743803825, 0.0006635540140609784, 0.018785406512217256, 0.008495326495403535, 0.005851620648845874, -0.015628403901533514, 0.004115754067655771, 0.013772699671002446, 0.010953325829532679, -0.008309815670595062, 0.013623797709277744, 0.014409060626245841, -0.008813412331632266, 0.0036393587291704975, 0.0006347600586652082, 0.007568447409397813, 0.0022464742941991456, -0.0029253568301711554, 0.0032108443781453597, 0.010677531021933407, -0.0023961288460251015, -0.024741770314323876, 0.005981246873765952, 0.002703561704772351, -0.005699098458087515, 0.012239119888768175, -0.007922553545014914, 0.0037469400436090164, -0.019123792067206453, 0.016022737946851072, -0.00046440190499904925, -0.005470436782702891, 0.011430511038506062, -0.02449925212838707, 0.0032118544426372494, -0.010442878770672778, 0.0112388793553399, 0.015919413160055598, 0.014039987865571101, -0.006796436299373508, 0.004541677403212268, 0.01048086260867547, -0.006437198771210207, 0.001505365451596193, -0.005362910071319535, -0.007588280909186574, 0.006823825166703865, 0.01718094145347099, 0.0002493623565565391, -0.01845759476007602, -0.012658406953106466, 0.0025652754894836863, -0.01671054129996591, 0.01080088660944987, -0.04948635650627708, 0.004075417420867878, -0.00807706242298422, -0.01461487995435838, 0.018257105676949832, -0.004503736293781639, 0.012729303306938246, 0.014784859925133233, -0.03803201366267332, 0.008407446690463087, 0.013579530716449042, 0.011547221445200453, -0.010868528488853061, -0.015511415289693384, 0.010437622478557536, 0.011290066804634992, -0.011642983400093948, -0.0325880844476974, -0.032365928971592454, -0.015236176175894138, 0.0037648701030145245, 0.007773891321629788, 0.012615555279160005, -0.018416060502948577, -0.006587605489767013, -0.01961110180484418, 0.002394276225484097, -0.023307666102136092, -0.014400340976878515, 0.008336542696896869, 0.011073766028219653, -0.002027725992737121, 0.005361195821790428, 0.008769043311556538, -0.0031821873865816473, 0.023976770059665108, 0.01379152059389917, -0.0028068720487034954, 0.002146731835893115, 0.006098873660041347, -0.007092633792926023, 0.0028009491428251476, -0.0065154957797338275, -0.017317753764948058, 0.023837832632282084, 0.008025973962733239, 0.002765813688545064, -0.005453829167401164, 0.0176531366021369, 0.003918766481416512, -0.0015758205713149182, 0.004305511346444786, -0.017683721460630214, -0.002563145479937449, 0.005166251899588744, -0.008627427606087024, 0.00535267986446027, -0.01842046946152932, -0.011019966892564875, -0.0201880637693109, 0.009195307794324432, -0.0007511781986159008, 0.014283803046298886, 0.009839403322284558, 0.013723488929933985, -0.005353311947615677, -0.0015279315934925881, -0.003373797887407443, -0.002565296436460946, 0.02137059552149045, -0.00898538010565898, -0.0056472354954302945, -0.009229001536182444, 0.013947971060331788, -0.010589953800608176, 0.017939449906674308, 0.0015160450396393842, -0.0016045207631470023, 0.005908516051263088, -0.011287500336174853, 0.018072441506436877, -0.00032813008662446557, -0.011176012680147347, 0.0063509896385393625, 0.013163059422410731, 0.017256254072656955, -0.01068376788716881, 0.002829041950936405, 0.0037337589934169444, -0.003954652349472304, 0.012066990293777743, 0.012804217886908319, -0.012215889037993982, 0.012172715578120814, -0.023902124254303284, 0.009238410473120554, -0.0062344792217001125, 0.025148086258010967, 0.013836399844126167, -0.019990632163257742, 0.004447051266243179, -0.011514311795969881, -0.016102481910550186, -0.012156544566943888, 0.008148820571291793, 0.007635628341743884, -0.006889465350315752, 0.0015378670819882398, 0.009898349535196214, 0.010327537878237893, -0.01163430972753887, 0.003856608827551427, 0.0011055824873288473, 0.004288497795237414, 0.01292314636366116, -0.009250451844131423, 0.01379948064166451, -0.0018162463317155746, 0.008027802190838908, -0.0040884885059933925, 0.008283234171334256, 0.016414287596033233, 0.013121758461880175, 0.014000052925942858, -0.02031041160745163, 0.015510978801286933, 0.008369305155672597, 0.01413037975869514, -0.002131061021634811, -0.0020214272366995555, -0.007568876887107212, 0.022029901171666175, 0.00492341536931548, -0.00802690760364352, -0.0027490219578024974, -0.014311980670199101, 0.0003164032720863932, -0.008143552843050746, 0.007289427583791893, 0.02174957907378596, 0.0036697926636863583, -0.003163066825904178, -0.005332638244739085, 0.0028388093093293138, 0.011726479827557526, 0.006912085668713699, 0.008400893544005273, -0.010131851676215303, 0.010287785221450409, -0.003082896779913275, 0.008208591370369635, -0.0005588457246943673, 0.008935659300847594, 0.006388368137147222, -0.007336549001249602, 0.002097625470561441, -0.004391621327555192, -0.0008068353012711434, 0.017465294061314113, 0.003930990790218419, 0.00014217895391989513, -0.0005647560389822307, -0.01048945485606702, 0.015802900415927468, 0.00646320158475762, -0.006599934784464023, -0.008834056371452176, -0.0029699996145868897, 0.014327554543344704, -0.009393640321930304, 0.00570514753244357, 0.016100146548286294, -0.014059503027826815, 0.001879565261653868, 0.005129345248246361, -0.0007045458474534737, 0.003930056069312185, -0.00653781006493708, -0.004568051720753361, 0.00179379718100176, 0.0036205357991637194, -0.01665279235631179, -0.0252513532110572, 0.001179701198153782, -0.014579838002194645, -0.004449228775552831, -0.001937476757991959, 0.008081595071539447, -0.014458952342742684, 0.018919699687911677, 0.002300953610672877, 0.013045198620244138, -0.0008017958616690598, -0.023120653627719312, -0.012203208320273114, 0.013753211400206331, -0.008085993290754155, -0.02207566920106082, -0.006743716224394383, -0.017149598885362233, -0.013804525231703803, -0.006646175450832712, 0.006995868161675044, -0.026490648275474763, -0.011570676407028965, 0.004493080371087824, 0.01099889853932079, -0.00216660526661196, -0.0034521415216297426, 0.018495658528370765, 0.010999364330786564, -0.004171232206756208, 0.005580465530640724, -0.01383524655189471, -0.020222662888363357, -0.0037828271234418218, -0.003811248866557335, 0.013558994729600428, -0.0037081764026697384, 0.004063753204770271, -0.014948524080958377, 0.0013804878354863436, 0.0033315388876152, -0.0022234414284630927, 0.00857856678347352, -0.004590326113727144, -0.020486634016267147, 0.026178085025260898, -0.01274596130648298, 0.019070083517874935, -0.007860662273558446, -0.00600082893962953, 0.008048061426701659, 0.010323065019364806, 0.015680641270344707, -0.030634312284549856, 0.01392247562082763, -0.003927166304941523, 0.0006041400016216707, 0.019291726903974654, 0.01066648987003894, -0.008630343338259976, 0.006256804383039083, 0.006609885990375593, 0.018302710134393427, 0.0019881092037663628, -0.02359016806493364, -0.011166204989022033, 0.016059001068184992, -0.014206587447897875, 0.0182185466222091, 0.0061721257458229135, 0.01115632980631684, -0.024013849909226056, -0.009594721920663974, 0.007264801849522791, -0.005112627210424021, 0.0009771165015998837, 0.013183161628416871, 0.0010110087532126785, 0.011420909057389185, 0.017087023571442428, -0.006987804942523974, 0.007457809987065262, 0.004600279399786338, 0.0021594320084421686, -0.011985136873835343, 0.009969065914761845, -0.001096150343011601, 0.01825798200011559, 0.003491049929441608, 0.0025449987872139504, 0.006275997935849916, -0.00780882542442007, 0.0005668367251449714, -0.0040056874613220005, -0.009205155479779422, -0.0038107403781046083, -0.006307577429849192, 0.012236577195536655, -0.030689172469667216, -0.005353390888770723, -0.009885098376456803, 0.0040002286783383505, -0.005022064799372972, -0.02026837288919496, 0.006044542762224004, 0.002031953635375291, -0.008609907752361336, -0.01819040090737926, -0.009106815461850206, -0.04031514808436791, 0.014953796025945796, -0.012490189802374358 ] } }, { "seed": 33, "temperature": 1.5367823253742512, "bootstrap": { "mean": -0.011514493749402574, "interval": [ -0.03964704432209044, 0.011288205657133968 ], "per_item_delta": [ 0.0014874206061791825, 0.15195244638917438, 0.003793544568691181, 0.0008628739393801759, 0.05708533270700385, 0.02841035939279759, 0.0010834536998720846, 0.04835543201133133, 0.007841556819940407, 0.0007690064508296179, 0.00023313685665056377, 0.0018779972420786331, 0.0029751169585221842, 0.14027365019648597, 0.0036106246642234425, 0.09644794991202524, 0.0001579509079237239, 0.07879076411912717, 0.0003813626540266984, 0.00041503618837683906, -0.3957782919835453, 0.0016939419109078455, 0.03382187604294315, 0.0027414818063270035, 0.0025425304201538742, 7.744114912997155e-05, 0.0009587276887592885, 0.0073476273060552625, 0.0014310348572207678, 0.00026301342604910136, 0.000573423528987792, 0.0015623174182077038, 0.00034402851171304376, 0.00035417985745006406, 0.07949511214331062, 0.01090502890384529, 0.00018859701819727098, 0.03967772212193302, 0.00022567871307505957, 0.00017165308799489603, 0.017133896136010227, 0.010567168639180408, 9.710496180650425e-07, 0.014613356074186257, 0.0016599018539709238, 0.014372978547915196, 0.005743579429166226, 0.0005916908049274115, 2.5912137997633962e-05, 0.0019533250659538753, 0.0005853301927980931, 0.002233390407154894, 0.21206450150431466, 0.002825654804179339, 0.0017419164797876341, 0.011553920191249071, 0.003521237618156232, 0.0004245834235999082, 0.011535801563434128, 0.18693756096377445, 0.0031054609435573976, 0.005986866964458806, 0.007464428526435366, 0.0002703927846485813, 0.12024739767857359, 0.19904421382971715, 0.0003553287799922739, 0.009217915179608134, 0.00010587354057058497, 0.017919562952401772, 0.007396814712301594, 2.034117435003529e-05, 0.008066298563824712, 7.173114080643247e-05, 0.0024759461624822105, 0.1929436367438825, 0.038275495272580985, 0.0004968624739390529, 0.0012961678081742972, 0.12988233699096535, 0.022443239563754717, 0.0038080895720736282, 0.00027505677414356645, 0.006413177783093594, 0.004343705777481203, 0.017303452693380867, 0.002553347433065266, 0.008080805317468328, 0.0003504181299907726, -0.9948490011200133, 0.0043049147051295806, 0.006562806187148959, 0.1454350606823191, 0.021468083665760637, 0.0031952203326697497, 0.030074732066575748, 0.010800126204589438, 4.890479944801976e-05, 0.007757837629755815, 0.0074417898407326604, 0.002883645200789392, 0.08036105761664591, 0.009517566532791545, 0.005444641781676969, 0.019945492136963866, 0.0022519513807675575, 0.005008686379024822, 0.032729629700794544, 0.1486151311594121, 0.0002614744091093586, 0.0003601921344558905, 0.00024314592033673513, 0.003807958738439642, 0.004728987421560992, 0.00040060015994088354, 0.041123453889224074, 0.018473852557402193, 0.00046219641152427636, -0.008042910685669646, 0.0027549269068815707, 4.692010511256096e-05, 0.0036547563973174206, 0.00018984508309872404, 0.00268779335336173, 0.02484810487949026, 0.0011520684782154711, 0.12123630185586445, 0.001684273849858228, 0.00035898711578154555, 0.01492434252750345, 0.00020119764865606182, 0.10638389650140523, 4.153950558718753e-05, 0.016324504174018126, 0.00022758353044643762, 0.0006190214932339808, 0.0010919665553636864, 0.02347138479401593, 0.11784353542249443, 0.000567322964588141, 0.0006768446022833247, 0.009490234398093815, 0.0003794145208431842, 0.218490780547414, -0.14735563743357138, 0.009777535920779505, 0.010809736641738092, 0.014813289468418785, 0.0028961995608768496, 4.187333992157744e-05, 0.01161537756311375, 0.0113154058767548, 0.03507324442283516, 0.022212857904110907, 4.008726116666889e-05, 0.0006124860812793789, 0.006800077934452527, 0.000809563450601068, 0.11410802669173642, 0.0005533030416639909, 0.004609816424872423, 0.0033460853132189117, 0.024926049423631868, -2.8523943396763336, 0.0006983207642169629, 0.041942231315543435, 0.1511730354613957, 0.00043734008170917375, 0.021260550706539722, 0.004756708026005401, 0.03897550215630119, 0.007754948773870832, 0.021357916048163013, 0.056672702640202736, -2.157148223669253, 0.19236189833250794, 0.007507930903099041, 0.12158784012562382, 0.004648971496139645, -0.2580203284083682, 0.003831785182979348, 0.00013435546910445947, 0.0002378047974254921, 0.018278603618528854, 3.285966983864629e-05, 2.5745875617595647e-05, 0.030499254716441118, 0.016257651235661787, 0.01600457520966056, 0.0004663570900785413, 0.02116828755564697, 3.161606802986574e-05, 0.0037495659879148253, -0.1471211126755918, 0.046097589377395665, 0.004483024516706698, 0.0014509546259784739, 0.15399834501875026, 0.000869316350871356, 0.009113422758582675, 0.01812286916891241, 0.014103622765631407, 0.003257866782114166, 0.003380430618204879, 0.0026790940479190332, 0.004437248767409032, 0.0139361775244339, 0.04578301335347758, 0.0005577318877178914, 0.06023981024028133, 5.953673488484226e-05, 0.031205276965858956, 0.002225092727951371, 0.00021917256693165086, 0.0005124506854147124, 0.140798977349575, 0.00016703186192087668, 0.17273631261861255, 0.0003148050567160888, 0.0004079183397279481, 0.0035767350086116207, 0.00018630713811814553, 3.566808728887173e-05, 0.0013692976760551138, 0.01746064897190415, 0.010916894646425956, 0.0011236959734588579, 0.004735432500411595, 0.00041170166258281744, 0.0001204655395643182, 0.006273185763538593, 0.0016149637388511917, 0.0015477289466234894, 0.0013332958767516014, 0.00013639475155005659, 0.01673814786302174, 0.08844874201689246, 9.31838508605233e-05, 0.05348376552461384, 0.0012496889258799377, 0.03300688325712934, 0.022697495634304392, 0.12354566820632662, 0.0001400885961710847, 0.02913900313185032, -0.4448648861541391, 0.03034159594060811, 0.023366180683162447, 0.009676343093045844, 0.00010563424630565661, 0.006708802238230645, 0.04313735307718805, 0.029082303427479684, 0.0010096623503356894, -1.4035242568046242, 0.026099018760823683, 0.024112729575753195, 0.03204070981517883, 0.016754884462984274, 0.002220359830135281, 0.028256569099808795, 0.0031194594961937525, 6.36183410101923e-05, 0.00021791203110159012, 0.011728313620820389, 0.00842713672983775, 0.07361830099474533, 0.01300905556315645, 1.2872510279695505e-05, 0.05680761591270564, 6.605217208609355e-05, 0.007336910058121562, 0.0025678491233109757, -0.5632003130814538, 0.0035925052915155354, 0.0008225132076309458, 7.82839875561803e-05, 0.00048656250396825093, 0.0047531845671311915, 0.0005741657218450624, 0.01454776621716294, 0.1747840873738954, 0.05454559819864166, 0.033034527595333756, 0.0013136999922571109, 0.06610671510711981, 0.05706775938673288, 0.005439794721459543, 0.0019815305478625583, 0.046248076167304544, 0.032827635005751955, 0.0005857021016312792, 0.06918908173565397, 0.00019801092705462703, 0.00012068086539693325, 0.0031977868353330603, 0.010554242228566234, 0.0002771504822580216, 0.12218546883820473, 0.00027424726331530086, 0.001997855103907343, 0.0030325917042585914, 0.022295311196938485, 0.0001985046783839771, 0.06481747993775733, 0.0017361617494736524, 0.00084346257581918, 0.0031863263190727393, 0.004680759992586383, 0.015571542482216115, 0.0048263940402477785, 0.0339346140106951, 0.0008454945378115425, 0.0033671863783103374, 0.0015082787434872688, 0.012893388220729192, 0.11690087405653338, 0.0028645655448003558, -0.5420329059925546, 0.000866313872189094, 0.011407140098045093, 0.08519982189335207, 0.15404019049012502, 0.012608589353066996, 0.022906810839963453, 0.007994898746277767, 0.0010322613478847434, 0.013556004882715066, 0.011501778180722923, 0.00031717895319999267, -2.155880544182102, 0.0014539738981281197, 0.0011411245989566422, 0.001555992018495463, 0.1054685836821001, 0.0563919547940163, 0.0009937489721568214, 0.0008033912993523699, 0.07409514315399326, 0.0013474347387996523, 0.00013132342978473068, 0.004174455627955076, 0.0021141770161482135, 0.00078800532050072, 0.00686440079420295, 0.005105226333100764, 0.00490997086179927, 0.020986695162045665, 0.014341813678215472, 0.004489407075336469, 0.004337983048204501, 0.003650440261917367, 0.12570587934616528, 0.004427857761209859, 0.0024807677817573455, 0.02018185563906483, 0.07125633582169906, 0.0001343343489881035, 0.0026106726511473384, 0.00015054017338357134 ], "resampled_means": [ -0.00034359370877989903, 0.002541190288942087, 0.00033273094954097015, 0.004280470965697315, -0.01052535834819562, -0.0007432649104074238, -0.0019171947354551895, -0.039043062136909806, -0.01769206527999031, -0.0016440130183653428, -0.01624965831079044, 0.005737573499094781, -0.014320208228629191, -0.004134844647776899, -0.012465349211352924, -0.04183193367335888, 0.010798517569884376, -0.00016659304476136135, -0.002922066120656207, -0.009773158368764618, -0.010986825083596542, -0.03059872244810092, -0.02408636151621267, 0.005165957882333115, -0.009183125612354117, -0.0051772575066333405, -0.011748439675323382, 0.001850878924802351, -0.001979402327617521, 0.003226653344850085, -0.019860675133455227, 0.0042620853728601, 0.004799814082523787, -0.00552985896891613, -0.024684352038274463, -0.008066783301104259, -0.021712022580860486, -0.01796111513175004, 0.003209480182381065, -0.041141009647422705, -0.024328743909693976, -0.011573208915330093, -0.005727716720440833, -0.029751694688683196, -0.011517736804560972, -0.007333830465053174, -0.03164810181052923, -0.03337130713104239, -0.0013403103027714981, -0.023684816238511253, -0.003660521715295453, -0.04208351441047361, -0.027508110199121527, 0.004145057486211882, -0.02302365285309797, 0.009054727700787706, -0.007768817815034459, -0.0071410535996799685, 0.0008940778954481236, -0.017910738626897488, -0.010266240333314783, -0.009509949541192315, 0.012010657141015544, 0.0021396277516845226, -0.0011366299746216372, 0.005149267748315979, -0.0011087511432282284, -0.006126149768769355, -0.016830698340488056, -0.0018242251176704832, 0.003053821961676673, -0.010860541837119408, -0.00123885292378259, -0.01744460008272506, -0.0007712365990599768, 0.0036911993192536956, -0.009747743468272452, -0.024787394695849986, -0.014532563867748868, -0.002699366792639131, -0.014370638747503415, -0.031819211277104235, -0.013573153100700027, -0.015890438882243833, -0.023991338416562993, -0.011281302305769129, -0.02261302263922669, -0.018084220974233085, 0.00350236180320078, -0.018079265427701056, 0.0023941153995499353, 0.0026478885838459292, -0.015727194712466808, -0.007904983691924711, -0.002390182880712653, 0.005295360741932309, 0.0002645592392670272, -0.02705636584635227, -0.013110157547542313, -0.004952044738121477, -0.03373917843641647, -0.002800845387621142, 0.01260531972283534, -0.026123175915696423, -0.01687399699480013, -0.0181178958541999, -0.020559756628948038, 0.010956921417155277, 0.004084337406531668, -0.015574325246409348, -0.026283460453808954, 0.002850791755031371, -0.010589328643900078, 0.0005104013882552531, -0.01995423930460111, -0.02497059093707946, -0.015055360898002043, -0.003917041497536317, -0.02166876482122816, -0.018469794635941464, -0.01564825402605711, -0.004944122278971896, -0.020583404205665175, -0.02294643713911783, -0.02144630999741349, -0.012264934916764602, -0.008756903769558866, -0.02188660090161476, 0.007711864018086852, -0.001778794245096632, -0.02004415178101021, -0.01517226429309542, -0.003397246279416667, -0.03451363780580682, -0.014779373370711271, -0.027795605771906817, -0.015924527544416266, -0.011560708298207026, -0.0191654429609429, -0.0180970387870988, -0.007682760328036229, -0.02655101731008508, -0.006121980819204979, -0.02001004212528305, -0.004049633259859278, -0.03482315698480266, -0.00046381845737919697, -0.009380323456202851, -0.009119797468320062, 0.0093792162712503, -0.0026044241290892572, -0.005371606508970376, 0.006517403818943215, -0.01646319897729266, -0.04634257197452951, -0.011790153711223581, -0.022545152330696105, -0.01894260272345759, -0.0024926875539297857, -0.0016475398653253482, -0.00394018394620508, -0.011459189178270495, 0.0010357425509197815, -0.015156773250058856, -0.0019213366810216655, -0.008175769791484034, 0.010838611034636729, -0.025977670252709657, -0.0014725720545330995, -0.0028163935103135765, -0.004302093876452598, 0.009186587013812076, -0.03414852466588862, -0.013010529041612857, 0.003071437542109467, -0.007993115834067492, 0.0001765357219634852, -0.005565789539758988, -0.015887010914120873, -0.014565515756509306, -0.027508023681301216, -0.001100146963109606, -0.00020697198516945797, 0.009218596888024856, -0.01365150662658747, -0.027639735100494268, -0.03805130991378478, 0.0015202182985167105, -0.010179637283570698, 0.006946082341112211, -0.0317935964223915, -0.029812384222189854, -0.015132456468398815, -0.025915517244593362, -0.010209286990482016, -0.004227282789561296, -0.01759037394939214, -0.012607853064962276, -0.008804305849616246, -0.008941685151617798, 0.00825616910609916, 0.01198631581363592, -0.01795197548398382, 0.001321423764327544, -0.008231359648958768, -0.020374883506622254, -0.012780414584034486, -0.027060947937467914, 0.010529438333398469, -0.011582694145062166, -0.013961263881613969, -0.009358448999278764, -0.01729040092451227, -0.006124533109867275, -0.021507108454247574, -0.01869714884967772, -0.03635442340392683, -0.0013929221046357501, -0.02377541770605046, -0.019865795510736473, -0.010668195815324882, -0.006333339747334053, 0.013467786508115875, -0.02177026999597995, 0.005597474355438716, 0.005111414891603263, -0.021614552459689143, -0.033693883192800395, 0.006590007130084288, -0.0053159360279018625, -0.014779907653567828, -0.006752030889137569, -0.0010359115286193439, -0.05583106769098078, -0.008705596307434438, -0.01783407177110582, -0.01784864059424866, -0.006557897480962673, 0.006508594364639789, 0.0011018046307718027, -0.022510816097044736, -0.034202308216433, -0.02181504108304638, -0.00015273505008371572, -0.0020583759480510095, 0.00592669448376553, -0.006571512883335716, -0.019479569877963513, -0.0038561782661727644, -0.010543541498928251, -0.016957941692976857, -0.0030982957758955917, -0.007816689370572522, 0.00010898697119557314, -0.015311206092078132, -0.04236019369441227, 0.00678725367564693, -0.010733480668627378, -0.013392486834688662, -0.0017880457251562274, -0.022271330827750107, 0.008326771783137246, 0.008186888674576388, -0.008641263507126297, 0.0020250337116165863, -0.012130948492559086, -0.0018160097699946001, -0.0016666535297892508, 0.007355466873383963, -0.002366633842406845, 0.006815614253675498, -0.00246372490698299, -0.021120056891707536, 0.008340622012421221, -0.013555704056512175, 0.015141052031901258, -0.02351094322461119, -0.025299030518679155, 0.0019046340267775204, 0.0033839751840314346, -0.015934287810535974, -0.03320120470077777, -0.024656721595104486, -0.020712080737899922, -0.040699450706507434, -0.008350104190325593, -0.030155120608523225, -0.04468527358951563, 0.0053732079037812595, -0.01902522218875705, -0.011694031087406777, -0.016929534160552685, -0.0012941813423732807, 0.0029378425753456253, -0.016420084830075336, -0.026300061378367712, -0.009927025629788411, -0.010726355845150807, -0.00281893235563045, -0.021520514225851443, -0.006512671849645336, 0.0072339722341863985, -0.005431227381353015, -0.008627243176740811, -0.027149360936811112, -0.02869365132443976, -0.008546980316101765, -0.006109515527675075, -0.017354485301466004, -0.0007849254922609085, -0.030796737726619384, -0.02483760103638419, 0.01742922970452597, -0.008101463482267448, -0.017667200096665975, -0.06439923753092681, -0.021723224303097637, -0.015978455640875876, -0.0010713530625496947, 0.01296445943350521, 0.019874941917688797, -0.019531306349482972, 0.0030340101737661614, -0.016588170417579, -0.0001420655207204814, -0.00679630369581632, -0.006236226485240161, -0.01689884473518941, -0.019432987275198075, -0.014626956728779606, -0.004758689193610452, -0.009114699545841317, -0.0261947819990415, -0.021544382139878568, 0.00994128183433605, -0.013143168905077699, -0.014102656715955161, -0.0036309876194338415, -0.021092302166726497, -0.01633261284054991, 0.0006508852738988775, -0.030302058620173808, -0.02605109990586732, -0.01493750137296816, -0.02851238941354268, -0.03889378108411991, -0.024025184701216215, -0.012674805666737189, -0.03676441459046776, -0.04404885934597553, -0.018782501407377406, -0.010798848388476263, -0.020808924964363085, -0.00461600892952301, -0.012723561836570729, 0.0030477630085073808, 0.005717320015925886, 0.013192466146442114, -0.02492140809992697, -0.008942985870575619, -0.012988812691146892, -0.005324109112877305, -0.005011278308131935, -0.004045308987776832, -0.027496866253478518, -0.027567249348791293, 0.016905815550953832, -0.00806601948032108, -0.008030620139611632, -0.013810330173378531, -0.023249532847487566, -0.012920015345891628, 0.0029501747206724656, -0.025543989839203397, -0.004575821718953586, -0.006333734585910092, -0.04674456513420182, -0.021492730303326116, 0.010139197170481347, 0.0076096828514603905, -0.028798916923952297, -0.012570624678336209, 0.004706269068714087, -0.009966389890980419, -0.05140612831586717, -0.01577020963095507, -0.01882606293009925, -0.02843395950509346, -0.006150593480330916, -0.015343949616946686, -0.009243443655140841, -0.0357201900549948, -0.032418312080972574, -0.015016072840263144, -3.160939218376544e-05, -0.00365833996872261, -0.009807664041068923, 0.0036391727076976717, 0.00872095052537766, -0.006265168735234139, -0.004949863642079307, 0.0048242866791796705, -0.014637036755825377, -0.03456583531515587, -0.0352701389898714, -0.04515991830163034, -0.02780066814985003, -0.025648590569384618, -0.00655315526137221, -0.0020215024860866064, -0.007697044174542384, -0.015348100575484238, -0.0013050379028292129, 0.008143751526642571, -0.022747573984708842, 0.00475364274731722, -0.014003695862816641, -0.027232437727616996, -0.019714012441495794, -0.018115185717462186, -0.0007171841459097235, -0.024239203577329595, -0.011448042744234482, -0.018209891684692902, 0.0020372895386428155, -0.0069269974648822055, -0.018741465373367776, 0.00027112977188936314, -0.015151310556294748, -0.02345224504262954, -0.04011097615631699, -0.009140157338717403, -0.00683590747532533, -0.025671706565325428, -0.005102094191685843, -0.004288554977647223, -0.001032425504320848, -0.02750629717883599, -0.01512224547348449, 0.0065239870059207805, -0.010221612426045641, 0.006459826428508792, -0.010294145002718278, -0.008993969316771856, 0.0002918985342197589, 0.012387550882845611, -0.02195524876396971, -0.03963514863403335, -0.01681346075229545, -0.012334447334418283, -0.012650138653920145, -0.023824819351282316, -0.04072049248516849, -0.004003680363953865, -0.011967703042234238, -0.0018247227034767716, -0.002322574728186315, -0.011797477259034947, 0.0032020839394006377, 0.006909125701674354, 0.008208019013923448, -0.009251017801203546, -0.0010071528639670065, -0.015274249710303832, -0.011359636897310824, -0.01449345505940301, -0.014965880646700225, -0.0016030150295936506, -0.002181612815938784, -0.03891382710162541, -0.005106237340071503, -0.010212096166582476, -0.031683193120215496, -0.003165450004835624, -0.005493402304010742, -0.027208439672349503, -0.00623589163275994, -0.0004014144290582111, -0.021340383415620067, 0.0035344536457890376, -0.03474703023450539, -0.034897383900291026, 0.002499115057651106, -0.01776087808755292, -0.008436010567910924, 0.007814042835214025, 0.006770063513497821, -0.012707214435009195, -0.05223730224230465, 0.003094652672546025, -0.0015169710123416429, 0.011210381533671038, 0.012437891956361644, 0.0035944739200303387, -0.02508496596527863, -0.03211973579851122, -0.002479649737434261, -0.010786196741996639, -0.003971496787332814, 0.006425804548090261, -0.01773937833698384, -0.0036965736891712855, -0.011274733377404412, -0.001377964538415448, -0.017673904206154313, -0.0077753179380090015, -0.004258793655793939, -0.018531495207550354, 0.011034369945145393, -0.027308377473641576, -0.003938358717909275, -0.02583458652084019, -0.026907385777468373, -0.018297984474673994, -0.01458195623834543, 0.01372277639497451, -0.008288078039214369, -0.053987221840013706, -0.0021654765131034486, -0.009260841236163412, -0.015060727755988906, -0.015678005168668466, -0.012268328021952166, -0.03478425499797199, -0.010788262608229791, -0.010765864688106558, -0.003672323351642219, -0.02731576058016677, -0.02435616288646147, -0.0196190626706964, -0.0047095244878745685, -0.024641848423037998, -0.0188096438256781, -0.01552255339178098, 0.008393875846987878, -0.013987039863819651, 0.010405081383416712, -0.023691085216462203, -0.01057208461624232, -0.018168032456434346, -0.018279465205139254, -0.012269872248327155, -0.02294317390009481, 0.0011227480649604817, 0.00823525099140978, -0.0012377704310637104, 0.003075336691670024, -0.04307150969732107, 0.004302269753576969, -0.0008608990845753506, 0.005725231869247034, 0.008798211306491463, -0.012799716466824079, -2.2772232593209732e-05, -0.027787187349704378, -0.007135874517524002, -0.036244637496238755, -0.020283371508669847, -0.017018733340834407, -0.02909949973253921, 0.002828681675288652, -0.015667328335998664, -0.008007841224874522, -0.005594722470299465, -0.043215296232057804, 0.015343951490261413, -0.01693002417092622, -0.028953845895772407, -0.013812145396982478, -0.021856446088516893, -0.007321279039792072, -0.03724208368136628, -0.013313820082946034, 0.0008278865965617015, -0.004581843576010609, -0.004608550773377679, -0.01644002231789892, -0.01006096938075089, -0.00964104442937615, 0.004504429171542698, -0.035113502591020085, 0.004724530379190202, -0.021205661208297907, -0.0007775943798305003, -0.018820704498548786, -0.009072513885042795, -0.04686037256543607, 0.002229321506427823, 0.008492525825307065, -0.020259682104625414, -0.0032368366478314784, -0.0032531482684465818, 0.006789750585382915, -0.016525817592587075, -0.02098353399354845, -0.020831793173059124, 0.00017077384735547034, 0.00896491054211654, -0.02470057617192763, 0.006285398384069723, -0.0071325301363061476, -0.015546099384672498, -0.011368715832690593, -0.016695386795894367, -0.015860608043507356, -0.02771801853099829, -0.02174154989168683, -0.0024138809521133072, -0.023019504141861366, -0.018473341960054238, -0.017465723552237616, -0.011426517674654428, -0.023221809973397494, -0.03633614315012541, 0.0020781820331061463, -0.0033984284642112326, -0.015856293448858402, -0.026007672137858033, -0.024183789512600085, -0.020856494999395247, -0.02255598189687434, -0.020668158449723462, -0.0045349787353369175, 0.0034704069505661177, -0.013895322845285519, -0.005419965528315921, 0.00018984899132540683, 0.0028897561184099184, -0.0031884421662972465, -0.021251337984577325, -0.02498792767258367, 0.016479321735052496, -0.010594858363499621, 0.01503274395376411, -0.004998039903839963, -0.010210659516045762, -0.003715998556199725, -0.018710628729614244, 0.012785888675554876, -0.006368076876493194, 0.007901854824199112, -0.007563195426552754, 0.000572975122047267, -0.005950870981350735, -0.0042918572659544, -0.026270479290008458, -0.005649885167258032, -0.04581852671781693, 0.010498454557707529, -0.010036087832256799, -0.016318043018302742, 0.00412866919111294, -0.020088482028889403, -0.0014551988943644792, -0.030626428411763582, -0.0074475153092212875, -0.007335260852883876, -0.016092203719388624, -0.030266352578171195, -0.03565512273371956, -0.016774880698712098, 0.0009049012929205508, 0.002760532879644303, -0.00931996363148857, 0.003243133556796185, -0.005214880602394197, -0.026420909774414945, -0.008610660876424151, 0.007150935022183168, -0.010238811299180712, -0.002404757957445382, -0.01067910818582132, -0.030468761821104865, -0.014665964141804516, 0.012626627664365378, -0.0019220308270981173, -0.020674903259245363, -0.05391161369480282, 0.0013567351513483316, -0.007044961830942513, -0.02083048178762714, -0.007245214541241826, -0.011477877653154906, -0.024277845735210444, -0.049075873904903866, -0.026446027772443523, -0.01648732963469808, -0.02681295036417734, -0.02708689622585006, -0.017646895828547022, 0.0031721018896492337, -0.006657909162810758, -0.003063831668743992, -0.019830378163937627, -0.03193390762433165, -0.00903206399179541, -0.013218726410589632, -0.013234928543369058, -0.03135255725614508, -0.024455844580170468, -0.004656718679453983, -0.011362092492242098, 0.003384746157951829, 0.0005166821874413517, 0.007612176656397246, 0.0023080408578944715, 0.003290878540763233, -0.007343924836669465, -0.013548552017098096, -0.01028434635731185, 0.0011891881450124047, -0.015502037898837407, -0.018780287769042038, -0.005717620502221103, -0.011962495427258859, -0.01485014614288108, 0.003377912181163069, 0.005217259076527473, -0.023261115520088714, -0.011725707703115915, 0.0003581846299570997, -0.007201025871641003, -0.00862928855674671, -0.026902685615806252, -0.02592988021405786, -0.007327557810041643, -0.01841659298387479, -0.011492189058471271, -0.0028158088276221964, -0.010924996830173273, -0.00899788947055193, -0.031186679308726983, -0.008184610961393689, -0.001980690666192787, -0.01209636868403816, 0.010683658122714573, -0.017570468827413255, -0.011447892120719352, -0.007880414511191607, -0.0012074018119969986, -0.03200321122892784, -0.0033365058502883437, -0.003937249670589896, -0.0002962994762279329, -0.03172374654552271, -0.030217678081627723, -0.01128122767682327, -0.013461913826004654, -0.026955184573218376, -0.022143547406029203, 0.01102365975547896, -0.02144988014751104, 0.0007428761193184709, -0.01508828710435136, -0.006227985518372702, 0.0063248083035167815, -0.019074102467320345, -0.01759667223385003, -0.028576418959119228, -0.0011627898688632004, -0.016898068428592868, -0.012899450262319942, -0.020201618676629497, -0.014362100064497552, -0.02064107937735143, -0.01590727278093378, -0.009178379506778354, -0.003304833287384704, -0.012639612340293245, -0.02210568663211067, 0.0016617096224143112, -0.028841530921692934, -0.009954503683154987, -0.010152915352487664, -0.021284067301083502, -0.021574967370616177, -0.01382557653550241, -0.009533394035952375, 0.016382805847748875, -0.015390069487315788, -0.010865980667566454, 0.0064055530133056695, -0.025838366034956848, -0.03429716365311286, -0.007184914863862688, 0.0038917588656438747, -0.0126682860944363, -0.009515282486911258, -0.01078246139820068, -0.011836654032297488, -0.02570748185202837, -0.017543486602501487, -0.0015432107596070178, -0.018826207295507654, -0.013661066480847283, -0.022473100380821585, -0.011853142457398168, 0.0030443124418900037, -0.009328050767828161, -0.006880108056594203, -0.008516132698035696, -0.02556405939846791, -0.01655749848547019, -0.01569718125476577, -0.005250953098483087, 0.005561908010049811, 0.0026451403918468686, 0.0026556500722263437, -0.021903658869178354, -0.007589200948505383, -0.023918802676618098, -0.018862184776392434, 0.006562198636328745, -0.011371332592792374, -0.024123985035941566, -0.014828557678429912, -0.012102460109818374, -0.02129774322708157, -0.016608250875683978, -0.010598309449044045, -0.009895504342104974, -0.01129747515396887, -0.012906947147141022, -0.016997363761041443, -0.008166781813722048, -0.004891293744372971, -0.029355381827760672, -0.011649342272547268, -0.00833066118839007, -0.013489680172415447, -0.03670539134737551, -0.019107164890178153, 0.01128755746758666, -0.018703055373922826, -0.014988832007031005, -0.025333833028873837, -0.007562426808529214, -0.015397219519265579, -0.01894167542733225, 0.014348188993716022, -0.029051071348173238, -0.008457090415509181, -0.0338385272492888, -0.008299612707346167, 0.0015602034607417232, -0.02575814061362668, -0.02616339418532738, 0.0022636875058053194, -0.011674086532610097, -0.020615303835958797, -0.04473151262704468, -0.014084327203007953, 0.003967006241080354, -0.012356636845585885, -0.01213944626290142, -0.00454872019587929, -0.0042512163118988795, -0.017703945223901743, -0.022923160857597734, 0.007912349291255465, 0.006260535489247102, -0.011234396219747768, -0.003774091171374747, -0.01959481467461445, -0.016256928219189247, -0.042184656088926825, -0.014719375925280224, -0.01975132045883635, -0.001609177794843885, -0.00457229503568262, -0.020018675679261782, -0.014525141474599731, -0.014133363842781702, -0.00705623304271318, 0.012810500276022308, -0.02441879523502471, -0.043069217807501205, -0.006924438063166849, -0.0006360757565683664, -0.020727894267086815, -0.005516290411217886, -0.002768989458177111, -0.016155783453221658, -0.01961577205431034, -0.01912419971845832, 0.0025340423566904994, -0.012230174353096767, -0.01096654020922934, -0.034377073420627555, -0.017848028809983128, -0.009761502452613256, -0.01006392993495067, -0.05706492474144945, -0.003525141774780664, -0.0016443739776604738, -0.03167298641512413, 0.015595941702898767, -0.025608381536679024, -0.009414501629628312, -0.035710322783914124, -0.025831448809905337, -0.010901218539189655, -0.01560756721995436, -0.02176657464734323, 0.006203010280569675, 0.015853203253934838, -0.0043225178652413964, 0.00357240201441916, -0.005552091887531719, -0.00266966455848841, -0.015694566883678717, -0.014375118332972987, -0.03366213421443999, -0.013333118070048364, 0.0003277568784845333, -3.047049601960937e-05, -0.01557018263214472, -0.018203918062359943, -0.004598197086493259, -0.007880943311733404, 0.002056885469087899, -0.025160742415872055, -0.0036178983232698853, -0.028948404761104, -0.0002219005403966307, -0.02221416694583916, -0.019450876200416904, 4.5493459978158494e-05, 0.01388081327122431, 0.004809767902213832, -0.006276002815660902, -0.003682884259897741, -0.018868572169000085, -0.006999649896235559, -0.03360730435575287, -0.037663189052698595, -0.013054664464108768, -0.00972344948522524, -0.0006710923939586734, -0.010774540171967465, -0.011960862465313853, -0.029487749054565863, -0.020859662177808498, -0.006777685856620273, -0.020195516370784566, -0.004971791299520198, -0.00488808537250019, 0.0001184019818954095, 0.0006262099853200099, -0.01731303083968095, -0.003236354071451835, -0.007677823314498689, -0.021238195919226044, -0.0035576327250239305, -0.008202123951706546, -0.0052199356170980985, 0.003683225217126735, -0.01881920824496646, -0.01603963948963122, -0.014009181352417653, -0.027098255945819875, -0.0020214434634259005, 0.008517259747046415, -0.004732970113825572, -0.004198401531720698, 0.013268096739897074, 0.006006921149298911, -0.008098179345285072, -0.01801949712983956, -0.02251358894571766, -0.0021132201369764275, -0.00838964146880175, -0.009239867624540975, -0.02991222366135608, 0.00827839283713177, -0.015156058510779788, -0.0005717826925548897, 0.004868666017742888, -0.012980752777457073, -0.005169120804034574, -0.011653418490530148, -0.035726652894957245, -0.024517906082218568, -0.013056553381741675, -0.020667121997595028, -0.016764954186802697, -0.014891525727424826, -0.005272844344139672, -0.023833010483796806, -0.010039964206200335, -0.006258035662837444, -0.010334666564674622, -0.021931774030801553, -0.030001716065185455, -0.0037956552750919286, -0.02095153245665829, 0.004044518933901231, -0.002324828598373076, -0.015339397240255856, -0.01679480866630101, -0.00810733576614131, -0.03564637403303607, 0.0016333107078403096, -0.004411533569669141, -0.008548456929228279, 0.002377400208865566, -0.03517245870524332, -0.005662051001541584, -0.0260372178916203, -0.0016710159397080556, -0.026759110598286468, 0.0026413398299815114, -0.052037463735447244, -0.028446470510864557, -0.008780956899586473, -0.018936197988564423, -0.0445872186205284, -0.013541470580232475, -0.029164642477940813, -0.02031388216198218, -0.03145717810042843, -0.005824803563659315, -0.026434133391828596, 0.003685373162485492, -0.013653676408382385, -0.021570482893269183, -0.007624049336056938, -0.003559664601193582, -0.0040639182251262715, -0.010968801239074158, -0.00972159863403238, -0.01168571311256325, -0.012767428819461548, -0.042789714554232035, -0.033497008851806995, -0.02638579980903018, -0.0077732487119419455, -0.014342563122620522, 0.007477204771494521, -0.032361083044241575, -0.014998196919813269, -0.0021041726506428955, -0.024127887607919195, -0.007621174090813643, -0.02902424914042189, -0.01156048320091001, -0.012019794407464894, -0.024894214664116048, -0.007196932290027469, -0.012672524705085016, -0.021598702922955605, -0.012155736488143236, -0.043076971422004, -0.005100937114069675, -0.014342522607769779, -0.00872237625078408, 0.0012934124114924968, 0.004602056017842068, -0.00342697005593046, -0.03533521122231294, -0.010163757920221663, -0.028755818206094505, -0.012571905895509307, -0.0230080439707546, -0.01682002392928789, -0.00608411380455047, -0.010797334922996837, -0.020339300538225382, -0.013759263471787696, -0.01958527157114496, -0.01658305812631678, -0.009544541076448892, -0.04525983428165063, -0.0019674165513196376, -0.008875797949948824, -0.015986709716177636, -0.03410201932423439, -0.007135919157045001, 0.00936148855407371, -0.011295557707656858, -9.790648631970557e-05, -0.004455917603849465, -0.02519611786155352, -0.02792483896984, 0.0019502473709806512, -0.004185650514801147, -0.01045655755280801, -0.012315905555350988, -0.02352378542113762, -0.031195610709947776, 0.0052374434078949535, -0.016712785258264463, 0.007584822739284844, 0.0020267544329961406, -0.018648876744941124, -0.034383818071043945, -0.02886922053404538, -0.015377183104471327, -0.010719228322793619, -0.03522592440402046, 0.006374528036433344, -0.01785810898144716, -0.0009879287044187801, -0.014007979404600406, -0.004044476477379722, 0.009287226402232257, -0.006619503558017316, -0.007859800238010393, -7.547216562233919e-07, 0.00027854684273389245, -0.023101528767652594, -0.0038288795564307343, -0.013718165243377451, -0.017110091459563125, -0.0060329931809711395, -0.012884150400987964, -0.01147790177975665, -0.010684736876738533, -0.016594559865160858, -0.01287393953070436, -0.00047462275852991776, 0.010974674768655945, -0.006450597701344379, 0.0061467852060564454, -0.011609246545752145, -0.005004496019940296, -0.006581352073845473, -0.009418513768477105, -0.017682347003145093, 0.0038438748524186927, -0.021384139358637734, -0.0404113223289792, -0.027335284899414447, -0.011269223182037122, -0.0007074818441689631, -0.020278192827863792, -0.034812727997285596, 0.014980780547351179, -0.022822858399623388, -0.0202821024160906, 0.002018694915392561, 0.00853903039648775, -0.01846570923470444, -0.010429644820556474, 0.006205844592247624, 0.0023714819588036055, -0.0028598165153489673, -0.005742854016308013, 0.0057875711987434115, 0.002598591334960153, 0.01051646436016853, -0.04629972176061065, -0.028453223051716, -0.01970239489992549, 0.007678150176407813, 0.001887317288117444, 0.005241672501415965, -0.017250690496741723, -0.003697487834830163, -0.013903905486113342, -0.0027069122806266032, -0.018596294743340357, -0.01576414078875577, -0.0181782402298237, 0.004541000751871457, -0.020879519338447967, -0.02496640967355601, -0.016495827185974997, -0.02471734157434711, -0.000262739382693976, -0.026801002334974195, 0.004290054361652664, -0.03256909396161148, -0.013169645529181153, -0.023502047201201606, -0.010055808137793054, -0.007556868352939293, -0.00641538287965169, 0.006566006966806727, -0.003493280352565346, -0.015168732392519606, 0.0007425303596090351, 0.002258017695173049, -0.02139263868751951, -0.018107831076620094, -0.01123437611062738, -0.006269805559048457, -0.0155893094646551, -0.01006585458814204, -0.011302950593420855, -0.004035001977923369, -0.03051868597275234, -0.015920373800892066, -0.020805821793913493, -0.011399802016031, -0.012534280840658246, -0.007931186219822299, -0.03009575399316217, -0.02299820667663631, -0.00454277026896647, -0.019312733367754947, -0.022769511284006467, -0.007828398580723135, 0.012938352712007448, -0.005331595786682085, -0.010521808673012513, -0.02586086211786702, -0.01963220131232076, -0.00507061503527619, -0.004541000424498491, -0.025422006153079702, -0.009631486846835041, -0.01702119408707998, -0.005047199710345312, -0.0004633858024722067, 0.010610447492345286, -0.006435729339075538, -0.03308255079980651, 0.0019807259694278423, -0.007096955894779041, 0.010720686605323226, -0.017263775157557334, -0.0168323227167321, -0.02695620235532526, 0.00409270114733618, -0.013102405596890625, 0.00226148444669787, -0.008077262338339814, -0.01773700103382893, 0.004185654275408645, -0.021058661333073977, -0.023820975806354054, -0.011690312785834446, -0.00936032329360876, -0.007617730658858337, -0.018261253858741756, -0.008772020875858974, -0.011447941571629245, -0.0034511188537946474, 0.004123998002640089, -0.014624793773569325, -0.002780498958250112, -0.007670515555978083, -0.023486974389154834, -0.024760698040231333, -0.0022402170839938983, -0.0017511162542642128, -0.005056277844187451, -0.019698915333690403, -0.018900781983436275, -0.011736385966449957, -0.00967801182128774, -0.0024538341420844312, -0.03720526476403076, 0.003761082011021047, -0.006482690376575485, -0.0053853440995016025, -0.0010381837413991695, -0.009327858688307268, -0.01387818595275011, 0.011193251239742737, -0.009467368622477214, 0.002308752111101805, -0.03028867748491052, -0.022161920578064282, -0.020540412960310533, -0.001264333373283191, -0.011640265602866165, 0.0008766851256221033, 0.00444117357271696, -0.030264525828691734, -0.013665601455041696, -0.015699395029033218, -0.014324486658217605, -0.018818971183590324, -0.03888343093026493, -0.01756724131734557, -0.03351992826349288, -0.025195669745707763, -0.009875649354274435, -0.008287544193307152, -0.019984990582354558, 0.012642681114250617, -0.007059753028884829, -0.01692779475167852, -0.00843395685983435, -0.007469588262034247, -0.0007506058896569578, 0.005702243745375778, -0.026884926099223216, -0.005820737743116692, -0.027838926972568605, -0.021225240471220982, -0.024875446411510044, -0.005170726067420787, -0.01468263478565018, -0.019548996791091823, -0.006340211832123936, -0.025091341470292586, -0.02546339074486851, -0.007817476762243526, -0.03624964707072107, 0.004371047523128326, -0.00823250382110632, -0.0001390519140790497, -0.01620009922381684, -0.008040422847240993, -0.01044363583730339, -0.0022952661051638228, -0.016872872672169238, -0.02921405747755054, 0.016456021426792894, -0.027655421045145928, -0.007383731755923437, 0.0007252768149843935, -0.008206462557984674, -0.012332948791236469, -0.016311970384636807, 0.0032019230341163657, 0.014640923610209806, 0.0016082446619814146, -0.041027206324109916, -0.01115904893079922, -0.0053569460364182595, -0.02388359175651384, -0.020507241632848196, -0.015160098866944798, -0.009959473569543612, -0.006771859098629333, -0.025842033473607817, 0.0039356401412400065, -0.011300475842969098, -0.004750101718442996, -0.01578418061407274, -0.015897690799673323, -0.013767970224318362, 0.014564494008779648, -0.0019941801584447516, -0.011374119349788445, -0.02437289323480237, -0.010957369410633532, 0.00481347945661254, -0.007315385531633379, -0.005183221936924526, -0.0013539739531354504, -0.024145098091696217, -0.011698240454238559, -0.002654169135655006, -0.013746584257443292, 0.015025340888120162, -0.021571265337553738, -0.02104548985207814, 0.0014266095498048773, -0.01971440427132515, -0.029616327160491993, -0.005101168902759386, -0.004621590609969373, 0.008288042058032193, -0.014951786563611735, -0.02426668880560547, -0.00725981365567299, -0.04440272532816742, -0.0028001095102095177, -0.021644031079346846, 0.0026128759506753685, -0.005851569508151236, -0.01357168515786188, -0.020005617778921313, 0.0012147643365241466, -0.036357922908203735, 0.011980230174806417, -0.010269514250110868, -0.010738551250648008, -0.008073171030370823, -0.01023394447129295, -0.014358120928155763, -0.016164132774609177, -0.00434658909750927, 0.008908407929278132, -0.004216452476099756, -0.041326247244584974, -0.0030709224624641, -0.0036445894523148416, 0.002675901536307448, 0.003842153861194829, -0.018353556785261096, -0.0027374411129652697, -0.022248189166881528, -0.022085129472555965, 0.003687197270113106, -0.027896123685736837, -0.016641089703504506, -0.04383538813949187, -0.0031562720426257184, 0.006286070600614788, -0.008024987936717163, -0.01024215199748902, -0.008905840453917591, 0.0016433498695625382, 0.014871288067341615, -0.016917338565510174, -0.03228210726643214, 0.010517183521938216, -0.018200407139015686, -0.026823755836165544, -0.015319334288306136, 0.0065191696141414365, -0.011204741328948465, -0.011763163894024493, -0.02001615795875943, -0.01279811546229027, -0.03291339077096032, -0.009602711760562285, -0.005393903997846015, -0.018809410379453804, -0.009559001319700734, -0.010520159159184327, -0.02027624195419444, -0.008513546365081361, 0.0051912189290601735, -0.024914159463488936, -0.0032381636193656027, -0.0027174540070752816, -0.04417067172199051, -0.00911117944705503, -0.015277982086639177, -0.012695857084322502, -0.02450542285951387, -0.02167057309762901, -0.018290097341244465, -0.024326382334119514, -0.02033456872918746, -0.023596479502397877, -0.01953159723439132, -0.022010825663436755, -0.026332183441770768, 0.00628023122971162, 0.013583739524511305, -0.03405405956860545, -0.020703948975508844, 0.0002520581240786147, -0.0053542612378262595, -0.008063452958053809, -0.0019350711812047752, -0.01597460657320463, -0.021909642502583464, -0.0027075459359102113, -0.006518842156145819, -0.0035422622570550574, -0.010875913857792512, -0.023289827949196484, -0.0027547171247618964, 0.0005201545457580429, -0.008724408827970155, -0.041560006466756054, -0.021454438971873553, -0.025604826413958254, -0.008810906176015888, -0.04069796948996876, -0.018537798285757988, -0.015504478587066734, -0.011485622485668352, -0.01989508991534144, -0.003150688115497048, -0.024424559433113866, -0.0037069095470283427, -0.032324825460775246, -0.013693866558378922, -0.01666168786310122, -0.0022340293737992054, -0.0060081451915516984, -0.030020753752294495, -0.009568409609679316, -0.01663927379010846, -0.03279810657739356, -0.01494132653828529, -0.01893249583867434, -0.019106464746774323, -0.017525906638626913, -0.017289396195212522, -0.002700021168799422, -0.0038942604010813804, -0.02903629754153221, -0.0288010807568656, -0.025382249764856847, -0.02075281824069097, -0.012580495266901717, -0.021469815279964496, -0.007006113326278358, -0.007896597748928192, -0.038854109949217154, -0.018837005300974343, 0.005230445565188048, -0.008910577667550833, -0.01885362614826396, -0.022354333967142558, -0.021458181512827572, -0.015199002068030569, -0.03215677776827645, -0.004310017464502596, -0.009196701656209895, -0.02195962363442381, -0.03401377010703764, -0.02378372863097052, -0.007526671025542353, -0.004754976414065192, -0.0029108283563518077, -0.01535208825094143, -0.04117000300727566, -0.018516357687973873, -0.004593892713523457, -0.011940325441295757, -0.016772653696256353, -0.0010973833153044084, 0.00745987988458482, -0.016235998008701766, -0.004249413686334908, 0.009423612704886083, -0.034622910463667465, -0.001360632724247343, -0.023749037180376225, -0.00622145836601542, -0.017369933729628785, -0.007124874886912505, 0.0063698468719412685, -0.02404605806747754, -0.0027381133335554007, -0.014399640014736188, -0.01861790705006678, -0.019292019919315236, -0.012267088861475742, 0.00046316641099780495, -0.017631596258697226, -0.012512210746976567, -0.008835797146354508, 0.013405840408580892, -0.0026875581529965586, -0.0006770228818235265, -0.012928485245107625, -0.006619215860762593, -0.03384436294470507, -0.018521386826630733, -0.004278952584453664, -0.0014293834193602295, -0.030228472654561637, -0.01903127390464748, -0.013568252266741408, -0.01611795545330016, -0.019590965918490466, -0.008525012412439505, 0.0013459026245605551, -0.024113864665528645, 0.007251296159939303, -0.04021266526690313, -0.012915275478543988, -0.03356977720052988, -0.010879223418475524, 0.015105277344540018, -0.0021044913254909437, -0.034610784721487195, -0.002480553444042318, -0.016671744059473352, -0.02759273519047179, -0.018931268216929847, -0.026467653270468967, -0.01789515521290348, -0.011688442361255078, -0.0019812639179228974, -0.008058375125740126, -0.012832058470628446, -0.03909628707899017, 0.005332904895249422, -0.007223968421148139, -0.004463712709553369, -0.024806506162069395, -0.022443551121205656, -0.005460206937866121, -0.031330658042264876, -0.03043657884635499, 0.01509833812403271, 0.0004289321392863517, -0.021315561904838853, -0.019089600035007426, -0.01873147935140951, -0.03493483361271804, -0.010506100999946229, 0.01437000722581054, -0.02830428194277859, 0.0014609721504888996, -0.008075420263826441, -0.013719872025183122, 0.007101604344110735, -0.03527184238248099, -0.02632812787004112, -0.022329156811842114, -0.004806664576092176, 0.002749461877969517, -0.029873902927967676, -0.020545883878563043, -0.007926190474225353, -0.022047923732945805, -0.009944866398157145, -0.01687769936792752, -0.009724889736918827, -0.008711338216458916, -0.009914749918595214, -0.0024857071797070264, 0.008753515110251745, 0.0023623564050134524, -0.028957038539717598, -0.0033309436361022354, -0.022513302477008793, -0.024607385302161906, -0.030105729160649748, -0.008349074616118848, 0.0045498490141843, -0.007277747628750836, -0.007334889040744714, -0.008583581246191883, -0.012181844934303656, -0.0033222583792368717, -0.004986595268203211, 0.010192759503563279, -0.03604962655058906, -0.022397255374261828, -0.014955508731365634, -0.008785357439962331, 0.007006249314498465, -0.011034668506150017, -0.023460066994942144, -0.028301164592714655, -0.015821576452728622, -0.031259966270114306, -0.019973489905808977, 0.011313485049479061, -0.028588192318695974, 0.011842662591193842, -0.002166021923341303, -0.006073290342744661, -0.02438196398469393, -0.036188299972858703, 0.004424547226090215, -0.014458611262432197, -0.015166020680414944, -0.00826728079001151, -0.007968229658477837, -0.007712982679525098, -0.0296247382335814, -0.016799480706254276, -0.025823530161628413, -0.0032401526205571313, -0.009529293687104399, 0.008582319311636291, -0.008536668849316989, 0.013429745555132473, -0.00025477128278093164, -0.006826163648814837, -0.02651499504186649, -0.02716624537588519, -0.0006214435793350953, -0.008199971761977468, -0.02171923878896557, 0.009746143193460248, 0.0018784511604392302, -0.020269217137730024, -0.01030952260086593, -0.011126033080393366, -0.00422899080594879, -0.012948790579636901, -0.003947784701508723, -0.010391686058557694, 0.00030347196850405433, -0.013430986614437394, -0.02469408908606849, 0.0017550218263572377, -0.0024274365805269926, -0.013120659799473348, 0.003976617158735838, -0.021362228861055575, -0.0208059615955321, -0.03577302378478996, 0.007648913275610383, -0.008416973097997868, -0.015639269551113548, -0.006864166606874949, -0.04251503550788716, -0.0010151396076861921, -0.023909384643593417, 0.00440083616235244, 0.009466993686482527, 0.005574487938921149, -0.023242491819286705, -0.009996995370930743, -0.003806880955461822, -0.013771488967906635, -0.00784265830773479, -0.02330993477606314, -0.01978173029551115, -0.0011156759743095662, 0.004297365732275996, -1.473155619214597e-05, -0.0344026183856565, -0.03625152443204709, -0.00708484294420458, -0.024479158907115064, -0.012107096511203662, -0.06026492588836147, -0.00865132457320423, -0.017098084293472787, -0.017934241822070797, 0.0016519317622916909, -0.01646771169852644, -0.014027832082319596, -0.005865234362251827, -0.03919375050789864, -0.00014683816148955666, -0.0022248735938082864, 1.9850295260729717e-06, -0.022735435264482382, -0.01864421110870739, -2.7392185992001278e-05, -0.0031307723981868684, -0.01828486090706289, -0.04285478432801333, -0.037572276051784426, -0.024218592182680267, -0.006325346988588055, 0.0011695851721391808, -0.0017747246791973703, -0.030013348021385087, -0.02788444427058253, -0.03618264610155176, -0.007152757856992468, -0.03181927428622719, -0.023982010174601207, 0.0015435445567811912, 0.0003860520410140918, -0.007746776225095944, -0.007894643493578768, -0.006619738303682058, -0.016633109814169236, 0.010812617456841879, 0.0062509793682755865, -0.01566557543006033, -0.012511753404195268, -0.005393533398339642, -0.01804322795658477, -0.000632497201717334, -0.010309135653206717, -0.022237026032755933, 0.016858747916066223, 0.0014395360776975254, -0.0033142232353299815, -0.019649256405736237, 0.003624942721668409, -0.012415696086823955, -0.013217394342767221, -0.003713956942032881, -0.022023728496949654, -0.013441276994157092, -0.004793919605799101, -0.01702336532518996, -0.003632392325777457, -0.03329877301142288, -0.026545331243906498, -0.03361708539796062, 0.0023289823404054966, -0.0192571837065082, 0.0030457024593787715, 0.003323089231051022, 0.006471392226106601, -0.02747945492547433, -0.004409568078922399, -0.006533463508909275, -0.008348138006450768, 0.013595572691665867, -0.02394124965326485, -0.016595171297674016, -0.02326928360077256, 0.005897719225372946, -0.01227120728030169, 0.008938346574177134, -0.008695360099612259, -0.016138949390901765, 0.0036136534424181708, -0.02523216349686834, 0.004555058170610274, -0.008413796719559722, -0.01652605706546151, -0.004777014193319226, 0.0034562022002369588, -0.0024620762306124975, -0.011137368194369696, 0.0013923693224990232, -0.010150985840201791, -0.019869867268274482, 0.0020425893908476715, -0.0006290845176696307, -0.02107102865848931, 0.007003013443238069, -0.03187900279900948, 0.007815009735488425, -0.0217419262063163, 0.016525838084337698, -0.00058366750441042, -0.0325779549444376, -0.0025603541358809874, -0.028310295124300344, -0.02347844396452143, -0.024070804881873867, 0.0014805160223392733, -0.00761634950437724, -0.019067134126819985, -0.011780417210677685, 0.003317987915693574, -0.014967440297397279, -0.027916454904158033, 0.0009480522229950257, -0.013918131172976302, -0.00955071089756705, -0.003994520081407533, -0.014568169473732313, 0.0015318608833386448, -0.023624146826362828, -0.008198448965269277, -0.020756643435154834, -0.00017947349275451656, 0.013001829062121339, 0.002888667479874687, 0.00066861519004394, -0.03557342349319418, 0.009001309151815113, -0.003194065033383742, 0.007270389866222637, -0.018818743333755657, -0.016340710629779265, -0.016881904048962833, 0.007732495074463065, -0.0014978464885399617, -0.028077825475568863, -0.015299428048258347, -0.01979455178278736, -0.008834802284716203, -0.024719212936720966, -0.007522699859111424, 0.010901795644732924, -0.01642189393909002, -0.018482409094246952, -0.007313344331687799, -0.012341773974713513, -0.0035867111351534703, -0.002139715023308624, -0.011440321448516548, -0.01753947754197401, -0.0010723873960430212, -0.009237605805008304, 0.002164174235513366, -0.008891642859964449, 0.004298356323410529, -0.00201594922795149, -0.027592828593016754, -0.005039607806209851, -0.007477159053706273, -0.010153368440642197, 0.014036539971332807, -0.01028537865560358, -0.01689642870615688, -0.009998163255139159, -0.02594019139621925, 0.00399785080543172, -0.00020748817916609936, -0.016048012735280018, -0.026655585448589356, -0.02486708860212456, 0.006459060862110471, -0.02120091386990931, -0.005770293644113707, 0.004870074773646588, -0.021211872804365664, -0.00522309400850953, -0.0008174418125899551, -0.012458493115353875, -0.0038865636777950974, -0.012557664989810556, -0.026163170926632024, -0.007160758743060608, -0.015830083492404935, -0.02250088225028686, -0.03009480090212208, -0.00901249259877709, -0.02492693843713728, -0.014835408569070044, -0.015358605637898363, -0.005594459653359895, -0.015440429040460444, 0.0060509227970446606, -0.006685973094198716, -0.005351361334518888, -0.017558310927387823, -0.03743811116294747, -0.021454211992938246, -0.002920736286697709, -0.018957288411348305, -0.029408666014117537, -0.030616595513020508, -0.03057295757796936, -0.024724694621021726, -0.013277500140739347, 0.0026588361746088737, -0.03362939775787637, -0.016741417164075654, -0.0005574168362462446, 0.003474158779427316, -0.004091384814168133, -0.004625918146266263, 0.011214517200357883, 0.0016468872990818364, -0.019603398863910614, -0.006229051547976164, -0.01847597158999139, -0.02836465600810781, -0.021681840382171483, -0.01210227325495978, 0.005634625751101065, -0.010403025773862973, -0.011560307468217835, -0.034856232963982076, -0.02487598456306341, -0.01423321228889697, -0.02174450848850095, -0.004730974628607284, -0.017652404570943928, -0.02762615580645375, 0.013314088930568967, -0.02148839124541788, 0.019520422841274986, -0.01907690900966422, -0.02137450512910585, -0.011471157012040134, -0.0009283166177569783, 0.0038996013586078356, -0.047701918084333134, -0.0011805010047731298, -0.016739377876899976, -0.006928059307443952, 0.0037504051187836206, 0.0015075229252962353, -0.01823023781020434, 0.0021769257462484273, -0.008627484394941652, 0.004775454474249536, -0.0115687328505978, -0.04205052221764777, -0.01589183145655528, 0.0046926874779688035, -0.029669618628844097, 0.005345206025163939, -0.0052653901679804785, -0.00026814872540666205, -0.0366427245257777, -0.018870918939851565, -0.001477718106221845, -0.026387997445268773, -0.014995606451970721, -0.002779462238413735, -0.01705592886270616, 0.0004089451797010652, 0.009456910784925728, -0.020335579144232927, -0.014441612576520407, -0.0076993833153821255, -0.0067570382048667355, -0.03517155167331142, -0.003196166744066067, -0.019503964399442172, 0.011695747509907591, -0.014121667402613664, -0.011707347968752549, -0.010961501827564994, -0.022084631439153207, -0.003488003044335555, -0.013381968030963096, -0.018942349710224182, -0.012126623027703141, -0.013277378811893472, 0.00619755338540139, -0.04191301559909782, -0.013004744968346284, -0.018317687698132752, 0.0017437977240780396, -0.020905123684337626, -0.02834063928389681, -0.00789747209733107, -0.01496929906147501, -0.027254526256305086, -0.027698731099376043, -0.030022323233357722, -0.06042820488625853, 0.002391913744961686, -0.016228568994197255 ] } } ], "bin_sensitivity": [ { "seed": 11, "bins": 5, "before": { "nll": 0.15970939605968382, "accuracy": 0.9583333333333334, "confidence": 0.9756299726754267, "ece": 0.019109432669800624, "bins": [ { "bin": 1, "count": 2, "confidence": 0.34289046189484873, "accuracy": 0.0 }, { "bin": 2, "count": 3, "confidence": 0.5578990670042329, "accuracy": 0.6666666666666666 }, { "bin": 3, "count": 10, "confidence": 0.7234086718068717, "accuracy": 0.6 }, { "bin": 4, "count": 345, "confidence": 0.990241232812413, "accuracy": 0.9768115942028985 } ] }, "after": { "nll": 0.14412838152046395, "accuracy": 0.9583333333333334, "confidence": 0.9535087222922699, "ece": 0.011300296490378694, "bins": [ { "bin": 1, "count": 2, "confidence": 0.3068881644682291, "accuracy": 0.0 }, { "bin": 2, "count": 5, "confidence": 0.5208618805122985, "accuracy": 0.8 }, { "bin": 3, "count": 18, "confidence": 0.6973248362189061, "accuracy": 0.6666666666666666 }, { "bin": 4, "count": 335, "confidence": 0.9775916634082954, "accuracy": 0.982089552238806 } ] } }, { "seed": 11, "bins": 10, "before": { "nll": 0.15970939605968382, "accuracy": 0.9583333333333334, "confidence": 0.9756299726754267, "ece": 0.019785490627386406, "bins": [ { "bin": 3, "count": 2, "confidence": 0.34289046189484873, "accuracy": 0.0 }, { "bin": 5, "count": 3, "confidence": 0.5578990670042329, "accuracy": 0.6666666666666666 }, { "bin": 6, "count": 3, "confidence": 0.6665815007599835, "accuracy": 0.6666666666666666 }, { "bin": 7, "count": 7, "confidence": 0.7477631736841096, "accuracy": 0.5714285714285714 }, { "bin": 8, "count": 8, "confidence": 0.8598206331693286, "accuracy": 0.875 }, { "bin": 9, "count": 337, "confidence": 0.9933372707861362, "accuracy": 0.9792284866468842 } ] }, "after": { "nll": 0.14412838152046395, "accuracy": 0.9583333333333334, "confidence": 0.9535087222922699, "ece": 0.01665789888272561, "bins": [ { "bin": 2, "count": 1, "confidence": 0.2635764243169535, "accuracy": 0.0 }, { "bin": 3, "count": 1, "confidence": 0.35019990461950473, "accuracy": 0.0 }, { "bin": 4, "count": 1, "confidence": 0.492679339063558, "accuracy": 1.0 }, { "bin": 5, "count": 4, "confidence": 0.5279075158744837, "accuracy": 0.75 }, { "bin": 6, "count": 10, "confidence": 0.6516215482562757, "accuracy": 0.5 }, { "bin": 7, "count": 8, "confidence": 0.754453946172194, "accuracy": 0.875 }, { "bin": 8, "count": 25, "confidence": 0.8572130163828384, "accuracy": 0.88 }, { "bin": 9, "count": 310, "confidence": 0.9872996188135742, "accuracy": 0.9903225806451613 } ] } }, { "seed": 11, "bins": 15, "before": { "nll": 0.15970939605968382, "accuracy": 0.9583333333333334, "confidence": 0.9756299726754267, "ece": 0.021206176375884088, "bins": [ { "bin": 4, "count": 1, "confidence": 0.3228541919475831, "accuracy": 0.0 }, { "bin": 5, "count": 1, "confidence": 0.3629267318421143, "accuracy": 0.0 }, { "bin": 8, "count": 3, "confidence": 0.5578990670042329, "accuracy": 0.6666666666666666 }, { "bin": 9, "count": 1, "confidence": 0.6225861329049945, "accuracy": 1.0 }, { "bin": 10, "count": 3, "confidence": 0.69635744433951, "accuracy": 0.6666666666666666 }, { "bin": 11, "count": 6, "confidence": 0.7537380420241989, "accuracy": 0.5 }, { "bin": 12, "count": 5, "confidence": 0.8420607167176215, "accuracy": 0.8 }, { "bin": 13, "count": 11, "confidence": 0.9085951389159753, "accuracy": 0.8181818181818182 }, { "bin": 14, "count": 329, "confidence": 0.9952230249502089, "accuracy": 0.9848024316109423 } ] }, "after": { "nll": 0.14412838152046395, "accuracy": 0.9583333333333334, "confidence": 0.9535087222922699, "ece": 0.018249305063047363, "bins": [ { "bin": 3, "count": 1, "confidence": 0.2635764243169535, "accuracy": 0.0 }, { "bin": 5, "count": 1, "confidence": 0.35019990461950473, "accuracy": 0.0 }, { "bin": 7, "count": 4, "confidence": 0.5062003460338178, "accuracy": 0.75 }, { "bin": 8, "count": 1, "confidence": 0.5795080184262218, "accuracy": 1.0 }, { "bin": 9, "count": 6, "confidence": 0.6337780991701131, "accuracy": 0.3333333333333333 }, { "bin": 10, "count": 6, "confidence": 0.6914824387328826, "accuracy": 0.8333333333333334 }, { "bin": 11, "count": 6, "confidence": 0.7667139707537225, "accuracy": 0.8333333333333334 }, { "bin": 12, "count": 14, "confidence": 0.8388363623044611, "accuracy": 0.9285714285714286 }, { "bin": 13, "count": 21, "confidence": 0.8953586751637714, "accuracy": 0.9047619047619048 }, { "bin": 14, "count": 300, "confidence": 0.9898232199702577, "accuracy": 0.99 } ] } }, { "seed": 11, "bins": 30, "before": { "nll": 0.15970939605968382, "accuracy": 0.9583333333333334, "confidence": 0.9756299726754267, "ece": 0.02701766397404052, "bins": [ { "bin": 9, "count": 1, "confidence": 0.3228541919475831, "accuracy": 0.0 }, { "bin": 10, "count": 1, "confidence": 0.3629267318421143, "accuracy": 0.0 }, { "bin": 16, "count": 2, "confidence": 0.5412021060112039, "accuracy": 0.5 }, { "bin": 17, "count": 1, "confidence": 0.5912929889902907, "accuracy": 1.0 }, { "bin": 18, "count": 1, "confidence": 0.6225861329049945, "accuracy": 1.0 }, { "bin": 20, "count": 2, "confidence": 0.6885791846874779, "accuracy": 0.5 }, { "bin": 21, "count": 1, "confidence": 0.711913963643574, "accuracy": 1.0 }, { "bin": 22, "count": 4, "confidence": 0.7438227843462917, "accuracy": 0.5 }, { "bin": 23, "count": 2, "confidence": 0.7735685573800134, "accuracy": 0.5 }, { "bin": 24, "count": 2, "confidence": 0.828080499472074, "accuracy": 1.0 }, { "bin": 25, "count": 3, "confidence": 0.8513808615479864, "accuracy": 0.6666666666666666 }, { "bin": 26, "count": 3, "confidence": 0.8894204939221737, "accuracy": 1.0 }, { "bin": 27, "count": 8, "confidence": 0.9157856307886509, "accuracy": 0.75 }, { "bin": 28, "count": 20, "confidence": 0.95394038653558, "accuracy": 0.95 }, { "bin": 29, "count": 309, "confidence": 0.9978950403815764, "accuracy": 0.9870550161812298 } ] }, "after": { "nll": 0.14412838152046395, "accuracy": 0.9583333333333334, "confidence": 0.9535087222922699, "ece": 0.023170418293590525, "bins": [ { "bin": 7, "count": 1, "confidence": 0.2635764243169535, "accuracy": 0.0 }, { "bin": 10, "count": 1, "confidence": 0.35019990461950473, "accuracy": 0.0 }, { "bin": 14, "count": 1, "confidence": 0.492679339063558, "accuracy": 1.0 }, { "bin": 15, "count": 3, "confidence": 0.5107073483572377, "accuracy": 0.6666666666666666 }, { "bin": 17, "count": 1, "confidence": 0.5795080184262218, "accuracy": 1.0 }, { "bin": 18, "count": 3, "confidence": 0.6202534470457935, "accuracy": 0.3333333333333333 }, { "bin": 19, "count": 3, "confidence": 0.6473027512944326, "accuracy": 0.3333333333333333 }, { "bin": 20, "count": 4, "confidence": 0.6783867218855195, "accuracy": 0.75 }, { "bin": 21, "count": 2, "confidence": 0.7176738724276086, "accuracy": 1.0 }, { "bin": 22, "count": 4, "confidence": 0.7584608325180272, "accuracy": 0.75 }, { "bin": 23, "count": 2, "confidence": 0.783220247225113, "accuracy": 1.0 }, { "bin": 24, "count": 4, "confidence": 0.815927734137541, "accuracy": 1.0 }, { "bin": 25, "count": 10, "confidence": 0.8479998135712293, "accuracy": 0.9 }, { "bin": 26, "count": 11, "confidence": 0.8806014852098643, "accuracy": 0.8181818181818182 }, { "bin": 27, "count": 10, "confidence": 0.9115915841130695, "accuracy": 1.0 }, { "bin": 28, "count": 26, "confidence": 0.953139433729237, "accuracy": 0.9615384615384616 }, { "bin": 29, "count": 274, "confidence": 0.9933041631902086, "accuracy": 0.9927007299270073 } ] } }, { "seed": 22, "bins": 5, "before": { "nll": 0.10809208384800954, "accuracy": 0.9666666666666667, "confidence": 0.9743925312832241, "ece": 0.008288102249318358, "bins": [ { "bin": 2, "count": 8, "confidence": 0.5433644751549982, "accuracy": 0.375 }, { "bin": 3, "count": 10, "confidence": 0.6898797226103056, "accuracy": 0.7 }, { "bin": 4, "count": 342, "confidence": 0.992794146884847, "accuracy": 0.9883040935672515 } ] }, "after": { "nll": 0.10774593556856667, "accuracy": 0.9666666666666667, "confidence": 0.9510981941735058, "ece": 0.018669946790388515, "bins": [ { "bin": 1, "count": 1, "confidence": 0.3784920212429425, "accuracy": 0.0 }, { "bin": 2, "count": 14, "confidence": 0.5128409537327172, "accuracy": 0.5 }, { "bin": 3, "count": 16, "confidence": 0.7288095026435738, "accuracy": 0.875 }, { "bin": 4, "count": 329, "confidence": 0.9822982750354528, "accuracy": 0.993920972644377 } ] } }, { "seed": 22, "bins": 10, "before": { "nll": 0.10809208384800954, "accuracy": 0.9666666666666667, "confidence": 0.9743925312832241, "ece": 0.01486792090871082, "bins": [ { "bin": 4, "count": 1, "confidence": 0.46949159973744253, "accuracy": 0.0 }, { "bin": 5, "count": 7, "confidence": 0.5539177430717919, "accuracy": 0.42857142857142855 }, { "bin": 6, "count": 5, "confidence": 0.632314625161008, "accuracy": 0.4 }, { "bin": 7, "count": 5, "confidence": 0.7474448200596033, "accuracy": 1.0 }, { "bin": 8, "count": 8, "confidence": 0.872150720889298, "accuracy": 0.875 }, { "bin": 9, "count": 334, "confidence": 0.9956838097829439, "accuracy": 0.9910179640718563 } ] }, "after": { "nll": 0.10774593556856667, "accuracy": 0.9666666666666667, "confidence": 0.9510981941735058, "ece": 0.01968257742631988, "bins": [ { "bin": 3, "count": 1, "confidence": 0.3784920212429425, "accuracy": 0.0 }, { "bin": 4, "count": 6, "confidence": 0.46962108092205895, "accuracy": 0.5 }, { "bin": 5, "count": 8, "confidence": 0.5452558583407109, "accuracy": 0.5 }, { "bin": 6, "count": 5, "confidence": 0.6467613542977034, "accuracy": 1.0 }, { "bin": 7, "count": 11, "confidence": 0.7661041155280603, "accuracy": 0.8181818181818182 }, { "bin": 8, "count": 18, "confidence": 0.8592319085853967, "accuracy": 0.9444444444444444 }, { "bin": 9, "count": 311, "confidence": 0.9894210872415653, "accuracy": 0.9967845659163987 } ] } }, { "seed": 22, "bins": 15, "before": { "nll": 0.10809208384800954, "accuracy": 0.9666666666666667, "confidence": 0.9743925312832241, "ece": 0.012973302322377521, "bins": [ { "bin": 7, "count": 4, "confidence": 0.5070546474294442, "accuracy": 0.25 }, { "bin": 8, "count": 4, "confidence": 0.5796743028805522, "accuracy": 0.5 }, { "bin": 9, "count": 4, "confidence": 0.6216933362054571, "accuracy": 0.5 }, { "bin": 10, "count": 4, "confidence": 0.7132264859229807, "accuracy": 0.75 }, { "bin": 11, "count": 2, "confidence": 0.7795589687946526, "accuracy": 1.0 }, { "bin": 12, "count": 2, "confidence": 0.821718665835595, "accuracy": 1.0 }, { "bin": 13, "count": 12, "confidence": 0.9020678070590029, "accuracy": 0.8333333333333334 }, { "bin": 14, "count": 328, "confidence": 0.9971565463970685, "accuracy": 0.9939024390243902 } ] }, "after": { "nll": 0.10774593556856667, "accuracy": 0.9666666666666667, "confidence": 0.9510981941735058, "ece": 0.02388533000822119, "bins": [ { "bin": 5, "count": 1, "confidence": 0.3784920212429425, "accuracy": 0.0 }, { "bin": 6, "count": 2, "confidence": 0.4454361404058881, "accuracy": 0.5 }, { "bin": 7, "count": 8, "confidence": 0.5034904163138741, "accuracy": 0.375 }, { "bin": 8, "count": 4, "confidence": 0.5652444352338178, "accuracy": 0.75 }, { "bin": 9, "count": 3, "confidence": 0.6231393350628363, "accuracy": 1.0 }, { "bin": 10, "count": 3, "confidence": 0.6968730003189773, "accuracy": 0.6666666666666666 }, { "bin": 11, "count": 10, "confidence": 0.7700915036151741, "accuracy": 0.9 }, { "bin": 12, "count": 8, "confidence": 0.8286711793196716, "accuracy": 1.0 }, { "bin": 13, "count": 16, "confidence": 0.8938094435063119, "accuracy": 0.9375 }, { "bin": 14, "count": 305, "confidence": 0.9909698752655921, "accuracy": 0.9967213114754099 } ] } }, { "seed": 22, "bins": 30, "before": { "nll": 0.10809208384800954, "accuracy": 0.9666666666666667, "confidence": 0.9743925312832241, "ece": 0.019243666075426682, "bins": [ { "bin": 14, "count": 1, "confidence": 0.46949159973744253, "accuracy": 0.0 }, { "bin": 15, "count": 3, "confidence": 0.519575663326778, "accuracy": 0.3333333333333333 }, { "bin": 17, "count": 4, "confidence": 0.5796743028805522, "accuracy": 0.5 }, { "bin": 18, "count": 4, "confidence": 0.6216933362054571, "accuracy": 0.5 }, { "bin": 20, "count": 1, "confidence": 0.6747997809832118, "accuracy": 0.0 }, { "bin": 21, "count": 3, "confidence": 0.7260353875695703, "accuracy": 1.0 }, { "bin": 23, "count": 2, "confidence": 0.7795589687946526, "accuracy": 1.0 }, { "bin": 24, "count": 1, "confidence": 0.8058248652604558, "accuracy": 1.0 }, { "bin": 25, "count": 1, "confidence": 0.8376124664107344, "accuracy": 1.0 }, { "bin": 26, "count": 6, "confidence": 0.888961405907199, "accuracy": 0.8333333333333334 }, { "bin": 27, "count": 6, "confidence": 0.9151742082108073, "accuracy": 0.8333333333333334 }, { "bin": 28, "count": 10, "confidence": 0.9546134305434368, "accuracy": 1.0 }, { "bin": 29, "count": 318, "confidence": 0.9984943802289438, "accuracy": 0.9937106918238994 } ] }, "after": { "nll": 0.10774593556856667, "accuracy": 0.9666666666666667, "confidence": 0.9510981941735058, "ece": 0.028423319038698366, "bins": [ { "bin": 11, "count": 1, "confidence": 0.3784920212429425, "accuracy": 0.0 }, { "bin": 13, "count": 2, "confidence": 0.4454361404058881, "accuracy": 0.5 }, { "bin": 14, "count": 4, "confidence": 0.48171355118014436, "accuracy": 0.5 }, { "bin": 15, "count": 4, "confidence": 0.5252672814476039, "accuracy": 0.25 }, { "bin": 16, "count": 2, "confidence": 0.5540404982532485, "accuracy": 0.5 }, { "bin": 17, "count": 2, "confidence": 0.5764483722143872, "accuracy": 1.0 }, { "bin": 18, "count": 2, "confidence": 0.6143637507988452, "accuracy": 1.0 }, { "bin": 19, "count": 1, "confidence": 0.6406905035908182, "accuracy": 1.0 }, { "bin": 20, "count": 2, "confidence": 0.6821943831500041, "accuracy": 1.0 }, { "bin": 21, "count": 1, "confidence": 0.7262302346569234, "accuracy": 0.0 }, { "bin": 22, "count": 5, "confidence": 0.7572831056207571, "accuracy": 1.0 }, { "bin": 23, "count": 5, "confidence": 0.782899901609591, "accuracy": 0.8 }, { "bin": 24, "count": 4, "confidence": 0.8150859517590435, "accuracy": 1.0 }, { "bin": 25, "count": 4, "confidence": 0.8422564068802999, "accuracy": 1.0 }, { "bin": 26, "count": 10, "confidence": 0.8836804919979766, "accuracy": 0.9 }, { "bin": 27, "count": 6, "confidence": 0.9106910293535373, "accuracy": 1.0 }, { "bin": 28, "count": 19, "confidence": 0.9498720932853422, "accuracy": 1.0 }, { "bin": 29, "count": 286, "confidence": 0.9937001474950493, "accuracy": 0.9965034965034965 } ] } }, { "seed": 33, "bins": 5, "before": { "nll": 0.14107690341861967, "accuracy": 0.9611111111111111, "confidence": 0.973709508912905, "ece": 0.023523663634778792, "bins": [ { "bin": 1, "count": 1, "confidence": 0.3925293881770926, "accuracy": 1.0 }, { "bin": 2, "count": 6, "confidence": 0.5146447422084369, "accuracy": 0.3333333333333333 }, { "bin": 3, "count": 12, "confidence": 0.7200768968238024, "accuracy": 0.8333333333333334 }, { "bin": 4, "count": 341, "confidence": 0.9924167231827932, "accuracy": 0.9765395894428153 } ] }, "after": { "nll": 0.12956240966921712, "accuracy": 0.9611111111111111, "confidence": 0.9521760967743357, "ece": 0.011743291598695513, "bins": [ { "bin": 1, "count": 4, "confidence": 0.37637247678639857, "accuracy": 0.25 }, { "bin": 2, "count": 6, "confidence": 0.5397290794194681, "accuracy": 0.6666666666666666 }, { "bin": 3, "count": 22, "confidence": 0.7017201041781514, "accuracy": 0.8181818181818182 }, { "bin": 4, "count": 328, "confidence": 0.9835417322048142, "accuracy": 0.9847560975609756 } ] } }, { "seed": 33, "bins": 10, "before": { "nll": 0.14107690341861967, "accuracy": 0.9611111111111111, "confidence": 0.973709508912905, "ece": 0.02938365032466806, "bins": [ { "bin": 3, "count": 1, "confidence": 0.3925293881770926, "accuracy": 1.0 }, { "bin": 4, "count": 3, "confidence": 0.47824068672419767, "accuracy": 0.0 }, { "bin": 5, "count": 3, "confidence": 0.5510487976926761, "accuracy": 0.6666666666666666 }, { "bin": 6, "count": 4, "confidence": 0.6769859993145225, "accuracy": 0.5 }, { "bin": 7, "count": 8, "confidence": 0.7416223455784423, "accuracy": 1.0 }, { "bin": 8, "count": 10, "confidence": 0.8630216745312869, "accuracy": 0.7 }, { "bin": 9, "count": 331, "confidence": 0.9963259391541378, "accuracy": 0.9848942598187311 } ] }, "after": { "nll": 0.12956240966921712, "accuracy": 0.9611111111111111, "confidence": 0.9521760967743357, "ece": 0.011743291598695494, "bins": [ { "bin": 3, "count": 4, "confidence": 0.37637247678639857, "accuracy": 0.25 }, { "bin": 4, "count": 1, "confidence": 0.4406541414396147, "accuracy": 1.0 }, { "bin": 5, "count": 5, "confidence": 0.5595440670154389, "accuracy": 0.6 }, { "bin": 6, "count": 9, "confidence": 0.6237365852497034, "accuracy": 0.8888888888888888 }, { "bin": 7, "count": 13, "confidence": 0.7557086942055385, "accuracy": 0.7692307692307693 }, { "bin": 8, "count": 9, "confidence": 0.8540676035124863, "accuracy": 0.8888888888888888 }, { "bin": 9, "count": 319, "confidence": 0.9871946073089866, "accuracy": 0.987460815047022 } ] } }, { "seed": 33, "bins": 15, "before": { "nll": 0.14107690341861967, "accuracy": 0.9611111111111111, "confidence": 0.973709508912905, "ece": 0.028407310550409556, "bins": [ { "bin": 5, "count": 1, "confidence": 0.3925293881770926, "accuracy": 1.0 }, { "bin": 6, "count": 1, "confidence": 0.46481783320226516, "accuracy": 0.0 }, { "bin": 7, "count": 3, "confidence": 0.5007023549539701, "accuracy": 0.0 }, { "bin": 8, "count": 2, "confidence": 0.5604717775932228, "accuracy": 1.0 }, { "bin": 10, "count": 7, "confidence": 0.6892445270758996, "accuracy": 0.7142857142857143 }, { "bin": 11, "count": 5, "confidence": 0.7632422144708664, "accuracy": 1.0 }, { "bin": 12, "count": 4, "confidence": 0.8300157154723514, "accuracy": 0.5 }, { "bin": 13, "count": 8, "confidence": 0.8912918544974587, "accuracy": 0.875 }, { "bin": 14, "count": 329, "confidence": 0.9968501668919859, "accuracy": 0.9848024316109423 } ] }, "after": { "nll": 0.12956240966921712, "accuracy": 0.9611111111111111, "confidence": 0.9521760967743357, "ece": 0.022418584576312864, "bins": [ { "bin": 5, "count": 4, "confidence": 0.37637247678639857, "accuracy": 0.25 }, { "bin": 6, "count": 1, "confidence": 0.4406541414396147, "accuracy": 1.0 }, { "bin": 7, "count": 1, "confidence": 0.5120048617854834, "accuracy": 0.0 }, { "bin": 8, "count": 4, "confidence": 0.5714288683229277, "accuracy": 0.75 }, { "bin": 9, "count": 9, "confidence": 0.6237365852497034, "accuracy": 0.8888888888888888 }, { "bin": 10, "count": 4, "confidence": 0.7133971072288425, "accuracy": 0.5 }, { "bin": 11, "count": 9, "confidence": 0.7745138439729591, "accuracy": 0.8888888888888888 }, { "bin": 12, "count": 6, "confidence": 0.8409547863181327, "accuracy": 1.0 }, { "bin": 13, "count": 17, "confidence": 0.9101148998521171, "accuracy": 0.8823529411764706 }, { "bin": 14, "count": 305, "confidence": 0.9904393644189649, "accuracy": 0.9901639344262295 } ] } }, { "seed": 33, "bins": 30, "before": { "nll": 0.14107690341861967, "accuracy": 0.9611111111111111, "confidence": 0.973709508912905, "ece": 0.03333932746541983, "bins": [ { "bin": 11, "count": 1, "confidence": 0.3925293881770926, "accuracy": 1.0 }, { "bin": 13, "count": 1, "confidence": 0.46481783320226516, "accuracy": 0.0 }, { "bin": 14, "count": 2, "confidence": 0.48495211348516387, "accuracy": 0.0 }, { "bin": 15, "count": 1, "confidence": 0.5322028378915827, "accuracy": 0.0 }, { "bin": 16, "count": 1, "confidence": 0.5377020099232558, "accuracy": 1.0 }, { "bin": 17, "count": 1, "confidence": 0.5832415452631897, "accuracy": 1.0 }, { "bin": 20, "count": 4, "confidence": 0.6769859993145225, "accuracy": 0.5 }, { "bin": 21, "count": 3, "confidence": 0.7055892307577355, "accuracy": 1.0 }, { "bin": 22, "count": 3, "confidence": 0.7541761736410632, "accuracy": 1.0 }, { "bin": 23, "count": 2, "confidence": 0.776841275715571, "accuracy": 1.0 }, { "bin": 24, "count": 2, "confidence": 0.8196958788157986, "accuracy": 0.5 }, { "bin": 25, "count": 2, "confidence": 0.8403355521289042, "accuracy": 0.5 }, { "bin": 26, "count": 6, "confidence": 0.8850256472372439, "accuracy": 0.8333333333333334 }, { "bin": 27, "count": 2, "confidence": 0.9100904762781037, "accuracy": 1.0 }, { "bin": 28, "count": 9, "confidence": 0.952448062941088, "accuracy": 0.8888888888888888 }, { "bin": 29, "count": 320, "confidence": 0.9980989760656047, "accuracy": 0.9875 } ] }, "after": { "nll": 0.12956240966921712, "accuracy": 0.9611111111111111, "confidence": 0.9521760967743357, "ece": 0.032197235288543784, "bins": [ { "bin": 10, "count": 1, "confidence": 0.35323827353430876, "accuracy": 1.0 }, { "bin": 11, "count": 3, "confidence": 0.3840838778704285, "accuracy": 0.0 }, { "bin": 13, "count": 1, "confidence": 0.4406541414396147, "accuracy": 1.0 }, { "bin": 15, "count": 1, "confidence": 0.5120048617854834, "accuracy": 0.0 }, { "bin": 16, "count": 1, "confidence": 0.5442519772860398, "accuracy": 1.0 }, { "bin": 17, "count": 3, "confidence": 0.5804878320018904, "accuracy": 0.6666666666666666 }, { "bin": 18, "count": 6, "confidence": 0.6075349727103297, "accuracy": 0.8333333333333334 }, { "bin": 19, "count": 3, "confidence": 0.6561398103284508, "accuracy": 1.0 }, { "bin": 21, "count": 4, "confidence": 0.7133971072288425, "accuracy": 0.5 }, { "bin": 22, "count": 3, "confidence": 0.7532173826813833, "accuracy": 0.6666666666666666 }, { "bin": 23, "count": 6, "confidence": 0.785162074618747, "accuracy": 1.0 }, { "bin": 24, "count": 2, "confidence": 0.8228584047038782, "accuracy": 1.0 }, { "bin": 25, "count": 4, "confidence": 0.8500029771252601, "accuracy": 1.0 }, { "bin": 26, "count": 3, "confidence": 0.8802932379011938, "accuracy": 0.6666666666666666 }, { "bin": 27, "count": 14, "confidence": 0.9165052559844579, "accuracy": 0.9285714285714286 }, { "bin": 28, "count": 27, "confidence": 0.9544008513058663, "accuracy": 0.9259259259259259 }, { "bin": 29, "count": 278, "confidence": 0.99393950777887, "accuracy": 0.9964028776978417 } ] } } ], "fit_count": 183, "seconds": 1.3911534580402076 }