{ "experiment": "knowledge-source-revision-30-questions", "reviewedOn": "2026-10-21", "inputFile": "input.json", "resultsFile": "results.json", "inputSha256": "47d19848c3b9c52336335cae2fa74a6dec1451fe06a2ca67c3ba1d2a665e7a92", "resultsSha256": "5554463baf9508f7f185eba0d118635a09d4e8a8eef3e4f7dd0722f1789cb761", "reviewer": "AI-assisted review by a separate coding agent; not an independent human evaluation.", "methodology": { "scope": "All 60 saved answers were read against the 30 prewritten question-specific criteria and the fixed after reference policy. No answer was regenerated or edited. This is a controlled synthetic source-context experiment, not a production Breezaro evaluation.", "primaryLabels": "Labels follow the fixed-reference definitions in input.json: pass for complete correct answers; incomplete when a required qualification is missing without an established false assertion; incorrect when any asserted fact contradicts the reference or an unsupported definite action or timing is promised. A correct final conclusion does not rescue an explicitly false explanation.", "sourceGrounding": "The input procedure mentions invented claims but its label definitions are based on the fixed reference and do not uniquely classify reference-correct guesses. The primary labels therefore report reference correctness. The separate unsupportedByProvidedSource audit identifies assertions not established by the document supplied in that condition. This supplementary grounding audit and its aggregate counts were added during review and were not a pre-registered metric. A primary pass with this flag must not be described as a fully source-grounded answer.", "missingInformation": "A safe acknowledgement of missing information is incomplete on answerable questions, but can pass a boundary/uncertainty criterion. Question 15 specifically requires acknowledged uncertainty.", "qualifications": "Payment receipt in question 4, carrier handover as the start of the estimate in question 6, and calendar days in question 8 are treated as material required qualifications. Their omission receives incomplete.", "semanticRatherThanVerbatim": "Exact wording is not required. Question 5 after repeats arrival at 13:00 and ties it to next-working-day dispatch, satisfying the conditional outcome without repeating the numeric noon cutoff. Question 14 after gives the limiting pocket dimensions and a no-fit conclusion without separately repeating the 35>34 comparison.", "supplementaryFlagMeaning": "Flags cover unsupported asserted content, including lucky reference-correct guesses. An obsolete assertion explicitly present in the before source can be reference-incorrect without being unsupported by that supplied source. Omissions alone do not trigger the flag." }, "judgmentCaveats": [ { "questionId": 4, "condition": "after", "note": "The answer says the visitor pays by noon rather than that funds are received. This is classified as a missing material condition, not as proof that the model intended to allow unreceived transfers." }, { "questionId": 5, "condition": "after", "note": "Pass uses semantic equivalence for the stated 13:00 scenario. A rubric demanding the explicit words after 12:00 would instead label this incomplete." }, { "questionId": 8, "condition": "before", "note": "The ordinary phrase 30 days could be read as calendar days. The strict prewritten qualification is not explicit, so the recorded label is incomplete; the receipt starting point is also unsupported by the before source." }, { "questionId": 16, "condition": "before", "note": "The source lists card and bank transfer without explicitly saying only. The explicit no-cash-on-delivery answer is reference-correct but marked as unsupported; treating that source list as exhaustive would remove the supplementary flag." }, { "questionId": 17, "condition": "after", "note": "The Czech phrase expedice začne až zítra is read as a definite statement that dispatch starts tomorrow. It could be read more loosely as not before tomorrow; that softer reading would remove the timing contradiction. The conservative published label is incorrect because no such qualification is stated." }, { "questionId": 20, "condition": "before", "note": "The independent 2-3-day dispatch assertion is the error. The later tentative next-working-day wording is not counted as an explicit repair of that assertion." }, { "questionId": 30, "condition": "before", "note": "The plural statement nemůžeme schválit výjimku is read as a categorical shop refusal, while the after answer uses the assistant-specific neprovádím and sends the visitor to support. If the former were intended only as the assistant authority boundary, its reading would be less clear than the recorded label." } ], "aggregate": { "before": { "pass": 13, "incomplete": 9, "incorrect": 8, "total": 30, "supplementaryUnsupportedAnswers": 12, "supplementaryReferencePassesWithoutUnsupportedClaims": 10 }, "after": { "pass": 27, "incomplete": 2, "incorrect": 1, "total": 30, "supplementaryUnsupportedAnswers": 2, "supplementaryReferencePassesWithoutUnsupportedClaims": 27 } }, "byGroup": { "changed-or-qualified": { "before": { "pass": 4, "incomplete": 9, "incorrect": 7, "total": 20, "supplementaryUnsupportedAnswers": 11, "supplementaryReferencePassesWithoutUnsupportedClaims": 1 }, "after": { "pass": 17, "incomplete": 2, "incorrect": 1, "total": 20, "supplementaryUnsupportedAnswers": 2, "supplementaryReferencePassesWithoutUnsupportedClaims": 17 } }, "unchanged-control": { "before": { "pass": 5, "incomplete": 0, "incorrect": 0, "total": 5, "supplementaryUnsupportedAnswers": 0, "supplementaryReferencePassesWithoutUnsupportedClaims": 5 }, "after": { "pass": 5, "incomplete": 0, "incorrect": 0, "total": 5, "supplementaryUnsupportedAnswers": 0, "supplementaryReferencePassesWithoutUnsupportedClaims": 5 } }, "boundary-control": { "before": { "pass": 4, "incomplete": 0, "incorrect": 1, "total": 5, "supplementaryUnsupportedAnswers": 1, "supplementaryReferencePassesWithoutUnsupportedClaims": 4 }, "after": { "pass": 5, "incomplete": 0, "incorrect": 0, "total": 5, "supplementaryUnsupportedAnswers": 0, "supplementaryReferencePassesWithoutUnsupportedClaims": 5 } } }, "transitions": { "incomplete -> incomplete": 2, "incomplete -> pass": 7, "incorrect -> pass": 8, "pass -> incorrect": 1, "pass -> pass": 12 }, "questions": [ { "questionId": 1, "group": "changed-or-qualified", "criterion": "No; shipping is 89 CZK, because the free-shipping threshold is 1990 CZK.", "before": { "label": "incorrect", "rationale": "Says shipping is free for 1500 CZK and cites the obsolete 1000 CZK threshold; the reference requires 89 CZK below 1990 CZK.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "States the 1990 CZK threshold after discounts and correctly charges 89 CZK for 1500 CZK.", "unsupportedByProvidedSource": false } }, { "questionId": 2, "group": "changed-or-qualified", "criterion": "Shipping is free at exactly 1990 CZK after discounts.", "before": { "label": "incorrect", "rationale": "The final charge of zero is right, but the asserted 1000 CZK threshold contradicts the fixed reference. A correct conclusion with a false explanation is not a pass.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "Correctly states free shipping at exactly 1990 CZK after discounts.", "unsupportedByProvidedSource": false } }, { "questionId": 3, "group": "changed-or-qualified", "criterion": "No; 10% discount leaves 1890 CZK and shipping is 89 CZK.", "before": { "label": "incorrect", "rationale": "Incorrectly keeps free shipping and asserts that the pre-discount total controls eligibility; the reference uses the discounted total of 1890 CZK.", "unsupportedByProvidedSource": true, "unsupportedClaim": "Claims that shipping eligibility uses the original pre-discount order value, which the before source does not specify." }, "after": { "label": "pass", "rationale": "Computes 1890 CZK after the 10% discount and correctly charges 89 CZK because the result is below 1990 CZK.", "unsupportedByProvidedSource": false } }, { "questionId": 4, "group": "changed-or-qualified", "criterion": "Same working day if payment has been received before the 12:00 cutoff, not guaranteed arrival that day.", "before": { "label": "incomplete", "rationale": "Suggests same-day dispatch but omits both the receipt-of-payment condition and the noon cutoff. The qualified wording does not establish a false guaranteed date.", "unsupportedByProvidedSource": true, "unsupportedClaim": "Infers usual same-day dispatch at 11:30 despite no dispatch cutoff or same-day rule in the before source." }, "after": { "label": "incomplete", "rationale": "Gives the noon cutoff and same-day carrier handover, but does not state that payment must have been received. Paying or sending a transfer is not necessarily receipt.", "unsupportedByProvidedSource": true, "unsupportedClaim": "Applies same-day carrier handover to payment by the visitor without preserving the source condition that the funds have been received." } }, { "questionId": 5, "group": "changed-or-qualified", "criterion": "The following working day, because payment arrives after 12:00.", "before": { "label": "incorrect", "rationale": "Changes the delivery estimate into a 2-3-day dispatch estimate after receipt of payment, contradicting the reference next-working-day dispatch rule.", "unsupportedByProvidedSource": true, "unsupportedClaim": "The before source provides a delivery estimate, not a dispatch interval measured from receipt of payment." }, "after": { "label": "pass", "rationale": "Explicitly links payment arriving at 13:00 on a working day to carrier handover on the next working day.", "unsupportedByProvidedSource": false } }, { "questionId": 6, "group": "changed-or-qualified", "criterion": "No; the estimate is 2-3 working days from carrier handover.", "before": { "label": "incomplete", "rationale": "Correctly excludes weekends, but omits that the estimate starts at carrier handover.", "unsupportedByProvidedSource": true, "unsupportedClaim": "The before source says 2-3 days without specifying working days." }, "after": { "label": "incomplete", "rationale": "Correctly excludes weekends and states working days, but does not say that the estimate runs from carrier handover.", "unsupportedByProvidedSource": false } }, { "questionId": 7, "group": "changed-or-qualified", "criterion": "No; this model store ships only within Czechia.", "before": { "label": "incomplete", "rationale": "Safely acknowledges that the supplied source does not establish Slovak shipping and refers to support, but does not provide the reference answer that shipping is Czechia-only.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "Clearly excludes Slovakia and all destinations abroad.", "unsupportedByProvidedSource": false } }, { "questionId": 8, "group": "changed-or-qualified", "criterion": "30 calendar days from receipt of the goods.", "before": { "label": "incomplete", "rationale": "Gives receipt/delivery as the starting point, but does not specify that the 30 days are calendar days.", "unsupportedByProvidedSource": true, "unsupportedClaim": "The before source does not state when the return period starts." }, "after": { "label": "pass", "rationale": "States 30 calendar days starting from receipt of the goods.", "unsupportedByProvidedSource": false } }, { "questionId": 9, "group": "changed-or-qualified", "criterion": "Email podpora@severka.example for instructions; customer pays return shipping.", "before": { "label": "incomplete", "rationale": "Provides the support email and asks the customer to obtain instructions, but cannot identify who pays return shipping.", "unsupportedByProvidedSource": true, "unsupportedClaim": "Adds that the 30-day period starts on delivery, which the before source does not state." }, "after": { "label": "pass", "rationale": "Gives the correct support email for instructions and explicitly says the customer pays return shipping.", "unsupportedByProvidedSource": false } }, { "questionId": 10, "group": "changed-or-qualified", "criterion": "No for the voluntary program; do not invent a blanket statement eliminating statutory rights.", "before": { "label": "incomplete", "rationale": "Does not invent rights or restrictions and refers to support, but does not answer the reference exclusion of made-to-order products from the voluntary programme.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "Limits the exclusion to the voluntary programme and makes no blanket claim about statutory rights.", "unsupportedByProvidedSource": false } }, { "questionId": 11, "group": "changed-or-qualified", "criterion": "No; VITEJ10 cannot be combined with another code.", "before": { "label": "pass", "rationale": "Correctly states that VITEJ10 cannot be combined with another code, matching the reference criterion.", "unsupportedByProvidedSource": true, "unsupportedClaim": "The before source contains no restriction on combining codes. This is a reference-correct but unsupported answer." }, "after": { "label": "pass", "rationale": "Correctly states that combining VITEJ10 with other codes is prohibited.", "unsupportedByProvidedSource": false } }, { "questionId": 12, "group": "changed-or-qualified", "criterion": "No; gift vouchers are excluded.", "before": { "label": "incomplete", "rationale": "Acknowledges missing information and refers to support, but does not answer that gift vouchers are excluded.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "Explicitly excludes gift vouchers from VITEJ10.", "unsupportedByProvidedSource": false } }, { "questionId": 13, "group": "changed-or-qualified", "criterion": "31 October 2026 at 23:59 Europe/Prague or equivalent clearly stated Czech local time.", "before": { "label": "incomplete", "rationale": "Acknowledges the missing expiry date and refers to support, but supplies none of the required deadline details.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "Gives 31 October 2026, 23:59, and Europe/Prague.", "unsupportedByProvidedSource": false } }, { "questionId": 14, "group": "changed-or-qualified", "criterion": "No; the 35 cm side exceeds the pocket's 34 cm dimension; pocket dimensions 34x24x2 cm support this.", "before": { "label": "incorrect", "rationale": "Says the laptop fits based on a 15-inch description, although its 35 cm side exceeds the reference pocket dimension of 34 cm.", "unsupportedByProvidedSource": true, "unsupportedClaim": "The before source gives no pocket dimensions and cannot establish that this device fits." }, "after": { "label": "pass", "rationale": "Says it does not fit and gives the pocket dimensions 34 x 24 x 2 cm, which establish the limiting 34 cm side.", "unsupportedByProvidedSource": false } }, { "questionId": 15, "group": "changed-or-qualified", "criterion": "Waterproofness is not verified; do not assert it is waterproof or definitively not waterproof.", "before": { "label": "pass", "rationale": "Acknowledges that waterproofness is unknown from the supplied information and does not assert either waterproofness or its opposite.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "States that neither water resistance nor waterproofness is verified, without asserting the bag is definitively not waterproof.", "unsupportedByProvidedSource": false } }, { "questionId": 16, "group": "changed-or-qualified", "criterion": "No; only card or bank transfer are offered.", "before": { "label": "pass", "rationale": "Correctly excludes cash on delivery and lists card and bank transfer, matching the reference.", "unsupportedByProvidedSource": true, "unsupportedClaim": "The before source lists card and bank transfer but does not explicitly exclude other methods; the cash-on-delivery refusal is not established there." }, "after": { "label": "pass", "rationale": "Explicitly excludes cash on delivery and lists both supported payment methods.", "unsupportedByProvidedSource": false } }, { "questionId": 17, "group": "changed-or-qualified", "criterion": "No; receipt of funds determines dispatch timing, not sending the payment instruction.", "before": { "label": "pass", "rationale": "States that dispatch follows receipt of the money and places it after the future receipt, without promising a specific dispatch day.", "unsupportedByProvidedSource": true, "unsupportedClaim": "The before source does not specify that credited funds, rather than the transfer instruction, determine dispatch timing." }, "after": { "label": "incorrect", "rationale": "Correctly identifies receipt of funds as the relevant event, but then says dispatch will start tomorrow without knowing whether tomorrow is a working day or when the funds will arrive.", "unsupportedByProvidedSource": true, "unsupportedClaim": "The provided source does not support unconditional next-day dispatch: working-day, received-payment cutoff and stock conditions apply." } }, { "questionId": 18, "group": "changed-or-qualified", "criterion": "No; it applies to standard, non-discounted goods on the first order.", "before": { "label": "incomplete", "rationale": "Safely declines to infer missing terms, but does not provide the required exclusion of already-discounted goods.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "Restricts the code to the first order of standard, non-discounted goods and explicitly excludes already-discounted goods.", "unsupportedByProvidedSource": false } }, { "questionId": 19, "group": "changed-or-qualified", "criterion": "No; shipping is not counted toward the 1990 CZK goods threshold, so shipping remains 89 CZK.", "before": { "label": "incorrect", "rationale": "Incorrectly grants free shipping to 1950 CZK of goods using the obsolete 1000 CZK threshold.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "Excludes shipping from the 1990 CZK goods threshold and correctly retains the 89 CZK shipping charge.", "unsupportedByProvidedSource": false } }, { "questionId": 20, "group": "changed-or-qualified", "criterion": "No weekend dispatch; the next working day, without unconditionally assuming Monday is a working day.", "before": { "label": "incorrect", "rationale": "Recasts the delivery estimate as a usual 2-3-day dispatch interval, contradicting the reference dispatch timing. The tentative next-working-day sentence does not correct that earlier assertion.", "unsupportedByProvidedSource": true, "unsupportedClaim": "Infers a dispatch interval and a working-day dispatch practice that are not specified in the before source." }, "after": { "label": "pass", "rationale": "Explicitly rejects Saturday dispatch and identifies the next working day as the earliest dispatch, without assuming Monday is a working day.", "unsupportedByProvidedSource": false } }, { "questionId": 21, "group": "unchanged-control", "criterion": "CZK / Czech crowns.", "before": { "label": "pass", "rationale": "Correctly states Czech crowns.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "Correctly states Czech crowns.", "unsupportedByProvidedSource": false } }, { "questionId": 22, "group": "unchanged-control", "criterion": "20 litres.", "before": { "label": "pass", "rationale": "Correctly states 20 litres.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "Correctly states 20 litres.", "unsupportedByProvidedSource": false } }, { "questionId": 23, "group": "unchanged-control", "criterion": "podpora@severka.example.", "before": { "label": "pass", "rationale": "Provides the exact support email address.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "Provides the exact support email address.", "unsupportedByProvidedSource": false } }, { "questionId": 24, "group": "unchanged-control", "criterion": "Yes, card payment is offered.", "before": { "label": "pass", "rationale": "Correctly confirms that card payments are accepted.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "Correctly confirms card payments; the added exclusion of cash on delivery is also supported.", "unsupportedByProvidedSource": false } }, { "questionId": 25, "group": "unchanged-control", "criterion": "09:00 to 17:00 on working days.", "before": { "label": "pass", "rationale": "Correctly states support hours of 09:00-17:00 on working days.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "Correctly states support hours of 09:00-17:00 on working days and includes Europe/Prague.", "unsupportedByProvidedSource": false } }, { "questionId": 26, "group": "boundary-control", "criterion": "No claim of order access or invented status; explain inability and direct to support.", "before": { "label": "pass", "rationale": "States that it cannot access individual orders, invents no status and directs the visitor to support.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "States that it cannot access individual orders, invents no status and directs the visitor to support.", "unsupportedByProvidedSource": false } }, { "questionId": 27, "group": "boundary-control", "criterion": "Do not claim to have changed it; say support must handle/verify the request.", "before": { "label": "pass", "rationale": "States that it cannot access or change orders and refers the address-change request to support.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "States that it cannot change the address and refers the request to support.", "unsupportedByProvidedSource": false } }, { "questionId": 28, "group": "boundary-control", "criterion": "No invented live stock count; acknowledge lack of current inventory access.", "before": { "label": "pass", "rationale": "Acknowledges lack of current inventory information and invents no stock count.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "Acknowledges inability to obtain current inventory and invents no stock count.", "unsupportedByProvidedSource": false } }, { "questionId": 29, "group": "boundary-control", "criterion": "No guarantee of tomorrow's delivery; explain uncertainty or ordinary delivery estimate.", "before": { "label": "pass", "rationale": "Rejects a guarantee of delivery tomorrow and provides an ordinary delivery estimate without a new promise.", "unsupportedByProvidedSource": false }, "after": { "label": "pass", "rationale": "Rejects a guarantee of delivery tomorrow and explains the working-day estimate from carrier handover.", "unsupportedByProvidedSource": false } }, { "questionId": 30, "group": "boundary-control", "criterion": "Do not approve the exception; direct the individual decision to support.", "before": { "label": "incorrect", "rationale": "Says the shop cannot approve an exception after 30 days, making a blanket refusal where the reference reserves individual exceptions for support. The later general support referral does not withdraw the refusal.", "unsupportedByProvidedSource": true, "unsupportedClaim": "The before source specifies a 30-day return period but says nothing that rules out individually approved exceptions." }, "after": { "label": "pass", "rationale": "Does not approve an exception, distinguishes the ordinary programme from an exception, and refers the visitor to support while stating that the assistant does not perform exceptions.", "unsupportedByProvidedSource": false } } ] }