{ "$schema": "https://www.tacitus.me/tcgc/schema-v0.1.json", "benchmark": "TCGC", "version": "v0.1-sample", "license": "MIT", "generated": "2026-04-18", "repo": "https://github.com/sargonxg/TCGC_TACITUS-Conflict-Grammar-Corpus_BENCHMARK", "note": "This is a public sample slice (5 items) from the TACITUS Conflict Grammar Corpus. The full corpus is under DUA and will be released in stages. See /research/tcgc.", "taskTypes": [ "actor-resolution", "claim-extraction", "interest-extraction", "constraint-extraction", "leverage-mapping", "commitment-tracking", "event-ordering", "narrative-drift", "causal-chain", "contradiction-detection", "provenance-attribution", "commitment-claim-mismatch", "position-interest-separation", "cross-document-synthesis" ], "itemSchema": { "id": "string: unique identifier", "task_type": "string: one of taskTypes", "domain": "string: e.g. 'workplace', 'commercial', 'peace-process'", "inputs": "object: task-specific inputs (text, documents, transcripts)", "gold": "object: expected extracted structure or answer", "rubric": "object: how to score (exact_match, f1, graph_overlap, etc.)", "notes": "string: annotator rationale and edge cases" }, "items": [ { "id": "tcgc-0001", "task_type": "commitment-tracking", "domain": "workplace", "inputs": { "messages": [ {"day": 1, "time": "Mon 09:14", "from": "Sam", "text": "So we're agreed: you own the Q4 launch deck content, I handle design. Lock it in by Thursday?"}, {"day": 1, "time": "Mon 09:47", "from": "Alex", "text": "Sounds good. I'll pick it up after the Jenkins pitch on Wednesday."}, {"day": 4, "time": "Thu 09:02", "from": "Alex", "text": "I never said I'd own it. Just help. I don't have the bandwidth honestly."} ], "question": "Was there a commitment on content ownership, when was it made, and who asserted it?" }, "gold": { "commitment_id": "cm1", "subject": "Q4 launch deck content", "deadline": "Thursday", "owner_asserted_by_sam": "Alex", "owner_as_alex_frames_it": "shared, assist", "status": "contested", "timestamps": { "made": "day1-mon-09:14", "acknowledged": "day1-mon-09:47", "contested": "day4-thu-09:02" }, "edges": [ {"from": "Sam", "to": "cm1", "type": "ASSERTED", "provenance": "msg1"}, {"from": "Alex", "to": "cm1", "type": "ACKNOWLEDGED_AMBIGUOUSLY", "provenance": "msg2"}, {"from": "Alex", "to": "cm1", "type": "DENIES_SCOPE", "provenance": "msg3"} ] }, "rubric": { "scoring": "graph_overlap + provenance_f1", "graph_overlap_target": 0.85, "provenance_f1_target": 1.0, "notes": "Partial credit if commitment_id missing but timestamps correct." } }, { "id": "tcgc-0002", "task_type": "contradiction-detection", "domain": "commercial", "inputs": { "documents": [ {"id": "doc1", "text": "Vendor confirmed shipment on 2026-02-03. Receiving log shows container arrived 2026-02-04 14:22."}, {"id": "doc2", "text": "Buyer asserts goods never delivered. Invoice paid by wire 2026-02-06."} ], "question": "Identify any contradictions between these accounts and cite the source span for each." }, "gold": { "contradictions": [ { "claim_a": {"actor": "vendor", "assertion": "shipment confirmed 2026-02-03", "provenance": "doc1:span[0..48]"}, "claim_b": {"actor": "buyer", "assertion": "goods never delivered", "provenance": "doc2:span[0..27]"}, "material": true, "resolution_hint": "Receiving log (doc1) shows container arrival; claim_b likely refers to a different shipment or a different definition of 'delivered'" } ] }, "rubric": { "scoring": "contradiction_pair_f1 + provenance_exact_match", "target_f1": 0.90, "notes": "Must identify both source spans. Partial credit for identifying one." } }, { "id": "tcgc-0003", "task_type": "position-interest-separation", "domain": "peace-process", "inputs": { "statement": "The KLF insists on full weapons retention until free elections are guaranteed by international observers.", "question": "Separate the surface Position from the underlying Interest for the KLF." }, "gold": { "position": { "subject": "weapons retention", "modifier": "full, until free elections observed internationally", "verb": "insists" }, "interest": { "subject": "security guarantee against post-election reprisal", "subject_alt": "political voice retained via armed leverage", "confidence": "medium: inferred, not explicitly stated in this span" }, "resolution_space": [ "Observed disarmament staged to electoral milestones", "Pre-negotiated amnesty framework", "Third-party security guarantor" ] }, "rubric": { "scoring": "interest_semantic_sim + position_exact_match + resolution_space_recall", "interest_target": 0.75, "notes": "Interest is interpretive; multiple acceptable answers. Scoring uses LLM-judge with human-annotator anchor set." } }, { "id": "tcgc-0004", "task_type": "event-ordering", "domain": "workplace", "inputs": { "prose": "After the HR meeting, Alex responded. The initial complaint had been filed the previous Friday. By the time Alex addressed it, three days of silence had already passed, during which Sam escalated to their manager.", "question": "Return the events in strict chronological order." }, "gold": { "ordered_events": [ {"id": "e1", "label": "Complaint filed", "time": "t0: Friday"}, {"id": "e2", "label": "3 days of silence", "time": "t0+1 to t0+3"}, {"id": "e3", "label": "Sam escalates to manager", "time": "t0+1 to t0+3"}, {"id": "e4", "label": "HR meeting held", "time": "t0+n"}, {"id": "e5", "label": "Alex responds", "time": "t0+n+delta"} ], "notes": "Dates are not explicit. Task is reconstruction from discourse cues." }, "rubric": { "scoring": "kendall_tau + event_recall", "tau_target": 0.90, "notes": "Items deliberately strip explicit dates. Ordering from cues only." } }, { "id": "tcgc-0005", "task_type": "narrative-drift", "domain": "commercial", "inputs": { "timeline": [ {"t": "week1", "actor": "Party A", "utterance": "This is a misunderstanding between colleagues."}, {"t": "week2", "actor": "Party A", "utterance": "We have a contractual dispute with the vendor."}, {"t": "week4", "actor": "Party A", "utterance": "We are the victims of vendor fraud."} ], "question": "Detect any narrative shift across the three utterances and characterise it." }, "gold": { "drift_detected": true, "frames": [ {"t": "week1", "frame": "interpersonal misunderstanding", "valence": "neutral"}, {"t": "week2", "frame": "contractual dispute", "valence": "adversarial-institutional"}, {"t": "week4", "frame": "fraud victim", "valence": "adversarial-criminal"} ], "shift_magnitude": "substantial", "implication": "Escalation vector: legal counsel likely engaged by week3; downstream resolution space narrows." }, "rubric": { "scoring": "frame_classification_f1 + drift_detection_accuracy + implication_recall", "f1_target": 0.80, "notes": "Drift must be explicit (frame named). Free-form implication scored by LLM-judge." } } ] }