"""Assemble draft-<date>.md from draft-prose-<date>.md and the frozen datasets: headline sentences, secondary numbers, site
coverage sentences, the three tables and the cohort summaries are generated, never typed. Usage: python3 scripts/build-draft.py 2026-09-02"""
import json,re,sys,statistics
date=sys.argv[1]; SITES=('Capterra','GetApp','Software Advice','G2','Trustpilot')
prose=open(f'draft-prose-{date}.md',encoding='utf-8').read(); tables=open(f'draft-tables-{date}.md',encoding='utf-8').read()
def section(name):
    i=tables.index('## '+name); j=tables.find('\n## ',i+3); return tables[i:j if j>0 else None].split('\n',2)[2].strip()
s3=open(f's3-results-cohort1-{date}.md').read()
head=s3[s3.index('## Headline sentences'):s3.index('## Displayed count')]
sents=[l.strip().strip('"') for l in head.split('\n') if l.strip().startswith('"')]
assert len(sents)==3
def num(label,txt):
    m=re.search(re.escape(label)+r'.*?: ([^\n]+)',txt); return m.group(1).strip()
prim=s3[s3.index('## Primary'):s3.index('## Sensitivity')]; sens=s3[s3.index('## Sensitivity'):s3.index('## Headline')]
F=num('F, products',prim); A=num('A, comparable',prim); pooled=num('Pooled',prim); medG=num('Median number of distinct',prim); medS=num('Median R_p',sens); pooledS=num('Pooled',sens)
secondary=f"coverage, F = {F} products with profiles found on two or more sites and A = {A} comparable products; the pooled volume-weighted share of displayed review-count entries classified as redundant under the exact-signature rule, {pooled}; the median number of distinct displayed signatures per comparable product, {medG}; and the sensitivity run that excludes profiles with fewer than ten displayed reviews, which gives a median R_p of {medS} and a pooled share of {pooledS}, with coverage unchanged."
rows=json.load(open(f'data/signatures-cohort1-{date}.json'))
cov=[]
for site in SITES:
    rs=[r for r in rows if r['site']==site]; el=sum(1 for r in rs if r.get('eligible')); zero=sum(1 for r in rs if r.get('saved') and r.get('count')==0); nor=sum(1 for r in rs if r.get('saved') and r.get('category_ok',True) and r.get('count') is None and r.get('capture_status','page')=='page'); unres=sum(1 for r in rs if r.get('saved') and r.get('category_ok',True) and r.get('capture_status','page')!='page'); nf=sum(1 for r in rs if not r.get('saved') or not r.get('category_ok',True))
    parts=[f"{site} displayed a count and a rating for {el} of the 12 products"]
    if zero: parts.append(f"a page with zero reviews for {zero}")
    if nor: parts.append(f"a profile with no count or rating displayed for {nor}")
    if unres: parts.append(f"a profile found in discovery whose capture returned an error response for {unres}")
    if nf: parts.append(f"and we found no profile in steps 1 to 3 for {nf}")
    cov.append(', '.join(parts)+'.')
site_cov=' '.join(cov)
def summary(c):
    t=open(f's3-results-{c}-{date}.md').read(); p=t[t.index('## Primary'):t.index('## Sensitivity')]
    A=num('A, comparable',p); a=int(A.split(' of ')[0]); B=num('B, comparable',p).split(' of ')[0]
    verb='was' if a==1 else 'were'
    bsent=(f" For {B} of those {a} comparable products, two or more review sites displayed an exact matching signature;" if a>1 else " For the one comparable product,") if a>0 else ""
    tail=(f" the median R_p was {num('Median R_p',p)} and the pooled share {num('Pooled',p)}." if a>1 else (f" the median R_p was {num('Median R_p',p)}." if a==1 else ""))
    cc=num('Corroboration',p).split(': ',1)[-1]; m=re.findall(r'corroborated (\d+), discordant (\d+), unassessable (\d+)',cc)
    cor=(f" Of the matching groups, {m[0][0]} were corroborated, {m[0][1]} discordant and {m[0][2]} unassessable under the sub-rating check." if m and a>1 else "")
    return f"{num('F, products',p)} products had profiles found on two or more sites and {A} {verb} comparable.{bsent}{tail}{cor}"
def corroboration():
    t=open(f's3-results-cohort1-{date}.md').read(); p=t[t.index('## Primary'):t.index('## Sensitivity')]
    rows=[l for l in p.split('\n') if l.startswith('| ') and not l.startswith('| Product')]
    ties=[];few=[];disc=[]
    for l in rows:
        c=l.split('|'); prod=c[1].strip(); cor=c[8].strip()
        if 'rounding tie' in cor: ties.append(prod)
        elif 'fewer than two' in cor: few.append(prod)
        elif 'discordant' in cor: disc.append(prod)
    cc=num('Corroboration',p).split(': ',1)[-1]
    m=re.findall(r'corroborated (\d+), discordant (\d+), unassessable (\d+)',cc)
    parts=[f"The corroboration check classified {m[0][0]} of the {len(rows)} matching groups as corroborated, {m[0][1]} as discordant and {m[0][2]} as unassessable." if m else f"Corroboration: {cc}."]
    if few: parts.append(f"For {', '.join(few[:-1])+' and '+few[-1] if len(few)>1 else few[0]}, Capterra and GetApp displayed the matching signature, but GetApp displayed no numeric sub-ratings, leaving fewer than two shared numeric labels.")
    if ties: parts.append(f"For {', '.join(ties[:-1])+' and '+ties[-1] if len(ties)>1 else ties[0]}, Software Advice displayed an Ease of use value with a rounding tie.")
    parts.append("No review site displayed a differing shared value in any group." if not disc else f"Shared values differed for {', '.join(disc)}.")
    return ' '.join(parts), f"Rounding ties left {len(ties)} group{'s' if len(ties)!=1 else ''} unassessable, and GetApp's glyph-only sub-ratings left {len(few)} more."

import os,hashlib
def dataset():
    base='/research/review-cross-listing-'+date+'/'
    man=f'dataset-{date}/MANIFEST.txt'
    if os.path.exists(man):
        lines=[l for l in open(man).read().splitlines() if l.strip()]
        ver=hashlib.sha256(open(man,'rb').read()).hexdigest()[:16]
        files=[l.split('  ',1)[1] for l in lines if '  ' in l]
        sig=', '.join(f'<a href="{base}{f}">{f.split("/")[-1]}</a>' for f in files if f.startswith('data/signatures-'))
        tr=', '.join(f'<a href="{base}{f}">{f.split("/")[-1]}</a>' for f in files if f.startswith('data/discovery-trace-'))
        res=', '.join(f'<a href="{base}{f}">{f}</a>' for f in files if f.startswith('s3-results-'))
        scripts=', '.join(f'<a href="{base}{f}">{f.split("/",1)[1]}</a>' for f in files if f.startswith('scripts/'))
        return (f"The frozen dataset behind every number on this page is published in one folder with a <a href=\"{base}MANIFEST.txt\">SHA-256 manifest</a> (dataset version {date}, manifest hash {ver}, {len(files)} files) and a <a href=\"{base}README.txt\">README</a>. The row datasets, one per cohort, are {sig}. The discovery traces, which record every search attempt and its outcome, are {tr}. The script outputs are {res}. The probe attempts, captures, supplementary pages and the file inventory are in the same folder, listed in the manifest. The scripts, in the exact version used, are {scripts}. The saved review-site pages (about 300 files) are not published because of their size; every row carries the SHA-256 of its saved page, and the pages are available on request from support@sharpassessment.com.")
    return "The frozen dataset, the discovery manifests and traces, the results files and the exact scripts are published alongside this page under /research/review-cross-listing-"+date+"/ with a SHA-256 manifest (published with the page)."
out=prose.replace('{{HEADLINE}}','\n\n'.join(f'**{x}**' for x in sents)).replace('{{SECONDARY}}',secondary).replace('{{CITE}}',sents[0]+' (SharpAssessment, Assessment review cross-listing study, https://sharpassessment.com/assessment-review-cross-listing/).').replace('{{STUDY_TABLE}}',section('Study table (32 products: cohort 1 headline frame, cohorts 2 and 3 sensitivity samples)')).replace('{{APPENDIX_TABLE}}',section('Appendix (4 products read the same day, in no cohort and no denominator)')).replace('{{DATASET}}',dataset()).replace('{{SITE_COVERAGE}}',site_cov).replace('{{CORROBORATION}}',corroboration()[0]).replace('{{TIES_SENTENCE}}',corroboration()[1]).replace('{{COHORT2_SUMMARY}}',summary('cohort2')).replace('{{COHORT3_SUMMARY}}',summary('cohort3'))
assert '{{' not in out and '–' not in out and '—' not in out
t=re.search(r"^title: '(.*)'$",out,re.M).group(1); d=re.search(r"^description: '(.*)'$",out,re.M).group(1)
assert len(t)<=70 and 50<=len(d)<=165,(len(t),len(d))
for w in ('the same reviews','duplicate','copied','recycled','redundant reviews','fake','fraud'): assert w not in out.lower(), w
open(f'draft-{date}.md','w',encoding='utf-8').write(out)
body=out.split('---',2)[2]; print('draft built:',len(out.split()),'words total,',len(re.sub(r'<[^>]+>|\|[^\n]*','',body).split()),'prose words; title',len(t),'description',len(d))