"""Package the frozen dataset for publication with the page (gate 3, item 8): copies the row datasets, discovery manifests and
traces, probe attempts, captures, supplements, results, file inventory and the exact scripts into dataset-<date>/, writes
MANIFEST.txt with the SHA-256 of every file (sorted, one per line: "<sha256>  <path>"), and mirrors the folder into the site
repo at public/research/review-cross-listing-<date>/ so it deploys with the page. Saved review-site pages are not included
(size); their hashes are inside the signatures files. Usage: python3 scripts/publish-dataset.py 2026-09-02"""
import sys,os,shutil,hashlib,glob,json
date=sys.argv[1]; out=f'dataset-{date}'; site=os.path.expanduser(f'~/Documents/GitHub/sharpassessment-site/public/research/review-cross-listing-{date}')
if os.path.exists(out): shutil.rmtree(out)
os.makedirs(out+'/scripts'); os.makedirs(out+'/data')
files=[]
for pat in ('data/signatures-*-%s.json','data/discovery-trace-*-%s.json','data/profiles-*-%s.json','data/capture-*-%s.json','data/supplement-*-%s.json','data/file-inventory-%s.json','data/protocol-evidence-%s.json'):
    for f in sorted(glob.glob(pat%date)): shutil.copy(f,out+'/data/'); files.append('data/'+os.path.basename(f))
for f in sorted(glob.glob(f's3-results-*-{date}.md'))+[f'draft-tables-{date}.md']:
    if os.path.exists(f): shutil.copy(f,out+'/'); files.append(os.path.basename(f))
for f in ('roster.py','discover.py','capture.py','freeze.py','probe2.py','supplement.py','trace.py','extract2.py','inventory.py','protocol-evidence.py','build-tables.py','build-draft.py','publish-dataset.py'):
    shutil.copy('scripts/'+f,out+'/scripts/'); files.append('scripts/'+f)
readme=f"""Assessment review cross-listing study, dataset {date} (sharpassessment.com/assessment-review-cross-listing/).
One row per product and review site in data/signatures-<cohort>-{date}.json: canonical URL, displayed review count and
one-decimal rating as read from the page's own structured data, listing name, sub-ratings where displayed as numbers, the
saved page's SHA-256 and byte size, UTC retrieval time, discovery step. data/discovery-trace-*.json records every step 1
and 2 search attempt (query, saved response files with SHA-256, the file used, the first accepted item, whether it
reproduces the recorded decision); data/profiles-probe-*.json every step 3 attempt; data/capture-*.json and
data/supplement-*.json the fetches; data/file-inventory-{date}.json every saved page on disk with its status. The scripts are
the exact versions used. Saved pages are available on request from support@sharpassessment.com (their hashes are in the
rows). Rows whose saved body was a fetch-layer error response rather than a review-site page carry capture_status, http_status
and error_type and the status 'profile found in discovery; capture unresolved (<status>)'; they enter no eligibility count.
Capterra, GetApp and Software Advice rows carry listing_category and category_ok; a listing outside the HR category family
is labelled 'listing rejected' and counts as not found (amendment 6 in SCOPE.md).
Cohort 1 is the headline frame; cohorts 2 and 3 are sensitivity samples; the appendix enters no denominator.
"""
open(out+'/README.txt','w').write(readme); files.append('README.txt')
for f in files:
    if f.startswith('scripts/'):
        assert open('scripts/'+f.split('/',1)[1],'rb').read()==open(out+'/'+f,'rb').read(), f'packaged script differs from the root copy: {f}'
lines=[]
for f in sorted(files):
    lines.append(hashlib.sha256(open(out+'/'+f,'rb').read()).hexdigest()+'  '+f)
open(out+'/MANIFEST.txt','w').write('\n'.join(lines)+'\n')
if '--mirror' in sys.argv:
    if os.path.exists(site): shutil.rmtree(site)
    shutil.copytree(out,site); print('mirrored to',site)
print('packaged',len(files),'files into',out); print('manifest sha256',hashlib.sha256(open(out+'/MANIFEST.txt','rb').read()).hexdigest()[:16])