import pandas as pd, numpy as np
W='/tmp/wdi/indicators/'
def ind(code, name):
    d=pd.read_csv(W+code+'/data.csv')
    d=d.rename(columns={'Country Code':'iso3','Year':'year','Value':name})[['iso3','year',name]]
    return d
panel=pd.read_csv('/sessions/wizardly-serene-wozniak/mnt/Shackles of Democracy/Regression - Evidence of Decline/panel.csv')
base=panel[['iso3','year','country','PR','Status','trade_open','log_gdppc','pop','gdppc_ppp']].copy()
for code,name in [('se.xpd.totl.gd.zs','educ'),('sh.imm.idpt','imm'),('sh.dyn.mort','u5mort'),
                  ('ny.gdp.totl.rt.zs','rents'),('dt.oda.odat.gn.zs','aid'),
                  ('it.cel.sets.p2','mobile'),('bx.klt.dinv.cd.wd','fdi_usd'),
                  ('bx.trf.pwkr.cd.dt','remit_usd'),('sm.pop.totl','migrants')]:
    base=base.merge(ind(code,name),on=['iso3','year'],how='left')
# glob_panel for gov effectiveness
g=pd.read_csv('/sessions/wizardly-serene-wozniak/mnt/Shackles of Democracy/Regression - Evidence of Decline/glob_panel.csv')[['code','year','ge']]
g.columns=['iso3','year','ge']
base=base.merge(g,on=['iso3','year'],how='left')
# per-capita door components
base['fdi_pc']=np.log1p((base['fdi_usd']/base['pop']).clip(lower=0))
base['remit_pc']=np.log1p((base['remit_usd']/base['pop']).clip(lower=0))
base['mig_share']=100*base['migrants']/base['pop']
base['log_u5mort']=np.log(base['u5mort'])
# doors index: z-scored mean of trade, mobile, fdi_pc, remit_pc, mig_share
comp=['trade_open','mobile','fdi_pc','remit_pc','mig_share']
for c in comp:
    base[c+'_z']=(base[c]-base[c].mean())/base[c].std()
base['doors']=base[[c+'_z' for c in comp]].mean(axis=1, skipna=True)
base['n_doors']=base[[c+'_z' for c in comp]].notna().sum(axis=1)
base.loc[base['n_doors']<3,'doors']=np.nan
# formal coalition proxy from FH political rights (1=most open..7=closed)
base['W_formal']=(8-base['PR'])/7.0
base['smallW']=(base['PR']>=5).astype(float)
base['nondem']=(base['Status']!='F').astype(float)
# lags
base=base.sort_values(['iso3','year'])
for v in ['doors','rents','aid','W_formal','smallW']:
    base[v+'_l1']=base.groupby('iso3')[v].shift(1)
base.to_csv('/tmp/m21/m21_panel.csv',index=False)
print('rows',len(base),'countries',base.iso3.nunique())
for v in ['educ','imm','log_u5mort','ge','doors','rents','aid']:
    print(v,'nonmissing',base[v].notna().sum())
import pandas as pd, numpy as np
from linearmodels.panel import PanelOLS
import warnings; warnings.filterwarnings('ignore')
d=pd.read_csv('/tmp/m21/m21_panel.csv')
d=d.set_index(['iso3','year'])

def run(dv, sample, rhs, label):
    cols=[dv]+rhs
    s=sample.dropna(subset=cols)
    if len(s)<300: print(f'{label}: N too small ({len(s)})'); return
    y=s[dv]; X=s[rhs]
    m=PanelOLS(y,X,entity_effects=True,time_effects=True).fit(cov_type='clustered',cluster_entity=True)
    n_c=s.reset_index().iso3.nunique()
    out=[f'{label}  N={m.nobs} countries={n_c} withinR2={m.rsquared_within:.3f}']
    for v in rhs:
        p=m.pvalues[v]
        star='***' if p<.01 else '**' if p<.05 else '*' if p<.1 else ''
        out.append(f'   {v:<22} b={m.params[v]:+.4f}  p={p:.4f}{star}')
    print('\n'.join(out)); print()

full=d
nd=d[d['nondem']==1]
rhs=['doors_l1','rents_l1','aid_l1','log_gdppc']

print('===== PUBLIC GOODS PROVISION, TWO-WAY FE, IVs lagged 1yr, SEs clustered by country =====\n')
print('--- FULL SAMPLE ---')
run('educ', full, rhs, 'M1  Education spending %GDP')
run('imm',  full, rhs, 'M2  DPT immunization %')
run('log_u5mort', full, rhs, 'M3  log under-5 mortality (neg = provision)')
run('ge',   full, rhs, 'M4  Gov effectiveness (WGI)')

print('--- NON-DEMOCRACIES ONLY (FH Status != Free) ---')
run('educ', nd, rhs, 'M5  Education spending %GDP')
run('imm',  nd, rhs, 'M6  DPT immunization %')
run('log_u5mort', nd, rhs, 'M7  log under-5 mortality')
run('ge',   nd, rhs, 'M8  Gov effectiveness')

print('--- INTERACTION: doors x small formal coalition (full sample) ---')
full2=full.copy()
full2['doorsXsmallW']=full2['doors_l1']*full2['smallW_l1']
rhs2=['doors_l1','smallW_l1','doorsXsmallW','rents_l1','aid_l1','log_gdppc']
run('imm', full2, rhs2, 'M9  DPT immunization %')
run('educ', full2, rhs2, 'M10 Education spending %GDP')
run('log_u5mort', full2, rhs2, 'M11 log under-5 mortality')
run('ge', full2, rhs2, 'M12 Gov effectiveness')
import pandas as pd, numpy as np
from linearmodels.panel import PanelOLS
import warnings; warnings.filterwarnings('ignore')
d=pd.read_csv('/tmp/m21/m21_panel.csv')
d=d.sort_values(['iso3','year'])
d['doors_lead1']=d.groupby('iso3')['doors'].shift(-1)
d['smallW6']=(d['PR']>=6).astype(float)
d['smallW6_l1']=d.groupby('iso3')['smallW6'].shift(1)
for c in ['trade_open','mobile','fdi_pc','remit_pc','mig_share']:
    d[c+'_z_l1']=d.groupby('iso3')[c+'_z'].shift(1)
d=d.set_index(['iso3','year'])
def run(dv,sample,rhs,label):
    s=sample.dropna(subset=[dv]+rhs)
    m=PanelOLS(s[dv],s[rhs],entity_effects=True,time_effects=True).fit(cov_type='clustered',cluster_entity=True)
    print(label,f'N={m.nobs}')
    for v in rhs:
        p=m.pvalues[v]; star='***' if p<.01 else '**' if p<.05 else '*' if p<.1 else ''
        print(f'   {v:<24} b={m.params[v]:+.4f}  p={p:.4f}{star}')
    print()

print('=== R1: stricter small-W (PR>=6) — u5mort ===')
d2=d.copy(); d2['dXs']=d2['doors_l1']*d2['smallW6_l1']
run('log_u5mort',d2,['doors_l1','smallW6_l1','dXs','rents_l1','aid_l1','log_gdppc'],'R1')

print('=== R2: post-1990 only (globalization era) — u5mort ===')
d3=d[d.index.get_level_values('year')>=1990].copy(); d3['dXs']=d3['doors_l1']*d3['smallW_l1']
run('log_u5mort',d3,['doors_l1','smallW_l1','dXs','rents_l1','aid_l1','log_gdppc'],'R2')

print('=== R3: placebo — FUTURE doors on today mortality (should be weaker/null) ===')
d4=d.copy(); d4['dXs_lead']=d4['doors_lead1']*d4['smallW']
run('log_u5mort',d4,['doors_lead1','smallW','dXs_lead','rents_l1','aid_l1','log_gdppc'],'R3')

print('=== R4: which door? component x smallW on u5mort ===')
for c in ['trade_open','mobile','fdi_pc','remit_pc','mig_share']:
    d5=d.copy(); d5['comp']=d5[c+'_z_l1']; d5['cXs']=d5['comp']*d5['smallW_l1']
    run('log_u5mort',d5,['comp','smallW_l1','cXs','rents_l1','aid_l1','log_gdppc'],f'R4-{c}')
