from jqdata import *
from jqfactor import winsorize_med
from jqfactor import standardlize
import pandas as pd
import numpy as np
from scipy.stats import mstats
from scipy import stats
import datetime as dt
from pandas import Series
from pandas import DataFrame
import matplotlib as mpl
import matplotlib.pyplot as plt
from jqfactor import get_factor_values
import statsmodels.api as sm
import statsmodels.tsa.stattools as ts
s = get_industries(name="sw_l1")

#因子共线性
trading_dates = get_trade_days('2014-04-01','2018-10-11')
num = int(len(trading_dates)/5)
for i in range(1):
    trdy = trading_dates[i*5]
    t1 = get_trade_days(trdy-dt.timedelta(30),trdy)[0]
    t2 = get_trade_days(trdy-dt.timedelta(180),trdy)[0]
    trdy0 = get_trade_days(trdy,trdy+dt.timedelta(7))[-1]

    stocks = get_index_stocks('000300.XSHG')
    prr = get_price(stocks, start_date=t1, end_date=trdy, frequency='daily', fields='close', skip_paused=False, fq='pre')
    pr = (prr.major_xs(trdy)-prr.major_xs(t1))/prr.major_xs(t1)
    prr = get_price(stocks, start_date=t2, end_date=trdy, frequency='daily', fields='close', skip_paused=False, fq='pre')
    pr['long'] =  (prr.major_xs(trdy)-prr.major_xs(t2))/prr.major_xs(t2)
    pr.rename(columns={'close':'short'}, inplace = True)

    dend = get_fundamentals(query(
              valuation.code, 
        valuation.market_cap,
        valuation.pb_ratio,
        valuation.pe_ratio,
        valuation.pcf_ratio,
          ).filter(
              # 这里不能使用 in 操作, 要使用in_()函数
              valuation.code.in_(stocks)
          ), date=trdy)
    dend = pd.merge(dend,pr,left_on='code',right_index=True)
    dend.index = dend['code']
    del dend['code']
    dend['market_cap'] = np.log(dend['market_cap'])

    list = ['operating_revenue_growth_rate','net_profit_growth_rate',
            'Variance20','Variance60','Variance120',
            'VOL20','VOL60','VOL120']
    dend1 = get_factor_values(securities=stocks, factors=list, start_date=trdy, end_date=trdy)
    for i in list:
        dend3 = dend1[i].T
        dend3.columns = [i]
        dend = pd.merge(dend,dend3,left_index=True,right_index=True)

    for i in dend.columns:
        md = dend[i].median()
        mad = ((dend[i] - md).abs()).median()
        minn = md-mad
        maxx = md+mad
        dend[i][dend[i]>maxx] = maxx
        dend[i][dend[i]<minn] = minn
        dend[i] = (dend[i] - dend[i].mean())/dend[i].std()


    dend['val'] = (dend['pb_ratio'] + dend['pe_ratio'] +dend['pcf_ratio'])/3
    dend['vol'] = (dend['Variance20']+dend['Variance60']+dend['Variance120'])/3
    dend['liquid'] = (dend['VOL20']+dend['VOL60']+dend['VOL120'])/3
    dend['grow'] = (dend['operating_revenue_growth_rate']+dend['net_profit_growth_rate'])/2
    del dend['pb_ratio']
    del dend['pe_ratio']
    del dend['pcf_ratio']
    del dend['Variance20']
    del dend['Variance60']
    del dend['Variance120']
    del dend['VOL20']
    del dend['VOL60']
    del dend['VOL120']
    del dend['operating_revenue_growth_rate']
    del dend['net_profit_growth_rate']

    dendf = get_fundamentals(query(
              valuation.code, 
        valuation.market_cap,
          ).filter(
              # 这里不能使用 in 操作, 要使用in_()函数
              valuation.code.in_(stocks)
          ), date=trdy)
    dendf['weight'] = np.sqrt(dendf['market_cap'])
    dendf.index = dendf['code']
    del dendf['code']
    del dendf['market_cap']
    dend = pd.merge(dend,dendf,left_index=True,right_index=True)
    price = get_price(stocks, start_date=trdy, end_date=trdy0, frequency='daily', fields='close', skip_paused=False, fq='pre')
    price1 = price.major_xs(trdy)
    price2 = price.major_xs(trdy0)
    retu = (price2 - price1)/price1
    retu.rename(columns={'close':'retu'}, inplace = True)
    dend = pd.merge(dend,retu,left_index=True,right_index=True)

    dend['market_cap'] = dend['market_cap']*dend['weight']
    dend['short'] = dend['weight']*dend['short'] 
    dend['long'] = dend['weight']*dend['long'] 
    dend['val'] = dend['weight']*dend['val'] 
    dend['vol'] = dend['weight']*dend['vol'] 
    dend['grow'] = dend['weight']*dend['grow'] 
    dend['liquid'] = dend['weight']*dend['liquid'] 
    dend['retu'] = dend['weight']*dend['retu'] 
    dec = dend.corr()
    

for i in range(1,num):
    trdy = trading_dates[i*5]
    t1 = get_trade_days(trdy-dt.timedelta(30),trdy)[0]
    t2 = get_trade_days(trdy-dt.timedelta(180),trdy)[0]
    trdy0 = get_trade_days(trdy,trdy+dt.timedelta(7))[-1]

    stocks = get_index_stocks('000300.XSHG')
    prr = get_price(stocks, start_date=t1, end_date=trdy, frequency='daily', fields='close', skip_paused=False, fq='pre')
    pr = (prr.major_xs(trdy)-prr.major_xs(t1))/prr.major_xs(t1)
    prr = get_price(stocks, start_date=t2, end_date=trdy, frequency='daily', fields='close', skip_paused=False, fq='pre')
    pr['long'] =  (prr.major_xs(trdy)-prr.major_xs(t2))/prr.major_xs(t2)
    pr.rename(columns={'close':'short'}, inplace = True)

    dend = get_fundamentals(query(
              valuation.code, 
        valuation.market_cap,
        valuation.pb_ratio,
        valuation.pe_ratio,
        valuation.pcf_ratio,
          ).filter(
              # 这里不能使用 in 操作, 要使用in_()函数
              valuation.code.in_(stocks)
          ), date=trdy)
    dend = pd.merge(dend,pr,left_on='code',right_index=True)
    dend.index = dend['code']
    del dend['code']
    dend['market_cap'] = np.log(dend['market_cap'])

    list = ['operating_revenue_growth_rate','net_profit_growth_rate',
            'Variance20','Variance60','Variance120',
            'VOL20','VOL60','VOL120']
    dend1 = get_factor_values(securities=stocks, factors=list, start_date=trdy, end_date=trdy)
    for i in list:
        dend3 = dend1[i].T
        dend3.columns = [i]
        dend = pd.merge(dend,dend3,left_index=True,right_index=True)

    for i in dend.columns:
        md = dend[i].median()
        mad = ((dend[i] - md).abs()).median()
        minn = md-mad
        maxx = md+mad
        dend[i][dend[i]>maxx] = maxx
        dend[i][dend[i]<minn] = minn
        dend[i] = (dend[i] - dend[i].mean())/dend[i].std()


    dend['val'] = (dend['pb_ratio'] + dend['pe_ratio'] +dend['pcf_ratio'])/3
    dend['vol'] = (dend['Variance20']+dend['Variance60']+dend['Variance120'])/3
    dend['liquid'] = (dend['VOL20']+dend['VOL60']+dend['VOL120'])/3
    dend['grow'] = (dend['operating_revenue_growth_rate']+dend['net_profit_growth_rate'])/2
    del dend['pb_ratio']
    del dend['pe_ratio']
    del dend['pcf_ratio']
    del dend['Variance20']
    del dend['Variance60']
    del dend['Variance120']
    del dend['VOL20']
    del dend['VOL60']
    del dend['VOL120']
    del dend['operating_revenue_growth_rate']
    del dend['net_profit_growth_rate']

    dendf = get_fundamentals(query(
              valuation.code, 
        valuation.market_cap,
          ).filter(
              # 这里不能使用 in 操作, 要使用in_()函数
              valuation.code.in_(stocks)
          ), date=trdy)
    dendf['weight'] = np.sqrt(dendf['market_cap'])
    dendf.index = dendf['code']
    del dendf['code']
    del dendf['market_cap']
    dend = pd.merge(dend,dendf,left_index=True,right_index=True)
    price = get_price(stocks, start_date=trdy, end_date=trdy0, frequency='daily', fields='close', skip_paused=False, fq='pre')
    price1 = price.major_xs(trdy)
    price2 = price.major_xs(trdy0)
    retu = (price2 - price1)/price1
    retu.rename(columns={'close':'retu'}, inplace = True)
    dend = pd.merge(dend,retu,left_index=True,right_index=True)

    dend['market_cap'] = dend['market_cap']*dend['weight']
    dend['short'] = dend['weight']*dend['short'] 
    dend['long'] = dend['weight']*dend['long'] 
    dend['val'] = dend['weight']*dend['val'] 
    dend['vol'] = dend['weight']*dend['vol'] 
    dend['grow'] = dend['weight']*dend['grow'] 
    dend['liquid'] = dend['weight']*dend['liquid'] 
    dend['retu'] = dend['weight']*dend['retu'] 
    dec = dec + dend.corr()
dec = dec/num

for i in range(1):
    trdy = trading_dates[i*5]
    t1 = get_trade_days(trdy-dt.timedelta(30),trdy)[0]
    t2 = get_trade_days(trdy-dt.timedelta(180),trdy)[0]
    trdy0 = get_trade_days(trdy,trdy+dt.timedelta(7))[-1]

    stocks = get_index_stocks('000300.XSHG')
    prr = get_price(stocks, start_date=t1, end_date=trdy, frequency='daily', fields='close', skip_paused=False, fq='pre')
    pr = (prr.major_xs(trdy)-prr.major_xs(t1))/prr.major_xs(t1)
    prr = get_price(stocks, start_date=t2, end_date=trdy, frequency='daily', fields='close', skip_paused=False, fq='pre')
    pr['long'] =  (prr.major_xs(trdy)-prr.major_xs(t2))/prr.major_xs(t2)
    pr.rename(columns={'close':'short'}, inplace = True)

    dend = get_fundamentals(query(
              valuation.code, 
        valuation.market_cap,
        valuation.pb_ratio,
        valuation.pe_ratio,
        valuation.pcf_ratio,
          ).filter(
              # 这里不能使用 in 操作, 要使用in_()函数
              valuation.code.in_(stocks)
          ), date=trdy)
    dend = pd.merge(dend,pr,left_on='code',right_index=True)
    dend.index = dend['code']
    del dend['code']
    dend['market_cap'] = np.log(dend['market_cap'])

    list = ['operating_revenue_growth_rate','net_profit_growth_rate',
            'Variance20','Variance60','Variance120',
            'VOL20','VOL60','VOL120']
    dend1 = get_factor_values(securities=stocks, factors=list, start_date=trdy, end_date=trdy)
    for i in list:
        dend3 = dend1[i].T
        dend3.columns = [i]
        dend = pd.merge(dend,dend3,left_index=True,right_index=True)

    for i in dend.columns:
        md = dend[i].median()
        mad = ((dend[i] - md).abs()).median()
        minn = md-mad
        maxx = md+mad
        dend[i][dend[i]>maxx] = maxx
        dend[i][dend[i]<minn] = minn
        dend[i] = (dend[i] - dend[i].mean())/dend[i].std()


    dend['val'] = (dend['pb_ratio'] + dend['pe_ratio'] +dend['pcf_ratio'])/3
    dend['vol'] = (dend['Variance20']+dend['Variance60']+dend['Variance120'])/3
    dend['liquid'] = (dend['VOL20']+dend['VOL60']+dend['VOL120'])/3
    dend['grow'] = (dend['operating_revenue_growth_rate']+dend['net_profit_growth_rate'])/2
    del dend['pb_ratio']
    del dend['pe_ratio']
    del dend['pcf_ratio']
    del dend['Variance20']
    del dend['Variance60']
    del dend['Variance120']
    del dend['VOL20']
    del dend['VOL60']
    del dend['VOL120']
    del dend['operating_revenue_growth_rate']
    del dend['net_profit_growth_rate']

    dendf = get_fundamentals(query(
              valuation.code, 
        valuation.market_cap,
          ).filter(
              # 这里不能使用 in 操作, 要使用in_()函数
              valuation.code.in_(stocks)
          ), date=trdy)
    dendf['weight'] = np.sqrt(dendf['market_cap'])
    dendf.index = dendf['code']
    del dendf['code']
    del dendf['market_cap']
    dend = pd.merge(dend,dendf,left_index=True,right_index=True)
    price = get_price(stocks, start_date=trdy, end_date=trdy0, frequency='daily', fields='close', skip_paused=False, fq='pre')
    price1 = price.major_xs(trdy)
    price2 = price.major_xs(trdy0)
    retu = (price2 - price1)/price1
    retu.rename(columns={'close':'retu'}, inplace = True)
    dend = pd.merge(dend,retu,left_index=True,right_index=True)

    dend['market_cap'] = dend['market_cap']*dend['weight']
    dend['short'] = dend['weight']*dend['short'] 
    dend['long'] = dend['weight']*dend['long'] 
    dend['val'] = dend['weight']*dend['val'] 
    dend['vol'] = dend['weight']*dend['vol'] 
    dend['grow'] = dend['weight']*dend['grow'] 
    dend['liquid'] = dend['weight']*dend['liquid'] 
    dend['retu'] = dend['weight']*dend['retu'] 



    dfn = pd.DataFrame()
    for i in s['name']:
        dend[i] = 0
        dfn[i] = dend[i]
        del dend[i]
    dfc = dfn.copy()
    for i in range(len(dfc.index)):
        n = get_industry(security=[dfc.index[i]], date="2018-10-01")
        dfc.iloc[i,:][dfc.columns == n[dfc.index[i]]["sw_l1"]['industry_name']] = 1

    dend = pd.merge(dend,dfc,left_index=True,right_index=True)
    dend = dend.fillna(value=0)
    x =dend[['market_cap','short','grow','vol','val','农林牧渔I',
          '采掘I',
    '化工I',
       '钢铁I',
     '有色金属I',
         '电子I',
       '家用电器I',
        '食品饮料I',
        '纺织服装I',
       '轻工制造I',
       '医药生物I',
      '公用事业I',
       '交通运输I',
       '房地产I',
        '商业贸易I',
      '休闲服务I',
          '综合I',
       '建筑材料I',
       '建筑装饰I',
       '电气设备I',
       '国防军工I',
        '计算机I',
         '传媒I',
          '通信I',
         '银行I',
        '非银金融I',
          '汽车I',
        '机械设备I']]
    y = dend['retu'] 
    x=sm.add_constant(x)
    est=sm.OLS(y,x).fit()
    res = est.summary()
    des = est.pvalues

trading_dates = get_trade_days('2014-04-01','2018-10-11')
num = int(len(trading_dates)/5)
for i in range(0,num):
    trdy = trading_dates[i*5]
    t1 = get_trade_days(trdy-dt.timedelta(30),trdy)[0]
    t2 = get_trade_days(trdy-dt.timedelta(180),trdy)[0]
    trdy0 = get_trade_days(trdy,trdy+dt.timedelta(7))[-1]

    stocks = get_index_stocks('000300.XSHG')
    prr = get_price(stocks, start_date=t1, end_date=trdy, frequency='daily', fields='close', skip_paused=False, fq='pre')
    pr = (prr.major_xs(trdy)-prr.major_xs(t1))/prr.major_xs(t1)
    prr = get_price(stocks, start_date=t2, end_date=trdy, frequency='daily', fields='close', skip_paused=False, fq='pre')
    pr['long'] =  (prr.major_xs(trdy)-prr.major_xs(t2))/prr.major_xs(t2)
    pr.rename(columns={'close':'short'}, inplace = True)

    dend = get_fundamentals(query(
              valuation.code, 
        valuation.market_cap,
        valuation.pb_ratio,
        valuation.pe_ratio,
        valuation.pcf_ratio,
          ).filter(
              # 这里不能使用 in 操作, 要使用in_()函数
              valuation.code.in_(stocks)
          ), date=trdy)
    dend = pd.merge(dend,pr,left_on='code',right_index=True)
    dend.index = dend['code']
    del dend['code']
    dend['market_cap'] = np.log(dend['market_cap'])

    list = ['operating_revenue_growth_rate','net_profit_growth_rate',
            'Variance20','Variance60','Variance120',
            'VOL20','VOL60','VOL120']
    dend1 = get_factor_values(securities=stocks, factors=list, start_date=trdy, end_date=trdy)
    for i in list:
        dend3 = dend1[i].T
        dend3.columns = [i]
        dend = pd.merge(dend,dend3,left_index=True,right_index=True)

    for i in dend.columns:
        md = dend[i].median()
        mad = ((dend[i] - md).abs()).median()
        minn = md-mad
        maxx = md+mad
        dend[i][dend[i]>maxx] = maxx
        dend[i][dend[i]<minn] = minn
        dend[i] = (dend[i] - dend[i].mean())/dend[i].std()


    dend['val'] = (dend['pb_ratio'] + dend['pe_ratio'] +dend['pcf_ratio'])/3
    dend['vol'] = (dend['Variance20']+dend['Variance60']+dend['Variance120'])/3
    dend['liquid'] = (dend['VOL20']+dend['VOL60']+dend['VOL120'])/3
    dend['grow'] = (dend['operating_revenue_growth_rate']+dend['net_profit_growth_rate'])/2
    del dend['pb_ratio']
    del dend['pe_ratio']
    del dend['pcf_ratio']
    del dend['Variance20']
    del dend['Variance60']
    del dend['Variance120']
    del dend['VOL20']
    del dend['VOL60']
    del dend['VOL120']
    del dend['operating_revenue_growth_rate']
    del dend['net_profit_growth_rate']

    dendf = get_fundamentals(query(
              valuation.code, 
        valuation.market_cap,
          ).filter(
              # 这里不能使用 in 操作, 要使用in_()函数
              valuation.code.in_(stocks)
          ), date=trdy)
    dendf['weight'] = np.sqrt(dendf['market_cap'])
    dendf.index = dendf['code']
    del dendf['code']
    del dendf['market_cap']
    dend = pd.merge(dend,dendf,left_index=True,right_index=True)
    price = get_price(stocks, start_date=trdy, end_date=trdy0, frequency='daily', fields='close', skip_paused=False, fq='pre')
    price1 = price.major_xs(trdy)
    price2 = price.major_xs(trdy0)
    retu = (price2 - price1)/price1
    retu.rename(columns={'close':'retu'}, inplace = True)
    dend = pd.merge(dend,retu,left_index=True,right_index=True)

    dend['market_cap'] = dend['market_cap']*dend['weight']
    dend['short'] = dend['weight']*dend['short'] 
    dend['long'] = dend['weight']*dend['long'] 
    dend['val'] = dend['weight']*dend['val'] 
    dend['vol'] = dend['weight']*dend['vol'] 
    dend['grow'] = dend['weight']*dend['grow'] 
    dend['liquid'] = dend['weight']*dend['liquid'] 
    dend['retu'] = dend['weight']*dend['retu'] 



    dfn = pd.DataFrame()
    for i in s['name']:
        dend[i] = 0
        dfn[i] = dend[i]
        del dend[i]
    dfc = dfn.copy()
    for i in range(len(dfc.index)):
        n = get_industry(security=[dfc.index[i]], date="2018-10-01")
        dfc.iloc[i,:][dfc.columns == n[dfc.index[i]]["sw_l1"]['industry_name']] = 1

    dend = pd.merge(dend,dfc,left_index=True,right_index=True)
    dend = dend.fillna(value=0)
    x =dend[['market_cap','short','grow','vol','val','农林牧渔I',
          '采掘I',
    '化工I',
       '钢铁I',
     '有色金属I',
         '电子I',
       '家用电器I',
        '食品饮料I',
        '纺织服装I',
       '轻工制造I',
       '医药生物I',
      '公用事业I',
       '交通运输I',
       '房地产I',
        '商业贸易I',
      '休闲服务I',
          '综合I',
       '建筑材料I',
       '建筑装饰I',
       '电气设备I',
       '国防军工I',
        '计算机I',
         '传媒I',
          '通信I',
         '银行I',
        '非银金融I',
          '汽车I',
        '机械设备I']]
    y = dend['retu'] 
    x=sm.add_constant(x)
    est=sm.OLS(y,x).fit()
    res = est.summary()
    des = des + est.pvalues

des =des/num
des