Datasets
Models
Downloads: 1
148 lines (148 with data), 14.6 kB
3
ZSöX│R Ń @ s╚ d dl mZ d dlmZ d dlZd dlZd dlm Z d dl
Z d dl m
Z
d dlZd dlZd dlmZ d dlmZ d dlmZ d d lmZ d d
lmZ d Zd Zd gZ d
Z dZ d Z dZ dZ!G ddä dâZ"dS )Ú )┌ OrderedDict)┌feature_selectionN)┌stats)┌partial)┌ CONSTANTS)┌PyConfigParser)┌ReadersWriters)┌FactoringThread)┌TransformThreadzMohsen Mesgarpourz-Copyright 2016, https://github.com/mesgarpour┌GPLz1.xzmohsen.mesgarpour@gmail.com┌ Developmentc @ sě e Zd Zddä Zd5ddäZddä Zd6d d
äZd d ä Zd
dä Zddä Z d7ddäZ
ddä Z ddä Z d8ddäZ
ddä Zd d ä Zd9d d!äZd:d#d$äZd;d'd(äZd)d*ä Zdeâ fd+d,äZeâ fd-d.äZeâ fd/d0äZd<d2d3äZd4S )=┌
PreProcessc C s, t jtjâ| _| jjtâ || _tâ | _ d S )N)
┌logging┌ getLoggerr ┌app_name┌_PreProcess__logger┌debug┌__name__┌_PreProcess__output_pathr ┌ _PreProcess__readers_writers)┌self┌ output_pathę r ˙PC:\Users\eagle\Documents\GitHub\UoW_Docobo\IntegratedCare_py\Stats\PreProcess.py┌__init__ s zPreProcess.__init__r c C s | j jtâ d }d }||| |k jjâ }xr|D ]j} tj|| â}
tj|| j| â} | j| |
â}
| j| | â} |d k rÇ|j |
ân|
}|d k rľ|j | ân| }q0W |j
â }|j
â }||fS )N) r r r ┌index┌tolistr ┌itemfreq┌loc┌ _PreProcess__stats_odds_ratio┌append┌ reset_index) r ┌df┌includes┌ df_target┌target┌
target_cutoff┌ summaries┌label_summaries┌indices┌f_name┌freq┌
label_freqr r r ┌stats_odds_ratio# s
zPreProcess.stats_odds_ratioc sR ç fddä|D â}t j|dddddgdŹ}|d jd â|d< |d jd
â|d< |S ) Nc sB g | ]:}ł |d |d |d t tjâ |d t tjâ d gĹqS )r Ú Úd )┌lenr" r )┌.0┌row)r* r r ˙
<listcomp>8 s z1PreProcess.__stats_odds_ratio.<locals>.<listcomp>┌ Feature_NameZState┌Odds┌
Odds_Ratio┌ Odds_Perc)┌columns┌i4┌i8)┌pd┌ DataFrame┌astype)r r* r+ r )r* r Z__stats_odds_ratio7 s
z PreProcess.__stats_odds_ratior. c C sÓ | j jtâ d }d }x▓|D ]¬} d|j|| |k| f< t|| || dk â}
t||| |k|| dk@ | â} | j| |
t|jââ}
| j| | t|jââ} |d k r¬|j|
ân|
}|d k r└|j| ân| }qW |jâ }|jâ }||fS )Nr. ) r r r ┌ixr0 ┌)_PreProcess__stats_odds_ratio_conditionalr r r! ) r r" r# r$ r% r& Z cond_cutoffr' r( r* r+ r, r r r ┌ stats_odds_ratio_conditional> s
$z'PreProcess.stats_odds_ratio_conditionalc C sR |dkr|| nd}||||d gg}t j|ddddgdŹ}|d jdâ|d< |S ) Nr r/ r4 r5 r6 r7 )r8 r: )r; r< r= )r r* r+ ┌lengthZ
odds_ratior r r Z __stats_odds_ratio_conditionalS s
z)PreProcess.__stats_odds_ratio_conditionalc C s░ | j jtâ d }| jj||g ddŹ xć|D ]~}||kr*| jj||d|gddŹ tj|| â}tj|d d ůdf |d d ůdf dťâ}|j ddd Ź}| jj|||ddd
Ź q*W |S ) NF)┌path┌title┌datar z Feature NameTr r. )┌valuer+ r+ )┌ ascending)rB rC rD r ┌header)
r r r r ┌save_csvr r r; r< ┌ sort_values)r r" r# r ┌ file_namer' r* r r r ┌stats_discrete_dfZ s*
( z PreProcess.stats_discrete_dfc C sľ | j jtâ d }| jj||g ddŹ xl|D ]d}||kr*| jj||d|gddŹ || jdddgdŹjâ }tjj |âjâ }| jj|||ddd Ź q*W |S )
NF)rB rC rD r z Feature NameTg đ?g Ó?g Ŕ?)┌ percentiles)rB rC rD r rG )
r r r r rH ┌describe┌ transposer; ┌Series┌to_frame)r r" r# r rJ r' r* r r r ┌stats_continuous_dfr s(
z PreProcess.stats_continuous_dfFc C sď | j jtâ | j jdâ t|â}|dk r8| j|||â}n| j|||â}g }x$|jâ D ]}|t|| jâ â7 }qTW |j j
â |ľ} t j | dgdŹj } x|D ]
}
| |
= qľW |j
|ddŹ}t j|g| ddŹ}|j| â}|S )N┌EncodingTr )r r. )┌axis)r r r ┌infor ┌(_PreProcess__factoring_group_wise_series┌*_PreProcess__factoring_group_wise_threaded┌keys┌list┌dtypes┌to_dictr; r< ┌drop┌concatr= ) r r" ┌categories_dic┌
labels_dic┌
dtypes_dic┌threaded┌pool_df_encoded┌labels_encoded┌ label_group┌
dtype_orig┌labelr r r ┌factoring_group_wiseł s"
z PreProcess.factoring_group_wisec C s~ t |||â}g }y&x |jâ D ]}|j|j|ââ q W W nB tk
rx } z&| jjtd t|â â t j
â W Y d d }~X nX |S )Nz - Invalid configuration(s): ) r rW r ┌factor_arr_group┌
ValueErrorr ┌errorr ┌str┌sys┌exit)r r" r] r^ ┌factoring_threadra rc ┌ exceptionr r r Z __factoring_group_wise_seriesú s z(PreProcess.__factoring_group_wise_seriesc C sÄ t |||â}y:tjtjâ d dŹĆ }|jt|jâ|jâ â}W d Q R X W nB tk
rł } z&| j j
t d t |â â t
jâ W Y d d }~X nX |S )Nr. )┌ processesz - Invalid configuration(s): )r ┌mp┌Pool┌ cpu_count┌mapr rg rW rh r ri r rj rk rl )r r" r] r^ rm ┌poolra rn r r r Z __factoring_group_wise_threaded» s z*PreProcess.__factoring_group_wise_threadedc
C s | j jtâ | j jdâ t|â}|dk r8| j|||â}n| j|||â}t|jâ â}|j j
â |ľ}t j |dgdŹj }x|D ]
} || = qxW |j
|ddŹ}t j|g| ddŹ}|j|â}|S )NrR Tr )r r. )rS )r r r rT r ┌*_PreProcess__factoring_feature_wise_series┌,_PreProcess__factoring_feature_wise_threadedrX rW rY rZ r; r< r[ r\ r= )
r r" r] r^ r_ r` ra rb rd re r r r ┌factoring_feature_wise║ s
z!PreProcess.factoring_feature_wisec C s~ t |||â}g }y&x |jâ D ]}|j|j|ââ q W W nB tk
rx } z&| jjtd t|â â t j
â W Y d d }~X nX |S )Nz - Invalid configuration(s): ) r rW r ┌
factor_arrrh r ri r rj rk rl )r r" r] r^ rm ra rc rn r r r Z __factoring_feature_wise_seriesË s z*PreProcess.__factoring_feature_wise_seriesc C sé t |||â}y.tjâ Ć }|jt|jâ|jâ â}W d Q R X W nB tk
r| } z&| jj t
d t |â â t j
â W Y d d }~X nX |S )Nz - Invalid configuration(s): )r rp rq rs r rx rW rh r ri r rj rk rl )r r" r] r^ rm rt ra rn r r r Z!__factoring_feature_wise_threaded▀ s
z,PreProcess.__factoring_feature_wise_threadedšffffffţ?Tc
sĺ | j jtâ | j jdâ d }|ł }tł âë g }tâ } |dkÉr |ç fddä|jD â jddŹ}x║|jjD ]«}
t |t
||
â|k j â} t | âdkrjy| j
|
â W n toČtk
r║ Y nX tj|| â}x"| D ]} | | jâ kr╬| j
| â q╬W t | âdkrj| | |
< td |
d
t| â â qjW | j|| || j| d â}xł D ]}
||
||
< Éq<W ttj|j ââÉrp|jdd Ź}|dkÉrŐ|| d
< || d< || fS )Nz/Finding high linear correlation (if applicable)Tc s g | ]}|ł kr|ĹqS r r )r1 ┌col)┌excludesr r r3 § s z9PreProcess.high_linear_correlation_df.<locals>.<listcomp>┌pearson)┌methodr. r zHigh Linear Correlation: z ~ z.ini)r[ zFeatures MatcheszCorrelation Matrix)r r r rT ┌setr r8 ┌corr┌valuesrX ┌absr r0 ┌removerh ┌AttributeError┌np┌union1drW ┌printrj ┌_PreProcess__remover ┌any┌isnanr! )r r" r{ rJ Zthresh_corr_cut┌ to_searchr ┌ df_excludes┌matchesr' re Z matches_temp┌match┌namer )r{ r ┌high_linear_correlation_dfŕ s@
z%PreProcess.high_linear_correlation_dfšÜÖÖÖÖÖę?c s | j jtâ | j jdâ || }t|â}g }tâ ë tâ }x"|jjâ D ]} | ł |jj| â< qDW |dkrČt j
|â}
|
j ddŹ} ç fddä| D â} x | D ]}
|
|krĺ||
g7 }qĺW | j |dt
|âi|| j| d â}x|D ]}|| ||< qďW ttj|jââÉr|jddŹ}|dkÉr||d < ||fS )
Nz*Finding near zero variance (if applicable)T)r) c s g | ] }ł | ĹqS r r )r1 r )r) r r r3 & s z7PreProcess.near_zero_var_df_sklearn.<locals>.<listcomp>┌NZVz.ini)r[ zFeatures Matches)r r r rT r~ r r8 rÇ ┌get_locr ┌VarianceThreshold┌ get_supportrç rX r rł rä rë r r! )r r" r{ rJ Zthresh_variancerŐ rő rî r' re ┌
variances_Zmatches_indicesZmatches_labelsrŹ rÄ r )r) r ┌near_zero_var_df_sklearn s0
"
z#PreProcess.near_zero_var_df_sklearnr/ ÚŔ c
C s. | j jtâ | j jdâ || }t|â}g }tâ } |dkr║x~|jjD ]r}
t||
d t âsnt
j t
j ||
âârz||
g7 }qD| j
||
|
|||â\} | |
< | dkrD||
g7 }td|
â qDW | j|dt|âi|| j| d â}x|D ]} || || < qÔW tt
j |jââÉr|jddŹ}|dkÉr&|| d< || fS ) Nz*Finding near zero variance (if applicable)Tr zNear Zero Variance: rĹ z.ini)r[ zFeatures Matches)r r r rT r~ r r8 rÇ ┌
isinstancerj rä rë ┌sum┌
near_zero_varrć rç rX r rł r r! )
r r" r{ rJ ┌thresh_unique_cut┌thresh_freq_cutrŐ rő rî r' re rŹ rÄ r r r ┌near_zero_var_df7 s0 &
"
zPreProcess.near_zero_var_dfc C s╩ t j|ddŹ\}}t|âdkr4dt|ât|âdťfS t|ddŹ}||krzt|âd tt|ââ |krzdt|ât|âdťfS ||kr░|d t|d â |kr░dt|ât|âdťfS dt|ât|âdťfS d S ) NT)┌
return_countsr. )┌unique┌counts)┌reverser/ r F)rä rč r0 rX ┌sorted┌float)r ┌arrre r{ rŤ rť rč rá r r r rÜ Z s $ zPreProcess.near_zero_varc
s▓ | j jtâ | j jdâ tł âë ç fddä|jjD â}|jjâ }x|D ] } d|| < qFW t j
|dgdŹj}|j |â}|dkrĺ| j |||f|Ä\}}n| j
||||f|Ä\}}||fS )NzRunning transformc s g | ]}|ł kr|ĹqS r r )r1 re )r{ r r r3 k s z+PreProcess.transform_df.<locals>.<listcomp>┌f8r )r F)r r r rT r~ r8 rÇ rY rZ r; r< r= ┌ _PreProcess__transform_df_series┌"_PreProcess__transform_df_threaded)
r r" r{ ┌transform_typer` ┌ method_args┌kwargsr# rd re r )r{ r ┌ transform_dfg s
zPreProcess.transform_dfc K st t f |Ä}Éy|dkr4x|D ]}|j|||â q W n|dkrZxŠ|D ]}|j|||â qBW n╩|dkrÇx└|D ]}|j|||â qhW nĄ|dkrŽxÜ|D ]}|j|||â qÄW n~|dkr╠xt|D ]}|j|||â q┤W nX|dkr˛xN|D ]}|j|||â q┌W n2|dkÉr x&|D ]}|j|||â ÉqW nt|âéW nD t k
Érj } z&| j
j t d t
|â â tjâ W Y d d }~X nX ||fS ) N┌scale┌ robust_scale┌max_abs_scalar┌
normalizer┌kernel_centerer┌ yeo_johnson┌box_coxz - Invalid configuration(s): )r
┌transform_scale_arr┌transform_robust_scale_arr┌ transform_max_abs_scalar_arr┌transform_normalizer_arr┌ transform_kernel_centerer_arr┌transform_yeo_johnson_arr┌transform_box_cox_arr┌ Exceptionrh r ri r rj rk rl ) r r" r# rĘ rę r¬ ┌transform_threadrÄ rn r r r Z__transform_df_series{ s8
z PreProcess.__transform_df_seriesc
K sŞ t jâ }|jtt|| j|| jjjâ âââ}t f |Ä}Éyt j
t j â d dŹĆ˛} |dkrr| j t
|j||â|â n╠|dkrĺ| j t
|j||â|â nČ|dkr▓| j t
|j||â|â nî|dkrĎ| j t
|j||â|â nl|dkr˛| j t
|j||â|â nL|dkÉr| j t
|j||â|â n*|d kÉr6| j t
|j||â|â nt|âéW d Q R X W nD tk
ÉrÄ }
z&| jjtd
t|
â â tj â W Y d d }
~
X nX x |j â D ]\} } | || < ÉqÜW ||fS ) Nr. )ro rČ rş r« r» r░ r▒ r▓ z - Invalid configuration(s): ) rp ┌Manager┌dictrX ┌zipr8 ┌TrÇ r r
rq rr rs r r│ r┤ rÁ r rĚ rŞ r╣ r║ rh r ri r rj rk rl ┌items)
r r" r# rĘ rę r¬ ┌manager┌dtr╗ rt rn ┌k┌vr r r Z__transform_df_threadedť s4 &
z"PreProcess.__transform_df_threaded┌featuresc sĘ t |tjâ}|dkrL|jâ |j||â | jjd| â}|dkrL|jâ ł S |jâ |j|â}| j j
ddj |â â ç fddä|j â D â}t
|âdkrĄł j|d d
Źë ł S ) NTz:the features defined in the following file to be removed: Fz
Removing: ˙,c s" g | ]}|D ]}|ł kr |Ĺq qS r r )r1 rc re )r" r r r3 Ë s z'PreProcess.__remove.<locals>.<listcomp>r r. )rS )r r r ┌reset┌
write_dictr ┌question_overwrite┌refresh┌ read_dictr r ┌joinrÇ r0 r[ ) r r" Z dict_matchesrŐ rB ┌section┌config┌response┌labelsr )r" r Z__removeż s
zPreProcess.__removeN)r )r. r. )F)F)ry T)rÉ T)r/ rŚ T)r┼ )r ┌
__module__┌ __qualname__r r- r r@ r? rK rQ rf rU rV rw ru rv rĆ rľ rŁ rÜ rŻ rź rŽ rž rç r r r r r
s*
+
"
#
!"r
)#┌ collectionsr ┌sklearnr ┌numpyrä ┌pandasr; ┌ scipy.statsr ┌multiprocessingrp ┌ functoolsr rk r ┌Configs.CONSTANTSr ┌ ReadersWrites.PyConfigParserr ┌ ReadersWrites.ReadersWritersr ZStats.FactoringThreadr ZStats.TransformThreadr
┌
__author__┌
__copyright__┌ __credits__┌ __license__┌ __version__┌__maintainer__┌ __email__┌
__status__r
r r r r ┌<module> s,