Datasets
Models
Downloads: 1
363 lines (357 with data), 44.6 kB
B
%Yfôà ã
@ s d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl
m Z ddl m
Z
mZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddlm mZ ddl Z ddl Z ddl m Z m!Z!m"Z" dd l#m$Z$m%Z%m&Z&m'Z' dd
l(m(Z( dd l)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 dd l2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZC dd
lDmEZE dZFdZGdZHeIe Jdd¡ZKeIe Jdd¡ZLeIe Jdd¡ZMeNe Jdd¡ O¡ dkZPx&e jQ R¡ D ]ZSe jQeS dkrÞP qÞW dd ZTd d ZUd d ZVd d! ZWG d"d# d#e'jXZYdKd(d)ZZG d*d+ d+e&j$Z[G d,d- d-Z\G d.d/ d/Z]G d0d1 d1Z^G d2d3 d3Z_d4d5 Z`G d6d7 d7e%Zae3d8 fd9d:Zbe3d8 fd;d<Zce3d= d>d$fd?d@ZddLdAdBZeG dCdD dDZfG dEdF dFe jgjhZidMdIdJZjdS )Nz
Dataloaders and dataset utils
é N)Úrepeat)ÚPoolÚ
ThreadPool)ÚPath)ÚThread)Úurlparse)ÚExifTagsÚImageÚImageOps)Ú
DataLoaderÚDatasetÚ
dataloaderÚ distributed)Útqdm)ÚAlbumentationsÚ augment_hsvÚclassify_albumentationsÚclassify_transformsÚ
copy_pasteÚ letterboxÚmixupÚrandom_perspective)Ú DATASETS_DIRÚLOGGERÚ NUM_THREADSÚTQDM_BAR_FORMATÚ
check_datasetÚcheck_requirementsÚ
check_yamlÚ clean_strÚcv2Úis_colabÚ is_kaggleÚsegments2boxesÚ
unzip_fileÚxyn2xyÚ xywh2xyxyÚ
xywhn2xyxyÚ
xyxy2xywhn)Ú torch_distributed_zero_firstzCSee https://docs.ultralytics.com/yolov5/tutorials/train_custom_data)
ÚbmpÚdngÚjpegÚjpgÚmpoÚpngÚtifÚtiffÚwebpÚpfm) ÚasfÚaviÚgifÚm4vÚmkvÚmovÚmp4ÚmpegÚmpgÚtsÚwmvÚ
LOCAL_RANKéÿÿÿÿÚRANKÚ
WORLD_SIZEé Ú
PIN_MEMORYTÚtrueÚ Orientationc C s@ t dd | D }t t| ¡ ¡}| d | ¡ ¡ ¡ | ¡ S )Nc s s& | ] }t j |¡rt j |¡V qd S )N)ÚosÚpathÚexistsÚgetsize)Ú.0Úp© rM úP/home/iml/Desktop/Talha/yolov5_P2_ROI_HCM_Micro_1000_Github/utils/dataloaders.pyú <genexpr>7 s zget_hash.<locals>.<genexpr>Ú )ÚsumÚhashlibÚsha256ÚstrÚencodeÚupdateÚjoinÚ hexdigest)ÚpathsÚsizeÚhrM rM rN Úget_hash5 s r\ c C sL | j }t t¡2 t| ¡ ¡ t }|dkr>|d |d f}W d Q R X |S )N)é é rC r )rZ Ú
contextlibÚsuppressÚ ExceptionÚdictÚ_getexifÚitemsÚ orientation)ÚimgÚsÚrotationrM rM rN Ú exif_size= s ri c C sn | ¡ }| dd¡}|dkrjtjtjtjtjtjtjtj d |¡}|dk rj|
|¡} |d= | ¡ | j d< | S )a
Transpose a PIL image accordingly if it has an EXIF Orientation tag.
Inplace version of https://github.com/python-pillow/Pillow/blob/master/src/PIL/ImageOps.py exif_transpose()
:param image: The image to transpose.
:return: An image.
i rC )é é é é r] é r^ NÚexif)
ÚgetexifÚgetr ÚFLIP_LEFT_RIGHTÚ
ROTATE_180ÚFLIP_TOP_BOTTOMÚ TRANSPOSEÚ
ROTATE_270Ú
TRANSVERSEÚ ROTATE_90Ú transposeÚtobytesÚinfo)Úimagero re ÚmethodrM rM rN Úexif_transposeG s
r~ c C s& t ¡ d }tj |¡ t |¡ d S )Nl )ÚtorchÚ initial_seedÚnpÚrandomÚseed)Ú worker_idÚ worker_seedrM rM rN Ú seed_workera s r c @ s e Zd Zdd ZdS )ÚSmartDistributedSamplerc C sÐ t ¡ }| | j| j ¡ tt| j| j d | j d }t j
||d}| j s\| ¡ d }|
¡ }| jrz|d | j
}nN| jt| }|t|krŠ||d |
7 }n"||t |t| ¡ d |
7 }t|S )NrC )Ú generatorr )r Ú GeneratorÚ manual_seedr ÚepochÚintÚlenÚdatasetÚrankÚ num_replicasÚrandpermÚshuffleÚsortÚtolistÚ drop_lastÚ num_samplesÚmathÚceilÚiter)ÚselfÚgÚnÚidxÚ padding_sizerM rM rN Ú__iter__l s " "z SmartDistributedSampler.__iter__N)Ú__name__Ú
__module__Ú __qualname__r rM rM rM rN r j s r Fç r^ rP c C s | r|rt d¡ d}t|
, t| ||||| ||t||| ||
d
}W d Q R X t|t|}tj ¡ }tt
¡ t |d |dkr|nd| g}|
dkrd n
t
||d}| r°tnt}t ¡ }| d| t ¡ ||||oÞ|d k||t|
rîtjntjt|d |fS )
NuT WARNING â ïž --rect is incompatible with DataLoader shuffle, setting shuffle=FalseF)
ÚaugmentÚhypÚrectÚ cache_imagesÚ
single_clsÚstrideÚpadÚ
image_weightsÚprefixr rC r r@ )r l UUª*UUª* )Ú
batch_sizer Ú num_workersÚsamplerÚ
pin_memoryÚ
collate_fnÚworker_init_fnr )r Úwarningr) ÚLoadImagesAndLabelsr Úminr r ÚcudaÚ device_countrG Ú cpu_countÚmaxr r ÚInfiniteDataLoaderr r rA rD Ú collate_fn4r± r )rH Úimgszr r© rš r¥ r€ Úcacherª rŠ r Úworkersr« Úquadr¬ r r r ÚndÚnwr¯ Úloaderr rM rM rN Úcreate_dataloader sD
(
rà c s0 e Zd ZdZ fddZdd Zdd Z ZS )rº zP Dataloader that reuses workers
Uses same syntax as vanilla DataLoader
c s2 t j|| t | dt| j¡ t ¡ | _d S )NÚ
batch_sampler)ÚsuperÚ__init__ÚobjectÚ __setattr__Ú_RepeatSamplerrÄ r Úiterator)r ÚargsÚkwargs)Ú __class__rM rN rÆ À s zInfiniteDataLoader.__init__c C s t | jjS )N)r rÄ r¯ )r rM rM rN Ú__len__Å s zInfiniteDataLoader.__len__c c s& x t t| D ]}t| jV qW d S )N)Úranger ÚnextrÊ )r Ú_rM rM rN r È s zInfiniteDataLoader.__iter__)r r¡ r¢ Ú__doc__rÆ rÎ r Ú
__classcell__rM rM )rÍ rN rº º s rº c @ s e Zd ZdZdd Zdd ZdS )rÉ zG Sampler that repeats forever
Args:
sampler (Sampler)
c C s
|| _ d S )N)r¯ )r r¯ rM rM rN rÆ Ô s z_RepeatSampler.__init__c c s xt | jE d H qW d S )N)r r¯ )r rM rM rN r × s z_RepeatSampler.__iter__N)r r¡ r¢ rÒ rÆ r rM rM rM rN rÉ Í s rÉ c @ s& e Zd Zd ddZdd Zd d
ZdS ) ÚLoadScreenshotsé é TNc
C sF t d dd l}| ¡ ^}}d\| _}} }
} t|dkrHt|d | _nJt|dkrldd |D \}} }
} n&t|dkrd d |D \| _}} }
} || _|| _|| _|| _ d
| _
d| _ | ¡ | _ | j j
| j } | d krÞ| d n
| d | | _|d krü| d n
| d | | _|
p| d
| _| p&| d | _| j| j| j| jd| _d S )NÚmssr )r NNNNrC rl c s s | ]}t |V qd S )N)r )rK ÚxrM rM rN rO è s z+LoadScreenshots.__init__.<locals>.<genexpr>rm c s s | ]}t |V qd S )N)r )rK rØ rM rM rN rO ê s ÚstreamÚtopÚleftÚwidthÚheight)rÛ rÚ rÜ rÝ )r r× ÚsplitÚscreenr r Úimg_sizer© Ú
transformsÚautoÚmodeÚframeÚsctÚmonitorsrÚ rÛ rÜ rÝ Úmonitor)
r Úsourcerà r© râ rá r× ÚparamsrÛ rÚ rÜ rÝ rç rM rM rN rÆ Þ s.
zLoadScreenshots.__init__c C s | S )NrM )r rM rM rN r û s zLoadScreenshots.__iter__c C sÄ t | j | j¡¡d d
d d
d d
f }d| j d| j d| j d| j d| j d }| j
rj|
|¡}n8t || j | j
| jdd }| d¡d d d
}t |¡}| jd
7 _t| j||d |fS ) Nrk zscreen z (LTWH): ú,z: )r© râ r )rj r rC r@ rC )r Úarrayrå Úgrabrç rß rÛ rÚ rÜ rÝ rá r rà r© râ ry Úascontiguousarrayrä rT )r Úim0rg ÚimrM rM rN Ú__next__þ s *.
zLoadScreenshots.__next__)rÕ rÖ TN)r r¡ r¢ rÆ r rð rM rM rM rN rÔ Ü s
rÔ c @ s> e Zd ZdddZdd Zd
d Zd d
Zdd Zdd ZdS )Ú
LoadImagesé rÖ TNrC c
C s t |tr(t|jdkr(t| ¡ ¡ }g }xšt |ttfrDt|n|gD ]}tt| ¡ }d|kr|
tt j |dd¡ qJt j
|¡r¬|
tt t j
|d¡¡¡ qJt j
|¡rÄ| |¡ qJt| dqJW dd |D } d d |D }
t| t|
} } || _|| _| |
| _| | | _d
g| dg| | _d | _|| _|| _|| _ t |
rl| |
d ¡ nd | _ | jd kst d
| dt! dt" d S )Nz.txtÚ*T)Ú recursivez*.*z does not existc S s& g | ] }| d ¡d ¡ tkr|qS )Ú.r@ )rÞ ÚlowerÚ IMG_FORMATS)rK rØ rM rM rN ú
<listcomp> s z'LoadImages.__init__.<locals>.<listcomp>c S s& g | ] }| d ¡d ¡ tkr|qS )rõ r@ )rÞ rö Ú VID_FORMATS)rK rØ rM rM rN rø s Fr| r z No images or videos found in z!. Supported formats are:
images: z
videos: )#Ú
isinstancerT r ÚsuffixÚ read_textÚrsplitÚlistÚtupleÚsortedÚresolveÚextendÚglobrG rH ÚisdirrW ÚisfileÚappendÚFileNotFoundErrorr rà r© ÚfilesÚnfÚ
video_flagrã râ rá Ú
vid_strideÚanyÚ
_new_videoÚcapÚAssertionErrorr÷ rù )
r rH rà r© râ rá r r rL ÚimagesÚvideosÚniÚnvrM rM rN rÆ s8 "
zLoadImages.__init__c C s
d| _ | S )Nr )Úcount)r rM rM rN r 2 s zLoadImages.__iter__c C sÊ | j | jkrt| j| j }| j| j rðd| _x t| jD ]}| j ¡ q:W | j
¡ \}}xT|s®| j d7 _ | j ¡ | j | jkrt| j| j }| |¡ | j
¡ \}}q\W | jd7 _d| j d d| j d| j d| j d| d }nz| j d7 _ t |¡}| ¡ | _t |¡}| | jtj¡}t |¡}|d k sNtd| d | j d| j d
| d}| j r~| |¡}n8t || j| j | j d d }| d
¡d d d
}t !|¡}|||| j|| jfS )NÚvideorC zvideo ú/z (z) z: zImage Not Found zimage Ú )r© râ r )rj r rC r@ )"r r Ú
StopIterationr r
rã rÏ r r rì ÚretrieveÚreleaser
Úreadrä Úframesr ÚimreadÚcopyÚorig_imgr Ú fromarrayÚresizerà ÚLANCZOSr rë r rá r r© râ ry rí ) r rH rÑ Úret_valrî rg Zpil_imgr| rï rM rM rN rð 6 s@
2
zLoadImages.__next__c C sD d| _ t |¡| _t| j tj¡| j | _t| j tj ¡| _
d S )Nr ) rä r Ú VideoCapturer r rq ÚCAP_PROP_FRAME_COUNTr r ÚCAP_PROP_ORIENTATION_METAre )r rH rM rM rN r
b s zLoadImages._new_videoc C sL | j dkrt |tj¡S | j dkr0t |tj¡S | j dkrHt |tj¡S |S )Nr éŽ éZ )re r ÚrotateÚROTATE_90_CLOCKWISEÚROTATE_90_COUNTERCLOCKWISErs )r rï rM rM rN Ú _cv2_rotatej s
zLoadImages._cv2_rotatec C s | j S )N)r )r rM rM rN rÎ t s zLoadImages.__len__)rò rÖ TNrC ) r r¡ r¢ rÆ r rð r
r, rÎ rM rM rM rN rñ
s
#,
rñ c @ s6 e Zd ZdddZd d
Zd d Zd
d Zdd ZdS )Ú LoadStreamsú file.streamsé rÖ TNrC c
s¶ dt jj_d| _| _| _|| _tj
|¡r>t | ¡
¡ n|g}t|}dd |D | _d g| dg| dg| d g| f\| _| _| _| _x¶t|D ]š\}} |d d| d| d }
t| jd
kròtd dd l} | | ¡jd d
j} | ¡ rt | n| } | dkr2t r"t dt r2t dt! "| ¡} | #¡ sXt |
d| t$| %t!j&¡}
t$| %t!j'¡}| %t!j(¡}t)t$| %t!j*¡dp¢t+d| j|< t)t, -|¡rŒ|ndd dpÌd| j|< | .¡ \}| j|< t/| j0|| | gdd| j|< t1 2|
d| j| d|
d| d| j| dd
¡ | j| 3¡ qW t1 2d¡ t4 5 fd d| jD ¡} t4j6| dd j7d dk| _8 o| j8| _9|| _:| j8s²t1 ;d ¡ d S ) NTrÙ c S s g | ] }t |qS rM )r )rK rØ rM rM rN rø s z(LoadStreams.__init__.<locals>.<listcomp>r rC r z: z... )zwww.youtube.comz youtube.comzyoutu.be)Úpafyzyoutube_dl==2020.12.2r: )ÚpreftypezM--source 0 webcam unsupported on Colab. Rerun command in a local environment.zN--source 0 webcam unsupported on Kaggle. Rerun command in a local environment.zFailed to open Úinféd é )ÚtargetrË Údaemonz
Success (z frames rØ z at z.2fz FPS)rP c s" g | ]}t | d d jqS ))r© râ r )r Úshape)rK rØ )râ rà r© rM rN rø s )Úaxisu] WARNING â ïž Stream shapes differ. For optimal performance supply similarly-shaped streams.)<r ÚbackendsÚcudnnÚ benchmarkrã rà r© r rG rH r r rü rý r ÚsourcesÚimgsÚfpsr ÚthreadsÚ enumerater Úhostnamer r0 ÚnewÚgetbestÚurlÚ isnumericÚevalr! r r" r r$ ÚisOpenedr rq ÚCAP_PROP_FRAME_WIDTHÚCAP_PROP_FRAME_HEIGHTÚ CAP_PROP_FPSr¹ r% Úfloatr Úisfiniter r rV r r{ Ústartr ÚstackÚuniquer7 rŠ râ rá r³ )r r< rà r© râ rá r r Úirg Ústr0 r Úwr[ r> rÑ rM )râ rà r© rN rÆ z sJ
"4
&* 6
zLoadStreams.__init__c C s d| j | }}x| ¡ r||k r|d7 }| ¡ || j dkr| ¡ \}}|r\|| j|< n*t d¡ t | j| ¡| j|< |
|¡ t d¡ qW d S )Nr rC uQ WARNING â ïž Video stream unresponsive, please check your IP camera connection.g )
r rG rì r r r= r r³ r Ú
zeros_likeÚopenÚtimeÚsleep)r rP r rÙ r ÚfÚsuccessrï rM rM rN rV Š s
zLoadStreams.updatec C s
d| _ | S )Nr@ )r )r rM rM rN r ¶ s zLoadStreams.__iter__c sŽ j d7 _ tdd jD r4t d¡tdkr@t ¡ t j ¡ } j
rjt fdd|D ¡}n:t fdd|D ¡}|dd d d
f
d
¡}t |¡} j||d d fS ) NrC c s s | ]}| ¡ V qd S )N)Úis_alive)rK rØ rM rM rN rO Œ s z'LoadStreams.__next__.<locals>.<genexpr>Úqc s g | ]} |¡qS rM )rá )rK rØ )r rM rN rø  s z(LoadStreams.__next__.<locals>.<listcomp>c s& g | ] }t | j j jd d qS ))r© râ r )r rà r© râ )rK rØ )r rM rN rø Ä s .r@ )r rk rC rj rP )r Úallr? r ÚwaitKeyÚordÚdestroyAllWindowsr r= r rá r rN ry rí r< )r rî rï rM )r rN rð º s &
zLoadStreams.__next__c C s
t | jS )N)r r< )r rM rM rN rÎ Ê s zLoadStreams.__len__)r. r/ rÖ TNrC )r r¡ r¢ rÆ rV r rð rÎ rM rM rM rN r- x s
,r- c s: t j dt j t j dt j fdd| D S )Nr Úlabelsc s. g | ]&} | d ¡¡ dd ¡d d qS )rC rõ r z.txt)rW rý )rK rØ )ÚsaÚsbrM rN rø Ñ s z#img2label_paths.<locals>.<listcomp>)rG Úsep)Ú img_pathsrM )r` ra rN Úimg2label_pathsÎ s &rd c @ s e Zd ZdZejejejejej gZ
d#d d Z d$ddZ e
dd fddZdd Zdd Zdd Zdd Zdd Zd d Zed d Zed!d" ZdS )%rŽ g333333ã?é é FNrÖ ç r rP r@ c 2 sº |_ |_|_|_|r dn|_jo2j _| d | d g_|
_|_|rft |dnd _
yèg }xŽt |t r|n|gD ]}t
|}| ¡ rŒ|tjt|d d dd7 }q| ¡ rt|<}| ¡ ¡ ¡ }t|jtj |fdd |D 7 }W d Q R X qt|
| d
qW td d |D _jsRt |
d
W nD t k
r } z$t |
d| d| dt |W d d }~X Y nX t j_ | ¡ rŽ|nt
j d j !d¡}yNt"j#|dd $¡ d } d j%ksøt d t&j j kst W n( t k
r@ '||
¡d }Y nX (d¡\}}}}}|rŒt)dkrŒd| d| d|| d| d }t*d |
| ||t+d d rŒt, -d . d ¡¡ |dksä|rät |
d | d t fd!d d"D t/ 0¡ \} } _1t2t" 3| d¡} | dksB|rBt |
d#| d t t | _4t" 5| ¡_6t 7¡ _t 7¡ _ r t" 5fd$d j4D ¡ 8¡ d 9t:¡} t, -|
|t2| d%| d&¡ fd'd | D _fd(d | D _ fd)d | D _4fd*d | D _1j6| _6t2j6}t" ;t" <|¡| ¡ 9t:¡} | d+ d, }!| _=|_>t" <|¡_?|d+krj?t"j@jA|d- B|¡tC tDk _?g }"t j1_1t" 5|"¡ Ed,d+¡}#xtFt/j4j1D ]~\}$\}%|"r,|%d d
dd,
f |#k Gd,¡}&|%|& j4|$< r,fd.d tF|&D j1|$< | rÌdj4|$ d d
df< qÌW jrj6}'|'d d
d,f |'d d
df }(|( H¡ })fd/d |)D _fd0d |)D _ fd1d |)D _4fd2d |)D _1|'|) _6|(|) }(d,d,gg|! } xbtI|!D ]V}$|(| |$k }*|* J¡ |* K¡ }+},|,d,k r8|,d,g| |$< n|+d,krþd,d,|+ g| |$< qþW t" Lt" 5| ¡| |
| ¡ 9t:¡|
_M|d3krjN|
d4sd}d g| _Od5d jD _P|r¶d6\}-}.d g| d g| _Q_R|d7kròjSnjTtUtV Wfd8d9j?¡}/t*|/t2j?t+t)dkd:}0x|0D ]x\}$}1|d7krZ|-jP|$ X¡ jY7 }-n0|1\jO|$< jQ|$< jR|$< |-jO|$ jZtC 7 }-|
d;|-|. d<d=| d>|0_[q2W |0 \¡ d S )?NFrj )rZ z**z*.*T)rô c s( g | ] }| d ¡r | d d¡n|qS )z./rC )Ú
startswithÚreplace)rK rØ )ÚparentrM rN rø ÿ s z0LoadImagesAndLabels.__init__.<locals>.<listcomp>z does not existc s s2 | ]*}| d ¡d ¡ tkr| dtj¡V qdS )rõ r@ r N)rÞ rö r÷ ri rG rb )rK rØ rM rM rN rO s z/LoadImagesAndLabels.__init__.<locals>.<genexpr>zNo images foundzError loading data from z: Ú
r z.cache)Ú allow_pickleÚversionÚhashÚresults> r r@ z Scanning z... z images, z backgrounds, z corrupt)ÚdescÚtotalÚinitialÚ
bar_formatÚmsgszNo labels found in z, can not start training. c s g | ]} |¡qS rM )Úpop)rK Úk)rœ rM rN rø s )rn rm rt zAll labels empty in c s g | ]}t | kqS rM )r )rK rØ )Ú min_itemsrM rN rø ( s r z images filtered from datasetc s g | ]} j | qS rM )Úim_files)rK rP )r rM rN rø * s c s g | ]} j | qS rM )Ú label_files)rK rP )r rM rN rø + s c s g | ]} j | qS rM )r_ )rK rP )r rM rN rø , s c s g | ]} j | qS rM )Úsegments)rK rP )r rM rN rø - s r@ rC )r c s g | ]\}}|r | qS rM rM )rK r Úelem)ÚsegmentrM rN rø D s c s g | ]} j | qS rM )rx )rK rP )r rM rN rø N s c s g | ]} j | qS rM )ry )rK rP )r rM rN rø O s c s g | ]} j | qS rM )r_ )rK rP )r rM rN rø P s c s g | ]} j | qS rM )rz )rK rP )r rM rN rø Q s Úram)r¬ c S s g | ]}t | d ¡qS )z.npy)r Ú with_suffix)rK rW rM rM rN rø e s )r i @Údiskc s | | fS )NrM )rP )ÚfcnrM rN Ú<lambda>j ó z.LoadImagesAndLabels.__init__.<locals>.<lambda>)rq rs ÚdisablezCaching images (z.1fzGB ú))]rà r€ r¥ r« rŠ ÚmosaicÚ
mosaic_borderr© rH r Úalbumentationsrú rþ r Úis_dirr rT Úis_filerT r ÚstripÚ
splitlinesrj rG rb r r rx r ra ÚHELP_URLrd ry r~ r ÚloadÚitemÚ
cache_versionr\ Ú cache_labelsru r? r r r r{ rW ÚzipÚvaluesrz r Ú concatenater_ rë ÚshapesÚkeysÚnonzeroÚastyper ÚfloorÚarangeÚbatchr Úindicesr Ú RandomStateÚ permutationrB rA Úreshaper@ r ÚargsortrÏ rµ r¹ r Ú batch_shapesÚcheck_cache_ramÚimsÚ npy_filesÚim_hw0Úim_hwÚcache_images_to_diskÚ
load_imager r ÚimapÚstatÚst_sizeÚnbytesrp Úclose)2r rH rà r r€ r¥ rŠ r« r§ rš r© rª rw r¬ r r rW rL ÚtÚeÚ
cache_pathrI r ÚnmÚneÚncr Údr_ r ÚnlÚincludeÚbiÚnbÚ
include_classÚinclude_class_arrayrP ÚlabelÚjrg ÚarÚirectÚariÚminiÚmaxiÚbÚgbro ÚpbarrØ rM )rœ r rw rj r| r rN rÆ Ù sà
"2 $""
((
("
$
(
$z LoadImagesAndLabels.__init__ç¹?c
C sØ d\}}t | jd}xPt|D ]D}t t | j¡¡}| jt |j
d |j
d }||j |d 7 }q W || j | } t
¡ }
| d| |
jk } | sÔt | | | dd|
j| dd|
j| dd | rÊd
nd ¡ | S ) N)r i @r4 r rC rj z.1fzGB RAM required, r zGB available, u caching images â
u not caching images â ïž)rµ r rÏ r r r Úchoicerx rà r¹ r7 r« ÚpsutilÚvirtual_memoryÚ availabler r{ rq ) r Ú
safety_marginr¬ rÁ r r rÑ rï ÚratioÚ mem_requiredÚmemrœ rM rM rN r¡ u s Bz#LoadImagesAndLabels.check_cache_ramz./labels.cachec C sø i }d}ddddg f\}}}}} | d|j |j d}
ttž} t| tt| j| j t
||¡|
t | jt d} x| D ]x\ }
}}}}}}}}||7 }||7 }||7 }||7 }|
rÀ|||g||
< |rÎ|
|¡ |
d| d|| d| d | _qxW W d Q R X | ¡ | r t d
| ¡¡ |dkr@t | d | d t ¡ t| j | j |d
< ||||t | jf|d< | |d< | j|d< y4t ||¡ | d¡ |¡ t | d| ¡ W n@ tk
rò } z t | d|j d| ¡ W d d }~X Y nX |S )NTr z Scanning z...)rp rq rs r z images, z backgrounds, z corruptrk u" WARNING â ïž No labels found in z. rn ro rt rm z
.cache.npyzNew cache created: u WARNING â ïž Cache directory z is not writeable: ) rj Ústemr r r rš Úverify_image_labelr rx ry r r r r rp r¬ r r{ rW r³ r r\ r r Úsaver~ Úrenamera )r rH r¬ rØ Úbloodr° r r± r² rt rp Úpoolrà Úim_fileÚlbr7 rz Únm_fÚnf_fÚne_fÚnc_fÚmsgr® rM rM rN r sF
0
.z LoadImagesAndLabels.cache_labelsc C s
t | jS )N)r rx )r rM rM rN rÎ s zLoadImagesAndLabels.__len__c C sÞ | j | }| j}| jo$t ¡ |d k }|rp| |¡\}}d }t ¡ |d k rnt||f| t | j ¡¡ \}}nþ| |¡\}\}}\} }
| jr| j | j
| n| j } t || d| j
d\}} }
||f| | |
| f|
ff}| j| ¡ }|jr8t|d d
dd
f | d |
| d | |
d |
d d|d d
dd
f< | j
rnt|||d |d
|d |d |d
d\}}t|}|rºt|d d
dd
f |jd |jd ddd|d d
dd
f< | j
rz| ||¡\}}t|}t||d |d |d d t ¡ |d k r8t |¡}|r8d|d d
df |d d
df< t ¡ |d k rzt |¡}|rzd|d d
df |d d
df< t |df¡}|ršt |¡|d d
dd
f< | d¡d d d
}t |¡}t |¡|| j | |fS ) Nr
r F)râ ÚscaleuprC rm r )ÚpadwÚpadhÚdegreesÚ translateÚscaleÚshearÚ perspective)rÝ rÞ rß rà rá Tgü©ñÒMbP?)rR r[ ÚclipÚepsÚhsv_hÚhsv_sÚhsv_v)ÚhgainÚsgainÚvgainÚflipudrj Úfliplré
)rj r rC r@ )!r r¥ r
r Ú load_mosaicr rÅ r§ rŠ r r rà r r€ r_ r rZ r' r r r( r7 r r r rê rë r ÚzerosÚ
from_numpyry rí rx )r Úindexr¥ r
rf r_ r Úh0Úw0r[ rR r7 rÊ rª rŽ Z
labels_outrM rM rN Ú __getitem__¶ sX
$ J>
z LoadImagesAndLabels.__getitem__c C sþ | j | | j| | j| }}}|d krâ| ¡ r>t |¡}n t |¡}|d k s^td| |j d d
\}}| j
t || }|dkrÊ| j s|dkrtj
ntj}tj|t || ¡t || ¡f|d}|||f|j d d
fS | j | | j| | j| fS )NzImage Not Found rj rC )Ú
interpolation)r¢ rx r£ rI r r r r r r7 rà r¹ r€ Ú INTER_LINEARÚ
INTER_AREAr! r r r€ r¥ ) r rP rï rW Úfnrñ rò ÚrÚinterprM rM rN r§ s "
(z LoadImagesAndLabels.load_imagec C s2 | j | }| ¡ s.t | ¡ t | j| ¡¡ d S )N)r£ rI r rÏ Úas_posixr r rx )r rP rW rM rM rN rŠ s
z(LoadImagesAndLabels.cache_images_to_diskc sr g g }}| j fdd| jD \}}|gtj| jdd }t |¡ x|t|D ]n\}}| |¡\}} \ |dkrðtj d d |j
d fdtj d}
t | dt | d||f\} } }
}|
| || f\}}}}n |d krN|t | dt
| d |f\} } }
}d || t
|
| f\}}}}nÀ|dkr¬t | d||t
d | f\} } }
}|
| dt
|| f\}}}}nb|dkr||t
| d t
d | f\} } }
}ddt
|
| t
|| f\}}}}|||
||
f |
| |
| |
f< | | | | | j| ¡ | j| ¡ }}|jr°t|d d
d d
f |d d
d d
f< fd
d |D }| |¡ | |¡ qTW t |d¡}x:|d d
d d
f f|D ] }tj|dd |d qðW t|
||| jd
d\}
}}t|
||| jd | jd | jd | jd | jd | jd \}
}|
|fS )Nc 3 s( | ] }t t | d | ¡V qdS )rj N)r r Úuniform)rK rØ )rg rM rN rO s z2LoadImagesAndLabels.load_mosaic.<locals>.<genexpr>rk )rv r rj ér )ÚdtyperC c s g | ]}t | qS rM )r% )rK rØ )r[ rÜ rÛ rR rM rN rø 9 s z3LoadImagesAndLabels.load_mosaic.<locals>.<listcomp>)Úoutr )rL rÝ rÞ rß rà rá )rÝ rÞ rß rà rá Úborder)rà r r Úchoicesr r r@ r§ r Úfullr7 Úuint8r¹ rµ r_ r rz rZ r' r r r râ r r¥ r )r rð Zlabels4Z segments4ÚycÚxcr rP rf rÑ Úimg4Úx1aÚy1aÚx2aÚy2aÚx1bÚy1bÚx2bÚy2br_ rz rØ rM )r[ rÜ rÛ rg rR rN rí sV
&((
,(
,(
0($ 0
z LoadImagesAndLabels.load_mosaicc s0 g g }}| j |gtj| jdd }t |¡ d\}}x¶t|D ]š\}}| |¡\}} \|dkr²tjd d |j d fdtj
d}
} } f n^|d krÒ f n>|dkrú| | f n|dkr"| | f nî|d
krR| | | | f nŸ|d kr| | | | f n|d krº| | | | | | f nV|d
krâ | | f n.|dkr | | | | f d d
\dd D \}
}}}| j | ¡ | j
| ¡ }}|jr€t|d d
d d
f |d d
d d
f< fdd|D }| |¡ | |¡ || d
|
d
f |
||
|
|
f< }}qDW fdd| jD \}}|
||d
||d
f }
t |d¡}|d d
d dgf |8 < |d d
dd
gf |8 < t ||g¡ fdd|D }x:|d d
d d
f f|D ] }tj|dd |d q®W t|
||| jd d\}
}}t|
||| jd | jd | jd | jd | jd | jd \}
}|
|fS ) Nr^ )rv )r@ r@ r rk rj rü )rý rC rl rm r] rn c s s | ]}t |d V qdS )r N)r¹ )rK rØ rM rM rN rO s s z3LoadImagesAndLabels.load_mosaic9.<locals>.<genexpr>c s g | ]}t | qS rM )r% )rK rØ )r[ ÚpadxÚpadyrR rM rN rø y s z4LoadImagesAndLabels.load_mosaic9.<locals>.<listcomp>c 3 s | ]}t t d ¡V qdS )r N)r r rû )rK rÑ )rg rM rN rO s c s g | ] }| qS rM rM )rK rØ )ÚcrM rN rø s )rþ r )rL rÝ rÞ rß rà rá )rÝ rÞ rß rà rá rÿ )rà r r r r r@ r§ r r r7 r r_ r rz rZ r' r r r r rë râ r r¥ r )r rð Zlabels9Z segments9r ÚhpÚwprP rf rÑ Úimg9rñ rò Úx1Úy1Úx2Úy2r_ rz r r rØ rM )r r[ r r rg rR rN Ú load_mosaic9Q sr
&
&
&
.
$ 0
,$ z LoadImagesAndLabels.load_mosaic9c C sR t | \}}}}x$t|D ]\}}||d d
df< qW t |d¡t |d¡||fS )Nr )r r@ r rN Úcat)r rï rº rH r rP rÔ rM rM rN r± s z LoadImagesAndLabels.collate_fnc
C sÊ t | \}}}}t|d }g g |d |
|d |
f\}}}} t ddddddgg¡}
t ddddddgg¡} t ddddddgg¡} xüt|D ]ð}
|
d9 }
t ¡ dk rætj||
d¡ ¡ dddd d
||
¡ ¡}||
}nt t ||
||
d fd¡t ||
d
||
d fd¡fd
¡}t ||
||
d |
||
d
| ||
d |
| fd¡| }| |¡ | |¡ qW x&t
|D ]\}
}|
|d d
df< qW t |d¡t |d¡|| fS ) Nrl g r rC g à?g @ÚbilinearF)Ú scale_factorrã Ú
align_cornersrj rk )r r r ÚtensorrÏ r ÚFÚ interpolateÚ unsqueezerK Útyper r r@ rN )r rï rº rH r r Zim4Zlabel4Zpath4Zshapes4ZhoÚworg rP Úim1rÔ rM rM rN r» ¥ s& $
DD
z LoadImagesAndLabels.collate_fn4)re rf FNFFFFrÖ rg r rP r@ r )rÄ rP )r r¡ r¢ r r Ú
INTER_NEARESTrõ Ú INTER_CUBICrö ÚINTER_LANCZOS4Úrand_interp_methodsrÆ r¡ r r rÎ ró r§ rŠ rí r Ú staticmethodr± r» rM rM rM rN rŽ Ô s6
' J:M rŽ Úcoco128c C sv t t| d}tj |¡r(t |¡ t |¡ x>tt j tt | dddD ]}t
||t |j ¡ qTW d S )NÚ_flatz/**/*.*T)rô ) r rT rG rH rI ÚshutilÚrmtreeÚmakedirsr r ÚcopyfileÚname)rH Únew_pathÚfilerM rM rN Úflatten_recursiveÁ s
&r2 c
C s& t | } | d ¡ r"t | d ¡nd t| d¡}t|}xât||dD ]Ð}|jdd
t krLt
t |¡dd d d
f }|j
d d
\}}t tt |gd }t | ¡ rLt|,}tjd d
| ¡ ¡ ¡ D tjd } W d Q R X x,t| D ] \}
} t| d } | d | | j d
|j d
|
d }|j ¡ sR|jjdd | dd
||||g }
|
dd
d d |
dd
< t |
dd¡ ¡ t¡}
t |
ddg d|¡|
ddg< t |
ddg d|¡|
ddg< t
!t |||
d |
d
|
d |
d
f ¡sút"d| qúW qLW d S )NÚclassificationz*.*)rq rC .r@ rj r c S s g | ] }| ¡ qS rM )rÞ )rK rØ rM rM rN rø Û s z!extract_boxes.<locals>.<listcomp>)rý Ú
classifierrÑ z.jpgT)Úparentsg333333ó?rk rl zbox failure in )#r r r+ r, rþ Úrglobr r rû r÷ r r rT r7 rd rI rT r rë r r r Úfloat32r@ r rÍ rj Úmkdirr& r Úravelr râ Úimwriter )rH r r rÓ rï r[ rR Úlb_filerW rÔ r» rØ r rÁ rM rM rN Ú
extract_boxesË s.
0 ( r< zcoco128/images)gÍÌÌÌÌÌì?g¹?g c
C s t | } tdd | d¡D }t|}t d¡ tjdddg||d}dd d
g}x(|D ] }| j| ¡ rX| j| ¡ qXW t
d | d | x|t t |||d
D ]f\}} |rÌt t
t| gd ¡ rŠt| j|| d&}
|
d| | j¡ ¡ d ¡ W dQ R X qŠW dS )aT Autosplit a dataset into train/val/test splits and save path/autosplit_*.txt files
Usage: from utils.dataloaders import *; autosplit()
Arguments
path: Path to images directory
weights: Train, val, test weights (list, tuple)
annotated_only: Only use images with an annotated txt file
c s s( | ] }|j d d
¡ tkr|V qdS )rC N)rû rö r÷ )rK rØ rM rM rN rO ö s z autosplit.<locals>.<genexpr>z*.*r rC rj )Úweightsrv zautosplit_train.txtzautosplit_val.txtzautosplit_test.txtzAutosplitting images from z!, using *.txt labeled images only)rq Úaz./rk N)r r r6 r r r r rj rI ÚunlinkÚprintr r rd rT rT ÚwriteÚ relative_torú ) rH r= Úannotated_onlyr r r ÚtxtrØ rP rf rW rM rM rN Ú autosplití s
rE c sB | \}}}dddddg f\}}}}} yÆt |¡}
|
¡ t|
} | d dk| d dk@ sltd| d|
j ¡ tkstd|
j |
j ¡ dkrôt|d L} | d
d ¡ | ¡ d krêt
t |¡¡j |d
ddd | d| d} W d Q R X t
j |¡rÆd}t|} dd | ¡ ¡ ¡ D }
|stdd |
D rtjdd |
D tjd}dd |
D t | dd¡t fd¡}
tj|
tjd}
W d Q R X t|
}|r°|
jd dksÖtd|
jd d |
dk ¡ sútd |
|
dk |
d d
dd
f dk ¡ sNtd |
d d
dd
f |
d d
dd
f dk tj |
dd d!\}}t||k rÄ|
| }
r fd"d|D | d| d#|t| d$} nd}tj d%tjd}
nd}tj d%tjd}
||
| ||||| f S t k
r< } z.d}| d| d&| } d d d d ||||| g S d }~X Y nX d S )'Nr rP é rC z image size z <10 pixelszinvalid image format )r- r, Úrbéþÿÿÿrj s ÿÙÚJPEGr3 )Ú subsamplingÚqualityu WARNING â ïž z!: corrupt JPEG restored and savedc S s g | ]}t |r| ¡ qS rM )r rÞ )rK rØ rM rM rN rø s z&verify_image_label.<locals>.<listcomp>c s s | ]}t |d kV qdS )r] N)r )rK rØ rM rM rN rO s z%verify_image_label.<locals>.<genexpr>c S s g | ] }|d qS )r rM )rK rØ rM rM rN rø s )rý c S s, g | ]$}t j|d d
t jd dd¡qS )rC N)rý r@ rj )r rë r7 r )rK rØ rM rM rN rø s r@ é zlabels require 5 columns, z columns detectedznegative label values rm z,non-normalized or out of bounds coordinates T)r8 Ú return_indexc s g | ] } | qS rM rM )rK rØ )rz rM rN rø , s z: z duplicate labels removed)r rL z : ignoring corrupt image/label: ) r rT Úverifyri r Úformatrö r÷ Úseekr r
r~ rÏ rG rH r r r r r rë r7 r r r# r r7 r[ rO rî ra )rË rÑ rÓ r; r¬ r° r r± r² rÙ rï r7 rW rÔ ÚclassesrŽ rÑ rP r® rM )rz rN rÎ sV
(
&$T rÎ c @ sJ e Zd ZdZdddZedd Zdd Zdd d Zdd
dZ dd Z
dS )ÚHUBDatasetStatsa
Class for generating HUB dataset JSON and `-hub` dataset directory
Arguments
path: Path to data.yaml or data.zip (with data.yaml inside data.zip)
autodownload: Attempt to download dataset if not found locally
Usage
from utils.dataloaders import HUBDatasetStats
stats = HUBDatasetStats('coco128.yaml', autodownload=True) # usage 1
stats = HUBDatasetStats('path/to/coco128.zip') # usage 2
stats.get_json(save=False)
stats.process_images()
ú coco128.yamlFc
C sÖ | t|¡\}}}y6tt|dd }t |¡}|r>||d< W d Q R X W n, tk
rv } ztd|W d d }~X Y nX t|| t|d d | _| jd | _ | j j
ddd |d t |d
¡ d | _
|| _d S ) NÚignore)ÚerrorsrH z!error/HUB/dataset_stats/yaml_loadz-hubr T)r5 Úexist_okr² Únames)r² rW )Ú_unzipr rT r ÚyamlÚ safe_loadra r Úhub_dirÚim_dirr8 rþ r ÚstatsÚdata) r rH Ú autodownloadÚzippedÚdata_dirÚ yaml_pathrW r^ r® rM rM rN rÆ J s
zHUBDatasetStats.__init__c s t d¡pt d¡}|s.td t|dkr` fdd|D }|s`td dt|dkstd| d |d
S ) Nz*.yamlzNo *.yaml file found in rC c s g | ]}|j j kr|qS rM )rÍ )rK rW )ÚdirrM rN rø b s z.HUBDatasetStats._find_yaml.<locals>.<listcomp>z Multiple *.yaml files found in z , only 1 *.yaml file allowedz Multiple *.yaml files found: z , only 1 *.yaml file allowed in r )rþ r r6 r r )rc r rM )rc rN Ú
_find_yaml\ s zHUBDatasetStats._find_yamlc C s~ t | d¡sdd |fS t| ¡ s4td| dt||jd | d¡}| ¡ sjtd| d| dd t || |¡fS )
Nz.zipFzError unzipping z, file not found)rH rP z, z6 not found. path/to/abc.zip MUST unzip to path/to/abc/T)
rT Úendswithr r r r$ rj r~ r rd )r rH rc rM rM rN rX g s
zHUBDatasetStats._unzipé c
C s | j t|j }y\t |¡}|t|j|j }|dk rX| t |j| t |j| f¡}|j
|dddd W n¢ t k
r } zt
d| d| ¡ t |¡}|jd d
\}}|t|| }|dk rîtj|t || t || ftjd }t t||¡ W d d }~X Y nX d S )
Ng ð?rI é2 T)rK Úoptimizeu# WARNING â ïž HUB ops PIL failure z: rj )rô )r\ r r/ r rT r¹ rÝ rÜ r! r rÏ ra r r{ r r r7 rö r: rT ) r rW Úmax_dimÚf_newrï rø r® Ú im_heightÚim_widthrM rM rN Ú_hub_opsq s
"
&zHUBDatasetStats._hub_opsc s> dd xÊdD ]Â}j |¡d kr.d j|< qtj | }t fddt|j|jddD ¡}t |
¡ |
d¡ ¡ d |jt t |dkd
¡
¡ |dk
d¡ ¡ d fd dt
|j|jD d
j|< qW |r jd }td| ¡ d t|d}t j|¡ W d Q R X |r8ttjjddd jS )Nc S s dd | D S )Nc S s( g | ] ^}}t |fd d |D qS )c s s | ]}t |d V qdS )rl N)Úround)rK rØ rM rM rN rO s zFHUBDatasetStats.get_json.<locals>._round.<locals>.<listcomp>.<genexpr>)r )rK r ÚpointsrM rM rN rø s z<HUBDatasetStats.get_json.<locals>._round.<locals>.<listcomp>rM )r_ rM rM rN Ú_round
s z(HUBDatasetStats.get_json.<locals>._round)ÚtrainÚvalÚtestc s2 g | ]*}t j|d d
df t¡ jd dqS )Nr r² )Ú minlength)r Úbincountr r r^ )rK rº )r rM rN rø s z,HUBDatasetStats.get_json.<locals>.<listcomp>Ú
Statistics)rq rp r )rq Ú per_classrC )rq Ú
unlabelledrw c s* g | ]"\}}t t|j | ¡ iqS rM )rT r r/ r )rK rv Úv)rp rM rN rø s )Úinstance_statsÚ image_statsr_ z
stats.jsonzSaving z...rR rj F)ÚindentÚ sort_keys)r^ rq r] rŽ r rë r r_ r r rQ r r[ r rx r[ r@ r rT ÚjsonÚdumpÚdumps)r rÏ ÚverboserÞ r rØ Ú
stats_pathrW rM )rp r rN Úget_json s0
zHUBDatasetStats.get_jsonc C sx x`dD ]X}| j |¡d kr qt| j | }| d}x(ttt | j|j¡|j |dD ]}qVW qW t
d| j | j S )N)rq rr rs z images)rq rp zDone. All images saved to ) r^ rq rŽ r r r rš rm rx r r@ r\ )r rÞ r rp rÑ rM rM rN Úprocess_imagesŠ s
&z HUBDatasetStats.process_imagesN)rS F)rf )FF) r r¡ r¢ rÒ rÆ r( rd rX rm r r rM rM rM rN rR ; s
#rR c s* e Zd ZdZd fdd Zdd Z ZS )ÚClassificationDatasetzÔ
YOLOv5 Classification Dataset.
Arguments
root: Dataset path
transform: torchvision transforms, used by default
album_transform: Albumentations transforms, used if installed
Fc s^ t j|d t|| _|r&t||nd | _|dkp:|dk| _|dk| _dd | jD | _d S )N)ÚrootTr} r c S s* g | ]"}t |t|d d¡dg qS )r z.npyN)rþ r r~ )rK rØ rM rM rN rø Ã s z2ClassificationDataset.__init__.<locals>.<listcomp>) rÅ rÆ r Útorch_transformsr Úalbum_transformsÚ cache_ramÚ
cache_diskÚsamples)r r r€ rŒ rœ )rÍ rM rN rÆ œ s
z ClassificationDataset.__init__c C sš | j | \}}}}| jr:|d kr:t |¡ }| j | d< n:| jrj| ¡ s^t | ¡ t |¡¡ t |¡}n
t |¡}| j
r| j
t |tj ¡dd }n
|
|¡}||fS )Nrk )r| r| )r r r r r rI r rÏ rú r r ÚcvtColorÚ
COLOR_BGR2RGBr )r rP rW r» r÷ rï ÚsamplerM rM rN ró Å s
z!ClassificationDataset.__getitem__)F)r r¡ r¢ rÒ rÆ ró rÓ rM rM )rÍ rN r
Ž s r
éà rf c
C sŽ t | t| |||d}W d Q R X t|t|}tj ¡ } tt ¡ t | d |dkrZ|nd|g}
|dkrpd n t
j ||d} t ¡ } |
dt ¡ t|||o€| d k|
| tt| dS )N)r rŒ r€ rœ rC r r@ )r l UUª*UUª* )r r r® r¯ r° r² r )r) r
rµ r r r¶ r· rG rž r¹ r ÚDistributedSamplerr r rA rº rD r )
rH rŒ r r€ rœ r rŸ r r rÀ rÁ r¯ r rM rM rN Ú create_classification_dataloaderÖ s
(
r )
FNFFr£ Fr@ r^ FFrP Fr )T)r rf TFr@ r^ T)krÒ r_ r rR r~ r rG r r+ rU Ú itertoolsr Úmultiprocessing.poolr r Úpathlibr Ú threadingr Ú urllib.parser Únumpyr rÆ r Útorch.nn.functionalÚnnÚ
functionalr Ú torchvisionrY ÚPILr r r
Útorch.utils.datar r r
r r Zutils.augmentationsr r r r r r r r Ú
utils.generalr r r r r r r r r r! r" r# r$ r% r& r' r( Úutils.torch_utilsr) r r÷ rù r Úgetenvr? rA rB rT rö rD ÚTAGSr re r\ ri r~ r r r rà rº rÉ rÔ rñ r- rd rŽ r2 r< rE rÎ rR ÚdatasetsÚ ImageFolderr
r rM rM rM rN Ú<module> s (L
&1kV p
"
4y#