U
    ¿d[                     @   s  d Z ddlZddlmZ ddlmZ ddlmZ ddlmZ ddl	m
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ eedZ G dd deZ!ej"G dd deZ#G dd deZ$dS )z>
Appraise evaluation framework

See LICENSE for usage details
    N)defaultdict)loads)
is_zipfile)ZipFile)User)models)format_lazy)ugettext_lazy)_get_loggerLANGUAGE_CODES_AND_NAMES)AnnotationTaskRegistry)BaseMetadata)EvalItem)MAX_REQUIREDANNOTATIONS_VALUE)MAX_SEGMENTID_LENGTH)MAX_SEGMENTTEXT_LENGTH)seconds_to_timedelta)namec                       s   e Zd ZdZejeedeededdZ	eje
edeede
ddZejeedeededdZeje
edeede
ddZejed	d
Z fddZ  ZS )TextPairWithImagez:
    Models a pair of two text segments and an image.
    z	Source IDz(max. {value} characters)value)
max_lengthverbose_name	help_textzSource textz	Target IDzTarget textz	image URL)r   c                    sr   t | jtdrdS t| j}|dk s.|tkr2dS t | jtdrFdS t| j}|dk s`|tkrddS tt|  S )zI
        Validates the current TextPair instance, checking text.
        zThis is a test sentence.F   )	
isinstance
sourceTexttypelenr   
targetTextsuperr   is_valid)self_len	__class__ U/var/www/rival/public_html/translation-eval/EvalData/models/multi_modal_assessment.pyr"   B   s    

zTextPairWithImage.is_valid)__name__
__module____qualname____doc__r   	CharFieldr   _fsourceIDr   r   targetIDr    URLFieldimageURLr"   __classcell__r'   r'   r%   r(   r   "   s.   r   c                
   @   sB  e Zd ZdZejddejddeddZej	e
dded	d
ZejedeededdZej	eddddededdZejededdZejdejdddddeddZdd Zdd Zdd Zdd  Zd!d" Zd#d$ Zd%d& Zd'd( Zd9d*d+Zed,d- Zed:d/d0Z ed1d2 Z!ed3d4 Z"d5d6 Z#d7d8 Z$d.S );MultiModalAssessmentTaskz9
    Models a multimodal assessment evaluation task.
    zCampaign.CampaignTz %(app_label)s_%(class)s_campaign%(app_label)s_%(class)ssCampaigndb_index	on_deleterelated_namerelated_query_namer   z%(app_label)s_%(class)s_itemsItems)r;   r<   r   zRequired annotationsz(value in range=[1,{value}])r   r   r   z"%(app_label)s_%(class)s_assignedTozAssigned toz(users working on this task))blankr9   r;   r<   r   r   zBatch numberz	(1-based)zCampaign.CampaignDataz!%(app_label)s_%(class)s_batchDataz
Batch data)r:   r?   r9   nullr;   r<   r   c                 C   s
   t | jS N)str	batchDatar#   r'   r'   r(   dataName   s    z!MultiModalAssessmentTask.dataNamec                 C   s   t | j jjS rA   )rB   itemsfirstmetadatamarketrD   r'   r'   r(   
marketName   s    z#MultiModalAssessmentTask.marketNamec                 C   sD   t | j jjd}t|dkr@|d t kr@t|d  S d S Nr.      r   	rB   rF   rG   rH   rI   splitr   r   keysr#   tokensr'   r'   r(   marketSourceLanguage   s    z-MultiModalAssessmentTask.marketSourceLanguagec                 C   s@   t | j jjd}t|dkr<|d t kr<|d S d S rK   rM   rP   r'   r'   r(   marketSourceLanguageCode   s    z1MultiModalAssessmentTask.marketSourceLanguageCodec                 C   sD   t | j jjd}t|dkr@|d t kr@t|d  S d S Nr.   rL   r   rM   rP   r'   r'   r(   marketTargetLanguage   s    z-MultiModalAssessmentTask.marketTargetLanguagec                 C   s@   t | j jjd}t|dkr<|d t kr<|d S d S rT   rM   rP   r'   r'   r(   marketTargetLanguageCode   s    z1MultiModalAssessmentTask.marketTargetLanguageCodec                 C   s*   t jj| dd|djddd}tt|S )NFT)task	activated	completed	createdByitem_idflat)MultiModalAssessmentResultobjectsfiltervalues_listr   set)r#   userresultsr'   r'   r(   completed_items_for_user   s        z1MultiModalAssessmentTask.completed_items_for_userc                 C   s&   ddl m} |jj|| jd}| S )Nr   )TrustedUser)rc   campaign)Campaign.modelsrf   r_   r`   rg   exists)r#   rc   rf   trusted_userr'   r'   r(   is_trusted_user   s    z(MultiModalAssessmentTask.is_trusted_userFc                 C   s  |  |}d }d}| j dD ]T}tjj|dd|d}| sntd	|j
|j| |rf|jdkrn|} qx|d7 }q"|std		| j
 tjj| ddd
jddd}tt|}	d}
|rd}
| j|
 }td	|	| |	|krtd	| j
 |   |   |r||fS |S )Nr   idFT)itemrX   rY   rZ   z-identified next item: {0}/{1} for trusted={2}TGTr   zNo next item found for task {0})rW   rX   rY   r[   r\   d   F   zUnique annotations={0}/{1}zCompleting task {0})rk   rF   allorder_byr^   r_   r`   ri   printformatrl   itemTypeLOGGERinfora   r   rb   requiredAnnotationscompletesave)r#   rc   return_completed_itemsrj   	next_itemcompleted_itemsrm   resultannotationsuniqueAnnotationsrequired_user_results_total_requiredr'   r'   r(   next_item_for_user   s^    
     
   


z+MultiModalAssessmentTask.next_item_for_userc                 C   s<   | j j|ddddD ]}||}|d k	r|  S qd S )NTF)
assignedTorX   rY   z-id)r_   r`   rr   r   )clsrc   active_taskr|   r'   r'   r(   get_task_for_user   s      

z*MultiModalAssessmentTask.get_task_for_userNc                 C   sp   | j jdd|d}|r"|j|d}|dD ]>}|j }t||j ||jk r,|r,||j kr,|  S q,d S )NTF)rX   rY   +items__metadata__market__targetLanguageCode)rg   rl   )r_   r`   rr   r   countrs   rx   rq   )r   coderg   rc   active_tasksr   active_usersr'   r'   r(   get_next_free_task_for_language   s    


z8MultiModalAssessmentTask.get_next_free_task_for_languagec                 C   s   |  ||S rA   )r   )r   r   rg   r'   r'   r(   ,get_next_free_task_for_language_and_campaign  s    zEMultiModalAssessmentTask.get_next_free_task_for_language_and_campaignc                 C   s  |j }|jj}|j}d}|drt|sBd|}	t|	 dS t|}
dd |
	 D }|D ]4}|

|d}tjdkrt|}q`t|dd}q`ntt|
 dd}d	d
lm} | }d	}d	}d	}|D ]}|d	kr||krd|}	t|	 | }t||   dS t||d d  g }|d D ]}t|d }t|d }||krnt||d  |}||krt||d d |}t|d |d |d |d |d ||d |d d}|| q6t|dksdt|}	t|	 q|d7 }|jj|ddi |  t||d d |d d ||d}|  |jj|  |  dt||d d }	t|	 qd||}	t|	 | }t||  dS ) zU
        Creates new MultiModalAssessmentTask instances based on JSON input.
        Nz.zipz!Batch {0} not a valid ZIP archivec                 S   s   g | ]}| d r|qS )z.json)endswith.0xr'   r'   r(   
<listcomp>$  s     
 z=MultiModalAssessmentTask.import_from_json.<locals>.<listcomp>zutf-8)rL   	   r   )encodingr   )datetimez'Stopping after max_count={0} iterationsrW   batchNorF   r1   r    r0   r   r3   itemIDru   )r0   r   r1   r    r3   rZ   r   ru   ro   z)Expected 100 items for task but found {0}r   bulkFrx   )rg   rx   r   rC   rZ   z&Success processing batch {0}, task {1}zMax length ID={0}, text={1})rH   dataFiler   r   r   rt   rv   warnr   namelistreaddecodesysversion_infor   rB   r   nowrw   rs   r   encoder   appendZtextpairwithimage_setaddrz   r5   rF   )r   rg   
batch_user
batch_data	max_count
batch_meta
batch_name
batch_file
batch_json_msg	batch_zipbatch_json_filesbatch_json_filebatch_contentr   t1current_countmax_length_idmax_length_text
batch_taskt2	new_itemsrm   current_length_idcurrent_length_textnew_itemnew_taskr'   r'   r(   import_from_json  s    













 

z)MultiModalAssessmentTask.import_from_jsonc                 C   sD   t | dr| j sdS t | ds&dS | jD ]}| s, dS q,dS )zS
        Validates the current DA task, checking campaign and items exist.
        rg   FrF   T)hasattrrg   r"   rF   )r#   rm   r'   r'   r(   r"     s    

z!MultiModalAssessmentTask.is_validc                 C   s   d | jj| j| j S )Nz{0}.{1}[1..{2}])rt   r&   r)   rg   rF   r   rD   r'   r'   r(   _generate_str_name  s
      z+MultiModalAssessmentTask._generate_str_name)F)NN)%r)   r*   r+   r,   r   
ForeignKeyPROTECTr.   rg   ManyToManyFieldr   rF   PositiveSmallIntegerFieldr/   r   rx   r   r   PositiveIntegerFieldr   rC   rE   rJ   rR   rS   rU   rV   re   rk   r   classmethodr   r   r   r   r"   r   r'   r'   r'   r(   r5   Y   s   	

 
3



tr5   c                
   @   s  e Zd ZdZejededdZejededdZ	ejededdZ
ejedejd	d
eddZejedddejdd
eddZdd Zdd Zdd Zed*ddZedd Zedd Zedd Zedd Zed d! Zed"d# Zed+d%d&Zed'd( Zd)S ),r^   z;
    Models a multimodal assessment evaluation result.
    Scorez(value in range=[1,100])r>   z
Start timez(in seconds)zEnd timeTz%(app_label)s_%(class)s_itemr6   Itemr8   z%(app_label)s_%(class)s_taskTask)r?   r9   r@   r:   r;   r<   r   c                 C   s   d | jj| j| jS )Nz{0}.{1}={2})rt   r&   r)   rm   scorerD   r'   r'   r(   r     s    z-MultiModalAssessmentResult._generate_str_namec                 C   s   | j | j }t|dS )Nr   )end_time
start_timeround)r#   dr'   r'   r(   duration  s    z#MultiModalAssessmentResult.durationc                 C   s   | j jS rA   )rm   ru   rD   r'   r'   r(   	item_type  s    z$MultiModalAssessmentResult.item_typec                 C   s0   | j j|ddd}|r(|d  S | S )NFTrZ   rX   rY   item__id)r_   r`   ra   distinctr   )r   rc   unique_only_queryr'   r'   r(   get_completed_for_user  s    z1MultiModalAssessmentResult.get_completed_for_userc                 C   sx   t t}| jj|dddddD ]*}|d  dkr8q"||d   d7  < q"t| }td	d
 | D }||fS )NFTr   task__iditem__itemTyper   tgtr   c                 S   s   g | ]}|d kr|qS )rp   r'   r   r'   r'   r(   r     s      zFMultiModalAssessmentResult.get_hit_status_for_user.<locals>.<listcomp>)	r   intr_   r`   ra   lowerr   rO   values)r   rc   	user_data	user_item
total_hitscompleted_hitsr'   r'   r(   get_hit_status_for_user  s       z2MultiModalAssessmentResult.get_hit_status_for_userc                 C   sB   | j j|ddd}g }|D ]}|j|j }|| qtt|S )NFTr   )r_   r`   r   r   r   r   sum)r   rc   rd   	durationsr~   r   r'   r'   r(   get_time_for_user  s    z,MultiModalAssessmentResult.get_time_for_userc                    s  ddl m  tt}| jjdd}|dddD ]4}|d  d	krFq0|d }|d
 }|| | q0tt}|D ]J}t	jj
|d}d fdd|j D }|sd}|| ||  qri }	|D ]H}
t||
 }d}|D ]}||
 |dkr|d7 }q|t|f|	|
< q|	S )Nr   r   TrY   rZ   r   r   r   r      pk;c                    s    g | ]}|j   kr|j qS r'   r   rO   r   r   r'   r(   r     s   zLMultiModalAssessmentResult.compute_accurate_group_status.<locals>.<listcomp>NoGroupInforp   )Dashboard.modelsr   r   listr_   r`   ra   r   r   r   getjoingroupsrq   extendrb   r   r   )r   user_statusqsr~   annotatorIDtaskIDgroup_statusrc   
usergroups
group_hits
group_nametask_idscompleted_taskstask_idr'   r   r(   compute_accurate_group_status  s:    

z8MultiModalAssessmentResult.compute_accurate_group_statusc                    s  ddl m  tt}i }| jjdd}|ddddd	d
ddddddD ]*}|d }|d }|d }|d }	tt|	t| d}
|d }|d }d	|d |d }|d }|d }|d }|d }||kr|| d }|| d }|| d }nPt
jj|d}|j}|j}d fdd |j D }|s6d!}|||f||< ||d" |  |||||||||	|
||f qF|}d#g}|D ]0}|| D ] }|d$d%d  |D  qqdd&lm} dd'lm} ||d(|}t|d)&}|D ]}|| |d* qW 5 Q R X d S )+Nr   r   Tr   item__targetIDr   r   r   rZ   item__itemID*item__metadata__market__sourceLanguageCode*item__metadata__market__targetLanguageCode"item__metadata__market__domainNamer   r   task__campaign__campaignNamer   r   rL         {0}-{1}         r   
      r   r   c                    s    g | ]}|j   kr|j qS r'   r   r   r   r'   r(   r   H  s   zKMultiModalAssessmentResult.dump_all_results_to_csv_file.<locals>.<listcomp>r   -zotaskID,systemID,username,email,groups,segmentID,score,startTime,endTime,durationInSeconds,itemType,campaignName,c                 S   s   g | ]}t |qS r'   )rB   )r   ar'   r'   r(   r   j  s     )r   )BASE_DIRmediaw
)r   r   r   r   r_   r`   ra   r   floatrt   r   r   usernameemailr   r   rq   r   os.pathAppraise.settingsr  openwrite)r   csv_filesystem_scoresr   r   r~   systemIDr   r   r   r   r   	segmentIDmarketID
domainNameru   r   campaignNamer  	useremailr   rc   r   slir   r  media_file_pathoutfilecr'   r   r(   dump_all_results_to_csv_file  s    

"
z7MultiModalAssessmentResult.dump_all_results_to_csv_filec           	      C   s   t t}| jjddd}|dddddd	D ]N}|d
 }|d }|d }|d }d|d |d }|| ||||f q,|S )NTrn   CHKrY   item__itemType__inr   r   rZ   r   r   r   r   r   r   rL   r  r  r  )r   r   r_   r`   ra   rt   r   )	r   r  r   r~   r  r   r   r  r  r'   r'   r(   get_system_annotationsu  s"    z1MultiModalAssessmentResult.get_system_annotationsc                 C   s\   t t}| jjddd}|ddD ]2}|d d}|d }|D ]}|| | qBq$|S )	NTr'  r)  r   r   r   +r   )r   r   r_   r`   ra   rN   r   )r   r  r   r~   
system_idsr   	system_idr'   r'   r(   get_system_scores  s    z,MultiModalAssessmentResult.get_system_scoresrL   c              	      s   |   }d}dd |D }i }|D ]4 i | <  fdd|D D ]}|| |  |< q@q"i }|D ] tdd |   D }g }	|  D ]>}|  | }
t|
| }|	|t|
t|
t|
 |f qtt|	fdddd	| < q`|S )
N)defrc                 S   s   g | ]}d  |qS )zen-{0})rt   r   r'   r'   r(   r     s     z@MultiModalAssessmentResult.get_system_status.<locals>.<listcomp>c                    s   g | ]} |kr|qS r'   r'   r   )r   r'   r(   r     s      c                 S   s   g | ]}t |qS r'   )r   r   r'   r'   r(   r     s     c                    s   |   S rA   r'   )r   )
sort_indexr'   r(   <lambda>      z>MultiModalAssessmentResult.get_system_status.<locals>.<lambda>T)keyreverse)r/  r   r   r   r   r   sorted)r   r2  r  non_english_codescodesdatar5  output_datatotal_annotationsoutput_localr   zr'   )r   r2  r(   get_system_status  s(    $
z,MultiModalAssessmentResult.get_system_statusc                 C   s*   | j jdd||djddd}tt|S )NFT)rX   rY   rZ   task__campaignr[   r\   )r_   r`   ra   r   rb   )r   rc   rg   rd   r'   r'   r(   'completed_results_for_user_and_campaign  s     zBMultiModalAssessmentResult.completed_results_for_user_and_campaignN)T)rL   )r)   r*   r+   r,   r   r   r.   r   
FloatFieldr   r   r   r   r   rm   r5   rW   r   r   r   r   r   r   r   r   r&  r+  r/  r?  rA  r'   r'   r'   r(   r^     sh      	



(
Y

r^   )%r,   r   collectionsr   jsonr   zipfiler   r   django.contrib.auth.modelsr   	django.dbr   django.utils.textr   r/   django.utils.translationr	   r.   Appraise.utilsr
   r   r   ZEvalData.models.base_modelsr   r   r   r   r   r   r   r)   rv   r   registerr5   r^   r'   r'   r'   r(   <module>   s2   
7  G