
    >Tfz3                        d dl Z d dlZd dlZd dlmZ d dlmZ d dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d dl m!Z! d dl"m#Z# ddl$T ddl%m&Z& dZ' ej(        d          Z) G d dee          Z* G d dej+                  Z, G d dej+                  Z-dS )    N)fnmatchcase)urlparse)settings)GenericForeignKey)ValidationError)RegexValidator)models)reverse)timezone)gettext)CENSOR_TOKENCENSOR_TOKEN_CHANGED)PrimaryModel)	JobsMixin)registry)RestrictedQuerySet   )*)	SyncError)AutoSyncRecordDataFile
DataSourceznetbox.core.datac                       e Zd ZdZ ej         ed          dd          Z ej         ed          d          Z ej        d	 ed
                    Z	 ej         ed          de
e
j        d          Z ej         ed          d          Z ej         ed          d ed                    Z ej         ed          dd          Z ej         ed          ddd          Z G d d          Zd Zd Zed             Zd Zd Zed             Zed              Zed!             Z fd"Z fd#Z d$ Z!d% Z"de"_#        d& Z$d' Z% xZ&S )(r   z[
    A remote source, such as a git repository, from which DataFiles are synchronized.
    named   T)verbose_name
max_lengthuniquetype2   )r   r      URL)r   r   statusF)r   r   choicesdefaulteditableenabled)r   r%   zignore rulesz=Patterns (one per line) matching files to ignore when syncing)r   blank	help_text
parameters)r   r(   nullzlast synced)r   r(   r+   r&   c                   >    e Zd ZdZ ed          Z ed          ZdS )DataSource.Metar   zdata sourcezdata sourcesN)__name__
__module____qualname__ordering_r   verbose_name_plural     5/var/www/html/netbox-4.1.3/netbox/core/models/data.pyMetar-   O   s4        q''a//r6   r8   c                     | j          S Nr.   selfs    r7   __str__zDataSource.__str__T   s    )~r6   c                 0    t          d| j        g          S )Nzcore:datasourceargsr
   pkr;   s    r7   get_absolute_urlzDataSource.get_absolute_urlW   s    (y9999r6   c                 T    t           j         d| j        j         d| j        j         dS )Nzdocs/models//)r   
STATIC_URL_meta	app_label
model_namer;   s    r7   docs_urlzDataSource.docs_urlZ   s.    %bb4:3Gbb$*J_bbbbr6   c                 b    t           d                             | j                  x}r|j        S d S Ndata_backends)r   getr   label)r<   backends     r7   get_type_displayzDataSource.get_type_display^   s6    /33DI>>>7 	!= 	! 	!r6   c                 J    t           j                            | j                  S r:   )DataSourceStatusChoicescolorsrN   r#   r;   s    r7   get_status_colorzDataSource.get_status_colorb   s    &-11$+>>>r6   c                 X    t          | j                  j                                        S r:   )r   
source_urlschemelowerr;   s    r7   
url_schemezDataSource.url_schemee   s!    ((/55777r6   c                 L    t           d                             | j                  S rL   )r   rN   r   r;   s    r7   backend_classzDataSource.backend_classi   s    (,,TY777r6   c                 N    | j         o| j        t          j        t          j        fvS r:   )r'   r#   rS   QUEUEDSYNCINGr;   s    r7   ready_for_synczDataSource.ready_for_syncm   s+    | 
#*#+4
 !
 	
r6   c                 <   t                                                       | j        rK| j        t          d         vr7t	          dt          d                    | j                            i          | j        j        r| j	        dvrt	          ddi          d S d S )NrM   r   zUnknown backend type: {type})r   )file rW   zEURLs for local sources must start with file:// (or specify no scheme))
supercleanr   r   r   r3   formatr\   is_localrZ   )r<   	__class__s    r7   re   zDataSource.cleant   s     9 	(?*CCC!8??TY?OOPP#   
 & 	4?,+N+N!f#   	 	+N+Nr6   c                    t                                          |          }i }i }|j        r|j                            d          pi }|j        r|j                            d          pi }| j        j        D ]j}|                    |          r4||         |                    |          k    rt          ||<   n
t          ||<   |                    |          r
t          ||<   k|S )Nr*   )	rd   to_objectchangeprechange_datarN   postchange_datar\   sensitive_parametersr   r   )r<   actionobjectchangepre_change_paramspost_change_paramsparamrh   s         r7   rj   zDataSource.to_objectchange   s   ww..v66 & 	T , ; ? ? M M SQS' 	V!-!=!A!A,!O!O!USU'< 	8 	8E!%%e,, =%e,0A0E0Ee0L0LLL0D&u--0<&u- $$U++ 8+7!%(r6   c                 :    | j         pi } | j        | j        fi |S r:   )r*   r\   rW   )r<   backend_paramss     r7   get_backendzDataSource.get_backend   s+    .B!t!$/DD^DDDr6   c                    ddl m}m} | j        t          j        k    rt           |d                    |                    | j        |            t          j        | _        t          j
                            | j                                      | j                   	 |                                 }n:# t          $ r-}t           |d          t!          |          z             d}~ww xY w|                                5 }t$                              d	|            | j                                        }d
 |D             }t$                              dt-          |           d           g }	g }
|D ]X}	 |                    |          r|	                    |           /# t2          $ r |
                    |j                   Y Uw xY wt4          j
                            |	d          }t$                              d| d           t4          j
                            |
                                          \  }}t$                              d| d           |                     |          |z
  }g }|D ]R}t5          | |          }|                    |           |                                 |                    |           St-          t4          j
                            |d                    }t$                              d| d           ddd           n# 1 swxY w Y   t          j         | _        tC          j"                    | _#        t          j
                            | j                                      | j        | j#                   |                    | j        |            dS )zj
        Create/update/delete child DataFiles as necessary to synchronize with the remote source.
        r   )	post_syncpre_syncz2Cannot initiate sync; syncing already in progress.)senderinstance)rB   )r#   zQThere was an error initializing the backend. A dependency needs to be installed: NzSyncing files from source root c                     h | ]	}|j         
S r5   path).0dfs     r7   	<setcomp>z"DataSource.sync.<locals>.<setcomp>   s    888r27888r6   zStarting with z known files)source_root)last_updatedsizehashdatazUpdated  files)pk__inzDeleted sourcer}   r   )
batch_sizezCreated z data files)r#   last_synced)$core.signalsrw   rx   r#   rS   r_   r   sendrh   r   objectsfilterrB   updateru   ModuleNotFoundErrorstrfetchloggerdebug	datafilesalllenrefresh_from_diskappendFileNotFoundErrorr   bulk_updatedelete_walk
full_cleanbulk_create	COMPLETEDr   nowr   )r<   rw   rx   r3   rP   e
local_path
data_filesknown_pathsupdated_filesdeleted_file_idsdatafileupdated_countdeleted_count	new_pathsnew_datafilesr}   created_counts                     r7   synczDataSource.sync   s    	54444444;1999AARSSTTT 	T^d;;;-5!!TW!--44DK4HHH	&&((GG" 	 	 	effilmnioioo  	 ]]__ '	@
LLG:GGHHH++--J88Z888KLLH#k*:*:HHHIII M!&  11j1II 7%,,X666(   $++HK888H %,88HpqqMLL9M999:::  (/66>N6OOVVXXM1LL9M999::: 

:..<I M! / /#4d;;;**z*BBB##%%%$$X.... 0 < <]WZ < [ [\\MLL>M>>>???O'	@ '	@ '	@ '	@ '	@ '	@ '	@ '	@ '	@ '	@ '	@ '	@ '	@ '	@ '	@T .7#<>>!!TW!--44DKUYUe4fff 	dnt<<<<<sP   "B7 7
C.(C))C.A6L*<+F('L*($GL*GEL**L.1L.c                    t                               d| d           t                      }t          j        |          D ]\  }}}|                    |          d                             d          }|                    d          rJ|D ]J}|                     |          s3|	                    t          j
                            ||                     Kt                               dt          |           d           |S )zN
        Return a set of all non-excluded files within the root path.
        zWalking z...   rE   .zFound r   )r   r   setoswalksplitlstrip
startswith_ignoreaddr}   joinr   )r<   rootpathsr}   	dir_names
file_names	file_names          r7   r   zDataSource._walk   s     	))))***+-74== 	= 	='D)Z::d##A&--c22Ds## ' = =	||I.. =IIbgll4;;<<<= 	0c%jj000111r6   c                     |                     d          rdS | j                                        D ]}t          ||          r dS dS )z
        Returns a boolean indicating whether the file should be ignored per the DataSource's configured
        ignore rules.
        r   TF)r   ignore_rules
splitlinesr   )r<   filenamerules      r7   r   zDataSource._ignore   s`    
 s## 	4%0022 	 	D8T** ttur6   )'r/   r0   r1   __doc__r	   	CharFieldr3   r   r   rW   rS   NEWr#   BooleanFieldr'   	TextFieldr   	JSONFieldr*   DateTimeFieldr   r8   r=   rC   propertyrJ   rQ   rU   rZ   r\   r`   re   rj   ru   r   alters_datar   r   __classcell__)rh   s   @r7   r   r   "   s         6QvYY  D
 6QvYY  D "!QuXX  J VQx[[''+  F "f!Qy\\  G $6#Q~&&!STT  L
 "!Q|__  J
 '&&Q}%%	  K0 0 0 0 0 0 0 0
  : : : c c Xc! ! !? ? ? 8 8 X8 8 8 X8 
 
 X
        ,E E EE= E= E=L D  $
 
 
 
 
 
 
r6   r   c                   6   e Zd ZdZ ej         ed          d          Z ej         ed          d          Z ej	        dej
        d	d
          Z ej         ed          dd ed                    Z ej        d ed                    Z ej         ed          dd ed ed                    g ed                    Z ej                    Z ej                    Z G d d          Zd Zd Zed             Zd Zd Zd!dZd S )"r   z
    The database representation of a remote file fetched from a remote DataSource. DataFile instances should be created,
    updated, or deleted only by calling DataSource.sync().
    createdT)r   auto_now_addzlast updatedF)r   r&   zcore.DataSourcer   )to	on_deleterelated_namer&   r}   i  z,File path relative to the data source's root)r   r   r&   r)   r   )r&   r   r   @   z^[0-9a-f]{64}$z)Length must be 64 hexadecimal characters.)regexmessagezSHA256 hash of the file data)r   r   r&   
validatorsr)   c                       e Zd ZdZ ej        dd          fZ ej        dd          gZ e	d          Z
 e	d          ZdS )DataFile.Metar   z*%(app_label)s_%(class)s_unique_source_pathfieldsr   core_datafile_source_pathz	data filez
data filesN)r/   r0   r1   r2   r	   UniqueConstraintconstraintsIndexindexesr3   r   r4   r5   r6   r7   r8   r   .  su        %#F#)A  
 FL 29TUUU
 q~~aoor6   r8   c                     | j         S r:   r|   r;   s    r7   r=   zDataFile.__str__<  s
    yr6   c                 0    t          d| j        g          S )Nzcore:datafiler?   rA   r;   s    r7   rC   zDataFile.get_absolute_url?  s    dgY7777r6   c                 l    | j         sd S 	 | j                             d          S # t          $ r Y d S w xY w)Nzutf-8)r   decodeUnicodeDecodeErrorr;   s    r7   data_as_stringzDataFile.data_as_stringB  sN    y 	4	9##G,,,! 	 	 	44	s   % 
33c                 4    t          j        | j                  S )z_
        Attempt to read the file data as JSON/YAML and return a native Python object.
        )yaml	safe_loadr   r;   s    r7   get_datazDataFile.get_dataK  s    
 ~d1222r6   c                 2   t           j                            || j                  }t          |d          5 }t	          j        |                                                                          }ddd           n# 1 swxY w Y   || j        k    x}rt          j
                    | _        t           j                            |          | _        || _        t          |d          5 }|                                | _        ddd           n# 1 swxY w Y   |S )zv
        Update instance attributes from the file on disk. Returns True if any attribute
        has changed.
        rbN)r   r}   r   openhashlibsha256read	hexdigestr   r   r   r   getsizer   r   )r<   r   	file_pathf	file_hashis_modifieds         r7   r   zDataFile.refresh_from_diskR  sY   
 GLLdi88	)T"" 	=aqvvxx00::<<I	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= $ty00; 	% (D	22DI!DIi&& %!FFHH	% % % % % % % % % % % % % % % s#   9A;;A?A?&DDDc                     t           j                            |          r|st                      t	          |d          5 }|                    | j                   ddd           dS # 1 swxY w Y   dS )zG
        Write the object's data to disk at the specified path
        zwb+N)r   r}   isfileFileExistsErrorr   writer   )r<   r}   	overwritenew_files       r7   write_to_diskzDataFile.write_to_diske  s    
 7>>$ 	$	 	$!###$ 	&(NN49%%%	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	&s    A((A,/A,N)F)r/   r0   r1   r   r	   r   r3   r   r   
ForeignKeyCASCADEr   r   r}   PositiveIntegerFieldr   r   r   BinaryFieldr   r   
as_managerr   r8   r=   rC   r   r   r   r   r   r5   r6   r7   r   r     s         #f"Qy\\  G (6'Q~&&  L V. 	  F 6QvYY!BCC	  D '6&QvYY  D 6QvYYN!111=h;i;ijjj
 !233  D 6D+ +--G. . . . . . . .  8 8 8   X3 3 3  &	& 	& 	& 	& 	& 	&r6   r   c                       e Zd ZdZ ej        eej        d          Z ej        dej        d          Z	 ej
                    Z edd          ZdZ G d	 d
          ZdS )r   zN
    Maps a DataFile to a synced object for efficient automatic updating.
    +)r   r   r   zcontenttypes.ContentTypeobject_type	object_id)ct_fieldfk_fieldTc                       e Zd Z ej        dd          fZ ej        d          fZ ed          Z	 ed          Z
dS )AutoSyncRecord.Meta)r  r  z%(app_label)s_%(class)s_objectr   )r   zauto sync recordzauto sync recordsN)r/   r0   r1   r	   r   r   r   r   r3   r   r4   r5   r6   r7   r8   r    ss        #F#35  
 FL <===
 q+,,a 344r6   r8   N)r/   r0   r1   r   r	   r   r   r  r   r  PositiveBigIntegerFieldr  r   object_netbox_privater8   r5   r6   r7   r   r   q  s          !v .  H
 $&#%.  K
 /.00I  F
 O5 5 5 5 5 5 5 5 5 5r6   r   ).r   loggingr   fnmatchr   urllib.parser   r   django.confr   "django.contrib.contenttypes.fieldsr   django.core.exceptionsr   django.core.validatorsr   	django.dbr	   django.urlsr
   django.utilsr   django.utils.translationr   r3   netbox.constantsr   r   netbox.modelsr   netbox.models.featuresr   netbox.registryr   utilities.querysetsr   r$   
exceptionsr   __all__	getLoggerr   r   Modelr   r   r5   r6   r7   <module>r$     s     				       ! ! ! ! ! !              @ @ @ @ @ @ 2 2 2 2 2 2 1 1 1 1 1 1             ! ! ! ! ! ! 1 1 1 1 1 1 ? ? ? ? ? ? ? ? & & & & & & , , , , , , $ $ $ $ $ $ 2 2 2 2 2 2     " " " " " " 
	-	.	._ _ _ _ _L _ _ _Dj& j& j& j& j&v| j& j& j&Z!5 !5 !5 !5 !5V\ !5 !5 !5 !5 !5r6   