Models:
Philip Bordner
philipB/
deidentify
0
Downloads: 1
[7fc5df]: / tests / surrogates / generators / test_location.py

Download this file

266 lines (224 with data), 8.6 kB 

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
from deidentify.surrogates.generators import LocationSurrogates, RandomData

from deidentify.surrogates.generators.location import (NUMBER_REGEX, ZIP_REGEX,

                                                       Location,

                                                       LocationDatabase,

                                                       _strip, parse_location)



from .util import RandomDataMock





def test_location_database():

    location_database = LocationDatabase()

    assert len(location_database.countries) >= 255

    assert 'Marokko' in location_database.countries

    assert 'Duitsland' in location_database.countries



    assert len(location_database.places) >= 2400

    assert 'Amsterdam' in location_database.places

    assert 'Enschede' in location_database.places

    assert 'Bennebroek' in location_database.places

    assert 'Haarlem' in location_database.places



    assert len(location_database.zip_codes) >= 452000

    assert '7141DC' in location_database.zip_codes



    assert len(location_database.streetnames) >= 127000

    assert 'Parallelweg' in location_database.streetnames

    assert 'Laan van Meerdervoort' in location_database.streetnames





def test_zip_regex():

    assert ZIP_REGEX.match('1234AB').group(0) == '1234AB'

    assert ZIP_REGEX.match('1234 AB').group(0) == '1234 AB'

    assert ZIP_REGEX.match('1234 ab').group(0) == '1234 ab'





def test_number_regex():

    assert NUMBER_REGEX.search('Parallelweg 122-3 a Groenlo').group(0) == '122-3 a'

    assert NUMBER_REGEX.search('Parallelweg 122-3 Groenlo').group(0) == '122-3'

    assert NUMBER_REGEX.search('Parallelweg 122A').group(0) == '122A'

    assert NUMBER_REGEX.search('Parallelweg 1').group(0) == '1'





def test_strip():

    assert _strip(' This is, a test, ') == 'This is, a test'

    assert _strip('This is, a test') == 'This is, a test'

    assert _strip('This is, a test,') == 'This is, a test'





def test_parse_location():

    # Case 1: ZIP Code => Left of ZIP is street, right is place and country

    assert parse_location('Parallelweg 2, 7141 DC Groenlo') == Location(

        raw='Parallelweg 2, 7141 DC Groenlo',

        country='',

        zip_code='7141 DC',

        place='Groenlo',

        street='Parallelweg',

        house_number='2')



    assert parse_location('Parallelweg 2, 7141 DC') == Location(

        raw='Parallelweg 2, 7141 DC',

        country='',

        zip_code='7141 DC',

        place='',

        street='Parallelweg',

        house_number='2')



    assert parse_location('7141DC') == Location(

        raw='7141DC', country='', zip_code='7141DC', place='', street='', house_number='')



    assert parse_location('7141 DC Groenlo') == Location(

        raw='7141 DC Groenlo', country='', zip_code='7141 DC', place='Groenlo', street='',

        house_number=''

    )



    assert parse_location('7141 DC te Haarlem Nederland') == Location(

        raw='7141 DC te Haarlem Nederland',

        country='Nederland',

        zip_code='7141 DC',

        place='te Haarlem',

        street='',

        house_number=''

    )



    # Case 2: No ZIP but number, split into left and right. Left of number (inclusive) is street,

    # right is place and country.

    assert parse_location('Parallelweg 2, Groenlo') == Location(

        raw='Parallelweg 2, Groenlo',

        country='',

        zip_code='',

        place='Groenlo',

        street='Parallelweg',

        house_number='2')



    assert parse_location('Parallelweg 2') == Location(

        raw='Parallelweg 2',

        country='',

        zip_code='',

        place='',

        street='Parallelweg',

        house_number='2')



    assert parse_location('Parallelweg 2A') == Location(

        raw='Parallelweg 2A',

        country='',

        zip_code='',

        place='',

        street='Parallelweg',

        house_number='2A')



    assert parse_location('Waterkant 11-3') == Location(

        raw='Waterkant 11-3',

        country='',

        zip_code='',

        place='',

        street='Waterkant',

        house_number='11-3')



    assert parse_location('Parallelweg 2 te Haarlem Nederland') == Location(

        raw='Parallelweg 2 te Haarlem Nederland',

        country='Nederland',

        zip_code='',

        place='te Haarlem',

        street='Parallelweg',

        house_number='2')



    # Case 3: no number, no ZIP code, split on road suffixes (e.g., weeg, straat). Left of suffix

    # is street, right is place and country.

    assert parse_location('Parallelweg Groenlo') == Location(

        raw='Parallelweg Groenlo',

        country='',

        zip_code='',

        place='Groenlo',

        street='Parallelweg',

        house_number='')



    assert parse_location('Parallelweg') == Location(

        raw='Parallelweg',

        country='',

        zip_code='',

        place='',

        street='Parallelweg',

        house_number='')



    assert parse_location('Parallelweg te Haarlem Nederland') == Location(

        raw='Parallelweg te Haarlem Nederland',

        country='Nederland',

        zip_code='',

        place='te Haarlem',

        street='Parallelweg',

        house_number='')



    assert parse_location('De singel te Haarlem Nederland') == Location(

        raw='De singel te Haarlem Nederland',

        country='Nederland',

        zip_code='',

        place='te Haarlem',

        street='De singel',

        house_number='')



    # Case 4: no number, no ZIP code, no street suffix => assume everything is a place,

    # except for names of countries

    assert parse_location('De diagonaal te Haarlem Nederland') == Location(

        raw='De diagonaal te Haarlem Nederland',

        country='Nederland',

        zip_code='',

        place='De diagonaal te Haarlem',

        street='',

        house_number='')



    assert parse_location('Groenlo') == Location(

        raw='Groenlo',

        country='',

        zip_code='',

        place='Groenlo',

        street='',

        house_number='')



    assert parse_location('Oostenrijk') == Location(

        raw='Oostenrijk',

        country='Oostenrijk',

        zip_code='',

        place='',

        street='',

        house_number='')



    assert parse_location('CARL MUCKSTRAAT') == Location(

        raw='CARL MUCKSTRAAT',

        country='',

        zip_code='',

        place='',

        street='CARL MUCKSTRAAT',

        house_number=''

    )



    assert parse_location('Waterkant 28-3, 7521PL Enschede') == Location(

        raw='Waterkant 28-3, 7521PL Enschede',

        country='',

        zip_code='7521PL',

        place='Enschede',

        street='Waterkant',

        house_number='28-3'

    )



    assert parse_location('Indonesië (Molukken)') == Location(

        raw='Indonesië (Molukken)',

        country='Indonesië',

        zip_code='',

        place='Molukken',

        street='',

        house_number=''

    )



    assert parse_location('De Giezen 15-002, 7461 BB') == Location(

        raw='De Giezen 15-002, 7461 BB',

        country='',

        zip_code='7461 BB',

        place='',

        street='De Giezen',

        house_number='15-002'

    )



    assert parse_location('Arnhem-Zuid') == Location(

        raw='Arnhem-Zuid',

        country='',

        zip_code='',

        place='Arnhem-Zuid',

        street='',

        house_number=''

    )



    assert parse_location('De Giezen 15-002, Arnhem-Zuid') == Location(

        raw='De Giezen 15-002, Arnhem-Zuid',

        country='',

        zip_code='',

        place='Arnhem-Zuid',

        street='De Giezen',

        house_number='15-002'

    )





def test_replace_all():

    locations = [

        ('7141DC', 'Groenlo', 'Parallelweg'),

    ]

    location_database = LocationDatabase(locations=locations)



    given_expected = [

        ('Waterkant 28-3, 7521PL Enschede', 'Parallelweg 11-1, 7141DC Groenlo'),

        ('Waterkant 28-3',                  'Parallelweg 11-1'),

        ('7521PL Enschede',                 '7141DC Groenlo'),

        ('7521 PL Enschede',                '7141 DC Groenlo'),

        ('Enschede',                        'Groenlo'),

        ('Arnhem-Zuid',                     'Groenlo'),

        # countries are be completely ignored during surrogate generation

        ('Oostenrijk',                      'Oostenrijk'),

        ('Duitsland',                       'Duitsland')

    ]



    annotations = [given for given, _ in given_expected]

    expected = [expected for _, expected in given_expected]



    location_surrogates = LocationSurrogates(annotations, location_database=location_database,

                                             random_data=RandomDataMock())

    surrogates = location_surrogates.replace_all()

    assert surrogates == expected