# install presidio via pip if not yet installed

#!pip install presidio-analyzer
#!pip install presidio-evaluator

import datetime
import pprint
from collections import Counter

import numpy as np
import pandas as pd

from presidio_evaluator import InputSample
from presidio_evaluator.data_generator import PresidioSentenceFaker

sentence_templates = [
    "My name is {{name}}",
    "Please send it to {{address}}",
    "I just moved to {{city}} from {{country}}",
]


sentence_faker = PresidioSentenceFaker(
    "en_US", lower_case_ratio=0.05, sentence_templates=sentence_templates
)
fake_sentence_results = sentence_faker.generate_new_fake_sentences(10)

# Print the spans of the first sample
print(fake_sentence_results[0].masked)
print(fake_sentence_results[0].spans)

Using default entity providers
Using default entity mapping between the entities in the templates and the ones in the output dataset
Using default provider aliases


Sampling: 100%|██████████| 10/10 [00:00<00:00, 4370.89it/s]

I just moved to {{GPE}} from {{GPE}}
[Span(type: GPE, value: Spain, char_span: [45: 50]), Span(type: GPE, value: Valverde de Valdelacasa, char_span: [16: 39])]

number_of_samples = 1500
lower_case_ratio = 0.05
locale = "en"
cur_time = datetime.date.today().strftime("%B_%d_%Y")

output_file = f"../data/generated_size_{number_of_samples}_date_{cur_time}.json"
output_conll = f"../data/generated_size_{number_of_samples}_date_{cur_time}.tsv"

sentence_faker = PresidioSentenceFaker("en_US", lower_case_ratio=0.05)

Using default entity providers
Using default entity mapping between the entities in the templates and the ones in the output dataset
Using default provider aliases

pd.DataFrame(sentence_faker._sentence_faker.records).head()

import random

from faker.providers import BaseProvider


class MarsIdProvider(BaseProvider):
    def mars_id(self):
        # Generate a random row number between 1 and 50
        row = random.randint(1, 50)
        # Generate a random letter for the seat location from A-K
        location = random.choice("ABCDEFGHIJK")
        # Return the seat in the format "row-letter" (e.g., "25A")
        return f"{row}{location}"


sentence_faker.add_provider(MarsIdProvider)
# Now a new `mars_id` entity can be generated if a template has `mars_id` in it.

from presidio_evaluator.data_generator.faker_extensions.providers import *

IpAddressProvider  # Both Ipv4 and IPv6 IP addresses
NationalityProvider  # Read countries + nationalities from file
OrganizationProvider  # Read organization names from file
UsDriverLicenseProvider  # Read US driver license numbers from file
AgeProvider  # Age values (unavailable on Faker
AddressProviderNew  # Extend the default address formats
PhoneNumberProviderNew  # Extend the default phone number formats
ReligionProvider  # Read religions from file

presidio_evaluator.data_generator.faker_extensions.providers.ReligionProvider

# Create entity aliases (e.g. if your provider supports "name" but templates contain "person").
provider_aliases = PresidioSentenceFaker.PROVIDER_ALIASES
provider_aliases

# To customize, call `PresidioSentenceFaker(locale="en_US",...,provider_aliases=provider_aliases)`

[('name', 'person'),
 ('credit_card_number', 'credit_card'),
 ('date_of_birth', 'birthday')]

fake_records = sentence_faker.generate_new_fake_sentences(num_samples=number_of_samples)
pprint.pprint(fake_records[0])

Sampling: 100%|██████████| 1500/1500 [00:00<00:00, 8316.22it/s]

Full text: I'll meet you at 323 Postbox 78
 Apt. 637
 Slædepatruljen Sirius
 Greenlander after the concert.
Spans: [Span(type: STREET_ADDRESS, value: 323 Postbox 78
 Apt. 637
 Slædepatruljen Sirius
 Greenlander, char_span: [17: 77])]

count_per_template_id = Counter([sample.template_id for sample in fake_records])

print(f"Total: {sum(count_per_template_id.values())}")
print(f"Avg # of records per template: {np.mean(list(count_per_template_id.values()))}")
print(
    f"Median # of records per template: {np.median(list(count_per_template_id.values()))}"
)
print(f"Std: {np.std(list(count_per_template_id.values()))}")

Total: 1500
Avg # of records per template: 7.142857142857143
Median # of records per template: 7.0
Std: 2.7513756608669206

count_per_entity = Counter()
for record in fake_records:
    count_per_entity.update(Counter([span.entity_type for span in record.spans]))

count_per_entity

Counter({'PERSON': 875,
         'STREET_ADDRESS': 647,
         'GPE': 462,
         'ORGANIZATION': 260,
         'CREDIT_CARD': 146,
         'PHONE_NUMBER': 101,
         'DATE_TIME': 96,
         'TITLE': 88,
         'AGE': 73,
         'NRP': 61,
         'EMAIL_ADDRESS': 47,
         'ZIP_CODE': 39,
         'DOMAIN_NAME': 28,
         'IBAN_CODE': 22,
         'US_SSN': 11,
         'IP_ADDRESS': 11,
         'US_DRIVER_LICENSE': 11})

for record in fake_records[:10]:
    print(record)

Full text: I'll meet you at 323 Postbox 78
 Apt. 637
 Slædepatruljen Sirius
 Greenlander after the concert.
Spans: [Span(type: STREET_ADDRESS, value: 323 Postbox 78
 Apt. 637
 Slædepatruljen Sirius
 Greenlander, char_span: [17: 77])]

Full text: The Adomos SA Orchestra was founded in 2014. Since then, it has grown from a volunteer community orchestra to a fully professional orchestra serving Portugal
Spans: [Span(type: GPE, value: Portugal, char_span: [149: 157]), Span(type: DATE_TIME, value: 2014, char_span: [39: 43]), Span(type: ORGANIZATION, value: Adomos SA, char_span: [4: 13])]

Full text: It's like that since 4/26/1954
Spans: [Span(type: DATE_TIME, value: 4/26/1954, char_span: [21: 30])]

Full text: One of the most depressing songs on the list. He's injured from the waist down from New Zealand, but Rinoka just has to get laid. Don't go to town, Lisa!
Spans: [Span(type: PERSON, value: Lisa, char_span: [148: 152]), Span(type: PERSON, value: Rinoka, char_span: [101: 107]), Span(type: GPE, value: New Zealand, char_span: [84: 95])]

Full text: Celebrating its 10th year in Maniitsoq, Marshall, Hernandez and Simpson is a 501(c)3 that invites songwriters from around the world to Tomášhaven to share the universal language of music in collaborations designed to bridge cultures, build friendships and cultivate peace.
Spans: [Span(type: GPE, value: Tomášhaven, char_span: [135: 145]), Span(type: ORGANIZATION, value: Marshall, Hernandez and Simpson, char_span: [40: 71]), Span(type: GPE, value: Maniitsoq, char_span: [29: 38])]

Full text: I would like to remove my kid Milada from the will. How do I do that?
Spans: [Span(type: PERSON, value: Milada, char_span: [30: 36])]

Full text: A great song made even greater by a mandolin coda (not by Hugolina Cazares).
Spans: [Span(type: PERSON, value: Hugolina Cazares, char_span: [58: 74])]

Full text: Who's coming to New Zealand with me?
Spans: [Span(type: GPE, value: New Zealand, char_span: [16: 27])]

Full text: For my take on Ms. Portič, see Guilty Pleasures: 5 Musicians Of The 70s You're Supposed To Hate (But Secretly Love)
Spans: [Span(type: PERSON, value: Portič, char_span: [19: 25]), Span(type: TITLE, value: Ms., char_span: [15: 18])]

Full text: Blink-182 pay tribute here to the Switzerland. Producer Jiří Lukášek explained to Fuse TV: "We all liked the idea of writing a song about our state, where we live and love. To me it's the most beautiful place in the world, this song was us giving credit to how lucky we are to have lived here and grown up here, raising families here, the whole thing."
Spans: [Span(type: PERSON, value: Jiří Lukášek, char_span: [56: 68]), Span(type: GPE, value: Switzerland, char_span: [34: 45])]

InputSample.to_json(dataset=fake_records, output_file=output_file)

output_file

'../data/generated_size_1500_date_January_08_2025.json'

conll = InputSample.create_conll_dataset(dataset=fake_records)
conll.head(10)

  0%|          | 0/1500 [00:00<?, ?it/s]

loading model en_core_web_sm


100%|██████████| 1500/1500 [00:04<00:00, 320.23it/s]

conll.to_csv(output_conll, sep="\t")
print(f"CoNLL2003 dataset structure output location: {output_conll}")

CoNLL2003 dataset structure output location: ../data/generated_size_1500_date_January_08_2025.tsv