Subject Request API

import requests
import csv
import time
import os

API_URL = "https://api.peopledatalabs.com/v5/person/subjectrequest"

def fetch_subject_requests(api_key: str):
    """Fetch subject requests from PDL API as a stream."""
    try:
        response = requests.get(API_URL, headers={'x-api-key': api_key}, stream=True)
        response.raise_for_status()

        for line in response.iter_lines(decode_unicode=True):
            if line:
                yield line.strip()
    except requests.RequestException as e:
        print(f"Error fetching data: {e}")
        raise

def save_to_csv(api_key: str, output_file: str, batch_size: int = 1000):
    """Save fetched subject requests to a CSV file in batches."""
    start_time = time.time()
    count = 0
    batch = []

    try:
        with open(output_file, 'w', newline='') as file:
            writer = csv.writer(file)

            for pdl_id in fetch_subject_requests(api_key):
                batch.append([pdl_id])
                count += 1

                if len(batch) >= batch_size:
                    writer.writerows(batch)
                    batch.clear()
                    print(f"Processed {count} IDs...")

            if batch:
                writer.writerows(batch)

        print(f"Saved {count} IDs to {output_file} in {time.time() - start_time:.2f} seconds")
    except Exception as e:
        print(f"Error saving data: {e}")
        raise

if __name__ == "__main__":
    api_key = os.getenv("API_KEY")
    output_file = "opt-out.csv"

    if not api_key:
        print("API_KEY is missing. Set it as an environment variable.")
    else:
        try:
            save_to_csv(api_key, output_file)
        except Exception as e:
            print(f"Script failed: {e}")