# Code Review Request: Tweet Filter & Merge Tools

**Please review these 2 Python CLI scripts for correctness, edge cases, and robustness.** These are utility scripts for filtering and merging tweet CSV files. Focus on:

1. **CSV handling**: Proper quoting for content with commas, quotes, newlines
2. **Error handling**: Edge cases like empty files, missing fields, None values
3. **Filename safety**: Special characters in usernames/content shouldn't break filenames
4. **Type safety**: Handling of None, missing keys, type mismatches
5. **Memory efficiency**: Any issues with large files?

---

## 1. tweet-filter

Filters tweets from a CSV by username or content search.

```python
#!/usr/bin/env python3
"""
Filter tweets from a CSV by name or content.

Usage:
  tweet-filter name <username> <csv-file> [-o <output>]
  tweet-filter content <search-string> <csv-file> [-o <output>]

Examples:
  tweet-filter name @sama feed.csv
  tweet-filter name sama feed.csv -o sama_tweets.csv
  tweet-filter content "claude code" feed.csv
  tweet-filter content AI feed.csv -o ai_tweets.csv
"""

import argparse
import csv
import re
import sys
from pathlib import Path


def safe_str(value):
    """Safely convert value to string, handling None."""
    if value is None:
        return ''
    return str(value)


def filter_by_name(rows, username):
    """Filter rows by username (case-insensitive, handles @ prefix)."""
    # Strip @ if present and lowercase
    username = username.lstrip('@').lower()
    results = []
    for row in rows:
        row_username = safe_str(row.get('username', '')).lstrip('@').lower()
        if row_username == username:
            results.append(row)
    return results


def filter_by_content(rows, search_string):
    """Filter rows by content in text and image_description (case-insensitive)."""
    search_lower = search_string.lower()
    results = []
    for row in rows:
        # Check main content/text field (could be named 'content' or 'text')
        text = safe_str(row.get('content', '')) or safe_str(row.get('text', ''))
        text = text.lower()

        # Check image description if it exists
        image_desc = safe_str(row.get('image_description', '')).lower()

        if search_lower in text or search_lower in image_desc:
            results.append(row)
    return results


def sanitize_filename(s, max_length=30):
    """
    Sanitize a string for use in filenames.
    Preserves @ for usernames, replaces other special chars with underscore.
    """
    # Keep alphanumeric, underscore, hyphen, and @
    sanitized = re.sub(r'[^\w\-@]', '_', s)
    # Collapse multiple underscores
    sanitized = re.sub(r'_+', '_', sanitized)
    # Strip leading/trailing underscores
    sanitized = sanitized.strip('_')
    # Truncate
    if len(sanitized) > max_length:
        sanitized = sanitized[:max_length].rstrip('_')
    # Fallback if empty
    if not sanitized:
        sanitized = 'filtered'
    return sanitized


def main():
    parser = argparse.ArgumentParser(
        description='Filter tweets from CSV by name or content',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  tweet-filter name @sama raw_feed.csv
  tweet-filter name elonmusk raw_feed.csv -o elon.csv
  tweet-filter content "claude code" with_images.csv
  tweet-filter content AI raw_feed.csv -o ai_tweets.csv
        """
    )
    parser.add_argument('filter_type', choices=['name', 'content'],
                        help='Filter type: "name" for username, "content" for text search')
    parser.add_argument('search', help='Search string (username or content to find)')
    parser.add_argument('csv_file', help='Input CSV file')
    parser.add_argument('-o', '--output', help='Output file (default: auto-generated name)')

    args = parser.parse_args()

    # Validate input file
    input_path = Path(args.csv_file)
    if not input_path.exists():
        print(f"Error: File not found: {args.csv_file}", file=sys.stderr)
        sys.exit(1)

    # Read CSV
    try:
        with open(args.csv_file, 'r', newline='', encoding='utf-8') as f:
            reader = csv.DictReader(f)
            fieldnames = reader.fieldnames

            if not fieldnames:
                print("Error: CSV file has no header row", file=sys.stderr)
                sys.exit(1)

            rows = list(reader)
    except csv.Error as e:
        print(f"Error: Failed to parse CSV: {e}", file=sys.stderr)
        sys.exit(1)

    if not rows:
        print("Warning: Input CSV is empty", file=sys.stderr)

    # Filter
    if args.filter_type == 'name':
        filtered = filter_by_name(rows, args.search)
        filter_desc = f"username '{args.search}'"
    else:
        filtered = filter_by_content(rows, args.search)
        filter_desc = f"content containing '{args.search}'"

    # Determine output path
    if args.output:
        output_path = Path(args.output)
    else:
        # Auto-generate: tweets-from-@sama.csv or tweets-containing-AI.csv
        safe_search = sanitize_filename(args.search)
        if args.filter_type == 'name':
            # Preserve @ for usernames
            if not safe_search.startswith('@'):
                safe_search = f"@{safe_search.lstrip('@')}"
            output_name = f"tweets-from-{safe_search}.csv"
        else:
            output_name = f"tweets-containing-{safe_search}.csv"
        output_path = input_path.parent / output_name

    # Ensure output directory exists
    output_dir = output_path.parent
    if output_dir and str(output_dir) != '.' and not output_dir.exists():
        output_dir.mkdir(parents=True, exist_ok=True)

    # Write output with proper quoting for special characters
    try:
        with open(output_path, 'w', newline='', encoding='utf-8') as f:
            writer = csv.DictWriter(
                f,
                fieldnames=fieldnames,
                quoting=csv.QUOTE_MINIMAL,
                extrasaction='ignore'
            )
            writer.writeheader()
            writer.writerows(filtered)
    except IOError as e:
        print(f"Error: Failed to write output file: {e}", file=sys.stderr)
        sys.exit(1)

    print(f"Filtered by {filter_desc}")
    print(f"  Input:  {len(rows)} tweets")
    print(f"  Output: {len(filtered)} tweets")
    print(f"  File:   {output_path}")


if __name__ == '__main__':
    main()
```

---

## 2. tweet-csv-merge

Merges multiple CSV files, deduplicating by tweet_id.

```python
#!/usr/bin/env python3
"""
Merge multiple tweet CSVs into one, deduplicating by tweet_id.

Usage:
  tweet-csv-merge <csv1> <csv2> [csv3...] [-o <output>]

Examples:
  tweet-csv-merge day1.csv day2.csv -o week.csv
  tweet-csv-merge archive/*/raw_feed.csv -o all_tweets.csv
"""

import argparse
import csv
import sys
from pathlib import Path
from datetime import datetime


def safe_str(value):
    """Safely convert value to string, handling None."""
    if value is None:
        return ''
    return str(value)


def main():
    parser = argparse.ArgumentParser(
        description='Merge multiple tweet CSVs, deduplicating by tweet_id',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  tweet-csv-merge day1.csv day2.csv -o week.csv
  tweet-csv-merge feed1.csv feed2.csv feed3.csv
  tweet-csv-merge archive/2025-01-*/raw_feed.csv -o january.csv
        """
    )
    parser.add_argument('csv_files', nargs='+', help='CSV files to merge (shell expands globs)')
    parser.add_argument('-o', '--output', help='Output file (default: merged_TIMESTAMP.csv)')
    parser.add_argument('--keep', choices=['first', 'last'], default='first',
                        help='When duplicate tweet_id found, keep first or last occurrence (default: first)')

    args = parser.parse_args()

    # Validate input files
    valid_files = []
    for csv_file in args.csv_files:
        path = Path(csv_file)
        if path.exists():
            valid_files.append(path)
        else:
            print(f"Warning: Skipping non-existent file: {csv_file}", file=sys.stderr)

    if not valid_files:
        print("Error: No valid input files found", file=sys.stderr)
        sys.exit(1)

    # Collect all rows, tracking by tweet_id for deduplication
    all_rows = {}  # tweet_id -> row
    all_fieldnames = []  # Union of all fieldnames, preserving order
    seen_fieldnames = set()
    files_processed = 0
    total_rows_read = 0

    for csv_file in valid_files:
        try:
            with open(csv_file, 'r', newline='', encoding='utf-8') as f:
                reader = csv.DictReader(f)
                file_fieldnames = reader.fieldnames

                if not file_fieldnames:
                    print(f"Warning: Skipping file with no header: {csv_file}", file=sys.stderr)
                    continue

                # Union fieldnames, preserving order of first occurrence
                for fn in file_fieldnames:
                    if fn not in seen_fieldnames:
                        all_fieldnames.append(fn)
                        seen_fieldnames.add(fn)

                file_count = 0
                for row in reader:
                    tweet_id = safe_str(row.get('tweet_id', ''))
                    total_rows_read += 1
                    file_count += 1

                    if not tweet_id:
                        # No tweet_id, include it anyway with a generated key
                        key = f"__no_id_{total_rows_read}"
                        all_rows[key] = row
                    elif tweet_id not in all_rows:
                        all_rows[tweet_id] = row
                    elif args.keep == 'last':
                        all_rows[tweet_id] = row
                    # else: keep first (already in dict)

                files_processed += 1
                print(f"  Read {file_count} rows from {csv_file.name}")
        except csv.Error as e:
            print(f"Warning: CSV error reading {csv_file}: {e}", file=sys.stderr)
        except IOError as e:
            print(f"Warning: IO error reading {csv_file}: {e}", file=sys.stderr)

    if not all_fieldnames:
        print("Error: No valid CSV headers found in any file", file=sys.stderr)
        sys.exit(1)

    if not all_rows:
        print("Warning: No data found in any input files", file=sys.stderr)

    # Sort by timestamp (newest first), with tweet_id as secondary key for stability
    def sort_key(row):
        ts = safe_str(row.get('timestamp', '')) or safe_str(row.get('created_at', ''))
        tweet_id = safe_str(row.get('tweet_id', ''))
        return (ts, tweet_id)

    sorted_rows = sorted(all_rows.values(), key=sort_key, reverse=True)

    # Determine output path
    if args.output:
        output_path = Path(args.output)
    else:
        output_path = Path(f"merged_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv")

    # Ensure output directory exists (skip if current directory)
    output_dir = output_path.parent
    if str(output_dir) not in ('', '.') and not output_dir.exists():
        output_dir.mkdir(parents=True, exist_ok=True)

    # Write output with proper quoting and handling of extra/missing keys
    try:
        with open(output_path, 'w', newline='', encoding='utf-8') as f:
            writer = csv.DictWriter(
                f,
                fieldnames=all_fieldnames,
                quoting=csv.QUOTE_MINIMAL,
                extrasaction='ignore'  # Ignore extra keys not in fieldnames
            )
            writer.writeheader()
            # Write rows, filling missing fields with empty string
            for row in sorted_rows:
                # Ensure all fieldnames have a value (empty string if missing)
                complete_row = {fn: safe_str(row.get(fn, '')) for fn in all_fieldnames}
                writer.writerow(complete_row)
    except IOError as e:
        print(f"Error: Failed to write output file: {e}", file=sys.stderr)
        sys.exit(1)

    duplicates = total_rows_read - len(sorted_rows)

    print(f"\nMerge complete:")
    print(f"  Files:      {files_processed}")
    print(f"  Total rows: {total_rows_read}")
    print(f"  Duplicates: {duplicates}")
    print(f"  Unique:     {len(sorted_rows)}")
    print(f"  Columns:    {len(all_fieldnames)}")
    print(f"  Output:     {output_path}")


if __name__ == '__main__':
    main()
```

---

## Edge Cases Already Handled

| Issue | Solution |
|-------|----------|
| `None` values in CSV | `safe_str()` helper |
| Empty search → empty filename | Falls back to `"filtered"` |
| All-special-chars search | Sanitized, fallback if empty |
| Different CSV schemas | Union of fieldnames |
| Missing fields in some rows | Filled with empty string |
| Duplicate tweet_ids | Deduped, configurable keep first/last |
| Rows without tweet_id | Assigned generated key |
| Embedded quotes/commas | `csv.QUOTE_MINIMAL` handles it |
| Sort stability | Secondary sort by tweet_id |

## Questions for Reviewer

1. Any CSV edge cases missed (embedded NULs, BOM, different line endings)?
2. Is loading entire CSV into memory acceptable, or should we stream for large files?
3. Any security concerns with filename sanitization (path traversal)?
4. Should `filter_by_content` support regex option?
