mirror of
https://github.com/house-of-vanity/OutFleet.git
synced 2025-08-21 14:37:16 +00:00
management command for cleanup old access logs
This commit is contained in:
Ultradesu
158
vpn/management/commands/cleanup_access_logs.py
Normal file
158
vpn/management/commands/cleanup_access_logs.py
Normal file
@@ -0,0 +1,158 @@
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db import connection, transaction
|
||||
from datetime import datetime, timedelta
|
||||
from vpn.models import AccessLog
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'Clean up old AccessLog entries without acl_link_id'
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
'--days',
|
||||
type=int,
|
||||
default=30,
|
||||
help='Delete logs older than this many days (default: 30)'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--batch-size',
|
||||
type=int,
|
||||
default=10000,
|
||||
help='Number of records to delete in each batch (default: 10000)'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--dry-run',
|
||||
action='store_true',
|
||||
help='Show what would be deleted without actually deleting'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--keep-recent',
|
||||
type=int,
|
||||
default=1000,
|
||||
help='Keep this many recent logs even if they have no link (default: 1000)'
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
days = options['days']
|
||||
batch_size = options['batch_size']
|
||||
dry_run = options['dry_run']
|
||||
keep_recent = options['keep_recent']
|
||||
|
||||
cutoff_date = datetime.now() - timedelta(days=days)
|
||||
|
||||
self.stdout.write(f"🔍 Analyzing AccessLog cleanup...")
|
||||
self.stdout.write(f" - Delete logs without acl_link_id older than {days} days")
|
||||
self.stdout.write(f" - Keep {keep_recent} most recent logs without links")
|
||||
self.stdout.write(f" - Batch size: {batch_size}")
|
||||
self.stdout.write(f" - Dry run: {dry_run}")
|
||||
|
||||
# Count total records to be deleted
|
||||
with connection.cursor() as cursor:
|
||||
# Count logs without acl_link_id
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) FROM vpn_accesslog
|
||||
WHERE (acl_link_id IS NULL OR acl_link_id = '')
|
||||
""")
|
||||
total_without_link = cursor.fetchone()[0]
|
||||
|
||||
# Count logs to be deleted (older than cutoff, excluding recent ones to keep)
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) FROM vpn_accesslog
|
||||
WHERE (acl_link_id IS NULL OR acl_link_id = '')
|
||||
AND timestamp < %s
|
||||
AND id NOT IN (
|
||||
SELECT id FROM (
|
||||
SELECT id FROM vpn_accesslog
|
||||
WHERE acl_link_id IS NULL OR acl_link_id = ''
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT %s
|
||||
) AS recent_logs
|
||||
)
|
||||
""", [cutoff_date, keep_recent])
|
||||
total_to_delete = cursor.fetchone()[0]
|
||||
|
||||
# Count total records
|
||||
cursor.execute("SELECT COUNT(*) FROM vpn_accesslog")
|
||||
total_records = cursor.fetchone()[0]
|
||||
|
||||
self.stdout.write(f"📊 Statistics:")
|
||||
self.stdout.write(f" - Total AccessLog records: {total_records:,}")
|
||||
self.stdout.write(f" - Records without acl_link_id: {total_without_link:,}")
|
||||
self.stdout.write(f" - Records to be deleted: {total_to_delete:,}")
|
||||
self.stdout.write(f" - Records to be kept (recent): {keep_recent:,}")
|
||||
|
||||
if total_to_delete == 0:
|
||||
self.stdout.write(self.style.SUCCESS("✅ No records to delete."))
|
||||
return
|
||||
|
||||
if dry_run:
|
||||
self.stdout.write(self.style.WARNING(f"🔍 DRY RUN: Would delete {total_to_delete:,} records"))
|
||||
return
|
||||
|
||||
# Confirm deletion
|
||||
if not options.get('verbosity', 1) == 0: # Only ask if not --verbosity=0
|
||||
confirm = input(f"❓ Delete {total_to_delete:,} records? (yes/no): ")
|
||||
if confirm.lower() != 'yes':
|
||||
self.stdout.write("❌ Cancelled.")
|
||||
return
|
||||
|
||||
self.stdout.write(f"🗑️ Starting deletion of {total_to_delete:,} records...")
|
||||
|
||||
deleted_total = 0
|
||||
batch_num = 0
|
||||
|
||||
while True:
|
||||
batch_num += 1
|
||||
|
||||
with transaction.atomic():
|
||||
with connection.cursor() as cursor:
|
||||
# Delete batch
|
||||
cursor.execute("""
|
||||
DELETE FROM vpn_accesslog
|
||||
WHERE (acl_link_id IS NULL OR acl_link_id = '')
|
||||
AND timestamp < %s
|
||||
AND id NOT IN (
|
||||
SELECT id FROM (
|
||||
SELECT id FROM vpn_accesslog
|
||||
WHERE acl_link_id IS NULL OR acl_link_id = ''
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT %s
|
||||
) AS recent_logs
|
||||
)
|
||||
LIMIT %s
|
||||
""", [cutoff_date, keep_recent, batch_size])
|
||||
|
||||
deleted_in_batch = cursor.rowcount
|
||||
|
||||
if deleted_in_batch == 0:
|
||||
break
|
||||
|
||||
deleted_total += deleted_in_batch
|
||||
progress = (deleted_total / total_to_delete) * 100
|
||||
|
||||
self.stdout.write(
|
||||
f" Batch {batch_num}: Deleted {deleted_in_batch:,} records "
|
||||
f"(Total: {deleted_total:,}/{total_to_delete:,}, {progress:.1f}%)"
|
||||
)
|
||||
|
||||
if deleted_in_batch < batch_size:
|
||||
break
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(f"✅ Cleanup completed!"))
|
||||
self.stdout.write(f" - Deleted {deleted_total:,} old AccessLog records")
|
||||
self.stdout.write(f" - Kept {keep_recent:,} recent records without links")
|
||||
|
||||
# Show final statistics
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("SELECT COUNT(*) FROM vpn_accesslog")
|
||||
final_total = cursor.fetchone()[0]
|
||||
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) FROM vpn_accesslog
|
||||
WHERE acl_link_id IS NULL OR acl_link_id = ''
|
||||
""")
|
||||
final_without_link = cursor.fetchone()[0]
|
||||
|
||||
self.stdout.write(f"📊 Final statistics:")
|
||||
self.stdout.write(f" - Total AccessLog records: {final_total:,}")
|
||||
self.stdout.write(f" - Records without acl_link_id: {final_without_link:,}")
|
||||
208
vpn/management/commands/cleanup_logs.py
Normal file
208
vpn/management/commands/cleanup_logs.py
Normal file
@@ -0,0 +1,208 @@
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db import connection
|
||||
from django.utils import timezone
|
||||
from datetime import timedelta
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = '''
|
||||
Clean up AccessLog entries efficiently using direct SQL.
|
||||
|
||||
Examples:
|
||||
# Delete logs without acl_link_id older than 30 days (recommended)
|
||||
python manage.py cleanup_logs --keep-days=30
|
||||
|
||||
# Keep only last 5000 logs without acl_link_id
|
||||
python manage.py cleanup_logs --keep-count=5000
|
||||
|
||||
# Delete ALL logs older than 7 days (including with acl_link_id)
|
||||
python manage.py cleanup_logs --keep-days=7 --target=all
|
||||
|
||||
# Preview what would be deleted
|
||||
python manage.py cleanup_logs --keep-days=30 --dry-run
|
||||
|
||||
# Force delete without confirmation
|
||||
python manage.py cleanup_logs --keep-days=30 --force
|
||||
'''
|
||||
|
||||
def add_arguments(self, parser):
|
||||
# Primary options (mutually exclusive)
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument(
|
||||
'--keep-days',
|
||||
type=int,
|
||||
help='Keep logs newer than this many days (delete older)'
|
||||
)
|
||||
group.add_argument(
|
||||
'--keep-count',
|
||||
type=int,
|
||||
help='Keep this many most recent logs (delete the rest)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--target',
|
||||
choices=['no-links', 'all'],
|
||||
default='no-links',
|
||||
help='Target: "no-links" = only logs without acl_link_id (default), "all" = all logs'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--dry-run',
|
||||
action='store_true',
|
||||
help='Show what would be deleted without actually deleting'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--force',
|
||||
action='store_true',
|
||||
help='Skip confirmation prompt'
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
keep_days = options.get('keep_days')
|
||||
keep_count = options.get('keep_count')
|
||||
target = options['target']
|
||||
dry_run = options['dry_run']
|
||||
force = options['force']
|
||||
|
||||
# Build SQL conditions
|
||||
if target == 'no-links':
|
||||
base_condition = "(acl_link_id IS NULL OR acl_link_id = '')"
|
||||
target_desc = "logs without acl_link_id"
|
||||
else:
|
||||
base_condition = "1=1"
|
||||
target_desc = "all logs"
|
||||
|
||||
# Get current statistics
|
||||
with connection.cursor() as cursor:
|
||||
# Total records
|
||||
cursor.execute("SELECT COUNT(*) FROM vpn_accesslog")
|
||||
total_records = cursor.fetchone()[0]
|
||||
|
||||
# Target records count
|
||||
cursor.execute(f"SELECT COUNT(*) FROM vpn_accesslog WHERE {base_condition}")
|
||||
target_records = cursor.fetchone()[0]
|
||||
|
||||
# Records to delete
|
||||
if keep_days:
|
||||
cutoff_date = timezone.now() - timedelta(days=keep_days)
|
||||
cursor.execute(f"""
|
||||
SELECT COUNT(*) FROM vpn_accesslog
|
||||
WHERE {base_condition} AND timestamp < %s
|
||||
""", [cutoff_date])
|
||||
to_delete = cursor.fetchone()[0]
|
||||
strategy = f"older than {keep_days} days"
|
||||
else: # keep_count
|
||||
to_delete = max(0, target_records - keep_count)
|
||||
strategy = f"keeping only {keep_count} most recent"
|
||||
|
||||
# Print statistics
|
||||
self.stdout.write("🗑️ AccessLog Cleanup" + (" (DRY RUN)" if dry_run else ""))
|
||||
self.stdout.write(f" Target: {target_desc}")
|
||||
self.stdout.write(f" Strategy: {strategy}")
|
||||
self.stdout.write("")
|
||||
self.stdout.write("📊 Statistics:")
|
||||
self.stdout.write(f" Total AccessLog records: {total_records:,}")
|
||||
self.stdout.write(f" Target records: {target_records:,}")
|
||||
self.stdout.write(f" Records to delete: {to_delete:,}")
|
||||
self.stdout.write(f" Records to keep: {target_records - to_delete:,}")
|
||||
|
||||
if total_records > 0:
|
||||
delete_percent = (to_delete / total_records) * 100
|
||||
self.stdout.write(f" Deletion percentage: {delete_percent:.1f}%")
|
||||
|
||||
if to_delete == 0:
|
||||
self.stdout.write(self.style.SUCCESS("✅ No records to delete."))
|
||||
return
|
||||
|
||||
# Show SQL that will be executed
|
||||
if dry_run or not force:
|
||||
self.stdout.write("")
|
||||
self.stdout.write("📝 SQL to execute:")
|
||||
if keep_days:
|
||||
sql_preview = f"""
|
||||
DELETE FROM vpn_accesslog
|
||||
WHERE {base_condition} AND timestamp < '{cutoff_date.strftime('%Y-%m-%d %H:%M:%S')}'
|
||||
"""
|
||||
else: # keep_count
|
||||
sql_preview = f"""
|
||||
DELETE FROM vpn_accesslog
|
||||
WHERE {base_condition}
|
||||
AND id NOT IN (
|
||||
SELECT id FROM (
|
||||
SELECT id FROM vpn_accesslog
|
||||
WHERE {base_condition}
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT {keep_count}
|
||||
) AS recent_logs
|
||||
)
|
||||
"""
|
||||
self.stdout.write(sql_preview.strip())
|
||||
|
||||
if dry_run:
|
||||
self.stdout.write("")
|
||||
self.stdout.write(self.style.WARNING(f"🔍 DRY RUN: Would delete {to_delete:,} records"))
|
||||
return
|
||||
|
||||
# Confirm deletion
|
||||
if not force:
|
||||
self.stdout.write("")
|
||||
self.stdout.write(self.style.ERROR(f"⚠️ About to DELETE {to_delete:,} records!"))
|
||||
confirm = input("Type 'DELETE' to confirm: ")
|
||||
if confirm != 'DELETE':
|
||||
self.stdout.write("❌ Cancelled.")
|
||||
return
|
||||
|
||||
# Execute deletion
|
||||
self.stdout.write("")
|
||||
self.stdout.write(f"🗑️ Deleting {to_delete:,} records...")
|
||||
|
||||
with connection.cursor() as cursor:
|
||||
if keep_days:
|
||||
# Simple time-based deletion
|
||||
cursor.execute(f"""
|
||||
DELETE FROM vpn_accesslog
|
||||
WHERE {base_condition} AND timestamp < %s
|
||||
""", [cutoff_date])
|
||||
else:
|
||||
# Keep count deletion (more complex)
|
||||
cursor.execute(f"""
|
||||
DELETE FROM vpn_accesslog
|
||||
WHERE {base_condition}
|
||||
AND id NOT IN (
|
||||
SELECT id FROM (
|
||||
SELECT id FROM vpn_accesslog
|
||||
WHERE {base_condition}
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT %s
|
||||
) AS recent_logs
|
||||
)
|
||||
""", [keep_count])
|
||||
|
||||
deleted_count = cursor.rowcount
|
||||
|
||||
# Final statistics
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("SELECT COUNT(*) FROM vpn_accesslog")
|
||||
final_total = cursor.fetchone()[0]
|
||||
|
||||
cursor.execute(f"SELECT COUNT(*) FROM vpn_accesslog WHERE {base_condition}")
|
||||
final_target = cursor.fetchone()[0]
|
||||
|
||||
self.stdout.write("")
|
||||
self.stdout.write(self.style.SUCCESS("✅ Cleanup completed!"))
|
||||
self.stdout.write(f" Deleted: {deleted_count:,} records")
|
||||
self.stdout.write(f" Remaining total: {final_total:,}")
|
||||
|
||||
if target == 'no-links':
|
||||
self.stdout.write(f" Remaining without links: {final_target:,}")
|
||||
|
||||
# Calculate space saved (rough estimate)
|
||||
if deleted_count > 0:
|
||||
# Rough estimate: ~200 bytes per AccessLog record
|
||||
space_saved_mb = (deleted_count * 200) / (1024 * 1024)
|
||||
if space_saved_mb > 1024:
|
||||
space_saved_gb = space_saved_mb / 1024
|
||||
self.stdout.write(f" Estimated space saved: ~{space_saved_gb:.1f} GB")
|
||||
else:
|
||||
self.stdout.write(f" Estimated space saved: ~{space_saved_mb:.1f} MB")
|
||||
51
vpn/management/commands/simple_cleanup_logs.py
Normal file
51
vpn/management/commands/simple_cleanup_logs.py
Normal file
@@ -0,0 +1,51 @@
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.utils import timezone
|
||||
from datetime import timedelta
|
||||
from vpn.models import AccessLog
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'Simple cleanup of AccessLog entries without acl_link_id using Django ORM'
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
'--days',
|
||||
type=int,
|
||||
default=30,
|
||||
help='Delete logs older than this many days (default: 30)'
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
days = options['days']
|
||||
cutoff_date = timezone.now() - timedelta(days=days)
|
||||
|
||||
# Count records to be deleted
|
||||
old_logs = AccessLog.objects.filter(
|
||||
acl_link_id__isnull=True,
|
||||
timestamp__lt=cutoff_date
|
||||
)
|
||||
|
||||
# Also include empty string acl_link_id
|
||||
empty_logs = AccessLog.objects.filter(
|
||||
acl_link_id='',
|
||||
timestamp__lt=cutoff_date
|
||||
)
|
||||
|
||||
total_old = old_logs.count()
|
||||
total_empty = empty_logs.count()
|
||||
total_to_delete = total_old + total_empty
|
||||
|
||||
self.stdout.write(f"Found {total_to_delete:,} old logs without acl_link_id to delete")
|
||||
|
||||
if total_to_delete == 0:
|
||||
self.stdout.write("Nothing to delete.")
|
||||
return
|
||||
|
||||
# Delete in batches to avoid memory issues
|
||||
self.stdout.write("Deleting old logs...")
|
||||
|
||||
deleted_count = 0
|
||||
deleted_count += old_logs._raw_delete(old_logs.db)
|
||||
deleted_count += empty_logs._raw_delete(empty_logs.db)
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(f"Deleted {deleted_count:,} old AccessLog records"))
|
||||
Reference in New Issue
Block a user