#!/usr/bin/env python
"""
LocationからLocation2025への完全データ移行スクリプト（統計検証付き）

機能：
- 全フィールド対応の完全データ移行
- リアルタイム統計検証
- データ品質チェック
- 移行前後の比較
- 詳細レポート生成
"""

import os
import django
from collections import defaultdict, Counter

# Django設定
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')
django.setup()

from rog.models import Location, Location2025, NewEvent2
from django.contrib.auth import get_user_model
from django.contrib.gis.geos import Point

def analyze_source_data():
    """移行前のデータ分析"""
    print('=== 移行前データ分析 ===')
    
    total_locations = Location.objects.count()
    print(f'総Location件数: {total_locations}件')
    
    # グループ別統計
    with_group = Location.objects.exclude(group__isnull=True).exclude(group='').count()
    without_group = total_locations - with_group
    print(f'groupありLocation: {with_group}件')
    print(f'groupなしLocation: {without_group}件')
    
    # 座標データ統計
    with_geom = Location.objects.exclude(geom__isnull=True).count()
    with_lat_lng = Location.objects.exclude(longitude__isnull=True).exclude(latitude__isnull=True).count()
    print(f'geom座標あり: {with_geom}件')
    print(f'lat/lng座標あり: {with_lat_lng}件')
    
    # フィールド統計
    fields_stats = {}
    text_fields = ['photos', 'videos', 'remark', 'tags', 'evaluation_value', 'sub_loc_id', 'subcategory']
    numeric_fields = ['checkin_point', 'buy_point']
    boolean_fields = ['hidden_location']
    
    for field in text_fields:
        if hasattr(Location, field):
            count = Location.objects.exclude(**{f'{field}__isnull': True}).exclude(**{field: ''}).count()
            fields_stats[field] = count
            print(f'{field}データあり: {count}件')
    
    for field in numeric_fields:
        if hasattr(Location, field):
            count = Location.objects.exclude(**{f'{field}__isnull': True}).exclude(**{field: 0}).count()
            fields_stats[field] = count
            print(f'{field}データあり: {count}件')
            
    for field in boolean_fields:
        if hasattr(Location, field):
            count = Location.objects.filter(**{field: True}).count()
            fields_stats[field] = count
            print(f'{field}データあり: {count}件')
    
    return {
        'total': total_locations,
        'with_group': with_group,
        'without_group': without_group,
        'with_geom': with_geom,
        'with_lat_lng': with_lat_lng,
        'fields': fields_stats
    }

def validate_migration_data(source_stats):
    """移行後データ検証"""
    print('\n=== 移行後データ検証 ===')
    
    total_migrated = Location2025.objects.count()
    print(f'移行完了件数: {total_migrated}件')
    
    # フィールド検証
    migrated_stats = {}
    field_mapping = {
        'photos': 'photos',
        'videos': 'videos', 
        'remark': 'remark',
        'tags': 'tags',
        'evaluation_value': 'evaluation_value',
        'hidden_location': 'hidden_location',
        'sub_loc_id': 'sub_loc_id',
        'subcategory': 'subcategory'
    }
    
    for source_field, target_field in field_mapping.items():
        if source_field == 'hidden_location':
            count = Location2025.objects.filter(**{target_field: True}).count()
        else:
            count = Location2025.objects.exclude(**{f'{target_field}__isnull': True}).exclude(**{target_field: ''}).count()
        migrated_stats[source_field] = count
        print(f'{target_field}データあり: {count}件')
    
    # 座標検証
    with_location = Location2025.objects.exclude(location__isnull=True).count()
    with_lat_lng = Location2025.objects.exclude(longitude__isnull=True).exclude(latitude__isnull=True).count()
    print(f'location座標あり: {with_location}件')
    print(f'lat/lng座標あり: {with_lat_lng}件')
    
    # 必須フィールド検証
    with_event = Location2025.objects.exclude(event__isnull=True).count()
    with_cp_name = Location2025.objects.exclude(cp_name__isnull=True).exclude(cp_name='').count()
    print(f'eventリンクあり: {with_event}件')
    print(f'cp_nameあり: {with_cp_name}件')
    
    return {
        'total': total_migrated,
        'fields': migrated_stats,
        'with_location': with_location,
        'with_lat_lng': with_lat_lng,
        'with_event': with_event,
        'with_cp_name': with_cp_name
    }

def generate_comparison_report(source_stats, migrated_stats):
    """移行前後比較レポート"""
    print('\n=== 移行前後比較レポート ===')
    
    print(f'総件数比較:')
    print(f'  移行前: {source_stats["total"]:,}件')
    print(f'  移行後: {migrated_stats["total"]:,}件')
    print(f'  移行率: {(migrated_stats["total"] / source_stats["total"] * 100):.1f}%')
    
    print(f'\nフィールド別データ保持率:')
    for field in source_stats['fields']:
        if field in migrated_stats['fields']:
            source_count = source_stats['fields'][field]
            migrated_count = migrated_stats['fields'][field]
            if source_count > 0:
                retention_rate = (migrated_count / source_count * 100)
                print(f'  {field}: {migrated_count:,}/{source_count:,}件 ({retention_rate:.1f}%)')
            else:
                print(f'  {field}: {migrated_count:,}/0件 (N/A)')

def analyze_event_distribution():
    """イベント別分布分析"""
    print('\n=== イベント別分布分析 ===')
    
    event_stats = {}
    for location in Location2025.objects.select_related('event'):
        event_name = location.event.event_name if location.event else 'No Event'
        event_code = location.event.event_code if location.event else 'No Code'
        key = f"{event_code} ({event_name})"
        event_stats[key] = event_stats.get(key, 0) + 1
    
    # 件数順でソート
    sorted_events = sorted(event_stats.items(), key=lambda x: x[1], reverse=True)
    
    print(f'総イベント数: {len(sorted_events)}件')
    print(f'上位イベント:')
    for i, (event_key, count) in enumerate(sorted_events[:10], 1):
        print(f'  {i:2d}. {event_key}: {count:,}件')
    
    return event_stats

def sample_data_verification():
    """サンプルデータ検証"""
    print('\n=== サンプルデータ検証 ===')
    
    # 各種データパターンのサンプルを取得
    samples = []
    
    # 写真データありのサンプル
    photo_sample = Location2025.objects.filter(photos__isnull=False).exclude(photos='').first()
    if photo_sample:
        samples.append(('写真データあり', photo_sample))
    
    # remarkデータありのサンプル
    remark_sample = Location2025.objects.filter(remark__isnull=False).exclude(remark='').first()
    if remark_sample:
        samples.append(('詳細説明あり', remark_sample))
    
    # 高ポイントのサンプル
    high_point_sample = Location2025.objects.filter(cp_point__gt=50).first()
    if high_point_sample:
        samples.append(('高ポイント', high_point_sample))
    
    # 通常サンプル
    if not samples:
        normal_sample = Location2025.objects.first()
        if normal_sample:
            samples.append(('通常データ', normal_sample))
    
    for sample_type, sample in samples[:3]:
        print(f'\n【{sample_type}サンプル】')
        print(f'  CP番号: {sample.cp_number}')
        print(f'  CP名: {sample.cp_name}')
        print(f'  CPポイント: {sample.cp_point}')
        print(f'  フォトポイント: {sample.photo_point}')
        print(f'  sub_loc_id: {sample.sub_loc_id}')
        print(f'  subcategory: {sample.subcategory}')
        
        # データ長を制限して表示
        def truncate_text(text, max_len=30):
            if not text:
                return '(空)'
            return text[:max_len] + '...' if len(text) > max_len else text
        
        print(f'  写真: {truncate_text(sample.photos)}')
        print(f'  動画: {truncate_text(sample.videos)}')
        print(f'  詳細: {truncate_text(sample.remark)}')
        print(f'  タグ: {truncate_text(sample.tags)}')
        print(f'  評価値: {truncate_text(sample.evaluation_value)}')
        print(f'  隠し: {sample.hidden_location}')
        print(f'  イベント: {sample.event.event_name if sample.event else "None"}')

def main():
    """メイン実行関数"""
    User = get_user_model()
    default_user = User.objects.first()
    
    print('='*60)
    print('Location → Location2025 完全移行スクリプト（統計検証付き）')
    print('='*60)
    
    # 1. 移行前データ分析
    source_stats = analyze_source_data()
    
    # 2. 既存Location2025データ削除
    print('\n=== 既存データクリア ===')
    deleted_count = Location2025.objects.count()
    Location2025.objects.all().delete()
    print(f'削除済み: {deleted_count}件')
    
    # 3. NewEvent2のevent_codeマップ作成
    print('\n=== Event Code マッピング ===')
    events = NewEvent2.objects.filter(event_code__isnull=False).exclude(event_code='')
    event_code_map = {}
    for event in events:
        event_code_map[event.event_code] = event
    print(f'有効なevent_code数: {len(event_code_map)}件')
    
    # 4. データ移行実行
    print('\n=== データ移行実行 ===')
    locations = Location.objects.all()
    processed_combinations = set()
    migrated_count = 0
    skipped_count = 0
    error_count = 0
    event_migration_stats = defaultdict(int)
    
    for location in locations:
        try:
            # groupが空の場合はスキップ
            if not location.group:
                skipped_count += 1
                continue
            
            # location.groupに含まれるevent_codeを検索
            matched_event = None
            matched_event_code = None
            
            for event_code, event in event_code_map.items():
                if event_code in location.group:
                    matched_event = event
                    matched_event_code = event_code
                    break
            
            # マッチするevent_codeがない場合はスキップ
            if not matched_event:
                skipped_count += 1
                continue
            
            # cp_number + event_idの組み合わせを確認
            combination_key = (location.cp, matched_event.id)
            if combination_key in processed_combinations:
                skipped_count += 1
                continue
            
            # この組み合わせを処理済みとしてマーク
            processed_combinations.add(combination_key)
            
            # MultiPointからPointに変換
            point_location = None
            if location.geom and len(location.geom) > 0:
                first_point = location.geom[0]
                point_location = Point(first_point.x, first_point.y)
            elif location.longitude and location.latitude:
                point_location = Point(location.longitude, location.latitude)
            
            # Location2025レコードを作成
            location2025, created = Location2025.objects.update_or_create(
                cp_number=location.cp,
                event=matched_event,
                defaults={
                    'cp_name': location.location_name or '',
                    'sub_loc_id': location.sub_loc_id or '',
                    'subcategory': location.subcategory or '',
                    'latitude': location.latitude or 0.0,
                    'longitude': location.longitude or 0.0,
                    'location': point_location,
                    'cp_point': int(location.checkin_point) if location.checkin_point else 0,
                    'photo_point': int(location.checkin_point) if location.checkin_point else 0,
                    'buy_point': int(location.buy_point) if location.buy_point else 0,
                    'checkin_radius': location.checkin_radius or 100.0,
                    'auto_checkin': location.auto_checkin or False,
                    'shop_closed': location.shop_closed or False,
                    'shop_shutdown': location.shop_shutdown or False,
                    'opening_hours': '',
                    'address': location.address or '',
                    'phone': location.phone or '',
                    'website': '',
                    'description': location.remark or '',
                    # 追加フィールド
                    'photos': location.photos or '',
                    'videos': location.videos or '',
                    'remark': location.remark or '',
                    'tags': location.tags or '',
                    'evaluation_value': location.evaluation_value or '',
                    'hidden_location': location.hidden_location or False,
                    # 管理情報
                    'is_active': True,
                    'sort_order': 0,
                    'csv_source_file': 'migration_from_location',
                    'created_by': default_user,
                    'updated_by': default_user,
                }
            )
            
            if created:
                migrated_count += 1
                event_migration_stats[matched_event_code] += 1
                
                if migrated_count % 100 == 0:
                    print(f'進捗: {migrated_count:,}件完了')
            
        except Exception as e:
            print(f'❌ エラー: CP {location.cp} - {str(e)}')
            error_count += 1
    
    # 5. 移行結果サマリー
    print(f'\n=== 移行結果サマリー ===')
    print(f'移行完了: {migrated_count:,}件')
    print(f'スキップ: {skipped_count:,}件')
    print(f'エラー: {error_count:,}件')
    print(f'総処理: {migrated_count + skipped_count + error_count:,}件')
    
    # 6. 移行後データ検証
    migrated_stats = validate_migration_data(source_stats)
    
    # 7. 比較レポート生成
    generate_comparison_report(source_stats, migrated_stats)
    
    # 8. イベント別分布分析
    event_distribution = analyze_event_distribution()
    
    # 9. サンプルデータ検証
    sample_data_verification()
    
    # 10. 最終検証サマリー
    print('\n' + '='*60)
    print('🎯 移行完了検証サマリー')
    print('='*60)
    
    success_rate = (migrated_count / source_stats['total'] * 100) if source_stats['total'] > 0 else 0
    print(f'✅ 総移行成功率: {success_rate:.1f}% ({migrated_count:,}/{source_stats["total"]:,}件)')
    print(f'✅ エラー率: {(error_count / source_stats["total"] * 100):.1f}% ({error_count:,}件)')
    print(f'✅ 最終Location2025件数: {Location2025.objects.count():,}件')
    print(f'✅ 対応イベント数: {len(event_distribution)}件')
    
    # データ品質スコア算出
    quality_score = 0
    if migrated_stats['with_event'] == migrated_stats['total']:
        quality_score += 25  # 全てにイベントがリンクされている
    if migrated_stats['with_cp_name'] >= migrated_stats['total'] * 0.95:
        quality_score += 25  # 95%以上にCP名がある
    if migrated_stats['fields']['photos'] >= migrated_stats['total'] * 0.8:
        quality_score += 25  # 80%以上に写真データがある
    if migrated_stats['fields']['remark'] >= migrated_stats['total'] * 0.8:
        quality_score += 25  # 80%以上に詳細説明がある
    
    print(f'✅ データ品質スコア: {quality_score}/100点')
    
    if quality_score >= 90:
        print('🏆 優秀：本格運用準備完了')
    elif quality_score >= 70:
        print('🥉 良好：運用可能レベル')
    elif quality_score >= 50:
        print('⚠️  要改善：一部データ補完推奨')
    else:
        print('❌ 要対応：データ品質に課題あり')
    
    print('\n✅ 全フィールド対応の完全データ移行が正常に完了しました')

if __name__ == "__main__":
    main()
