[{"data":1,"prerenderedAt":8340},["ShallowReactive",2],{"tutorials-pandas":3,"category-items-pandas":387,"content-/pandas/removing-duplicates":388},[4,21,31,40,49,58,67,78,87,96,105,116,125,134,143,154,163,172,181,192,201,210,219,228,239,248,257,266,276,285,294,303,313,322,331,341,350,359,369,378],{"_path":5,"_dir":6,"_draft":7,"_partial":7,"_locale":6,"title":8,"description":9,"slug":10,"order":11,"isParent":12,"icon":13,"_type":14,"_id":15,"_source":16,"_file":17,"_stem":18,"_extension":19,"children":20,"expanded":12},"/pandas","",false,"Getting Started","Learn the basics of Pandas, Python's powerful data analysis library for working with spreadsheet-like data","getting-started",1,true,"pi pi-play","markdown","content:pandas:1.index.md","content","pandas/1.index.md","pandas/1.index","md",[21,31,40,49,58],{"_path":22,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":24,"description":25,"slug":26,"parentId":5,"order":27,"_type":14,"_id":28,"_source":16,"_file":29,"_stem":30,"_extension":19},"/pandas/what-is-pandas","pandas","What is Pandas","Understand what Pandas is, why it's essential for data analysis, and how it makes working with data simple and powerful","what-is-pandas",2,"content:pandas:2.what-is-pandas.md","pandas/2.what-is-pandas.md","pandas/2.what-is-pandas",{"_path":32,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":33,"description":34,"slug":35,"parentId":5,"order":36,"_type":14,"_id":37,"_source":16,"_file":38,"_stem":39,"_extension":19},"/pandas/installing-pandas","Installing Pandas","Learn how to install Pandas on your computer and verify it's working properly","installing-pandas",3,"content:pandas:3.installing-pandas.md","pandas/3.installing-pandas.md","pandas/3.installing-pandas",{"_path":41,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":42,"description":43,"slug":44,"parentId":5,"order":45,"_type":14,"_id":46,"_source":16,"_file":47,"_stem":48,"_extension":19},"/pandas/your-first-dataframe","Your First DataFrame","Create your first Pandas DataFrame and learn the basics of working with tabular data","your-first-dataframe",4,"content:pandas:4.your-first-dataframe.md","pandas/4.your-first-dataframe.md","pandas/4.your-first-dataframe",{"_path":50,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":51,"description":52,"slug":53,"parentId":5,"order":54,"_type":14,"_id":55,"_source":16,"_file":56,"_stem":57,"_extension":19},"/pandas/series-vs-dataframe","Series vs DataFrame","Understand the difference between Pandas Series and DataFrames, and learn when to use each one","series-vs-dataframe",5,"content:pandas:5.series-vs-dataframe.md","pandas/5.series-vs-dataframe.md","pandas/5.series-vs-dataframe",{"_path":59,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":60,"description":61,"slug":62,"parentId":5,"order":63,"_type":14,"_id":64,"_source":16,"_file":65,"_stem":66,"_extension":19},"/pandas/basic-dataframe-info","Basic DataFrame Info","Learn how to explore and get basic information about your DataFrames to understand your data","basic-dataframe-info",6,"content:pandas:6.basic-dataframe-info.md","pandas/6.basic-dataframe-info.md","pandas/6.basic-dataframe-info",{"_path":68,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":69,"description":70,"slug":71,"order":72,"isParent":12,"icon":73,"_type":14,"_id":74,"_source":16,"_file":75,"_stem":76,"_extension":19,"children":77,"expanded":12},"/pandas/loading-data","Loading Data","Learn how to load data from various sources into Pandas DataFrames","loading-data",7,"pi pi-download","content:pandas:7.loading-data.md","pandas/7.loading-data.md","pandas/7.loading-data",[78,87,96],{"_path":79,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":80,"description":81,"slug":82,"parentId":68,"order":83,"_type":14,"_id":84,"_source":16,"_file":85,"_stem":86,"_extension":19},"/pandas/reading-files","Reading Files (CSV, Excel, JSON)","Learn how to load data from CSV, Excel, and JSON files into Pandas DataFrames","reading-files",8,"content:pandas:8.reading-files.md","pandas/8.reading-files.md","pandas/8.reading-files",{"_path":88,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":89,"description":90,"slug":91,"parentId":68,"order":92,"_type":14,"_id":93,"_source":16,"_file":94,"_stem":95,"_extension":19},"/pandas/loading-from-databases","Loading from Databases","Learn how to connect to databases and load data directly into Pandas DataFrames","loading-from-databases",9,"content:pandas:9.loading-from-databases.md","pandas/9.loading-from-databases.md","pandas/9.loading-from-databases",{"_path":97,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":98,"description":99,"slug":100,"parentId":68,"order":101,"_type":14,"_id":102,"_source":16,"_file":103,"_stem":104,"_extension":19},"/pandas/handling-file-errors","Handling File Errors","Learn how to handle common errors when loading data files and how to troubleshoot loading problems","handling-file-errors",10,"content:pandas:10.handling-file-errors.md","pandas/10.handling-file-errors.md","pandas/10.handling-file-errors",{"_path":106,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":107,"description":108,"slug":109,"order":110,"isParent":12,"icon":111,"_type":14,"_id":112,"_source":16,"_file":113,"_stem":114,"_extension":19,"children":115,"expanded":12},"/pandas/data-exploration","Data Exploration","Learn how to explore and understand your data using Pandas methods and techniques","data-exploration",11,"pi pi-search","content:pandas:11.data-exploration.md","pandas/11.data-exploration.md","pandas/11.data-exploration",[116,125,134],{"_path":117,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":118,"description":119,"slug":120,"parentId":106,"order":121,"_type":14,"_id":122,"_source":16,"_file":123,"_stem":124,"_extension":19},"/pandas/viewing-your-data","Viewing Your Data","Learn different ways to look at and sample your DataFrame data for exploration and understanding","viewing-your-data",12,"content:pandas:12.viewing-your-data.md","pandas/12.viewing-your-data.md","pandas/12.viewing-your-data",{"_path":126,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":127,"description":128,"slug":129,"parentId":106,"order":130,"_type":14,"_id":131,"_source":16,"_file":132,"_stem":133,"_extension":19},"/pandas/data-shape-and-statistics","Data Shape and Statistics","Learn how to understand your DataFrame's size, structure, and get basic statistical summaries","data-shape-and-statistics",13,"content:pandas:13.data-shape-and-statistics.md","pandas/13.data-shape-and-statistics.md","pandas/13.data-shape-and-statistics",{"_path":135,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":136,"description":137,"slug":138,"parentId":106,"order":139,"_type":14,"_id":140,"_source":16,"_file":141,"_stem":142,"_extension":19},"/pandas/column-information-and-data-types","Column Information and Data Types","Learn how to explore individual columns and understand different data types in your DataFrames","column-information-and-data-types",14,"content:pandas:14.column-information-and-data-types.md","pandas/14.column-information-and-data-types.md","pandas/14.column-information-and-data-types",{"_path":144,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":145,"description":146,"slug":147,"order":148,"isParent":12,"icon":149,"_type":14,"_id":150,"_source":16,"_file":151,"_stem":152,"_extension":19,"children":153,"expanded":12},"/pandas/selecting-data","Selecting Data","Learn how to select specific columns, rows, and subsets of data from your DataFrames","selecting-data",15,"pi pi-filter","content:pandas:15.selecting-data.md","pandas/15.selecting-data.md","pandas/15.selecting-data",[154,163,172],{"_path":155,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":156,"description":157,"slug":158,"parentId":144,"order":159,"_type":14,"_id":160,"_source":16,"_file":161,"_stem":162,"_extension":19},"/pandas/selecting-columns-and-rows","Selecting Columns and Rows","Learn how to select specific columns and rows from your DataFrames using basic selection techniques","selecting-columns-and-rows",16,"content:pandas:16.selecting-columns-and-rows.md","pandas/16.selecting-columns-and-rows.md","pandas/16.selecting-columns-and-rows",{"_path":164,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":165,"description":166,"slug":167,"parentId":144,"order":168,"_type":14,"_id":169,"_source":16,"_file":170,"_stem":171,"_extension":19},"/pandas/using-loc-and-iloc","Using loc and iloc","Master precise data selection with label-based (loc) and position-based (iloc) indexing in pandas","using-loc-and-iloc",17,"content:pandas:17.using-loc-and-iloc.md","pandas/17.using-loc-and-iloc.md","pandas/17.using-loc-and-iloc",{"_path":173,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":174,"description":175,"slug":176,"parentId":144,"order":177,"_type":14,"_id":178,"_source":16,"_file":179,"_stem":180,"_extension":19},"/pandas/boolean-and-conditional-selection","Boolean and Conditional Selection","Learn to filter DataFrames using conditions, boolean logic, and multiple criteria for precise data selection","boolean-and-conditional-selection",18,"content:pandas:18.boolean-and-conditional-selection.md","pandas/18.boolean-and-conditional-selection.md","pandas/18.boolean-and-conditional-selection",{"_path":182,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":183,"description":184,"slug":185,"order":186,"isParent":12,"icon":187,"_type":14,"_id":188,"_source":16,"_file":189,"_stem":190,"_extension":19,"children":191,"expanded":12},"/pandas/data-cleaning","Data Cleaning","Learn essential data cleaning techniques to prepare messy data for analysis","data-cleaning",19,"pi pi-wrench","content:pandas:19.data-cleaning.md","pandas/19.data-cleaning.md","pandas/19.data-cleaning",[192,201,210,219],{"_path":193,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":194,"description":195,"slug":196,"parentId":182,"order":197,"_type":14,"_id":198,"_source":16,"_file":199,"_stem":200,"_extension":19},"/pandas/handling-missing-data","Handling Missing Data","Learn to identify, fill, and remove missing values in your DataFrames effectively","handling-missing-data",20,"content:pandas:20.handling-missing-data.md","pandas/20.handling-missing-data.md","pandas/20.handling-missing-data",{"_path":202,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":203,"description":204,"slug":205,"parentId":182,"order":206,"_type":14,"_id":207,"_source":16,"_file":208,"_stem":209,"_extension":19},"/pandas/removing-duplicates","Removing Duplicates","Learn to find and eliminate duplicate rows from your DataFrames to ensure data quality","removing-duplicates",21,"content:pandas:21.removing-duplicates.md","pandas/21.removing-duplicates.md","pandas/21.removing-duplicates",{"_path":211,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":212,"description":213,"slug":214,"parentId":182,"order":215,"_type":14,"_id":216,"_source":16,"_file":217,"_stem":218,"_extension":19},"/pandas/data-type-conversion","Data Type Conversion","Learn to convert data to the right types for proper analysis and calculations","data-type-conversion",22,"content:pandas:22.data-type-conversion.md","pandas/22.data-type-conversion.md","pandas/22.data-type-conversion",{"_path":220,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":221,"description":222,"slug":223,"parentId":182,"order":224,"_type":14,"_id":225,"_source":16,"_file":226,"_stem":227,"_extension":19},"/pandas/string-cleaning-operations","String Cleaning Operations","Learn to clean and standardize text data for consistent analysis using pandas string methods","string-cleaning-operations",23,"content:pandas:23.string-cleaning-operations.md","pandas/23.string-cleaning-operations.md","pandas/23.string-cleaning-operations",{"_path":229,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":230,"description":231,"slug":232,"order":233,"isParent":12,"icon":234,"_type":14,"_id":235,"_source":16,"_file":236,"_stem":237,"_extension":19,"children":238},"/pandas/data-transformation","Data Transformation","Learn how to modify, add, and transform data in your DataFrames for analysis","data-transformation",24,"pi pi-folder","content:pandas:24.data-transformation.md","pandas/24.data-transformation.md","pandas/24.data-transformation",[239,248,257],{"_path":240,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":241,"description":242,"slug":243,"parentId":229,"order":244,"_type":14,"_id":245,"_source":16,"_file":246,"_stem":247,"_extension":19},"/pandas/adding-and-modifying-columns","Adding and Modifying Columns","Learn how to create new columns and update existing ones with calculations and conditional logic","adding-and-modifying-columns",25,"content:pandas:25.adding-and-modifying-columns.md","pandas/25.adding-and-modifying-columns.md","pandas/25.adding-and-modifying-columns",{"_path":249,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":250,"description":251,"slug":252,"parentId":229,"order":253,"_type":14,"_id":254,"_source":16,"_file":255,"_stem":256,"_extension":19},"/pandas/applying-functions","Applying Functions","Use built-in and custom functions to transform your data efficiently","applying-functions",26,"content:pandas:26.applying-functions.md","pandas/26.applying-functions.md","pandas/26.applying-functions",{"_path":258,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":259,"description":260,"slug":261,"parentId":229,"order":262,"_type":14,"_id":263,"_source":16,"_file":264,"_stem":265,"_extension":19},"/pandas/renaming-and-sorting-data","Renaming and Sorting Data","Organize your DataFrame with better column names and logical data ordering","renaming-and-sorting-data",27,"content:pandas:27.renaming-and-sorting-data.md","pandas/27.renaming-and-sorting-data.md","pandas/27.renaming-and-sorting-data",{"_path":267,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":268,"description":269,"slug":270,"order":271,"isParent":12,"icon":234,"_type":14,"_id":272,"_source":16,"_file":273,"_stem":274,"_extension":19,"children":275},"/pandas/grouping-and-aggregation","Grouping and Aggregation","Learn to group data and calculate summary statistics for powerful data analysis","grouping-and-aggregation",28,"content:pandas:28.grouping-and-aggregation.md","pandas/28.grouping-and-aggregation.md","pandas/28.grouping-and-aggregation",[276,285,294],{"_path":277,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":278,"description":279,"slug":280,"parentId":267,"order":281,"_type":14,"_id":282,"_source":16,"_file":283,"_stem":284,"_extension":19},"/pandas/groupby-operations","GroupBy Operations","Learn the fundamentals of splitting data into groups and basic aggregations","groupby-operations",29,"content:pandas:29.groupby-operations.md","pandas/29.groupby-operations.md","pandas/29.groupby-operations",{"_path":286,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":287,"description":288,"slug":289,"parentId":267,"order":290,"_type":14,"_id":291,"_source":16,"_file":292,"_stem":293,"_extension":19},"/pandas/aggregation-functions","Aggregation Functions","Explore different ways to summarize your grouped data with built-in and custom functions","aggregation-functions",30,"content:pandas:30.aggregation-functions.md","pandas/30.aggregation-functions.md","pandas/30.aggregation-functions",{"_path":295,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":296,"description":297,"slug":298,"parentId":267,"order":299,"_type":14,"_id":300,"_source":16,"_file":301,"_stem":302,"_extension":19},"/pandas/pivot-tables-and-cross-tabulation","Pivot Tables and Cross Tabulation","Create powerful summary tables and cross-tabulations for analysis","pivot-tables-and-cross-tabulation",31,"content:pandas:31.pivot-tables-and-cross-tabulation.md","pandas/31.pivot-tables-and-cross-tabulation.md","pandas/31.pivot-tables-and-cross-tabulation",{"_path":304,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":305,"description":306,"slug":307,"order":308,"isParent":12,"icon":234,"_type":14,"_id":309,"_source":16,"_file":310,"_stem":311,"_extension":19,"children":312},"/pandas/combining-data","Combining Data","Learn to combine data from multiple DataFrames for comprehensive analysis","combining-data",32,"content:pandas:32.combining-data.md","pandas/32.combining-data.md","pandas/32.combining-data",[313,322],{"_path":314,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":315,"description":316,"slug":317,"parentId":304,"order":318,"_type":14,"_id":319,"_source":16,"_file":320,"_stem":321,"_extension":19},"/pandas/concatenating-dataframes","Concatenating DataFrames","Learn to stack DataFrames vertically and horizontally for simple data combination","concatenating-dataframes",33,"content:pandas:33.concatenating-dataframes.md","pandas/33.concatenating-dataframes.md","pandas/33.concatenating-dataframes",{"_path":323,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":324,"description":325,"slug":326,"parentId":304,"order":327,"_type":14,"_id":328,"_source":16,"_file":329,"_stem":330,"_extension":19},"/pandas/merging-and-join-operations","Merging and Join Operations","Master different types of joins to combine related data from multiple sources","merging-and-join-operations",34,"content:pandas:34.merging-and-join-operations.md","pandas/34.merging-and-join-operations.md","pandas/34.merging-and-join-operations",{"_path":332,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":333,"description":334,"slug":335,"order":336,"isParent":12,"icon":234,"_type":14,"_id":337,"_source":16,"_file":338,"_stem":339,"_extension":19,"children":340},"/pandas/working-with-dates","Working with Dates","Master date and time operations for time-based analysis and reporting","working-with-dates",35,"content:pandas:35.working-with-dates.md","pandas/35.working-with-dates.md","pandas/35.working-with-dates",[341,350],{"_path":342,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":343,"description":344,"slug":345,"parentId":332,"order":346,"_type":14,"_id":347,"_source":16,"_file":348,"_stem":349,"_extension":19},"/pandas/datetime-operations","DateTime Operations","Learn to convert, manipulate, and extract information from dates and times","datetime-operations",36,"content:pandas:36.datetime-operations.md","pandas/36.datetime-operations.md","pandas/36.datetime-operations",{"_path":351,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":352,"description":353,"slug":354,"parentId":332,"order":355,"_type":14,"_id":356,"_source":16,"_file":357,"_stem":358,"_extension":19},"/pandas/time-series-and-resampling","Time Series and Resampling","Discover how to analyze data over time periods and aggregate by different time intervals","time-series-and-resampling",37,"content:pandas:37.time-series-and-resampling.md","pandas/37.time-series-and-resampling.md","pandas/37.time-series-and-resampling",{"_path":360,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":361,"description":362,"slug":363,"order":364,"isParent":12,"icon":234,"_type":14,"_id":365,"_source":16,"_file":366,"_stem":367,"_extension":19,"children":368},"/pandas/saving-your-work","Saving Your Work","Learn to export your analyzed data to different formats for sharing and reporting","saving-your-work",38,"content:pandas:38.saving-your-work.md","pandas/38.saving-your-work.md","pandas/38.saving-your-work",[369,378],{"_path":370,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":371,"description":372,"slug":373,"parentId":360,"order":374,"_type":14,"_id":375,"_source":16,"_file":376,"_stem":377,"_extension":19},"/pandas/exporting-data","Exporting Data (CSV, Excel, JSON)","Learn to save your DataFrames to common file formats for sharing and reporting","exporting-data",39,"content:pandas:39.exporting-data.md","pandas/39.exporting-data.md","pandas/39.exporting-data",{"_path":379,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":380,"description":381,"slug":382,"parentId":360,"order":383,"_type":14,"_id":384,"_source":16,"_file":385,"_stem":386,"_extension":19},"/pandas/database-export","Database Export","Discover how to save data back to databases for integration with other systems","database-export",40,"content:pandas:40.database-export.md","pandas/40.database-export.md","pandas/40.database-export",[201],{"_path":202,"_dir":23,"_draft":7,"_partial":7,"_locale":6,"title":203,"description":204,"slug":205,"parentId":182,"order":206,"body":389,"_type":14,"_id":207,"_source":16,"_file":208,"_stem":209,"_extension":19},{"type":390,"children":391,"toc":8328},"root",[392,400,406,537,544,549,1309,1315,1320,2105,2111,2116,2918,2924,2929,3958,3964,3969,5015,5021,5026,6219,6225,6230,6375,6941,6947,7164,7170,7175,8291,8297,8302,8317,8322],{"type":393,"tag":394,"props":395,"children":396},"element","h1",{"id":205},[397],{"type":398,"value":399},"text","🧹 Removing Duplicates",{"type":393,"tag":401,"props":402,"children":403},"p",{},[404],{"type":398,"value":405},"Duplicate data is like having identical photos in your album - they take up space and can skew your analysis. Whether from data entry errors, system glitches, or merging datasets, duplicates are common and need to be handled properly.",{"type":393,"tag":407,"props":408,"children":412},"info-box",{"emoji":409,"title":410,"type":411},"🧹","What Are Duplicate Rows?","info",[413,422,430,465,473,496,506,514],{"type":393,"tag":401,"props":414,"children":415},{},[416],{"type":393,"tag":417,"props":418,"children":419},"strong",{},[420],{"type":398,"value":421},"Duplicates = Identical or Nearly Identical Rows",{"type":393,"tag":401,"props":423,"children":424},{},[425],{"type":393,"tag":417,"props":426,"children":427},{},[428],{"type":398,"value":429},"Types of Duplicates:",{"type":393,"tag":431,"props":432,"children":433},"ul",{},[434,445,455],{"type":393,"tag":435,"props":436,"children":437},"li",{},[438,443],{"type":393,"tag":417,"props":439,"children":440},{},[441],{"type":398,"value":442},"Exact duplicates",{"type":398,"value":444},": All columns exactly the same",{"type":393,"tag":435,"props":446,"children":447},{},[448,453],{"type":393,"tag":417,"props":449,"children":450},{},[451],{"type":398,"value":452},"Partial duplicates",{"type":398,"value":454},": Key columns same, other columns different",{"type":393,"tag":435,"props":456,"children":457},{},[458,463],{"type":393,"tag":417,"props":459,"children":460},{},[461],{"type":398,"value":462},"Near duplicates",{"type":398,"value":464},": Similar but not identical (like \"John\" vs \"Jon\")",{"type":393,"tag":401,"props":466,"children":467},{},[468],{"type":393,"tag":417,"props":469,"children":470},{},[471],{"type":398,"value":472},"Common Causes:",{"type":393,"tag":431,"props":474,"children":475},{},[476,481,486,491],{"type":393,"tag":435,"props":477,"children":478},{},[479],{"type":398,"value":480},"Data entry errors (accidental double-entry)",{"type":393,"tag":435,"props":482,"children":483},{},[484],{"type":398,"value":485},"System bugs (form submitted twice)",{"type":393,"tag":435,"props":487,"children":488},{},[489],{"type":398,"value":490},"Merging datasets with overlapping records",{"type":393,"tag":435,"props":492,"children":493},{},[494],{"type":398,"value":495},"Import errors (file processed multiple times)",{"type":393,"tag":401,"props":497,"children":498},{},[499,504],{"type":393,"tag":417,"props":500,"children":501},{},[502],{"type":398,"value":503},"Think of it like:",{"type":398,"value":505}," Finding identical items in your shopping cart - you probably only need one!",{"type":393,"tag":401,"props":507,"children":508},{},[509],{"type":393,"tag":417,"props":510,"children":511},{},[512],{"type":398,"value":513},"Why Remove Duplicates:",{"type":393,"tag":431,"props":515,"children":516},{},[517,522,527,532],{"type":393,"tag":435,"props":518,"children":519},{},[520],{"type":398,"value":521},"Accurate counts and statistics",{"type":393,"tag":435,"props":523,"children":524},{},[525],{"type":398,"value":526},"Prevent double-counting in analysis",{"type":393,"tag":435,"props":528,"children":529},{},[530],{"type":398,"value":531},"Save storage space",{"type":393,"tag":435,"props":533,"children":534},{},[535],{"type":398,"value":536},"Improve data quality",{"type":393,"tag":538,"props":539,"children":541},"h2",{"id":540},"identifying-duplicates",[542],{"type":398,"value":543},"🔍 Identifying Duplicates",{"type":393,"tag":401,"props":545,"children":546},{},[547],{"type":398,"value":548},"First, find where duplicates exist in your data:",{"type":393,"tag":550,"props":551,"children":552},"code-try",{},[553],{"type":393,"tag":554,"props":555,"children":559},"pre",{"className":556,"code":557,"language":558,"meta":6,"style":6},"language-python shiki shiki-themes github-light github-dark","import pandas as pd\n\n# Customer data with duplicates\ncustomers = pd.DataFrame({\n    'customer_id': [1, 2, 3, 2, 4, 3],\n    'name': ['Alice', 'Bob', 'Charlie', 'Bob', 'Diana', 'Charlie'],\n    'email': ['alice@email.com', 'bob@email.com', 'charlie@email.com', \n              'bob@email.com', 'diana@email.com', 'charlie@email.com'],\n    'purchase_amount': [100, 250, 150, 250, 75, 150]\n})\n\nprint(\"📊 Customer Data with Duplicates:\")\nprint(customers)\nprint()\n\nprint(\"🔍 Finding Duplicates:\")\nprint()\n\nprint(\"1️⃣ Check which rows are duplicates:\")\nprint(customers.duplicated())\nprint()\n\nprint(\"2️⃣ Count total duplicates:\")\nprint(f\"Number of duplicate rows: {customers.duplicated().sum()}\")\nprint()\n\nprint(\"3️⃣ See the actual duplicate rows:\")\nduplicates = customers[customers.duplicated()]\nprint(\"Duplicate rows:\")\nprint(duplicates)\nprint()\n\nprint(\"4️⃣ See all rows that have duplicates (including originals):\")\nhas_duplicates = customers[customers.duplicated(keep=False)]\nprint(\"All rows involved in duplication:\")\nprint(has_duplicates)\n","python",[560],{"type":393,"tag":561,"props":562,"children":563},"code",{"__ignoreMap":6},[564,591,599,608,626,695,759,799,828,893,901,908,931,943,955,962,982,993,1000,1020,1032,1043,1050,1070,1115,1126,1133,1153,1170,1190,1202,1213,1220,1240,1277,1297],{"type":393,"tag":565,"props":566,"children":568},"span",{"class":567,"line":11},"line",[569,575,581,586],{"type":393,"tag":565,"props":570,"children":572},{"style":571},"--shiki-default:#D73A49;--shiki-app-dark-mode:#F97583",[573],{"type":398,"value":574},"import",{"type":393,"tag":565,"props":576,"children":578},{"style":577},"--shiki-default:#24292E;--shiki-app-dark-mode:#E1E4E8",[579],{"type":398,"value":580}," pandas ",{"type":393,"tag":565,"props":582,"children":583},{"style":571},[584],{"type":398,"value":585},"as",{"type":393,"tag":565,"props":587,"children":588},{"style":577},[589],{"type":398,"value":590}," pd\n",{"type":393,"tag":565,"props":592,"children":593},{"class":567,"line":27},[594],{"type":393,"tag":565,"props":595,"children":596},{"emptyLinePlaceholder":12},[597],{"type":398,"value":598},"\n",{"type":393,"tag":565,"props":600,"children":601},{"class":567,"line":36},[602],{"type":393,"tag":565,"props":603,"children":605},{"style":604},"--shiki-default:#6A737D;--shiki-app-dark-mode:#6A737D",[606],{"type":398,"value":607},"# Customer data with duplicates\n",{"type":393,"tag":565,"props":609,"children":610},{"class":567,"line":45},[611,616,621],{"type":393,"tag":565,"props":612,"children":613},{"style":577},[614],{"type":398,"value":615},"customers ",{"type":393,"tag":565,"props":617,"children":618},{"style":571},[619],{"type":398,"value":620},"=",{"type":393,"tag":565,"props":622,"children":623},{"style":577},[624],{"type":398,"value":625}," pd.DataFrame({\n",{"type":393,"tag":565,"props":627,"children":628},{"class":567,"line":54},[629,635,640,646,651,656,660,665,669,673,677,682,686,690],{"type":393,"tag":565,"props":630,"children":632},{"style":631},"--shiki-default:#032F62;--shiki-app-dark-mode:#9ECBFF",[633],{"type":398,"value":634},"    'customer_id'",{"type":393,"tag":565,"props":636,"children":637},{"style":577},[638],{"type":398,"value":639},": [",{"type":393,"tag":565,"props":641,"children":643},{"style":642},"--shiki-default:#005CC5;--shiki-app-dark-mode:#79B8FF",[644],{"type":398,"value":645},"1",{"type":393,"tag":565,"props":647,"children":648},{"style":577},[649],{"type":398,"value":650},", ",{"type":393,"tag":565,"props":652,"children":653},{"style":642},[654],{"type":398,"value":655},"2",{"type":393,"tag":565,"props":657,"children":658},{"style":577},[659],{"type":398,"value":650},{"type":393,"tag":565,"props":661,"children":662},{"style":642},[663],{"type":398,"value":664},"3",{"type":393,"tag":565,"props":666,"children":667},{"style":577},[668],{"type":398,"value":650},{"type":393,"tag":565,"props":670,"children":671},{"style":642},[672],{"type":398,"value":655},{"type":393,"tag":565,"props":674,"children":675},{"style":577},[676],{"type":398,"value":650},{"type":393,"tag":565,"props":678,"children":679},{"style":642},[680],{"type":398,"value":681},"4",{"type":393,"tag":565,"props":683,"children":684},{"style":577},[685],{"type":398,"value":650},{"type":393,"tag":565,"props":687,"children":688},{"style":642},[689],{"type":398,"value":664},{"type":393,"tag":565,"props":691,"children":692},{"style":577},[693],{"type":398,"value":694},"],\n",{"type":393,"tag":565,"props":696,"children":697},{"class":567,"line":63},[698,703,707,712,716,721,725,730,734,738,742,747,751,755],{"type":393,"tag":565,"props":699,"children":700},{"style":631},[701],{"type":398,"value":702},"    'name'",{"type":393,"tag":565,"props":704,"children":705},{"style":577},[706],{"type":398,"value":639},{"type":393,"tag":565,"props":708,"children":709},{"style":631},[710],{"type":398,"value":711},"'Alice'",{"type":393,"tag":565,"props":713,"children":714},{"style":577},[715],{"type":398,"value":650},{"type":393,"tag":565,"props":717,"children":718},{"style":631},[719],{"type":398,"value":720},"'Bob'",{"type":393,"tag":565,"props":722,"children":723},{"style":577},[724],{"type":398,"value":650},{"type":393,"tag":565,"props":726,"children":727},{"style":631},[728],{"type":398,"value":729},"'Charlie'",{"type":393,"tag":565,"props":731,"children":732},{"style":577},[733],{"type":398,"value":650},{"type":393,"tag":565,"props":735,"children":736},{"style":631},[737],{"type":398,"value":720},{"type":393,"tag":565,"props":739,"children":740},{"style":577},[741],{"type":398,"value":650},{"type":393,"tag":565,"props":743,"children":744},{"style":631},[745],{"type":398,"value":746},"'Diana'",{"type":393,"tag":565,"props":748,"children":749},{"style":577},[750],{"type":398,"value":650},{"type":393,"tag":565,"props":752,"children":753},{"style":631},[754],{"type":398,"value":729},{"type":393,"tag":565,"props":756,"children":757},{"style":577},[758],{"type":398,"value":694},{"type":393,"tag":565,"props":760,"children":761},{"class":567,"line":72},[762,767,771,776,780,785,789,794],{"type":393,"tag":565,"props":763,"children":764},{"style":631},[765],{"type":398,"value":766},"    'email'",{"type":393,"tag":565,"props":768,"children":769},{"style":577},[770],{"type":398,"value":639},{"type":393,"tag":565,"props":772,"children":773},{"style":631},[774],{"type":398,"value":775},"'alice@email.com'",{"type":393,"tag":565,"props":777,"children":778},{"style":577},[779],{"type":398,"value":650},{"type":393,"tag":565,"props":781,"children":782},{"style":631},[783],{"type":398,"value":784},"'bob@email.com'",{"type":393,"tag":565,"props":786,"children":787},{"style":577},[788],{"type":398,"value":650},{"type":393,"tag":565,"props":790,"children":791},{"style":631},[792],{"type":398,"value":793},"'charlie@email.com'",{"type":393,"tag":565,"props":795,"children":796},{"style":577},[797],{"type":398,"value":798},", \n",{"type":393,"tag":565,"props":800,"children":801},{"class":567,"line":83},[802,807,811,816,820,824],{"type":393,"tag":565,"props":803,"children":804},{"style":631},[805],{"type":398,"value":806},"              'bob@email.com'",{"type":393,"tag":565,"props":808,"children":809},{"style":577},[810],{"type":398,"value":650},{"type":393,"tag":565,"props":812,"children":813},{"style":631},[814],{"type":398,"value":815},"'diana@email.com'",{"type":393,"tag":565,"props":817,"children":818},{"style":577},[819],{"type":398,"value":650},{"type":393,"tag":565,"props":821,"children":822},{"style":631},[823],{"type":398,"value":793},{"type":393,"tag":565,"props":825,"children":826},{"style":577},[827],{"type":398,"value":694},{"type":393,"tag":565,"props":829,"children":830},{"class":567,"line":92},[831,836,840,845,849,854,858,863,867,871,875,880,884,888],{"type":393,"tag":565,"props":832,"children":833},{"style":631},[834],{"type":398,"value":835},"    'purchase_amount'",{"type":393,"tag":565,"props":837,"children":838},{"style":577},[839],{"type":398,"value":639},{"type":393,"tag":565,"props":841,"children":842},{"style":642},[843],{"type":398,"value":844},"100",{"type":393,"tag":565,"props":846,"children":847},{"style":577},[848],{"type":398,"value":650},{"type":393,"tag":565,"props":850,"children":851},{"style":642},[852],{"type":398,"value":853},"250",{"type":393,"tag":565,"props":855,"children":856},{"style":577},[857],{"type":398,"value":650},{"type":393,"tag":565,"props":859,"children":860},{"style":642},[861],{"type":398,"value":862},"150",{"type":393,"tag":565,"props":864,"children":865},{"style":577},[866],{"type":398,"value":650},{"type":393,"tag":565,"props":868,"children":869},{"style":642},[870],{"type":398,"value":853},{"type":393,"tag":565,"props":872,"children":873},{"style":577},[874],{"type":398,"value":650},{"type":393,"tag":565,"props":876,"children":877},{"style":642},[878],{"type":398,"value":879},"75",{"type":393,"tag":565,"props":881,"children":882},{"style":577},[883],{"type":398,"value":650},{"type":393,"tag":565,"props":885,"children":886},{"style":642},[887],{"type":398,"value":862},{"type":393,"tag":565,"props":889,"children":890},{"style":577},[891],{"type":398,"value":892},"]\n",{"type":393,"tag":565,"props":894,"children":895},{"class":567,"line":101},[896],{"type":393,"tag":565,"props":897,"children":898},{"style":577},[899],{"type":398,"value":900},"})\n",{"type":393,"tag":565,"props":902,"children":903},{"class":567,"line":110},[904],{"type":393,"tag":565,"props":905,"children":906},{"emptyLinePlaceholder":12},[907],{"type":398,"value":598},{"type":393,"tag":565,"props":909,"children":910},{"class":567,"line":121},[911,916,921,926],{"type":393,"tag":565,"props":912,"children":913},{"style":642},[914],{"type":398,"value":915},"print",{"type":393,"tag":565,"props":917,"children":918},{"style":577},[919],{"type":398,"value":920},"(",{"type":393,"tag":565,"props":922,"children":923},{"style":631},[924],{"type":398,"value":925},"\"📊 Customer Data with Duplicates:\"",{"type":393,"tag":565,"props":927,"children":928},{"style":577},[929],{"type":398,"value":930},")\n",{"type":393,"tag":565,"props":932,"children":933},{"class":567,"line":130},[934,938],{"type":393,"tag":565,"props":935,"children":936},{"style":642},[937],{"type":398,"value":915},{"type":393,"tag":565,"props":939,"children":940},{"style":577},[941],{"type":398,"value":942},"(customers)\n",{"type":393,"tag":565,"props":944,"children":945},{"class":567,"line":139},[946,950],{"type":393,"tag":565,"props":947,"children":948},{"style":642},[949],{"type":398,"value":915},{"type":393,"tag":565,"props":951,"children":952},{"style":577},[953],{"type":398,"value":954},"()\n",{"type":393,"tag":565,"props":956,"children":957},{"class":567,"line":148},[958],{"type":393,"tag":565,"props":959,"children":960},{"emptyLinePlaceholder":12},[961],{"type":398,"value":598},{"type":393,"tag":565,"props":963,"children":964},{"class":567,"line":159},[965,969,973,978],{"type":393,"tag":565,"props":966,"children":967},{"style":642},[968],{"type":398,"value":915},{"type":393,"tag":565,"props":970,"children":971},{"style":577},[972],{"type":398,"value":920},{"type":393,"tag":565,"props":974,"children":975},{"style":631},[976],{"type":398,"value":977},"\"🔍 Finding Duplicates:\"",{"type":393,"tag":565,"props":979,"children":980},{"style":577},[981],{"type":398,"value":930},{"type":393,"tag":565,"props":983,"children":984},{"class":567,"line":168},[985,989],{"type":393,"tag":565,"props":986,"children":987},{"style":642},[988],{"type":398,"value":915},{"type":393,"tag":565,"props":990,"children":991},{"style":577},[992],{"type":398,"value":954},{"type":393,"tag":565,"props":994,"children":995},{"class":567,"line":177},[996],{"type":393,"tag":565,"props":997,"children":998},{"emptyLinePlaceholder":12},[999],{"type":398,"value":598},{"type":393,"tag":565,"props":1001,"children":1002},{"class":567,"line":186},[1003,1007,1011,1016],{"type":393,"tag":565,"props":1004,"children":1005},{"style":642},[1006],{"type":398,"value":915},{"type":393,"tag":565,"props":1008,"children":1009},{"style":577},[1010],{"type":398,"value":920},{"type":393,"tag":565,"props":1012,"children":1013},{"style":631},[1014],{"type":398,"value":1015},"\"1️⃣ Check which rows are duplicates:\"",{"type":393,"tag":565,"props":1017,"children":1018},{"style":577},[1019],{"type":398,"value":930},{"type":393,"tag":565,"props":1021,"children":1022},{"class":567,"line":197},[1023,1027],{"type":393,"tag":565,"props":1024,"children":1025},{"style":642},[1026],{"type":398,"value":915},{"type":393,"tag":565,"props":1028,"children":1029},{"style":577},[1030],{"type":398,"value":1031},"(customers.duplicated())\n",{"type":393,"tag":565,"props":1033,"children":1034},{"class":567,"line":206},[1035,1039],{"type":393,"tag":565,"props":1036,"children":1037},{"style":642},[1038],{"type":398,"value":915},{"type":393,"tag":565,"props":1040,"children":1041},{"style":577},[1042],{"type":398,"value":954},{"type":393,"tag":565,"props":1044,"children":1045},{"class":567,"line":215},[1046],{"type":393,"tag":565,"props":1047,"children":1048},{"emptyLinePlaceholder":12},[1049],{"type":398,"value":598},{"type":393,"tag":565,"props":1051,"children":1052},{"class":567,"line":224},[1053,1057,1061,1066],{"type":393,"tag":565,"props":1054,"children":1055},{"style":642},[1056],{"type":398,"value":915},{"type":393,"tag":565,"props":1058,"children":1059},{"style":577},[1060],{"type":398,"value":920},{"type":393,"tag":565,"props":1062,"children":1063},{"style":631},[1064],{"type":398,"value":1065},"\"2️⃣ Count total duplicates:\"",{"type":393,"tag":565,"props":1067,"children":1068},{"style":577},[1069],{"type":398,"value":930},{"type":393,"tag":565,"props":1071,"children":1072},{"class":567,"line":233},[1073,1077,1081,1086,1091,1096,1101,1106,1111],{"type":393,"tag":565,"props":1074,"children":1075},{"style":642},[1076],{"type":398,"value":915},{"type":393,"tag":565,"props":1078,"children":1079},{"style":577},[1080],{"type":398,"value":920},{"type":393,"tag":565,"props":1082,"children":1083},{"style":571},[1084],{"type":398,"value":1085},"f",{"type":393,"tag":565,"props":1087,"children":1088},{"style":631},[1089],{"type":398,"value":1090},"\"Number of duplicate rows: ",{"type":393,"tag":565,"props":1092,"children":1093},{"style":642},[1094],{"type":398,"value":1095},"{",{"type":393,"tag":565,"props":1097,"children":1098},{"style":577},[1099],{"type":398,"value":1100},"customers.duplicated().sum()",{"type":393,"tag":565,"props":1102,"children":1103},{"style":642},[1104],{"type":398,"value":1105},"}",{"type":393,"tag":565,"props":1107,"children":1108},{"style":631},[1109],{"type":398,"value":1110},"\"",{"type":393,"tag":565,"props":1112,"children":1113},{"style":577},[1114],{"type":398,"value":930},{"type":393,"tag":565,"props":1116,"children":1117},{"class":567,"line":244},[1118,1122],{"type":393,"tag":565,"props":1119,"children":1120},{"style":642},[1121],{"type":398,"value":915},{"type":393,"tag":565,"props":1123,"children":1124},{"style":577},[1125],{"type":398,"value":954},{"type":393,"tag":565,"props":1127,"children":1128},{"class":567,"line":253},[1129],{"type":393,"tag":565,"props":1130,"children":1131},{"emptyLinePlaceholder":12},[1132],{"type":398,"value":598},{"type":393,"tag":565,"props":1134,"children":1135},{"class":567,"line":262},[1136,1140,1144,1149],{"type":393,"tag":565,"props":1137,"children":1138},{"style":642},[1139],{"type":398,"value":915},{"type":393,"tag":565,"props":1141,"children":1142},{"style":577},[1143],{"type":398,"value":920},{"type":393,"tag":565,"props":1145,"children":1146},{"style":631},[1147],{"type":398,"value":1148},"\"3️⃣ See the actual duplicate rows:\"",{"type":393,"tag":565,"props":1150,"children":1151},{"style":577},[1152],{"type":398,"value":930},{"type":393,"tag":565,"props":1154,"children":1155},{"class":567,"line":271},[1156,1161,1165],{"type":393,"tag":565,"props":1157,"children":1158},{"style":577},[1159],{"type":398,"value":1160},"duplicates ",{"type":393,"tag":565,"props":1162,"children":1163},{"style":571},[1164],{"type":398,"value":620},{"type":393,"tag":565,"props":1166,"children":1167},{"style":577},[1168],{"type":398,"value":1169}," customers[customers.duplicated()]\n",{"type":393,"tag":565,"props":1171,"children":1172},{"class":567,"line":281},[1173,1177,1181,1186],{"type":393,"tag":565,"props":1174,"children":1175},{"style":642},[1176],{"type":398,"value":915},{"type":393,"tag":565,"props":1178,"children":1179},{"style":577},[1180],{"type":398,"value":920},{"type":393,"tag":565,"props":1182,"children":1183},{"style":631},[1184],{"type":398,"value":1185},"\"Duplicate rows:\"",{"type":393,"tag":565,"props":1187,"children":1188},{"style":577},[1189],{"type":398,"value":930},{"type":393,"tag":565,"props":1191,"children":1192},{"class":567,"line":290},[1193,1197],{"type":393,"tag":565,"props":1194,"children":1195},{"style":642},[1196],{"type":398,"value":915},{"type":393,"tag":565,"props":1198,"children":1199},{"style":577},[1200],{"type":398,"value":1201},"(duplicates)\n",{"type":393,"tag":565,"props":1203,"children":1204},{"class":567,"line":299},[1205,1209],{"type":393,"tag":565,"props":1206,"children":1207},{"style":642},[1208],{"type":398,"value":915},{"type":393,"tag":565,"props":1210,"children":1211},{"style":577},[1212],{"type":398,"value":954},{"type":393,"tag":565,"props":1214,"children":1215},{"class":567,"line":308},[1216],{"type":393,"tag":565,"props":1217,"children":1218},{"emptyLinePlaceholder":12},[1219],{"type":398,"value":598},{"type":393,"tag":565,"props":1221,"children":1222},{"class":567,"line":318},[1223,1227,1231,1236],{"type":393,"tag":565,"props":1224,"children":1225},{"style":642},[1226],{"type":398,"value":915},{"type":393,"tag":565,"props":1228,"children":1229},{"style":577},[1230],{"type":398,"value":920},{"type":393,"tag":565,"props":1232,"children":1233},{"style":631},[1234],{"type":398,"value":1235},"\"4️⃣ See all rows that have duplicates (including originals):\"",{"type":393,"tag":565,"props":1237,"children":1238},{"style":577},[1239],{"type":398,"value":930},{"type":393,"tag":565,"props":1241,"children":1242},{"class":567,"line":327},[1243,1248,1252,1257,1263,1267,1272],{"type":393,"tag":565,"props":1244,"children":1245},{"style":577},[1246],{"type":398,"value":1247},"has_duplicates ",{"type":393,"tag":565,"props":1249,"children":1250},{"style":571},[1251],{"type":398,"value":620},{"type":393,"tag":565,"props":1253,"children":1254},{"style":577},[1255],{"type":398,"value":1256}," customers[customers.duplicated(",{"type":393,"tag":565,"props":1258,"children":1260},{"style":1259},"--shiki-default:#E36209;--shiki-app-dark-mode:#FFAB70",[1261],{"type":398,"value":1262},"keep",{"type":393,"tag":565,"props":1264,"children":1265},{"style":571},[1266],{"type":398,"value":620},{"type":393,"tag":565,"props":1268,"children":1269},{"style":642},[1270],{"type":398,"value":1271},"False",{"type":393,"tag":565,"props":1273,"children":1274},{"style":577},[1275],{"type":398,"value":1276},")]\n",{"type":393,"tag":565,"props":1278,"children":1279},{"class":567,"line":336},[1280,1284,1288,1293],{"type":393,"tag":565,"props":1281,"children":1282},{"style":642},[1283],{"type":398,"value":915},{"type":393,"tag":565,"props":1285,"children":1286},{"style":577},[1287],{"type":398,"value":920},{"type":393,"tag":565,"props":1289,"children":1290},{"style":631},[1291],{"type":398,"value":1292},"\"All rows involved in duplication:\"",{"type":393,"tag":565,"props":1294,"children":1295},{"style":577},[1296],{"type":398,"value":930},{"type":393,"tag":565,"props":1298,"children":1299},{"class":567,"line":346},[1300,1304],{"type":393,"tag":565,"props":1301,"children":1302},{"style":642},[1303],{"type":398,"value":915},{"type":393,"tag":565,"props":1305,"children":1306},{"style":577},[1307],{"type":398,"value":1308},"(has_duplicates)\n",{"type":393,"tag":538,"props":1310,"children":1312},{"id":1311},"️-removing-all-duplicates",[1313],{"type":398,"value":1314},"🗑️ Removing All Duplicates",{"type":393,"tag":401,"props":1316,"children":1317},{},[1318],{"type":398,"value":1319},"Remove duplicate rows to clean your dataset:",{"type":393,"tag":550,"props":1321,"children":1322},{},[1323],{"type":393,"tag":554,"props":1324,"children":1326},{"className":556,"code":1325,"language":558,"meta":6,"style":6},"import pandas as pd\n\n# Order data with exact duplicates\norders = pd.DataFrame({\n    'order_id': [1001, 1002, 1003, 1002, 1004, 1003],\n    'customer': ['Alice', 'Bob', 'Charlie', 'Bob', 'Diana', 'Charlie'],\n    'product': ['Laptop', 'Mouse', 'Keyboard', 'Mouse', 'Monitor', 'Keyboard'],\n    'amount': [999, 25, 75, 25, 300, 75]\n})\n\nprint(\"📦 Orders with Duplicates:\")\nprint(orders)\nprint(f\"Total orders: {len(orders)}\")\nprint()\n\nprint(\"🗑️ Removing Duplicates:\")\nprint()\n\nprint(\"1️⃣ Remove duplicates (keep first occurrence):\")\nclean_orders = orders.drop_duplicates()\nprint(clean_orders)\nprint(f\"Clean orders: {len(clean_orders)} (removed {len(orders) - len(clean_orders)} duplicates)\")\nprint()\n\nprint(\"2️⃣ Remove duplicates (keep last occurrence):\")\nclean_orders_last = orders.drop_duplicates(keep='last')\nprint(clean_orders_last)\nprint()\n\nprint(\"3️⃣ Remove ALL duplicates (keep none):\")\nclean_orders_none = orders.drop_duplicates(keep=False)\nprint(clean_orders_none)\nprint(f\"Remaining after removing all duplicates: {len(clean_orders_none)}\")\n",[1327],{"type":393,"tag":561,"props":1328,"children":1329},{"__ignoreMap":6},[1330,1349,1356,1364,1380,1444,1504,1568,1631,1638,1645,1665,1677,1719,1730,1737,1757,1768,1775,1795,1812,1824,1898,1909,1916,1936,1970,1982,1993,2000,2020,2052,2064],{"type":393,"tag":565,"props":1331,"children":1332},{"class":567,"line":11},[1333,1337,1341,1345],{"type":393,"tag":565,"props":1334,"children":1335},{"style":571},[1336],{"type":398,"value":574},{"type":393,"tag":565,"props":1338,"children":1339},{"style":577},[1340],{"type":398,"value":580},{"type":393,"tag":565,"props":1342,"children":1343},{"style":571},[1344],{"type":398,"value":585},{"type":393,"tag":565,"props":1346,"children":1347},{"style":577},[1348],{"type":398,"value":590},{"type":393,"tag":565,"props":1350,"children":1351},{"class":567,"line":27},[1352],{"type":393,"tag":565,"props":1353,"children":1354},{"emptyLinePlaceholder":12},[1355],{"type":398,"value":598},{"type":393,"tag":565,"props":1357,"children":1358},{"class":567,"line":36},[1359],{"type":393,"tag":565,"props":1360,"children":1361},{"style":604},[1362],{"type":398,"value":1363},"# Order data with exact duplicates\n",{"type":393,"tag":565,"props":1365,"children":1366},{"class":567,"line":45},[1367,1372,1376],{"type":393,"tag":565,"props":1368,"children":1369},{"style":577},[1370],{"type":398,"value":1371},"orders ",{"type":393,"tag":565,"props":1373,"children":1374},{"style":571},[1375],{"type":398,"value":620},{"type":393,"tag":565,"props":1377,"children":1378},{"style":577},[1379],{"type":398,"value":625},{"type":393,"tag":565,"props":1381,"children":1382},{"class":567,"line":54},[1383,1388,1392,1397,1401,1406,1410,1415,1419,1423,1427,1432,1436,1440],{"type":393,"tag":565,"props":1384,"children":1385},{"style":631},[1386],{"type":398,"value":1387},"    'order_id'",{"type":393,"tag":565,"props":1389,"children":1390},{"style":577},[1391],{"type":398,"value":639},{"type":393,"tag":565,"props":1393,"children":1394},{"style":642},[1395],{"type":398,"value":1396},"1001",{"type":393,"tag":565,"props":1398,"children":1399},{"style":577},[1400],{"type":398,"value":650},{"type":393,"tag":565,"props":1402,"children":1403},{"style":642},[1404],{"type":398,"value":1405},"1002",{"type":393,"tag":565,"props":1407,"children":1408},{"style":577},[1409],{"type":398,"value":650},{"type":393,"tag":565,"props":1411,"children":1412},{"style":642},[1413],{"type":398,"value":1414},"1003",{"type":393,"tag":565,"props":1416,"children":1417},{"style":577},[1418],{"type":398,"value":650},{"type":393,"tag":565,"props":1420,"children":1421},{"style":642},[1422],{"type":398,"value":1405},{"type":393,"tag":565,"props":1424,"children":1425},{"style":577},[1426],{"type":398,"value":650},{"type":393,"tag":565,"props":1428,"children":1429},{"style":642},[1430],{"type":398,"value":1431},"1004",{"type":393,"tag":565,"props":1433,"children":1434},{"style":577},[1435],{"type":398,"value":650},{"type":393,"tag":565,"props":1437,"children":1438},{"style":642},[1439],{"type":398,"value":1414},{"type":393,"tag":565,"props":1441,"children":1442},{"style":577},[1443],{"type":398,"value":694},{"type":393,"tag":565,"props":1445,"children":1446},{"class":567,"line":63},[1447,1452,1456,1460,1464,1468,1472,1476,1480,1484,1488,1492,1496,1500],{"type":393,"tag":565,"props":1448,"children":1449},{"style":631},[1450],{"type":398,"value":1451},"    'customer'",{"type":393,"tag":565,"props":1453,"children":1454},{"style":577},[1455],{"type":398,"value":639},{"type":393,"tag":565,"props":1457,"children":1458},{"style":631},[1459],{"type":398,"value":711},{"type":393,"tag":565,"props":1461,"children":1462},{"style":577},[1463],{"type":398,"value":650},{"type":393,"tag":565,"props":1465,"children":1466},{"style":631},[1467],{"type":398,"value":720},{"type":393,"tag":565,"props":1469,"children":1470},{"style":577},[1471],{"type":398,"value":650},{"type":393,"tag":565,"props":1473,"children":1474},{"style":631},[1475],{"type":398,"value":729},{"type":393,"tag":565,"props":1477,"children":1478},{"style":577},[1479],{"type":398,"value":650},{"type":393,"tag":565,"props":1481,"children":1482},{"style":631},[1483],{"type":398,"value":720},{"type":393,"tag":565,"props":1485,"children":1486},{"style":577},[1487],{"type":398,"value":650},{"type":393,"tag":565,"props":1489,"children":1490},{"style":631},[1491],{"type":398,"value":746},{"type":393,"tag":565,"props":1493,"children":1494},{"style":577},[1495],{"type":398,"value":650},{"type":393,"tag":565,"props":1497,"children":1498},{"style":631},[1499],{"type":398,"value":729},{"type":393,"tag":565,"props":1501,"children":1502},{"style":577},[1503],{"type":398,"value":694},{"type":393,"tag":565,"props":1505,"children":1506},{"class":567,"line":72},[1507,1512,1516,1521,1525,1530,1534,1539,1543,1547,1551,1556,1560,1564],{"type":393,"tag":565,"props":1508,"children":1509},{"style":631},[1510],{"type":398,"value":1511},"    'product'",{"type":393,"tag":565,"props":1513,"children":1514},{"style":577},[1515],{"type":398,"value":639},{"type":393,"tag":565,"props":1517,"children":1518},{"style":631},[1519],{"type":398,"value":1520},"'Laptop'",{"type":393,"tag":565,"props":1522,"children":1523},{"style":577},[1524],{"type":398,"value":650},{"type":393,"tag":565,"props":1526,"children":1527},{"style":631},[1528],{"type":398,"value":1529},"'Mouse'",{"type":393,"tag":565,"props":1531,"children":1532},{"style":577},[1533],{"type":398,"value":650},{"type":393,"tag":565,"props":1535,"children":1536},{"style":631},[1537],{"type":398,"value":1538},"'Keyboard'",{"type":393,"tag":565,"props":1540,"children":1541},{"style":577},[1542],{"type":398,"value":650},{"type":393,"tag":565,"props":1544,"children":1545},{"style":631},[1546],{"type":398,"value":1529},{"type":393,"tag":565,"props":1548,"children":1549},{"style":577},[1550],{"type":398,"value":650},{"type":393,"tag":565,"props":1552,"children":1553},{"style":631},[1554],{"type":398,"value":1555},"'Monitor'",{"type":393,"tag":565,"props":1557,"children":1558},{"style":577},[1559],{"type":398,"value":650},{"type":393,"tag":565,"props":1561,"children":1562},{"style":631},[1563],{"type":398,"value":1538},{"type":393,"tag":565,"props":1565,"children":1566},{"style":577},[1567],{"type":398,"value":694},{"type":393,"tag":565,"props":1569,"children":1570},{"class":567,"line":83},[1571,1576,1580,1585,1589,1594,1598,1602,1606,1610,1614,1619,1623,1627],{"type":393,"tag":565,"props":1572,"children":1573},{"style":631},[1574],{"type":398,"value":1575},"    'amount'",{"type":393,"tag":565,"props":1577,"children":1578},{"style":577},[1579],{"type":398,"value":639},{"type":393,"tag":565,"props":1581,"children":1582},{"style":642},[1583],{"type":398,"value":1584},"999",{"type":393,"tag":565,"props":1586,"children":1587},{"style":577},[1588],{"type":398,"value":650},{"type":393,"tag":565,"props":1590,"children":1591},{"style":642},[1592],{"type":398,"value":1593},"25",{"type":393,"tag":565,"props":1595,"children":1596},{"style":577},[1597],{"type":398,"value":650},{"type":393,"tag":565,"props":1599,"children":1600},{"style":642},[1601],{"type":398,"value":879},{"type":393,"tag":565,"props":1603,"children":1604},{"style":577},[1605],{"type":398,"value":650},{"type":393,"tag":565,"props":1607,"children":1608},{"style":642},[1609],{"type":398,"value":1593},{"type":393,"tag":565,"props":1611,"children":1612},{"style":577},[1613],{"type":398,"value":650},{"type":393,"tag":565,"props":1615,"children":1616},{"style":642},[1617],{"type":398,"value":1618},"300",{"type":393,"tag":565,"props":1620,"children":1621},{"style":577},[1622],{"type":398,"value":650},{"type":393,"tag":565,"props":1624,"children":1625},{"style":642},[1626],{"type":398,"value":879},{"type":393,"tag":565,"props":1628,"children":1629},{"style":577},[1630],{"type":398,"value":892},{"type":393,"tag":565,"props":1632,"children":1633},{"class":567,"line":92},[1634],{"type":393,"tag":565,"props":1635,"children":1636},{"style":577},[1637],{"type":398,"value":900},{"type":393,"tag":565,"props":1639,"children":1640},{"class":567,"line":101},[1641],{"type":393,"tag":565,"props":1642,"children":1643},{"emptyLinePlaceholder":12},[1644],{"type":398,"value":598},{"type":393,"tag":565,"props":1646,"children":1647},{"class":567,"line":110},[1648,1652,1656,1661],{"type":393,"tag":565,"props":1649,"children":1650},{"style":642},[1651],{"type":398,"value":915},{"type":393,"tag":565,"props":1653,"children":1654},{"style":577},[1655],{"type":398,"value":920},{"type":393,"tag":565,"props":1657,"children":1658},{"style":631},[1659],{"type":398,"value":1660},"\"📦 Orders with Duplicates:\"",{"type":393,"tag":565,"props":1662,"children":1663},{"style":577},[1664],{"type":398,"value":930},{"type":393,"tag":565,"props":1666,"children":1667},{"class":567,"line":121},[1668,1672],{"type":393,"tag":565,"props":1669,"children":1670},{"style":642},[1671],{"type":398,"value":915},{"type":393,"tag":565,"props":1673,"children":1674},{"style":577},[1675],{"type":398,"value":1676},"(orders)\n",{"type":393,"tag":565,"props":1678,"children":1679},{"class":567,"line":130},[1680,1684,1688,1692,1697,1702,1707,1711,1715],{"type":393,"tag":565,"props":1681,"children":1682},{"style":642},[1683],{"type":398,"value":915},{"type":393,"tag":565,"props":1685,"children":1686},{"style":577},[1687],{"type":398,"value":920},{"type":393,"tag":565,"props":1689,"children":1690},{"style":571},[1691],{"type":398,"value":1085},{"type":393,"tag":565,"props":1693,"children":1694},{"style":631},[1695],{"type":398,"value":1696},"\"Total orders: ",{"type":393,"tag":565,"props":1698,"children":1699},{"style":642},[1700],{"type":398,"value":1701},"{len",{"type":393,"tag":565,"props":1703,"children":1704},{"style":577},[1705],{"type":398,"value":1706},"(orders)",{"type":393,"tag":565,"props":1708,"children":1709},{"style":642},[1710],{"type":398,"value":1105},{"type":393,"tag":565,"props":1712,"children":1713},{"style":631},[1714],{"type":398,"value":1110},{"type":393,"tag":565,"props":1716,"children":1717},{"style":577},[1718],{"type":398,"value":930},{"type":393,"tag":565,"props":1720,"children":1721},{"class":567,"line":139},[1722,1726],{"type":393,"tag":565,"props":1723,"children":1724},{"style":642},[1725],{"type":398,"value":915},{"type":393,"tag":565,"props":1727,"children":1728},{"style":577},[1729],{"type":398,"value":954},{"type":393,"tag":565,"props":1731,"children":1732},{"class":567,"line":148},[1733],{"type":393,"tag":565,"props":1734,"children":1735},{"emptyLinePlaceholder":12},[1736],{"type":398,"value":598},{"type":393,"tag":565,"props":1738,"children":1739},{"class":567,"line":159},[1740,1744,1748,1753],{"type":393,"tag":565,"props":1741,"children":1742},{"style":642},[1743],{"type":398,"value":915},{"type":393,"tag":565,"props":1745,"children":1746},{"style":577},[1747],{"type":398,"value":920},{"type":393,"tag":565,"props":1749,"children":1750},{"style":631},[1751],{"type":398,"value":1752},"\"🗑️ Removing Duplicates:\"",{"type":393,"tag":565,"props":1754,"children":1755},{"style":577},[1756],{"type":398,"value":930},{"type":393,"tag":565,"props":1758,"children":1759},{"class":567,"line":168},[1760,1764],{"type":393,"tag":565,"props":1761,"children":1762},{"style":642},[1763],{"type":398,"value":915},{"type":393,"tag":565,"props":1765,"children":1766},{"style":577},[1767],{"type":398,"value":954},{"type":393,"tag":565,"props":1769,"children":1770},{"class":567,"line":177},[1771],{"type":393,"tag":565,"props":1772,"children":1773},{"emptyLinePlaceholder":12},[1774],{"type":398,"value":598},{"type":393,"tag":565,"props":1776,"children":1777},{"class":567,"line":186},[1778,1782,1786,1791],{"type":393,"tag":565,"props":1779,"children":1780},{"style":642},[1781],{"type":398,"value":915},{"type":393,"tag":565,"props":1783,"children":1784},{"style":577},[1785],{"type":398,"value":920},{"type":393,"tag":565,"props":1787,"children":1788},{"style":631},[1789],{"type":398,"value":1790},"\"1️⃣ Remove duplicates (keep first occurrence):\"",{"type":393,"tag":565,"props":1792,"children":1793},{"style":577},[1794],{"type":398,"value":930},{"type":393,"tag":565,"props":1796,"children":1797},{"class":567,"line":197},[1798,1803,1807],{"type":393,"tag":565,"props":1799,"children":1800},{"style":577},[1801],{"type":398,"value":1802},"clean_orders ",{"type":393,"tag":565,"props":1804,"children":1805},{"style":571},[1806],{"type":398,"value":620},{"type":393,"tag":565,"props":1808,"children":1809},{"style":577},[1810],{"type":398,"value":1811}," orders.drop_duplicates()\n",{"type":393,"tag":565,"props":1813,"children":1814},{"class":567,"line":206},[1815,1819],{"type":393,"tag":565,"props":1816,"children":1817},{"style":642},[1818],{"type":398,"value":915},{"type":393,"tag":565,"props":1820,"children":1821},{"style":577},[1822],{"type":398,"value":1823},"(clean_orders)\n",{"type":393,"tag":565,"props":1825,"children":1826},{"class":567,"line":215},[1827,1831,1835,1839,1844,1848,1853,1857,1862,1866,1871,1876,1881,1885,1889,1894],{"type":393,"tag":565,"props":1828,"children":1829},{"style":642},[1830],{"type":398,"value":915},{"type":393,"tag":565,"props":1832,"children":1833},{"style":577},[1834],{"type":398,"value":920},{"type":393,"tag":565,"props":1836,"children":1837},{"style":571},[1838],{"type":398,"value":1085},{"type":393,"tag":565,"props":1840,"children":1841},{"style":631},[1842],{"type":398,"value":1843},"\"Clean orders: ",{"type":393,"tag":565,"props":1845,"children":1846},{"style":642},[1847],{"type":398,"value":1701},{"type":393,"tag":565,"props":1849,"children":1850},{"style":577},[1851],{"type":398,"value":1852},"(clean_orders)",{"type":393,"tag":565,"props":1854,"children":1855},{"style":642},[1856],{"type":398,"value":1105},{"type":393,"tag":565,"props":1858,"children":1859},{"style":631},[1860],{"type":398,"value":1861}," (removed ",{"type":393,"tag":565,"props":1863,"children":1864},{"style":642},[1865],{"type":398,"value":1701},{"type":393,"tag":565,"props":1867,"children":1868},{"style":577},[1869],{"type":398,"value":1870},"(orders) ",{"type":393,"tag":565,"props":1872,"children":1873},{"style":571},[1874],{"type":398,"value":1875},"-",{"type":393,"tag":565,"props":1877,"children":1878},{"style":642},[1879],{"type":398,"value":1880}," len",{"type":393,"tag":565,"props":1882,"children":1883},{"style":577},[1884],{"type":398,"value":1852},{"type":393,"tag":565,"props":1886,"children":1887},{"style":642},[1888],{"type":398,"value":1105},{"type":393,"tag":565,"props":1890,"children":1891},{"style":631},[1892],{"type":398,"value":1893}," duplicates)\"",{"type":393,"tag":565,"props":1895,"children":1896},{"style":577},[1897],{"type":398,"value":930},{"type":393,"tag":565,"props":1899,"children":1900},{"class":567,"line":224},[1901,1905],{"type":393,"tag":565,"props":1902,"children":1903},{"style":642},[1904],{"type":398,"value":915},{"type":393,"tag":565,"props":1906,"children":1907},{"style":577},[1908],{"type":398,"value":954},{"type":393,"tag":565,"props":1910,"children":1911},{"class":567,"line":233},[1912],{"type":393,"tag":565,"props":1913,"children":1914},{"emptyLinePlaceholder":12},[1915],{"type":398,"value":598},{"type":393,"tag":565,"props":1917,"children":1918},{"class":567,"line":244},[1919,1923,1927,1932],{"type":393,"tag":565,"props":1920,"children":1921},{"style":642},[1922],{"type":398,"value":915},{"type":393,"tag":565,"props":1924,"children":1925},{"style":577},[1926],{"type":398,"value":920},{"type":393,"tag":565,"props":1928,"children":1929},{"style":631},[1930],{"type":398,"value":1931},"\"2️⃣ Remove duplicates (keep last occurrence):\"",{"type":393,"tag":565,"props":1933,"children":1934},{"style":577},[1935],{"type":398,"value":930},{"type":393,"tag":565,"props":1937,"children":1938},{"class":567,"line":253},[1939,1944,1948,1953,1957,1961,1966],{"type":393,"tag":565,"props":1940,"children":1941},{"style":577},[1942],{"type":398,"value":1943},"clean_orders_last ",{"type":393,"tag":565,"props":1945,"children":1946},{"style":571},[1947],{"type":398,"value":620},{"type":393,"tag":565,"props":1949,"children":1950},{"style":577},[1951],{"type":398,"value":1952}," orders.drop_duplicates(",{"type":393,"tag":565,"props":1954,"children":1955},{"style":1259},[1956],{"type":398,"value":1262},{"type":393,"tag":565,"props":1958,"children":1959},{"style":571},[1960],{"type":398,"value":620},{"type":393,"tag":565,"props":1962,"children":1963},{"style":631},[1964],{"type":398,"value":1965},"'last'",{"type":393,"tag":565,"props":1967,"children":1968},{"style":577},[1969],{"type":398,"value":930},{"type":393,"tag":565,"props":1971,"children":1972},{"class":567,"line":262},[1973,1977],{"type":393,"tag":565,"props":1974,"children":1975},{"style":642},[1976],{"type":398,"value":915},{"type":393,"tag":565,"props":1978,"children":1979},{"style":577},[1980],{"type":398,"value":1981},"(clean_orders_last)\n",{"type":393,"tag":565,"props":1983,"children":1984},{"class":567,"line":271},[1985,1989],{"type":393,"tag":565,"props":1986,"children":1987},{"style":642},[1988],{"type":398,"value":915},{"type":393,"tag":565,"props":1990,"children":1991},{"style":577},[1992],{"type":398,"value":954},{"type":393,"tag":565,"props":1994,"children":1995},{"class":567,"line":281},[1996],{"type":393,"tag":565,"props":1997,"children":1998},{"emptyLinePlaceholder":12},[1999],{"type":398,"value":598},{"type":393,"tag":565,"props":2001,"children":2002},{"class":567,"line":290},[2003,2007,2011,2016],{"type":393,"tag":565,"props":2004,"children":2005},{"style":642},[2006],{"type":398,"value":915},{"type":393,"tag":565,"props":2008,"children":2009},{"style":577},[2010],{"type":398,"value":920},{"type":393,"tag":565,"props":2012,"children":2013},{"style":631},[2014],{"type":398,"value":2015},"\"3️⃣ Remove ALL duplicates (keep none):\"",{"type":393,"tag":565,"props":2017,"children":2018},{"style":577},[2019],{"type":398,"value":930},{"type":393,"tag":565,"props":2021,"children":2022},{"class":567,"line":299},[2023,2028,2032,2036,2040,2044,2048],{"type":393,"tag":565,"props":2024,"children":2025},{"style":577},[2026],{"type":398,"value":2027},"clean_orders_none ",{"type":393,"tag":565,"props":2029,"children":2030},{"style":571},[2031],{"type":398,"value":620},{"type":393,"tag":565,"props":2033,"children":2034},{"style":577},[2035],{"type":398,"value":1952},{"type":393,"tag":565,"props":2037,"children":2038},{"style":1259},[2039],{"type":398,"value":1262},{"type":393,"tag":565,"props":2041,"children":2042},{"style":571},[2043],{"type":398,"value":620},{"type":393,"tag":565,"props":2045,"children":2046},{"style":642},[2047],{"type":398,"value":1271},{"type":393,"tag":565,"props":2049,"children":2050},{"style":577},[2051],{"type":398,"value":930},{"type":393,"tag":565,"props":2053,"children":2054},{"class":567,"line":308},[2055,2059],{"type":393,"tag":565,"props":2056,"children":2057},{"style":642},[2058],{"type":398,"value":915},{"type":393,"tag":565,"props":2060,"children":2061},{"style":577},[2062],{"type":398,"value":2063},"(clean_orders_none)\n",{"type":393,"tag":565,"props":2065,"children":2066},{"class":567,"line":318},[2067,2071,2075,2079,2084,2088,2093,2097,2101],{"type":393,"tag":565,"props":2068,"children":2069},{"style":642},[2070],{"type":398,"value":915},{"type":393,"tag":565,"props":2072,"children":2073},{"style":577},[2074],{"type":398,"value":920},{"type":393,"tag":565,"props":2076,"children":2077},{"style":571},[2078],{"type":398,"value":1085},{"type":393,"tag":565,"props":2080,"children":2081},{"style":631},[2082],{"type":398,"value":2083},"\"Remaining after removing all duplicates: ",{"type":393,"tag":565,"props":2085,"children":2086},{"style":642},[2087],{"type":398,"value":1701},{"type":393,"tag":565,"props":2089,"children":2090},{"style":577},[2091],{"type":398,"value":2092},"(clean_orders_none)",{"type":393,"tag":565,"props":2094,"children":2095},{"style":642},[2096],{"type":398,"value":1105},{"type":393,"tag":565,"props":2098,"children":2099},{"style":631},[2100],{"type":398,"value":1110},{"type":393,"tag":565,"props":2102,"children":2103},{"style":577},[2104],{"type":398,"value":930},{"type":393,"tag":538,"props":2106,"children":2108},{"id":2107},"partial-duplicate-detection",[2109],{"type":398,"value":2110},"🎯 Partial Duplicate Detection",{"type":393,"tag":401,"props":2112,"children":2113},{},[2114],{"type":398,"value":2115},"Sometimes only certain columns matter for identifying duplicates:",{"type":393,"tag":550,"props":2117,"children":2118},{},[2119],{"type":393,"tag":554,"props":2120,"children":2122},{"className":556,"code":2121,"language":558,"meta":6,"style":6},"import pandas as pd\n\n# Employee data with partial duplicates\nemployees = pd.DataFrame({\n    'emp_id': [101, 102, 103, 104, 102],\n    'name': ['Alice', 'Bob', 'Charlie', 'Diana', 'Bob'],\n    'department': ['Sales', 'IT', 'HR', 'Sales', 'IT'],\n    'salary': [50000, 75000, 60000, 52000, 78000],  # Bob got a raise!\n    'hire_date': ['2020-01', '2019-03', '2021-06', '2020-08', '2019-03']\n})\n\nprint(\"👥 Employee Data:\")\nprint(employees)\nprint()\n\nprint(\"🎯 Partial Duplicate Detection:\")\nprint()\n\nprint(\"1️⃣ Duplicates based on employee ID only:\")\nid_duplicates = employees.duplicated(subset=['emp_id'])\nprint(\"Duplicate employee IDs:\")\nprint(employees[id_duplicates])\nprint()\n\nprint(\"2️⃣ Duplicates based on name and department:\")\nname_dept_duplicates = employees.duplicated(subset=['name', 'department'])\nprint(\"Same name and department:\")\nprint(employees[name_dept_duplicates])\nprint()\n\nprint(\"3️⃣ Remove duplicates keeping employee with higher salary:\")\n# For business logic: keep the record with higher salary\nunique_employees = employees.sort_values('salary').drop_duplicates(\n    subset=['emp_id'], keep='last'\n)\nprint(\"Unique employees (keeping higher salary):\")\nprint(unique_employees)\n",[2123],{"type":393,"tag":561,"props":2124,"children":2125},{"__ignoreMap":6},[2126,2145,2152,2160,2176,2232,2283,2338,2401,2457,2464,2471,2491,2503,2514,2521,2541,2552,2559,2579,2620,2640,2652,2663,2670,2690,2736,2756,2768,2779,2786,2806,2814,2841,2879,2886,2906],{"type":393,"tag":565,"props":2127,"children":2128},{"class":567,"line":11},[2129,2133,2137,2141],{"type":393,"tag":565,"props":2130,"children":2131},{"style":571},[2132],{"type":398,"value":574},{"type":393,"tag":565,"props":2134,"children":2135},{"style":577},[2136],{"type":398,"value":580},{"type":393,"tag":565,"props":2138,"children":2139},{"style":571},[2140],{"type":398,"value":585},{"type":393,"tag":565,"props":2142,"children":2143},{"style":577},[2144],{"type":398,"value":590},{"type":393,"tag":565,"props":2146,"children":2147},{"class":567,"line":27},[2148],{"type":393,"tag":565,"props":2149,"children":2150},{"emptyLinePlaceholder":12},[2151],{"type":398,"value":598},{"type":393,"tag":565,"props":2153,"children":2154},{"class":567,"line":36},[2155],{"type":393,"tag":565,"props":2156,"children":2157},{"style":604},[2158],{"type":398,"value":2159},"# Employee data with partial duplicates\n",{"type":393,"tag":565,"props":2161,"children":2162},{"class":567,"line":45},[2163,2168,2172],{"type":393,"tag":565,"props":2164,"children":2165},{"style":577},[2166],{"type":398,"value":2167},"employees ",{"type":393,"tag":565,"props":2169,"children":2170},{"style":571},[2171],{"type":398,"value":620},{"type":393,"tag":565,"props":2173,"children":2174},{"style":577},[2175],{"type":398,"value":625},{"type":393,"tag":565,"props":2177,"children":2178},{"class":567,"line":54},[2179,2184,2188,2193,2197,2202,2206,2211,2215,2220,2224,2228],{"type":393,"tag":565,"props":2180,"children":2181},{"style":631},[2182],{"type":398,"value":2183},"    'emp_id'",{"type":393,"tag":565,"props":2185,"children":2186},{"style":577},[2187],{"type":398,"value":639},{"type":393,"tag":565,"props":2189,"children":2190},{"style":642},[2191],{"type":398,"value":2192},"101",{"type":393,"tag":565,"props":2194,"children":2195},{"style":577},[2196],{"type":398,"value":650},{"type":393,"tag":565,"props":2198,"children":2199},{"style":642},[2200],{"type":398,"value":2201},"102",{"type":393,"tag":565,"props":2203,"children":2204},{"style":577},[2205],{"type":398,"value":650},{"type":393,"tag":565,"props":2207,"children":2208},{"style":642},[2209],{"type":398,"value":2210},"103",{"type":393,"tag":565,"props":2212,"children":2213},{"style":577},[2214],{"type":398,"value":650},{"type":393,"tag":565,"props":2216,"children":2217},{"style":642},[2218],{"type":398,"value":2219},"104",{"type":393,"tag":565,"props":2221,"children":2222},{"style":577},[2223],{"type":398,"value":650},{"type":393,"tag":565,"props":2225,"children":2226},{"style":642},[2227],{"type":398,"value":2201},{"type":393,"tag":565,"props":2229,"children":2230},{"style":577},[2231],{"type":398,"value":694},{"type":393,"tag":565,"props":2233,"children":2234},{"class":567,"line":63},[2235,2239,2243,2247,2251,2255,2259,2263,2267,2271,2275,2279],{"type":393,"tag":565,"props":2236,"children":2237},{"style":631},[2238],{"type":398,"value":702},{"type":393,"tag":565,"props":2240,"children":2241},{"style":577},[2242],{"type":398,"value":639},{"type":393,"tag":565,"props":2244,"children":2245},{"style":631},[2246],{"type":398,"value":711},{"type":393,"tag":565,"props":2248,"children":2249},{"style":577},[2250],{"type":398,"value":650},{"type":393,"tag":565,"props":2252,"children":2253},{"style":631},[2254],{"type":398,"value":720},{"type":393,"tag":565,"props":2256,"children":2257},{"style":577},[2258],{"type":398,"value":650},{"type":393,"tag":565,"props":2260,"children":2261},{"style":631},[2262],{"type":398,"value":729},{"type":393,"tag":565,"props":2264,"children":2265},{"style":577},[2266],{"type":398,"value":650},{"type":393,"tag":565,"props":2268,"children":2269},{"style":631},[2270],{"type":398,"value":746},{"type":393,"tag":565,"props":2272,"children":2273},{"style":577},[2274],{"type":398,"value":650},{"type":393,"tag":565,"props":2276,"children":2277},{"style":631},[2278],{"type":398,"value":720},{"type":393,"tag":565,"props":2280,"children":2281},{"style":577},[2282],{"type":398,"value":694},{"type":393,"tag":565,"props":2284,"children":2285},{"class":567,"line":72},[2286,2291,2295,2300,2304,2309,2313,2318,2322,2326,2330,2334],{"type":393,"tag":565,"props":2287,"children":2288},{"style":631},[2289],{"type":398,"value":2290},"    'department'",{"type":393,"tag":565,"props":2292,"children":2293},{"style":577},[2294],{"type":398,"value":639},{"type":393,"tag":565,"props":2296,"children":2297},{"style":631},[2298],{"type":398,"value":2299},"'Sales'",{"type":393,"tag":565,"props":2301,"children":2302},{"style":577},[2303],{"type":398,"value":650},{"type":393,"tag":565,"props":2305,"children":2306},{"style":631},[2307],{"type":398,"value":2308},"'IT'",{"type":393,"tag":565,"props":2310,"children":2311},{"style":577},[2312],{"type":398,"value":650},{"type":393,"tag":565,"props":2314,"children":2315},{"style":631},[2316],{"type":398,"value":2317},"'HR'",{"type":393,"tag":565,"props":2319,"children":2320},{"style":577},[2321],{"type":398,"value":650},{"type":393,"tag":565,"props":2323,"children":2324},{"style":631},[2325],{"type":398,"value":2299},{"type":393,"tag":565,"props":2327,"children":2328},{"style":577},[2329],{"type":398,"value":650},{"type":393,"tag":565,"props":2331,"children":2332},{"style":631},[2333],{"type":398,"value":2308},{"type":393,"tag":565,"props":2335,"children":2336},{"style":577},[2337],{"type":398,"value":694},{"type":393,"tag":565,"props":2339,"children":2340},{"class":567,"line":83},[2341,2346,2350,2355,2359,2364,2368,2373,2377,2382,2386,2391,2396],{"type":393,"tag":565,"props":2342,"children":2343},{"style":631},[2344],{"type":398,"value":2345},"    'salary'",{"type":393,"tag":565,"props":2347,"children":2348},{"style":577},[2349],{"type":398,"value":639},{"type":393,"tag":565,"props":2351,"children":2352},{"style":642},[2353],{"type":398,"value":2354},"50000",{"type":393,"tag":565,"props":2356,"children":2357},{"style":577},[2358],{"type":398,"value":650},{"type":393,"tag":565,"props":2360,"children":2361},{"style":642},[2362],{"type":398,"value":2363},"75000",{"type":393,"tag":565,"props":2365,"children":2366},{"style":577},[2367],{"type":398,"value":650},{"type":393,"tag":565,"props":2369,"children":2370},{"style":642},[2371],{"type":398,"value":2372},"60000",{"type":393,"tag":565,"props":2374,"children":2375},{"style":577},[2376],{"type":398,"value":650},{"type":393,"tag":565,"props":2378,"children":2379},{"style":642},[2380],{"type":398,"value":2381},"52000",{"type":393,"tag":565,"props":2383,"children":2384},{"style":577},[2385],{"type":398,"value":650},{"type":393,"tag":565,"props":2387,"children":2388},{"style":642},[2389],{"type":398,"value":2390},"78000",{"type":393,"tag":565,"props":2392,"children":2393},{"style":577},[2394],{"type":398,"value":2395},"],  ",{"type":393,"tag":565,"props":2397,"children":2398},{"style":604},[2399],{"type":398,"value":2400},"# Bob got a raise!\n",{"type":393,"tag":565,"props":2402,"children":2403},{"class":567,"line":92},[2404,2409,2413,2418,2422,2427,2431,2436,2440,2445,2449,2453],{"type":393,"tag":565,"props":2405,"children":2406},{"style":631},[2407],{"type":398,"value":2408},"    'hire_date'",{"type":393,"tag":565,"props":2410,"children":2411},{"style":577},[2412],{"type":398,"value":639},{"type":393,"tag":565,"props":2414,"children":2415},{"style":631},[2416],{"type":398,"value":2417},"'2020-01'",{"type":393,"tag":565,"props":2419,"children":2420},{"style":577},[2421],{"type":398,"value":650},{"type":393,"tag":565,"props":2423,"children":2424},{"style":631},[2425],{"type":398,"value":2426},"'2019-03'",{"type":393,"tag":565,"props":2428,"children":2429},{"style":577},[2430],{"type":398,"value":650},{"type":393,"tag":565,"props":2432,"children":2433},{"style":631},[2434],{"type":398,"value":2435},"'2021-06'",{"type":393,"tag":565,"props":2437,"children":2438},{"style":577},[2439],{"type":398,"value":650},{"type":393,"tag":565,"props":2441,"children":2442},{"style":631},[2443],{"type":398,"value":2444},"'2020-08'",{"type":393,"tag":565,"props":2446,"children":2447},{"style":577},[2448],{"type":398,"value":650},{"type":393,"tag":565,"props":2450,"children":2451},{"style":631},[2452],{"type":398,"value":2426},{"type":393,"tag":565,"props":2454,"children":2455},{"style":577},[2456],{"type":398,"value":892},{"type":393,"tag":565,"props":2458,"children":2459},{"class":567,"line":101},[2460],{"type":393,"tag":565,"props":2461,"children":2462},{"style":577},[2463],{"type":398,"value":900},{"type":393,"tag":565,"props":2465,"children":2466},{"class":567,"line":110},[2467],{"type":393,"tag":565,"props":2468,"children":2469},{"emptyLinePlaceholder":12},[2470],{"type":398,"value":598},{"type":393,"tag":565,"props":2472,"children":2473},{"class":567,"line":121},[2474,2478,2482,2487],{"type":393,"tag":565,"props":2475,"children":2476},{"style":642},[2477],{"type":398,"value":915},{"type":393,"tag":565,"props":2479,"children":2480},{"style":577},[2481],{"type":398,"value":920},{"type":393,"tag":565,"props":2483,"children":2484},{"style":631},[2485],{"type":398,"value":2486},"\"👥 Employee Data:\"",{"type":393,"tag":565,"props":2488,"children":2489},{"style":577},[2490],{"type":398,"value":930},{"type":393,"tag":565,"props":2492,"children":2493},{"class":567,"line":130},[2494,2498],{"type":393,"tag":565,"props":2495,"children":2496},{"style":642},[2497],{"type":398,"value":915},{"type":393,"tag":565,"props":2499,"children":2500},{"style":577},[2501],{"type":398,"value":2502},"(employees)\n",{"type":393,"tag":565,"props":2504,"children":2505},{"class":567,"line":139},[2506,2510],{"type":393,"tag":565,"props":2507,"children":2508},{"style":642},[2509],{"type":398,"value":915},{"type":393,"tag":565,"props":2511,"children":2512},{"style":577},[2513],{"type":398,"value":954},{"type":393,"tag":565,"props":2515,"children":2516},{"class":567,"line":148},[2517],{"type":393,"tag":565,"props":2518,"children":2519},{"emptyLinePlaceholder":12},[2520],{"type":398,"value":598},{"type":393,"tag":565,"props":2522,"children":2523},{"class":567,"line":159},[2524,2528,2532,2537],{"type":393,"tag":565,"props":2525,"children":2526},{"style":642},[2527],{"type":398,"value":915},{"type":393,"tag":565,"props":2529,"children":2530},{"style":577},[2531],{"type":398,"value":920},{"type":393,"tag":565,"props":2533,"children":2534},{"style":631},[2535],{"type":398,"value":2536},"\"🎯 Partial Duplicate Detection:\"",{"type":393,"tag":565,"props":2538,"children":2539},{"style":577},[2540],{"type":398,"value":930},{"type":393,"tag":565,"props":2542,"children":2543},{"class":567,"line":168},[2544,2548],{"type":393,"tag":565,"props":2545,"children":2546},{"style":642},[2547],{"type":398,"value":915},{"type":393,"tag":565,"props":2549,"children":2550},{"style":577},[2551],{"type":398,"value":954},{"type":393,"tag":565,"props":2553,"children":2554},{"class":567,"line":177},[2555],{"type":393,"tag":565,"props":2556,"children":2557},{"emptyLinePlaceholder":12},[2558],{"type":398,"value":598},{"type":393,"tag":565,"props":2560,"children":2561},{"class":567,"line":186},[2562,2566,2570,2575],{"type":393,"tag":565,"props":2563,"children":2564},{"style":642},[2565],{"type":398,"value":915},{"type":393,"tag":565,"props":2567,"children":2568},{"style":577},[2569],{"type":398,"value":920},{"type":393,"tag":565,"props":2571,"children":2572},{"style":631},[2573],{"type":398,"value":2574},"\"1️⃣ Duplicates based on employee ID only:\"",{"type":393,"tag":565,"props":2576,"children":2577},{"style":577},[2578],{"type":398,"value":930},{"type":393,"tag":565,"props":2580,"children":2581},{"class":567,"line":197},[2582,2587,2591,2596,2601,2605,2610,2615],{"type":393,"tag":565,"props":2583,"children":2584},{"style":577},[2585],{"type":398,"value":2586},"id_duplicates ",{"type":393,"tag":565,"props":2588,"children":2589},{"style":571},[2590],{"type":398,"value":620},{"type":393,"tag":565,"props":2592,"children":2593},{"style":577},[2594],{"type":398,"value":2595}," employees.duplicated(",{"type":393,"tag":565,"props":2597,"children":2598},{"style":1259},[2599],{"type":398,"value":2600},"subset",{"type":393,"tag":565,"props":2602,"children":2603},{"style":571},[2604],{"type":398,"value":620},{"type":393,"tag":565,"props":2606,"children":2607},{"style":577},[2608],{"type":398,"value":2609},"[",{"type":393,"tag":565,"props":2611,"children":2612},{"style":631},[2613],{"type":398,"value":2614},"'emp_id'",{"type":393,"tag":565,"props":2616,"children":2617},{"style":577},[2618],{"type":398,"value":2619},"])\n",{"type":393,"tag":565,"props":2621,"children":2622},{"class":567,"line":206},[2623,2627,2631,2636],{"type":393,"tag":565,"props":2624,"children":2625},{"style":642},[2626],{"type":398,"value":915},{"type":393,"tag":565,"props":2628,"children":2629},{"style":577},[2630],{"type":398,"value":920},{"type":393,"tag":565,"props":2632,"children":2633},{"style":631},[2634],{"type":398,"value":2635},"\"Duplicate employee IDs:\"",{"type":393,"tag":565,"props":2637,"children":2638},{"style":577},[2639],{"type":398,"value":930},{"type":393,"tag":565,"props":2641,"children":2642},{"class":567,"line":215},[2643,2647],{"type":393,"tag":565,"props":2644,"children":2645},{"style":642},[2646],{"type":398,"value":915},{"type":393,"tag":565,"props":2648,"children":2649},{"style":577},[2650],{"type":398,"value":2651},"(employees[id_duplicates])\n",{"type":393,"tag":565,"props":2653,"children":2654},{"class":567,"line":224},[2655,2659],{"type":393,"tag":565,"props":2656,"children":2657},{"style":642},[2658],{"type":398,"value":915},{"type":393,"tag":565,"props":2660,"children":2661},{"style":577},[2662],{"type":398,"value":954},{"type":393,"tag":565,"props":2664,"children":2665},{"class":567,"line":233},[2666],{"type":393,"tag":565,"props":2667,"children":2668},{"emptyLinePlaceholder":12},[2669],{"type":398,"value":598},{"type":393,"tag":565,"props":2671,"children":2672},{"class":567,"line":244},[2673,2677,2681,2686],{"type":393,"tag":565,"props":2674,"children":2675},{"style":642},[2676],{"type":398,"value":915},{"type":393,"tag":565,"props":2678,"children":2679},{"style":577},[2680],{"type":398,"value":920},{"type":393,"tag":565,"props":2682,"children":2683},{"style":631},[2684],{"type":398,"value":2685},"\"2️⃣ Duplicates based on name and department:\"",{"type":393,"tag":565,"props":2687,"children":2688},{"style":577},[2689],{"type":398,"value":930},{"type":393,"tag":565,"props":2691,"children":2692},{"class":567,"line":253},[2693,2698,2702,2706,2710,2714,2718,2723,2727,2732],{"type":393,"tag":565,"props":2694,"children":2695},{"style":577},[2696],{"type":398,"value":2697},"name_dept_duplicates ",{"type":393,"tag":565,"props":2699,"children":2700},{"style":571},[2701],{"type":398,"value":620},{"type":393,"tag":565,"props":2703,"children":2704},{"style":577},[2705],{"type":398,"value":2595},{"type":393,"tag":565,"props":2707,"children":2708},{"style":1259},[2709],{"type":398,"value":2600},{"type":393,"tag":565,"props":2711,"children":2712},{"style":571},[2713],{"type":398,"value":620},{"type":393,"tag":565,"props":2715,"children":2716},{"style":577},[2717],{"type":398,"value":2609},{"type":393,"tag":565,"props":2719,"children":2720},{"style":631},[2721],{"type":398,"value":2722},"'name'",{"type":393,"tag":565,"props":2724,"children":2725},{"style":577},[2726],{"type":398,"value":650},{"type":393,"tag":565,"props":2728,"children":2729},{"style":631},[2730],{"type":398,"value":2731},"'department'",{"type":393,"tag":565,"props":2733,"children":2734},{"style":577},[2735],{"type":398,"value":2619},{"type":393,"tag":565,"props":2737,"children":2738},{"class":567,"line":262},[2739,2743,2747,2752],{"type":393,"tag":565,"props":2740,"children":2741},{"style":642},[2742],{"type":398,"value":915},{"type":393,"tag":565,"props":2744,"children":2745},{"style":577},[2746],{"type":398,"value":920},{"type":393,"tag":565,"props":2748,"children":2749},{"style":631},[2750],{"type":398,"value":2751},"\"Same name and department:\"",{"type":393,"tag":565,"props":2753,"children":2754},{"style":577},[2755],{"type":398,"value":930},{"type":393,"tag":565,"props":2757,"children":2758},{"class":567,"line":271},[2759,2763],{"type":393,"tag":565,"props":2760,"children":2761},{"style":642},[2762],{"type":398,"value":915},{"type":393,"tag":565,"props":2764,"children":2765},{"style":577},[2766],{"type":398,"value":2767},"(employees[name_dept_duplicates])\n",{"type":393,"tag":565,"props":2769,"children":2770},{"class":567,"line":281},[2771,2775],{"type":393,"tag":565,"props":2772,"children":2773},{"style":642},[2774],{"type":398,"value":915},{"type":393,"tag":565,"props":2776,"children":2777},{"style":577},[2778],{"type":398,"value":954},{"type":393,"tag":565,"props":2780,"children":2781},{"class":567,"line":290},[2782],{"type":393,"tag":565,"props":2783,"children":2784},{"emptyLinePlaceholder":12},[2785],{"type":398,"value":598},{"type":393,"tag":565,"props":2787,"children":2788},{"class":567,"line":299},[2789,2793,2797,2802],{"type":393,"tag":565,"props":2790,"children":2791},{"style":642},[2792],{"type":398,"value":915},{"type":393,"tag":565,"props":2794,"children":2795},{"style":577},[2796],{"type":398,"value":920},{"type":393,"tag":565,"props":2798,"children":2799},{"style":631},[2800],{"type":398,"value":2801},"\"3️⃣ Remove duplicates keeping employee with higher salary:\"",{"type":393,"tag":565,"props":2803,"children":2804},{"style":577},[2805],{"type":398,"value":930},{"type":393,"tag":565,"props":2807,"children":2808},{"class":567,"line":308},[2809],{"type":393,"tag":565,"props":2810,"children":2811},{"style":604},[2812],{"type":398,"value":2813},"# For business logic: keep the record with higher salary\n",{"type":393,"tag":565,"props":2815,"children":2816},{"class":567,"line":318},[2817,2822,2826,2831,2836],{"type":393,"tag":565,"props":2818,"children":2819},{"style":577},[2820],{"type":398,"value":2821},"unique_employees ",{"type":393,"tag":565,"props":2823,"children":2824},{"style":571},[2825],{"type":398,"value":620},{"type":393,"tag":565,"props":2827,"children":2828},{"style":577},[2829],{"type":398,"value":2830}," employees.sort_values(",{"type":393,"tag":565,"props":2832,"children":2833},{"style":631},[2834],{"type":398,"value":2835},"'salary'",{"type":393,"tag":565,"props":2837,"children":2838},{"style":577},[2839],{"type":398,"value":2840},").drop_duplicates(\n",{"type":393,"tag":565,"props":2842,"children":2843},{"class":567,"line":327},[2844,2849,2853,2857,2861,2866,2870,2874],{"type":393,"tag":565,"props":2845,"children":2846},{"style":1259},[2847],{"type":398,"value":2848},"    subset",{"type":393,"tag":565,"props":2850,"children":2851},{"style":571},[2852],{"type":398,"value":620},{"type":393,"tag":565,"props":2854,"children":2855},{"style":577},[2856],{"type":398,"value":2609},{"type":393,"tag":565,"props":2858,"children":2859},{"style":631},[2860],{"type":398,"value":2614},{"type":393,"tag":565,"props":2862,"children":2863},{"style":577},[2864],{"type":398,"value":2865},"], ",{"type":393,"tag":565,"props":2867,"children":2868},{"style":1259},[2869],{"type":398,"value":1262},{"type":393,"tag":565,"props":2871,"children":2872},{"style":571},[2873],{"type":398,"value":620},{"type":393,"tag":565,"props":2875,"children":2876},{"style":631},[2877],{"type":398,"value":2878},"'last'\n",{"type":393,"tag":565,"props":2880,"children":2881},{"class":567,"line":336},[2882],{"type":393,"tag":565,"props":2883,"children":2884},{"style":577},[2885],{"type":398,"value":930},{"type":393,"tag":565,"props":2887,"children":2888},{"class":567,"line":346},[2889,2893,2897,2902],{"type":393,"tag":565,"props":2890,"children":2891},{"style":642},[2892],{"type":398,"value":915},{"type":393,"tag":565,"props":2894,"children":2895},{"style":577},[2896],{"type":398,"value":920},{"type":393,"tag":565,"props":2898,"children":2899},{"style":631},[2900],{"type":398,"value":2901},"\"Unique employees (keeping higher salary):\"",{"type":393,"tag":565,"props":2903,"children":2904},{"style":577},[2905],{"type":398,"value":930},{"type":393,"tag":565,"props":2907,"children":2908},{"class":567,"line":355},[2909,2913],{"type":393,"tag":565,"props":2910,"children":2911},{"style":642},[2912],{"type":398,"value":915},{"type":393,"tag":565,"props":2914,"children":2915},{"style":577},[2916],{"type":398,"value":2917},"(unique_employees)\n",{"type":393,"tag":538,"props":2919,"children":2921},{"id":2920},"real-world-duplicate-scenarios",[2922],{"type":398,"value":2923},"📊 Real-World Duplicate Scenarios",{"type":393,"tag":401,"props":2925,"children":2926},{},[2927],{"type":398,"value":2928},"Practical examples of handling duplicates in business contexts:",{"type":393,"tag":550,"props":2930,"children":2931},{},[2932],{"type":393,"tag":554,"props":2933,"children":2935},{"className":556,"code":2934,"language":558,"meta":6,"style":6},"import pandas as pd\n\n# E-commerce transaction data\ntransactions = pd.DataFrame({\n    'transaction_id': ['T001', 'T002', 'T003', 'T002', 'T004', 'T005'],\n    'customer_email': ['alice@email.com', 'bob@email.com', 'charlie@email.com', \n                       'bob@email.com', 'diana@email.com', 'eve@email.com'],\n    'product': ['Laptop', 'Mouse', 'Keyboard', 'Mouse', 'Monitor', 'Tablet'],\n    'amount': [999, 25, 75, 25, 300, 450],\n    'timestamp': ['2023-01-15 10:30', '2023-01-15 11:15', '2023-01-15 14:20',\n                  '2023-01-15 11:15', '2023-01-16 09:45', '2023-01-16 16:30']\n})\n\nprint(\"💳 Transaction Data with Potential Duplicates:\")\nprint(transactions)\nprint()\n\nprint(\"📊 Business Duplicate Analysis:\")\nprint()\n\nprint(\"1️⃣ Exact duplicate transactions (system error):\")\nexact_dups = transactions.duplicated()\nif exact_dups.any():\n    print(\"Exact duplicates found:\")\n    print(transactions[exact_dups])\nelse:\n    print(\"No exact duplicates\")\nprint()\n\nprint(\"2️⃣ Same transaction ID (should be unique):\")\nid_dups = transactions.duplicated(subset=['transaction_id'])\nprint(\"Duplicate transaction IDs:\")\nprint(transactions[id_dups])\nprint()\n\nprint(\"3️⃣ Same customer, product, amount (possible double-click):\")\nbusiness_dups = transactions.duplicated(subset=['customer_email', 'product', 'amount'])\nprint(\"Potential duplicate purchases:\")\nprint(transactions[business_dups])\nprint()\n\nprint(\"4️⃣ Clean data for analysis:\")\n# Remove exact duplicates and transaction ID duplicates\nclean_transactions = transactions.drop_duplicates()\nclean_transactions = clean_transactions.drop_duplicates(subset=['transaction_id'])\nprint(\"Clean transaction data:\")\nprint(clean_transactions)\nprint(f\"Transactions: {len(transactions)} → {len(clean_transactions)}\")\n",[2936],{"type":393,"tag":561,"props":2937,"children":2938},{"__ignoreMap":6},[2939,2958,2965,2973,2989,3054,3090,3119,3179,3239,3279,3309,3316,3323,3343,3355,3366,3373,3393,3404,3411,3431,3448,3461,3482,3494,3507,3527,3538,3545,3565,3603,3623,3635,3646,3653,3673,3728,3748,3760,3771,3779,3800,3809,3827,3864,3885,3898],{"type":393,"tag":565,"props":2940,"children":2941},{"class":567,"line":11},[2942,2946,2950,2954],{"type":393,"tag":565,"props":2943,"children":2944},{"style":571},[2945],{"type":398,"value":574},{"type":393,"tag":565,"props":2947,"children":2948},{"style":577},[2949],{"type":398,"value":580},{"type":393,"tag":565,"props":2951,"children":2952},{"style":571},[2953],{"type":398,"value":585},{"type":393,"tag":565,"props":2955,"children":2956},{"style":577},[2957],{"type":398,"value":590},{"type":393,"tag":565,"props":2959,"children":2960},{"class":567,"line":27},[2961],{"type":393,"tag":565,"props":2962,"children":2963},{"emptyLinePlaceholder":12},[2964],{"type":398,"value":598},{"type":393,"tag":565,"props":2966,"children":2967},{"class":567,"line":36},[2968],{"type":393,"tag":565,"props":2969,"children":2970},{"style":604},[2971],{"type":398,"value":2972},"# E-commerce transaction data\n",{"type":393,"tag":565,"props":2974,"children":2975},{"class":567,"line":45},[2976,2981,2985],{"type":393,"tag":565,"props":2977,"children":2978},{"style":577},[2979],{"type":398,"value":2980},"transactions ",{"type":393,"tag":565,"props":2982,"children":2983},{"style":571},[2984],{"type":398,"value":620},{"type":393,"tag":565,"props":2986,"children":2987},{"style":577},[2988],{"type":398,"value":625},{"type":393,"tag":565,"props":2990,"children":2991},{"class":567,"line":54},[2992,2997,3001,3006,3010,3015,3019,3024,3028,3032,3036,3041,3045,3050],{"type":393,"tag":565,"props":2993,"children":2994},{"style":631},[2995],{"type":398,"value":2996},"    'transaction_id'",{"type":393,"tag":565,"props":2998,"children":2999},{"style":577},[3000],{"type":398,"value":639},{"type":393,"tag":565,"props":3002,"children":3003},{"style":631},[3004],{"type":398,"value":3005},"'T001'",{"type":393,"tag":565,"props":3007,"children":3008},{"style":577},[3009],{"type":398,"value":650},{"type":393,"tag":565,"props":3011,"children":3012},{"style":631},[3013],{"type":398,"value":3014},"'T002'",{"type":393,"tag":565,"props":3016,"children":3017},{"style":577},[3018],{"type":398,"value":650},{"type":393,"tag":565,"props":3020,"children":3021},{"style":631},[3022],{"type":398,"value":3023},"'T003'",{"type":393,"tag":565,"props":3025,"children":3026},{"style":577},[3027],{"type":398,"value":650},{"type":393,"tag":565,"props":3029,"children":3030},{"style":631},[3031],{"type":398,"value":3014},{"type":393,"tag":565,"props":3033,"children":3034},{"style":577},[3035],{"type":398,"value":650},{"type":393,"tag":565,"props":3037,"children":3038},{"style":631},[3039],{"type":398,"value":3040},"'T004'",{"type":393,"tag":565,"props":3042,"children":3043},{"style":577},[3044],{"type":398,"value":650},{"type":393,"tag":565,"props":3046,"children":3047},{"style":631},[3048],{"type":398,"value":3049},"'T005'",{"type":393,"tag":565,"props":3051,"children":3052},{"style":577},[3053],{"type":398,"value":694},{"type":393,"tag":565,"props":3055,"children":3056},{"class":567,"line":63},[3057,3062,3066,3070,3074,3078,3082,3086],{"type":393,"tag":565,"props":3058,"children":3059},{"style":631},[3060],{"type":398,"value":3061},"    'customer_email'",{"type":393,"tag":565,"props":3063,"children":3064},{"style":577},[3065],{"type":398,"value":639},{"type":393,"tag":565,"props":3067,"children":3068},{"style":631},[3069],{"type":398,"value":775},{"type":393,"tag":565,"props":3071,"children":3072},{"style":577},[3073],{"type":398,"value":650},{"type":393,"tag":565,"props":3075,"children":3076},{"style":631},[3077],{"type":398,"value":784},{"type":393,"tag":565,"props":3079,"children":3080},{"style":577},[3081],{"type":398,"value":650},{"type":393,"tag":565,"props":3083,"children":3084},{"style":631},[3085],{"type":398,"value":793},{"type":393,"tag":565,"props":3087,"children":3088},{"style":577},[3089],{"type":398,"value":798},{"type":393,"tag":565,"props":3091,"children":3092},{"class":567,"line":72},[3093,3098,3102,3106,3110,3115],{"type":393,"tag":565,"props":3094,"children":3095},{"style":631},[3096],{"type":398,"value":3097},"                       'bob@email.com'",{"type":393,"tag":565,"props":3099,"children":3100},{"style":577},[3101],{"type":398,"value":650},{"type":393,"tag":565,"props":3103,"children":3104},{"style":631},[3105],{"type":398,"value":815},{"type":393,"tag":565,"props":3107,"children":3108},{"style":577},[3109],{"type":398,"value":650},{"type":393,"tag":565,"props":3111,"children":3112},{"style":631},[3113],{"type":398,"value":3114},"'eve@email.com'",{"type":393,"tag":565,"props":3116,"children":3117},{"style":577},[3118],{"type":398,"value":694},{"type":393,"tag":565,"props":3120,"children":3121},{"class":567,"line":83},[3122,3126,3130,3134,3138,3142,3146,3150,3154,3158,3162,3166,3170,3175],{"type":393,"tag":565,"props":3123,"children":3124},{"style":631},[3125],{"type":398,"value":1511},{"type":393,"tag":565,"props":3127,"children":3128},{"style":577},[3129],{"type":398,"value":639},{"type":393,"tag":565,"props":3131,"children":3132},{"style":631},[3133],{"type":398,"value":1520},{"type":393,"tag":565,"props":3135,"children":3136},{"style":577},[3137],{"type":398,"value":650},{"type":393,"tag":565,"props":3139,"children":3140},{"style":631},[3141],{"type":398,"value":1529},{"type":393,"tag":565,"props":3143,"children":3144},{"style":577},[3145],{"type":398,"value":650},{"type":393,"tag":565,"props":3147,"children":3148},{"style":631},[3149],{"type":398,"value":1538},{"type":393,"tag":565,"props":3151,"children":3152},{"style":577},[3153],{"type":398,"value":650},{"type":393,"tag":565,"props":3155,"children":3156},{"style":631},[3157],{"type":398,"value":1529},{"type":393,"tag":565,"props":3159,"children":3160},{"style":577},[3161],{"type":398,"value":650},{"type":393,"tag":565,"props":3163,"children":3164},{"style":631},[3165],{"type":398,"value":1555},{"type":393,"tag":565,"props":3167,"children":3168},{"style":577},[3169],{"type":398,"value":650},{"type":393,"tag":565,"props":3171,"children":3172},{"style":631},[3173],{"type":398,"value":3174},"'Tablet'",{"type":393,"tag":565,"props":3176,"children":3177},{"style":577},[3178],{"type":398,"value":694},{"type":393,"tag":565,"props":3180,"children":3181},{"class":567,"line":92},[3182,3186,3190,3194,3198,3202,3206,3210,3214,3218,3222,3226,3230,3235],{"type":393,"tag":565,"props":3183,"children":3184},{"style":631},[3185],{"type":398,"value":1575},{"type":393,"tag":565,"props":3187,"children":3188},{"style":577},[3189],{"type":398,"value":639},{"type":393,"tag":565,"props":3191,"children":3192},{"style":642},[3193],{"type":398,"value":1584},{"type":393,"tag":565,"props":3195,"children":3196},{"style":577},[3197],{"type":398,"value":650},{"type":393,"tag":565,"props":3199,"children":3200},{"style":642},[3201],{"type":398,"value":1593},{"type":393,"tag":565,"props":3203,"children":3204},{"style":577},[3205],{"type":398,"value":650},{"type":393,"tag":565,"props":3207,"children":3208},{"style":642},[3209],{"type":398,"value":879},{"type":393,"tag":565,"props":3211,"children":3212},{"style":577},[3213],{"type":398,"value":650},{"type":393,"tag":565,"props":3215,"children":3216},{"style":642},[3217],{"type":398,"value":1593},{"type":393,"tag":565,"props":3219,"children":3220},{"style":577},[3221],{"type":398,"value":650},{"type":393,"tag":565,"props":3223,"children":3224},{"style":642},[3225],{"type":398,"value":1618},{"type":393,"tag":565,"props":3227,"children":3228},{"style":577},[3229],{"type":398,"value":650},{"type":393,"tag":565,"props":3231,"children":3232},{"style":642},[3233],{"type":398,"value":3234},"450",{"type":393,"tag":565,"props":3236,"children":3237},{"style":577},[3238],{"type":398,"value":694},{"type":393,"tag":565,"props":3240,"children":3241},{"class":567,"line":101},[3242,3247,3251,3256,3260,3265,3269,3274],{"type":393,"tag":565,"props":3243,"children":3244},{"style":631},[3245],{"type":398,"value":3246},"    'timestamp'",{"type":393,"tag":565,"props":3248,"children":3249},{"style":577},[3250],{"type":398,"value":639},{"type":393,"tag":565,"props":3252,"children":3253},{"style":631},[3254],{"type":398,"value":3255},"'2023-01-15 10:30'",{"type":393,"tag":565,"props":3257,"children":3258},{"style":577},[3259],{"type":398,"value":650},{"type":393,"tag":565,"props":3261,"children":3262},{"style":631},[3263],{"type":398,"value":3264},"'2023-01-15 11:15'",{"type":393,"tag":565,"props":3266,"children":3267},{"style":577},[3268],{"type":398,"value":650},{"type":393,"tag":565,"props":3270,"children":3271},{"style":631},[3272],{"type":398,"value":3273},"'2023-01-15 14:20'",{"type":393,"tag":565,"props":3275,"children":3276},{"style":577},[3277],{"type":398,"value":3278},",\n",{"type":393,"tag":565,"props":3280,"children":3281},{"class":567,"line":110},[3282,3287,3291,3296,3300,3305],{"type":393,"tag":565,"props":3283,"children":3284},{"style":631},[3285],{"type":398,"value":3286},"                  '2023-01-15 11:15'",{"type":393,"tag":565,"props":3288,"children":3289},{"style":577},[3290],{"type":398,"value":650},{"type":393,"tag":565,"props":3292,"children":3293},{"style":631},[3294],{"type":398,"value":3295},"'2023-01-16 09:45'",{"type":393,"tag":565,"props":3297,"children":3298},{"style":577},[3299],{"type":398,"value":650},{"type":393,"tag":565,"props":3301,"children":3302},{"style":631},[3303],{"type":398,"value":3304},"'2023-01-16 16:30'",{"type":393,"tag":565,"props":3306,"children":3307},{"style":577},[3308],{"type":398,"value":892},{"type":393,"tag":565,"props":3310,"children":3311},{"class":567,"line":121},[3312],{"type":393,"tag":565,"props":3313,"children":3314},{"style":577},[3315],{"type":398,"value":900},{"type":393,"tag":565,"props":3317,"children":3318},{"class":567,"line":130},[3319],{"type":393,"tag":565,"props":3320,"children":3321},{"emptyLinePlaceholder":12},[3322],{"type":398,"value":598},{"type":393,"tag":565,"props":3324,"children":3325},{"class":567,"line":139},[3326,3330,3334,3339],{"type":393,"tag":565,"props":3327,"children":3328},{"style":642},[3329],{"type":398,"value":915},{"type":393,"tag":565,"props":3331,"children":3332},{"style":577},[3333],{"type":398,"value":920},{"type":393,"tag":565,"props":3335,"children":3336},{"style":631},[3337],{"type":398,"value":3338},"\"💳 Transaction Data with Potential Duplicates:\"",{"type":393,"tag":565,"props":3340,"children":3341},{"style":577},[3342],{"type":398,"value":930},{"type":393,"tag":565,"props":3344,"children":3345},{"class":567,"line":148},[3346,3350],{"type":393,"tag":565,"props":3347,"children":3348},{"style":642},[3349],{"type":398,"value":915},{"type":393,"tag":565,"props":3351,"children":3352},{"style":577},[3353],{"type":398,"value":3354},"(transactions)\n",{"type":393,"tag":565,"props":3356,"children":3357},{"class":567,"line":159},[3358,3362],{"type":393,"tag":565,"props":3359,"children":3360},{"style":642},[3361],{"type":398,"value":915},{"type":393,"tag":565,"props":3363,"children":3364},{"style":577},[3365],{"type":398,"value":954},{"type":393,"tag":565,"props":3367,"children":3368},{"class":567,"line":168},[3369],{"type":393,"tag":565,"props":3370,"children":3371},{"emptyLinePlaceholder":12},[3372],{"type":398,"value":598},{"type":393,"tag":565,"props":3374,"children":3375},{"class":567,"line":177},[3376,3380,3384,3389],{"type":393,"tag":565,"props":3377,"children":3378},{"style":642},[3379],{"type":398,"value":915},{"type":393,"tag":565,"props":3381,"children":3382},{"style":577},[3383],{"type":398,"value":920},{"type":393,"tag":565,"props":3385,"children":3386},{"style":631},[3387],{"type":398,"value":3388},"\"📊 Business Duplicate Analysis:\"",{"type":393,"tag":565,"props":3390,"children":3391},{"style":577},[3392],{"type":398,"value":930},{"type":393,"tag":565,"props":3394,"children":3395},{"class":567,"line":186},[3396,3400],{"type":393,"tag":565,"props":3397,"children":3398},{"style":642},[3399],{"type":398,"value":915},{"type":393,"tag":565,"props":3401,"children":3402},{"style":577},[3403],{"type":398,"value":954},{"type":393,"tag":565,"props":3405,"children":3406},{"class":567,"line":197},[3407],{"type":393,"tag":565,"props":3408,"children":3409},{"emptyLinePlaceholder":12},[3410],{"type":398,"value":598},{"type":393,"tag":565,"props":3412,"children":3413},{"class":567,"line":206},[3414,3418,3422,3427],{"type":393,"tag":565,"props":3415,"children":3416},{"style":642},[3417],{"type":398,"value":915},{"type":393,"tag":565,"props":3419,"children":3420},{"style":577},[3421],{"type":398,"value":920},{"type":393,"tag":565,"props":3423,"children":3424},{"style":631},[3425],{"type":398,"value":3426},"\"1️⃣ Exact duplicate transactions (system error):\"",{"type":393,"tag":565,"props":3428,"children":3429},{"style":577},[3430],{"type":398,"value":930},{"type":393,"tag":565,"props":3432,"children":3433},{"class":567,"line":215},[3434,3439,3443],{"type":393,"tag":565,"props":3435,"children":3436},{"style":577},[3437],{"type":398,"value":3438},"exact_dups ",{"type":393,"tag":565,"props":3440,"children":3441},{"style":571},[3442],{"type":398,"value":620},{"type":393,"tag":565,"props":3444,"children":3445},{"style":577},[3446],{"type":398,"value":3447}," transactions.duplicated()\n",{"type":393,"tag":565,"props":3449,"children":3450},{"class":567,"line":224},[3451,3456],{"type":393,"tag":565,"props":3452,"children":3453},{"style":571},[3454],{"type":398,"value":3455},"if",{"type":393,"tag":565,"props":3457,"children":3458},{"style":577},[3459],{"type":398,"value":3460}," exact_dups.any():\n",{"type":393,"tag":565,"props":3462,"children":3463},{"class":567,"line":233},[3464,3469,3473,3478],{"type":393,"tag":565,"props":3465,"children":3466},{"style":642},[3467],{"type":398,"value":3468},"    print",{"type":393,"tag":565,"props":3470,"children":3471},{"style":577},[3472],{"type":398,"value":920},{"type":393,"tag":565,"props":3474,"children":3475},{"style":631},[3476],{"type":398,"value":3477},"\"Exact duplicates found:\"",{"type":393,"tag":565,"props":3479,"children":3480},{"style":577},[3481],{"type":398,"value":930},{"type":393,"tag":565,"props":3483,"children":3484},{"class":567,"line":244},[3485,3489],{"type":393,"tag":565,"props":3486,"children":3487},{"style":642},[3488],{"type":398,"value":3468},{"type":393,"tag":565,"props":3490,"children":3491},{"style":577},[3492],{"type":398,"value":3493},"(transactions[exact_dups])\n",{"type":393,"tag":565,"props":3495,"children":3496},{"class":567,"line":253},[3497,3502],{"type":393,"tag":565,"props":3498,"children":3499},{"style":571},[3500],{"type":398,"value":3501},"else",{"type":393,"tag":565,"props":3503,"children":3504},{"style":577},[3505],{"type":398,"value":3506},":\n",{"type":393,"tag":565,"props":3508,"children":3509},{"class":567,"line":262},[3510,3514,3518,3523],{"type":393,"tag":565,"props":3511,"children":3512},{"style":642},[3513],{"type":398,"value":3468},{"type":393,"tag":565,"props":3515,"children":3516},{"style":577},[3517],{"type":398,"value":920},{"type":393,"tag":565,"props":3519,"children":3520},{"style":631},[3521],{"type":398,"value":3522},"\"No exact duplicates\"",{"type":393,"tag":565,"props":3524,"children":3525},{"style":577},[3526],{"type":398,"value":930},{"type":393,"tag":565,"props":3528,"children":3529},{"class":567,"line":271},[3530,3534],{"type":393,"tag":565,"props":3531,"children":3532},{"style":642},[3533],{"type":398,"value":915},{"type":393,"tag":565,"props":3535,"children":3536},{"style":577},[3537],{"type":398,"value":954},{"type":393,"tag":565,"props":3539,"children":3540},{"class":567,"line":281},[3541],{"type":393,"tag":565,"props":3542,"children":3543},{"emptyLinePlaceholder":12},[3544],{"type":398,"value":598},{"type":393,"tag":565,"props":3546,"children":3547},{"class":567,"line":290},[3548,3552,3556,3561],{"type":393,"tag":565,"props":3549,"children":3550},{"style":642},[3551],{"type":398,"value":915},{"type":393,"tag":565,"props":3553,"children":3554},{"style":577},[3555],{"type":398,"value":920},{"type":393,"tag":565,"props":3557,"children":3558},{"style":631},[3559],{"type":398,"value":3560},"\"2️⃣ Same transaction ID (should be unique):\"",{"type":393,"tag":565,"props":3562,"children":3563},{"style":577},[3564],{"type":398,"value":930},{"type":393,"tag":565,"props":3566,"children":3567},{"class":567,"line":299},[3568,3573,3577,3582,3586,3590,3594,3599],{"type":393,"tag":565,"props":3569,"children":3570},{"style":577},[3571],{"type":398,"value":3572},"id_dups ",{"type":393,"tag":565,"props":3574,"children":3575},{"style":571},[3576],{"type":398,"value":620},{"type":393,"tag":565,"props":3578,"children":3579},{"style":577},[3580],{"type":398,"value":3581}," transactions.duplicated(",{"type":393,"tag":565,"props":3583,"children":3584},{"style":1259},[3585],{"type":398,"value":2600},{"type":393,"tag":565,"props":3587,"children":3588},{"style":571},[3589],{"type":398,"value":620},{"type":393,"tag":565,"props":3591,"children":3592},{"style":577},[3593],{"type":398,"value":2609},{"type":393,"tag":565,"props":3595,"children":3596},{"style":631},[3597],{"type":398,"value":3598},"'transaction_id'",{"type":393,"tag":565,"props":3600,"children":3601},{"style":577},[3602],{"type":398,"value":2619},{"type":393,"tag":565,"props":3604,"children":3605},{"class":567,"line":308},[3606,3610,3614,3619],{"type":393,"tag":565,"props":3607,"children":3608},{"style":642},[3609],{"type":398,"value":915},{"type":393,"tag":565,"props":3611,"children":3612},{"style":577},[3613],{"type":398,"value":920},{"type":393,"tag":565,"props":3615,"children":3616},{"style":631},[3617],{"type":398,"value":3618},"\"Duplicate transaction IDs:\"",{"type":393,"tag":565,"props":3620,"children":3621},{"style":577},[3622],{"type":398,"value":930},{"type":393,"tag":565,"props":3624,"children":3625},{"class":567,"line":318},[3626,3630],{"type":393,"tag":565,"props":3627,"children":3628},{"style":642},[3629],{"type":398,"value":915},{"type":393,"tag":565,"props":3631,"children":3632},{"style":577},[3633],{"type":398,"value":3634},"(transactions[id_dups])\n",{"type":393,"tag":565,"props":3636,"children":3637},{"class":567,"line":327},[3638,3642],{"type":393,"tag":565,"props":3639,"children":3640},{"style":642},[3641],{"type":398,"value":915},{"type":393,"tag":565,"props":3643,"children":3644},{"style":577},[3645],{"type":398,"value":954},{"type":393,"tag":565,"props":3647,"children":3648},{"class":567,"line":336},[3649],{"type":393,"tag":565,"props":3650,"children":3651},{"emptyLinePlaceholder":12},[3652],{"type":398,"value":598},{"type":393,"tag":565,"props":3654,"children":3655},{"class":567,"line":346},[3656,3660,3664,3669],{"type":393,"tag":565,"props":3657,"children":3658},{"style":642},[3659],{"type":398,"value":915},{"type":393,"tag":565,"props":3661,"children":3662},{"style":577},[3663],{"type":398,"value":920},{"type":393,"tag":565,"props":3665,"children":3666},{"style":631},[3667],{"type":398,"value":3668},"\"3️⃣ Same customer, product, amount (possible double-click):\"",{"type":393,"tag":565,"props":3670,"children":3671},{"style":577},[3672],{"type":398,"value":930},{"type":393,"tag":565,"props":3674,"children":3675},{"class":567,"line":355},[3676,3681,3685,3689,3693,3697,3701,3706,3710,3715,3719,3724],{"type":393,"tag":565,"props":3677,"children":3678},{"style":577},[3679],{"type":398,"value":3680},"business_dups ",{"type":393,"tag":565,"props":3682,"children":3683},{"style":571},[3684],{"type":398,"value":620},{"type":393,"tag":565,"props":3686,"children":3687},{"style":577},[3688],{"type":398,"value":3581},{"type":393,"tag":565,"props":3690,"children":3691},{"style":1259},[3692],{"type":398,"value":2600},{"type":393,"tag":565,"props":3694,"children":3695},{"style":571},[3696],{"type":398,"value":620},{"type":393,"tag":565,"props":3698,"children":3699},{"style":577},[3700],{"type":398,"value":2609},{"type":393,"tag":565,"props":3702,"children":3703},{"style":631},[3704],{"type":398,"value":3705},"'customer_email'",{"type":393,"tag":565,"props":3707,"children":3708},{"style":577},[3709],{"type":398,"value":650},{"type":393,"tag":565,"props":3711,"children":3712},{"style":631},[3713],{"type":398,"value":3714},"'product'",{"type":393,"tag":565,"props":3716,"children":3717},{"style":577},[3718],{"type":398,"value":650},{"type":393,"tag":565,"props":3720,"children":3721},{"style":631},[3722],{"type":398,"value":3723},"'amount'",{"type":393,"tag":565,"props":3725,"children":3726},{"style":577},[3727],{"type":398,"value":2619},{"type":393,"tag":565,"props":3729,"children":3730},{"class":567,"line":364},[3731,3735,3739,3744],{"type":393,"tag":565,"props":3732,"children":3733},{"style":642},[3734],{"type":398,"value":915},{"type":393,"tag":565,"props":3736,"children":3737},{"style":577},[3738],{"type":398,"value":920},{"type":393,"tag":565,"props":3740,"children":3741},{"style":631},[3742],{"type":398,"value":3743},"\"Potential duplicate purchases:\"",{"type":393,"tag":565,"props":3745,"children":3746},{"style":577},[3747],{"type":398,"value":930},{"type":393,"tag":565,"props":3749,"children":3750},{"class":567,"line":374},[3751,3755],{"type":393,"tag":565,"props":3752,"children":3753},{"style":642},[3754],{"type":398,"value":915},{"type":393,"tag":565,"props":3756,"children":3757},{"style":577},[3758],{"type":398,"value":3759},"(transactions[business_dups])\n",{"type":393,"tag":565,"props":3761,"children":3762},{"class":567,"line":383},[3763,3767],{"type":393,"tag":565,"props":3764,"children":3765},{"style":642},[3766],{"type":398,"value":915},{"type":393,"tag":565,"props":3768,"children":3769},{"style":577},[3770],{"type":398,"value":954},{"type":393,"tag":565,"props":3772,"children":3774},{"class":567,"line":3773},41,[3775],{"type":393,"tag":565,"props":3776,"children":3777},{"emptyLinePlaceholder":12},[3778],{"type":398,"value":598},{"type":393,"tag":565,"props":3780,"children":3782},{"class":567,"line":3781},42,[3783,3787,3791,3796],{"type":393,"tag":565,"props":3784,"children":3785},{"style":642},[3786],{"type":398,"value":915},{"type":393,"tag":565,"props":3788,"children":3789},{"style":577},[3790],{"type":398,"value":920},{"type":393,"tag":565,"props":3792,"children":3793},{"style":631},[3794],{"type":398,"value":3795},"\"4️⃣ Clean data for analysis:\"",{"type":393,"tag":565,"props":3797,"children":3798},{"style":577},[3799],{"type":398,"value":930},{"type":393,"tag":565,"props":3801,"children":3803},{"class":567,"line":3802},43,[3804],{"type":393,"tag":565,"props":3805,"children":3806},{"style":604},[3807],{"type":398,"value":3808},"# Remove exact duplicates and transaction ID duplicates\n",{"type":393,"tag":565,"props":3810,"children":3812},{"class":567,"line":3811},44,[3813,3818,3822],{"type":393,"tag":565,"props":3814,"children":3815},{"style":577},[3816],{"type":398,"value":3817},"clean_transactions ",{"type":393,"tag":565,"props":3819,"children":3820},{"style":571},[3821],{"type":398,"value":620},{"type":393,"tag":565,"props":3823,"children":3824},{"style":577},[3825],{"type":398,"value":3826}," transactions.drop_duplicates()\n",{"type":393,"tag":565,"props":3828,"children":3830},{"class":567,"line":3829},45,[3831,3835,3839,3844,3848,3852,3856,3860],{"type":393,"tag":565,"props":3832,"children":3833},{"style":577},[3834],{"type":398,"value":3817},{"type":393,"tag":565,"props":3836,"children":3837},{"style":571},[3838],{"type":398,"value":620},{"type":393,"tag":565,"props":3840,"children":3841},{"style":577},[3842],{"type":398,"value":3843}," clean_transactions.drop_duplicates(",{"type":393,"tag":565,"props":3845,"children":3846},{"style":1259},[3847],{"type":398,"value":2600},{"type":393,"tag":565,"props":3849,"children":3850},{"style":571},[3851],{"type":398,"value":620},{"type":393,"tag":565,"props":3853,"children":3854},{"style":577},[3855],{"type":398,"value":2609},{"type":393,"tag":565,"props":3857,"children":3858},{"style":631},[3859],{"type":398,"value":3598},{"type":393,"tag":565,"props":3861,"children":3862},{"style":577},[3863],{"type":398,"value":2619},{"type":393,"tag":565,"props":3865,"children":3867},{"class":567,"line":3866},46,[3868,3872,3876,3881],{"type":393,"tag":565,"props":3869,"children":3870},{"style":642},[3871],{"type":398,"value":915},{"type":393,"tag":565,"props":3873,"children":3874},{"style":577},[3875],{"type":398,"value":920},{"type":393,"tag":565,"props":3877,"children":3878},{"style":631},[3879],{"type":398,"value":3880},"\"Clean transaction data:\"",{"type":393,"tag":565,"props":3882,"children":3883},{"style":577},[3884],{"type":398,"value":930},{"type":393,"tag":565,"props":3886,"children":3888},{"class":567,"line":3887},47,[3889,3893],{"type":393,"tag":565,"props":3890,"children":3891},{"style":642},[3892],{"type":398,"value":915},{"type":393,"tag":565,"props":3894,"children":3895},{"style":577},[3896],{"type":398,"value":3897},"(clean_transactions)\n",{"type":393,"tag":565,"props":3899,"children":3901},{"class":567,"line":3900},48,[3902,3906,3910,3914,3919,3923,3928,3932,3937,3941,3946,3950,3954],{"type":393,"tag":565,"props":3903,"children":3904},{"style":642},[3905],{"type":398,"value":915},{"type":393,"tag":565,"props":3907,"children":3908},{"style":577},[3909],{"type":398,"value":920},{"type":393,"tag":565,"props":3911,"children":3912},{"style":571},[3913],{"type":398,"value":1085},{"type":393,"tag":565,"props":3915,"children":3916},{"style":631},[3917],{"type":398,"value":3918},"\"Transactions: ",{"type":393,"tag":565,"props":3920,"children":3921},{"style":642},[3922],{"type":398,"value":1701},{"type":393,"tag":565,"props":3924,"children":3925},{"style":577},[3926],{"type":398,"value":3927},"(transactions)",{"type":393,"tag":565,"props":3929,"children":3930},{"style":642},[3931],{"type":398,"value":1105},{"type":393,"tag":565,"props":3933,"children":3934},{"style":631},[3935],{"type":398,"value":3936}," → ",{"type":393,"tag":565,"props":3938,"children":3939},{"style":642},[3940],{"type":398,"value":1701},{"type":393,"tag":565,"props":3942,"children":3943},{"style":577},[3944],{"type":398,"value":3945},"(clean_transactions)",{"type":393,"tag":565,"props":3947,"children":3948},{"style":642},[3949],{"type":398,"value":1105},{"type":393,"tag":565,"props":3951,"children":3952},{"style":631},[3953],{"type":398,"value":1110},{"type":393,"tag":565,"props":3955,"children":3956},{"style":577},[3957],{"type":398,"value":930},{"type":393,"tag":538,"props":3959,"children":3961},{"id":3960},"advanced-duplicate-handling",[3962],{"type":398,"value":3963},"🔧 Advanced Duplicate Handling",{"type":393,"tag":401,"props":3965,"children":3966},{},[3967],{"type":398,"value":3968},"Sophisticated techniques for complex duplicate scenarios:",{"type":393,"tag":550,"props":3970,"children":3971},{},[3972],{"type":393,"tag":554,"props":3973,"children":3975},{"className":556,"code":3974,"language":558,"meta":6,"style":6},"import pandas as pd\n\n# Customer database with merge conflicts\ncustomers = pd.DataFrame({\n    'customer_id': [1, 2, 3, 1, 4],\n    'name': ['Alice Smith', 'Bob Jones', 'Charlie Brown', 'Alice Smith', 'Diana Lee'],\n    'email': ['alice@email.com', 'bob@email.com', 'charlie@email.com', \n              'alice.smith@email.com', 'diana@email.com'],  # Alice changed email\n    'phone': ['123-456-7890', '555-123-4567', '555-987-6543', \n              '123-456-7890', '555-111-2222'],\n    'last_purchase': ['2023-01-15', '2023-02-10', '2023-01-25', \n                      '2023-03-05', '2023-02-20'],  # Alice made a recent purchase\n    'total_spent': [500, 250, 150, 750, 300]  # Alice's total is higher\n})\n\nprint(\"👤 Customer Database with Conflicts:\")\nprint(customers)\nprint()\n\nprint(\"🔧 Advanced Duplicate Resolution:\")\nprint()\n\nprint(\"1️⃣ Identify conflicts by customer ID:\")\nid_conflicts = customers[customers.duplicated(subset=['customer_id'], keep=False)]\nprint(\"Customer ID conflicts:\")\nprint(id_conflicts)\nprint()\n\nprint(\"2️⃣ Resolve by keeping most recent purchase:\")\nrecent_customers = customers.sort_values('last_purchase').drop_duplicates(\n    subset=['customer_id'], keep='last'\n)\nprint(\"Resolved by most recent purchase:\")\nprint(recent_customers)\nprint()\n\nprint(\"3️⃣ Resolve by keeping highest total spent:\")\nhigh_value_customers = customers.sort_values('total_spent').drop_duplicates(\n    subset=['customer_id'], keep='last'\n)\nprint(\"Resolved by highest spending:\")\nprint(high_value_customers)\nprint()\n\nprint(\"4️⃣ Manual resolution for critical conflicts:\")\n# For important data, you might need manual review\nprint(\"Manual review needed for:\")\nmanual_review = customers[customers['customer_id'].duplicated(keep=False)]\nprint(manual_review[['customer_id', 'name', 'email', 'total_spent']])\n",[3976],{"type":393,"tag":561,"props":3977,"children":3978},{"__ignoreMap":6},[3979,3998,4005,4013,4028,4079,4134,4169,4194,4233,4254,4293,4319,4379,4386,4393,4413,4424,4435,4442,4462,4473,4480,4500,4553,4573,4585,4596,4603,4623,4649,4684,4691,4711,4723,4734,4741,4761,4786,4821,4828,4848,4860,4871,4878,4898,4906,4926,4968],{"type":393,"tag":565,"props":3980,"children":3981},{"class":567,"line":11},[3982,3986,3990,3994],{"type":393,"tag":565,"props":3983,"children":3984},{"style":571},[3985],{"type":398,"value":574},{"type":393,"tag":565,"props":3987,"children":3988},{"style":577},[3989],{"type":398,"value":580},{"type":393,"tag":565,"props":3991,"children":3992},{"style":571},[3993],{"type":398,"value":585},{"type":393,"tag":565,"props":3995,"children":3996},{"style":577},[3997],{"type":398,"value":590},{"type":393,"tag":565,"props":3999,"children":4000},{"class":567,"line":27},[4001],{"type":393,"tag":565,"props":4002,"children":4003},{"emptyLinePlaceholder":12},[4004],{"type":398,"value":598},{"type":393,"tag":565,"props":4006,"children":4007},{"class":567,"line":36},[4008],{"type":393,"tag":565,"props":4009,"children":4010},{"style":604},[4011],{"type":398,"value":4012},"# Customer database with merge conflicts\n",{"type":393,"tag":565,"props":4014,"children":4015},{"class":567,"line":45},[4016,4020,4024],{"type":393,"tag":565,"props":4017,"children":4018},{"style":577},[4019],{"type":398,"value":615},{"type":393,"tag":565,"props":4021,"children":4022},{"style":571},[4023],{"type":398,"value":620},{"type":393,"tag":565,"props":4025,"children":4026},{"style":577},[4027],{"type":398,"value":625},{"type":393,"tag":565,"props":4029,"children":4030},{"class":567,"line":54},[4031,4035,4039,4043,4047,4051,4055,4059,4063,4067,4071,4075],{"type":393,"tag":565,"props":4032,"children":4033},{"style":631},[4034],{"type":398,"value":634},{"type":393,"tag":565,"props":4036,"children":4037},{"style":577},[4038],{"type":398,"value":639},{"type":393,"tag":565,"props":4040,"children":4041},{"style":642},[4042],{"type":398,"value":645},{"type":393,"tag":565,"props":4044,"children":4045},{"style":577},[4046],{"type":398,"value":650},{"type":393,"tag":565,"props":4048,"children":4049},{"style":642},[4050],{"type":398,"value":655},{"type":393,"tag":565,"props":4052,"children":4053},{"style":577},[4054],{"type":398,"value":650},{"type":393,"tag":565,"props":4056,"children":4057},{"style":642},[4058],{"type":398,"value":664},{"type":393,"tag":565,"props":4060,"children":4061},{"style":577},[4062],{"type":398,"value":650},{"type":393,"tag":565,"props":4064,"children":4065},{"style":642},[4066],{"type":398,"value":645},{"type":393,"tag":565,"props":4068,"children":4069},{"style":577},[4070],{"type":398,"value":650},{"type":393,"tag":565,"props":4072,"children":4073},{"style":642},[4074],{"type":398,"value":681},{"type":393,"tag":565,"props":4076,"children":4077},{"style":577},[4078],{"type":398,"value":694},{"type":393,"tag":565,"props":4080,"children":4081},{"class":567,"line":63},[4082,4086,4090,4095,4099,4104,4108,4113,4117,4121,4125,4130],{"type":393,"tag":565,"props":4083,"children":4084},{"style":631},[4085],{"type":398,"value":702},{"type":393,"tag":565,"props":4087,"children":4088},{"style":577},[4089],{"type":398,"value":639},{"type":393,"tag":565,"props":4091,"children":4092},{"style":631},[4093],{"type":398,"value":4094},"'Alice Smith'",{"type":393,"tag":565,"props":4096,"children":4097},{"style":577},[4098],{"type":398,"value":650},{"type":393,"tag":565,"props":4100,"children":4101},{"style":631},[4102],{"type":398,"value":4103},"'Bob Jones'",{"type":393,"tag":565,"props":4105,"children":4106},{"style":577},[4107],{"type":398,"value":650},{"type":393,"tag":565,"props":4109,"children":4110},{"style":631},[4111],{"type":398,"value":4112},"'Charlie Brown'",{"type":393,"tag":565,"props":4114,"children":4115},{"style":577},[4116],{"type":398,"value":650},{"type":393,"tag":565,"props":4118,"children":4119},{"style":631},[4120],{"type":398,"value":4094},{"type":393,"tag":565,"props":4122,"children":4123},{"style":577},[4124],{"type":398,"value":650},{"type":393,"tag":565,"props":4126,"children":4127},{"style":631},[4128],{"type":398,"value":4129},"'Diana Lee'",{"type":393,"tag":565,"props":4131,"children":4132},{"style":577},[4133],{"type":398,"value":694},{"type":393,"tag":565,"props":4135,"children":4136},{"class":567,"line":72},[4137,4141,4145,4149,4153,4157,4161,4165],{"type":393,"tag":565,"props":4138,"children":4139},{"style":631},[4140],{"type":398,"value":766},{"type":393,"tag":565,"props":4142,"children":4143},{"style":577},[4144],{"type":398,"value":639},{"type":393,"tag":565,"props":4146,"children":4147},{"style":631},[4148],{"type":398,"value":775},{"type":393,"tag":565,"props":4150,"children":4151},{"style":577},[4152],{"type":398,"value":650},{"type":393,"tag":565,"props":4154,"children":4155},{"style":631},[4156],{"type":398,"value":784},{"type":393,"tag":565,"props":4158,"children":4159},{"style":577},[4160],{"type":398,"value":650},{"type":393,"tag":565,"props":4162,"children":4163},{"style":631},[4164],{"type":398,"value":793},{"type":393,"tag":565,"props":4166,"children":4167},{"style":577},[4168],{"type":398,"value":798},{"type":393,"tag":565,"props":4170,"children":4171},{"class":567,"line":83},[4172,4177,4181,4185,4189],{"type":393,"tag":565,"props":4173,"children":4174},{"style":631},[4175],{"type":398,"value":4176},"              'alice.smith@email.com'",{"type":393,"tag":565,"props":4178,"children":4179},{"style":577},[4180],{"type":398,"value":650},{"type":393,"tag":565,"props":4182,"children":4183},{"style":631},[4184],{"type":398,"value":815},{"type":393,"tag":565,"props":4186,"children":4187},{"style":577},[4188],{"type":398,"value":2395},{"type":393,"tag":565,"props":4190,"children":4191},{"style":604},[4192],{"type":398,"value":4193},"# Alice changed email\n",{"type":393,"tag":565,"props":4195,"children":4196},{"class":567,"line":92},[4197,4202,4206,4211,4215,4220,4224,4229],{"type":393,"tag":565,"props":4198,"children":4199},{"style":631},[4200],{"type":398,"value":4201},"    'phone'",{"type":393,"tag":565,"props":4203,"children":4204},{"style":577},[4205],{"type":398,"value":639},{"type":393,"tag":565,"props":4207,"children":4208},{"style":631},[4209],{"type":398,"value":4210},"'123-456-7890'",{"type":393,"tag":565,"props":4212,"children":4213},{"style":577},[4214],{"type":398,"value":650},{"type":393,"tag":565,"props":4216,"children":4217},{"style":631},[4218],{"type":398,"value":4219},"'555-123-4567'",{"type":393,"tag":565,"props":4221,"children":4222},{"style":577},[4223],{"type":398,"value":650},{"type":393,"tag":565,"props":4225,"children":4226},{"style":631},[4227],{"type":398,"value":4228},"'555-987-6543'",{"type":393,"tag":565,"props":4230,"children":4231},{"style":577},[4232],{"type":398,"value":798},{"type":393,"tag":565,"props":4234,"children":4235},{"class":567,"line":101},[4236,4241,4245,4250],{"type":393,"tag":565,"props":4237,"children":4238},{"style":631},[4239],{"type":398,"value":4240},"              '123-456-7890'",{"type":393,"tag":565,"props":4242,"children":4243},{"style":577},[4244],{"type":398,"value":650},{"type":393,"tag":565,"props":4246,"children":4247},{"style":631},[4248],{"type":398,"value":4249},"'555-111-2222'",{"type":393,"tag":565,"props":4251,"children":4252},{"style":577},[4253],{"type":398,"value":694},{"type":393,"tag":565,"props":4255,"children":4256},{"class":567,"line":110},[4257,4262,4266,4271,4275,4280,4284,4289],{"type":393,"tag":565,"props":4258,"children":4259},{"style":631},[4260],{"type":398,"value":4261},"    'last_purchase'",{"type":393,"tag":565,"props":4263,"children":4264},{"style":577},[4265],{"type":398,"value":639},{"type":393,"tag":565,"props":4267,"children":4268},{"style":631},[4269],{"type":398,"value":4270},"'2023-01-15'",{"type":393,"tag":565,"props":4272,"children":4273},{"style":577},[4274],{"type":398,"value":650},{"type":393,"tag":565,"props":4276,"children":4277},{"style":631},[4278],{"type":398,"value":4279},"'2023-02-10'",{"type":393,"tag":565,"props":4281,"children":4282},{"style":577},[4283],{"type":398,"value":650},{"type":393,"tag":565,"props":4285,"children":4286},{"style":631},[4287],{"type":398,"value":4288},"'2023-01-25'",{"type":393,"tag":565,"props":4290,"children":4291},{"style":577},[4292],{"type":398,"value":798},{"type":393,"tag":565,"props":4294,"children":4295},{"class":567,"line":121},[4296,4301,4305,4310,4314],{"type":393,"tag":565,"props":4297,"children":4298},{"style":631},[4299],{"type":398,"value":4300},"                      '2023-03-05'",{"type":393,"tag":565,"props":4302,"children":4303},{"style":577},[4304],{"type":398,"value":650},{"type":393,"tag":565,"props":4306,"children":4307},{"style":631},[4308],{"type":398,"value":4309},"'2023-02-20'",{"type":393,"tag":565,"props":4311,"children":4312},{"style":577},[4313],{"type":398,"value":2395},{"type":393,"tag":565,"props":4315,"children":4316},{"style":604},[4317],{"type":398,"value":4318},"# Alice made a recent purchase\n",{"type":393,"tag":565,"props":4320,"children":4321},{"class":567,"line":130},[4322,4327,4331,4336,4340,4344,4348,4352,4356,4361,4365,4369,4374],{"type":393,"tag":565,"props":4323,"children":4324},{"style":631},[4325],{"type":398,"value":4326},"    'total_spent'",{"type":393,"tag":565,"props":4328,"children":4329},{"style":577},[4330],{"type":398,"value":639},{"type":393,"tag":565,"props":4332,"children":4333},{"style":642},[4334],{"type":398,"value":4335},"500",{"type":393,"tag":565,"props":4337,"children":4338},{"style":577},[4339],{"type":398,"value":650},{"type":393,"tag":565,"props":4341,"children":4342},{"style":642},[4343],{"type":398,"value":853},{"type":393,"tag":565,"props":4345,"children":4346},{"style":577},[4347],{"type":398,"value":650},{"type":393,"tag":565,"props":4349,"children":4350},{"style":642},[4351],{"type":398,"value":862},{"type":393,"tag":565,"props":4353,"children":4354},{"style":577},[4355],{"type":398,"value":650},{"type":393,"tag":565,"props":4357,"children":4358},{"style":642},[4359],{"type":398,"value":4360},"750",{"type":393,"tag":565,"props":4362,"children":4363},{"style":577},[4364],{"type":398,"value":650},{"type":393,"tag":565,"props":4366,"children":4367},{"style":642},[4368],{"type":398,"value":1618},{"type":393,"tag":565,"props":4370,"children":4371},{"style":577},[4372],{"type":398,"value":4373},"]  ",{"type":393,"tag":565,"props":4375,"children":4376},{"style":604},[4377],{"type":398,"value":4378},"# Alice's total is higher\n",{"type":393,"tag":565,"props":4380,"children":4381},{"class":567,"line":139},[4382],{"type":393,"tag":565,"props":4383,"children":4384},{"style":577},[4385],{"type":398,"value":900},{"type":393,"tag":565,"props":4387,"children":4388},{"class":567,"line":148},[4389],{"type":393,"tag":565,"props":4390,"children":4391},{"emptyLinePlaceholder":12},[4392],{"type":398,"value":598},{"type":393,"tag":565,"props":4394,"children":4395},{"class":567,"line":159},[4396,4400,4404,4409],{"type":393,"tag":565,"props":4397,"children":4398},{"style":642},[4399],{"type":398,"value":915},{"type":393,"tag":565,"props":4401,"children":4402},{"style":577},[4403],{"type":398,"value":920},{"type":393,"tag":565,"props":4405,"children":4406},{"style":631},[4407],{"type":398,"value":4408},"\"👤 Customer Database with Conflicts:\"",{"type":393,"tag":565,"props":4410,"children":4411},{"style":577},[4412],{"type":398,"value":930},{"type":393,"tag":565,"props":4414,"children":4415},{"class":567,"line":168},[4416,4420],{"type":393,"tag":565,"props":4417,"children":4418},{"style":642},[4419],{"type":398,"value":915},{"type":393,"tag":565,"props":4421,"children":4422},{"style":577},[4423],{"type":398,"value":942},{"type":393,"tag":565,"props":4425,"children":4426},{"class":567,"line":177},[4427,4431],{"type":393,"tag":565,"props":4428,"children":4429},{"style":642},[4430],{"type":398,"value":915},{"type":393,"tag":565,"props":4432,"children":4433},{"style":577},[4434],{"type":398,"value":954},{"type":393,"tag":565,"props":4436,"children":4437},{"class":567,"line":186},[4438],{"type":393,"tag":565,"props":4439,"children":4440},{"emptyLinePlaceholder":12},[4441],{"type":398,"value":598},{"type":393,"tag":565,"props":4443,"children":4444},{"class":567,"line":197},[4445,4449,4453,4458],{"type":393,"tag":565,"props":4446,"children":4447},{"style":642},[4448],{"type":398,"value":915},{"type":393,"tag":565,"props":4450,"children":4451},{"style":577},[4452],{"type":398,"value":920},{"type":393,"tag":565,"props":4454,"children":4455},{"style":631},[4456],{"type":398,"value":4457},"\"🔧 Advanced Duplicate Resolution:\"",{"type":393,"tag":565,"props":4459,"children":4460},{"style":577},[4461],{"type":398,"value":930},{"type":393,"tag":565,"props":4463,"children":4464},{"class":567,"line":206},[4465,4469],{"type":393,"tag":565,"props":4466,"children":4467},{"style":642},[4468],{"type":398,"value":915},{"type":393,"tag":565,"props":4470,"children":4471},{"style":577},[4472],{"type":398,"value":954},{"type":393,"tag":565,"props":4474,"children":4475},{"class":567,"line":215},[4476],{"type":393,"tag":565,"props":4477,"children":4478},{"emptyLinePlaceholder":12},[4479],{"type":398,"value":598},{"type":393,"tag":565,"props":4481,"children":4482},{"class":567,"line":224},[4483,4487,4491,4496],{"type":393,"tag":565,"props":4484,"children":4485},{"style":642},[4486],{"type":398,"value":915},{"type":393,"tag":565,"props":4488,"children":4489},{"style":577},[4490],{"type":398,"value":920},{"type":393,"tag":565,"props":4492,"children":4493},{"style":631},[4494],{"type":398,"value":4495},"\"1️⃣ Identify conflicts by customer ID:\"",{"type":393,"tag":565,"props":4497,"children":4498},{"style":577},[4499],{"type":398,"value":930},{"type":393,"tag":565,"props":4501,"children":4502},{"class":567,"line":233},[4503,4508,4512,4516,4520,4524,4528,4533,4537,4541,4545,4549],{"type":393,"tag":565,"props":4504,"children":4505},{"style":577},[4506],{"type":398,"value":4507},"id_conflicts ",{"type":393,"tag":565,"props":4509,"children":4510},{"style":571},[4511],{"type":398,"value":620},{"type":393,"tag":565,"props":4513,"children":4514},{"style":577},[4515],{"type":398,"value":1256},{"type":393,"tag":565,"props":4517,"children":4518},{"style":1259},[4519],{"type":398,"value":2600},{"type":393,"tag":565,"props":4521,"children":4522},{"style":571},[4523],{"type":398,"value":620},{"type":393,"tag":565,"props":4525,"children":4526},{"style":577},[4527],{"type":398,"value":2609},{"type":393,"tag":565,"props":4529,"children":4530},{"style":631},[4531],{"type":398,"value":4532},"'customer_id'",{"type":393,"tag":565,"props":4534,"children":4535},{"style":577},[4536],{"type":398,"value":2865},{"type":393,"tag":565,"props":4538,"children":4539},{"style":1259},[4540],{"type":398,"value":1262},{"type":393,"tag":565,"props":4542,"children":4543},{"style":571},[4544],{"type":398,"value":620},{"type":393,"tag":565,"props":4546,"children":4547},{"style":642},[4548],{"type":398,"value":1271},{"type":393,"tag":565,"props":4550,"children":4551},{"style":577},[4552],{"type":398,"value":1276},{"type":393,"tag":565,"props":4554,"children":4555},{"class":567,"line":244},[4556,4560,4564,4569],{"type":393,"tag":565,"props":4557,"children":4558},{"style":642},[4559],{"type":398,"value":915},{"type":393,"tag":565,"props":4561,"children":4562},{"style":577},[4563],{"type":398,"value":920},{"type":393,"tag":565,"props":4565,"children":4566},{"style":631},[4567],{"type":398,"value":4568},"\"Customer ID conflicts:\"",{"type":393,"tag":565,"props":4570,"children":4571},{"style":577},[4572],{"type":398,"value":930},{"type":393,"tag":565,"props":4574,"children":4575},{"class":567,"line":253},[4576,4580],{"type":393,"tag":565,"props":4577,"children":4578},{"style":642},[4579],{"type":398,"value":915},{"type":393,"tag":565,"props":4581,"children":4582},{"style":577},[4583],{"type":398,"value":4584},"(id_conflicts)\n",{"type":393,"tag":565,"props":4586,"children":4587},{"class":567,"line":262},[4588,4592],{"type":393,"tag":565,"props":4589,"children":4590},{"style":642},[4591],{"type":398,"value":915},{"type":393,"tag":565,"props":4593,"children":4594},{"style":577},[4595],{"type":398,"value":954},{"type":393,"tag":565,"props":4597,"children":4598},{"class":567,"line":271},[4599],{"type":393,"tag":565,"props":4600,"children":4601},{"emptyLinePlaceholder":12},[4602],{"type":398,"value":598},{"type":393,"tag":565,"props":4604,"children":4605},{"class":567,"line":281},[4606,4610,4614,4619],{"type":393,"tag":565,"props":4607,"children":4608},{"style":642},[4609],{"type":398,"value":915},{"type":393,"tag":565,"props":4611,"children":4612},{"style":577},[4613],{"type":398,"value":920},{"type":393,"tag":565,"props":4615,"children":4616},{"style":631},[4617],{"type":398,"value":4618},"\"2️⃣ Resolve by keeping most recent purchase:\"",{"type":393,"tag":565,"props":4620,"children":4621},{"style":577},[4622],{"type":398,"value":930},{"type":393,"tag":565,"props":4624,"children":4625},{"class":567,"line":290},[4626,4631,4635,4640,4645],{"type":393,"tag":565,"props":4627,"children":4628},{"style":577},[4629],{"type":398,"value":4630},"recent_customers ",{"type":393,"tag":565,"props":4632,"children":4633},{"style":571},[4634],{"type":398,"value":620},{"type":393,"tag":565,"props":4636,"children":4637},{"style":577},[4638],{"type":398,"value":4639}," customers.sort_values(",{"type":393,"tag":565,"props":4641,"children":4642},{"style":631},[4643],{"type":398,"value":4644},"'last_purchase'",{"type":393,"tag":565,"props":4646,"children":4647},{"style":577},[4648],{"type":398,"value":2840},{"type":393,"tag":565,"props":4650,"children":4651},{"class":567,"line":299},[4652,4656,4660,4664,4668,4672,4676,4680],{"type":393,"tag":565,"props":4653,"children":4654},{"style":1259},[4655],{"type":398,"value":2848},{"type":393,"tag":565,"props":4657,"children":4658},{"style":571},[4659],{"type":398,"value":620},{"type":393,"tag":565,"props":4661,"children":4662},{"style":577},[4663],{"type":398,"value":2609},{"type":393,"tag":565,"props":4665,"children":4666},{"style":631},[4667],{"type":398,"value":4532},{"type":393,"tag":565,"props":4669,"children":4670},{"style":577},[4671],{"type":398,"value":2865},{"type":393,"tag":565,"props":4673,"children":4674},{"style":1259},[4675],{"type":398,"value":1262},{"type":393,"tag":565,"props":4677,"children":4678},{"style":571},[4679],{"type":398,"value":620},{"type":393,"tag":565,"props":4681,"children":4682},{"style":631},[4683],{"type":398,"value":2878},{"type":393,"tag":565,"props":4685,"children":4686},{"class":567,"line":308},[4687],{"type":393,"tag":565,"props":4688,"children":4689},{"style":577},[4690],{"type":398,"value":930},{"type":393,"tag":565,"props":4692,"children":4693},{"class":567,"line":318},[4694,4698,4702,4707],{"type":393,"tag":565,"props":4695,"children":4696},{"style":642},[4697],{"type":398,"value":915},{"type":393,"tag":565,"props":4699,"children":4700},{"style":577},[4701],{"type":398,"value":920},{"type":393,"tag":565,"props":4703,"children":4704},{"style":631},[4705],{"type":398,"value":4706},"\"Resolved by most recent purchase:\"",{"type":393,"tag":565,"props":4708,"children":4709},{"style":577},[4710],{"type":398,"value":930},{"type":393,"tag":565,"props":4712,"children":4713},{"class":567,"line":327},[4714,4718],{"type":393,"tag":565,"props":4715,"children":4716},{"style":642},[4717],{"type":398,"value":915},{"type":393,"tag":565,"props":4719,"children":4720},{"style":577},[4721],{"type":398,"value":4722},"(recent_customers)\n",{"type":393,"tag":565,"props":4724,"children":4725},{"class":567,"line":336},[4726,4730],{"type":393,"tag":565,"props":4727,"children":4728},{"style":642},[4729],{"type":398,"value":915},{"type":393,"tag":565,"props":4731,"children":4732},{"style":577},[4733],{"type":398,"value":954},{"type":393,"tag":565,"props":4735,"children":4736},{"class":567,"line":346},[4737],{"type":393,"tag":565,"props":4738,"children":4739},{"emptyLinePlaceholder":12},[4740],{"type":398,"value":598},{"type":393,"tag":565,"props":4742,"children":4743},{"class":567,"line":355},[4744,4748,4752,4757],{"type":393,"tag":565,"props":4745,"children":4746},{"style":642},[4747],{"type":398,"value":915},{"type":393,"tag":565,"props":4749,"children":4750},{"style":577},[4751],{"type":398,"value":920},{"type":393,"tag":565,"props":4753,"children":4754},{"style":631},[4755],{"type":398,"value":4756},"\"3️⃣ Resolve by keeping highest total spent:\"",{"type":393,"tag":565,"props":4758,"children":4759},{"style":577},[4760],{"type":398,"value":930},{"type":393,"tag":565,"props":4762,"children":4763},{"class":567,"line":364},[4764,4769,4773,4777,4782],{"type":393,"tag":565,"props":4765,"children":4766},{"style":577},[4767],{"type":398,"value":4768},"high_value_customers ",{"type":393,"tag":565,"props":4770,"children":4771},{"style":571},[4772],{"type":398,"value":620},{"type":393,"tag":565,"props":4774,"children":4775},{"style":577},[4776],{"type":398,"value":4639},{"type":393,"tag":565,"props":4778,"children":4779},{"style":631},[4780],{"type":398,"value":4781},"'total_spent'",{"type":393,"tag":565,"props":4783,"children":4784},{"style":577},[4785],{"type":398,"value":2840},{"type":393,"tag":565,"props":4787,"children":4788},{"class":567,"line":374},[4789,4793,4797,4801,4805,4809,4813,4817],{"type":393,"tag":565,"props":4790,"children":4791},{"style":1259},[4792],{"type":398,"value":2848},{"type":393,"tag":565,"props":4794,"children":4795},{"style":571},[4796],{"type":398,"value":620},{"type":393,"tag":565,"props":4798,"children":4799},{"style":577},[4800],{"type":398,"value":2609},{"type":393,"tag":565,"props":4802,"children":4803},{"style":631},[4804],{"type":398,"value":4532},{"type":393,"tag":565,"props":4806,"children":4807},{"style":577},[4808],{"type":398,"value":2865},{"type":393,"tag":565,"props":4810,"children":4811},{"style":1259},[4812],{"type":398,"value":1262},{"type":393,"tag":565,"props":4814,"children":4815},{"style":571},[4816],{"type":398,"value":620},{"type":393,"tag":565,"props":4818,"children":4819},{"style":631},[4820],{"type":398,"value":2878},{"type":393,"tag":565,"props":4822,"children":4823},{"class":567,"line":383},[4824],{"type":393,"tag":565,"props":4825,"children":4826},{"style":577},[4827],{"type":398,"value":930},{"type":393,"tag":565,"props":4829,"children":4830},{"class":567,"line":3773},[4831,4835,4839,4844],{"type":393,"tag":565,"props":4832,"children":4833},{"style":642},[4834],{"type":398,"value":915},{"type":393,"tag":565,"props":4836,"children":4837},{"style":577},[4838],{"type":398,"value":920},{"type":393,"tag":565,"props":4840,"children":4841},{"style":631},[4842],{"type":398,"value":4843},"\"Resolved by highest spending:\"",{"type":393,"tag":565,"props":4845,"children":4846},{"style":577},[4847],{"type":398,"value":930},{"type":393,"tag":565,"props":4849,"children":4850},{"class":567,"line":3781},[4851,4855],{"type":393,"tag":565,"props":4852,"children":4853},{"style":642},[4854],{"type":398,"value":915},{"type":393,"tag":565,"props":4856,"children":4857},{"style":577},[4858],{"type":398,"value":4859},"(high_value_customers)\n",{"type":393,"tag":565,"props":4861,"children":4862},{"class":567,"line":3802},[4863,4867],{"type":393,"tag":565,"props":4864,"children":4865},{"style":642},[4866],{"type":398,"value":915},{"type":393,"tag":565,"props":4868,"children":4869},{"style":577},[4870],{"type":398,"value":954},{"type":393,"tag":565,"props":4872,"children":4873},{"class":567,"line":3811},[4874],{"type":393,"tag":565,"props":4875,"children":4876},{"emptyLinePlaceholder":12},[4877],{"type":398,"value":598},{"type":393,"tag":565,"props":4879,"children":4880},{"class":567,"line":3829},[4881,4885,4889,4894],{"type":393,"tag":565,"props":4882,"children":4883},{"style":642},[4884],{"type":398,"value":915},{"type":393,"tag":565,"props":4886,"children":4887},{"style":577},[4888],{"type":398,"value":920},{"type":393,"tag":565,"props":4890,"children":4891},{"style":631},[4892],{"type":398,"value":4893},"\"4️⃣ Manual resolution for critical conflicts:\"",{"type":393,"tag":565,"props":4895,"children":4896},{"style":577},[4897],{"type":398,"value":930},{"type":393,"tag":565,"props":4899,"children":4900},{"class":567,"line":3866},[4901],{"type":393,"tag":565,"props":4902,"children":4903},{"style":604},[4904],{"type":398,"value":4905},"# For important data, you might need manual review\n",{"type":393,"tag":565,"props":4907,"children":4908},{"class":567,"line":3887},[4909,4913,4917,4922],{"type":393,"tag":565,"props":4910,"children":4911},{"style":642},[4912],{"type":398,"value":915},{"type":393,"tag":565,"props":4914,"children":4915},{"style":577},[4916],{"type":398,"value":920},{"type":393,"tag":565,"props":4918,"children":4919},{"style":631},[4920],{"type":398,"value":4921},"\"Manual review needed for:\"",{"type":393,"tag":565,"props":4923,"children":4924},{"style":577},[4925],{"type":398,"value":930},{"type":393,"tag":565,"props":4927,"children":4928},{"class":567,"line":3900},[4929,4934,4938,4943,4947,4952,4956,4960,4964],{"type":393,"tag":565,"props":4930,"children":4931},{"style":577},[4932],{"type":398,"value":4933},"manual_review ",{"type":393,"tag":565,"props":4935,"children":4936},{"style":571},[4937],{"type":398,"value":620},{"type":393,"tag":565,"props":4939,"children":4940},{"style":577},[4941],{"type":398,"value":4942}," customers[customers[",{"type":393,"tag":565,"props":4944,"children":4945},{"style":631},[4946],{"type":398,"value":4532},{"type":393,"tag":565,"props":4948,"children":4949},{"style":577},[4950],{"type":398,"value":4951},"].duplicated(",{"type":393,"tag":565,"props":4953,"children":4954},{"style":1259},[4955],{"type":398,"value":1262},{"type":393,"tag":565,"props":4957,"children":4958},{"style":571},[4959],{"type":398,"value":620},{"type":393,"tag":565,"props":4961,"children":4962},{"style":642},[4963],{"type":398,"value":1271},{"type":393,"tag":565,"props":4965,"children":4966},{"style":577},[4967],{"type":398,"value":1276},{"type":393,"tag":565,"props":4969,"children":4971},{"class":567,"line":4970},49,[4972,4976,4981,4985,4989,4993,4997,5002,5006,5010],{"type":393,"tag":565,"props":4973,"children":4974},{"style":642},[4975],{"type":398,"value":915},{"type":393,"tag":565,"props":4977,"children":4978},{"style":577},[4979],{"type":398,"value":4980},"(manual_review[[",{"type":393,"tag":565,"props":4982,"children":4983},{"style":631},[4984],{"type":398,"value":4532},{"type":393,"tag":565,"props":4986,"children":4987},{"style":577},[4988],{"type":398,"value":650},{"type":393,"tag":565,"props":4990,"children":4991},{"style":631},[4992],{"type":398,"value":2722},{"type":393,"tag":565,"props":4994,"children":4995},{"style":577},[4996],{"type":398,"value":650},{"type":393,"tag":565,"props":4998,"children":4999},{"style":631},[5000],{"type":398,"value":5001},"'email'",{"type":393,"tag":565,"props":5003,"children":5004},{"style":577},[5005],{"type":398,"value":650},{"type":393,"tag":565,"props":5007,"children":5008},{"style":631},[5009],{"type":398,"value":4781},{"type":393,"tag":565,"props":5011,"children":5012},{"style":577},[5013],{"type":398,"value":5014},"]])\n",{"type":393,"tag":538,"props":5016,"children":5018},{"id":5017},"duplicate-prevention-strategies",[5019],{"type":398,"value":5020},"📋 Duplicate Prevention Strategies",{"type":393,"tag":401,"props":5022,"children":5023},{},[5024],{"type":398,"value":5025},"Best practices to prevent duplicates from entering your data:",{"type":393,"tag":550,"props":5027,"children":5028},{},[5029],{"type":393,"tag":554,"props":5030,"children":5032},{"className":556,"code":5031,"language":558,"meta":6,"style":6},"import pandas as pd\n\n# Simulating data validation during import\nnew_records = pd.DataFrame({\n    'user_id': [201, 202, 203, 201, 204],  # 201 is duplicate\n    'username': ['user201', 'user202', 'user203', 'user201_new', 'user204'],\n    'email': ['user201@email.com', 'user202@email.com', 'user203@email.com',\n              'user201@email.com', 'user204@email.com'],\n    'registration_date': ['2023-03-01', '2023-03-02', '2023-03-03', \n                          '2023-03-04', '2023-03-05']\n})\n\nexisting_users = pd.DataFrame({\n    'user_id': [101, 102, 103],\n    'username': ['user101', 'user102', 'user103'],\n    'email': ['user101@email.com', 'user102@email.com', 'user103@email.com'],\n    'registration_date': ['2023-01-01', '2023-01-02', '2023-01-03']\n})\n\nprint(\"📋 Duplicate Prevention Example:\")\nprint()\nprint(\"New records to import:\")\nprint(new_records)\nprint()\nprint(\"Existing users:\")\nprint(existing_users)\nprint()\n\nprint(\"🛡️ Validation Steps:\")\nprint()\n\nprint(\"1️⃣ Check for duplicates within new records:\")\ninternal_dups = new_records.duplicated(subset=['user_id'])\nif internal_dups.any():\n    print(\"Internal duplicates found:\")\n    print(new_records[internal_dups])\n    print(\"Action: Remove internal duplicates before import\")\n    clean_new = new_records.drop_duplicates(subset=['user_id'])\nelse:\n    clean_new = new_records\n    print(\"No internal duplicates\")\nprint()\n\nprint(\"2️⃣ Check for conflicts with existing data:\")\n# Simulate checking if user IDs already exist\nexisting_ids = existing_users['user_id'].tolist()\nconflicts = clean_new[clean_new['user_id'].isin(existing_ids)]\nif not conflicts.empty:\n    print(\"Conflicts with existing data:\")\n    print(conflicts)\n    print(\"Action: Skip conflicting records or assign new IDs\")\nelse:\n    print(\"No conflicts with existing data\")\nprint()\n\nprint(\"3️⃣ Safe records to import:\")\nsafe_records = clean_new[~clean_new['user_id'].isin(existing_ids)]\nprint(safe_records)\nprint(f\"Ready to import: {len(safe_records)} records\")\n",[5033],{"type":393,"tag":561,"props":5034,"children":5035},{"__ignoreMap":6},[5036,5055,5062,5070,5086,5147,5204,5242,5263,5302,5323,5330,5337,5353,5388,5426,5464,5502,5509,5516,5536,5547,5567,5579,5590,5610,5622,5633,5640,5660,5671,5678,5698,5736,5748,5768,5780,5800,5837,5848,5864,5884,5895,5902,5922,5930,5956,5982,5999,6019,6032,6053,6065,6086,6098,6106,6127,6163,6176],{"type":393,"tag":565,"props":5037,"children":5038},{"class":567,"line":11},[5039,5043,5047,5051],{"type":393,"tag":565,"props":5040,"children":5041},{"style":571},[5042],{"type":398,"value":574},{"type":393,"tag":565,"props":5044,"children":5045},{"style":577},[5046],{"type":398,"value":580},{"type":393,"tag":565,"props":5048,"children":5049},{"style":571},[5050],{"type":398,"value":585},{"type":393,"tag":565,"props":5052,"children":5053},{"style":577},[5054],{"type":398,"value":590},{"type":393,"tag":565,"props":5056,"children":5057},{"class":567,"line":27},[5058],{"type":393,"tag":565,"props":5059,"children":5060},{"emptyLinePlaceholder":12},[5061],{"type":398,"value":598},{"type":393,"tag":565,"props":5063,"children":5064},{"class":567,"line":36},[5065],{"type":393,"tag":565,"props":5066,"children":5067},{"style":604},[5068],{"type":398,"value":5069},"# Simulating data validation during import\n",{"type":393,"tag":565,"props":5071,"children":5072},{"class":567,"line":45},[5073,5078,5082],{"type":393,"tag":565,"props":5074,"children":5075},{"style":577},[5076],{"type":398,"value":5077},"new_records ",{"type":393,"tag":565,"props":5079,"children":5080},{"style":571},[5081],{"type":398,"value":620},{"type":393,"tag":565,"props":5083,"children":5084},{"style":577},[5085],{"type":398,"value":625},{"type":393,"tag":565,"props":5087,"children":5088},{"class":567,"line":54},[5089,5094,5098,5103,5107,5112,5116,5121,5125,5129,5133,5138,5142],{"type":393,"tag":565,"props":5090,"children":5091},{"style":631},[5092],{"type":398,"value":5093},"    'user_id'",{"type":393,"tag":565,"props":5095,"children":5096},{"style":577},[5097],{"type":398,"value":639},{"type":393,"tag":565,"props":5099,"children":5100},{"style":642},[5101],{"type":398,"value":5102},"201",{"type":393,"tag":565,"props":5104,"children":5105},{"style":577},[5106],{"type":398,"value":650},{"type":393,"tag":565,"props":5108,"children":5109},{"style":642},[5110],{"type":398,"value":5111},"202",{"type":393,"tag":565,"props":5113,"children":5114},{"style":577},[5115],{"type":398,"value":650},{"type":393,"tag":565,"props":5117,"children":5118},{"style":642},[5119],{"type":398,"value":5120},"203",{"type":393,"tag":565,"props":5122,"children":5123},{"style":577},[5124],{"type":398,"value":650},{"type":393,"tag":565,"props":5126,"children":5127},{"style":642},[5128],{"type":398,"value":5102},{"type":393,"tag":565,"props":5130,"children":5131},{"style":577},[5132],{"type":398,"value":650},{"type":393,"tag":565,"props":5134,"children":5135},{"style":642},[5136],{"type":398,"value":5137},"204",{"type":393,"tag":565,"props":5139,"children":5140},{"style":577},[5141],{"type":398,"value":2395},{"type":393,"tag":565,"props":5143,"children":5144},{"style":604},[5145],{"type":398,"value":5146},"# 201 is duplicate\n",{"type":393,"tag":565,"props":5148,"children":5149},{"class":567,"line":63},[5150,5155,5159,5164,5168,5173,5177,5182,5186,5191,5195,5200],{"type":393,"tag":565,"props":5151,"children":5152},{"style":631},[5153],{"type":398,"value":5154},"    'username'",{"type":393,"tag":565,"props":5156,"children":5157},{"style":577},[5158],{"type":398,"value":639},{"type":393,"tag":565,"props":5160,"children":5161},{"style":631},[5162],{"type":398,"value":5163},"'user201'",{"type":393,"tag":565,"props":5165,"children":5166},{"style":577},[5167],{"type":398,"value":650},{"type":393,"tag":565,"props":5169,"children":5170},{"style":631},[5171],{"type":398,"value":5172},"'user202'",{"type":393,"tag":565,"props":5174,"children":5175},{"style":577},[5176],{"type":398,"value":650},{"type":393,"tag":565,"props":5178,"children":5179},{"style":631},[5180],{"type":398,"value":5181},"'user203'",{"type":393,"tag":565,"props":5183,"children":5184},{"style":577},[5185],{"type":398,"value":650},{"type":393,"tag":565,"props":5187,"children":5188},{"style":631},[5189],{"type":398,"value":5190},"'user201_new'",{"type":393,"tag":565,"props":5192,"children":5193},{"style":577},[5194],{"type":398,"value":650},{"type":393,"tag":565,"props":5196,"children":5197},{"style":631},[5198],{"type":398,"value":5199},"'user204'",{"type":393,"tag":565,"props":5201,"children":5202},{"style":577},[5203],{"type":398,"value":694},{"type":393,"tag":565,"props":5205,"children":5206},{"class":567,"line":72},[5207,5211,5215,5220,5224,5229,5233,5238],{"type":393,"tag":565,"props":5208,"children":5209},{"style":631},[5210],{"type":398,"value":766},{"type":393,"tag":565,"props":5212,"children":5213},{"style":577},[5214],{"type":398,"value":639},{"type":393,"tag":565,"props":5216,"children":5217},{"style":631},[5218],{"type":398,"value":5219},"'user201@email.com'",{"type":393,"tag":565,"props":5221,"children":5222},{"style":577},[5223],{"type":398,"value":650},{"type":393,"tag":565,"props":5225,"children":5226},{"style":631},[5227],{"type":398,"value":5228},"'user202@email.com'",{"type":393,"tag":565,"props":5230,"children":5231},{"style":577},[5232],{"type":398,"value":650},{"type":393,"tag":565,"props":5234,"children":5235},{"style":631},[5236],{"type":398,"value":5237},"'user203@email.com'",{"type":393,"tag":565,"props":5239,"children":5240},{"style":577},[5241],{"type":398,"value":3278},{"type":393,"tag":565,"props":5243,"children":5244},{"class":567,"line":83},[5245,5250,5254,5259],{"type":393,"tag":565,"props":5246,"children":5247},{"style":631},[5248],{"type":398,"value":5249},"              'user201@email.com'",{"type":393,"tag":565,"props":5251,"children":5252},{"style":577},[5253],{"type":398,"value":650},{"type":393,"tag":565,"props":5255,"children":5256},{"style":631},[5257],{"type":398,"value":5258},"'user204@email.com'",{"type":393,"tag":565,"props":5260,"children":5261},{"style":577},[5262],{"type":398,"value":694},{"type":393,"tag":565,"props":5264,"children":5265},{"class":567,"line":92},[5266,5271,5275,5280,5284,5289,5293,5298],{"type":393,"tag":565,"props":5267,"children":5268},{"style":631},[5269],{"type":398,"value":5270},"    'registration_date'",{"type":393,"tag":565,"props":5272,"children":5273},{"style":577},[5274],{"type":398,"value":639},{"type":393,"tag":565,"props":5276,"children":5277},{"style":631},[5278],{"type":398,"value":5279},"'2023-03-01'",{"type":393,"tag":565,"props":5281,"children":5282},{"style":577},[5283],{"type":398,"value":650},{"type":393,"tag":565,"props":5285,"children":5286},{"style":631},[5287],{"type":398,"value":5288},"'2023-03-02'",{"type":393,"tag":565,"props":5290,"children":5291},{"style":577},[5292],{"type":398,"value":650},{"type":393,"tag":565,"props":5294,"children":5295},{"style":631},[5296],{"type":398,"value":5297},"'2023-03-03'",{"type":393,"tag":565,"props":5299,"children":5300},{"style":577},[5301],{"type":398,"value":798},{"type":393,"tag":565,"props":5303,"children":5304},{"class":567,"line":101},[5305,5310,5314,5319],{"type":393,"tag":565,"props":5306,"children":5307},{"style":631},[5308],{"type":398,"value":5309},"                          '2023-03-04'",{"type":393,"tag":565,"props":5311,"children":5312},{"style":577},[5313],{"type":398,"value":650},{"type":393,"tag":565,"props":5315,"children":5316},{"style":631},[5317],{"type":398,"value":5318},"'2023-03-05'",{"type":393,"tag":565,"props":5320,"children":5321},{"style":577},[5322],{"type":398,"value":892},{"type":393,"tag":565,"props":5324,"children":5325},{"class":567,"line":110},[5326],{"type":393,"tag":565,"props":5327,"children":5328},{"style":577},[5329],{"type":398,"value":900},{"type":393,"tag":565,"props":5331,"children":5332},{"class":567,"line":121},[5333],{"type":393,"tag":565,"props":5334,"children":5335},{"emptyLinePlaceholder":12},[5336],{"type":398,"value":598},{"type":393,"tag":565,"props":5338,"children":5339},{"class":567,"line":130},[5340,5345,5349],{"type":393,"tag":565,"props":5341,"children":5342},{"style":577},[5343],{"type":398,"value":5344},"existing_users ",{"type":393,"tag":565,"props":5346,"children":5347},{"style":571},[5348],{"type":398,"value":620},{"type":393,"tag":565,"props":5350,"children":5351},{"style":577},[5352],{"type":398,"value":625},{"type":393,"tag":565,"props":5354,"children":5355},{"class":567,"line":139},[5356,5360,5364,5368,5372,5376,5380,5384],{"type":393,"tag":565,"props":5357,"children":5358},{"style":631},[5359],{"type":398,"value":5093},{"type":393,"tag":565,"props":5361,"children":5362},{"style":577},[5363],{"type":398,"value":639},{"type":393,"tag":565,"props":5365,"children":5366},{"style":642},[5367],{"type":398,"value":2192},{"type":393,"tag":565,"props":5369,"children":5370},{"style":577},[5371],{"type":398,"value":650},{"type":393,"tag":565,"props":5373,"children":5374},{"style":642},[5375],{"type":398,"value":2201},{"type":393,"tag":565,"props":5377,"children":5378},{"style":577},[5379],{"type":398,"value":650},{"type":393,"tag":565,"props":5381,"children":5382},{"style":642},[5383],{"type":398,"value":2210},{"type":393,"tag":565,"props":5385,"children":5386},{"style":577},[5387],{"type":398,"value":694},{"type":393,"tag":565,"props":5389,"children":5390},{"class":567,"line":148},[5391,5395,5399,5404,5408,5413,5417,5422],{"type":393,"tag":565,"props":5392,"children":5393},{"style":631},[5394],{"type":398,"value":5154},{"type":393,"tag":565,"props":5396,"children":5397},{"style":577},[5398],{"type":398,"value":639},{"type":393,"tag":565,"props":5400,"children":5401},{"style":631},[5402],{"type":398,"value":5403},"'user101'",{"type":393,"tag":565,"props":5405,"children":5406},{"style":577},[5407],{"type":398,"value":650},{"type":393,"tag":565,"props":5409,"children":5410},{"style":631},[5411],{"type":398,"value":5412},"'user102'",{"type":393,"tag":565,"props":5414,"children":5415},{"style":577},[5416],{"type":398,"value":650},{"type":393,"tag":565,"props":5418,"children":5419},{"style":631},[5420],{"type":398,"value":5421},"'user103'",{"type":393,"tag":565,"props":5423,"children":5424},{"style":577},[5425],{"type":398,"value":694},{"type":393,"tag":565,"props":5427,"children":5428},{"class":567,"line":159},[5429,5433,5437,5442,5446,5451,5455,5460],{"type":393,"tag":565,"props":5430,"children":5431},{"style":631},[5432],{"type":398,"value":766},{"type":393,"tag":565,"props":5434,"children":5435},{"style":577},[5436],{"type":398,"value":639},{"type":393,"tag":565,"props":5438,"children":5439},{"style":631},[5440],{"type":398,"value":5441},"'user101@email.com'",{"type":393,"tag":565,"props":5443,"children":5444},{"style":577},[5445],{"type":398,"value":650},{"type":393,"tag":565,"props":5447,"children":5448},{"style":631},[5449],{"type":398,"value":5450},"'user102@email.com'",{"type":393,"tag":565,"props":5452,"children":5453},{"style":577},[5454],{"type":398,"value":650},{"type":393,"tag":565,"props":5456,"children":5457},{"style":631},[5458],{"type":398,"value":5459},"'user103@email.com'",{"type":393,"tag":565,"props":5461,"children":5462},{"style":577},[5463],{"type":398,"value":694},{"type":393,"tag":565,"props":5465,"children":5466},{"class":567,"line":168},[5467,5471,5475,5480,5484,5489,5493,5498],{"type":393,"tag":565,"props":5468,"children":5469},{"style":631},[5470],{"type":398,"value":5270},{"type":393,"tag":565,"props":5472,"children":5473},{"style":577},[5474],{"type":398,"value":639},{"type":393,"tag":565,"props":5476,"children":5477},{"style":631},[5478],{"type":398,"value":5479},"'2023-01-01'",{"type":393,"tag":565,"props":5481,"children":5482},{"style":577},[5483],{"type":398,"value":650},{"type":393,"tag":565,"props":5485,"children":5486},{"style":631},[5487],{"type":398,"value":5488},"'2023-01-02'",{"type":393,"tag":565,"props":5490,"children":5491},{"style":577},[5492],{"type":398,"value":650},{"type":393,"tag":565,"props":5494,"children":5495},{"style":631},[5496],{"type":398,"value":5497},"'2023-01-03'",{"type":393,"tag":565,"props":5499,"children":5500},{"style":577},[5501],{"type":398,"value":892},{"type":393,"tag":565,"props":5503,"children":5504},{"class":567,"line":177},[5505],{"type":393,"tag":565,"props":5506,"children":5507},{"style":577},[5508],{"type":398,"value":900},{"type":393,"tag":565,"props":5510,"children":5511},{"class":567,"line":186},[5512],{"type":393,"tag":565,"props":5513,"children":5514},{"emptyLinePlaceholder":12},[5515],{"type":398,"value":598},{"type":393,"tag":565,"props":5517,"children":5518},{"class":567,"line":197},[5519,5523,5527,5532],{"type":393,"tag":565,"props":5520,"children":5521},{"style":642},[5522],{"type":398,"value":915},{"type":393,"tag":565,"props":5524,"children":5525},{"style":577},[5526],{"type":398,"value":920},{"type":393,"tag":565,"props":5528,"children":5529},{"style":631},[5530],{"type":398,"value":5531},"\"📋 Duplicate Prevention Example:\"",{"type":393,"tag":565,"props":5533,"children":5534},{"style":577},[5535],{"type":398,"value":930},{"type":393,"tag":565,"props":5537,"children":5538},{"class":567,"line":206},[5539,5543],{"type":393,"tag":565,"props":5540,"children":5541},{"style":642},[5542],{"type":398,"value":915},{"type":393,"tag":565,"props":5544,"children":5545},{"style":577},[5546],{"type":398,"value":954},{"type":393,"tag":565,"props":5548,"children":5549},{"class":567,"line":215},[5550,5554,5558,5563],{"type":393,"tag":565,"props":5551,"children":5552},{"style":642},[5553],{"type":398,"value":915},{"type":393,"tag":565,"props":5555,"children":5556},{"style":577},[5557],{"type":398,"value":920},{"type":393,"tag":565,"props":5559,"children":5560},{"style":631},[5561],{"type":398,"value":5562},"\"New records to import:\"",{"type":393,"tag":565,"props":5564,"children":5565},{"style":577},[5566],{"type":398,"value":930},{"type":393,"tag":565,"props":5568,"children":5569},{"class":567,"line":224},[5570,5574],{"type":393,"tag":565,"props":5571,"children":5572},{"style":642},[5573],{"type":398,"value":915},{"type":393,"tag":565,"props":5575,"children":5576},{"style":577},[5577],{"type":398,"value":5578},"(new_records)\n",{"type":393,"tag":565,"props":5580,"children":5581},{"class":567,"line":233},[5582,5586],{"type":393,"tag":565,"props":5583,"children":5584},{"style":642},[5585],{"type":398,"value":915},{"type":393,"tag":565,"props":5587,"children":5588},{"style":577},[5589],{"type":398,"value":954},{"type":393,"tag":565,"props":5591,"children":5592},{"class":567,"line":244},[5593,5597,5601,5606],{"type":393,"tag":565,"props":5594,"children":5595},{"style":642},[5596],{"type":398,"value":915},{"type":393,"tag":565,"props":5598,"children":5599},{"style":577},[5600],{"type":398,"value":920},{"type":393,"tag":565,"props":5602,"children":5603},{"style":631},[5604],{"type":398,"value":5605},"\"Existing users:\"",{"type":393,"tag":565,"props":5607,"children":5608},{"style":577},[5609],{"type":398,"value":930},{"type":393,"tag":565,"props":5611,"children":5612},{"class":567,"line":253},[5613,5617],{"type":393,"tag":565,"props":5614,"children":5615},{"style":642},[5616],{"type":398,"value":915},{"type":393,"tag":565,"props":5618,"children":5619},{"style":577},[5620],{"type":398,"value":5621},"(existing_users)\n",{"type":393,"tag":565,"props":5623,"children":5624},{"class":567,"line":262},[5625,5629],{"type":393,"tag":565,"props":5626,"children":5627},{"style":642},[5628],{"type":398,"value":915},{"type":393,"tag":565,"props":5630,"children":5631},{"style":577},[5632],{"type":398,"value":954},{"type":393,"tag":565,"props":5634,"children":5635},{"class":567,"line":271},[5636],{"type":393,"tag":565,"props":5637,"children":5638},{"emptyLinePlaceholder":12},[5639],{"type":398,"value":598},{"type":393,"tag":565,"props":5641,"children":5642},{"class":567,"line":281},[5643,5647,5651,5656],{"type":393,"tag":565,"props":5644,"children":5645},{"style":642},[5646],{"type":398,"value":915},{"type":393,"tag":565,"props":5648,"children":5649},{"style":577},[5650],{"type":398,"value":920},{"type":393,"tag":565,"props":5652,"children":5653},{"style":631},[5654],{"type":398,"value":5655},"\"🛡️ Validation Steps:\"",{"type":393,"tag":565,"props":5657,"children":5658},{"style":577},[5659],{"type":398,"value":930},{"type":393,"tag":565,"props":5661,"children":5662},{"class":567,"line":290},[5663,5667],{"type":393,"tag":565,"props":5664,"children":5665},{"style":642},[5666],{"type":398,"value":915},{"type":393,"tag":565,"props":5668,"children":5669},{"style":577},[5670],{"type":398,"value":954},{"type":393,"tag":565,"props":5672,"children":5673},{"class":567,"line":299},[5674],{"type":393,"tag":565,"props":5675,"children":5676},{"emptyLinePlaceholder":12},[5677],{"type":398,"value":598},{"type":393,"tag":565,"props":5679,"children":5680},{"class":567,"line":308},[5681,5685,5689,5694],{"type":393,"tag":565,"props":5682,"children":5683},{"style":642},[5684],{"type":398,"value":915},{"type":393,"tag":565,"props":5686,"children":5687},{"style":577},[5688],{"type":398,"value":920},{"type":393,"tag":565,"props":5690,"children":5691},{"style":631},[5692],{"type":398,"value":5693},"\"1️⃣ Check for duplicates within new records:\"",{"type":393,"tag":565,"props":5695,"children":5696},{"style":577},[5697],{"type":398,"value":930},{"type":393,"tag":565,"props":5699,"children":5700},{"class":567,"line":318},[5701,5706,5710,5715,5719,5723,5727,5732],{"type":393,"tag":565,"props":5702,"children":5703},{"style":577},[5704],{"type":398,"value":5705},"internal_dups ",{"type":393,"tag":565,"props":5707,"children":5708},{"style":571},[5709],{"type":398,"value":620},{"type":393,"tag":565,"props":5711,"children":5712},{"style":577},[5713],{"type":398,"value":5714}," new_records.duplicated(",{"type":393,"tag":565,"props":5716,"children":5717},{"style":1259},[5718],{"type":398,"value":2600},{"type":393,"tag":565,"props":5720,"children":5721},{"style":571},[5722],{"type":398,"value":620},{"type":393,"tag":565,"props":5724,"children":5725},{"style":577},[5726],{"type":398,"value":2609},{"type":393,"tag":565,"props":5728,"children":5729},{"style":631},[5730],{"type":398,"value":5731},"'user_id'",{"type":393,"tag":565,"props":5733,"children":5734},{"style":577},[5735],{"type":398,"value":2619},{"type":393,"tag":565,"props":5737,"children":5738},{"class":567,"line":327},[5739,5743],{"type":393,"tag":565,"props":5740,"children":5741},{"style":571},[5742],{"type":398,"value":3455},{"type":393,"tag":565,"props":5744,"children":5745},{"style":577},[5746],{"type":398,"value":5747}," internal_dups.any():\n",{"type":393,"tag":565,"props":5749,"children":5750},{"class":567,"line":336},[5751,5755,5759,5764],{"type":393,"tag":565,"props":5752,"children":5753},{"style":642},[5754],{"type":398,"value":3468},{"type":393,"tag":565,"props":5756,"children":5757},{"style":577},[5758],{"type":398,"value":920},{"type":393,"tag":565,"props":5760,"children":5761},{"style":631},[5762],{"type":398,"value":5763},"\"Internal duplicates found:\"",{"type":393,"tag":565,"props":5765,"children":5766},{"style":577},[5767],{"type":398,"value":930},{"type":393,"tag":565,"props":5769,"children":5770},{"class":567,"line":346},[5771,5775],{"type":393,"tag":565,"props":5772,"children":5773},{"style":642},[5774],{"type":398,"value":3468},{"type":393,"tag":565,"props":5776,"children":5777},{"style":577},[5778],{"type":398,"value":5779},"(new_records[internal_dups])\n",{"type":393,"tag":565,"props":5781,"children":5782},{"class":567,"line":355},[5783,5787,5791,5796],{"type":393,"tag":565,"props":5784,"children":5785},{"style":642},[5786],{"type":398,"value":3468},{"type":393,"tag":565,"props":5788,"children":5789},{"style":577},[5790],{"type":398,"value":920},{"type":393,"tag":565,"props":5792,"children":5793},{"style":631},[5794],{"type":398,"value":5795},"\"Action: Remove internal duplicates before import\"",{"type":393,"tag":565,"props":5797,"children":5798},{"style":577},[5799],{"type":398,"value":930},{"type":393,"tag":565,"props":5801,"children":5802},{"class":567,"line":364},[5803,5808,5812,5817,5821,5825,5829,5833],{"type":393,"tag":565,"props":5804,"children":5805},{"style":577},[5806],{"type":398,"value":5807},"    clean_new ",{"type":393,"tag":565,"props":5809,"children":5810},{"style":571},[5811],{"type":398,"value":620},{"type":393,"tag":565,"props":5813,"children":5814},{"style":577},[5815],{"type":398,"value":5816}," new_records.drop_duplicates(",{"type":393,"tag":565,"props":5818,"children":5819},{"style":1259},[5820],{"type":398,"value":2600},{"type":393,"tag":565,"props":5822,"children":5823},{"style":571},[5824],{"type":398,"value":620},{"type":393,"tag":565,"props":5826,"children":5827},{"style":577},[5828],{"type":398,"value":2609},{"type":393,"tag":565,"props":5830,"children":5831},{"style":631},[5832],{"type":398,"value":5731},{"type":393,"tag":565,"props":5834,"children":5835},{"style":577},[5836],{"type":398,"value":2619},{"type":393,"tag":565,"props":5838,"children":5839},{"class":567,"line":374},[5840,5844],{"type":393,"tag":565,"props":5841,"children":5842},{"style":571},[5843],{"type":398,"value":3501},{"type":393,"tag":565,"props":5845,"children":5846},{"style":577},[5847],{"type":398,"value":3506},{"type":393,"tag":565,"props":5849,"children":5850},{"class":567,"line":383},[5851,5855,5859],{"type":393,"tag":565,"props":5852,"children":5853},{"style":577},[5854],{"type":398,"value":5807},{"type":393,"tag":565,"props":5856,"children":5857},{"style":571},[5858],{"type":398,"value":620},{"type":393,"tag":565,"props":5860,"children":5861},{"style":577},[5862],{"type":398,"value":5863}," new_records\n",{"type":393,"tag":565,"props":5865,"children":5866},{"class":567,"line":3773},[5867,5871,5875,5880],{"type":393,"tag":565,"props":5868,"children":5869},{"style":642},[5870],{"type":398,"value":3468},{"type":393,"tag":565,"props":5872,"children":5873},{"style":577},[5874],{"type":398,"value":920},{"type":393,"tag":565,"props":5876,"children":5877},{"style":631},[5878],{"type":398,"value":5879},"\"No internal duplicates\"",{"type":393,"tag":565,"props":5881,"children":5882},{"style":577},[5883],{"type":398,"value":930},{"type":393,"tag":565,"props":5885,"children":5886},{"class":567,"line":3781},[5887,5891],{"type":393,"tag":565,"props":5888,"children":5889},{"style":642},[5890],{"type":398,"value":915},{"type":393,"tag":565,"props":5892,"children":5893},{"style":577},[5894],{"type":398,"value":954},{"type":393,"tag":565,"props":5896,"children":5897},{"class":567,"line":3802},[5898],{"type":393,"tag":565,"props":5899,"children":5900},{"emptyLinePlaceholder":12},[5901],{"type":398,"value":598},{"type":393,"tag":565,"props":5903,"children":5904},{"class":567,"line":3811},[5905,5909,5913,5918],{"type":393,"tag":565,"props":5906,"children":5907},{"style":642},[5908],{"type":398,"value":915},{"type":393,"tag":565,"props":5910,"children":5911},{"style":577},[5912],{"type":398,"value":920},{"type":393,"tag":565,"props":5914,"children":5915},{"style":631},[5916],{"type":398,"value":5917},"\"2️⃣ Check for conflicts with existing data:\"",{"type":393,"tag":565,"props":5919,"children":5920},{"style":577},[5921],{"type":398,"value":930},{"type":393,"tag":565,"props":5923,"children":5924},{"class":567,"line":3829},[5925],{"type":393,"tag":565,"props":5926,"children":5927},{"style":604},[5928],{"type":398,"value":5929},"# Simulate checking if user IDs already exist\n",{"type":393,"tag":565,"props":5931,"children":5932},{"class":567,"line":3866},[5933,5938,5942,5947,5951],{"type":393,"tag":565,"props":5934,"children":5935},{"style":577},[5936],{"type":398,"value":5937},"existing_ids ",{"type":393,"tag":565,"props":5939,"children":5940},{"style":571},[5941],{"type":398,"value":620},{"type":393,"tag":565,"props":5943,"children":5944},{"style":577},[5945],{"type":398,"value":5946}," existing_users[",{"type":393,"tag":565,"props":5948,"children":5949},{"style":631},[5950],{"type":398,"value":5731},{"type":393,"tag":565,"props":5952,"children":5953},{"style":577},[5954],{"type":398,"value":5955},"].tolist()\n",{"type":393,"tag":565,"props":5957,"children":5958},{"class":567,"line":3887},[5959,5964,5968,5973,5977],{"type":393,"tag":565,"props":5960,"children":5961},{"style":577},[5962],{"type":398,"value":5963},"conflicts ",{"type":393,"tag":565,"props":5965,"children":5966},{"style":571},[5967],{"type":398,"value":620},{"type":393,"tag":565,"props":5969,"children":5970},{"style":577},[5971],{"type":398,"value":5972}," clean_new[clean_new[",{"type":393,"tag":565,"props":5974,"children":5975},{"style":631},[5976],{"type":398,"value":5731},{"type":393,"tag":565,"props":5978,"children":5979},{"style":577},[5980],{"type":398,"value":5981},"].isin(existing_ids)]\n",{"type":393,"tag":565,"props":5983,"children":5984},{"class":567,"line":3900},[5985,5989,5994],{"type":393,"tag":565,"props":5986,"children":5987},{"style":571},[5988],{"type":398,"value":3455},{"type":393,"tag":565,"props":5990,"children":5991},{"style":571},[5992],{"type":398,"value":5993}," not",{"type":393,"tag":565,"props":5995,"children":5996},{"style":577},[5997],{"type":398,"value":5998}," conflicts.empty:\n",{"type":393,"tag":565,"props":6000,"children":6001},{"class":567,"line":4970},[6002,6006,6010,6015],{"type":393,"tag":565,"props":6003,"children":6004},{"style":642},[6005],{"type":398,"value":3468},{"type":393,"tag":565,"props":6007,"children":6008},{"style":577},[6009],{"type":398,"value":920},{"type":393,"tag":565,"props":6011,"children":6012},{"style":631},[6013],{"type":398,"value":6014},"\"Conflicts with existing data:\"",{"type":393,"tag":565,"props":6016,"children":6017},{"style":577},[6018],{"type":398,"value":930},{"type":393,"tag":565,"props":6020,"children":6022},{"class":567,"line":6021},50,[6023,6027],{"type":393,"tag":565,"props":6024,"children":6025},{"style":642},[6026],{"type":398,"value":3468},{"type":393,"tag":565,"props":6028,"children":6029},{"style":577},[6030],{"type":398,"value":6031},"(conflicts)\n",{"type":393,"tag":565,"props":6033,"children":6035},{"class":567,"line":6034},51,[6036,6040,6044,6049],{"type":393,"tag":565,"props":6037,"children":6038},{"style":642},[6039],{"type":398,"value":3468},{"type":393,"tag":565,"props":6041,"children":6042},{"style":577},[6043],{"type":398,"value":920},{"type":393,"tag":565,"props":6045,"children":6046},{"style":631},[6047],{"type":398,"value":6048},"\"Action: Skip conflicting records or assign new IDs\"",{"type":393,"tag":565,"props":6050,"children":6051},{"style":577},[6052],{"type":398,"value":930},{"type":393,"tag":565,"props":6054,"children":6056},{"class":567,"line":6055},52,[6057,6061],{"type":393,"tag":565,"props":6058,"children":6059},{"style":571},[6060],{"type":398,"value":3501},{"type":393,"tag":565,"props":6062,"children":6063},{"style":577},[6064],{"type":398,"value":3506},{"type":393,"tag":565,"props":6066,"children":6068},{"class":567,"line":6067},53,[6069,6073,6077,6082],{"type":393,"tag":565,"props":6070,"children":6071},{"style":642},[6072],{"type":398,"value":3468},{"type":393,"tag":565,"props":6074,"children":6075},{"style":577},[6076],{"type":398,"value":920},{"type":393,"tag":565,"props":6078,"children":6079},{"style":631},[6080],{"type":398,"value":6081},"\"No conflicts with existing data\"",{"type":393,"tag":565,"props":6083,"children":6084},{"style":577},[6085],{"type":398,"value":930},{"type":393,"tag":565,"props":6087,"children":6089},{"class":567,"line":6088},54,[6090,6094],{"type":393,"tag":565,"props":6091,"children":6092},{"style":642},[6093],{"type":398,"value":915},{"type":393,"tag":565,"props":6095,"children":6096},{"style":577},[6097],{"type":398,"value":954},{"type":393,"tag":565,"props":6099,"children":6101},{"class":567,"line":6100},55,[6102],{"type":393,"tag":565,"props":6103,"children":6104},{"emptyLinePlaceholder":12},[6105],{"type":398,"value":598},{"type":393,"tag":565,"props":6107,"children":6109},{"class":567,"line":6108},56,[6110,6114,6118,6123],{"type":393,"tag":565,"props":6111,"children":6112},{"style":642},[6113],{"type":398,"value":915},{"type":393,"tag":565,"props":6115,"children":6116},{"style":577},[6117],{"type":398,"value":920},{"type":393,"tag":565,"props":6119,"children":6120},{"style":631},[6121],{"type":398,"value":6122},"\"3️⃣ Safe records to import:\"",{"type":393,"tag":565,"props":6124,"children":6125},{"style":577},[6126],{"type":398,"value":930},{"type":393,"tag":565,"props":6128,"children":6130},{"class":567,"line":6129},57,[6131,6136,6140,6145,6150,6155,6159],{"type":393,"tag":565,"props":6132,"children":6133},{"style":577},[6134],{"type":398,"value":6135},"safe_records ",{"type":393,"tag":565,"props":6137,"children":6138},{"style":571},[6139],{"type":398,"value":620},{"type":393,"tag":565,"props":6141,"children":6142},{"style":577},[6143],{"type":398,"value":6144}," clean_new[",{"type":393,"tag":565,"props":6146,"children":6147},{"style":571},[6148],{"type":398,"value":6149},"~",{"type":393,"tag":565,"props":6151,"children":6152},{"style":577},[6153],{"type":398,"value":6154},"clean_new[",{"type":393,"tag":565,"props":6156,"children":6157},{"style":631},[6158],{"type":398,"value":5731},{"type":393,"tag":565,"props":6160,"children":6161},{"style":577},[6162],{"type":398,"value":5981},{"type":393,"tag":565,"props":6164,"children":6166},{"class":567,"line":6165},58,[6167,6171],{"type":393,"tag":565,"props":6168,"children":6169},{"style":642},[6170],{"type":398,"value":915},{"type":393,"tag":565,"props":6172,"children":6173},{"style":577},[6174],{"type":398,"value":6175},"(safe_records)\n",{"type":393,"tag":565,"props":6177,"children":6179},{"class":567,"line":6178},59,[6180,6184,6188,6192,6197,6201,6206,6210,6215],{"type":393,"tag":565,"props":6181,"children":6182},{"style":642},[6183],{"type":398,"value":915},{"type":393,"tag":565,"props":6185,"children":6186},{"style":577},[6187],{"type":398,"value":920},{"type":393,"tag":565,"props":6189,"children":6190},{"style":571},[6191],{"type":398,"value":1085},{"type":393,"tag":565,"props":6193,"children":6194},{"style":631},[6195],{"type":398,"value":6196},"\"Ready to import: ",{"type":393,"tag":565,"props":6198,"children":6199},{"style":642},[6200],{"type":398,"value":1701},{"type":393,"tag":565,"props":6202,"children":6203},{"style":577},[6204],{"type":398,"value":6205},"(safe_records)",{"type":393,"tag":565,"props":6207,"children":6208},{"style":642},[6209],{"type":398,"value":1105},{"type":393,"tag":565,"props":6211,"children":6212},{"style":631},[6213],{"type":398,"value":6214}," records\"",{"type":393,"tag":565,"props":6216,"children":6217},{"style":577},[6218],{"type":398,"value":930},{"type":393,"tag":538,"props":6220,"children":6222},{"id":6221},"️-duplicate-removal-pitfalls",[6223],{"type":398,"value":6224},"⚠️ Duplicate Removal Pitfalls",{"type":393,"tag":401,"props":6226,"children":6227},{},[6228],{"type":398,"value":6229},"Understanding when data that looks like duplicates is actually legitimate:",{"type":393,"tag":407,"props":6231,"children":6235},{"emoji":6232,"title":6233,"type":6234},"⚠️","Duplicate Removal Mistakes","warning",[6236,6244,6252,6275,6283,6305,6313,6335,6343,6365],{"type":393,"tag":401,"props":6237,"children":6238},{},[6239],{"type":393,"tag":417,"props":6240,"children":6241},{},[6242],{"type":398,"value":6243},"Critical Mistakes to Avoid:",{"type":393,"tag":401,"props":6245,"children":6246},{},[6247],{"type":393,"tag":417,"props":6248,"children":6249},{},[6250],{"type":398,"value":6251},"1. Blindly Removing \"Duplicates\"",{"type":393,"tag":431,"props":6253,"children":6254},{},[6255,6260,6270],{"type":393,"tag":435,"props":6256,"children":6257},{},[6258],{"type":398,"value":6259},"❌ Wrong: Removing all rows with same customer + date",{"type":393,"tag":435,"props":6261,"children":6262},{},[6263,6268],{"type":393,"tag":417,"props":6264,"children":6265},{},[6266],{"type":398,"value":6267},"Problem:",{"type":398,"value":6269}," Customer might make multiple legitimate purchases",{"type":393,"tag":435,"props":6271,"children":6272},{},[6273],{"type":398,"value":6274},"✅ Better: Only remove exact duplicates across ALL relevant fields",{"type":393,"tag":401,"props":6276,"children":6277},{},[6278],{"type":393,"tag":417,"props":6279,"children":6280},{},[6281],{"type":398,"value":6282},"2. Wrong Subset Selection",{"type":393,"tag":431,"props":6284,"children":6285},{},[6286,6291,6300],{"type":393,"tag":435,"props":6287,"children":6288},{},[6289],{"type":398,"value":6290},"❌ Wrong: Using only partial columns for duplicate detection",{"type":393,"tag":435,"props":6292,"children":6293},{},[6294,6298],{"type":393,"tag":417,"props":6295,"children":6296},{},[6297],{"type":398,"value":6267},{"type":398,"value":6299}," Might remove legitimate variations",{"type":393,"tag":435,"props":6301,"children":6302},{},[6303],{"type":398,"value":6304},"✅ Better: Include all distinguishing fields in subset",{"type":393,"tag":401,"props":6306,"children":6307},{},[6308],{"type":393,"tag":417,"props":6309,"children":6310},{},[6311],{"type":398,"value":6312},"3. Not Understanding Business Context",{"type":393,"tag":431,"props":6314,"children":6315},{},[6316,6321,6330],{"type":393,"tag":435,"props":6317,"children":6318},{},[6319],{"type":398,"value":6320},"❌ Wrong: Applying same logic to all datasets",{"type":393,"tag":435,"props":6322,"children":6323},{},[6324,6328],{"type":393,"tag":417,"props":6325,"children":6326},{},[6327],{"type":398,"value":6267},{"type":398,"value":6329}," What's duplicate in one context isn't in another",{"type":393,"tag":435,"props":6331,"children":6332},{},[6333],{"type":398,"value":6334},"✅ Better: Define duplicates based on business rules",{"type":393,"tag":401,"props":6336,"children":6337},{},[6338],{"type":393,"tag":417,"props":6339,"children":6340},{},[6341],{"type":398,"value":6342},"4. Ignoring Data Entry Patterns",{"type":393,"tag":431,"props":6344,"children":6345},{},[6346,6351,6360],{"type":393,"tag":435,"props":6347,"children":6348},{},[6349],{"type":398,"value":6350},"❌ Wrong: Not considering how duplicates actually occur",{"type":393,"tag":435,"props":6352,"children":6353},{},[6354,6358],{"type":393,"tag":417,"props":6355,"children":6356},{},[6357],{"type":398,"value":6267},{"type":398,"value":6359}," Miss real duplicates or remove legitimate data",{"type":393,"tag":435,"props":6361,"children":6362},{},[6363],{"type":398,"value":6364},"✅ Better: Understand your data source and entry patterns",{"type":393,"tag":401,"props":6366,"children":6367},{},[6368,6373],{"type":393,"tag":417,"props":6369,"children":6370},{},[6371],{"type":398,"value":6372},"Key Principle:",{"type":398,"value":6374}," Define what constitutes a \"duplicate\" in YOUR specific business context!",{"type":393,"tag":550,"props":6376,"children":6377},{},[6378],{"type":393,"tag":554,"props":6379,"children":6381},{"className":556,"code":6380,"language":558,"meta":6,"style":6},"import pandas as pd\n\n# Sales data demonstrating legitimate vs duplicate scenarios\nsales = pd.DataFrame({\n    'date': ['2023-01-15', '2023-01-15', '2023-01-16', '2023-01-16', '2023-01-17'],\n    'customer': ['Alice', 'Alice', 'Bob', 'Bob', 'Charlie'],\n    'product': ['Laptop', 'Mouse', 'Keyboard', 'Mouse', 'Monitor'],\n    'amount': [999, 25, 75, 25, 300]\n})\n\nprint(\"📊 Sales Data Analysis:\")\nprint(sales)\nprint()\n\nprint(\"✅ Correct Approach: Context-Aware Duplicate Handling\")\nprint()\n\n# Correct: Only remove exact duplicates\ncorrect_approach = sales.drop_duplicates()\nprint(\"Exact duplicates removed (none found):\")\nprint(correct_approach)\nprint(f\"All {len(correct_approach)} transactions preserved\")\nprint()\n\nprint(\"🎯 Business Logic Applied:\")\nprint(\"- Same customer + date + different product = Legitimate\")\nprint(\"- Same customer + date + same product = Check carefully\")\nprint(\"- Exact match on all fields = Likely duplicate\")\n",[6382],{"type":393,"tag":561,"props":6383,"children":6384},{"__ignoreMap":6},[6385,6404,6411,6419,6435,6489,6540,6591,6642,6649,6656,6676,6688,6699,6706,6726,6737,6744,6752,6769,6789,6801,6843,6854,6861,6881,6901,6921],{"type":393,"tag":565,"props":6386,"children":6387},{"class":567,"line":11},[6388,6392,6396,6400],{"type":393,"tag":565,"props":6389,"children":6390},{"style":571},[6391],{"type":398,"value":574},{"type":393,"tag":565,"props":6393,"children":6394},{"style":577},[6395],{"type":398,"value":580},{"type":393,"tag":565,"props":6397,"children":6398},{"style":571},[6399],{"type":398,"value":585},{"type":393,"tag":565,"props":6401,"children":6402},{"style":577},[6403],{"type":398,"value":590},{"type":393,"tag":565,"props":6405,"children":6406},{"class":567,"line":27},[6407],{"type":393,"tag":565,"props":6408,"children":6409},{"emptyLinePlaceholder":12},[6410],{"type":398,"value":598},{"type":393,"tag":565,"props":6412,"children":6413},{"class":567,"line":36},[6414],{"type":393,"tag":565,"props":6415,"children":6416},{"style":604},[6417],{"type":398,"value":6418},"# Sales data demonstrating legitimate vs duplicate scenarios\n",{"type":393,"tag":565,"props":6420,"children":6421},{"class":567,"line":45},[6422,6427,6431],{"type":393,"tag":565,"props":6423,"children":6424},{"style":577},[6425],{"type":398,"value":6426},"sales ",{"type":393,"tag":565,"props":6428,"children":6429},{"style":571},[6430],{"type":398,"value":620},{"type":393,"tag":565,"props":6432,"children":6433},{"style":577},[6434],{"type":398,"value":625},{"type":393,"tag":565,"props":6436,"children":6437},{"class":567,"line":54},[6438,6443,6447,6451,6455,6459,6463,6468,6472,6476,6480,6485],{"type":393,"tag":565,"props":6439,"children":6440},{"style":631},[6441],{"type":398,"value":6442},"    'date'",{"type":393,"tag":565,"props":6444,"children":6445},{"style":577},[6446],{"type":398,"value":639},{"type":393,"tag":565,"props":6448,"children":6449},{"style":631},[6450],{"type":398,"value":4270},{"type":393,"tag":565,"props":6452,"children":6453},{"style":577},[6454],{"type":398,"value":650},{"type":393,"tag":565,"props":6456,"children":6457},{"style":631},[6458],{"type":398,"value":4270},{"type":393,"tag":565,"props":6460,"children":6461},{"style":577},[6462],{"type":398,"value":650},{"type":393,"tag":565,"props":6464,"children":6465},{"style":631},[6466],{"type":398,"value":6467},"'2023-01-16'",{"type":393,"tag":565,"props":6469,"children":6470},{"style":577},[6471],{"type":398,"value":650},{"type":393,"tag":565,"props":6473,"children":6474},{"style":631},[6475],{"type":398,"value":6467},{"type":393,"tag":565,"props":6477,"children":6478},{"style":577},[6479],{"type":398,"value":650},{"type":393,"tag":565,"props":6481,"children":6482},{"style":631},[6483],{"type":398,"value":6484},"'2023-01-17'",{"type":393,"tag":565,"props":6486,"children":6487},{"style":577},[6488],{"type":398,"value":694},{"type":393,"tag":565,"props":6490,"children":6491},{"class":567,"line":63},[6492,6496,6500,6504,6508,6512,6516,6520,6524,6528,6532,6536],{"type":393,"tag":565,"props":6493,"children":6494},{"style":631},[6495],{"type":398,"value":1451},{"type":393,"tag":565,"props":6497,"children":6498},{"style":577},[6499],{"type":398,"value":639},{"type":393,"tag":565,"props":6501,"children":6502},{"style":631},[6503],{"type":398,"value":711},{"type":393,"tag":565,"props":6505,"children":6506},{"style":577},[6507],{"type":398,"value":650},{"type":393,"tag":565,"props":6509,"children":6510},{"style":631},[6511],{"type":398,"value":711},{"type":393,"tag":565,"props":6513,"children":6514},{"style":577},[6515],{"type":398,"value":650},{"type":393,"tag":565,"props":6517,"children":6518},{"style":631},[6519],{"type":398,"value":720},{"type":393,"tag":565,"props":6521,"children":6522},{"style":577},[6523],{"type":398,"value":650},{"type":393,"tag":565,"props":6525,"children":6526},{"style":631},[6527],{"type":398,"value":720},{"type":393,"tag":565,"props":6529,"children":6530},{"style":577},[6531],{"type":398,"value":650},{"type":393,"tag":565,"props":6533,"children":6534},{"style":631},[6535],{"type":398,"value":729},{"type":393,"tag":565,"props":6537,"children":6538},{"style":577},[6539],{"type":398,"value":694},{"type":393,"tag":565,"props":6541,"children":6542},{"class":567,"line":72},[6543,6547,6551,6555,6559,6563,6567,6571,6575,6579,6583,6587],{"type":393,"tag":565,"props":6544,"children":6545},{"style":631},[6546],{"type":398,"value":1511},{"type":393,"tag":565,"props":6548,"children":6549},{"style":577},[6550],{"type":398,"value":639},{"type":393,"tag":565,"props":6552,"children":6553},{"style":631},[6554],{"type":398,"value":1520},{"type":393,"tag":565,"props":6556,"children":6557},{"style":577},[6558],{"type":398,"value":650},{"type":393,"tag":565,"props":6560,"children":6561},{"style":631},[6562],{"type":398,"value":1529},{"type":393,"tag":565,"props":6564,"children":6565},{"style":577},[6566],{"type":398,"value":650},{"type":393,"tag":565,"props":6568,"children":6569},{"style":631},[6570],{"type":398,"value":1538},{"type":393,"tag":565,"props":6572,"children":6573},{"style":577},[6574],{"type":398,"value":650},{"type":393,"tag":565,"props":6576,"children":6577},{"style":631},[6578],{"type":398,"value":1529},{"type":393,"tag":565,"props":6580,"children":6581},{"style":577},[6582],{"type":398,"value":650},{"type":393,"tag":565,"props":6584,"children":6585},{"style":631},[6586],{"type":398,"value":1555},{"type":393,"tag":565,"props":6588,"children":6589},{"style":577},[6590],{"type":398,"value":694},{"type":393,"tag":565,"props":6592,"children":6593},{"class":567,"line":83},[6594,6598,6602,6606,6610,6614,6618,6622,6626,6630,6634,6638],{"type":393,"tag":565,"props":6595,"children":6596},{"style":631},[6597],{"type":398,"value":1575},{"type":393,"tag":565,"props":6599,"children":6600},{"style":577},[6601],{"type":398,"value":639},{"type":393,"tag":565,"props":6603,"children":6604},{"style":642},[6605],{"type":398,"value":1584},{"type":393,"tag":565,"props":6607,"children":6608},{"style":577},[6609],{"type":398,"value":650},{"type":393,"tag":565,"props":6611,"children":6612},{"style":642},[6613],{"type":398,"value":1593},{"type":393,"tag":565,"props":6615,"children":6616},{"style":577},[6617],{"type":398,"value":650},{"type":393,"tag":565,"props":6619,"children":6620},{"style":642},[6621],{"type":398,"value":879},{"type":393,"tag":565,"props":6623,"children":6624},{"style":577},[6625],{"type":398,"value":650},{"type":393,"tag":565,"props":6627,"children":6628},{"style":642},[6629],{"type":398,"value":1593},{"type":393,"tag":565,"props":6631,"children":6632},{"style":577},[6633],{"type":398,"value":650},{"type":393,"tag":565,"props":6635,"children":6636},{"style":642},[6637],{"type":398,"value":1618},{"type":393,"tag":565,"props":6639,"children":6640},{"style":577},[6641],{"type":398,"value":892},{"type":393,"tag":565,"props":6643,"children":6644},{"class":567,"line":92},[6645],{"type":393,"tag":565,"props":6646,"children":6647},{"style":577},[6648],{"type":398,"value":900},{"type":393,"tag":565,"props":6650,"children":6651},{"class":567,"line":101},[6652],{"type":393,"tag":565,"props":6653,"children":6654},{"emptyLinePlaceholder":12},[6655],{"type":398,"value":598},{"type":393,"tag":565,"props":6657,"children":6658},{"class":567,"line":110},[6659,6663,6667,6672],{"type":393,"tag":565,"props":6660,"children":6661},{"style":642},[6662],{"type":398,"value":915},{"type":393,"tag":565,"props":6664,"children":6665},{"style":577},[6666],{"type":398,"value":920},{"type":393,"tag":565,"props":6668,"children":6669},{"style":631},[6670],{"type":398,"value":6671},"\"📊 Sales Data Analysis:\"",{"type":393,"tag":565,"props":6673,"children":6674},{"style":577},[6675],{"type":398,"value":930},{"type":393,"tag":565,"props":6677,"children":6678},{"class":567,"line":121},[6679,6683],{"type":393,"tag":565,"props":6680,"children":6681},{"style":642},[6682],{"type":398,"value":915},{"type":393,"tag":565,"props":6684,"children":6685},{"style":577},[6686],{"type":398,"value":6687},"(sales)\n",{"type":393,"tag":565,"props":6689,"children":6690},{"class":567,"line":130},[6691,6695],{"type":393,"tag":565,"props":6692,"children":6693},{"style":642},[6694],{"type":398,"value":915},{"type":393,"tag":565,"props":6696,"children":6697},{"style":577},[6698],{"type":398,"value":954},{"type":393,"tag":565,"props":6700,"children":6701},{"class":567,"line":139},[6702],{"type":393,"tag":565,"props":6703,"children":6704},{"emptyLinePlaceholder":12},[6705],{"type":398,"value":598},{"type":393,"tag":565,"props":6707,"children":6708},{"class":567,"line":148},[6709,6713,6717,6722],{"type":393,"tag":565,"props":6710,"children":6711},{"style":642},[6712],{"type":398,"value":915},{"type":393,"tag":565,"props":6714,"children":6715},{"style":577},[6716],{"type":398,"value":920},{"type":393,"tag":565,"props":6718,"children":6719},{"style":631},[6720],{"type":398,"value":6721},"\"✅ Correct Approach: Context-Aware Duplicate Handling\"",{"type":393,"tag":565,"props":6723,"children":6724},{"style":577},[6725],{"type":398,"value":930},{"type":393,"tag":565,"props":6727,"children":6728},{"class":567,"line":159},[6729,6733],{"type":393,"tag":565,"props":6730,"children":6731},{"style":642},[6732],{"type":398,"value":915},{"type":393,"tag":565,"props":6734,"children":6735},{"style":577},[6736],{"type":398,"value":954},{"type":393,"tag":565,"props":6738,"children":6739},{"class":567,"line":168},[6740],{"type":393,"tag":565,"props":6741,"children":6742},{"emptyLinePlaceholder":12},[6743],{"type":398,"value":598},{"type":393,"tag":565,"props":6745,"children":6746},{"class":567,"line":177},[6747],{"type":393,"tag":565,"props":6748,"children":6749},{"style":604},[6750],{"type":398,"value":6751},"# Correct: Only remove exact duplicates\n",{"type":393,"tag":565,"props":6753,"children":6754},{"class":567,"line":186},[6755,6760,6764],{"type":393,"tag":565,"props":6756,"children":6757},{"style":577},[6758],{"type":398,"value":6759},"correct_approach ",{"type":393,"tag":565,"props":6761,"children":6762},{"style":571},[6763],{"type":398,"value":620},{"type":393,"tag":565,"props":6765,"children":6766},{"style":577},[6767],{"type":398,"value":6768}," sales.drop_duplicates()\n",{"type":393,"tag":565,"props":6770,"children":6771},{"class":567,"line":197},[6772,6776,6780,6785],{"type":393,"tag":565,"props":6773,"children":6774},{"style":642},[6775],{"type":398,"value":915},{"type":393,"tag":565,"props":6777,"children":6778},{"style":577},[6779],{"type":398,"value":920},{"type":393,"tag":565,"props":6781,"children":6782},{"style":631},[6783],{"type":398,"value":6784},"\"Exact duplicates removed (none found):\"",{"type":393,"tag":565,"props":6786,"children":6787},{"style":577},[6788],{"type":398,"value":930},{"type":393,"tag":565,"props":6790,"children":6791},{"class":567,"line":206},[6792,6796],{"type":393,"tag":565,"props":6793,"children":6794},{"style":642},[6795],{"type":398,"value":915},{"type":393,"tag":565,"props":6797,"children":6798},{"style":577},[6799],{"type":398,"value":6800},"(correct_approach)\n",{"type":393,"tag":565,"props":6802,"children":6803},{"class":567,"line":215},[6804,6808,6812,6816,6821,6825,6830,6834,6839],{"type":393,"tag":565,"props":6805,"children":6806},{"style":642},[6807],{"type":398,"value":915},{"type":393,"tag":565,"props":6809,"children":6810},{"style":577},[6811],{"type":398,"value":920},{"type":393,"tag":565,"props":6813,"children":6814},{"style":571},[6815],{"type":398,"value":1085},{"type":393,"tag":565,"props":6817,"children":6818},{"style":631},[6819],{"type":398,"value":6820},"\"All ",{"type":393,"tag":565,"props":6822,"children":6823},{"style":642},[6824],{"type":398,"value":1701},{"type":393,"tag":565,"props":6826,"children":6827},{"style":577},[6828],{"type":398,"value":6829},"(correct_approach)",{"type":393,"tag":565,"props":6831,"children":6832},{"style":642},[6833],{"type":398,"value":1105},{"type":393,"tag":565,"props":6835,"children":6836},{"style":631},[6837],{"type":398,"value":6838}," transactions preserved\"",{"type":393,"tag":565,"props":6840,"children":6841},{"style":577},[6842],{"type":398,"value":930},{"type":393,"tag":565,"props":6844,"children":6845},{"class":567,"line":224},[6846,6850],{"type":393,"tag":565,"props":6847,"children":6848},{"style":642},[6849],{"type":398,"value":915},{"type":393,"tag":565,"props":6851,"children":6852},{"style":577},[6853],{"type":398,"value":954},{"type":393,"tag":565,"props":6855,"children":6856},{"class":567,"line":233},[6857],{"type":393,"tag":565,"props":6858,"children":6859},{"emptyLinePlaceholder":12},[6860],{"type":398,"value":598},{"type":393,"tag":565,"props":6862,"children":6863},{"class":567,"line":244},[6864,6868,6872,6877],{"type":393,"tag":565,"props":6865,"children":6866},{"style":642},[6867],{"type":398,"value":915},{"type":393,"tag":565,"props":6869,"children":6870},{"style":577},[6871],{"type":398,"value":920},{"type":393,"tag":565,"props":6873,"children":6874},{"style":631},[6875],{"type":398,"value":6876},"\"🎯 Business Logic Applied:\"",{"type":393,"tag":565,"props":6878,"children":6879},{"style":577},[6880],{"type":398,"value":930},{"type":393,"tag":565,"props":6882,"children":6883},{"class":567,"line":253},[6884,6888,6892,6897],{"type":393,"tag":565,"props":6885,"children":6886},{"style":642},[6887],{"type":398,"value":915},{"type":393,"tag":565,"props":6889,"children":6890},{"style":577},[6891],{"type":398,"value":920},{"type":393,"tag":565,"props":6893,"children":6894},{"style":631},[6895],{"type":398,"value":6896},"\"- Same customer + date + different product = Legitimate\"",{"type":393,"tag":565,"props":6898,"children":6899},{"style":577},[6900],{"type":398,"value":930},{"type":393,"tag":565,"props":6902,"children":6903},{"class":567,"line":262},[6904,6908,6912,6917],{"type":393,"tag":565,"props":6905,"children":6906},{"style":642},[6907],{"type":398,"value":915},{"type":393,"tag":565,"props":6909,"children":6910},{"style":577},[6911],{"type":398,"value":920},{"type":393,"tag":565,"props":6913,"children":6914},{"style":631},[6915],{"type":398,"value":6916},"\"- Same customer + date + same product = Check carefully\"",{"type":393,"tag":565,"props":6918,"children":6919},{"style":577},[6920],{"type":398,"value":930},{"type":393,"tag":565,"props":6922,"children":6923},{"class":567,"line":271},[6924,6928,6932,6937],{"type":393,"tag":565,"props":6925,"children":6926},{"style":642},[6927],{"type":398,"value":915},{"type":393,"tag":565,"props":6929,"children":6930},{"style":577},[6931],{"type":398,"value":920},{"type":393,"tag":565,"props":6933,"children":6934},{"style":631},[6935],{"type":398,"value":6936},"\"- Exact match on all fields = Likely duplicate\"",{"type":393,"tag":565,"props":6938,"children":6939},{"style":577},[6940],{"type":398,"value":930},{"type":393,"tag":538,"props":6942,"children":6944},{"id":6943},"key-takeaways",[6945],{"type":398,"value":6946},"🎯 Key Takeaways",{"type":393,"tag":407,"props":6948,"children":6951},{"emoji":409,"title":6949,"type":6950},"Duplicate Removal Mastery","success",[6952,6960,6996,7004,7051,7059,7087,7095,7123,7131,7154],{"type":393,"tag":401,"props":6953,"children":6954},{},[6955],{"type":393,"tag":417,"props":6956,"children":6957},{},[6958],{"type":398,"value":6959},"Detection Methods:",{"type":393,"tag":431,"props":6961,"children":6962},{},[6963,6974,6985],{"type":393,"tag":435,"props":6964,"children":6965},{},[6966,6972],{"type":393,"tag":561,"props":6967,"children":6969},{"className":6968},[],[6970],{"type":398,"value":6971},".duplicated()",{"type":398,"value":6973}," - identify duplicate rows",{"type":393,"tag":435,"props":6975,"children":6976},{},[6977,6983],{"type":393,"tag":561,"props":6978,"children":6980},{"className":6979},[],[6981],{"type":398,"value":6982},".duplicated(subset=['col'])",{"type":398,"value":6984}," - duplicates based on specific columns",{"type":393,"tag":435,"props":6986,"children":6987},{},[6988,6994],{"type":393,"tag":561,"props":6989,"children":6991},{"className":6990},[],[6992],{"type":398,"value":6993},".duplicated(keep=False)",{"type":398,"value":6995}," - show all rows involved in duplication",{"type":393,"tag":401,"props":6997,"children":6998},{},[6999],{"type":393,"tag":417,"props":7000,"children":7001},{},[7002],{"type":398,"value":7003},"Removal Strategies:",{"type":393,"tag":431,"props":7005,"children":7006},{},[7007,7018,7029,7040],{"type":393,"tag":435,"props":7008,"children":7009},{},[7010,7016],{"type":393,"tag":561,"props":7011,"children":7013},{"className":7012},[],[7014],{"type":398,"value":7015},".drop_duplicates()",{"type":398,"value":7017}," - remove exact duplicates",{"type":393,"tag":435,"props":7019,"children":7020},{},[7021,7027],{"type":393,"tag":561,"props":7022,"children":7024},{"className":7023},[],[7025],{"type":398,"value":7026},".drop_duplicates(subset=['col'])",{"type":398,"value":7028}," - remove based on specific columns",{"type":393,"tag":435,"props":7030,"children":7031},{},[7032,7038],{"type":393,"tag":561,"props":7033,"children":7035},{"className":7034},[],[7036],{"type":398,"value":7037},".drop_duplicates(keep='last')",{"type":398,"value":7039}," - keep last occurrence",{"type":393,"tag":435,"props":7041,"children":7042},{},[7043,7049],{"type":393,"tag":561,"props":7044,"children":7046},{"className":7045},[],[7047],{"type":398,"value":7048},".drop_duplicates(keep=False)",{"type":398,"value":7050}," - remove all duplicates",{"type":393,"tag":401,"props":7052,"children":7053},{},[7054],{"type":393,"tag":417,"props":7055,"children":7056},{},[7057],{"type":398,"value":7058},"Best Practices:",{"type":393,"tag":431,"props":7060,"children":7061},{},[7062,7067,7072,7077,7082],{"type":393,"tag":435,"props":7063,"children":7064},{},[7065],{"type":398,"value":7066},"Understand what constitutes a \"duplicate\" in your context",{"type":393,"tag":435,"props":7068,"children":7069},{},[7070],{"type":398,"value":7071},"Check for duplicates within new data before importing",{"type":393,"tag":435,"props":7073,"children":7074},{},[7075],{"type":398,"value":7076},"Use business logic to resolve conflicts (recent date, higher value)",{"type":393,"tag":435,"props":7078,"children":7079},{},[7080],{"type":398,"value":7081},"Sort data before removing duplicates to control which record is kept",{"type":393,"tag":435,"props":7083,"children":7084},{},[7085],{"type":398,"value":7086},"Validate results to ensure legitimate data wasn't removed",{"type":393,"tag":401,"props":7088,"children":7089},{},[7090],{"type":393,"tag":417,"props":7091,"children":7092},{},[7093],{"type":398,"value":7094},"Common Patterns:",{"type":393,"tag":431,"props":7096,"children":7097},{},[7098,7108,7113,7118],{"type":393,"tag":435,"props":7099,"children":7100},{},[7101,7103],{"type":398,"value":7102},"Exact duplicates: Remove with ",{"type":393,"tag":561,"props":7104,"children":7106},{"className":7105},[],[7107],{"type":398,"value":7015},{"type":393,"tag":435,"props":7109,"children":7110},{},[7111],{"type":398,"value":7112},"ID conflicts: Keep record with most recent date or highest value",{"type":393,"tag":435,"props":7114,"children":7115},{},[7116],{"type":398,"value":7117},"Near duplicates: May need manual review or fuzzy matching",{"type":393,"tag":435,"props":7119,"children":7120},{},[7121],{"type":398,"value":7122},"Import validation: Check new data against existing data",{"type":393,"tag":401,"props":7124,"children":7125},{},[7126],{"type":393,"tag":417,"props":7127,"children":7128},{},[7129],{"type":398,"value":7130},"Prevention Tips:",{"type":393,"tag":431,"props":7132,"children":7133},{},[7134,7139,7144,7149],{"type":393,"tag":435,"props":7135,"children":7136},{},[7137],{"type":398,"value":7138},"Add unique constraints in databases",{"type":393,"tag":435,"props":7140,"children":7141},{},[7142],{"type":398,"value":7143},"Validate data at entry point",{"type":393,"tag":435,"props":7145,"children":7146},{},[7147],{"type":398,"value":7148},"Use transaction IDs or timestamps to identify true duplicates",{"type":393,"tag":435,"props":7150,"children":7151},{},[7152],{"type":398,"value":7153},"Implement data quality checks in your data pipeline",{"type":393,"tag":401,"props":7155,"children":7156},{},[7157,7162],{"type":393,"tag":417,"props":7158,"children":7159},{},[7160],{"type":398,"value":7161},"Remember:",{"type":398,"value":7163}," The goal is to remove duplicate information while preserving legitimate data!",{"type":393,"tag":538,"props":7165,"children":7167},{"id":7166},"practice-challenge",[7168],{"type":398,"value":7169},"🎮 Practice Challenge",{"type":393,"tag":401,"props":7171,"children":7172},{},[7173],{"type":398,"value":7174},"Apply duplicate removal to a realistic customer service scenario:",{"type":393,"tag":550,"props":7176,"children":7177},{},[7178],{"type":393,"tag":554,"props":7179,"children":7181},{"className":556,"code":7180,"language":558,"meta":6,"style":6},"import pandas as pd\n\n# Customer support tickets with various duplicate issues\ntickets = pd.DataFrame({\n    'ticket_id': ['T001', 'T002', 'T003', 'T002', 'T004', 'T005', 'T006'],\n    'customer_email': ['alice@email.com', 'bob@email.com', 'charlie@email.com',\n                       'bob@email.com', 'diana@email.com', 'alice@email.com', 'eve@email.com'],\n    'issue_type': ['Login Problem', 'Billing', 'Feature Request', 'Billing',\n                   'Bug Report', 'Login Problem', 'Billing'],\n    'priority': ['High', 'Medium', 'Low', 'Medium', 'High', 'Medium', 'Low'],\n    'created_date': ['2023-03-01', '2023-03-01', '2023-03-02', '2023-03-01',\n                     '2023-03-03', '2023-03-04', '2023-03-05'],\n    'status': ['Open', 'Resolved', 'Open', 'Resolved', 'Open', 'In Progress', 'Open']\n})\n\nprint(\"🎮 Customer Support Duplicate Challenge:\")\nprint(tickets)\nprint()\n\nprint(\"Mission: Clean this support ticket data\")\nprint()\n\n# Analysis and solution\nprint(\"🔍 Duplicate Analysis:\")\nprint(f\"1️⃣ Total tickets: {len(tickets)}\")\nprint(f\"2️⃣ Exact duplicates: {tickets.duplicated().sum()}\")\nprint(f\"3️⃣ Duplicate ticket IDs: {tickets.duplicated(subset=['ticket_id']).sum()}\")\nprint()\n\n# Solution steps\nprint(\"🧹 Cleaning Steps:\")\ncleaned_tickets = tickets.copy()\n\nprint(\"Step 1: Remove exact duplicates\")\ncleaned_tickets = cleaned_tickets.drop_duplicates()\nprint(f\"After exact duplicate removal: {len(cleaned_tickets)} tickets\")\n\nprint(\"Step 2: Handle duplicate ticket IDs (keep most recent)\")\ncleaned_tickets = cleaned_tickets.sort_values('created_date').drop_duplicates(\n    subset=['ticket_id'], keep='last'\n)\nprint(f\"After ID deduplication: {len(cleaned_tickets)} tickets\")\n\nprint(\"Step 3: Review result\")\nprint(cleaned_tickets)\nprint()\nprint(\"✅ Clean dataset ready for support team analysis!\")\n",[7182],{"type":393,"tag":561,"props":7183,"children":7184},{"__ignoreMap":6},[7185,7204,7211,7219,7235,7304,7339,7374,7421,7449,7520,7564,7593,7664,7671,7678,7698,7710,7721,7728,7748,7759,7766,7774,7794,7835,7876,7939,7950,7957,7965,7985,8002,8009,8029,8045,8087,8094,8114,8139,8174,8181,8221,8228,8248,8260,8271],{"type":393,"tag":565,"props":7186,"children":7187},{"class":567,"line":11},[7188,7192,7196,7200],{"type":393,"tag":565,"props":7189,"children":7190},{"style":571},[7191],{"type":398,"value":574},{"type":393,"tag":565,"props":7193,"children":7194},{"style":577},[7195],{"type":398,"value":580},{"type":393,"tag":565,"props":7197,"children":7198},{"style":571},[7199],{"type":398,"value":585},{"type":393,"tag":565,"props":7201,"children":7202},{"style":577},[7203],{"type":398,"value":590},{"type":393,"tag":565,"props":7205,"children":7206},{"class":567,"line":27},[7207],{"type":393,"tag":565,"props":7208,"children":7209},{"emptyLinePlaceholder":12},[7210],{"type":398,"value":598},{"type":393,"tag":565,"props":7212,"children":7213},{"class":567,"line":36},[7214],{"type":393,"tag":565,"props":7215,"children":7216},{"style":604},[7217],{"type":398,"value":7218},"# Customer support tickets with various duplicate issues\n",{"type":393,"tag":565,"props":7220,"children":7221},{"class":567,"line":45},[7222,7227,7231],{"type":393,"tag":565,"props":7223,"children":7224},{"style":577},[7225],{"type":398,"value":7226},"tickets ",{"type":393,"tag":565,"props":7228,"children":7229},{"style":571},[7230],{"type":398,"value":620},{"type":393,"tag":565,"props":7232,"children":7233},{"style":577},[7234],{"type":398,"value":625},{"type":393,"tag":565,"props":7236,"children":7237},{"class":567,"line":54},[7238,7243,7247,7251,7255,7259,7263,7267,7271,7275,7279,7283,7287,7291,7295,7300],{"type":393,"tag":565,"props":7239,"children":7240},{"style":631},[7241],{"type":398,"value":7242},"    'ticket_id'",{"type":393,"tag":565,"props":7244,"children":7245},{"style":577},[7246],{"type":398,"value":639},{"type":393,"tag":565,"props":7248,"children":7249},{"style":631},[7250],{"type":398,"value":3005},{"type":393,"tag":565,"props":7252,"children":7253},{"style":577},[7254],{"type":398,"value":650},{"type":393,"tag":565,"props":7256,"children":7257},{"style":631},[7258],{"type":398,"value":3014},{"type":393,"tag":565,"props":7260,"children":7261},{"style":577},[7262],{"type":398,"value":650},{"type":393,"tag":565,"props":7264,"children":7265},{"style":631},[7266],{"type":398,"value":3023},{"type":393,"tag":565,"props":7268,"children":7269},{"style":577},[7270],{"type":398,"value":650},{"type":393,"tag":565,"props":7272,"children":7273},{"style":631},[7274],{"type":398,"value":3014},{"type":393,"tag":565,"props":7276,"children":7277},{"style":577},[7278],{"type":398,"value":650},{"type":393,"tag":565,"props":7280,"children":7281},{"style":631},[7282],{"type":398,"value":3040},{"type":393,"tag":565,"props":7284,"children":7285},{"style":577},[7286],{"type":398,"value":650},{"type":393,"tag":565,"props":7288,"children":7289},{"style":631},[7290],{"type":398,"value":3049},{"type":393,"tag":565,"props":7292,"children":7293},{"style":577},[7294],{"type":398,"value":650},{"type":393,"tag":565,"props":7296,"children":7297},{"style":631},[7298],{"type":398,"value":7299},"'T006'",{"type":393,"tag":565,"props":7301,"children":7302},{"style":577},[7303],{"type":398,"value":694},{"type":393,"tag":565,"props":7305,"children":7306},{"class":567,"line":63},[7307,7311,7315,7319,7323,7327,7331,7335],{"type":393,"tag":565,"props":7308,"children":7309},{"style":631},[7310],{"type":398,"value":3061},{"type":393,"tag":565,"props":7312,"children":7313},{"style":577},[7314],{"type":398,"value":639},{"type":393,"tag":565,"props":7316,"children":7317},{"style":631},[7318],{"type":398,"value":775},{"type":393,"tag":565,"props":7320,"children":7321},{"style":577},[7322],{"type":398,"value":650},{"type":393,"tag":565,"props":7324,"children":7325},{"style":631},[7326],{"type":398,"value":784},{"type":393,"tag":565,"props":7328,"children":7329},{"style":577},[7330],{"type":398,"value":650},{"type":393,"tag":565,"props":7332,"children":7333},{"style":631},[7334],{"type":398,"value":793},{"type":393,"tag":565,"props":7336,"children":7337},{"style":577},[7338],{"type":398,"value":3278},{"type":393,"tag":565,"props":7340,"children":7341},{"class":567,"line":72},[7342,7346,7350,7354,7358,7362,7366,7370],{"type":393,"tag":565,"props":7343,"children":7344},{"style":631},[7345],{"type":398,"value":3097},{"type":393,"tag":565,"props":7347,"children":7348},{"style":577},[7349],{"type":398,"value":650},{"type":393,"tag":565,"props":7351,"children":7352},{"style":631},[7353],{"type":398,"value":815},{"type":393,"tag":565,"props":7355,"children":7356},{"style":577},[7357],{"type":398,"value":650},{"type":393,"tag":565,"props":7359,"children":7360},{"style":631},[7361],{"type":398,"value":775},{"type":393,"tag":565,"props":7363,"children":7364},{"style":577},[7365],{"type":398,"value":650},{"type":393,"tag":565,"props":7367,"children":7368},{"style":631},[7369],{"type":398,"value":3114},{"type":393,"tag":565,"props":7371,"children":7372},{"style":577},[7373],{"type":398,"value":694},{"type":393,"tag":565,"props":7375,"children":7376},{"class":567,"line":83},[7377,7382,7386,7391,7395,7400,7404,7409,7413,7417],{"type":393,"tag":565,"props":7378,"children":7379},{"style":631},[7380],{"type":398,"value":7381},"    'issue_type'",{"type":393,"tag":565,"props":7383,"children":7384},{"style":577},[7385],{"type":398,"value":639},{"type":393,"tag":565,"props":7387,"children":7388},{"style":631},[7389],{"type":398,"value":7390},"'Login Problem'",{"type":393,"tag":565,"props":7392,"children":7393},{"style":577},[7394],{"type":398,"value":650},{"type":393,"tag":565,"props":7396,"children":7397},{"style":631},[7398],{"type":398,"value":7399},"'Billing'",{"type":393,"tag":565,"props":7401,"children":7402},{"style":577},[7403],{"type":398,"value":650},{"type":393,"tag":565,"props":7405,"children":7406},{"style":631},[7407],{"type":398,"value":7408},"'Feature Request'",{"type":393,"tag":565,"props":7410,"children":7411},{"style":577},[7412],{"type":398,"value":650},{"type":393,"tag":565,"props":7414,"children":7415},{"style":631},[7416],{"type":398,"value":7399},{"type":393,"tag":565,"props":7418,"children":7419},{"style":577},[7420],{"type":398,"value":3278},{"type":393,"tag":565,"props":7422,"children":7423},{"class":567,"line":92},[7424,7429,7433,7437,7441,7445],{"type":393,"tag":565,"props":7425,"children":7426},{"style":631},[7427],{"type":398,"value":7428},"                   'Bug Report'",{"type":393,"tag":565,"props":7430,"children":7431},{"style":577},[7432],{"type":398,"value":650},{"type":393,"tag":565,"props":7434,"children":7435},{"style":631},[7436],{"type":398,"value":7390},{"type":393,"tag":565,"props":7438,"children":7439},{"style":577},[7440],{"type":398,"value":650},{"type":393,"tag":565,"props":7442,"children":7443},{"style":631},[7444],{"type":398,"value":7399},{"type":393,"tag":565,"props":7446,"children":7447},{"style":577},[7448],{"type":398,"value":694},{"type":393,"tag":565,"props":7450,"children":7451},{"class":567,"line":101},[7452,7457,7461,7466,7470,7475,7479,7484,7488,7492,7496,7500,7504,7508,7512,7516],{"type":393,"tag":565,"props":7453,"children":7454},{"style":631},[7455],{"type":398,"value":7456},"    'priority'",{"type":393,"tag":565,"props":7458,"children":7459},{"style":577},[7460],{"type":398,"value":639},{"type":393,"tag":565,"props":7462,"children":7463},{"style":631},[7464],{"type":398,"value":7465},"'High'",{"type":393,"tag":565,"props":7467,"children":7468},{"style":577},[7469],{"type":398,"value":650},{"type":393,"tag":565,"props":7471,"children":7472},{"style":631},[7473],{"type":398,"value":7474},"'Medium'",{"type":393,"tag":565,"props":7476,"children":7477},{"style":577},[7478],{"type":398,"value":650},{"type":393,"tag":565,"props":7480,"children":7481},{"style":631},[7482],{"type":398,"value":7483},"'Low'",{"type":393,"tag":565,"props":7485,"children":7486},{"style":577},[7487],{"type":398,"value":650},{"type":393,"tag":565,"props":7489,"children":7490},{"style":631},[7491],{"type":398,"value":7474},{"type":393,"tag":565,"props":7493,"children":7494},{"style":577},[7495],{"type":398,"value":650},{"type":393,"tag":565,"props":7497,"children":7498},{"style":631},[7499],{"type":398,"value":7465},{"type":393,"tag":565,"props":7501,"children":7502},{"style":577},[7503],{"type":398,"value":650},{"type":393,"tag":565,"props":7505,"children":7506},{"style":631},[7507],{"type":398,"value":7474},{"type":393,"tag":565,"props":7509,"children":7510},{"style":577},[7511],{"type":398,"value":650},{"type":393,"tag":565,"props":7513,"children":7514},{"style":631},[7515],{"type":398,"value":7483},{"type":393,"tag":565,"props":7517,"children":7518},{"style":577},[7519],{"type":398,"value":694},{"type":393,"tag":565,"props":7521,"children":7522},{"class":567,"line":110},[7523,7528,7532,7536,7540,7544,7548,7552,7556,7560],{"type":393,"tag":565,"props":7524,"children":7525},{"style":631},[7526],{"type":398,"value":7527},"    'created_date'",{"type":393,"tag":565,"props":7529,"children":7530},{"style":577},[7531],{"type":398,"value":639},{"type":393,"tag":565,"props":7533,"children":7534},{"style":631},[7535],{"type":398,"value":5279},{"type":393,"tag":565,"props":7537,"children":7538},{"style":577},[7539],{"type":398,"value":650},{"type":393,"tag":565,"props":7541,"children":7542},{"style":631},[7543],{"type":398,"value":5279},{"type":393,"tag":565,"props":7545,"children":7546},{"style":577},[7547],{"type":398,"value":650},{"type":393,"tag":565,"props":7549,"children":7550},{"style":631},[7551],{"type":398,"value":5288},{"type":393,"tag":565,"props":7553,"children":7554},{"style":577},[7555],{"type":398,"value":650},{"type":393,"tag":565,"props":7557,"children":7558},{"style":631},[7559],{"type":398,"value":5279},{"type":393,"tag":565,"props":7561,"children":7562},{"style":577},[7563],{"type":398,"value":3278},{"type":393,"tag":565,"props":7565,"children":7566},{"class":567,"line":121},[7567,7572,7576,7581,7585,7589],{"type":393,"tag":565,"props":7568,"children":7569},{"style":631},[7570],{"type":398,"value":7571},"                     '2023-03-03'",{"type":393,"tag":565,"props":7573,"children":7574},{"style":577},[7575],{"type":398,"value":650},{"type":393,"tag":565,"props":7577,"children":7578},{"style":631},[7579],{"type":398,"value":7580},"'2023-03-04'",{"type":393,"tag":565,"props":7582,"children":7583},{"style":577},[7584],{"type":398,"value":650},{"type":393,"tag":565,"props":7586,"children":7587},{"style":631},[7588],{"type":398,"value":5318},{"type":393,"tag":565,"props":7590,"children":7591},{"style":577},[7592],{"type":398,"value":694},{"type":393,"tag":565,"props":7594,"children":7595},{"class":567,"line":130},[7596,7601,7605,7610,7614,7619,7623,7627,7631,7635,7639,7643,7647,7652,7656,7660],{"type":393,"tag":565,"props":7597,"children":7598},{"style":631},[7599],{"type":398,"value":7600},"    'status'",{"type":393,"tag":565,"props":7602,"children":7603},{"style":577},[7604],{"type":398,"value":639},{"type":393,"tag":565,"props":7606,"children":7607},{"style":631},[7608],{"type":398,"value":7609},"'Open'",{"type":393,"tag":565,"props":7611,"children":7612},{"style":577},[7613],{"type":398,"value":650},{"type":393,"tag":565,"props":7615,"children":7616},{"style":631},[7617],{"type":398,"value":7618},"'Resolved'",{"type":393,"tag":565,"props":7620,"children":7621},{"style":577},[7622],{"type":398,"value":650},{"type":393,"tag":565,"props":7624,"children":7625},{"style":631},[7626],{"type":398,"value":7609},{"type":393,"tag":565,"props":7628,"children":7629},{"style":577},[7630],{"type":398,"value":650},{"type":393,"tag":565,"props":7632,"children":7633},{"style":631},[7634],{"type":398,"value":7618},{"type":393,"tag":565,"props":7636,"children":7637},{"style":577},[7638],{"type":398,"value":650},{"type":393,"tag":565,"props":7640,"children":7641},{"style":631},[7642],{"type":398,"value":7609},{"type":393,"tag":565,"props":7644,"children":7645},{"style":577},[7646],{"type":398,"value":650},{"type":393,"tag":565,"props":7648,"children":7649},{"style":631},[7650],{"type":398,"value":7651},"'In Progress'",{"type":393,"tag":565,"props":7653,"children":7654},{"style":577},[7655],{"type":398,"value":650},{"type":393,"tag":565,"props":7657,"children":7658},{"style":631},[7659],{"type":398,"value":7609},{"type":393,"tag":565,"props":7661,"children":7662},{"style":577},[7663],{"type":398,"value":892},{"type":393,"tag":565,"props":7665,"children":7666},{"class":567,"line":139},[7667],{"type":393,"tag":565,"props":7668,"children":7669},{"style":577},[7670],{"type":398,"value":900},{"type":393,"tag":565,"props":7672,"children":7673},{"class":567,"line":148},[7674],{"type":393,"tag":565,"props":7675,"children":7676},{"emptyLinePlaceholder":12},[7677],{"type":398,"value":598},{"type":393,"tag":565,"props":7679,"children":7680},{"class":567,"line":159},[7681,7685,7689,7694],{"type":393,"tag":565,"props":7682,"children":7683},{"style":642},[7684],{"type":398,"value":915},{"type":393,"tag":565,"props":7686,"children":7687},{"style":577},[7688],{"type":398,"value":920},{"type":393,"tag":565,"props":7690,"children":7691},{"style":631},[7692],{"type":398,"value":7693},"\"🎮 Customer Support Duplicate Challenge:\"",{"type":393,"tag":565,"props":7695,"children":7696},{"style":577},[7697],{"type":398,"value":930},{"type":393,"tag":565,"props":7699,"children":7700},{"class":567,"line":168},[7701,7705],{"type":393,"tag":565,"props":7702,"children":7703},{"style":642},[7704],{"type":398,"value":915},{"type":393,"tag":565,"props":7706,"children":7707},{"style":577},[7708],{"type":398,"value":7709},"(tickets)\n",{"type":393,"tag":565,"props":7711,"children":7712},{"class":567,"line":177},[7713,7717],{"type":393,"tag":565,"props":7714,"children":7715},{"style":642},[7716],{"type":398,"value":915},{"type":393,"tag":565,"props":7718,"children":7719},{"style":577},[7720],{"type":398,"value":954},{"type":393,"tag":565,"props":7722,"children":7723},{"class":567,"line":186},[7724],{"type":393,"tag":565,"props":7725,"children":7726},{"emptyLinePlaceholder":12},[7727],{"type":398,"value":598},{"type":393,"tag":565,"props":7729,"children":7730},{"class":567,"line":197},[7731,7735,7739,7744],{"type":393,"tag":565,"props":7732,"children":7733},{"style":642},[7734],{"type":398,"value":915},{"type":393,"tag":565,"props":7736,"children":7737},{"style":577},[7738],{"type":398,"value":920},{"type":393,"tag":565,"props":7740,"children":7741},{"style":631},[7742],{"type":398,"value":7743},"\"Mission: Clean this support ticket data\"",{"type":393,"tag":565,"props":7745,"children":7746},{"style":577},[7747],{"type":398,"value":930},{"type":393,"tag":565,"props":7749,"children":7750},{"class":567,"line":206},[7751,7755],{"type":393,"tag":565,"props":7752,"children":7753},{"style":642},[7754],{"type":398,"value":915},{"type":393,"tag":565,"props":7756,"children":7757},{"style":577},[7758],{"type":398,"value":954},{"type":393,"tag":565,"props":7760,"children":7761},{"class":567,"line":215},[7762],{"type":393,"tag":565,"props":7763,"children":7764},{"emptyLinePlaceholder":12},[7765],{"type":398,"value":598},{"type":393,"tag":565,"props":7767,"children":7768},{"class":567,"line":224},[7769],{"type":393,"tag":565,"props":7770,"children":7771},{"style":604},[7772],{"type":398,"value":7773},"# Analysis and solution\n",{"type":393,"tag":565,"props":7775,"children":7776},{"class":567,"line":233},[7777,7781,7785,7790],{"type":393,"tag":565,"props":7778,"children":7779},{"style":642},[7780],{"type":398,"value":915},{"type":393,"tag":565,"props":7782,"children":7783},{"style":577},[7784],{"type":398,"value":920},{"type":393,"tag":565,"props":7786,"children":7787},{"style":631},[7788],{"type":398,"value":7789},"\"🔍 Duplicate Analysis:\"",{"type":393,"tag":565,"props":7791,"children":7792},{"style":577},[7793],{"type":398,"value":930},{"type":393,"tag":565,"props":7795,"children":7796},{"class":567,"line":244},[7797,7801,7805,7809,7814,7818,7823,7827,7831],{"type":393,"tag":565,"props":7798,"children":7799},{"style":642},[7800],{"type":398,"value":915},{"type":393,"tag":565,"props":7802,"children":7803},{"style":577},[7804],{"type":398,"value":920},{"type":393,"tag":565,"props":7806,"children":7807},{"style":571},[7808],{"type":398,"value":1085},{"type":393,"tag":565,"props":7810,"children":7811},{"style":631},[7812],{"type":398,"value":7813},"\"1️⃣ Total tickets: ",{"type":393,"tag":565,"props":7815,"children":7816},{"style":642},[7817],{"type":398,"value":1701},{"type":393,"tag":565,"props":7819,"children":7820},{"style":577},[7821],{"type":398,"value":7822},"(tickets)",{"type":393,"tag":565,"props":7824,"children":7825},{"style":642},[7826],{"type":398,"value":1105},{"type":393,"tag":565,"props":7828,"children":7829},{"style":631},[7830],{"type":398,"value":1110},{"type":393,"tag":565,"props":7832,"children":7833},{"style":577},[7834],{"type":398,"value":930},{"type":393,"tag":565,"props":7836,"children":7837},{"class":567,"line":253},[7838,7842,7846,7850,7855,7859,7864,7868,7872],{"type":393,"tag":565,"props":7839,"children":7840},{"style":642},[7841],{"type":398,"value":915},{"type":393,"tag":565,"props":7843,"children":7844},{"style":577},[7845],{"type":398,"value":920},{"type":393,"tag":565,"props":7847,"children":7848},{"style":571},[7849],{"type":398,"value":1085},{"type":393,"tag":565,"props":7851,"children":7852},{"style":631},[7853],{"type":398,"value":7854},"\"2️⃣ Exact duplicates: ",{"type":393,"tag":565,"props":7856,"children":7857},{"style":642},[7858],{"type":398,"value":1095},{"type":393,"tag":565,"props":7860,"children":7861},{"style":577},[7862],{"type":398,"value":7863},"tickets.duplicated().sum()",{"type":393,"tag":565,"props":7865,"children":7866},{"style":642},[7867],{"type":398,"value":1105},{"type":393,"tag":565,"props":7869,"children":7870},{"style":631},[7871],{"type":398,"value":1110},{"type":393,"tag":565,"props":7873,"children":7874},{"style":577},[7875],{"type":398,"value":930},{"type":393,"tag":565,"props":7877,"children":7878},{"class":567,"line":262},[7879,7883,7887,7891,7896,7900,7905,7909,7913,7917,7922,7927,7931,7935],{"type":393,"tag":565,"props":7880,"children":7881},{"style":642},[7882],{"type":398,"value":915},{"type":393,"tag":565,"props":7884,"children":7885},{"style":577},[7886],{"type":398,"value":920},{"type":393,"tag":565,"props":7888,"children":7889},{"style":571},[7890],{"type":398,"value":1085},{"type":393,"tag":565,"props":7892,"children":7893},{"style":631},[7894],{"type":398,"value":7895},"\"3️⃣ Duplicate ticket IDs: ",{"type":393,"tag":565,"props":7897,"children":7898},{"style":642},[7899],{"type":398,"value":1095},{"type":393,"tag":565,"props":7901,"children":7902},{"style":577},[7903],{"type":398,"value":7904},"tickets.duplicated(",{"type":393,"tag":565,"props":7906,"children":7907},{"style":1259},[7908],{"type":398,"value":2600},{"type":393,"tag":565,"props":7910,"children":7911},{"style":571},[7912],{"type":398,"value":620},{"type":393,"tag":565,"props":7914,"children":7915},{"style":577},[7916],{"type":398,"value":2609},{"type":393,"tag":565,"props":7918,"children":7919},{"style":631},[7920],{"type":398,"value":7921},"'ticket_id'",{"type":393,"tag":565,"props":7923,"children":7924},{"style":577},[7925],{"type":398,"value":7926},"]).sum()",{"type":393,"tag":565,"props":7928,"children":7929},{"style":642},[7930],{"type":398,"value":1105},{"type":393,"tag":565,"props":7932,"children":7933},{"style":631},[7934],{"type":398,"value":1110},{"type":393,"tag":565,"props":7936,"children":7937},{"style":577},[7938],{"type":398,"value":930},{"type":393,"tag":565,"props":7940,"children":7941},{"class":567,"line":271},[7942,7946],{"type":393,"tag":565,"props":7943,"children":7944},{"style":642},[7945],{"type":398,"value":915},{"type":393,"tag":565,"props":7947,"children":7948},{"style":577},[7949],{"type":398,"value":954},{"type":393,"tag":565,"props":7951,"children":7952},{"class":567,"line":281},[7953],{"type":393,"tag":565,"props":7954,"children":7955},{"emptyLinePlaceholder":12},[7956],{"type":398,"value":598},{"type":393,"tag":565,"props":7958,"children":7959},{"class":567,"line":290},[7960],{"type":393,"tag":565,"props":7961,"children":7962},{"style":604},[7963],{"type":398,"value":7964},"# Solution steps\n",{"type":393,"tag":565,"props":7966,"children":7967},{"class":567,"line":299},[7968,7972,7976,7981],{"type":393,"tag":565,"props":7969,"children":7970},{"style":642},[7971],{"type":398,"value":915},{"type":393,"tag":565,"props":7973,"children":7974},{"style":577},[7975],{"type":398,"value":920},{"type":393,"tag":565,"props":7977,"children":7978},{"style":631},[7979],{"type":398,"value":7980},"\"🧹 Cleaning Steps:\"",{"type":393,"tag":565,"props":7982,"children":7983},{"style":577},[7984],{"type":398,"value":930},{"type":393,"tag":565,"props":7986,"children":7987},{"class":567,"line":308},[7988,7993,7997],{"type":393,"tag":565,"props":7989,"children":7990},{"style":577},[7991],{"type":398,"value":7992},"cleaned_tickets ",{"type":393,"tag":565,"props":7994,"children":7995},{"style":571},[7996],{"type":398,"value":620},{"type":393,"tag":565,"props":7998,"children":7999},{"style":577},[8000],{"type":398,"value":8001}," tickets.copy()\n",{"type":393,"tag":565,"props":8003,"children":8004},{"class":567,"line":318},[8005],{"type":393,"tag":565,"props":8006,"children":8007},{"emptyLinePlaceholder":12},[8008],{"type":398,"value":598},{"type":393,"tag":565,"props":8010,"children":8011},{"class":567,"line":327},[8012,8016,8020,8025],{"type":393,"tag":565,"props":8013,"children":8014},{"style":642},[8015],{"type":398,"value":915},{"type":393,"tag":565,"props":8017,"children":8018},{"style":577},[8019],{"type":398,"value":920},{"type":393,"tag":565,"props":8021,"children":8022},{"style":631},[8023],{"type":398,"value":8024},"\"Step 1: Remove exact duplicates\"",{"type":393,"tag":565,"props":8026,"children":8027},{"style":577},[8028],{"type":398,"value":930},{"type":393,"tag":565,"props":8030,"children":8031},{"class":567,"line":336},[8032,8036,8040],{"type":393,"tag":565,"props":8033,"children":8034},{"style":577},[8035],{"type":398,"value":7992},{"type":393,"tag":565,"props":8037,"children":8038},{"style":571},[8039],{"type":398,"value":620},{"type":393,"tag":565,"props":8041,"children":8042},{"style":577},[8043],{"type":398,"value":8044}," cleaned_tickets.drop_duplicates()\n",{"type":393,"tag":565,"props":8046,"children":8047},{"class":567,"line":346},[8048,8052,8056,8060,8065,8069,8074,8078,8083],{"type":393,"tag":565,"props":8049,"children":8050},{"style":642},[8051],{"type":398,"value":915},{"type":393,"tag":565,"props":8053,"children":8054},{"style":577},[8055],{"type":398,"value":920},{"type":393,"tag":565,"props":8057,"children":8058},{"style":571},[8059],{"type":398,"value":1085},{"type":393,"tag":565,"props":8061,"children":8062},{"style":631},[8063],{"type":398,"value":8064},"\"After exact duplicate removal: ",{"type":393,"tag":565,"props":8066,"children":8067},{"style":642},[8068],{"type":398,"value":1701},{"type":393,"tag":565,"props":8070,"children":8071},{"style":577},[8072],{"type":398,"value":8073},"(cleaned_tickets)",{"type":393,"tag":565,"props":8075,"children":8076},{"style":642},[8077],{"type":398,"value":1105},{"type":393,"tag":565,"props":8079,"children":8080},{"style":631},[8081],{"type":398,"value":8082}," tickets\"",{"type":393,"tag":565,"props":8084,"children":8085},{"style":577},[8086],{"type":398,"value":930},{"type":393,"tag":565,"props":8088,"children":8089},{"class":567,"line":355},[8090],{"type":393,"tag":565,"props":8091,"children":8092},{"emptyLinePlaceholder":12},[8093],{"type":398,"value":598},{"type":393,"tag":565,"props":8095,"children":8096},{"class":567,"line":364},[8097,8101,8105,8110],{"type":393,"tag":565,"props":8098,"children":8099},{"style":642},[8100],{"type":398,"value":915},{"type":393,"tag":565,"props":8102,"children":8103},{"style":577},[8104],{"type":398,"value":920},{"type":393,"tag":565,"props":8106,"children":8107},{"style":631},[8108],{"type":398,"value":8109},"\"Step 2: Handle duplicate ticket IDs (keep most recent)\"",{"type":393,"tag":565,"props":8111,"children":8112},{"style":577},[8113],{"type":398,"value":930},{"type":393,"tag":565,"props":8115,"children":8116},{"class":567,"line":374},[8117,8121,8125,8130,8135],{"type":393,"tag":565,"props":8118,"children":8119},{"style":577},[8120],{"type":398,"value":7992},{"type":393,"tag":565,"props":8122,"children":8123},{"style":571},[8124],{"type":398,"value":620},{"type":393,"tag":565,"props":8126,"children":8127},{"style":577},[8128],{"type":398,"value":8129}," cleaned_tickets.sort_values(",{"type":393,"tag":565,"props":8131,"children":8132},{"style":631},[8133],{"type":398,"value":8134},"'created_date'",{"type":393,"tag":565,"props":8136,"children":8137},{"style":577},[8138],{"type":398,"value":2840},{"type":393,"tag":565,"props":8140,"children":8141},{"class":567,"line":383},[8142,8146,8150,8154,8158,8162,8166,8170],{"type":393,"tag":565,"props":8143,"children":8144},{"style":1259},[8145],{"type":398,"value":2848},{"type":393,"tag":565,"props":8147,"children":8148},{"style":571},[8149],{"type":398,"value":620},{"type":393,"tag":565,"props":8151,"children":8152},{"style":577},[8153],{"type":398,"value":2609},{"type":393,"tag":565,"props":8155,"children":8156},{"style":631},[8157],{"type":398,"value":7921},{"type":393,"tag":565,"props":8159,"children":8160},{"style":577},[8161],{"type":398,"value":2865},{"type":393,"tag":565,"props":8163,"children":8164},{"style":1259},[8165],{"type":398,"value":1262},{"type":393,"tag":565,"props":8167,"children":8168},{"style":571},[8169],{"type":398,"value":620},{"type":393,"tag":565,"props":8171,"children":8172},{"style":631},[8173],{"type":398,"value":2878},{"type":393,"tag":565,"props":8175,"children":8176},{"class":567,"line":3773},[8177],{"type":393,"tag":565,"props":8178,"children":8179},{"style":577},[8180],{"type":398,"value":930},{"type":393,"tag":565,"props":8182,"children":8183},{"class":567,"line":3781},[8184,8188,8192,8196,8201,8205,8209,8213,8217],{"type":393,"tag":565,"props":8185,"children":8186},{"style":642},[8187],{"type":398,"value":915},{"type":393,"tag":565,"props":8189,"children":8190},{"style":577},[8191],{"type":398,"value":920},{"type":393,"tag":565,"props":8193,"children":8194},{"style":571},[8195],{"type":398,"value":1085},{"type":393,"tag":565,"props":8197,"children":8198},{"style":631},[8199],{"type":398,"value":8200},"\"After ID deduplication: ",{"type":393,"tag":565,"props":8202,"children":8203},{"style":642},[8204],{"type":398,"value":1701},{"type":393,"tag":565,"props":8206,"children":8207},{"style":577},[8208],{"type":398,"value":8073},{"type":393,"tag":565,"props":8210,"children":8211},{"style":642},[8212],{"type":398,"value":1105},{"type":393,"tag":565,"props":8214,"children":8215},{"style":631},[8216],{"type":398,"value":8082},{"type":393,"tag":565,"props":8218,"children":8219},{"style":577},[8220],{"type":398,"value":930},{"type":393,"tag":565,"props":8222,"children":8223},{"class":567,"line":3802},[8224],{"type":393,"tag":565,"props":8225,"children":8226},{"emptyLinePlaceholder":12},[8227],{"type":398,"value":598},{"type":393,"tag":565,"props":8229,"children":8230},{"class":567,"line":3811},[8231,8235,8239,8244],{"type":393,"tag":565,"props":8232,"children":8233},{"style":642},[8234],{"type":398,"value":915},{"type":393,"tag":565,"props":8236,"children":8237},{"style":577},[8238],{"type":398,"value":920},{"type":393,"tag":565,"props":8240,"children":8241},{"style":631},[8242],{"type":398,"value":8243},"\"Step 3: Review result\"",{"type":393,"tag":565,"props":8245,"children":8246},{"style":577},[8247],{"type":398,"value":930},{"type":393,"tag":565,"props":8249,"children":8250},{"class":567,"line":3829},[8251,8255],{"type":393,"tag":565,"props":8252,"children":8253},{"style":642},[8254],{"type":398,"value":915},{"type":393,"tag":565,"props":8256,"children":8257},{"style":577},[8258],{"type":398,"value":8259},"(cleaned_tickets)\n",{"type":393,"tag":565,"props":8261,"children":8262},{"class":567,"line":3866},[8263,8267],{"type":393,"tag":565,"props":8264,"children":8265},{"style":642},[8266],{"type":398,"value":915},{"type":393,"tag":565,"props":8268,"children":8269},{"style":577},[8270],{"type":398,"value":954},{"type":393,"tag":565,"props":8272,"children":8273},{"class":567,"line":3887},[8274,8278,8282,8287],{"type":393,"tag":565,"props":8275,"children":8276},{"style":642},[8277],{"type":398,"value":915},{"type":393,"tag":565,"props":8279,"children":8280},{"style":577},[8281],{"type":398,"value":920},{"type":393,"tag":565,"props":8283,"children":8284},{"style":631},[8285],{"type":398,"value":8286},"\"✅ Clean dataset ready for support team analysis!\"",{"type":393,"tag":565,"props":8288,"children":8289},{"style":577},[8290],{"type":398,"value":930},{"type":393,"tag":538,"props":8292,"children":8294},{"id":8293},"whats-next",[8295],{"type":398,"value":8296},"🚀 What's Next?",{"type":393,"tag":401,"props":8298,"children":8299},{},[8300],{"type":398,"value":8301},"Excellent work! You now know how to identify and remove duplicates effectively. Next, let's learn about converting data to the right types for proper analysis.",{"type":393,"tag":401,"props":8303,"children":8304},{},[8305,8310,8312],{"type":393,"tag":417,"props":8306,"children":8307},{},[8308],{"type":398,"value":8309},"Continue to",{"type":398,"value":8311},": ",{"type":393,"tag":8313,"props":8314,"children":8315},"a",{"href":211},[8316],{"type":398,"value":212},{"type":393,"tag":401,"props":8318,"children":8319},{},[8320],{"type":398,"value":8321},"You're mastering data quality control! 🧹✨",{"type":393,"tag":8323,"props":8324,"children":8325},"style",{},[8326],{"type":398,"value":8327},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .app-dark-mode .shiki span {color: var(--shiki-app-dark-mode);background: var(--shiki-app-dark-mode-bg);font-style: var(--shiki-app-dark-mode-font-style);font-weight: var(--shiki-app-dark-mode-font-weight);text-decoration: var(--shiki-app-dark-mode-text-decoration);}html.app-dark-mode .shiki span {color: var(--shiki-app-dark-mode);background: var(--shiki-app-dark-mode-bg);font-style: var(--shiki-app-dark-mode-font-style);font-weight: var(--shiki-app-dark-mode-font-weight);text-decoration: var(--shiki-app-dark-mode-text-decoration);}",{"title":6,"searchDepth":36,"depth":36,"links":8329},[8330,8331,8332,8333,8334,8335,8336,8337,8338,8339],{"id":540,"depth":27,"text":543},{"id":1311,"depth":27,"text":1314},{"id":2107,"depth":27,"text":2110},{"id":2920,"depth":27,"text":2923},{"id":3960,"depth":27,"text":3963},{"id":5017,"depth":27,"text":5020},{"id":6221,"depth":27,"text":6224},{"id":6943,"depth":27,"text":6946},{"id":7166,"depth":27,"text":7169},{"id":8293,"depth":27,"text":8296},1782373723311]