\documentclass[11pt,twoside]{article}\makeatletter \IfFileExists{xcolor.sty}% {\RequirePackage{xcolor}}% {\RequirePackage{color}} \usepackage{colortbl} \usepackage{wrapfig} \usepackage{ifxetex} \ifxetex \usepackage{fontspec} \usepackage{xunicode} \catcode`⃥=\active \def⃥{\textbackslash} \catcode`❴=\active \def❴{\{} \catcode`❵=\active \def❵{\}} \def\textJapanese{\fontspec{Noto Sans CJK JP}} \def\textChinese{\fontspec{Noto Sans CJK SC}} \def\textKorean{\fontspec{Noto Sans CJK KR}} \setmonofont{DejaVu Sans Mono} \else \IfFileExists{utf8x.def}% {\usepackage[utf8x]{inputenc} \PrerenderUnicode{–} }% {\usepackage[utf8]{inputenc}} \usepackage[english]{babel} \usepackage[T1]{fontenc} \usepackage{float} \usepackage[]{ucs} \uc@dclc{8421}{default}{\textbackslash } \uc@dclc{10100}{default}{\{} \uc@dclc{10101}{default}{\}} \uc@dclc{8491}{default}{\AA{}} \uc@dclc{8239}{default}{\,} \uc@dclc{20154}{default}{ } \uc@dclc{10148}{default}{>} \def\textschwa{\rotatebox{-90}{e}} \def\textJapanese{} \def\textChinese{} \IfFileExists{tipa.sty}{\usepackage{tipa}}{} \fi \def\exampleFont{\ttfamily\small} \DeclareTextSymbol{\textpi}{OML}{25} \usepackage{relsize} \RequirePackage{array} \def\@testpach{\@chclass \ifnum \@lastchclass=6 \@ne \@chnum \@ne \else \ifnum \@lastchclass=7 5 \else \ifnum \@lastchclass=8 \tw@ \else \ifnum \@lastchclass=9 \thr@@ \else \z@ \ifnum \@lastchclass = 10 \else \edef\@nextchar{\expandafter\string\@nextchar}% \@chnum \if \@nextchar c\z@ \else \if \@nextchar l\@ne \else \if \@nextchar r\tw@ \else \z@ \@chclass \if\@nextchar |\@ne \else \if \@nextchar !6 \else \if \@nextchar @7 \else \if \@nextchar (8 \else \if \@nextchar )9 \else 10 \@chnum \if \@nextchar m\thr@@\else \if \@nextchar p4 \else \if \@nextchar b5 \else \z@ \@chclass \z@ \@preamerr \z@ \fi \fi \fi \fi \fi \fi \fi \fi \fi \fi \fi \fi \fi \fi \fi \fi} \gdef\arraybackslash{\let\\=\@arraycr} \def\@textsubscript#1{{\m@th\ensuremath{_{\mbox{\fontsize\sf@size\z@#1}}}}} \def\Panel#1#2#3#4{\multicolumn{#3}{){\columncolor{#2}}#4}{#1}} \def\abbr{} \def\corr{} \def\expan{} \def\gap{} \def\orig{} \def\reg{} \def\ref{} \def\sic{} \def\persName{}\def\name{} \def\placeName{} \def\orgName{} \def\textcal#1{{\fontspec{Lucida Calligraphy}#1}} \def\textgothic#1{{\fontspec{Lucida Blackletter}#1}} \def\textlarge#1{{\large #1}} \def\textoverbar#1{\ensuremath{\overline{#1}}} \def\textquoted#1{‘#1’} \def\textsmall#1{{\small #1}} \def\textsubscript#1{\@textsubscript{\selectfont#1}} \def\textxi{\ensuremath{\xi}} \def\titlem{\itshape} \newenvironment{biblfree}{}{\ifvmode\par\fi } \newenvironment{bibl}{}{} \newenvironment{byline}{\vskip6pt\itshape\fontsize{16pt}{18pt}\selectfont}{\par } \newenvironment{citbibl}{}{\ifvmode\par\fi } \newenvironment{docAuthor}{\ifvmode\vskip4pt\fontsize{16pt}{18pt}\selectfont\fi\itshape}{\ifvmode\par\fi } \newenvironment{docDate}{}{\ifvmode\par\fi } \newenvironment{docImprint}{\vskip 6pt}{\ifvmode\par\fi } \newenvironment{docTitle}{\vskip6pt\bfseries\fontsize{22pt}{25pt}\selectfont}{\par } \newenvironment{msHead}{\vskip 6pt}{\par} \newenvironment{msItem}{\vskip 6pt}{\par} \newenvironment{rubric}{}{} \newenvironment{titlePart}{}{\par } \newcolumntype{L}[1]{){\raggedright\arraybackslash}p{#1}} \newcolumntype{C}[1]{){\centering\arraybackslash}p{#1}} \newcolumntype{R}[1]{){\raggedleft\arraybackslash}p{#1}} \newcolumntype{P}[1]{){\arraybackslash}p{#1}} \newcolumntype{B}[1]{){\arraybackslash}b{#1}} \newcolumntype{M}[1]{){\arraybackslash}m{#1}} \definecolor{label}{gray}{0.75} \def\unusedattribute#1{\sout{\textcolor{label}{#1}}} \DeclareRobustCommand*{\xref}{\hyper@normalise\xref@} \def\xref@#1#2{\hyper@linkurl{#2}{#1}} \begingroup \catcode`\_=\active \gdef_#1{\ensuremath{\sb{\mathrm{#1}}}} \endgroup \mathcode`\_=\string"8000 \catcode`\_=12\relax \usepackage[a4paper,twoside,lmargin=1in,rmargin=1in,tmargin=1in,bmargin=1in,marginparwidth=0.75in]{geometry} \usepackage{framed} \definecolor{shadecolor}{gray}{0.95} \usepackage{longtable} \usepackage[normalem]{ulem} \usepackage{fancyvrb} \usepackage{fancyhdr} \usepackage{graphicx} \usepackage{marginnote} \renewcommand{\@cite}[1]{#1} \renewcommand*{\marginfont}{\itshape\footnotesize} \def\Gin@extensions{.pdf,.png,.jpg,.mps,.tif} \pagestyle{fancy} \usepackage[pdftitle={Entity Matching for Digital World: A Modern Approach using Artificial Intelligence and Machine Learning}, pdfauthor={}]{hyperref} \hyperbaseurl{} \paperwidth210mm \paperheight297mm \def\@pnumwidth{1.55em} \def\@tocrmarg {2.55em} \def\@dotsep{4.5} \setcounter{tocdepth}{3} \clubpenalty=8000 \emergencystretch 3em \hbadness=4000 \hyphenpenalty=400 \pretolerance=750 \tolerance=2000 \vbadness=4000 \widowpenalty=10000 \renewcommand\section{\@startsection {section}{1}{\z@}% {-1.75ex \@plus -0.5ex \@minus -.2ex}% {0.5ex \@plus .2ex}% {\reset@font\Large\bfseries}} \renewcommand\subsection{\@startsection{subsection}{2}{\z@}% {-1.75ex\@plus -0.5ex \@minus- .2ex}% {0.5ex \@plus .2ex}% {\reset@font\Large}} \renewcommand\subsubsection{\@startsection{subsubsection}{3}{\z@}% {-1.5ex\@plus -0.35ex \@minus -.2ex}% {0.5ex \@plus .2ex}% {\reset@font\large}} \renewcommand\paragraph{\@startsection{paragraph}{4}{\z@}% {-1ex \@plus-0.35ex \@minus -0.2ex}% {0.5ex \@plus .2ex}% {\reset@font\normalsize}} \renewcommand\subparagraph{\@startsection{subparagraph}{5}{\parindent}% {1.5ex \@plus1ex \@minus .2ex}% {-1em}% {\reset@font\normalsize\bfseries}} \def\l@section#1#2{\addpenalty{\@secpenalty} \addvspace{1.0em plus 1pt} \@tempdima 1.5em \begingroup \parindent \z@ \rightskip \@pnumwidth \parfillskip -\@pnumwidth \bfseries \leavevmode #1\hfil \hbox to\@pnumwidth{\hss #2}\par \endgroup} \def\l@subsection{\@dottedtocline{2}{1.5em}{2.3em}} \def\l@subsubsection{\@dottedtocline{3}{3.8em}{3.2em}} \def\l@paragraph{\@dottedtocline{4}{7.0em}{4.1em}} \def\l@subparagraph{\@dottedtocline{5}{10em}{5em}} \@ifundefined{c@section}{\newcounter{section}}{} \@ifundefined{c@chapter}{\newcounter{chapter}}{} \newif\if@mainmatter \@mainmattertrue \def\chaptername{Chapter} \def\frontmatter{% \pagenumbering{roman} \def\thechapter{\@roman\c@chapter} \def\theHchapter{\roman{chapter}} \def\thesection{\@roman\c@section} \def\theHsection{\roman{section}} \def\@chapapp{}% } \def\mainmatter{% \cleardoublepage \def\thechapter{\@arabic\c@chapter} \setcounter{chapter}{0} \setcounter{section}{0} \pagenumbering{arabic} \setcounter{secnumdepth}{6} \def\@chapapp{\chaptername}% \def\theHchapter{\arabic{chapter}} \def\thesection{\@arabic\c@section} \def\theHsection{\arabic{section}} } \def\backmatter{% \cleardoublepage \setcounter{chapter}{0} \setcounter{section}{0} \setcounter{secnumdepth}{2} \def\@chapapp{\appendixname}% \def\thechapter{\@Alph\c@chapter} \def\theHchapter{\Alph{chapter}} \appendix } \newenvironment{bibitemlist}[1]{% \list{\@biblabel{\@arabic\c@enumiv}}% {\settowidth\labelwidth{\@biblabel{#1}}% \leftmargin\labelwidth \advance\leftmargin\labelsep \@openbib@code \usecounter{enumiv}% \let\p@enumiv\@empty \renewcommand\theenumiv{\@arabic\c@enumiv}% }% \sloppy \clubpenalty4000 \@clubpenalty \clubpenalty \widowpenalty4000% \sfcode`\.\@m}% {\def\@noitemerr {\@latex@warning{Empty `bibitemlist' environment}}% \endlist} \def\tableofcontents{\section*{\contentsname}\@starttoc{toc}} \parskip0pt \parindent1em \def\Panel#1#2#3#4{\multicolumn{#3}{){\columncolor{#2}}#4}{#1}} \newenvironment{reflist}{% \begin{raggedright}\begin{list}{} {% \setlength{\topsep}{0pt}% \setlength{\rightmargin}{0.25in}% \setlength{\itemsep}{0pt}% \setlength{\itemindent}{0pt}% \setlength{\parskip}{0pt}% \setlength{\parsep}{2pt}% \def\makelabel##1{\itshape ##1}}% } {\end{list}\end{raggedright}} \newenvironment{sansreflist}{% \begin{raggedright}\begin{list}{} {% \setlength{\topsep}{0pt}% \setlength{\rightmargin}{0.25in}% \setlength{\itemindent}{0pt}% \setlength{\parskip}{0pt}% \setlength{\itemsep}{0pt}% \setlength{\parsep}{2pt}% \def\makelabel##1{\upshape ##1}}% } {\end{list}\end{raggedright}} \newenvironment{specHead}[2]% {\vspace{20pt}\hrule\vspace{10pt}% \phantomsection\label{#1}\markright{#2}% \pdfbookmark[2]{#2}{#1}% \hspace{-0.75in}{\bfseries\fontsize{16pt}{18pt}\selectfont#2}% }{} \def\TheFullDate{1970-01-01 (revised: 01 January 1970)} \def\TheID{\makeatother } \def\TheDate{1970-01-01} \title{Entity Matching for Digital World: A Modern Approach using Artificial Intelligence and Machine Learning} \author{}\makeatletter \makeatletter \newcommand*{\cleartoleftpage}{% \clearpage \if@twoside \ifodd\c@page \hbox{}\newpage \if@twocolumn \hbox{}\newpage \fi \fi \fi } \makeatother \makeatletter \thispagestyle{empty} \markright{\@title}\markboth{\@title}{\@author} \renewcommand\small{\@setfontsize\small{9pt}{11pt}\abovedisplayskip 8.5\p@ plus3\p@ minus4\p@ \belowdisplayskip \abovedisplayskip \abovedisplayshortskip \z@ plus2\p@ \belowdisplayshortskip 4\p@ plus2\p@ minus2\p@ \def\@listi{\leftmargin\leftmargini \topsep 2\p@ plus1\p@ minus1\p@ \parsep 2\p@ plus\p@ minus\p@ \itemsep 1pt} } \makeatother \fvset{frame=single,numberblanklines=false,xleftmargin=5mm,xrightmargin=5mm} \fancyhf{} \setlength{\headheight}{14pt} \fancyhead[LE]{\bfseries\leftmark} \fancyhead[RO]{\bfseries\rightmark} \fancyfoot[RO]{} \fancyfoot[CO]{\thepage} \fancyfoot[LO]{\TheID} \fancyfoot[LE]{} \fancyfoot[CE]{\thepage} \fancyfoot[RE]{\TheID} \hypersetup{citebordercolor=0.75 0.75 0.75,linkbordercolor=0.75 0.75 0.75,urlbordercolor=0.75 0.75 0.75,bookmarksnumbered=true} \fancypagestyle{plain}{\fancyhead{}\renewcommand{\headrulewidth}{0pt}} \date{} \usepackage{authblk} \providecommand{\keywords}[1] { \footnotesize \textbf{\textit{Index terms---}} #1 } \usepackage{graphicx,xcolor} \definecolor{GJBlue}{HTML}{273B81} \definecolor{GJLightBlue}{HTML}{0A9DD9} \definecolor{GJMediumGrey}{HTML}{6D6E70} \definecolor{GJLightGrey}{HTML}{929497} \renewenvironment{abstract}{% \setlength{\parindent}{0pt}\raggedright \textcolor{GJMediumGrey}{\rule{\textwidth}{2pt}} \vskip16pt \textcolor{GJBlue}{\large\bfseries\abstractname\space} }{% \vskip8pt \textcolor{GJMediumGrey}{\rule{\textwidth}{2pt}} \vskip16pt } \usepackage[absolute,overlay]{textpos} \makeatother \usepackage{lineno} \linenumbers \begin{document} \author[1]{K. Victor Rajan} \author[2]{Edward Lambert} \affil[1]{ Atlantic International University, USA} \renewcommand\Authands{ and } \date{\small \em Received: 1 January 1970 Accepted: 1 January 1970 Published: 1 January 1970} \maketitle \begin{abstract} Entity matching is the field of research solving the problem of identifying similar records which refer to the same real-world entity. In today?s digital world, business organizations deal with large amount of data like customers, vendors, manufacturers, etc. Entities are spread across various data sources and failure to correlate two records as one entity can lead to confusion. Relationships and patterns would be missed. Aggregations and calculations won?t make any sense. It is a significant data integration effort that often arises when data originate from different sources. In such scenarios, we understand the situation by linking records and then track entities from a person to a product, etc. There is appreciable value in integrating the data silos across various industries. \end{abstract} \keywords{entity matching, entity resolution, record linkage, de-duplication, machine learning} \begin{textblock*}{18cm}(1cm,1cm) % {block width} (coords) \textcolor{GJBlue}{\LARGE Global Journals \LaTeX\ JournalKaleidoscope\texttrademark} \end{textblock*} \begin{textblock*}{18cm}(1.4cm,1.5cm) % {block width} (coords) \textcolor{GJBlue}{\footnotesize \\ Artificial Intelligence formulated this projection for compatibility purposes from the original article published at Global Journals. However, this technology is currently in beta. \emph{Therefore, kindly ignore odd layouts, missed formulae, text, tables, or figures.}} \end{textblock*} \begin{textblock*}{10cm}(1.05cm,3cm) {{\textit{CrossRef DOI of original article:}} \underline{}} \end{textblock*}\let\tabcellsep& \section[{Introduction}]{Introduction}\par ur world is moving towards digitized business. This opens up numerous avenues to increase revenue through digital marketing, sales forecast, etc. Huge amount of historical data is available to analyze customer behavior, buying patterns and make predictions for future. However, it also comes with challenges along the way. A substantial amount of the value to be harvested from digitization depends on successful integration of large volume of data from different sources. Unfortunately, many of the existing data sources do not share a common frame of reference. For example, let us say, a marketing team wants to use statistics from retail stores, e-commerce sites etc., to find out potential buyers for a product. Sadly, these two systems do not refer to customers in the same way -i.e., there are no common identifiers or names across the two systems. Duplicate emails or messages may be sent to same customer again and again unless customer records are tagged uniquely. Recommendations to a customer and an effective marketing scheme cannot be performed based on distinct data silos. A group of similar problems has been studied for a long time in a variety of fields under different names like entity resolution, de-duplication etc. Entity matching is the field of research dedicated to solving the problem of matching which records refer to the same real-world entity. Organizations often struggle with a plethora of customer data captured multiple times in different sources by various people in their own ways. Despite having been studied for decades, entity matching remains a challenging problem in practice. In general, there are several factors that make it difficult to solve:\par Poor Data Quality: Real-world data is seldom completely structured, cleansed, and homogeneous. Data originating from manual insertion may contain alternative spellings, typos, or fail to comply with the schema (e.g., mixing of first and last name).\par Dependency on Human Knowledge: Same data may be represented in different formats by various users like abbreviations, suffixes, prefixes, etc. To perform matching, our solution must interact with human experts and make use of their knowledge. Human interaction in itself is a complex domain.\par For example, let's look at a customer table from which analyst is trying to identify distinct customers. Without manual inspection and good understanding of geographical locations, it is difficult to guess whether record 2 is duplicate of 1 or 3. Somewhat ironically, as often pointed out, entity matching suffers from the problem of being referenced by different names, some referring to the exact same problem, while others are slight variations, generalizations, or specializations. In addition, the names are also not used completely consistently. Deduplication or duplicate detection is the problem of identifying records in the same data source that refer to the same entity and can be seen as the special case 1 = 2. Given such representation variations, an unprecedented number of permutations and combinations, the entity matching would be a herculean job when we handle large volume of data. Artificial intelligence and machine learning has become an essential part of multiple research fields in recent years, most notably in natural language processing and computer vision, which are concerned with unstructured data. Its most prominent advantage over systematic approaches is its ability to learn features instead of relying on step-by-step calculations. \section[{a) Problem Definition}]{a) Problem Definition}\par Researchers have already realized the potential advantage of machine learning for entity matching. In this paper, we aim to propose a machine learning model for entity matching.\par Let E be a data source containing entities. E has the attributes (??1,??2, ...,????), and we denote entities as e = (e1, e2, ..., e??) ? E. A data source is a set of records, and a record is a tuple having a specific schema of attributes. An attribute is defined by the intended semantics of its values. So, entities e?? = e?? if and only if attributes ???? of e?? are intended to carry the same information as attributes a?? of e??, and the specific syntactics of the attribute values are irrelevant. Attributes can also have metadata (like a name) associated with them, but this does not affect the equality between them.\par The goal of entity matching is to find the largest possible binary relation ?? ? E × E such that ?? and ?? refer to the same entity for all (??, ??) ? ??. In other words, we would like to find all record pairs across data source that refer to the same entity. We define an entity to be something of unique existence. Attribute values are often assumed to be strings, but that is not always the case. The records are assumed to operate with the same taxonomic granularity. In this research, we will stick to the definition of deduplication (or duplicate detection) as the problem of identifying which records in the same data source refer to the same entity.\par The remainder of this paper is organized as follows. We discuss related work in section 2. In Section 3, we formally formulate the problem and propose our methodology. Section 4 describes how our approach is used to detect similarity in a real-world data set and the results of our experiment are explained. Finally, the paper is concluded in Section 5. \section[{II.}]{II.} \section[{Related Work}]{Related Work}\par Entity resolution, record linkage, deduplication and entity matching are frequently used for more or less the same problem as we mentioned earlier. It is a technique to identify data records in a single data source or across multiple data sources that refer to the same real-world entity and to correlate the records together. In entity matching, the strings that are nearly identical, but not exactly the same, are matched without explicitly having a unique identifier. Entity matching is crucial as it matches non-identical records despite all the data inconsistencies without the constant need for formulating rules. By combining databases using fuzzy matching, we can refine the data and analyze the information. Comparing big data records having nonstandard and inconsistent data from diverse sources that do not provide any unique identifier is a complex problem. In this section, we present an overview of the previous work done by researchers in entity matching. \section[{Researchers use two major techniques as shown below:}]{Researchers use two major techniques as shown below:}\par Rule-Based: Rule-based systems perform matching based on a set of manually crafted rules. To match any two records of the same entity, various string-based comparison rules are defined. Each record then would run with every other record on all these rules to decide if the two are identical.\par Automatic: These systems rely on machine learning algorithms to learn from data. Computers first learn from data provided for training so that they can later make predictions on unknown input data items.\par Usually, a rule-based system uses a set of human-crafted rules to help identify subjectivity. As the number of records increases, the number of comparisons increases exponentially in rule-based systems. With large volume of records, rule-based data matching becomes computationally challenging and unscalable. Automatic methods, contrary to rule-based systems, do not rely on manually crafted rules but on machine learning algorithms. There has been an uptick in interest on machine learning as a solution for entity matching in recent years. We note that this process is machine-oriented and does not highlight any iterative human interactions or feedback loops. First, there are several books that provide an overview. Christen \hyperref[b14]{[15]} is a dedicated and comprehensive source on entity matching. Anhai Doan et al. \hyperref[b1]{[2]} and Talburt \hyperref[b9]{[10]} introduce entity matching in the context of data quality and integration. Quite early on, statisticians dominated the field of entity matching. Probabilistic methods were first developed by Newcombe et al. \hyperref[b14]{[15]}. A solid theoretical framework was presented by Fellegi and Sunter \hyperref[b8]{[9]}. Blocking, which is surveyed by Papadakis et al. \hyperref[b7]{[8,}\hyperref[b8]{9]}, is considered an important subtask of entity matching. This is meant to tackle the quadratic complexity of potential matches. Christophides et al. \hyperref[b23]{[24]} specifically review entity matching techniques in the context of big data. Significant research has gone into active learning approaches by Arvind \hyperref[b2]{[3]}, Jungo \hyperref[b10]{[11]} and Kun \hyperref[b11]{[12]}. Interestingly, Jungo et al. \hyperref[b10]{[11]} use a deep neural network in their active learning approach. Such human-in-the-loop factors are often crucial for entity matching in practice as analyzed by Anhai et al. \hyperref[b1]{[2]}. Many state-of-the-art models for natural language processing are based on deep learning networks. Central to all these approaches is how text is transformed to a numerical format suitable for a neural network. This is mainly done through embeddings, which are translations from text units to a vector spacetraditionally available in a lookup table. The text units will usually be characters or words. An embeddings lookup table may be seen as parameters to the network and can be learned together with the rest of the network endto-end. That way the network is able to learn good, distributed character or word representations for the problem at hand. The words used in a data set are often not unique to that data set, but rather just typical words from some language. Therefore, one may often get a head start by using pretrained word embeddings like word2vec, GloVe or fastText, which have been trained on enormous general corpora. One particular influential recent trend is the ability to leverage huge pretrained models that have been trained unsupervised for language modeling on massive text corpora similar to what the computer vision community has done for image recognition. They produce contextualized word embeddings that consider the surrounding words. These contextual embeddings can be used as a much more powerful variant of the classical word embeddings, but as popularized by BERT. However, with neural networks, the actual line between the initial feature extraction part and the rest is an artificial one and not necessarily indicative of how the networks actually learn and work. But they do reflect design decisions to a certain degree and help us compare them in that regard. Often these approaches use pre-built word embeddings for a specific set of values. Our research focuses on entity matching based on attributes where the number of attributes may vary from one use case to another. Also, we try to address the problem of multiple domains, i.e., the machine learning model must be suitable for entities from various categories like customers, products, vendors, etc. In this paper, we present a machine learning model which will perform attribute-based matching of entities. The type, number of attributes may vary over the time, but our approach does not require re-design. Merely a re-training of the model on the new data set will suffice. The model is robust enough to handle slight variations in ordinality and type of the attributes. \section[{III.}]{III.} \section[{Methodology}]{Methodology}\par Most neural network-based methods perform entity matching by producing so-called knowledge graph embeddings, embeddings of entries which incorporate information about their relationship with other entries. The embeddings work mainly at word level or character level. Embeddings offer neural networks an initial mapping from the actual input to a suitable numeric representation. When we surveyed the earlier methods, we found that researchers focus on explicit levels of representation of entities into single word or text. However, we try to address two problems mainly,\par ? How to perform matching of entities containing attributes of different data types, say string, boolean, and categorical? ? Will the machine learning algorithm continue to work even if the number of attributes change over the time? Let's say there are few entities in a data set as shown in Table \hyperref[tab_0]{1}. It has two duplicates. Following is a generalized notation. The entities e1 and e2 are same, though they might vary slightly in their attribute values but have similar meanings. Our aim is to design an approach which will combine the attribute level similarity and artificial intelligence to classify entities as unique or duplicate. We propose a two-step methodology where the first step involves calculating attribute level similarity scores and the second step is classification using supervised learning. Feature extraction involves use of a distance function for every pair of attributes. It transforms every pair of entities into numerical vector. For any give pair of attributes (?? ???? , ?? ???? ), the distance function ?? produces a numerical value such that 0 <= ??(?? ???? , ?? ???? ) <= 1\par If the two attributes are exactly same, then the distance metric is zero. If they are completely unrelated, then the distance is 1. Partial match will result in value between 0 and 1. We call it as similarity score of the attributes.\par A sample set of vectors for a set of three entities will be as shown below. The extracted values correspond to two class labels duplicate (D) and unique (U). If we extract feature vectors of a data set and plot the points in a 3-dimensional space, then we will see two clusters as shown below. Our approach takes every pair of entities and produces a numerical vector. This is in turn fed to a machine learning algorithm for classification. We use supervised learning algorithm for classification. The ML model learns from the training data set and makes accurate predictions on the incoming test data. \section[{a) Feature Extraction using Similarity Score}]{a) Feature Extraction using Similarity Score}\par The first step in ML modeling is data preprocessing, which is usually a crucial step in many data analytics tasks. Typical transformations involve lowercasing all letters, removing excess punctuation, normalizing values, and tokenizing. There are two other major steps in our process. Second one being the feature vector construction using similarity score and For example, consider a similarity function Levenshtein Distance. The Levenshtein distance between 'new yrk' and 'new york' is one since it needs at least one edit (insertion, deletion, or substitution) to transform from 'new yrk' to 'new york'. It is advisable to normalize the similarity scores between 0 and 1 for improved accuracy of the machine learning algorithm. \section[{b) Classification using Supervised Learning}]{b) Classification using Supervised Learning}\par The matching phase aims to develop the prediction model, which takes a candidate pair as input and predicts whether they are matching or nonmatching. Figure \hyperref[fig_1]{2} illustrates that the model predicts an output label Duplicate (D) or Unique (U). This is a binary classification problem. Data scientists need to decide which algorithm is most suitable for their classification task. Based on our study and experiments, we found three classification algorithms suitable for this task. \section[{i. K-Nearest Neighbors (KNN)}]{i. K-Nearest Neighbors (KNN)}\par K-Nearest Neighbors (KNN) is an algorithm that learns all available cases from data set and classifies new data item by a majority vote of its K neighbors. A case assigned to the data is majority of its K nearest neighbors measured by a distance (metric) function. The metric functions include Euclidean, Manhattan, Minkowski, and Hamming distances. KNN can be used for both regression and classification problems. However, it is widely used in classification problems in the industry.\par ii. XG Boost XG Boost stands for Extreme Gradient Boosting. It is a scalable, distributed gradient-boosted decision tree (GBDT) machine learning library. It provides parallel tree boosting and is the leading machine learning library for regression, and classification problems. \section[{iii. Support Vector Machines (SVM)}]{iii. Support Vector Machines (SVM)}\par Support Vector Machine is a supervised algorithm in which the learning algorithm analyzes data and recognizes patterns. We plot the data as points in an n-dimensional space. The value of each feature is then tied to a particular co-ordinate, making it easy to classify the data.\par And finally, we need to tune hyper-parameters in order to get the best model performance.\par IV. \section[{Experiments and Results}]{Experiments and Results}\par Automatic entity matching makes the life of commercial organizations easier. A company that maintains thousands of customer records cannot afford to employ many people to verify manually and identify duplicates. Artificial Intelligence based entity matching is an efficient and cost-effective analytics tool for operational efficiency. We used open-source data sets for our experiments. While several open-source datasets are available, we picked up few commercial data sets for analysis. In this section, we describe the evaluation tasks, the data sets used, and the experimental results of our approach.\par Evaluation Tasks: 1. We evaluate our approach on real-world data set. 2. We evaluate our approach on popular benchmarks.\par Our goal is to provide real-life solution using our approach. We aim to evaluate the quality of entity matching. The empirical result is compared with realtime data to harness the accuracy. The results show promising output. \section[{a) Data Set}]{a) Data Set}\par We conducted extensive experiments on realworld benchmark entity datasets to evaluate the performance of approach. Following are few opensource data sets available for evaluating entity matching algorithms. Many commercial organizations are nowadays struggling with customer de-duplication. Automatic deduplication has significance in various sectors like Banking and Finance, Insurance, Telecom, Retail, etc. Hence our results mainly focus on the evaluation metrics accuracy on the customer data set. \section[{b) Popular Metrics}]{b) Popular Metrics}\par In this section, we first describe a set of metrics commonly used for evaluating the performance of our classification model. Then we present a quantitative analysis of the performance using popular benchmarks.\par Accuracy and Error Rate: These are primary metrics to evaluate the quality of a classification model. Let TP, FP, TN, FN denote true positive, false positive, true negative, and false negative, respectively. The classification Accuracy and Error Rate are defined in Equation \hyperref[formula_0]{1}.\par ()\textbf{1}\par where ?? is the total number of samples. Obviously, we have Error Rate = 1 -Accuracy.\par Precision, Recall, and F1 Score: These are also primary metrics and are more often used than accuracy or error rate for imbalanced test sets. Precision and recall for binary classification are defined in Equation {\ref 2}. The F1 score is the harmonic mean of the precision and recall, as in Equation {\ref 2}. F1 score reaches its best value at 1 (perfect precision and recall) and worst at 0. \section[{c) Empirical Results}]{c) Empirical Results}\par We aim to use our entity matching systemin real world applications like retail, e-commerce etc. We analyzed the presence of duplicate customer data and results showed more that 80\% accuracy in read-world data sets. Following is a set of predictions made by our system from Fodors-Zagats dataset. From the above table, we observe that customers, vendors can easily get their ambiguities resolved using automatic entity matching system. AIbased entity matching is an alternative to traditional manual or other text analysis-based tools, and it is costeffective solution for decision-makers.\par V. \section[{Conclusion and Future Work}]{Conclusion and Future Work}\par The proposed method accomplished superior performance in terms of time and cost. The overall benefits of AI-based entity matching include: Sorting Data at Scale: Manually screening thousands of customer records, or product details is complex and time-consuming. AI-based entity matching helps businesses process large amount of data in an efficient and cost-effective way. \section[{Real-Time Analysis:}]{Real-Time Analysis:}\par The automatic entity matching can help organizations quickly identify duplicates on realtime basis and act swiftly before duplicate marketing or promotional offers are sent out. Though many deep learning models are being developed nowadays for entity matching, we propose a supervised learning model for few major reasons.\par Explainability and Ease of Debugging: For many applications, it is crucial to trust the data source, and try to understand why something does not work is key. Unfortunately, deep learning models are notoriously hard to interpret. As steps in the entity matching process increasingly coalesce into a large neural network, we get fewer checkpoints along the way in the process that can easily be inspected. We can't see the output from each step in the same way anymore. Therefore, figuring out why two records where matched or not matched is usually nontrivial while inspecting deep learning models. There are a few techniques that are already used, such as looking at alignment scores, but we are still far away from a comprehensive way of debugging neural networks for entity matching. Our model addresses the challenges of explainability, running time in interactive settings, and the large need for training examples. Explainability of our supervised learning algorithm helps researchers to improve accuracy through inspection, comparison of algorithms and meet the real-world demands. We also see a lot of opportunities in trying to develop more open datasets, standardized benchmarks, and publicly available pretrained models for entity matching.\begin{figure}[htbp] \noindent\textbf{1}\includegraphics[]{image-2.png} \caption{\label{fig_0}Figure 1 :}\end{figure} \begin{figure}[htbp] \noindent\textbf{2}\includegraphics[]{image-3.png} \caption{\label{fig_1}Figure 2 :}\end{figure} \begin{figure}[htbp] \noindent\textbf{2}\includegraphics[]{image-4.png} \caption{\label{fig_2}( 2 )}\end{figure} \begin{figure}[htbp] \noindent\textbf{3}\includegraphics[]{image-5.png} \caption{\label{fig_3}Figure 3 :}\end{figure} \begin{figure}[htbp] \noindent\textbf{1} \par \begin{longtable}{P{0.027127659574468083\textwidth}P{0.2079787234042553\textwidth}P{0.425\textwidth}P{0.18989361702127658\textwidth}} No.\tabcellsep Name\tabcellsep Address\tabcellsep Email\\ 1\tabcellsep Alexander Great\tabcellsep 2/13, Philip France Street, Paris,\tabcellsep alex.gr@gmail.com\\ 2\tabcellsep Alexander G\tabcellsep 2/13, Philip Street, Paris\tabcellsep n/a\\ 3\tabcellsep Alexander Graham\tabcellsep 10, Middle Street, New York\tabcellsep alex.gr@yahoo.com\end{longtable} \par \caption{\label{tab_0}Table 1 :}\end{figure} \begin{figure}[htbp] \noindent\textbf{2} \par \begin{longtable}{P{0.3787128712871287\textwidth}P{0.07574257425742574\textwidth}P{0.07574257425742574\textwidth}P{0.07574257425742574\textwidth}P{0.24405940594059403\textwidth}} \multicolumn{4}{l}{Entity Attribute1 Attribute2 Attribute3}\tabcellsep Label\\ e1\tabcellsep a11\tabcellsep a12\tabcellsep a13\tabcellsep Duplicate\\ e2\tabcellsep a21\tabcellsep a22\tabcellsep a23\tabcellsep (e1 = e2)\\ e3\tabcellsep a31\tabcellsep a32\tabcellsep a33\tabcellsep Unique\end{longtable} \par \caption{\label{tab_1}Table 2 :}\end{figure} \begin{figure}[htbp] \noindent\textbf{3} \par \begin{longtable}{P{0.3090909090909091\textwidth}P{0.2125\textwidth}P{0.11590909090909089\textwidth}P{0.11590909090909089\textwidth}P{0.09659090909090909\textwidth}} Entity Pair\tabcellsep Score1\tabcellsep Score2\tabcellsep Score3\tabcellsep Label\\ e1,e2\tabcellsep ??(??\tabcellsep \tabcellsep \tabcellsep \end{longtable} \par {\small\itshape [Note: 11 , ?? 21 ) = 0.8 ??(?? 12 , ?? 22 ) = 0.6 ??(?? 13 , ?? 23 ) = 1 D e2,e3 ??(?? 21 , ?? 31 ) = 0.5 ??(?? 22 , ?? 32 ) = 0.6 ??(?? 23 , ?? 33 ) = 0 U e1,e3 ??(?? 11 , ?? 31 ) = 0.6 ??(?? 12 , ?? 32 ) = 0.4 ??(?? 13 , ?? 33 ) = 1 U]} \caption{\label{tab_2}Table 3 :}\end{figure} \begin{figure}[htbp] \noindent\textbf{4} \par \begin{longtable}{P{0.04473684210526315\textwidth}P{0.2013157894736842\textwidth}P{0.6039473684210526\textwidth}} \tabcellsep \tabcellsep popular\\ \tabcellsep \tabcellsep algorithms.\\ No.\tabcellsep Data Type\tabcellsep Similarity Function\\ 1\tabcellsep \tabcellsep Exact Match\\ 2\tabcellsep \tabcellsep Levenshtein Distance\\ 3\tabcellsep Single Word String\tabcellsep Jaro Distance\end{longtable} \par {\small\itshape [Note: last step is machine learning. One might also view second as feature extraction, since records are transformed to a feature space. The success of this entity matching systems depends upon careful selection of right algorithms. Attributes are often assumed to be strings, but that is not the case always. Attributes of an entity may be of any data type like string, numeric, categorical, boolean etc. One single function will not be able to calculate similarity score for various attributes to attribute. It is useful to compare various functions available for similarity score and pick the right choice. To this end, we present a high-level overview of few]} \caption{\label{tab_3}Table 4 :}\end{figure} \begin{figure}[htbp] \noindent\textbf{5} \par \begin{longtable}{P{0.012022630834512021\textwidth}P{0.1045968882602546\textwidth}P{0.6107496463932108\textwidth}P{0.048090523338048086\textwidth}P{0.04568599717114569\textwidth}P{0.028854314002828854\textwidth}} No.\tabcellsep Dataset\tabcellsep Description\tabcellsep Training Size\tabcellsep Testing Size\tabcellsep No. of Attributes\\ 1\tabcellsep Fodors-Zagats\tabcellsep Customer records with name, address, city, phone, type, and category code.\tabcellsep 757\tabcellsep 189\tabcellsep 6\\ 2\tabcellsep iTunes-Amazon\tabcellsep Records of songs with song name, artist name, album name, genre, etc.\tabcellsep 430\tabcellsep 109\tabcellsep 8\\ 3\tabcellsep DBLP-ACM\tabcellsep Publication dataset with paper title, author, venue etc.\tabcellsep 9890\tabcellsep 2473\tabcellsep 4\\ 4\tabcellsep DBLP-Scholar\tabcellsep Publication dataset with title, authors, venue, and year.\tabcellsep 22965\tabcellsep 5742\tabcellsep 4\\ 5\tabcellsep Amazon-Google\tabcellsep Software product dataset with attributes product title, manufacturer, and price.\tabcellsep 9167\tabcellsep 2293\tabcellsep 3\\ 6\tabcellsep Walmart-Amazon\tabcellsep Electronic product dataset with attributes product name, category, brand, model number, etc.\tabcellsep 8193\tabcellsep 2049\tabcellsep 5\\ 7\tabcellsep Abt-Buy\tabcellsep Product dataset with attributes product name, price, and description.\tabcellsep 7659\tabcellsep 1916\tabcellsep 3\end{longtable} \par \caption{\label{tab_4}Table 5 :}\end{figure} \begin{figure}[htbp] \noindent\textbf{6} \par \begin{longtable}{P{0.7121621621621621\textwidth}P{0.06891891891891892\textwidth}P{0.06891891891891892\textwidth}} \multicolumn{3}{l}{CLASSIFICATION ALGORITHM}\\ XGBoost\tabcellsep KNN\tabcellsep SVM\end{longtable} \par \caption{\label{tab_5}Table 6 :}\end{figure} \begin{figure}[htbp] \noindent\textbf{7} \par \begin{longtable}{P{0.0195852534562212\textwidth}P{0.23306451612903223\textwidth}P{0.2565668202764977\textwidth}P{0.1880184331797235\textwidth}P{0.13513824884792627\textwidth}P{0.01762672811059908\textwidth}} No.\tabcellsep Name\tabcellsep Address\tabcellsep City\tabcellsep Phone\tabcellsep Label\\ 1\tabcellsep restaurant ritz-carlton atlanta\tabcellsep 181 Peachtree st.\tabcellsep Atlanta\tabcellsep 404/659 -0400\tabcellsep \\ 2\tabcellsep ritz-carlton restaurant\tabcellsep 181 Peachtree st.\tabcellsep Atlanta\tabcellsep 404/659 -0400\tabcellsep D\\ 3\tabcellsep posterior\tabcellsep 545 post st.\tabcellsep \multicolumn{2}{l}{San Francisco 415/776 -7825}\tabcellsep \\ 4\tabcellsep postrio\tabcellsep 545 post street.\tabcellsep \multicolumn{2}{l}{San Francisco 415/776 -7825}\tabcellsep D\\ 5\tabcellsep tavern on the green\tabcellsep in central park at 67th st\tabcellsep New York\tabcellsep 212/873 -3200\tabcellsep \\ 6\tabcellsep tavern on the green\tabcellsep central park west\tabcellsep New York\tabcellsep 212/873 -3200\tabcellsep D\\ 7\tabcellsep carey's\tabcellsep 1021 cobb pkwy . se\tabcellsep marietta\tabcellsep 770-422-8042\tabcellsep U\end{longtable} \par \caption{\label{tab_6}Table 7 :}\end{figure} \footnote{© 2023 Global Journals} \backmatter \begin{bibitemlist}{1} \bibitem[Christen (2012)]{b19}\label{b19} ‘A Survey of Indexing Techniques for Scalable Record Linkage and Deduplication’. P Christen . \textit{IEEE Trans. Knowl. Data Eng} 2012. Sept. 2012. 24 p. . \bibitem[Ivan et al. (1969)]{b10}\label{b10} ‘A Theory for Record Linkage’. P Ivan , Alan B Fellegi , Sunter . \textit{Jr. Am. Stat. Assoc} 1969. Dec. 1969. 64 p. . \bibitem[Kun Qian et al. ()]{b13}\label{b13} ‘Active Learning for Large-Scale Entity Resolution’. Lucian Kun Qian , Prithviraj Popa , Sen . \textit{Proceedings of the 2017 ACM on Conference on Information and Knowledge Management (CIKM '17)}, (the 2017 ACM on Conference on Information and Knowledge Management (CIKM '17)) 2017. \bibitem[Naumann and Herschel ()]{b6}\label{b6} \textit{An Introduction to Duplicate Detection}, Felix Naumann , Melanie Herschel . 2010. Morgan and Claypool Publishers. \bibitem[Newcombe et al. (1959)]{b16}\label{b16} ‘Automatic linkage of vital records’. H B Newcombe , J M Kennedy , S J Axford , A P James . \textit{Science} 1959. Oct. 1959. 130 p. . \bibitem[Papadakis and Svirsky (2016)]{b8}\label{b8} ‘Comparative Analysis of Approximate Blocking Techniques for Entity Resolution’. George Papadakis , Jonathan Svirsky . \textit{Proceedings VLDB Endowment} May 2016. 9 p. . (Avigdor Gal, and Themis Palpanas) \bibitem[Christen ()]{b18}\label{b18} \textit{Data Matching: Concepts and Techniques for Record Linkage, Entity Resolution, and Duplicate Detection}, Peter Christen . 2012. Springer Science \& Business Media. \bibitem[Thomas et al. ()]{b21}\label{b21} \textit{Data Quality and Record Linkage Techniques}, N Thomas , Herzog , J Fritz , William E Scheuren , Winkler . 2007. Springer Science \& Business Media. \bibitem[Li et al. (2020)]{b24}\label{b24} \textit{Deep Entity Matching with Pre-Trained Language Models}, Yuliang Li , Jinfeng Li , Yoshihiko Suhara , Anhai Doan , Wang-Chiew Tan . arXiv:cs.DB/2004.00584. 2020. April 2020. \bibitem[Kooli et al. ()]{b17}\label{b17} \textit{Deep Learning Based Approach for Entity Resolution in Databases. In Intelligent Information and Database Systems}, Nihel Kooli , Robin Allesiardo , Erwan Pigneul . 2018. Springer International Publishing. p. . \bibitem[Muhammad et al. (2018)]{b15}\label{b15} ‘Distributed representations of tuples for entity resolution’. E Muhammad , T Saravanan , Shafiq Joty , Mourad Ouzzani , Nan Tang . \textit{Proceedings VLDB Endowment} 2018. July 2018. 11 (11) p. . \bibitem[A K Elmagarmid et al. (2007)]{b3}\label{b3} ‘Duplicate Record Detection: A Survey’. P G A K Elmagarmid , V S Ipeirotis , Verykios . \textit{IEEE Trans. Knowl. Data Eng} 2007. Jan. 2007. 19 p. . \bibitem[Christophides et al. (2019)]{b23}\label{b23} \textit{End-to-End Entity Resolution for Big Data: A Survey}, Vassilis Christophides , Vasilis Efthymiou , Themis Palpanas , George Papadakis , Kostas Stefanidis . arXiv:cs. 2019. May 2019. \bibitem[Fu et al. ()]{b4}\label{b4} ‘End-to-end Multi-perspective Matching for Entity Resolution’. Cheng Fu , Xianpei Han , Le Sun , Bo Chen , Wei Zhang , Suhui Wu , Hao Kong . \textit{Proceedings of the 28th International Joint Conference on Artificial Intelligence (IJCAI'19)}, (the 28th International Joint Conference on Artificial Intelligence (IJCAI'19)Macao, China) 2019. AAAI Press. p. . \bibitem[Brunner and Stockinger (2020)]{b22}\label{b22} ‘Entity matching with transformer architectures-a step forward in data integration’. Ursin Brunner , Kurt Stockinger . \textit{International Conference on Extending Database Technology}, (Copenhagen) 2020. 30 March-2 April 2020. \bibitem[John R Talburt ()]{b11}\label{b11} \textit{Entity Resolution and Information Quality}, John R Talburt . 2011. Elsevier. \bibitem[Deepak Gottapu et al. (2016)]{b20}\label{b20} ‘Entity Resolution Using Convolutional Neural Network’. Ram Deepak Gottapu , Cihan Dagli , Bharami Ali . \textit{Procedia Comput. Sci} 2016. Jan. 2016. 95 p. . \bibitem[Getoor and Machanavajjhala (2012)]{b14}\label{b14} ‘Entity Resolution: Theory, Practice \& Open Challenges’. Lise Getoor , Ashwin Machanavajjhala . \textit{Proceedings VLDB Endowment} 2012. Aug. 2012. 5 p. . \bibitem[Köpcke et al. (2010)]{b9}\label{b9} ‘Evaluation of Entity Resolution Approaches on Real-world Match Problems’. Hanna Köpcke , Andreas Thor , Erhard Rahm . \textit{Proceedings VLDB Endowment} 2010. Sept. 2010. 3 p. . \bibitem[Doan et al. ()]{b1}\label{b1} ‘Human-in-the-loop challenges for entity matching: A midterm report’. Anhai Doan , Adel Ardalan , Jeffrey Ballard , Sanjib Das , Yash Govind , Pradap Konda , Han Li , Sidharth Mudgal , Erik Paulson , G C Paul Suganthan , Haojun Zhang . \textit{Proceedings of the 2Nd Workshop on Human-In-the-Loop Data Analytics (HILDA'17). dl.acm.org}, (the 2Nd Workshop on Human-In-the-Loop Data Analytics (HILDA'17). dl.acm.orgNew York, NY, USA) 2017. 12 p. 6. \bibitem[Kasai et al. ()]{b12}\label{b12} ‘Low-resource Deep Entity Resolution with Transfer and Active Learning’. Jungo Kasai , Sairam Kun Qian , Yunyao Gurajada , Lucian Li , Popa . \textit{Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics}, (the 57th Annual Meeting of the Association for Computational LinguisticsStroudsburg, PA, USA) 2019. Association for Computational Linguistics. p. . \bibitem[Arasu et al. ()]{b2}\label{b2} ‘On Active Learning of Record Matching Packages’. Arvind Arasu , Michaela Götz , Raghav Kaushik . \textit{Proceedings of the 2010 ACM SIGMOD International Conference on Management of Data (SIGMOD '10)}, (the 2010 ACM SIGMOD International Conference on Management of Data (SIGMOD '10)New York, NY, USA) 2010. ACM. p. . \bibitem[Ioannou et al. (2013)]{b5}\label{b5} ‘On Generating Benchmark Data for Entity Matching’. Ekaterini Ioannou , Nataliya Rassadko , Yannis Velegrakis . \textit{Jr. Data Semant} 2013. March 2013. 2 p. . \bibitem[Papadakis and Skoutas (2019)]{b7}\label{b7} George Papadakis , Dimitrios Skoutas . arXiv:cs.DB/1905.06167. \textit{Emmanouil Thanos, and Themis Palpanas. A Survey of Blocking and Filtering Techniques for Entity Resolution}, May 2019. \bibitem[Arasu et al. ()]{b0}\label{b0} ‘Transformation-based Framework for Record Matching’. A Arasu , R Chaudhuri , Kaushik . \textit{IEEE 24th International Conference on Data Engineering.ieeexplore.ieee. org}, 2008. 2008. p. . \end{bibitemlist} \end{document}