https://spark.apache.org/docs/3.5.1/ml-guide.html

掃描 ID：: 7a6398f2-af8a-4ef8-9dd6-fd8b68916154已完成
已提交的 URL：: https://spark.apache.org/docs/3.5.1/ml-guide.html
報告完成時間：: 2024年12月16日 08:38:46
連結 · 找到 31 個

從頁面中識別的傳出連結
連結	Text
http://www.scalanlp.org/	Breeze
https://github.com/luhenry/netlib	dev.ludovic.netlib
https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html	Intel MKL
http://www.openblas.net	OpenBLAS
http://www.numpy.org	NumPy
https://issues.apache.org/jira/browse/SPARK-23578	SPARK-23578
https://issues.apache.org/jira/browse/SPARK-11215	SPARK-11215
https://issues.apache.org/jira/browse/SPARK-29808	SPARK-29808
https://issues.apache.org/jira/browse/SPARK-22796	SPARK-22796
https://issues.apache.org/jira/browse/SPARK-13677	SPARK-13677
JavaScript 變數 · 找到 26 個

在頁面的視窗物件上載入的全域 JavaScript 變數是在函數外部宣告的變數，可從目前範圍內程式碼中的任何位置存取
名稱	類型
onbeforetoggle	object
documentPictureInPicture	object
onscrollend	object
html5	object
Modernizr	object
yepnope	function
respond	object
_paq	object
uidEvent	number
bootstrap	object
主控台記錄訊息 · 找到 0 條

記錄到 Web 主控台的訊息
HTML

頁面的原始 HTML 主體
<!DOCTYPE html><!--[if lt IE 7]>      <html class="no-js lt-ie9 lt-ie8 lt-ie7"> <![endif]--><!--[if IE 7]>         <html class="no-js lt-ie9 lt-ie8"> <![endif]--><!--[if IE 8]>         <html class="no-js lt-ie9"> <![endif]--><!--[if gt IE 8]><!--><html class=" js flexbox canvas canvastext webgl no-touch geolocation postmessage no-websqldatabase indexeddb hashchange history draganddrop websockets rgba hsla multiplebgs backgroundsize borderimage borderradius boxshadow textshadow opacity cssanimations csscolumns cssgradients cssreflections csstransforms csstransforms3d csstransitions fontface generatedcontent video audio localstorage sessionstorage webworkers no-applicationcache svg inlinesvg smil svgclippaths"><!--<![endif]--><head>
        <meta charset="utf-8">
        <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
        <meta name="viewport" content="width=device-width, initial-scale=1.0">

        <title>MLlib: Main Guide - Spark 3.5.1 Documentation</title>
        

        


        <style class="anchorjs"></style><link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
        <link rel="preconnect" href="https://fonts.googleapis.com">
        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
        <link href="https://fonts.googleapis.com/css2?family=DM+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&amp;Courier+Prime:wght@400;700&amp;display=swap" rel="stylesheet">
        <link href="css/custom.css" rel="stylesheet">
        <script async="" src="https://analytics.apache.org/matomo.js"></script><script src="js/vendor/modernizr-2.6.1-respond-1.1.0.min.js"></script>

        <link rel="stylesheet" href="css/pygments-default.css">
        <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.css">
        <link rel="stylesheet" href="css/docsearch.css">

        
        <!-- Matomo -->
        <script>
            var _paq = window._paq = window._paq || [];
            /* tracker methods like "setCustomDimension" should be called before "trackPageView" */
            _paq.push(["disableCookies"]);
            _paq.push(['trackPageView']);
            _paq.push(['enableLinkTracking']);
            (function() {
              var u="https://analytics.apache.org/";
              _paq.push(['setTrackerUrl', u+'matomo.php']);
              _paq.push(['setSiteId', '40']);
              var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
              g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
            })();
        </script>
        <!-- End Matomo Code -->
        

    <script type="text/javascript" async="" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script><style type="text/css">.MathJax_Hover_Frame {border-radius: .25em; -webkit-border-radius: .25em; -moz-border-radius: .25em; -khtml-border-radius: .25em; box-shadow: 0px 0px 15px #83A; -webkit-box-shadow: 0px 0px 15px #83A; -moz-box-shadow: 0px 0px 15px #83A; -khtml-box-shadow: 0px 0px 15px #83A; border: 1px solid #A6D ! important; display: inline-block; position: absolute}
.MathJax_Menu_Button .MathJax_Hover_Arrow {position: absolute; cursor: pointer; display: inline-block; border: 2px solid #AAA; border-radius: 4px; -webkit-border-radius: 4px; -moz-border-radius: 4px; -khtml-border-radius: 4px; font-family: 'Courier New',Courier; font-size: 9px; color: #F0F0F0}
.MathJax_Menu_Button .MathJax_Hover_Arrow span {display: block; background-color: #AAA; border: 1px solid; border-radius: 3px; line-height: 0; padding: 4px}
.MathJax_Hover_Arrow:hover {color: white!important; border: 2px solid #CCC!important}
.MathJax_Hover_Arrow:hover span {background-color: #CCC!important}
</style><style type="text/css">#MathJax_About {position: fixed; left: 50%; width: auto; text-align: center; border: 3px outset; padding: 1em 2em; background-color: #DDDDDD; color: black; cursor: default; font-family: message-box; font-size: 120%; font-style: normal; text-indent: 0; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; z-index: 201; border-radius: 15px; -webkit-border-radius: 15px; -moz-border-radius: 15px; -khtml-border-radius: 15px; box-shadow: 0px 10px 20px #808080; -webkit-box-shadow: 0px 10px 20px #808080; -moz-box-shadow: 0px 10px 20px #808080; -khtml-box-shadow: 0px 10px 20px #808080; filter: progid:DXImageTransform.Microsoft.dropshadow(OffX=2, OffY=2, Color='gray', Positive='true')}
#MathJax_About.MathJax_MousePost {outline: none}
.MathJax_Menu {position: absolute; background-color: white; color: black; width: auto; padding: 5px 0px; border: 1px solid #CCCCCC; margin: 0; cursor: default; font: menu; text-align: left; text-indent: 0; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; z-index: 201; border-radius: 5px; -webkit-border-radius: 5px; -moz-border-radius: 5px; -khtml-border-radius: 5px; box-shadow: 0px 10px 20px #808080; -webkit-box-shadow: 0px 10px 20px #808080; -moz-box-shadow: 0px 10px 20px #808080; -khtml-box-shadow: 0px 10px 20px #808080; filter: progid:DXImageTransform.Microsoft.dropshadow(OffX=2, OffY=2, Color='gray', Positive='true')}
.MathJax_MenuItem {padding: 1px 2em; background: transparent}
.MathJax_MenuArrow {position: absolute; right: .5em; padding-top: .25em; color: #666666; font-size: .75em}
.MathJax_MenuActive .MathJax_MenuArrow {color: white}
.MathJax_MenuArrow.RTL {left: .5em; right: auto}
.MathJax_MenuCheck {position: absolute; left: .7em}
.MathJax_MenuCheck.RTL {right: .7em; left: auto}
.MathJax_MenuRadioCheck {position: absolute; left: .7em}
.MathJax_MenuRadioCheck.RTL {right: .7em; left: auto}
.MathJax_MenuLabel {padding: 1px 2em 3px 1.33em; font-style: italic}
.MathJax_MenuRule {border-top: 1px solid #DDDDDD; margin: 4px 3px}
.MathJax_MenuDisabled {color: GrayText}
.MathJax_MenuActive {background-color: #606872; color: white}
.MathJax_MenuDisabled:focus, .MathJax_MenuLabel:focus {background-color: #E8E8E8}
.MathJax_ContextMenu:focus {outline: none}
.MathJax_ContextMenu .MathJax_MenuItem:focus {outline: none}
#MathJax_AboutClose {top: .2em; right: .2em}
.MathJax_Menu .MathJax_MenuClose {top: -10px; left: -10px}
.MathJax_MenuClose {position: absolute; cursor: pointer; display: inline-block; border: 2px solid #AAA; border-radius: 18px; -webkit-border-radius: 18px; -moz-border-radius: 18px; -khtml-border-radius: 18px; font-family: 'Courier New',Courier; font-size: 24px; color: #F0F0F0}
.MathJax_MenuClose span {display: block; background-color: #AAA; border: 1.5px solid; border-radius: 18px; -webkit-border-radius: 18px; -moz-border-radius: 18px; -khtml-border-radius: 18px; line-height: 0; padding: 8px 0 6px}
.MathJax_MenuClose:hover {color: white!important; border: 2px solid #CCC!important}
.MathJax_MenuClose:hover span {background-color: #CCC!important}
.MathJax_MenuClose:hover:focus {outline: none}
</style><style type="text/css">.MathJax_Preview .MJXf-math {color: inherit!important}
</style><style type="text/css">.MJX_Assistive_MathML {position: absolute!important; top: 0; left: 0; clip: rect(1px, 1px, 1px, 1px); padding: 1px 0 0 0!important; border: 0!important; height: 1px!important; width: 1px!important; overflow: hidden!important; display: block!important; -webkit-touch-callout: none; -webkit-user-select: none; -khtml-user-select: none; -moz-user-select: none; -ms-user-select: none; user-select: none}
.MJX_Assistive_MathML.MJX_Assistive_MathML_Block {width: 100%!important}
</style><style type="text/css">#MathJax_Zoom {position: absolute; background-color: #F0F0F0; overflow: auto; display: block; z-index: 301; padding: .5em; border: 1px solid black; margin: 0; font-weight: normal; font-style: normal; text-align: left; text-indent: 0; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; -webkit-box-sizing: content-box; -moz-box-sizing: content-box; box-sizing: content-box; box-shadow: 5px 5px 15px #AAAAAA; -webkit-box-shadow: 5px 5px 15px #AAAAAA; -moz-box-shadow: 5px 5px 15px #AAAAAA; -khtml-box-shadow: 5px 5px 15px #AAAAAA; filter: progid:DXImageTransform.Microsoft.dropshadow(OffX=2, OffY=2, Color='gray', Positive='true')}
#MathJax_ZoomOverlay {position: absolute; left: 0; top: 0; z-index: 300; display: inline-block; width: 100%; height: 100%; border: 0; padding: 0; margin: 0; background-color: white; opacity: 0; filter: alpha(opacity=0)}
#MathJax_ZoomFrame {position: relative; display: inline-block; height: 0; width: 0}
#MathJax_ZoomEventTrap {position: absolute; left: 0; top: 0; z-index: 302; display: inline-block; border: 0; padding: 0; margin: 0; background-color: white; opacity: 0; filter: alpha(opacity=0)}
</style><style type="text/css">.MathJax_Preview {color: #888}
#MathJax_Message {position: fixed; left: 1em; bottom: 1.5em; background-color: #E6E6E6; border: 1px solid #959595; margin: 0px; padding: 2px 8px; z-index: 102; color: black; font-size: 80%; width: auto; white-space: nowrap}
#MathJax_MSIE_Frame {position: absolute; top: 0; left: 0; width: 0px; z-index: 101; border: 0px; margin: 0px; padding: 0px}
.MathJax_Error {color: #CC0000; font-style: italic}
</style><style type="text/css">.MJXp-script {font-size: .8em}
.MJXp-right {-webkit-transform-origin: right; -moz-transform-origin: right; -ms-transform-origin: right; -o-transform-origin: right; transform-origin: right}
.MJXp-bold {font-weight: bold}
.MJXp-italic {font-style: italic}
.MJXp-scr {font-family: MathJax_Script,'Times New Roman',Times,STIXGeneral,serif}
.MJXp-frak {font-family: MathJax_Fraktur,'Times New Roman',Times,STIXGeneral,serif}
.MJXp-sf {font-family: MathJax_SansSerif,'Times New Roman',Times,STIXGeneral,serif}
.MJXp-cal {font-family: MathJax_Caligraphic,'Times New Roman',Times,STIXGeneral,serif}
.MJXp-mono {font-family: MathJax_Typewriter,'Times New Roman',Times,STIXGeneral,serif}
.MJXp-largeop {font-size: 150%}
.MJXp-largeop.MJXp-int {vertical-align: -.2em}
.MJXp-math {display: inline-block; line-height: 1.2; text-indent: 0; font-family: 'Times New Roman',Times,STIXGeneral,serif; white-space: nowrap; border-collapse: collapse}
.MJXp-display {display: block; text-align: center; margin: 1em 0}
.MJXp-math span {display: inline-block}
.MJXp-box {display: block!important; text-align: center}
.MJXp-box:after {content: " "}
.MJXp-rule {display: block!important; margin-top: .1em}
.MJXp-char {display: block!important}
.MJXp-mo {margin: 0 .15em}
.MJXp-mfrac {margin: 0 .125em; vertical-align: .25em}
.MJXp-denom {display: inline-table!important; width: 100%}
.MJXp-denom > * {display: table-row!important}
.MJXp-surd {vertical-align: top}
.MJXp-surd > * {display: block!important}
.MJXp-script-box > *  {display: table!important; height: 50%}
.MJXp-script-box > * > * {display: table-cell!important; vertical-align: top}
.MJXp-script-box > *:last-child > * {vertical-align: bottom}
.MJXp-script-box > * > * > * {display: block!important}
.MJXp-mphantom {visibility: hidden}
.MJXp-munderover {display: inline-table!important}
.MJXp-over {display: inline-block!important; text-align: center}
.MJXp-over > * {display: block!important}
.MJXp-munderover > * {display: table-row!important}
.MJXp-mtable {vertical-align: .25em; margin: 0 .125em}
.MJXp-mtable > * {display: inline-table!important; vertical-align: middle}
.MJXp-mtr {display: table-row!important}
.MJXp-mtd {display: table-cell!important; text-align: center; padding: .5em 0 0 .5em}
.MJXp-mtr > .MJXp-mtd:first-child {padding-left: 0}
.MJXp-mtr:first-child > .MJXp-mtd {padding-top: 0}
.MJXp-mlabeledtr {display: table-row!important}
.MJXp-mlabeledtr > .MJXp-mtd:first-child {padding-left: 0}
.MJXp-mlabeledtr:first-child > .MJXp-mtd {padding-top: 0}
.MJXp-merror {background-color: #FFFF88; color: #CC0000; border: 1px solid #CC0000; padding: 1px 3px; font-style: normal; font-size: 90%}
.MJXp-scale0 {-webkit-transform: scaleX(.0); -moz-transform: scaleX(.0); -ms-transform: scaleX(.0); -o-transform: scaleX(.0); transform: scaleX(.0)}
.MJXp-scale1 {-webkit-transform: scaleX(.1); -moz-transform: scaleX(.1); -ms-transform: scaleX(.1); -o-transform: scaleX(.1); transform: scaleX(.1)}
.MJXp-scale2 {-webkit-transform: scaleX(.2); -moz-transform: scaleX(.2); -ms-transform: scaleX(.2); -o-transform: scaleX(.2); transform: scaleX(.2)}
.MJXp-scale3 {-webkit-transform: scaleX(.3); -moz-transform: scaleX(.3); -ms-transform: scaleX(.3); -o-transform: scaleX(.3); transform: scaleX(.3)}
.MJXp-scale4 {-webkit-transform: scaleX(.4); -moz-transform: scaleX(.4); -ms-transform: scaleX(.4); -o-transform: scaleX(.4); transform: scaleX(.4)}
.MJXp-scale5 {-webkit-transform: scaleX(.5); -moz-transform: scaleX(.5); -ms-transform: scaleX(.5); -o-transform: scaleX(.5); transform: scaleX(.5)}
.MJXp-scale6 {-webkit-transform: scaleX(.6); -moz-transform: scaleX(.6); -ms-transform: scaleX(.6); -o-transform: scaleX(.6); transform: scaleX(.6)}
.MJXp-scale7 {-webkit-transform: scaleX(.7); -moz-transform: scaleX(.7); -ms-transform: scaleX(.7); -o-transform: scaleX(.7); transform: scaleX(.7)}
.MJXp-scale8 {-webkit-transform: scaleX(.8); -moz-transform: scaleX(.8); -ms-transform: scaleX(.8); -o-transform: scaleX(.8); transform: scaleX(.8)}
.MJXp-scale9 {-webkit-transform: scaleX(.9); -moz-transform: scaleX(.9); -ms-transform: scaleX(.9); -o-transform: scaleX(.9); transform: scaleX(.9)}
.MathJax_PHTML .noError {vertical-align: ; font-size: 90%; text-align: left; color: black; padding: 1px 3px; border: 1px solid}
</style></head>
    <body class="global"><div id="MathJax_Message" style="display: none;"></div>
        <!--[if lt IE 7]>
            <p class="chromeframe">You are using an outdated browser. <a href="https://browsehappy.com/">Upgrade your browser today</a> or <a href="http://www.google.com/chromeframe/?redirect=true">install Google Chrome Frame</a> to better experience this site.</p>
        <![endif]-->

        <!-- This code is taken from http://twitter.github.com/bootstrap/examples/hero.html -->

        <nav class="navbar navbar-expand-lg navbar-dark p-0 px-4 fixed-top" style="background: #1d6890;" id="topbar">
            <div class="navbar-brand"><a href="index.html">
                <img src="img/spark-logo-rev.svg" width="141" height="72"></a><span class="version">3.5.1</span>
            </div>
            <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarCollapse" aria-controls="navbarCollapse" aria-expanded="false" aria-label="Toggle navigation">
                <span class="navbar-toggler-icon"></span>
            </button>
            <div class="collapse navbar-collapse" id="navbarCollapse">
                <ul class="navbar-nav me-auto">
                    <li class="nav-item"><a href="index.html" class="nav-link">Overview</a></li>

                    <li class="nav-item dropdown">
                        <a href="#" class="nav-link dropdown-toggle" id="navbarQuickStart" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Programming Guides</a>
                        <div class="dropdown-menu" aria-labelledby="navbarQuickStart">
                            <a class="dropdown-item" href="quick-start.html">Quick Start</a>
                            <a class="dropdown-item" href="rdd-programming-guide.html">RDDs, Accumulators, Broadcasts Vars</a>
                            <a class="dropdown-item" href="sql-programming-guide.html">SQL, DataFrames, and Datasets</a>
                            <a class="dropdown-item" href="structured-streaming-programming-guide.html">Structured Streaming</a>
                            <a class="dropdown-item" href="streaming-programming-guide.html">Spark Streaming (DStreams)</a>
                            <a class="dropdown-item" href="ml-guide.html">MLlib (Machine Learning)</a>
                            <a class="dropdown-item" href="graphx-programming-guide.html">GraphX (Graph Processing)</a>
                            <a class="dropdown-item" href="sparkr.html">SparkR (R on Spark)</a>
                            <a class="dropdown-item" href="api/python/getting_started/index.html">PySpark (Python on Spark)</a>
                        </div>
                    </li>

                    <li class="nav-item dropdown">
                        <a href="#" class="nav-link dropdown-toggle" id="navbarAPIDocs" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">API Docs</a>
                        <div class="dropdown-menu" aria-labelledby="navbarAPIDocs">
                            <a class="dropdown-item" href="api/scala/org/apache/spark/index.html">Scala</a>
                            <a class="dropdown-item" href="api/java/index.html">Java</a>
                            <a class="dropdown-item" href="api/python/index.html">Python</a>
                            <a class="dropdown-item" href="api/R/index.html">R</a>
                            <a class="dropdown-item" href="api/sql/index.html">SQL, Built-in Functions</a>
                        </div>
                    </li>

                    <li class="nav-item dropdown">
                        <a href="#" class="nav-link dropdown-toggle" id="navbarDeploying" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Deploying</a>
                        <div class="dropdown-menu" aria-labelledby="navbarDeploying">
                            <a class="dropdown-item" href="cluster-overview.html">Overview</a>
                            <a class="dropdown-item" href="submitting-applications.html">Submitting Applications</a>
                            <div class="dropdown-divider"></div>
                            <a class="dropdown-item" href="spark-standalone.html">Spark Standalone</a>
                            <a class="dropdown-item" href="running-on-mesos.html">Mesos</a>
                            <a class="dropdown-item" href="running-on-yarn.html">YARN</a>
                            <a class="dropdown-item" href="running-on-kubernetes.html">Kubernetes</a>
                        </div>
                    </li>

                    <li class="nav-item dropdown">
                        <a href="#" class="nav-link dropdown-toggle" id="navbarMore" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">More</a>
                        <div class="dropdown-menu" aria-labelledby="navbarMore">
                            <a class="dropdown-item" href="configuration.html">Configuration</a>
                            <a class="dropdown-item" href="monitoring.html">Monitoring</a>
                            <a class="dropdown-item" href="tuning.html">Tuning Guide</a>
                            <a class="dropdown-item" href="job-scheduling.html">Job Scheduling</a>
                            <a class="dropdown-item" href="security.html">Security</a>
                            <a class="dropdown-item" href="hardware-provisioning.html">Hardware Provisioning</a>
                            <a class="dropdown-item" href="migration-guide.html">Migration Guide</a>
                            <div class="dropdown-divider"></div>
                            <a class="dropdown-item" href="building-spark.html">Building Spark</a>
                            <a class="dropdown-item" href="https://spark.apache.org/contributing.html">Contributing to Spark</a>
                            <a class="dropdown-item" href="https://spark.apache.org/third-party-projects.html">Third Party Projects</a>
                        </div>
                    </li>

                    <li class="nav-item">
                        
  <form novalidate="novalidate" onsubmit="return false;" class="searchbox">
    <div role="search" class="searchbox__wrapper">
      <span class="algolia-autocomplete" style="position: relative; display: inline-block; direction: ltr;"><input id="docsearch" type="search" name="search" placeholder="Search the docs" autocomplete="off" required="required" class="searchbox__input ds-input" spellcheck="false" role="combobox" aria-autocomplete="list" aria-expanded="false" aria-label="search input" aria-owns="algolia-autocomplete-listbox-0" dir="auto" style="position: relative; vertical-align: top;"><pre aria-hidden="true" style="position: absolute; visibility: hidden; white-space: pre; font-family: &quot;DM Sans&quot;, sans-serif; font-size: 12px; font-style: normal; font-variant: normal; font-weight: 500; word-spacing: 0px; letter-spacing: normal; text-indent: 0px; text-rendering: auto; text-transform: none;"></pre><span class="ds-dropdown-menu" role="listbox" id="algolia-autocomplete-listbox-0" style="position: absolute; top: 100%; z-index: 100; display: none; left: 0px; right: auto;"><div class="ds-dataset-1"></div></span></span>
      <button type="submit" title="Submit your search query." class="searchbox__submit">
        <svg width="12" height="12" role="img" aria-label="Search">
          <use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#sbx-icon-search-13"></use>
        </svg>
      </button>
      <button type="reset" title="Clear the search query." class="searchbox__reset hide">
        <svg width="12" height="12" role="img" aria-label="Reset">
          <use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#sbx-icon-clear-3"></use>
        </svg>
      </button>
    </div>
</form>

<div class="svg-icons" style="height: 0; width: 0; position: absolute; visibility: hidden">
  <svg xmlns="http://www.w3.org/2000/svg">
    <symbol id="sbx-icon-clear-3" viewBox="0 0 40 40"><path d="M16.228 20L1.886 5.657 0 3.772 3.772 0l1.885 1.886L20 16.228 34.343 1.886 36.228 0 40 3.772l-1.886 1.885L23.772 20l14.342 14.343L40 36.228 36.228 40l-1.885-1.886L20 23.772 5.657 38.114 3.772 40 0 36.228l1.886-1.885L16.228 20z" fill-rule="evenodd"></path></symbol>
    <symbol id="sbx-icon-search-13" viewBox="0 0 40 40"><path d="M26.806 29.012a16.312 16.312 0 0 1-10.427 3.746C7.332 32.758 0 25.425 0 16.378 0 7.334 7.333 0 16.38 0c9.045 0 16.378 7.333 16.378 16.38 0 3.96-1.406 7.593-3.746 10.426L39.547 37.34c.607.608.61 1.59-.004 2.203a1.56 1.56 0 0 1-2.202.004L26.807 29.012zm-10.427.627c7.322 0 13.26-5.938 13.26-13.26 0-7.324-5.938-13.26-13.26-13.26-7.324 0-13.26 5.936-13.26 13.26 0 7.322 5.936 13.26 13.26 13.26z" fill-rule="evenodd"></path></symbol>
  </svg>
</div>
  
                    </li>
                </ul>
                <!--<span class="navbar-text navbar-right"><span class="version-text">v3.5.1</span></span>-->
            </div>
        </nav>

        

        <div class="container">

            
                
                    <div class="left-menu-wrapper">
    <div class="left-menu">
        <h3 id="mllib-main-guide"><a href="ml-guide.html">MLlib: Main Guide</a><a class="anchorjs-link " href="#mllib-main-guide" aria-label="Anchor link for: mllib main guide" data-anchorjs-icon="" style="font-family: anchorjs-icons; font-style: normal; font-variant: normal; font-weight: normal; padding-left: 0.375em;"></a></h3>
        
<ul>

    <li>
        <a href="ml-statistics.html">
            
                Basic statistics
            
        </a>
    </li>
    
    

    <li>
        <a href="ml-datasource.html">
            
                Data sources
            
        </a>
    </li>
    
    

    <li>
        <a href="ml-pipeline.html">
            
                Pipelines
            
        </a>
    </li>
    
    

    <li>
        <a href="ml-features.html">
            
                Extracting, transforming and selecting features
            
        </a>
    </li>
    
    

    <li>
        <a href="ml-classification-regression.html">
            
                Classification and Regression
            
        </a>
    </li>
    
    

    <li>
        <a href="ml-clustering.html">
            
                Clustering
            
        </a>
    </li>
    
    

    <li>
        <a href="ml-collaborative-filtering.html">
            
                Collaborative filtering
            
        </a>
    </li>
    
    

    <li>
        <a href="ml-frequent-pattern-mining.html">
            
                Frequent Pattern Mining
            
        </a>
    </li>
    
    

    <li>
        <a href="ml-tuning.html">
            
                Model selection and tuning
            
        </a>
    </li>
    
    

    <li>
        <a href="ml-advanced.html">
            
                Advanced topics
            
        </a>
    </li>
    
    

</ul>

        <h3 id="mllib-rdd-based-api-guide"><a href="mllib-guide.html">MLlib: RDD-based API Guide</a><a class="anchorjs-link " href="#mllib-rdd-based-api-guide" aria-label="Anchor link for: mllib rdd based api guide" data-anchorjs-icon="" style="font-family: anchorjs-icons; font-style: normal; font-variant: normal; font-weight: normal; padding-left: 0.375em;"></a></h3>
        
<ul>

    <li>
        <a href="mllib-data-types.html">
            
                Data types
            
        </a>
    </li>
    
    

    <li>
        <a href="mllib-statistics.html">
            
                Basic statistics
            
        </a>
    </li>
    
    

    <li>
        <a href="mllib-classification-regression.html">
            
                Classification and regression
            
        </a>
    </li>
    
    

    <li>
        <a href="mllib-collaborative-filtering.html">
            
                Collaborative filtering
            
        </a>
    </li>
    
    

    <li>
        <a href="mllib-clustering.html">
            
                Clustering
            
        </a>
    </li>
    
    

    <li>
        <a href="mllib-dimensionality-reduction.html">
            
                Dimensionality reduction
            
        </a>
    </li>
    
    

    <li>
        <a href="mllib-feature-extraction.html">
            
                Feature extraction and transformation
            
        </a>
    </li>
    
    

    <li>
        <a href="mllib-frequent-pattern-mining.html">
            
                Frequent pattern mining
            
        </a>
    </li>
    
    

    <li>
        <a href="mllib-evaluation-metrics.html">
            
                Evaluation metrics
            
        </a>
    </li>
    
    

    <li>
        <a href="mllib-pmml-model-export.html">
            
                PMML model export
            
        </a>
    </li>
    
    

    <li>
        <a href="mllib-optimization.html">
            
                Optimization (developer)
            
        </a>
    </li>
    
    

</ul>

    </div>
</div>
                
                <input id="nav-trigger" class="nav-trigger" checked="" type="checkbox">
                <label for="nav-trigger"></label>
                <div class="content-with-sidebar mr-3" id="content">
                    
                        <h1 class="title" id="machine-learning-library-mllib-guide">Machine Learning Library (MLlib) Guide<a class="anchorjs-link " href="#machine-learning-library-mllib-guide" aria-label="Anchor link for: machine learning library mllib guide" data-anchorjs-icon="" style="font-family: anchorjs-icons; font-style: normal; font-variant: normal; font-weight: normal; padding-left: 0.375em;"></a></h1>
                    

                    <p>MLlib is Spark’s machine learning (ML) library.
Its goal is to make practical machine learning scalable and easy.
At a high level, it provides tools such as:</p>

<ul>
  <li>ML Algorithms: common learning algorithms such as classification, regression, clustering, and collaborative filtering</li>
  <li>Featurization: feature extraction, transformation, dimensionality reduction, and selection</li>
  <li>Pipelines: tools for constructing, evaluating, and tuning ML Pipelines</li>
  <li>Persistence: saving and load algorithms, models, and Pipelines</li>
  <li>Utilities: linear algebra, statistics, data handling, etc.</li>
</ul>

<h1 id="announcement-dataframe-based-api-is-primary-api">Announcement: DataFrame-based API is primary API<a class="anchorjs-link " href="#announcement-dataframe-based-api-is-primary-api" aria-label="Anchor link for: announcement dataframe based api is primary api" data-anchorjs-icon="" style="font-family: anchorjs-icons; font-style: normal; font-variant: normal; font-weight: normal; padding-left: 0.375em;"></a></h1>

<p><strong>The MLlib RDD-based API is now in maintenance mode.</strong></p>

<p>As of Spark 2.0, the <a href="rdd-programming-guide.html#resilient-distributed-datasets-rdds">RDD</a>-based APIs in the <code class="language-plaintext highlighter-rouge">spark.mllib</code> package have entered maintenance mode.
The primary Machine Learning API for Spark is now the <a href="sql-programming-guide.html">DataFrame</a>-based API in the <code class="language-plaintext highlighter-rouge">spark.ml</code> package.</p>

<p><em>What are the implications?</em></p>

<ul>
  <li>MLlib will still support the RDD-based API in <code class="language-plaintext highlighter-rouge">spark.mllib</code> with bug fixes.</li>
  <li>MLlib will not add new features to the RDD-based API.</li>
  <li>In the Spark 2.x releases, MLlib will add features to the DataFrames-based API to reach feature parity with the RDD-based API.</li>
</ul>

<p><em>Why is MLlib switching to the DataFrame-based API?</em></p>

<ul>
  <li>DataFrames provide a more user-friendly API than RDDs.  The many benefits of DataFrames include Spark Datasources, SQL/DataFrame queries, Tungsten and Catalyst optimizations, and uniform APIs across languages.</li>
  <li>The DataFrame-based API for MLlib provides a uniform API across ML algorithms and across multiple languages.</li>
  <li>DataFrames facilitate practical ML Pipelines, particularly feature transformations.  See the <a href="ml-pipeline.html">Pipelines guide</a> for details.</li>
</ul>

<p><em>What is “Spark ML”?</em></p>

<ul>
  <li>“Spark ML” is not an official name but occasionally used to refer to the MLlib DataFrame-based API.
This is majorly due to the <code class="language-plaintext highlighter-rouge">org.apache.spark.ml</code> Scala package name used by the DataFrame-based API, 
and the “Spark ML Pipelines” term we used initially to emphasize the pipeline concept.</li>
</ul>

<p><em>Is MLlib deprecated?</em></p>

<ul>
  <li>No. MLlib includes both the RDD-based API and the DataFrame-based API.
The RDD-based API is now in maintenance mode.
But neither API is deprecated, nor MLlib as a whole.</li>
</ul>

<h1 id="dependencies">Dependencies<a class="anchorjs-link " href="#dependencies" aria-label="Anchor link for: dependencies" data-anchorjs-icon="" style="font-family: anchorjs-icons; font-style: normal; font-variant: normal; font-weight: normal; padding-left: 0.375em;"></a></h1>

<p>MLlib uses linear algebra packages <a href="http://www.scalanlp.org/">Breeze</a> and <a href="https://github.com/luhenry/netlib">dev.ludovic.netlib</a> for optimised numerical processing<sup id="fnref:1" role="doc-noteref"><a href="#fn:1" class="footnote" rel="footnote">1</a></sup>. Those packages may call native acceleration libraries such as <a href="https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html">Intel MKL</a> or <a href="http://www.openblas.net">OpenBLAS</a> if they are available as system libraries or in runtime library paths.</p>

<p>However, native acceleration libraries can’t be distributed with Spark. See <a href="ml-linalg-guide.html">MLlib Linear Algebra Acceleration Guide</a> for how to enable accelerated linear algebra processing. If accelerated native libraries are not enabled, you will see a warning message like below and a pure JVM implementation will be used instead:</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>WARNING: Failed to load implementation from:dev.ludovic.netlib.blas.JNIBLAS
</code></pre></div></div>

<p>To use MLlib in Python, you will need <a href="http://www.numpy.org">NumPy</a> version 1.4 or newer.</p>

<h1 id="highlights-in-30">Highlights in 3.0<a class="anchorjs-link " href="#highlights-in-30" aria-label="Anchor link for: highlights in 30" data-anchorjs-icon="" style="font-family: anchorjs-icons; font-style: normal; font-variant: normal; font-weight: normal; padding-left: 0.375em;"></a></h1>

<p>The list below highlights some of the new features and enhancements added to MLlib in the <code class="language-plaintext highlighter-rouge">3.0</code>
release of Spark:</p>

<ul>
  <li>Multiple columns support was added to <code class="language-plaintext highlighter-rouge">Binarizer</code> (<a href="https://issues.apache.org/jira/browse/SPARK-23578">SPARK-23578</a>), <code class="language-plaintext highlighter-rouge">StringIndexer</code> (<a href="https://issues.apache.org/jira/browse/SPARK-11215">SPARK-11215</a>), <code class="language-plaintext highlighter-rouge">StopWordsRemover</code> (<a href="https://issues.apache.org/jira/browse/SPARK-29808">SPARK-29808</a>) and PySpark <code class="language-plaintext highlighter-rouge">QuantileDiscretizer</code> (<a href="https://issues.apache.org/jira/browse/SPARK-22796">SPARK-22796</a>).</li>
  <li>Tree-Based Feature Transformation was added
(<a href="https://issues.apache.org/jira/browse/SPARK-13677">SPARK-13677</a>).</li>
  <li>Two new evaluators <code class="language-plaintext highlighter-rouge">MultilabelClassificationEvaluator</code> (<a href="https://issues.apache.org/jira/browse/SPARK-16692">SPARK-16692</a>) and <code class="language-plaintext highlighter-rouge">RankingEvaluator</code> (<a href="https://issues.apache.org/jira/browse/SPARK-28045">SPARK-28045</a>) were added.</li>
  <li>Sample weights support was added in <code class="language-plaintext highlighter-rouge">DecisionTreeClassifier/Regressor</code> (<a href="https://issues.apache.org/jira/browse/SPARK-19591">SPARK-19591</a>), <code class="language-plaintext highlighter-rouge">RandomForestClassifier/Regressor</code> (<a href="https://issues.apache.org/jira/browse/SPARK-9478">SPARK-9478</a>), <code class="language-plaintext highlighter-rouge">GBTClassifier/Regressor</code> (<a href="https://issues.apache.org/jira/browse/SPARK-9612">SPARK-9612</a>),  <code class="language-plaintext highlighter-rouge">MulticlassClassificationEvaluator</code> (<a href="https://issues.apache.org/jira/browse/SPARK-24101">SPARK-24101</a>), <code class="language-plaintext highlighter-rouge">RegressionEvaluator</code> (<a href="https://issues.apache.org/jira/browse/SPARK-24102">SPARK-24102</a>), <code class="language-plaintext highlighter-rouge">BinaryClassificationEvaluator</code> (<a href="https://issues.apache.org/jira/browse/SPARK-24103">SPARK-24103</a>), <code class="language-plaintext highlighter-rouge">BisectingKMeans</code> (<a href="https://issues.apache.org/jira/browse/SPARK-30351">SPARK-30351</a>), <code class="language-plaintext highlighter-rouge">KMeans</code> (<a href="https://issues.apache.org/jira/browse/SPARK-29967">SPARK-29967</a>) and <code class="language-plaintext highlighter-rouge">GaussianMixture</code> (<a href="https://issues.apache.org/jira/browse/SPARK-30102">SPARK-30102</a>).</li>
  <li>R API for <code class="language-plaintext highlighter-rouge">PowerIterationClustering</code> was added
(<a href="https://issues.apache.org/jira/browse/SPARK-19827">SPARK-19827</a>).</li>
  <li>Added Spark ML listener for tracking ML pipeline status
(<a href="https://issues.apache.org/jira/browse/SPARK-23674">SPARK-23674</a>).</li>
  <li>Fit with validation set was added to Gradient Boosted Trees in Python
(<a href="https://issues.apache.org/jira/browse/SPARK-24333">SPARK-24333</a>).</li>
  <li><a href="ml-features.html#robustscaler"><code class="language-plaintext highlighter-rouge">RobustScaler</code></a> transformer was added
(<a href="https://issues.apache.org/jira/browse/SPARK-28399">SPARK-28399</a>).</li>
  <li><a href="ml-classification-regression.html#factorization-machines"><code class="language-plaintext highlighter-rouge">Factorization Machines</code></a> classifier and regressor were added
(<a href="https://issues.apache.org/jira/browse/SPARK-29224">SPARK-29224</a>).</li>
  <li>Gaussian Naive Bayes Classifier (<a href="https://issues.apache.org/jira/browse/SPARK-16872">SPARK-16872</a>) and Complement Naive Bayes Classifier (<a href="https://issues.apache.org/jira/browse/SPARK-29942">SPARK-29942</a>) were added.</li>
  <li>ML function parity between Scala and Python
(<a href="https://issues.apache.org/jira/browse/SPARK-28958">SPARK-28958</a>).</li>
  <li><code class="language-plaintext highlighter-rouge">predictRaw</code> is made public in all the Classification models. <code class="language-plaintext highlighter-rouge">predictProbability</code> is made public in all the Classification models except <code class="language-plaintext highlighter-rouge">LinearSVCModel</code>
(<a href="https://issues.apache.org/jira/browse/SPARK-30358">SPARK-30358</a>).</li>
</ul>

<h1 id="migration-guide">Migration Guide<a class="anchorjs-link " href="#migration-guide" aria-label="Anchor link for: migration guide" data-anchorjs-icon="" style="font-family: anchorjs-icons; font-style: normal; font-variant: normal; font-weight: normal; padding-left: 0.375em;"></a></h1>

<p>The migration guide is now archived <a href="ml-migration-guide.html">on this page</a>.</p>

<div class="footnotes" role="doc-endnotes">
  <ol>
    <li id="fn:1" role="doc-endnote">
      <p>To learn more about the benefits and background of system optimised natives, you may wish to
watch Sam Halliday’s ScalaX talk on <a href="http://fommil.github.io/scalax14/#/">High Performance Linear Algebra in Scala</a>.&nbsp;<a href="#fnref:1" class="reversefootnote" role="doc-backlink">↩</a></p>
    </li>
  </ol>
</div>


                </div>
            
             <!-- /container -->
        </div>

        <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
        <script src="https://code.jquery.com/jquery.js"></script>

        <script src="js/vendor/anchor.min.js"></script>
        <script src="js/main.js"></script>

        <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.js"></script>
        <script type="text/javascript">
            // DocSearch is entirely free and automated. DocSearch is built in two parts:
            // 1. a crawler which we run on our own infrastructure every 24 hours. It follows every link
            //    in your website and extract content from every page it traverses. It then pushes this
            //    content to an Algolia index.
            // 2. a JavaScript snippet to be inserted in your website that will bind this Algolia index
            //    to your search input and display its results in a dropdown UI. If you want to find more
            //    details on how works DocSearch, check the docs of DocSearch.
            docsearch({
    apiKey: 'd62f962a82bc9abb53471cb7b89da35e',
    appId: 'RAI69RXRSK',
    indexName: 'apache_spark',
    inputSelector: '#docsearch-input',
    enhancedSearchInput: true,
    algoliaOptions: {
      'facetFilters': ["version:3.5.1"]
    },
    debug: false // Set debug to true if you want to inspect the dropdown
});

        </script>

        <!-- MathJax Section -->
        <script type="text/x-mathjax-config;executed=true">
            MathJax.Hub.Config({
                TeX: { equationNumbers: { autoNumber: "AMS" } }
            });
        </script>
        <script>
            // Note that we load MathJax this way to work with local file (file://), HTTP and HTTPS.
            // We could use "//cdn.mathjax...", but that won't support "file://".
            (function(d, script) {
                script = d.createElement('script');
                script.type = 'text/javascript';
                script.async = true;
                script.onload = function(){
                    MathJax.Hub.Config({
                        tex2jax: {
                            inlineMath: [ ["$", "$"], ["\\\\(","\\\\)"] ],
                            displayMath: [ ["$$","$$"], ["\\[", "\\]"] ],
                            processEscapes: true,
                            skipTags: ['script', 'noscript', 'style', 'textarea', 'pre']
                        }
                    });
                };
                script.src = ('https:' == document.location.protocol ? 'https://' : 'http://') +
                    'cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js' +
                    '?config=TeX-AMS-MML_HTMLorMML';
                d.getElementsByTagName('head')[0].appendChild(script);
            }(document));
        </script>
    

</body></html>