- Scan ID:
- b30813d2-b465-4841-bf58-517ea5d6d2a3Finished
- Submitted URL:
- https://indeed.techRedirected
- Report Finished:
Links · 79 found
The outgoing links identified from the page
| Link | Text |
|---|---|
| http://opensource.indeedeng.io/ | Open Source |
| http://indeed.jobs | Work at Indeed |
| https://www.indeed.com/about?isid=press_us&ikw=press_us_press%2Freleases%2Faward-winning-actress-viola-davis-to-keynote-indeed-futureworks-2023_textlink_https%3A%2F%2Fwww.indeed.com%2Fabout | #1 job site |
| https://www.indeed.com/about/methodology | over 350M+ unique visitors |
| https://en.wikipedia.org/wiki/Hudson_(software) | Hudson |
| https://aws.amazon.com/ec2/ | AWS EC2 |
| https://kubernetes.io/ | Kubernetes |
| https://tag-app-delivery.cncf.io/whitepapers/platforms/ | Golden Path |
| https://gitlab.com/gitlab-org/gitlab/-/issues/365101 | issue |
| https://gitlab.com/gitlab-org/gitlab/-/issues/335138 | bug |
JavaScript Variables · 36 found
Global JavaScript variables loaded on the window object of a page, are variables declared outside of functions and accessible from anywhere in the code within the current scope
| Name | Type |
|---|---|
| onbeforetoggle | object |
| documentPictureInPicture | object |
| onscrollend | object |
| dataLayer | object |
| _wpemojiSettings | object |
| $ | function |
| jQuery | function |
| WPMLLanguageSwitcherDropdownClick | object |
| scriptUrl | object |
| ttPolicy | object |
Console log messages · 0 found
Messages logged to the web console
HTML
The raw HTML body of the page
<!DOCTYPE html><html lang="en-us"><head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Blog - Indeed Engineering Blog</title>
<link rel="profile" href="http://gmpg.org/xfn/11">
<link rel="pingback" href="https://engineering.indeedblog.com/xmlrpc.php">
<link rel="shortcut icon" href="https://engblogs.wpenginepowered.com/wp-content/themes/indeedblog/favicon.ico">
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
<!--[if lt IE 9]>
<script src="https://engblogs.wpenginepowered.com/wp-content/themes/indeedblog/dist/js/html5shiv.js"></script>
<script src="https://engblogs.wpenginepowered.com/wp-content/themes/indeedblog/dist/js/respond.min.js"></script>
<![endif]-->
<!--[if lt IE 8]>
<link href="https://engblogs.wpenginepowered.com/wp-content/themes/indeedblog/dist/css/bootstrap-ie7.css" rel="stylesheet">
<![endif]-->
<meta name="robots" content="index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1">
<link rel="alternate" hreflang="en-us" href="https://engineering.indeedblog.com/blog/">
<link rel="alternate" hreflang="ja" href="https://jp.engineering.indeedblog.com/blog/">
<link rel="alternate" hreflang="x-default" href="https://engineering.indeedblog.com/blog/">
<!-- Start Google Analytics tracking code here -->
<script type="text/javascript" async="" src="//munchkin.marketo.net/163/munchkin.js"></script><script type="text/javascript" async="" src="https://www.google-analytics.com/analytics.js"></script><script type="text/javascript" async="" src="https://www.googletagmanager.com/gtag/js?id=G-JYBL0QNF55&l=dataLayer&cx=c"></script><script type="text/javascript" id="www-widgetapi-script" src="https://www.youtube.com/s/player/8579e400/www-widgetapi.vflset/www-widgetapi.js" async=""></script><script async="" src="https://www.googletagmanager.com/gtm.js?id=GTM-K8RKLN"></script><script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer','GTM-K8RKLN');
</script>
<!-- END Google Analytics tracking code here -->
<!-- This site is optimized with the Yoast SEO plugin v23.4 - https://yoast.com/wordpress/plugins/seo/ -->
<link rel="next" href="https://engineering.indeedblog.com/blog/page/2/">
<meta property="og:locale" content="en_US">
<meta property="og:type" content="article">
<meta property="og:title" content="Blog - Indeed Engineering Blog">
<meta property="og:url" content="https://engineering.indeedblog.com/blog/">
<meta property="og:site_name" content="Indeed Engineering Blog">
<meta property="og:image" content="https://engineering.indeedblog.com/wp-content/uploads/2014/04/indeedengineeringblog.jpg">
<meta property="og:image:width" content="1200">
<meta property="og:image:height" content="630">
<meta property="og:image:type" content="image/jpeg">
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:site" content="@IndeedEng">
<script type="application/ld+json" class="yoast-schema-graph">{"@context":"https://schema.org","@graph":[{"@type":["WebPage","CollectionPage"],"@id":"https://engineering.indeedblog.com/blog/","url":"https://engineering.indeedblog.com/blog/","name":"Blog - Indeed Engineering Blog","isPartOf":{"@id":"https://engineering.indeedblog.com/#website"},"datePublished":"2013-12-12T16:11:27+00:00","dateModified":"2021-02-09T21:21:49+00:00","breadcrumb":{"@id":"https://engineering.indeedblog.com/blog/#breadcrumb"},"inLanguage":"en-us"},{"@type":"BreadcrumbList","@id":"https://engineering.indeedblog.com/blog/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https://engineering.indeedblog.com/"},{"@type":"ListItem","position":2,"name":"Blog"}]},{"@type":"WebSite","@id":"https://engineering.indeedblog.com/#website","url":"https://engineering.indeedblog.com/","name":"Indeed Engineering Blog","description":"We help people get jobs.","publisher":{"@id":"https://engineering.indeedblog.com/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https://engineering.indeedblog.com/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-us"},{"@type":"Organization","@id":"https://engineering.indeedblog.com/#organization","name":"Indeed.com","url":"https://engineering.indeedblog.com/","logo":{"@type":"ImageObject","inLanguage":"en-us","@id":"https://engineering.indeedblog.com/#/schema/logo/image/","url":"https://engineering.indeedblog.com/wp-content/uploads/2015/04/1024x1024-553d53a6v1_site_icon.png","contentUrl":"https://engineering.indeedblog.com/wp-content/uploads/2015/04/1024x1024-553d53a6v1_site_icon.png","width":512,"height":512,"caption":"Indeed.com"},"image":{"@id":"https://engineering.indeedblog.com/#/schema/logo/image/"},"sameAs":["https://www.facebook.com/Indeed","https://x.com/IndeedEng","https://www.youtube.com/user/IndeedEng"]}]}</script>
<!-- / Yoast SEO plugin. -->
<link rel="dns-prefetch" href="//cpwebassets.codepen.io">
<link rel="dns-prefetch" href="//www.youtube.com">
<link rel="alternate" type="application/rss+xml" title="Indeed Engineering Blog » Feed" href="https://engineering.indeedblog.com/feed/">
<link rel="alternate" type="application/rss+xml" title="Indeed Engineering Blog » Comments Feed" href="https://engineering.indeedblog.com/comments/feed/">
<script type="text/javascript">
/* <![CDATA[ */
window._wpemojiSettings = {"baseUrl":"https:\/\/s.w.org\/images\/core\/emoji\/15.0.3\/72x72\/","ext":".png","svgUrl":"https:\/\/s.w.org\/images\/core\/emoji\/15.0.3\/svg\/","svgExt":".svg","source":{"concatemoji":"https:\/\/engineering.indeedblog.com\/wp-includes\/js\/wp-emoji-release.min.js?ver=6.6.1"}};
/*! This file is auto-generated */
!function(i,n){var o,s,e;function c(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(o,JSON.stringify(t))}catch(e){}}function p(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data),r=(e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0),new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data));return t.every(function(e,t){return e===r[t]})}function u(e,t,n){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\uddfa\ud83c\uddf3","\ud83c\uddfa\u200b\ud83c\uddf3")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!n(e,"\ud83d\udc26\u200d\u2b1b","\ud83d\udc26\u200b\u2b1b")}return!1}function f(e,t,n){var r="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):i.createElement("canvas"),a=r.getContext("2d",{willReadFrequently:!0}),o=(a.textBaseline="top",a.font="600 32px Arial",{});return e.forEach(function(e){o[e]=t(a,e,n)}),o}function t(e){var t=i.createElement("script");t.src=e,t.defer=!0,i.head.appendChild(t)}"undefined"!=typeof Promise&&(o="wpEmojiSettingsSupports",s=["flag","emoji"],n.supports={everything:!0,everythingExceptFlag:!0},e=new Promise(function(e){i.addEventListener("DOMContentLoaded",e,{once:!0})}),new Promise(function(t){var n=function(){try{var e=JSON.parse(sessionStorage.getItem(o));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(s),u.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"}),a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=function(e){c(n=e.data),a.terminate(),t(n)})}catch(e){}c(n=f(s,u,p))}t(n)}).then(function(e){for(var t in e)n.supports[t]=e[t],n.supports.everything=n.supports.everything&&n.supports[t],"flag"!==t&&(n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&n.supports[t]);n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&!n.supports.flag,n.DOMReady=!1,n.readyCallback=function(){n.DOMReady=!0}}).then(function(){return e}).then(function(){var e;n.supports.everything||(n.readyCallback(),(e=n.source||{}).concatemoji?t(e.concatemoji):e.wpemoji&&e.twemoji&&(t(e.twemoji),t(e.wpemoji)))}))}((window,document),window._wpemojiSettings);
/* ]]> */
</script>
<style id="wp-emoji-styles-inline-css" type="text/css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
</style>
<link rel="stylesheet" id="wp-block-library-css" href="https://engblogs.wpenginepowered.com/wp-includes/css/dist/block-library/style.min.css?ver=6.6.1" type="text/css" media="all">
<style id="co-authors-plus-coauthors-style-inline-css" type="text/css">
.wp-block-co-authors-plus-coauthors.is-layout-flow [class*=wp-block-co-authors-plus]{display:inline}
</style>
<style id="co-authors-plus-avatar-style-inline-css" type="text/css">
.wp-block-co-authors-plus-avatar :where(img){height:auto;max-width:100%;vertical-align:bottom}.wp-block-co-authors-plus-coauthors.is-layout-flow .wp-block-co-authors-plus-avatar :where(img){vertical-align:middle}.wp-block-co-authors-plus-avatar:is(.alignleft,.alignright){display:table}.wp-block-co-authors-plus-avatar.aligncenter{display:table;margin-inline:auto}
</style>
<style id="co-authors-plus-image-style-inline-css" type="text/css">
.wp-block-co-authors-plus-image{margin-bottom:0}.wp-block-co-authors-plus-image :where(img){height:auto;max-width:100%;vertical-align:bottom}.wp-block-co-authors-plus-coauthors.is-layout-flow .wp-block-co-authors-plus-image :where(img){vertical-align:middle}.wp-block-co-authors-plus-image:is(.alignfull,.alignwide) :where(img){width:100%}.wp-block-co-authors-plus-image:is(.alignleft,.alignright){display:table}.wp-block-co-authors-plus-image.aligncenter{display:table;margin-inline:auto}
</style>
<style id="classic-theme-styles-inline-css" type="text/css">
/*! This file is auto-generated */
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
</style>
<style id="global-styles-inline-css" type="text/css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgba(6,147,227,1) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgba(252,185,0,1) 0%,rgba(255,105,0,1) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgba(255,105,0,1) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgba(255, 255, 255, 1), 6px 6px rgba(0, 0, 0, 1);--wp--preset--shadow--crisp: 6px 6px 0px rgba(0, 0, 0, 1);}:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}:where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;}:where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
:where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;}
:where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;}
:root :where(.wp-block-pullquote){font-size: 1.5em;line-height: 1.6;}
</style>
<link rel="stylesheet" id="style.css-css" href="https://engblogs.wpenginepowered.com/wp-content/plugins/indeed-sidebar-widgets/style.css?ver=6.6.1" type="text/css" media="all">
<link rel="stylesheet" id="wpml-legacy-dropdown-click-0-css" href="https://engblogs.wpenginepowered.com/wp-content/plugins/sitepress-multilingual-cms/templates/language-switchers/legacy-dropdown-click/style.min.css?ver=1" type="text/css" media="all">
<style id="wpml-legacy-dropdown-click-0-inline-css" type="text/css">
.wpml-ls-statics-shortcode_actions, .wpml-ls-statics-shortcode_actions .wpml-ls-sub-menu, .wpml-ls-statics-shortcode_actions a {border-color:#cdcdcd;}.wpml-ls-statics-shortcode_actions a, .wpml-ls-statics-shortcode_actions .wpml-ls-sub-menu a, .wpml-ls-statics-shortcode_actions .wpml-ls-sub-menu a:link, .wpml-ls-statics-shortcode_actions li:not(.wpml-ls-current-language) .wpml-ls-link, .wpml-ls-statics-shortcode_actions li:not(.wpml-ls-current-language) .wpml-ls-link:link {color:#444444;background-color:#ffffff;}.wpml-ls-statics-shortcode_actions a, .wpml-ls-statics-shortcode_actions .wpml-ls-sub-menu a:hover,.wpml-ls-statics-shortcode_actions .wpml-ls-sub-menu a:focus, .wpml-ls-statics-shortcode_actions .wpml-ls-sub-menu a:link:hover, .wpml-ls-statics-shortcode_actions .wpml-ls-sub-menu a:link:focus {color:#000000;background-color:#eeeeee;}.wpml-ls-statics-shortcode_actions .wpml-ls-current-language > a {color:#444444;background-color:#ffffff;}.wpml-ls-statics-shortcode_actions .wpml-ls-current-language:hover>a, .wpml-ls-statics-shortcode_actions .wpml-ls-current-language>a:focus {color:#000000;background-color:#eeeeee;}
</style>
<link rel="stylesheet" id="cms-navigation-style-base-css" href="https://engblogs.wpenginepowered.com/wp-content/plugins/wpml-cms-nav/res/css/cms-navigation-base.css?ver=1.5.5" type="text/css" media="screen">
<link rel="stylesheet" id="cms-navigation-style-css" href="https://engblogs.wpenginepowered.com/wp-content/plugins/wpml-cms-nav/res/css/cms-navigation.css?ver=1.5.5" type="text/css" media="screen">
<link rel="stylesheet" id="parent-style-css" href="https://engblogs.wpenginepowered.com/wp-content/themes/indeedblog/style.css?ver=1.5.6" type="text/css" media="all">
<link rel="stylesheet" id="bootstrap-style-css" href="https://engblogs.wpenginepowered.com/wp-content/themes/indeedblog/dist/css/bootstrap.min.css?ver=2.6.0" type="text/css" media="all">
<link rel="stylesheet" id="indeedblog-style-css" href="https://engblogs.wpenginepowered.com/wp-content/themes/indeedblog/style.min.css?ver=2.6.0" type="text/css" media="all">
<link rel="stylesheet" id="style_login_widget-css" href="https://engblogs.wpenginepowered.com/wp-content/plugins/miniorange-oauth-oidc-single-sign-on/resources/css/style_login_widget.css?ver=6.6.1" type="text/css" media="all">
<script type="text/javascript" src="https://engblogs.wpenginepowered.com/wp-content/themes/indeedblog/dist/js/jquery-1-11-1.min.js?ver=2.6.0" id="jquery-js"></script>
<script type="text/javascript" src="https://engblogs.wpenginepowered.com/wp-content/plugins/sitepress-multilingual-cms/templates/language-switchers/legacy-dropdown-click/script.min.js?ver=1" id="wpml-legacy-dropdown-click-0-js"></script>
<script type="text/javascript" src="https://engblogs.wpenginepowered.com/wp-content/themes/indeedblog/dist/js/scripts.js?ver=2.6.0" id="scripts-js-js"></script>
<script type="text/javascript" src="https://www.youtube.com/iframe_api?ver=20210524" id="youtube_iframe_api-js"></script>
<script type="text/javascript" id="wpml-xdomain-data-js-extra">
/* <![CDATA[ */
var wpml_xdomain_data = {"css_selector":"wpml-ls-item","ajax_url":"https:\/\/engineering.indeedblog.com\/wp-admin\/admin-ajax.php","current_lang":"en","_nonce":"0c100599a1"};
/* ]]> */
</script>
<script type="text/javascript" src="https://engblogs.wpenginepowered.com/wp-content/plugins/sitepress-multilingual-cms/res/js/xdomain-data.js?ver=4.6.13" id="wpml-xdomain-data-js" defer="defer" data-wp-strategy="defer"></script>
<link rel="https://api.w.org/" href="https://engineering.indeedblog.com/wp-json/"><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://engineering.indeedblog.com/xmlrpc.php?rsd">
<meta name="generator" content="WPML ver:4.6.13 stt:1,28;">
<link rel="icon" href="https://engblogs.wpenginepowered.com/wp-content/uploads/2015/04/1024x1024-553d53a6v1_site_icon-32x32.png" sizes="32x32">
<link rel="icon" href="https://engblogs.wpenginepowered.com/wp-content/uploads/2015/04/1024x1024-553d53a6v1_site_icon-256x256.png" sizes="192x192">
<link rel="apple-touch-icon" href="https://engblogs.wpenginepowered.com/wp-content/uploads/2015/04/1024x1024-553d53a6v1_site_icon-256x256.png">
<meta name="msapplication-TileImage" content="https://engineering.indeedblog.com/wp-content/uploads/2015/04/1024x1024-553d53a6v1_site_icon.png">
<style type="text/css" id="wp-custom-css">
/*
Welcome to Custom CSS!
CSS (Cascading Style Sheets) is a kind of code that tells the browser how
to render a web page. You may delete these comments and get started with
your customizations.
By default, your stylesheet will be loaded after the theme stylesheets,
which means that your rules can take precedence and override the theme CSS
rules. Just write here what you want to change, you don't need to copy all
your theme's stylesheet content.
*/ </style>
<script src="https://engineering.indeedblog.com/wp-includes/js/wp-emoji-release.min.js?ver=6.6.1" defer=""></script><style type="text/css">.MathJax_Hover_Frame {border-radius: .25em; -webkit-border-radius: .25em; -moz-border-radius: .25em; -khtml-border-radius: .25em; box-shadow: 0px 0px 15px #83A; -webkit-box-shadow: 0px 0px 15px #83A; -moz-box-shadow: 0px 0px 15px #83A; -khtml-box-shadow: 0px 0px 15px #83A; border: 1px solid #A6D ! important; display: inline-block; position: absolute}
.MathJax_Menu_Button .MathJax_Hover_Arrow {position: absolute; cursor: pointer; display: inline-block; border: 2px solid #AAA; border-radius: 4px; -webkit-border-radius: 4px; -moz-border-radius: 4px; -khtml-border-radius: 4px; font-family: 'Courier New',Courier; font-size: 9px; color: #F0F0F0}
.MathJax_Menu_Button .MathJax_Hover_Arrow span {display: block; background-color: #AAA; border: 1px solid; border-radius: 3px; line-height: 0; padding: 4px}
.MathJax_Hover_Arrow:hover {color: white!important; border: 2px solid #CCC!important}
.MathJax_Hover_Arrow:hover span {background-color: #CCC!important}
</style><style type="text/css">#MathJax_About {position: fixed; left: 50%; width: auto; text-align: center; border: 3px outset; padding: 1em 2em; background-color: #DDDDDD; color: black; cursor: default; font-family: message-box; font-size: 120%; font-style: normal; text-indent: 0; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; z-index: 201; border-radius: 15px; -webkit-border-radius: 15px; -moz-border-radius: 15px; -khtml-border-radius: 15px; box-shadow: 0px 10px 20px #808080; -webkit-box-shadow: 0px 10px 20px #808080; -moz-box-shadow: 0px 10px 20px #808080; -khtml-box-shadow: 0px 10px 20px #808080; filter: progid:DXImageTransform.Microsoft.dropshadow(OffX=2, OffY=2, Color='gray', Positive='true')}
#MathJax_About.MathJax_MousePost {outline: none}
.MathJax_Menu {position: absolute; background-color: white; color: black; width: auto; padding: 5px 0px; border: 1px solid #CCCCCC; margin: 0; cursor: default; font: menu; text-align: left; text-indent: 0; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; z-index: 201; border-radius: 5px; -webkit-border-radius: 5px; -moz-border-radius: 5px; -khtml-border-radius: 5px; box-shadow: 0px 10px 20px #808080; -webkit-box-shadow: 0px 10px 20px #808080; -moz-box-shadow: 0px 10px 20px #808080; -khtml-box-shadow: 0px 10px 20px #808080; filter: progid:DXImageTransform.Microsoft.dropshadow(OffX=2, OffY=2, Color='gray', Positive='true')}
.MathJax_MenuItem {padding: 1px 2em; background: transparent}
.MathJax_MenuArrow {position: absolute; right: .5em; padding-top: .25em; color: #666666; font-size: .75em}
.MathJax_MenuActive .MathJax_MenuArrow {color: white}
.MathJax_MenuArrow.RTL {left: .5em; right: auto}
.MathJax_MenuCheck {position: absolute; left: .7em}
.MathJax_MenuCheck.RTL {right: .7em; left: auto}
.MathJax_MenuRadioCheck {position: absolute; left: .7em}
.MathJax_MenuRadioCheck.RTL {right: .7em; left: auto}
.MathJax_MenuLabel {padding: 1px 2em 3px 1.33em; font-style: italic}
.MathJax_MenuRule {border-top: 1px solid #DDDDDD; margin: 4px 3px}
.MathJax_MenuDisabled {color: GrayText}
.MathJax_MenuActive {background-color: #606872; color: white}
.MathJax_MenuDisabled:focus, .MathJax_MenuLabel:focus {background-color: #E8E8E8}
.MathJax_ContextMenu:focus {outline: none}
.MathJax_ContextMenu .MathJax_MenuItem:focus {outline: none}
#MathJax_AboutClose {top: .2em; right: .2em}
.MathJax_Menu .MathJax_MenuClose {top: -10px; left: -10px}
.MathJax_MenuClose {position: absolute; cursor: pointer; display: inline-block; border: 2px solid #AAA; border-radius: 18px; -webkit-border-radius: 18px; -moz-border-radius: 18px; -khtml-border-radius: 18px; font-family: 'Courier New',Courier; font-size: 24px; color: #F0F0F0}
.MathJax_MenuClose span {display: block; background-color: #AAA; border: 1.5px solid; border-radius: 18px; -webkit-border-radius: 18px; -moz-border-radius: 18px; -khtml-border-radius: 18px; line-height: 0; padding: 8px 0 6px}
.MathJax_MenuClose:hover {color: white!important; border: 2px solid #CCC!important}
.MathJax_MenuClose:hover span {background-color: #CCC!important}
.MathJax_MenuClose:hover:focus {outline: none}
</style><style type="text/css">.MathJax_Preview .MJXf-math {color: inherit!important}
</style><style type="text/css">.MJX_Assistive_MathML {position: absolute!important; top: 0; left: 0; clip: rect(1px, 1px, 1px, 1px); padding: 1px 0 0 0!important; border: 0!important; height: 1px!important; width: 1px!important; overflow: hidden!important; display: block!important; -webkit-touch-callout: none; -webkit-user-select: none; -khtml-user-select: none; -moz-user-select: none; -ms-user-select: none; user-select: none}
.MJX_Assistive_MathML.MJX_Assistive_MathML_Block {width: 100%!important}
</style><style type="text/css">#MathJax_Zoom {position: absolute; background-color: #F0F0F0; overflow: auto; display: block; z-index: 301; padding: .5em; border: 1px solid black; margin: 0; font-weight: normal; font-style: normal; text-align: left; text-indent: 0; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; -webkit-box-sizing: content-box; -moz-box-sizing: content-box; box-sizing: content-box; box-shadow: 5px 5px 15px #AAAAAA; -webkit-box-shadow: 5px 5px 15px #AAAAAA; -moz-box-shadow: 5px 5px 15px #AAAAAA; -khtml-box-shadow: 5px 5px 15px #AAAAAA; filter: progid:DXImageTransform.Microsoft.dropshadow(OffX=2, OffY=2, Color='gray', Positive='true')}
#MathJax_ZoomOverlay {position: absolute; left: 0; top: 0; z-index: 300; display: inline-block; width: 100%; height: 100%; border: 0; padding: 0; margin: 0; background-color: white; opacity: 0; filter: alpha(opacity=0)}
#MathJax_ZoomFrame {position: relative; display: inline-block; height: 0; width: 0}
#MathJax_ZoomEventTrap {position: absolute; left: 0; top: 0; z-index: 302; display: inline-block; border: 0; padding: 0; margin: 0; background-color: white; opacity: 0; filter: alpha(opacity=0)}
</style><style type="text/css">.MathJax_Preview {color: #888; display: contents}
#MathJax_Message {position: fixed; left: 1em; bottom: 1.5em; background-color: #E6E6E6; border: 1px solid #959595; margin: 0px; padding: 2px 8px; z-index: 102; color: black; font-size: 80%; width: auto; white-space: nowrap}
#MathJax_MSIE_Frame {position: absolute; top: 0; left: 0; width: 0px; z-index: 101; border: 0px; margin: 0px; padding: 0px}
.MathJax_Error {color: #CC0000; font-style: italic}
</style><style type="text/css">.MJXp-script {font-size: .8em}
.MJXp-right {-webkit-transform-origin: right; -moz-transform-origin: right; -ms-transform-origin: right; -o-transform-origin: right; transform-origin: right}
.MJXp-bold {font-weight: bold}
.MJXp-italic {font-style: italic}
.MJXp-scr {font-family: MathJax_Script,'Times New Roman',Times,STIXGeneral,serif}
.MJXp-frak {font-family: MathJax_Fraktur,'Times New Roman',Times,STIXGeneral,serif}
.MJXp-sf {font-family: MathJax_SansSerif,'Times New Roman',Times,STIXGeneral,serif}
.MJXp-cal {font-family: MathJax_Caligraphic,'Times New Roman',Times,STIXGeneral,serif}
.MJXp-mono {font-family: MathJax_Typewriter,'Times New Roman',Times,STIXGeneral,serif}
.MJXp-largeop {font-size: 150%}
.MJXp-largeop.MJXp-int {vertical-align: -.2em}
.MJXp-math {display: inline-block; line-height: 1.2; text-indent: 0; font-family: 'Times New Roman',Times,STIXGeneral,serif; white-space: nowrap; border-collapse: collapse}
.MJXp-display {display: block; text-align: center; margin: 1em 0}
.MJXp-math span {display: inline-block}
.MJXp-box {display: block!important; text-align: center}
.MJXp-box:after {content: " "}
.MJXp-rule {display: block!important; margin-top: .1em}
.MJXp-char {display: block!important}
.MJXp-mo {margin: 0 .15em}
.MJXp-mfrac {margin: 0 .125em; vertical-align: .25em}
.MJXp-denom {display: inline-table!important; width: 100%}
.MJXp-denom > * {display: table-row!important}
.MJXp-surd {vertical-align: top}
.MJXp-surd > * {display: block!important}
.MJXp-script-box > * {display: table!important; height: 50%}
.MJXp-script-box > * > * {display: table-cell!important; vertical-align: top}
.MJXp-script-box > *:last-child > * {vertical-align: bottom}
.MJXp-script-box > * > * > * {display: block!important}
.MJXp-mphantom {visibility: hidden}
.MJXp-munderover, .MJXp-munder {display: inline-table!important}
.MJXp-over {display: inline-block!important; text-align: center}
.MJXp-over > * {display: block!important}
.MJXp-munderover > *, .MJXp-munder > * {display: table-row!important}
.MJXp-mtable {vertical-align: .25em; margin: 0 .125em}
.MJXp-mtable > * {display: inline-table!important; vertical-align: middle}
.MJXp-mtr {display: table-row!important}
.MJXp-mtd {display: table-cell!important; text-align: center; padding: .5em 0 0 .5em}
.MJXp-mtr > .MJXp-mtd:first-child {padding-left: 0}
.MJXp-mtr:first-child > .MJXp-mtd {padding-top: 0}
.MJXp-mlabeledtr {display: table-row!important}
.MJXp-mlabeledtr > .MJXp-mtd:first-child {padding-left: 0}
.MJXp-mlabeledtr:first-child > .MJXp-mtd {padding-top: 0}
.MJXp-merror {background-color: #FFFF88; color: #CC0000; border: 1px solid #CC0000; padding: 1px 3px; font-style: normal; font-size: 90%}
.MJXp-scale0 {-webkit-transform: scaleX(.0); -moz-transform: scaleX(.0); -ms-transform: scaleX(.0); -o-transform: scaleX(.0); transform: scaleX(.0)}
.MJXp-scale1 {-webkit-transform: scaleX(.1); -moz-transform: scaleX(.1); -ms-transform: scaleX(.1); -o-transform: scaleX(.1); transform: scaleX(.1)}
.MJXp-scale2 {-webkit-transform: scaleX(.2); -moz-transform: scaleX(.2); -ms-transform: scaleX(.2); -o-transform: scaleX(.2); transform: scaleX(.2)}
.MJXp-scale3 {-webkit-transform: scaleX(.3); -moz-transform: scaleX(.3); -ms-transform: scaleX(.3); -o-transform: scaleX(.3); transform: scaleX(.3)}
.MJXp-scale4 {-webkit-transform: scaleX(.4); -moz-transform: scaleX(.4); -ms-transform: scaleX(.4); -o-transform: scaleX(.4); transform: scaleX(.4)}
.MJXp-scale5 {-webkit-transform: scaleX(.5); -moz-transform: scaleX(.5); -ms-transform: scaleX(.5); -o-transform: scaleX(.5); transform: scaleX(.5)}
.MJXp-scale6 {-webkit-transform: scaleX(.6); -moz-transform: scaleX(.6); -ms-transform: scaleX(.6); -o-transform: scaleX(.6); transform: scaleX(.6)}
.MJXp-scale7 {-webkit-transform: scaleX(.7); -moz-transform: scaleX(.7); -ms-transform: scaleX(.7); -o-transform: scaleX(.7); transform: scaleX(.7)}
.MJXp-scale8 {-webkit-transform: scaleX(.8); -moz-transform: scaleX(.8); -ms-transform: scaleX(.8); -o-transform: scaleX(.8); transform: scaleX(.8)}
.MJXp-scale9 {-webkit-transform: scaleX(.9); -moz-transform: scaleX(.9); -ms-transform: scaleX(.9); -o-transform: scaleX(.9); transform: scaleX(.9)}
.MathJax_PHTML .noError {vertical-align: ; font-size: 90%; text-align: left; color: black; padding: 1px 3px; border: 1px solid}
</style></head>
<body class="blog group-blog"><div id="MathJax_Message" style="display: none;"></div>
<!-- Start Google Tag Manager (noscript) -->
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-K8RKLN"
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager (noscript) -->
<a href="#mainContent" class="skip-link">Skip to Main Content</a>
<div id="blogHeader" class="full-width-wrapper">
<div class="container">
<div class="row">
<nav class="navbar navbar-default navbar-static-top">
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#headerNavLinks">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a href="https://engineering.indeedblog.com" class="navbar-brand">
<img src="https://engblogs.wpenginepowered.com/wp-content/themes/indeedblog/images/ind-eng-logo.png" alt="Indeed Engineering" width="300">
</a>
</div>
<div id="headerNavLinks" class="collapse navbar-collapse"><ul class=""><li id="menu-item-1211" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-1211"><a href="/talks">Tech Talks</a></li>
<li id="menu-item-1653" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-1653"><a href="http://opensource.indeedeng.io/">Open Source</a></li>
<li id="menu-item-1212" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-1212"><a href="http://indeed.jobs">Work at Indeed</a></li>
</ul></div> </nav>
</div>
</div>
</div><!-- end #blogHeader -->
<main id="mainContent" class="container">
<div class="row">
<div class="col-xs-12 col-sm-8 post-container">
<article id="post-4693" class="clearfix post-4693 post type-post status-publish format-standard hentry category-unsorted">
<header class="entry-header">
<h1 class="entry-title"><a href="https://engineering.indeedblog.com/blog/2024/08/indeed-gitlab-ci-migration/">How Indeed Replaced Its CI Platform with Gitlab CI</a></h1>
<p class="date entry-meta">
<span class="posted-on">Posted on <time class="entry-date published" datetime="2024-08-06T10:03:51-05:00">August 6, 2024</time></span><span class="byline"> by Carl Myers</span> </p><!-- .entry-meta -->
</header><!-- .entry-header -->
<div class="entry-content">
<div class="post-excerpt">
<p><span style="font-weight: 400;">Here at Indeed, our mission is to help people get jobs. </span><span style="font-weight: 400;">Indeed is the</span><a href="https://www.indeed.com/about?isid=press_us&ikw=press_us_press%2Freleases%2Faward-winning-actress-viola-davis-to-keynote-indeed-futureworks-2023_textlink_https%3A%2F%2Fwww.indeed.com%2Fabout"><span style="font-weight: 400;"> #1 job site</span></a><span style="font-weight: 400;"> in the world with </span><a href="https://www.indeed.com/about/methodology"><span style="font-weight: 400;">over 350M+ unique visitors</span></a><span style="font-weight: 400;"> every month. </span><span style="font-weight: 400;">For Indeed’s Engineering Platform teams, we have a slightly different motto: “We </span><i><span style="font-weight: 400;">help people to</span></i><span style="font-weight: 400;"> help people get jobs”. As part of a data-driven engineering culture that has spent the better part of two decades always putting the job seeker first, we are responsible for building the tools that not only make this possible, but empower engineers to deliver positive outcomes to job seekers every day.</span></p>
<h2><span style="font-weight: 400;">Do you want to build a Jenkins snowman?</span></h2>
<p><span style="font-weight: 400;">Like many large technology companies, our Continuous Integration (CI) platform was built organically as the company scaled. In fact, Indeed was using </span><a href="https://en.wikipedia.org/wiki/Hudson_(software)"><span style="font-weight: 400;">Hudson</span></a><span style="font-weight: 400;">, Jenkins’ direct predecessor, back in 2007. At the time, Indeed had fewer than 20 engineers. Today, through nearly two decades of growth, we have thousands of engineers. We built our platform on top of the de facto open source and industry standard solutions available at the time. As new technology became available, we made incremental improvements, switching to Jenkins after Oracle bought Sun and caused the Jenkins/Hudson fork around 2011. Another improvement allowed us to move most of our workloads to dynamic cloud worker nodes using </span><a href="https://aws.amazon.com/ec2/"><span style="font-weight: 400;">AWS EC2</span></a><span style="font-weight: 400;">. As we entered the </span><a href="https://kubernetes.io/"><span style="font-weight: 400;">Kubernetes</span></a><span style="font-weight: 400;"> age, however, the system architecture reached its limits. Hudson was first released in 2005. In 2005, J2SE 5.0 was less than a year old. Java with generics was novel! AWS was not a thing. Clouds were made of water vapor, not servers and software defined networking.</span></p>
<p><span style="font-weight: 400;">Suffice it to say, Jenkins’ architecture was not created with the cloud in mind and could not have been, because the cloud did not yet exist. Jenkins operates by having a “controller” node, a single point of failure which runs critical parts of a pipeline and farms out certain steps to worker nodes (which can scale horizontally to some extent). Controllers are not only a single point of failure, they are also a manual scaling axis. If you have too many jobs to fit on one controller, you must partition your jobs across controllers </span><b>manually</b><span style="font-weight: 400;">. Cloudbees, the largest company offering Jenkins enterprise support, has some mitigations for this including the Cloudbees Jenkins Operations Center (CJOC), which allows you to manage your constellation of controllers from a single centralized place, but they remain challenging to run in a Kubernetes environment because each controller is a fragile single-point-of-failure. Activities like node rollouts or hardware failures cause downtime.</span></p>
<h2><span style="font-weight: 400;">Follow the yellow brick road</span></h2>
<p><span style="font-weight: 400;">Besides the technical limitations baked into Jenkins itself, our CI platform also had several problems of our own making. We used the Groovy Jenkins DSL to generate jobs from code which were checked into each repository – an industry best practice and the minimum necessary for sanity. However, these scripts were based upon shared code using a library model, rather than a template model. This means that a large portion of the job logic was essentially copy-pasted into each project repository and only called out to shared modules leveraging shared code.</span></p>
<p><span style="font-weight: 400;">This pattern had several drawbacks. Each project had its own copy-pasted version of the job pipeline, which was copied from the skeleton for that project type at the time of creation and then rarely, if ever, updated. This resulted in hundreds of different versions of our various pipelines all existing at the same time and depending upon our shared library modules. That in turn made them extremely difficult to update without breaking pipelines. Testing changes against the wide variety of pipelines was an intractable challenge. Furthermore, modifying pipelines to adopt new features often required asking our users to manually update their own build code, since hundreds of divergent versions existed across the company, many with customization implemented by the teams.</span></p>
<p><span style="font-weight: 400;">To understand why things were this way, it is important to understand that Indeed’s engineering culture includes a core value of flexibility. We accept that there are many valid ways to do something and different teams and products may have different optimal choices. Furthermore, being agile and data-driven often requires a degree of flexibility. We do not subscribe to a monorepo model and instead each project lives in its own repository (we have tens of thousands of repositories).</span></p>
<p><span style="font-weight: 400;">This flexibility serves us well in many contexts but unfortunately, too much flexibility can be a double-edged sword. The inevitable result of this balance was that teams were spending an unacceptable portion of their time just addressing “platform asks”. This is our term for regular maintenance that would come up when we needed teams to modify their build, as we deployed new versions of our platform, moved resources to the cloud, or made other changes to our infrastructure. The flexibility we gave our users (other engineers at Indeed) meant we couldn’t easily make the changes for them. It was around the time that we were looking to solve the hardware scaling and resiliency problems of Jenkins that we realized the scope and depth of our self-imposed technical debt for our build platform code. The solution came from the </span><a href="https://tag-app-delivery.cncf.io/whitepapers/platforms/"><span style="font-weight: 400;">Golden Path</span></a><span style="font-weight: 400;"> pattern. Using this pattern, we could give our users the flexibility to do things their own way while still making sure it was easy to choose the default way when possible, and modify only the parts of the path they really needed to while leveraging the shared path as much as possible for the rest.</span></p>
<h2><span style="font-weight: 400;">The CI Platform team at Indeed</span></h2>
<p><span style="font-weight: 400;">The CI Platform team at Indeed is not very large. Our team of ~11 engineers supports thousands of users, fielding support requests, performing upgrades and maintenance, and enabling follow-the-sun support for our global company. </span></p>
<p><span style="font-weight: 400;">Because our team not only supports Gitlab but also the entire CI platform including the artifact server, our shared build code, and multiple other custom components of our platform, we had our work cut out for us. We needed a plan to get where we were going that makes the most efficient use of the resources we have.</span></p>
<h2><span style="font-weight: 400;">A plan comes together</span></h2>
<p><span style="font-weight: 400;">After a careful design review with key stakeholders, we successfully built consensus for the new CI Platform. We would migrate the entire company from Jenkins to Gitlab CI. The primary reasons for choosing Gitlab CI were:</span></p>
<ul>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Gitlab is a complete offering (already in use for SCM) which provides everything we need for CI</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Gitlab CI is designed for scalability and the cloud</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Gitlab CI enables us to write templates that extend other templates, which is compatible with our golden path strategy.</span></li>
</ul>
<p><span style="font-weight: 400;">By the time we officially announced that the Gitlab CI Platform would be generally available to users, we already had 23% of all builds happening in Gitlab CI from a combination of grassroots efforts and early adopters wanting to switch ASAP. The challenge of the migration, however, would be the long tail. Due to the number of custom builds in Jenkins, an automated migration tool would not work for the majority of teams. Most of the benefits of the new system would not come until the old system was at 0%. Only then could we turn off the hardware and save the Cloudbees license fee.</span></p>
<h2><span style="font-weight: 400;">Gitlab CI is Open Source Software</span></h2>
<p><span style="font-weight: 400;">Another factor that influenced our decision-making process and ended up being critical to our success was that Gitlab itself is Open Source software. As a proof of concept, we had a project to make a small change to Gitlab. We picked a few simple looking bugs (a Gitlab Geo </span><a href="https://gitlab.com/gitlab-org/gitlab/-/issues/365101"><span style="font-weight: 400;">issue</span></a><span style="font-weight: 400;">, and a template parsing </span><a href="https://gitlab.com/gitlab-org/gitlab/-/issues/335138"><span style="font-weight: 400;">bug</span></a><span style="font-weight: 400;">) we had noticed and submitted the fixes. Gitlab was massively supportive of this and helped us shepherd our changes through. This reduced uncertainty because we knew we could always fix our own issues if Gitlab was not able to prioritize fixing them for us.</span></p>
<p><span style="font-weight: 400;">This foresight would become especially prescient the next year when we discovered an unexpected behavior in the CI job runner that caused an internal security issue due to Indeed’s unique access configuration. We were able to leverage our experience from contributing to Gitlab and compile and run a fork of the Gitlab CI job runner immediately to mitigate the issue. Meanwhile, we were able to submit the fork as an </span><a href="https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3801"><span style="font-weight: 400;">MR to Gitlab</span></a><span style="font-weight: 400;"> so they could understand the vulnerability and come up with an acceptable long-term fix. In the end we only had to run a fork for a few months, but that flexibility proved the value of choosing open source software.</span></p>
<h2><span style="font-weight: 400;">Feature parity and the benefits of starting over</span></h2>
<p><span style="font-weight: 400;">Though we support many different technologies at Indeed, the three most common languages are Java, Python, and Javascript. These language stacks are used to make libraries, deployables (i.e. web services or applications), and cron jobs (a process that runs at regular intervals, for example, to build a data set in our data lake). Each of these formed a matrix of project types (Java Library, Python Cronjob, Javascript Webapp, etc) for which we had a skeleton in Jenkins. Therefore, we had to produce a golden path template in Gitlab CI for each of these project types. Most users could use these recommended paths without change, but for those who did require customization, the golden path would still be a valuable starting point and enable them to change only what they needed, while still benefiting from centralized template updates in the future.</span></p>
<p><span style="font-weight: 400;">We quickly realized that most users, even those with customizations, were happy to take the golden path and at least try it. If they missed their customizations, they could always add them later. This was a surprising result! We thought that teams who had invested in significant customization would be loath to give them up, but in the majority of cases teams just didn’t care about them anymore. This allowed us to migrate many projects very quickly – we could just drop the golden path (a small file about 6 lines long with includes) into their project, and they could take it from there.</span></p>
<h2><span style="font-weight: 400;">InnerSource to the rescue</span></h2>
<p><span style="font-weight: 400;">The CI Platform team also adopted a policy of “external contributions first” to encourage everyone in the company to participate. This is sometimes called </span><a href="https://en.wikipedia.org/wiki/Inner_source"><span style="font-weight: 400;">InnerSource</span></a><span style="font-weight: 400;">. We wrote tests and documentation to enable external contributions – contributions from outside our immediate team – so teams that wanted to write customizations could instead include them in the golden path behind a feature flag. This let them share their work with others and ensure we didn’t break them moving forward (because they became part of our codebase, not theirs). </span></p>
<p><span style="font-weight: 400;">This also had the benefit that particular teams who were blocked waiting for a feature they needed were empowered to work on the feature themselves. We could say “we plan to implement the feature in a few weeks, but if you need it earlier than that we are happy to accept a contribution”. In the end, many core features necessary for parity were developed in this manner, more quickly and better than our team had resources to do it. The migration would not have been a success without this model.</span></p>
<h2><span style="font-weight: 400;">Ahead of schedule and under budget</span></h2>
<p><span style="font-weight: 400;">Our Cloudbees license expired on April 1, 2024. This gave us an aggressive target to achieve the full migration. This was particularly aggressive considering at the time, 80% of all builds (60% of all projects) still used Jenkins for their CI. This meant over 2000 </span><a href="https://www.jenkins.io/doc/book/pipeline/jenkinsfile/"><span style="font-weight: 400;">Jenkinsfiles</span></a><span style="font-weight: 400;"> would still need to be rewritten or replaced with our golden path templates. The wide consensus was that this date was extremely aggressive and an alternative (such as a smaller license engagement for the teams that still required Jenkins) would be needed. Nonetheless, we took the approach that one must aim for the stars to land on the moon. We made documentation and examples available, implemented features where possible, and helped our users contribute features where they were able.</span></p>
<p><span style="font-weight: 400;">We started regular office hours, where anyone could come and ask questions or seek our help to migrate. We additionally prioritized support questions relating to migration ahead of almost everything else. Our team became Gitlab CI experts and shared that expertise inside our team and across the organization.</span></p>
<p><span style="font-weight: 400;">Automatic migration for most projects was not possible, but we discovered it could work for a small subset of projects where customization was rare. We created a Sourcegraph batch change campaign to submit merge requests (MRs) to migrate hundreds of projects, and poked and prodded our users to accept these MRs. We took success stories from our users and shared them widely. As users contributed new features to our golden paths, we advertised that these features “came free” when you migrated to Gitlab CI. Some examples included built in security and compliance scanning, Slack notifications for CI builds, and integrations with other internal systems.</span></p>
<p><span style="font-weight: 400;">We also conducted a campaign of aggressive “scream tests”. We automatically disabled Jenkins jobs that hadn’t run in a while or hadn’t succeeded in a while, telling users “if you need these, turn them back on, it is self-service”. This was a low-friction way to get some signal about what jobs were actually needed. We had thousands of jobs that hadn’t been run a single time since our </span><i><span style="font-weight: 400;">last</span></i><span style="font-weight: 400;"> CI migration (which was Jenkins to Jenkins). This allowed us to know we could safely ignore almost all of them.</span></p>
<p><span style="font-weight: 400;">In January 2024, we nudged our users by announcing that all Jenkins controllers would become read-only (no builds) unless an exception was explicitly requested. We had much better ownership information for controllers and they generally aligned with our organization’s structure, so it made sense to focus on controllers rather than jobs. The list of controllers was also a much more manageable list than the list of jobs. The only thing we asked of our users in order to obtain an exception was to find their controllers in a spreadsheet and put their contact information next to it. This enabled us to get a guaranteed up-to-date list of stakeholders we could follow up with as we sprinted to the finish line, but also enabled users to clearly say “we need these jobs, please don’t break them without talking to us”. At peak we had about 400 controllers, by January we had 220, but only 54 controllers required exceptions (several of them owned by us, to run our tests and canaries).</span></p>
<p><img fetchpriority="high" decoding="async" class="alignnone size-full wp-image-4694" src="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/08/image1.png" alt="" width="740" height="460" srcset="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/08/image1.png 740w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/08/image1-300x186.png 300w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/08/image1-676x420.png 676w" sizes="(max-width: 740px) 100vw, 740px"></p>
<p><span style="font-weight: 400;">With a list of ~50 teams to reach out to, we had an approachable list we could divide among our team and start doing the work of understanding where they were at. We spent January and February discovering that some teams planned to finish their migration without our help before February 28th, others were planning to deprecate their projects before then, and a very small number were very worried they wouldn’t make it.</span></p>
<p><span style="font-weight: 400;">We were able to work with this smaller set of teams and provide them with “white-glove” service. We still explained that while we lacked the expertise necessary to do it for them, we could pair together with a subject matter expert from their team. For some projects we wrote and they reviewed, for others they wrote and we reviewed. In the end, all of our work paid off and we turned off Jenkins on the very day we had announced 8 months earlier.</span></p>
<h2><span style="font-weight: 400;">All’s well that ends well</span></h2>
<p><span style="font-weight: 400;">At peak, our Jenkins CI platform ran over 14,000 pipelines per day and serviced our thousands of projects. Today, our Gitlab CI platform has run over 40,000 pipelines in a single day and regularly runs over 25,000 per day. The incremental cost of each job of each pipeline is similar to Jenkins, but without the overhead of hardware to run the controllers. Additionally, these controllers served as single points of failure and scaling limiters that forced us to artificially divide our platform into segments. While an apples-to-apples comparison is difficult, we find that with this overhead gone our CI hardware costs are 10-20% lower. Additionally, the support burden of Gitlab CI is lower since the application automatically scales in the cloud, has cross-availability-zone resiliency, and the templating language has excellent public documentation available.</span></p>
<p><span style="font-weight: 400;">A benefit just as important, if not moreso, is that now we are at over 70% adoption of our golden paths. This means that we can roll out an improvement and over 5000 projects at Indeed will benefit immediately with no action required on their part. This has enabled us to move some jobs to more cost-effective ARM64 instances, keep users’ build images updated more easily, and better manage other cost saving opportunities. Most importantly, our users are happier with the new platform.</span></p>
<p><span style="font-weight: 400;">This post is long enough, so I will leave you with two of my favorite graphs of my entire career.</span></p>
<p><img decoding="async" class="alignnone size-full wp-image-4698" src="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/08/image3.png" alt="" width="738" height="454" srcset="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/08/image3.png 738w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/08/image3-300x185.png 300w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/08/image3-683x420.png 683w" sizes="(max-width: 738px) 100vw, 738px"></p>
<p><img decoding="async" class="alignnone size-full wp-image-4696" src="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/08/image2.png" alt="" width="739" height="456" srcset="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/08/image2.png 739w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/08/image2-300x185.png 300w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/08/image2-681x420.png 681w" sizes="(max-width: 739px) 100vw, 739px"></p>
<h2><span style="font-weight: 400;">Acknowledgements</span></h2>
<p><em><span style="font-weight: 400;">This migration would not have been possible without the tireless efforts of Tron Nedelea, Eddie Huang, Vivek Nynaru, Carlos Gonzalez, </span><span style="font-weight: 400;">Lane Van Elderen, </span><span style="font-weight: 400;">and the rest of the CI Platform team. The team also especially appreciates the leadership of Deepak Bitragunta, and Irina Tyree for helping secure buy-in, resources and company wide alignment throughout this long project. Finally, our thanks go out to everyone across Indeed who contributed code, feedback, bug reports, and helped migrate projects.</span></em></p>
<!-- <p><a class="moretag" href="https://engineering.indeedblog.com/blog/2024/08/indeed-gitlab-ci-migration/"> Read the full article <span class="link-chevron light-link">»</span></a></p> --> </div>
</div><!-- .entry-content -->
<footer class="entry-meta">
</footer><!-- .entry-meta -->
</article><!-- #post-## -->
<article id="post-4681" class="clearfix post-4681 post type-post status-publish format-standard hentry category-unsorted">
<header class="entry-header">
<h1 class="entry-title"><a href="https://engineering.indeedblog.com/blog/2024/07/workload-identity-with-spire-oidc-for-k8s-istio/">Secure Workload Identity with SPIRE and OIDC: A Guide for Kubernetes and Istio Users</a></h1>
<p class="date entry-meta">
<span class="posted-on">Posted on <time class="entry-date published" datetime="2024-07-03T10:52:26-05:00">July 3, 2024</time></span><span class="byline"> by Nikhil Arora</span> </p><!-- .entry-meta -->
</header><!-- .entry-header -->
<div class="entry-content">
<div class="post-excerpt">
<h1><span style="font-weight: 400;">Goal</span></h1>
<p><span style="font-weight: 400;">This blog is for engineering teams, architects, and leaders responsible for defining and implementing a workload identity platform and access controls rooted in Zero Trust principles to mitigate the risks from compromised services. It is relevant for companies using Kubernetes to manage workloads, using Istio for service mesh, and aiming to define identities in a way that aligns with internal standards, free from platform-specific constraints. Specifically, we’ll discuss Indeed’s solution for third-party authentication, opinionated best practices, and challenges faced. It is not practical to share all the alternatives, trade-offs and engineering insights supporting our decisions; we want to share design choices and implementation details that can accelerate decision making and problem solving for others in similar situations.</span></p>
<h1><span style="font-weight: 400;">Introduction</span></h1>
<p><span style="font-weight: 400;">Passwords are a tale as old as ancient civilizations. Modern systems routinely rely on API key & ID pairs (analogous to username and passwords) to access other systems. These API keys in theory are complex, managed by developers, and stored securely. The reality is more complicated. We all have heard stories of passwords hiding in plain sight, unencrypted, in code repositories, in log messages, in headers, in terminal history, wherever it’s convenient to just get the job done. Rotating old API keys can even be scarier. Who knows if keys have been shared, how many times they have been shared, and where all they have been shared? Did Alice delete the old API key? Was the new API key deployed everywhere!?</span></p>
<p><span style="font-weight: 400;">So what’s the solution? Step 1: Articulate and measure the problem. At Indeed, we embody our core value of being data-driven. Through our analysis, we recognized the risk posed by compromised credentials used by services. Our data revealed that half of our AWS IAM keys have access to some type of restricted data. We observed shared API keys being used across a wide range of our workloads. We discovered roughly eight times as many stored secrets as there are unique keys in all of our major authorization systems. This indicates a significant duplication of secrets, though we have not yet determined the exact scale of this duplication. Step 2: Implement a solution that works for Indeed’s heterogeneous workloads across third-party SaaS cloud vendors and Indeed’s own (first-party) apps.</span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-4686 size-full" src="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image3.png" alt="Image showing API keys from a shared vault being used to access resources in multiple cloud providers" width="1522" height="1027" srcset="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image3.png 1522w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image3-300x202.png 300w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image3-1024x691.png 1024w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image3-768x518.png 768w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image3-622x420.png 622w" sizes="(max-width: 1522px) 100vw, 1522px"></p>
<p><span style="font-weight: 400;">The starting point is to build an identity platform capable of provisioning </span><b>temporary, verifiable, attestable, unique, and cryptographically secure</b> <b>workload credentials</b><span style="font-weight: 400;"> for access to third-party systems like Confluent Cloud and AWS, and first-party services as well. Indeed promotes responsible use of Open Source Software and dedicated platforms with clear responsibilities leveraging industry standards to solve common problems. Our workload identity platform is built on </span><a href="https://spiffe.io/docs/latest/spire-about/"><span style="font-weight: 400;">SPIRE</span></a><span style="font-weight: 400;">, embracing open standards like </span><a href="https://spiffe.io/"><span style="font-weight: 400;">SPIFFE</span></a><span style="font-weight: 400;">, </span><a href="https://oauth.net/2/"><span style="font-weight: 400;">OAuth 2.0</span></a><span style="font-weight: 400;"> and </span><a href="https://openid.net/"><span style="font-weight: 400;">OIDC</span></a><span style="font-weight: 400;"> to provide managed identities in </span><a href="https://github.com/spiffe/spiffe/blob/main/standards/X509-SVID.md"><span style="font-weight: 400;">x509 PKI Certificate</span></a><span style="font-weight: 400;"> or </span><a href="https://jwt.io/introduction"><span style="font-weight: 400;">JSON Web Tokens</span></a><span style="font-weight: 400;"> standards.</span></p>
<h1><span style="font-weight: 400;">SPIRE</span></h1>
<p><a href="https://www.cncf.io/projects/spire/"><span style="font-weight: 400;">SPIRE is a PKI project</span></a><span style="font-weight: 400;"> that graduated from the </span><a href="https://www.cncf.io/"><span style="font-weight: 400;">Cloud Native Computing Foundation</span></a><span style="font-weight: 400;">. SPIRE is open source, </span><a href="https://spiffe.io/docs/latest/spire-about/case-studies/"><span style="font-weight: 400;">widely used</span></a><span style="font-weight: 400;"> in the industry and has a vibrant and active community of engineers. SPIRE can be deployed in a scalable and resilient manner and has been operating reliably at scale in production at Indeed for over a year now. SPIRE-issued x509 identities are used in our Istio service mesh for mTLS, and JWT identities are used to enable OIDC-based federated access with </span><a href="https://docs.confluent.io/cloud/current/access-management/authenticate/oauth/overview.html"><span style="font-weight: 400;">Confluent</span></a><span style="font-weight: 400;"> and</span><a href="https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers.html#id_roles_providers_iam"> <span style="font-weight: 400;">AWS</span></a><span style="font-weight: 400;"> resources.</span></p>
<h2><span style="font-weight: 400;">Istio Opinions</span></h2>
<p><span style="font-weight: 400;">Adopting Istio to replace our legacy service mesh created conflicts with certain SPIRE configurations already in production.</span></p>
<h3><span style="font-weight: 400;">SPIFFE Format</span></h3>
<p><span style="font-weight: 400;">We debated the granularity and uniqueness of identities suitable to represent an Indeed application. In this context </span><i><span style="font-weight: 400;">identity</span></i><span style="font-weight: 400;"> refers to the subject, i.e., the SPIFFE ID of a workload. The discussion revolved around the SPIFFE template and its constituent parts, e.g.:</span></p>
<pre><strong><i>spiffe://<trust_domain>/<scheduling_platform>/<environment>/ns/<namespace>/sa/<service-account></i></strong></pre>
<p><span style="font-weight: 400;">However, Istio is <a href="https://istio.io/latest/docs/ops/integrations/spire/#option-2-manual-registration">highly opinionated</a> about the SPIFFE ID format a workload must have:</span></p>
<pre><b><i>spiffe://<trust.domain>/ns/<namespace>/sa/<service-account></i></b></pre>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-4684 size-full" src="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image4.png" alt="An image showing a cautionary note on workload ID formatting from the Istio / SPIRE documentation" width="1812" height="226" srcset="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image4.png 1812w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image4-300x37.png 300w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image4-1024x128.png 1024w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image4-768x96.png 768w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image4-1536x192.png 1536w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image4-747x93.png 747w" sizes="(max-width: 1812px) 100vw, 1812px"></p>
<p><span style="font-weight: 400;"> </span><span style="font-weight: 400;">This is a known problem that is still open with Istio: </span><a href="https://github.com/istio/istio/issues/43105"><span style="font-weight: 400;">Customizing SPIFFE ID format if using an external SPIFFE-compliant SDS should be supported · Issue #43105 · istio/istio · GitHub</span></a></p>
<p><span style="font-weight: 400;">If you have a SPIRE deployment already in production with a different SPIFFE ID format for your Kubernetes workloads, be aware of Istio requirements. Updating the subject of your workloads is not trivial. While it’s only a configuration change in SPIRE, the subject likely appears wherever access control and authorization rules are defined for your workloads. </span></p>
<h3><span style="font-weight: 400;">SPIRE Agent Socket Name</span></h3>
<p>Istio requires SPIRE Agent APIs be available on the <i>/var/run/secrets/workload-spiffe-uds/socket</i> Unix domain socket only—another (unnecessary) <a href="https://github.com/istio/istio/blob/c8d5b492e7d6305b1da503a39025c468378c22a0/pkg/security/security.go#L49">Istio opinion</a> that affects the entirety of the platform and will require careful planning to accommodate. Since we already had SPIRE in production, we used K8s to mount our socket path to <i>/var/run/secrets/workload-spiffe-uds</i> and only had to update the file name from <i>agent.socket</i> to <i>socket</i>. We made the practical choice of temporarily disabling mTLS in the mesh and rolling out our SPIRE Agent socket name changes one cluster at a time, as it affected the <a href="https://www.envoyproxy.io/docs/envoy/latest/configuration/security/secret#secret-discovery-service-sds">proxy SDS (Secret Discovery Service</a>) configuration as well. During this time, our mesh was only protected by the network perimeter behind the VPN. After both SPIRE Agent and service mesh SDS configuration were updated, mTLS was turned back on.</p>
<h2><span style="font-weight: 400;">SPIRE Architecture</span></h2>
<h3><span style="font-weight: 400;">Topology and Trust Domain</span></h3>
<p><span style="font-weight: 400;">At Indeed, we manage a single trust domain in SPIRE deployed in a </span><a href="https://spiffe.io/docs/latest/planning/scaling_spire/#nested-spire"><span style="font-weight: 400;">nested topology</span></a><span style="font-weight: 400;">. We run multiple SPIRE Servers in each Kubernetes cluster for redundancy. SPIRE Servers in each cluster have a </span><a href="https://spiffe.io/docs/latest/planning/scaling_spire/#spire-servers-in-high-availability-mode"><span style="font-weight: 400;">common datastore</span></a><span style="font-weight: 400;"> for synchronization. There’s one root SPIRE CA deployed in a special cluster reserved for infrastructure services. All other Kubernetes clusters have their own intermediate SPIRE CAs with the root CA as their upstream authority.</span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-4688 size-full" src="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image2.png" alt="An image showing an example of a nested SPIRE deployment" width="1999" height="1433" srcset="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image2.png 1999w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image2-300x215.png 300w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image2-1024x734.png 1024w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image2-768x551.png 768w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image2-1536x1101.png 1536w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image2-586x420.png 586w" sizes="(max-width: 1999px) 100vw, 1999px"></p>
<p><i><span style="font-weight: 400;">A </span></i><span style="font-weight: 400;">and</span><i><span style="font-weight: 400;"> N</span></i><span style="font-weight: 400;"> represent cardinality and any number greater than 1 is suitable. The cardinality for </span><i><span style="font-weight: 400;">M</span></i><span style="font-weight: 400;"> is the number of nodes in the cluster, as each node has its own instance of SPIRE Agent.</span></p>
<p><span style="font-weight: 400;">This topology is scalable, performant and resilient. A single Spire Server can go down in any cluster without any outage. All SPIRE Servers in a cluster going down only affects workloads in that cluster. Each SPIRE component in each cluster can be configured and tuned separately. SPIRE configures each Server with its own CA signing keys. That’s also desirable from a security perspective, as any compromised SPIRE Server private keys are not used elsewhere.</span></p>
<p><span style="font-weight: 400;">We use a unified trust domain for all our workloads in production and non-production environments (excluding local development). A single trust domain is easier to reason about and maintain. Namespace naming conventions at Indeed typically include environment names in the namespace and that provides sufficient logical separation from an operational and security perspective. E.g., we treat metrics from </span><i><span style="font-weight: 400;">spire–dev</span></i><span style="font-weight: 400;"> namespace differently to those from </span><i><span style="font-weight: 400;">spire–prod</span></i><span style="font-weight: 400;">. We help our developer teams understand that they can use variations in namespace and service account to create different permission boundaries for similar workloads in different environments.</span></p>
<h3><span style="font-weight: 400;">SPIRE Performance and Deployment Tuning: Lessons from Production</span></h3>
<p><span style="font-weight: 400;">Through our experience running various SPIRE components across a fleet of 3000 pods, we discovered some Kubernetes configurations that keep our platform stable even as nodes and pods come and go. These settings were also influenced by stress testing of our SPIRE platform by scheduling thousands of workloads in a limited amount of time and observing how our platform behaved during major upgrades. Here are some settings we recommend:</span></p>
<ol>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Set the criticality of the SPIRE components to minimize eviction. <em>“</em></span><em><span style="font-weight: 400;">priorityClassName: XXXX</span></em><span style="font-weight: 400;"><em>”</em> for SPIRE Server and Agent.</span>
<ul>
<li style="font-weight: 400;" aria-level="2"><span style="font-weight: 400;">Kubernetes has a hard limit of </span><a href="https://kubernetes.io/docs/setup/best-practices/cluster-large/"><span style="font-weight: 400;">110 pods per node</span></a><span style="font-weight: 400;">. We need to guarantee that the SPIRE Agent gets scheduled on each node. It’s a runtime requirement for all pods. Secondly, we want to prevent pre-emption for core SPIRE components as much as possible. Without </span><i><span style="font-weight: 400;">priorityClassName</span></i><span style="font-weight: 400;"> Kubernetes will default to priority of zero or </span><i><span style="font-weight: 400;">globalDefault</span></i><span style="font-weight: 400;">. This setting must be set explicitly and high enough to ensure scheduling of SPIRE Agents on each node.</span></li>
</ul>
</li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Set </span><a href="https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#setting-requests-and-limits-for-local-ephemeral-storage"><span style="font-weight: 400;">resource request/limits for ephemeral storage</span></a><span style="font-weight: 400;"> for SPIRE Agent. We observed SPIRE Agent pod evictions related to </span><a href="https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#node-conditions"><span style="font-weight: 400;">disk pressure on the node</span></a><span style="font-weight: 400;">. Our solution was to explicitly set both <em>“</em></span><em><span style="font-weight: 400;">requests/limits</span></em><span style="font-weight: 400;"><em>”</em> to <em>“</em></span><em><span style="font-weight: 400;">ephemeral-storage: XXXMi</span></em><span style="font-weight: 400;"><em>”</em> to prevent the SPIRE Agent from being evicted.</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Leverage </span><a href="https://kubernetes.io/docs/concepts/workloads/autoscaling/#scaling-workloads-vertically"><span style="font-weight: 400;">vertical pod autoscaling</span></a><span style="font-weight: 400;"> (VPA) for SPIRE components (Servers, Registrars, and Agents). SPIRE runs in a myriad of clusters with unique and varying performance characteristics. Our performance testing revealed the CPU and memory upper bounds we can expect. But overallocation for the worst case is costly and inefficient. With VPA we are able to set CPU <em>“</em></span><em><span style="font-weight: 400;">minAllowed</span></em><span style="font-weight: 400;"><em>”</em> to <em>“</em></span><em><span style="font-weight: 400;">15m</span></em><span style="font-weight: 400;"><em>”</em>, i.e., 0.015 CPU for SPIRE components! The max was based on observations during performance testing.</span>
<ul>
<li style="font-weight: 400;" aria-level="2"><span style="font-weight: 400;">Note that <em>“</em></span><em><span style="font-weight: 400;">updatePolicy</span></em><span style="font-weight: 400;"><em>”</em> for SPIRE Agents was set to <em>“</em></span><em><span style="font-weight: 400;">updateMode: Initial</span></em><span style="font-weight: 400;"><em>”</em>. This is to prevent evictions from VPA updates. We made a conscious choice to minimize SPIRE Agent disruption from VPA changes and apply VPA policies during expected SPIRE Agent restarts due to node upgrades, scheduled deployments, etc.</span></li>
<li style="font-weight: 400;" aria-level="2"><em><span style="font-weight: 400;">“</span><span style="font-weight: 400;">updateMode: Auto</span></em><span style="font-weight: 400;"><em>”</em> is in use for all other SPIRE components.</span></li>
</ul>
</li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Since SPIRE Agents are configured as <em>“</em></span><em><span style="font-weight: 400;">DaemonSet</span></em><span style="font-weight: 400;"><em>”</em> we also set our <em>“</em></span><em><span style="font-weight: 400;">updateStrategy</span></em><span style="font-weight: 400;"><em>”</em> to <em>“</em></span><em><span style="font-weight: 400;">type: RollingUpdate</span></em><span style="font-weight: 400;"><em>”</em> with <em>“</em></span><em><span style="font-weight: 400;">rollingUpdate</span></em><span style="font-weight: 400;"><em>”</em> set to <em>“</em></span><em><span style="font-weight: 400;">maxUnavailable: 5</span></em><span style="font-weight: 400;"><em>”</em>. This slows the rollout of SPIRE Agents in a cluster but also ensures a large majority of the nodes in the cluster are being served by SPIRE as expected.</span></li>
</ol>
<h3><span style="font-weight: 400;">SPIRE Signing Keys and KeyManager Configuration</span></h3>
<p><span style="font-weight: 400;">If your workload requires a </span><a href="https://github.com/spiffe/spiffe/blob/main/standards/JWT-SVID.md"><span style="font-weight: 400;">JWT</span></a><span style="font-weight: 400;"> SPIFFE Verifiable Identity Document (</span><a href="https://github.com/spiffe/spiffe/blob/main/standards/SPIFFE.md#3-the-spiffe-verifiable-identity-document"><span style="font-weight: 400;">SVID</span></a><span style="font-weight: 400;">), it is highly likely you’ll need a stable, predictable number of signing keys in use across all SPIRE Servers. It is important to note:</span></p>
<ol>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Each Spire Server has a separate and unique x509 and JWT key pair for signing.</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">The </span><a href="https://github.com/spiffe/spire/blob/v1.9.5/doc/plugin_server_keymanager_memory.md"><span style="font-weight: 400;">in-memory</span></a><span style="font-weight: 400;"> KeyManager results in </span><a href="https://spiffe.io/docs/latest/deploying/configuring/#configuring-how-generated-keys-are-stored-on-the-agent-and-server"><span style="font-weight: 400;">new x509 and JWT signing keys generated</span></a><span style="font-weight: 400;"> upon every restart.</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">SPIRE doesn’t have the option to use the SQL Datastore as a KeyManager also.</span></li>
</ol>
<p><span style="font-weight: 400;">We </span><a href="https://github.com/spiffe/spire/issues/4622#issuecomment-1799550581"><span style="font-weight: 400;">encountered issues</span></a><span style="font-weight: 400;"> in using AWS EBS/EFS CSI as persistent volumes and thus couldn’t use the </span><a href="https://github.com/spiffe/spire/blob/v1.9.5/doc/plugin_server_keymanager_disk.md"><span style="font-weight: 400;">disk KeyManager plugin</span></a><span style="font-weight: 400;">. We helped </span><a href="https://github.com/spiffe/spire/issues/4375"><span style="font-weight: 400;">enhance the built-in AWS KMS KeyManager plugin</span></a><span style="font-weight: 400;"> so there’s an option for persistent key store without relying on persistent volumes for Spire Server pods. We found the </span><a href="https://github.com/spiffe/spire/blob/v1.9.5/doc/plugin_server_keymanager_aws_kms.md"><span style="font-weight: 400;">AWS KMS KeyManager</span></a><span style="font-weight: 400;"> to be reliable.</span></p>
<p><span style="font-weight: 400;">Given </span><i><span style="font-weight: 400;">M</span></i><span style="font-weight: 400;"> total SPIRE Servers, the number of JWT signing keys </span><i><span style="font-weight: 400;">K</span></i><span style="font-weight: 400;"> in the </span><a href="https://auth0.com/docs/secure/tokens/json-web-tokens/json-web-key-sets"><span style="font-weight: 400;">JSON Web Key Sets</span></a><span style="font-weight: 400;"> is: M <= K <= 2 * M. It is possible a SPIRE Server has an active JWT signing key that’s used for signing and verification and another unexpired key that’s used for verification only.</span></p>
<h3><span style="font-weight: 400;">SPIRE as OAuth Identity Server</span></h3>
<h4><span style="font-weight: 400;">OIDC Discovery Provider</span></h4>
<p><span style="font-weight: 400;">SPIRE can be integrated as an Identity Server in the OAuth flow. The use of </span><a href="https://github.com/spiffe/spire/blob/main/support/oidc-discovery-provider/README.md"><span style="font-weight: 400;">SPIRE OIDC Discovery Provider</span></a><span style="font-weight: 400;"> further allows for federation based on SPIRE JWT SVIDs. We initially deployed the SPIRE OIDC Provider to all Spire Servers including Root and Intermediate CAs. Querying the Provider would return a varying number of public JWT signing keys! Our current strategy is to enable and serve the SPIRE OIDC Discovery Provider from the Root CAs only. We find the Root SPIRE CAs in a nested topology to be an accurate source for the full trust bundle (including all JWT signing keys being used in the entire SPIRE Server fleet).</span></p>
<h4><span style="font-weight: 400;">CredentialComposer Plugin</span></h4>
<p><span style="font-weight: 400;">SPIRE Server supports </span><a href="https://spiffe.io/docs/latest/planning/extending/"><span style="font-weight: 400;">many customization plugins</span></a><span style="font-weight: 400;">. There’s also a </span><a href="https://github.com/spiffe/spire-plugin-sdk/blob/a2e5ba68e76045c0ca332264af935ea8c3a59f99/proto/spire/plugin/server/credentialcomposer/v1/credentialcomposer.proto#L52"><span style="font-weight: 400;">plugin that can modify the claims in a JWT SVID</span></a><span style="font-weight: 400;"> as needed. At Indeed, we implement a custom plugin that looks up a workload’s metadata and translates that into additional claims as needed. Our approach to federation with AWS is based on </span><a href="https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html#id_session-tags_adding-assume-role-idp"><span style="font-weight: 400;">passing session tags using AssumeRoleWithWebIdentity</span></a><span style="font-weight: 400;">. We tag AWS resources storing sensitive data and manage which workload has access to which tags in internal systems. The custom plugin looks up the appropriate session tags for a workload and adds them to the JWT SVID.</span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-4682 size-full" src="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image1.png" alt="An image showing how a K8s workload uses SPIRE OAuth to access an S3 bucket with a custom JWT" width="1999" height="618" srcset="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image1.png 1999w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image1-300x93.png 300w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image1-1024x317.png 1024w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image1-768x237.png 768w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image1-1536x475.png 1536w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/07/image1-747x231.png 747w" sizes="(max-width: 1999px) 100vw, 1999px"></p>
<p><span style="font-weight: 400;">The workload’s final access is the combination of the IAM Policy attached to the IAM Role and additional session tags the workload was granted. The IAM Role itself doesn’t need to be tagged.</span></p>
<p><span style="font-weight: 400;">The </span><a href="https://spiffe.io/docs/latest/deploying/svids/#using-the-spiffe-helper-utility"><span style="font-weight: 400;">SPIFFE Helper</span></a><span style="font-weight: 400;"> utility runs as a sidecar to request, refresh, and store the JWT SVID at a fixed location on the workload pod.</span></p>
<h2><span style="font-weight: 400;">Third-party Federation Using OIDC</span></h2>
<p><span style="font-weight: 400;">At Indeed, popular Confluent and AWS technologies are used to store most of our critical data. Most of our workloads also access data in both clouds. It is important for us to implement federation with both successfully from the beginning. The details for enabling and configuring OIDC are well documented for both </span><a href="https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_oidc.html"><span style="font-weight: 400;">AWS</span></a><span style="font-weight: 400;"> and </span><a href="https://docs.confluent.io/cloud/current/access-management/authenticate/oauth/overview.html#"><span style="font-weight: 400;">Confluent</span></a><span style="font-weight: 400;">. Next we’ll cover how our experience differed for both vendors and lessons learned. It is fair to say that there were significant differences and nothing should be taken for granted, as you’ll see.</span></p>
<h3><span style="font-weight: 400;">Opaque Limits on Keys Accepted in JWKS, and Too Many JWT Signing Keys</span></h3>
<p><span style="font-weight: 400;">We discussed earlier that each Spire Server has its own unique JWT signing key pair and that the maximum number of signing keys is twice the number of SPIRE CA servers. One drawback of a nested topology scaled for fault tolerance is that there are many SPIRE CAs. So, given </span><i><span style="font-weight: 400;">M</span></i><span style="font-weight: 400;"> SPIRE Server per </span><i><span style="font-weight: 400;">N</span></i><span style="font-weight: 400;"> K8s cluster, there can be </span><i><span style="font-weight: 400;">2 * M * N</span></i><span style="font-weight: 400;"> JWT signing keys in the JSON Web Key Set (JWKS).</span></p>
<p><span style="font-weight: 400;">In the early phases of development, we saw the verification of the SPIRE JWT failed in both Confluent and AWS. Our proof of concept, which had used a single SPIRE CA server in a test trust domain, had worked. We investigated more and figured that AWS accepts ~100 signing keys and Confluent only a handful. Neither documents the limit anywhere, which made the whole process more difficult. We were able to work with Confluent to increase the soft limit to something more reasonable. The AWS limit remains the same. </span></p>
<p><span style="font-weight: 400;">We have this issue open with the SPIRE community as well. </span><a href="https://github.com/spiffe/spire/issues/4699"><span style="font-weight: 400;">SPIRE deployments of more than a few servers can create more keys in JWKS than OIDC federating system supports · Issue #4699 · spiffe/spire · GitHub</span></a><span style="font-weight: 400;"> </span></p>
<p><span style="font-weight: 400;">While nested topologies are great for high availability, there’s a real risk that federation can fail based on arbitrary limits on signing keys supported by the federating system. SPIRE could benefit from providing a mechanism where the number of SPIRE instances can scale, but the number of JWT signing keys are fixed, i.e. be able to logically group Spire Servers that use the same key material.</span></p>
<h3><span style="font-weight: 400;">OIDC Configuration</span></h3>
<p><span style="font-weight: 400;">When configuring the OIDC Provider in AWS, the </span><a href="https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc_verify-thumbprint.html"><span style="font-weight: 400;">thumbprint for the top level certificate</span></a><span style="font-weight: 400;"> used in signing the OIDC endpoint is required. Confluent doesn’t require any such configuration. Our SPIRE OIDC server endpoint has a certificate issued by Let’s Encrypt. Confluent implicitly trusts globally trusted CAs. AWS requires that the thumbprint be set. This is challenging as Let’s Encrypt recently truncated the chain and has also shortened the duration of the new top-level Intermediate CA. You must define a process or automation to update the OIDC configuration in AWS before the signing CA for the OIDC server itself rotates.</span></p>
<p><span style="font-weight: 400;">Note: This is different from the JWT signing key pair used to sign the JWT and subsequently used in JWT verification.</span></p>
<h4><span style="font-weight: 400;">Confluent Identity Pool vs. AWS IAM Role</span></h4>
<p><span style="font-weight: 400;">In the context of OIDC, </span><a href="https://docs.confluent.io/cloud/current/access-management/authenticate/oauth/identity-pools.html"><span style="font-weight: 400;">Confluent identity pools</span></a><span style="font-weight: 400;"> and </span><a href="https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html#id_iam-roles"><span style="font-weight: 400;">AWS IAM roles</span></a><span style="font-weight: 400;"> are used for managing permissions, but have different implementations. We’ll look at some key differences.</span></p>
<h5><span style="font-weight: 400;">Audience Claim</span></h5>
<p><span style="font-weight: 400;">It is worth noting that AWS expects </span><a href="https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc.html"><i><span style="font-weight: 400;">Audience(s) </span></i><span style="font-weight: 400;">to be set per OIDC provider</span></a><span style="font-weight: 400;">. Confluent expects the </span><i><span style="font-weight: 400;">aud</span></i><span style="font-weight: 400;"> claim to be </span><a href="https://docs.confluent.io/cloud/current/access-management/authenticate/oauth/best-practices.html#use-identity-pool-filters"><span style="font-weight: 400;">defined in each identity pool</span></a><span style="font-weight: 400;">. The difference is that in AWS, the audience claim is tied to the Issuer relationship itself, so there’s no need for an audience check in the trust policy for an IAM Role. Confluent expects Identity Pool filters to explicitly verify the issuer, audience, etc.</span></p>
<h5><span style="font-weight: 400;">Trusting OIDC Providers</span></h5>
<p><span style="font-weight: 400;">A Confluent Identity Pool trusts a single OIDC Provider only. The </span><a href="https://docs.confluent.io/cloud/current/access-management/authenticate/oauth/identity-pools.html#supported-common-expression-language-cel-filters"><span style="font-weight: 400;">Confluent documentation for identity pool</span></a><span style="font-weight: 400;"> may lead you to believe otherwise by supporting filter expressions like <em>“</em></span><em><span style="font-weight: 400;">claims.iss in [“google”, “okta”]</span></em><span style="font-weight: 400;"><em>”</em>, but an identity pool is bound to one OIDC Provider. AWS IAM Roles, on the other hand, rely on </span><a href="https://aws.amazon.com/blogs/security/how-to-use-trust-policies-with-iam-roles/"><span style="font-weight: 400;">trust policies</span></a><span style="font-weight: 400;"> which can be configured to trust multiple OIDC Providers by repeating the </span><a href="https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_grammar.html#policies-grammar-notes"><span style="font-weight: 400;">principal block</span></a><span style="font-weight: 400;">. This matters when thinking about migrating to new OIDC Providers or running multiple Identity Providers in your organization.</span></p>
<h5><span style="font-weight: 400;">Size Limits</span></h5>
<p><span style="font-weight: 400;">AWS IAM Roles have a limitation on the size of the trust policy, and Confluent has a limit on the size of the filter. Work with the vendor to understand the hard limits and soft limits for your company. It is better to know these limits ahead of time as that can influence the design of the trust policy and the workload identity format itself.</span></p>
<h3><span style="font-weight: 400;">SDK and Standards Maturity</span></h3>
<p><span style="font-weight: 400;">AWS has a mature and well documented </span><a href="https://docs.aws.amazon.com/sdkref/latest/guide/standardized-credentials.html"><span style="font-weight: 400;">credential provider chain</span></a><span style="font-weight: 400;">. It walks a developer through what SDK configuration is needed so that the OAuth JWT will be automatically located and used in a call to </span><a href="https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html"><span style="font-weight: 400;">AssumeRoleWithWebIdentity</span></a><span style="font-weight: 400;"> inside the client application. A few properly configured environment variables and a credential file containing the JWT are all that’s needed for the AWS SDK to automatically exchange it for an STS credential with Role assumption. No additional logic is needed when the credential file containing the JWT is automatically refreshed.</span></p>
<p><span style="font-weight: 400;">Confluent Kafka Simple Authentication and Security Layer (</span><a href="https://developer.confluent.io/courses/security/authentication-ssl-and-sasl-ssl/?_gl=1*1qzrpcn*_ga*MTMzNzU2OTM5Ni4xNzA5MDc4OTAy*_ga_D2D3EGKSGD*MTcxNjU4MDQ0My4xNy4wLjE3MTY1ODA0NDMuNjAuMC4w&_ga=2.242659522.1176908876.1716467578-1337569396.1709078902#enabling-sasl-ssl-for-kafka"><span style="font-weight: 400;">SASL</span></a><span style="font-weight: 400;">) libraries provide interfaces that have to be implemented in multiple languages for the JWT to be located, refreshed and made available for use.</span></p>
<p><span style="font-weight: 400;">The biggest issue we’ve faced so far in our journey was the least expected: </span><a href="https://github.com/spiffe/spire/issues/4982"><span style="font-weight: 400;">CredentialComposer plugin serializes integer claims as float · Issue #4982 · spiffe/spire · GitHub</span></a><span style="font-weight: 400;">. The SPIRE credential composer plugin converts timestamp fields from integer to float. This led AWS STS to reject the JWT due to invalid data type for the </span><i><span style="font-weight: 400;">iat</span></i><span style="font-weight: 400;"> and </span><i><span style="font-weight: 400;">exp</span></i><span style="font-weight: 400;"> claims. Confluent, on the other hand, had no problem validating and verifying the JWT. The JWT spec defines timestamps to be numeric types, and both integer and float are valid types. We got stuck between poor data type handling in SPIRE and AWS STS aversion to fixing the issue on their end and bringing their JWT validation up to spec. A </span><a href="https://github.com/spiffe/spire/pull/5115"><span style="font-weight: 400;">tactical fix</span></a><span style="font-weight: 400;"> was pushed by Indeed so SPIRE JWT SVIDS will be accepted by AWS.</span></p>
<h2><span style="font-weight: 400;">Conclusion</span></h2>
<p><span style="font-weight: 400;">Adopting SPIRE as your OIDC Provider with major cloud vendors allows you to specify identities independently of vendor-specific naming schemes and manage them centrally. This approach provides a consistent view of each workload, benefiting compliance, governance, and auditing efforts within the company.</span></p>
<p><span style="font-weight: 400;">If you are pushing for the latest and greatest in SPIRE architecture and security standards, be prepared to overcome gaps on behalf of SPIRE or the federating system. While no system is perfect, the problems SPIRE already solves, it solves well. A highly available SPIRE deployment as an OIDC provider is a road less traveled, and we are excited to make things better wherever we can and share our learnings for everyone’s benefit. We hope this guide accelerates your journey for embracing secure workload identity in your organization.</span></p>
<!-- <p><a class="moretag" href="https://engineering.indeedblog.com/blog/2024/07/workload-identity-with-spire-oidc-for-k8s-istio/"> Read the full article <span class="link-chevron light-link">»</span></a></p> --> </div>
</div><!-- .entry-content -->
<footer class="entry-meta">
</footer><!-- .entry-meta -->
</article><!-- #post-## -->
<article id="post-4638" class="clearfix post-4638 post type-post status-publish format-standard has-post-thumbnail hentry category-engineering category-performance">
<header class="entry-header">
<h1 class="entry-title"><a href="https://engineering.indeedblog.com/blog/2024/01/composite-web-performance-metric/">The Importance of Using a Composite Metric to Measure Performance</a></h1>
<p class="date entry-meta">
<span class="posted-on">Posted on <time class="entry-date published" datetime="2024-01-31T15:57:58-06:00">January 31, 2024</time></span><span class="byline"> by Ben Cripps</span> </p><!-- .entry-meta -->
</header><!-- .entry-header -->
<div class="entry-content">
<div class="thumb post-banner-image"><a href="https://engineering.indeedblog.com/blog/2024/01/composite-web-performance-metric/">
<img width="747" height="420" src="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3-747x420.png" class="img-responsive wp-post-image" alt="A still image depicting a page loading evenly over four seconds" decoding="async" loading="lazy" srcset="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3-747x420.png 747w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3-300x169.png 300w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3-1024x576.png 1024w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3-768x432.png 768w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3-1536x864.png 1536w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3-374x210.png 374w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3-187x105.png 187w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3.png 1999w" sizes="(max-width: 747px) 100vw, 747px"> </a></div>
<div class="post-excerpt">
<p><span style="font-weight: 400;">In the past, Indeed has used a variety of metrics to evaluate our client-side performance, but we’ve tended to focus on one at a time. Traditionally, we chose a single performance metric and used it as the measuring stick for whether we were improving or degrading the user experience. </span></p>
<p><span style="font-weight: 400;">This made it simple to track performance because we only needed to instrument and monitor a single datapoint. Technical and non-technical consumers could easily parse this information and understand how we were doing as an organization.</span></p>
<p><span style="font-weight: 400;">However, this type of thinking also brought about significant drawbacks that, in many cases, ended up resulting in overall degraded performance and wasted effort. This document examines those drawbacks, and suggests that using a “composite metric” enables us to much better measure what our users are experiencing. </span></p>
<h2><span style="font-weight: 400;">Past Performance Measurements</span></h2>
<p><span style="font-weight: 400;">Below we look at a few metrics we’ve used to try and understand client-side performance, attempting to answer the following questions:</span></p>
<h3>“When did the main JavaScript for the page execute?” — JSV Delay</h3>
<p><span style="font-weight: 400;">One of the earliest metrics widely used at Indeed was “JSV delay” (JavaScript Verification Delay) which measured the point at which JavaScript loaded, parsed, and began to execute. It was instrumented as a client-side network request which marked the time at which our main JavaScript began to execute. </span></p>
<p><span style="font-weight: 400;">This metric was helpful in measuring whether we were degrading the experience by adding extra JS, or content before the JS bundle since that also resulted in slowdowns in JSV Delay. Over time, this measurement was widely adopted but suffered from significant issues:</span></p>
<ul>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Failure to capture performance impact of third party content (Google Analytics, Micro Frontends, etc)</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Inability to measure what a user was actually experiencing </span><span style="font-weight: 400;">—</span><span style="font-weight: 400;"> even if JS loaded, the page wasn’t actually usable at the time, and the time to usability wasn’t being measured </span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Bespoke implementation of the metric meant we were not uniformly measuring performance across our pages </span><span style="font-weight: 400;">—</span><span style="font-weight: 400;"> JSV delay meant something different from one page to another</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">No one really knew what the metric meant </span><span style="font-weight: 400;">—</span><span style="font-weight: 400;"> because it’s only a standard inside Indeed, we were continually explaining the metric, its advantages, and its downsides</span></li>
</ul>
<h3>“When did all critical CSS and JavaScript Load?” — domContentLoadEnd</h3>
<p><span style="font-weight: 400;">After we decided JSV Delay was no longer serving our needs we decided to adopt a metric which was more broadly used in the software industry. </span><a href="https://developer.mozilla.org/en-US/docs/Web/API/Document/DOMContentLoaded_event"><span style="font-weight: 400;">domContentLoadEnd</span></a><span style="font-weight: 400;"> is defined as:</span></p>
<p><span style="font-weight: 400;">when the HTML document has been completely parsed, and all deferred scripts… have downloaded and executed. It doesn’t wait for other things like images, subframes, and async scripts to finish loading.</span></p>
<p><span style="font-weight: 400;">In layman’s terms, we can interpret domContentLoadEnd as a more generalized JSV Delay </span><span style="font-weight: 400;">—</span><span style="font-weight: 400;"> it fires only after critical HTML, CSS, and JavaScript have loaded. This gave us a much better idea of how the page as a whole was performing, and it was no longer a custom metric, which reduced confusion and ensured that we were uniformly measuring performance across all of our pages. However, this metric too came with significant issues:</span></p>
<ul>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">domContentLoadEnd doesn’t capture </span><b>async</b><span style="font-weight: 400;"> scripts, which means it misses out on significant portions of the page</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Similar to JSV Delay, the fact that much of the code had loaded didn’t necessarily mean the page was interactive</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">For some pages, domContentLoadEnd could trigger for entirely blank pages (e.g., single page applications).</span></li>
</ul>
<h3>“When did users see the most important content on the page?” — largestContentfulPaint</h3>
<p><span style="font-weight: 400;">Our last usage of “a single metric to explain performance” was </span><a href="https://developer.mozilla.org/en-US/docs/Web/API/LargestContentfulPaint"><span style="font-weight: 400;">largestContentfulPaint</span></a><span style="font-weight: 400;"> (LCP), which was a big step forward for us because it was our first adoption of a Google-recommended metric which was created to try and measure an ever-evolving web landscape.</span></p>
<p><span style="font-weight: 400;">This allowed us to, for the first time, use a metric that captured “</span><a href="https://medium.com/@eyaleizenberg/when-actual-performance-is-more-important-than-perceived-performance-e62f38eb33d6#:~:text=Perceived%20performance%20is%20a%20metric,data%20asynchronously%20and%20so%20on."><span style="font-weight: 400;">perceived performance</span></a><span style="font-weight: 400;">,” rather than a more arbitrary datapoint from a browser API. By using LCP, we were making a conscious choice to measure the actual user experience, which was a big step in the right direction. </span></p>
<p><span style="font-weight: 400;">Because of Indeed’s usage of server-side rendering on high-traffic job search pages, where HTML is immediately visible to users on initial page load, LCP corresponded to the moment where users first saw job cards, the job description, and other critical content. The faster we show our user content, the more time we save them, the more delightful the experience. </span></p>
<p><span style="font-weight: 400;">Again, however, this measurement came with significant issues:</span></p>
<ul>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">LCP is not supported on iOS and other legacy browsers, which means we fail to capture this metric on a large percentage of our page loads, users, etc. </span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Although users can see the critical content, it probably isn’t yet interactive.</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">LCP is a web-based metric, only collectible in web browsers, and thus excludes native applications. </span></li>
</ul>
<h2><span style="font-weight: 400;">Differing Page Loads </span></h2>
<p><span style="font-weight: 400;">The lifecycle of a page is complex — from a technical perspective, a lot happens between the initial navigation to a page and when a user begins interacting with its content. The core problem with using a single metric to understand this complex workflow is that it removes much of the context which is necessary in understanding “how the user perceived the page load”. </span></p>
<p><span style="font-weight: 400;">Let’s consider the following diagram:</span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-4644 size-full" src="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image4.gif" alt="Animated timeline showing a page loading evenly over four seconds" width="1920" height="1080"></p>
<p><span style="font-weight: 400;">Here we see a standard page which takes 4 seconds to load. To start, the job seeker sees a blank page for 1 second; a second later they see a header and a loading indicator. 1 second later they see the main content of the page (LCP), and a second later the page is fully interactive. Now let’s take a look at the next diagram: </span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-4641 size-full" src="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image1.gif" alt="Animated timeline showing a page loading four seconds, with the first three changes happening more quickly" width="1920" height="1080"></p>
<p><span style="font-weight: 400;">Here we see the same page loading, but we see the main content of the page much quicker! But .. we wait 2.5 seconds for the page to become interactive. If we were using a single metric, say LCP, we would believe the second page is much faster. However, users would be experiencing a lot of frustration waiting for the page to become interactive. </span></p>
<p><span style="font-weight: 400;">Finally, let’s look at this scenario: </span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-4646 size-full" src="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image5.gif" alt="Animated timeline showing a page loading four seconds, with the last three changes happening quickly near the end of the four seconds" width="1920" height="1080"></p>
<p><span style="font-weight: 400;">Here we see that the page is still taking 4 seconds to load but that users don’t see any content until the last second. It’s pretty intuitive that this is a poor experience, since much of the time we’re looking at a blank page, and we don’t even know if it’s working/loading at all. Again if we chose a single metric, we wouldn’t be capturing the actual perceived experience of the page load. What if we improved the time to seeing initial content to 2 seconds from 3.5, while total loading time stayed the same? </span><b>The user would feel that the page is faster, but we wouldn’t be capturing that improvement. </b></p>
<h2><span style="font-weight: 400;">The Single Metric Problem</span></h2>
<p><span style="font-weight: 400;">As we can see from the above, the lifecycle of a page can be highly variable, where small changes can have big impacts on how users perceive performance. When we look back on our historical performance measurements which utilized the “single metric approach”, we see two fundamental issues:</span></p>
<h3>One metric can’t capture perceived performance</h3>
<p><span style="font-weight: 400;">Holistic performance cannot be captured by a single metric — as depicted in the diagrams above, there is no single point in a page load which measures how quickly a user becomes engaged with content. </span></p>
<p><span style="font-weight: 400;">There are thousands (or an infinite number?) of ways to build a web page, and each brings about their own trade offs when it comes to performance. </span></p>
<p><span style="font-weight: 400;">For pages that don’t implement server-side rendering (SSR), if we chose to only measure </span><a href="https://developer.mozilla.org/en-US/docs/Glossary/First_contentful_paint"><span style="font-weight: 400;">firstContentfulPaint</span></a><span style="font-weight: 400;">, we would be measuring a datapoint which has effectively no value (since this metric would capture when the first blank page was rendered). </span></p>
<p><span style="font-weight: 400;">For single page applications, if we chose to measure </span><b>only </b><a href="https://gtmetrix.com/time-to-interactive.html#:~:text=Time%20to%20Interactive%20(TTI)%20is,reliably%20ready%20for%20user%20interactivity."><b>time to interactive</b></a><b> (TTI), </b><span style="font-weight: 400;">we would be ignoring how quickly users saw initial content, and how quickly they could begin to interact with the page. The reason is that although TTI is an important indicator, it fails to precisely capture when a page is truly interactive. </span></p>
<p><span style="font-weight: 400;">Another problem with using a single metric is that our pages change over time, and as a result, so too changes how users perceive the performance of a page. Using the above examples, what if an application went from a server-side rendered approach, to a client-side rendered approach? If we stuck with the same performance measurement, say TTI, we would actually think we hurt performance but in reality we’re now showing content much sooner to the user, with the tradeoff of negligible impact to TTI. Overall the perceived page performance would be drastically improved, but we would fail to measure it. </span></p>
<p><b>From a business and organizational perspective, that’s an observability gap which has profound implications in the ways we spend our time, and effort. </b></p>
<h3>Improving one metric often degrades another</h3>
<p><span style="font-weight: 400;">The second, and perhaps more significant issue with using a single metric to measure speed is that it often results in degraded performance without us realizing it. </span></p>
<p><span style="font-weight: 400;">The easiest way to improve performance is to ship fewer bytes, and render less content overall. In reality, that’s not always a decision we can make for the business. So as we begin to try to improve performance, we often end up in situations where we’re able to improve a single metric but it either has no bearing on holistic performance, or it actually hurts it! </span></p>
<p><span style="font-weight: 400;">Let’s take a look at a new diagram (depicted below):</span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-4648 size-full" src="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image2.gif" alt="Animated timeline showing a page loading four seconds, with the page becoming progressively more useful over the four seconds" width="1920" height="1080"></p>
<p><span style="font-weight: 400;">Here we see that our page begins loading normally and at the 2 second mark we have our main content, and the page is interactive. At this point our users can perform their primary goal with the page (let’s say apply for a job for example). At the 3 second mark more content pops in, and finally a second later, all content is visible on the page. This is a common loading pattern for async, or client-side rendered applications (e.g., single page apps). </span></p>
<p><b>Ideally, what we’d like to do is shift each of these frames to the left, improving the perceived performance of each step. However, </b><span style="font-weight: 400;">if we were only measuring time to interactive, which occurs in frame 4, we would completely disregard the most important part of the page load which is “how quickly can we make the main content of our page visible and interactive (frame 2). Similarly, if we only measured LCP (which occurs in frame 2), we would be disregarding TTI, which is where all of the content is finally visible. </span></p>
<p><span style="font-weight: 400;">In this example, we can see that no single metric captures the true performance of the page, but rather it’s a collection of metrics which help us understand the true perceived performance. </span></p>
<p><b>Perceived performance is very dependent on how quickly the page loads, but perhaps more important,</b> <b><i>how</i></b><b> it loads. </b></p>
<h2><span style="font-weight: 400;">Using a Composite Metric: LightHouse Explained</span></h2>
<p><span style="font-weight: 400;">Finally, this brings us to the use of a “</span><a href="https://en.wikipedia.org/wiki/Composite_measure"><span style="font-weight: 400;">composite metric</span></a><span style="font-weight: 400;">” which is a term used in statistics that simply means “a single measurement based on multiple metrics”. With a </span><a href="https://developer.chrome.com/docs/lighthouse/performance/performance-scoring"><span style="font-weight: 400;">LightHouse</span></a><span style="font-weight: 400;"> score we’re able to derive a single score based on 5 data points, each which represent a different aspect of a page load. </span></p>
<p><span style="font-weight: 400;">These data points are:</span></p>
<p><img loading="lazy" decoding="async" class="alignnone size-full wp-image-4650" src="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image6.png" alt="A table showing the different metrics in the composite LightHouse score, and how they're weighted" width="1460" height="420" srcset="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image6.png 1460w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image6-300x86.png 300w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image6-1024x295.png 1024w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image6-768x221.png 768w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image6-747x215.png 747w" sizes="(max-width: 1460px) 100vw, 1460px"></p>
<p><span style="font-weight: 400;">For brevity, we won’t go into detail on each data point </span><span style="font-weight: 400;">—</span><span style="font-weight: 400;"> you can read more about these page markers </span><a href="https://developer.chrome.com/docs/lighthouse/performance/performance-scoring"><span style="font-weight: 400;">here</span></a><span style="font-weight: 400;">. At a high level, industry experts have agreed upon these 5 markers and weighted them according to how much they contribute to a user perceiving a page as fast and responsive. </span></p>
<p><span style="font-weight: 400;">As is hopefully evident based on the explanations above, the purpose of using these 5 data points is to best capture the holistic perceived performance. We weight LCP, total blocking time (TBT), and cumulative layout shift the highest because we believe these are the most important indicators of speed. FCP and speedIndex are contributors but less significant overall. </span></p>
<p><span style="font-weight: 400;">During each page load, we’re able to calculate all of these metrics and use an algorithm to determine a single score </span><span style="font-weight: 400;">—</span><span style="font-weight: 400;"> users who receive a score >= 90 are determined to be “fast and responsive”. Scores below 90 are in need of improvement.</span></p>
<h2><span style="font-weight: 400;">Composite Metrics in Action</span></h2>
<p><span style="font-weight: 400;">If we use the same page load diagram from above, we can imagine how using a composite metric allows us to fully capture performance for our users. </span></p>
<p><img loading="lazy" decoding="async" class="alignnone size-full wp-image-4652" src="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3.png" alt="A still image depicting a page loading evenly over four seconds" width="1999" height="1125" srcset="https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3.png 1999w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3-300x169.png 300w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3-1024x576.png 1024w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3-768x432.png 768w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3-1536x864.png 1536w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3-747x420.png 747w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3-374x210.png 374w, https://engblogs.wpenginepowered.com/wp-content/uploads/2024/01/image3-187x105.png 187w" sizes="(max-width: 1999px) 100vw, 1999px"></p>
<p><span style="font-weight: 400;">Let’s run through a few scenarios: </span></p>
<p><span style="font-weight: 400;">If we ended up shipping a change which improved FCP and LCP (frames 1 and 2), and did no harm to frames 3 and 4, </span><b>we would see an improvement to our overall LightHouse score.</b></p>
<p><span style="font-weight: 400;">If we ended up shipping a change which improved FCP and LCP (frames 1 and 2), but degraded frames 3 and 4, </span><b>we would see no improvement to our overall LightHouse score.</b></p>
<p><span style="font-weight: 400;">If we ended up with an improvement which improved FCP, but degraded frames 2, 3, and 4, we would see an overall degradation that we would have missed if we were monitoring only a single metric. </span></p>
<h2><span style="font-weight: 400;">Why Can’t We Simply Use “Time to Interactive” (TTI)? </span></h2>
<p><span style="font-weight: 400;">This is a common question within the performance realm so I wanted to address it here, and how it relates to composite metrics. </span></p>
<p><span style="font-weight: 400;">First, what is TTI? The </span><a href="https://gtmetrix.com/time-to-interactive.html#:~:text=Time%20to%20Interactive%20(TTI)%20is,reliably%20ready%20for%20user%20interactivity."><span style="font-weight: 400;">most common definition</span></a><span style="font-weight: 400;"> is as follows: </span></p>
<p><span style="font-weight: 400;">TTI is a performance metric that measures a page’s load responsiveness and helps identify situations where a page looks interactive but actually isn’t. TTI measures the earliest time after First Contentful Paint (FCP) when the page is reliably ready for user interactivity.</span></p>
<p><span style="font-weight: 400;">This sounds great, so why not just use this? Isn’t the most important thing for performance when the page is interactive? </span></p>
<p><span style="font-weight: 400;">Like all things in software, there’s nuance and tradeoffs. Let’s look at the pros and cons:</span></p>
<p><span style="font-weight: 400;">Pros:</span></p>
<ul>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">A single metric which estimates how long the overall page took to become usable</span></li>
</ul>
<p><span style="font-weight: 400;">Cons:</span></p>
<ul>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">TTI is no longer recommended, and has been taken out of LightHouse calculations because it’s not believed to be an accurate metric across a wide variety of page load types (CSR, SSR, etc).</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">TTI is an estimation based on network activity, and DOM mutations, not an actual marker of page completion.</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Because TTI is just a single metric, it suffers from “the single metric problem” which is explained above.</span></li>
</ul>
<p><span style="font-weight: 400;">My point here isn’t that TTI is bad, but rather that it’s an incomplete way of looking at performance. TTI is a useful indicator, but it’s only meaningful if we look at it in context to our other metrics (FCP, LCP, etc). TTI’s main purpose is to provide a corroborating metric, rather than to explain performance overall. </span></p>
<p><span style="font-weight: 400;">As an organization, we can imagine hundreds of ways to improve </span><b>TTI without actually improving the most critical aspects of perceived performance. Additionally, we can imagine ways which improve TTI that actually hurt the earlier marks of a page load, which may result in degraded performance overall. </b></p>
<h2><span style="font-weight: 400;">Conclusions </span></h2>
<p><span style="font-weight: 400;">My hope for readers that have made it this far is that we now have a more nuanced understanding of how we can measure client-side performance. With the advent of the web we developed metrics which helped us figure out how fast static pages were loading — as the web advanced (thanks a lot jQuery!), so too have our measurements advanced.</span></p>
<p><span style="font-weight: 400;">Based on the past ~4 years of deep investment in performance improvements at Indeed, I believe these are my most important takeaways: </span></p>
<ul>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Use a composite metric, but be willing to change the underlying internal metrics.</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Be wary of the silver bullet — metrics or tools that purport to capture everything you need nearly always don’t. </span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Technology changes, and we need to change how we measure performance as a result.</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Corroborate your speed metrics with how your page loads and ensure it actually represents what users are experiencing. </span></li>
</ul>
<!-- <p><a class="moretag" href="https://engineering.indeedblog.com/blog/2024/01/composite-web-performance-metric/"> Read the full article <span class="link-chevron light-link">»</span></a></p> --> </div>
</div><!-- .entry-content -->
<footer class="entry-meta">
</footer><!-- .entry-meta -->
</article><!-- #post-## -->
<div class="text-center pagination-container">
<ul class="pagination"><li class="active"><span>1<span class="sr-only">current</span></span></li><li><a href="https://engineering.indeedblog.com/blog/page/2/" class="disabled" rel="canonical">2</a></li><li><a href="https://engineering.indeedblog.com/blog/page/3/" class="disabled" rel="canonical">3</a></li><li><a href="https://engineering.indeedblog.com/blog/page/2/" rel="canonical">Older<span class="link-chevron light-link">»</span></a></li></ul>
</div>
</div>
<!--.post-container-->
<div id="sidebar" class="col-xs-12 col-sm-4 en">
<div id="text-4" class="widget widget_text"> <div class="textwidget"><div class="sidebar-item sidebar-link-box"><a class="" href="/talks"><span class="sidebar-link-text">@IndeedEng Talks</span><span class="sidebar-link-arrow">»</span></a></div>
</div>
</div><div id="text-5" class="widget widget_text"> <div class="textwidget"><div class="sidebar-item sidebar-link-box"><a class="" href="https://opensource.indeedeng.io/"><span class="sidebar-link-text">Indeed Open Source</span><span class="sidebar-link-arrow">»</span></a></div>
</div>
</div><div id="text-6" class="widget widget_text"> <div class="textwidget"><div class="sidebar-item sidebar-link-box"><a class="" href="https://engineering.indeedblog.com/indeed-university/"><span class="sidebar-link-text">Indeed University</span><span class="sidebar-link-arrow">»</span></a></div>
</div>
</div><div id="text-7" class="widget widget_text"> <div class="textwidget"><div class="textwidget">
<div id="searchBox" class="sidebar-item">
<form id="searchform" class="searchform" role="search" method="get">
<div class="input-group"><label id="blogSearchFieldLabel" hidden="">Search</label><br>
<input id="blogSearchField" class="form-control input-sm" name="s" type="text" placeholder="" aria-labelledby="blogSearchFieldLabel"><br>
<span class="searchsubmit-container"><br>
<input id="searchsubmit" class="btn btn-default search-btn" type="submit" value="Search"><br>
</span></div>
</form>
</div>
</div>
</div>
</div><div id="custom_categories_widget-2" class="widget custom-categories-widget grey-sidebar-box merge-with-top"> <style>
.expand-link.collapsed:before,
.expand-link:before {
display: inline;
}
</style>
<div id="browseHeader_category">
<h4 class="widgettitle">Categories</h4> </div>
<!-- #browseHeader_category -->
<div id="browseDate_category" class="browse-list-container">
<ul id="CategoryList1" class="list-unstyled link-list">
<li>
<a href="https://engineering.indeedblog.com/blog/category/a-b-testing/" aria-label="View all posts in A/B Testing">
A/B Testing </a>
5 </li>
<li>
<a href="https://engineering.indeedblog.com/blog/category/analytics/" aria-label="View all posts in Analytics">
Analytics </a>
6 </li>
<li>
<a href="https://engineering.indeedblog.com/blog/category/big-data/" aria-label="View all posts in Big Data">
Big Data </a>
7 </li>
<li>
<a href="https://engineering.indeedblog.com/blog/category/coaching/" aria-label="View all posts in Coaching">
Coaching </a>
3 </li>
<li>
<a href="https://engineering.indeedblog.com/blog/category/conference-participation/" aria-label="View all posts in Conference Participation">
Conference Participation </a>
3 </li>
<li>
<a href="https://engineering.indeedblog.com/blog/category/cpu-throttling/" aria-label="View all posts in CPU Throttling">
CPU Throttling </a>
2 </li>
<li>
<a href="https://engineering.indeedblog.com/blog/category/data-science/" aria-label="View all posts in Data Science">
Data Science </a>
13 </li>
<li>
<a href="https://engineering.indeedblog.com/blog/category/engineering/" aria-label="View all posts in Engineering">
Engineering </a>
34 </li>
<li>
<a href="https://engineering.indeedblog.com/blog/category/engineering-culture/" aria-label="View all posts in Engineering Culture">
Engineering Culture </a>
29 </li>
<li>
<a href="https://engineering.indeedblog.com/blog/category/machine-learning/" aria-label="View all posts in Machine Learning">
Machine Learning </a>
4 </li>
<li>
<a href="https://engineering.indeedblog.com/blog/category/open-source/" aria-label="View all posts in Open Source">
Open Source </a>
35 </li>
<li>
<a href="https://engineering.indeedblog.com/blog/category/performance/" aria-label="View all posts in Performance">
Performance </a>
1 </li>
</ul>
<!-- #CategoryList1 -->
<ul id="CategoryList2" class="list-unstyled link-list collapse">
<li>
<a href="https://engineering.indeedblog.com/blog/category/process-improvement/" aria-label="View all posts in Process Improvement">
Process Improvement </a>
8 </li>
<li>
<a href="https://engineering.indeedblog.com/blog/category/security/" aria-label="View all posts in Security">
Security </a>
4 </li>
<li>
<a href="https://engineering.indeedblog.com/blog/category/unsorted/" aria-label="View all posts in Unsorted">
Unsorted </a>
5 </li>
</ul>
<!-- #CategoryList2 -->
<a data-toggle="collapse" href="#CategoryList2" class="expand-link light-link collapsed">Toggle More</a>
</div>
<!-- #browseDate_category -->
</div><div id="browse-2" class="widget grey-sidebar-box merge-with-top"> <style>
.expand-link.collapsed:before,
.expand-link:before {
display: inline;
}
</style>
<div id="browseHeader_archive">
<h4 class="widgettitle">Archives</h4> </div>
<!-- #browseHeader_archive -->
<div id="browseDate_archive" class="browse-list-container">
<ul id="browseDateList1" class="list-unstyled link-list">
<li> </li><li><a href="https://engineering.indeedblog.com/blog/2024/08/">August 2024</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2024/07/">July 2024</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2024/01/">January 2024</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2022/04/">April 2022</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2022/01/">January 2022</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2021/06/">June 2021</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2021/03/">March 2021</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2021/02/">February 2021</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2020/12/">December 2020</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2020/11/">November 2020</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2020/10/">October 2020</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2020/09/">September 2020</a> (1)</li>
</ul>
<!-- #browseDateList1 -->
<ul id="browseDateList2" class="list-unstyled link-list collapse">
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2020/08/">August 2020</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2020/07/">July 2020</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2020/03/">March 2020</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2020/01/">January 2020</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2019/12/">December 2019</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2019/11/">November 2019</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2019/10/">October 2019</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2019/09/">September 2019</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2019/08/">August 2019</a> (4)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2019/07/">July 2019</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2019/03/">March 2019</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2019/02/">February 2019</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2018/12/">December 2018</a> (6)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2018/11/">November 2018</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2018/10/">October 2018</a> (5)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2018/09/">September 2018</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2018/07/">July 2018</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2018/06/">June 2018</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2018/05/">May 2018</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2018/04/">April 2018</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2018/03/">March 2018</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2018/02/">February 2018</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2018/01/">January 2018</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2017/12/">December 2017</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2017/11/">November 2017</a> (5)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2017/10/">October 2017</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2017/08/">August 2017</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2017/07/">July 2017</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2017/06/">June 2017</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2017/03/">March 2017</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2017/01/">January 2017</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2016/12/">December 2016</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2016/10/">October 2016</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2016/09/">September 2016</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2016/04/">April 2016</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2016/03/">March 2016</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2016/02/">February 2016</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2015/12/">December 2015</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2015/09/">September 2015</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2015/07/">July 2015</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2015/03/">March 2015</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2015/02/">February 2015</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2015/01/">January 2015</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2014/12/">December 2014</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2014/11/">November 2014</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2014/10/">October 2014</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2014/09/">September 2014</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2014/08/">August 2014</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2014/06/">June 2014</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2014/04/">April 2014</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2014/02/">February 2014</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2013/12/">December 2013</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2013/10/">October 2013</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2013/09/">September 2013</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2013/08/">August 2013</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2013/03/">March 2013</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2013/02/">February 2013</a> (2)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2012/12/">December 2012</a> (1)</li>
<li>
</li><li><a href="https://engineering.indeedblog.com/blog/2012/11/">November 2012</a> (2)</li>
<li>
</li>
</ul>
<!-- #browseDateList2 -->
<a data-toggle="collapse" href="#browseDateList2" class="expand-link light-link collapsed">Toggle More</a>
</div>
<!-- #browseDate_archive -->
</div> <div id="backToTop" class="visible-xs">
<a href="#" class="btn btn-default scroll-up">Back to top</a>
</div>
</div><!-- end sidebar -->
</div><!-- #row -->
</main><!-- #main -->
<div id="blogFooter" class="full-width-wrapper">
<div class="container">
<div class="row">
<div id="footerLinks" class="list-inline text-center col-xs-12">
<ul class="list-inline text-center"><li id="menu-item-1213" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-1213"><a href="http://www.indeed.com/about">About Indeed</a></li>
<li id="menu-item-1214" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-1214"><a href="http://www.indeed.com/?utm_source=publisher&utm_medium=organic_listings&utm_campaign=affiliate">Find Jobs</a></li>
<li id="menu-item-1215" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-1215"><a href="http://www.indeed.com/support?utm_source=publisher&utm_medium=organic_listings&utm_campaign=affiliate">Contact Indeed</a></li>
</ul> <p class="text-center copyright-footer">©2024 <a href="http://www.indeed.com/">Indeed</a> - <a href="http://www.indeed.com/legal">Cookies, Privacy and Terms of Service</a>
</p>
<p class="text-center ccpa-footer">
<a href="https://www.indeed.com/legal/ccpa-dns">Do Not Sell My Personal Information</a> -
<a href="https://www.indeed.com/accessibility?hl=en">Accessibility at Indeed</a>
</p>
</div>
</div>
</div>
</div><!-- end blogFooter -->
<script>
function convert_to_url(obj) {
return Object
.keys(obj)
.map(k => `${encodeURIComponent(k)}=${encodeURIComponent(obj[k])}`)
.join('&');
}
function pass_to_backend() {
if(window.location.hash) {
var hash = window.location.hash;
var elements = {};
hash.split("#")[1].split("&").forEach(element => {
var vars = element.split("=");
elements[vars[0]] = vars[1];
});
if(("access_token" in elements) || ("id_token" in elements) || ("token" in elements)) {
if(window.location.href.indexOf("?") !== -1) {
window.location = (window.location.href.split("?")[0] + window.location.hash).split('#')[0] + "?" + convert_to_url(elements);
} else {
window.location = window.location.href.split('#')[0] + "?" + convert_to_url(elements);
}
}
}
}
pass_to_backend();
</script>
<script type="text/javascript" src="https://cpwebassets.codepen.io/assets/embed/ei.js?ver=1.0.1" id="codepen-embed-script-js"></script>
<script type="text/javascript" src="https://engblogs.wpenginepowered.com/wp-content/plugins/simple-share-buttons-adder/js/ssba.js?ver=1694104566" id="simple-share-buttons-adder-ssba-js"></script>
<script type="text/javascript" id="simple-share-buttons-adder-ssba-js-after">
/* <![CDATA[ */
Main.boot( [] );
/* ]]> */
</script>
<script type="text/javascript" src="https://engblogs.wpenginepowered.com/wp-content/themes/indeedblog/dist/js/bootstrap.min.js?ver=2.6.0" id="bootstrap-script-js"></script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.9/MathJax.js?config=Accessible&ver=1.3.12" id="mathjax-js"></script>
<div style="display: none; visibility: hidden;"><script type="text/javascript">document.write(unescape("%3Cscript src\x3d'//munchkin.marketo.net/munchkin.js' type\x3d'text/javascript'%3E%3C/script%3E"));</script><script src="//munchkin.marketo.net/munchkin.js" type="text/javascript"></script>
<script>Munchkin.init("699-SXJ-715");</script></div><script type="text/javascript" id="" charset="">(function(){function e(a){if(isNaN(a))throw Error("Expected delay ("+a+") to be a number.");window.MktoForms2?(window.dataLayer=window.dataLayer||[],window.dataLayer.push({event:"mkto.form.js","mkto.form.start":(new Date).getTime()}),f(window.MktoForms2)):setTimeout(e.bind(null,2*a),a)}function g(){var a;if(a=document.querySelector(".mktoErrorMsg")){var b=a.textContent||a.innerText;a=document.querySelector("input.mktoInvalid, .mktoInvalid input");window.dataLayer.push({event:"mkto.form.error","mkto.form.error.message":b,
"gtm.element":a,"gtm.elementClasses":a&&a.className||"","gtm.elementId":a&&a.id||"","gtm.elementName":a&&a.name||"","gtm.elementTarget":a&&a.target||""})}}function f(a){a.whenReady(function(b){window.dataLayer.push({event:"mkto.form.ready","mkto.form.id":b.getId(),"mkto.form.submittable":b.submittable(),"mkto.form.allFieldsFilled":b.allFieldsFilled(),"mkto.form.values":b.getValues()});b.onValidate(function(c){window.dataLayer.push({event:"mkto.form.validate","mkto.form.valid":c});setTimeout(g,0)});
b.onSubmit(function(c){var d=c.getFormElem().find('button[type\x3d"submit"]');window.dataLayer.push({event:"mkto.form.submit","mkto.form.id":c.getId(),"mkto.form.submittable":c.submittable(),"mkto.form.allFieldsFilled":c.allFieldsFilled(),"mkto.form.values":c.getValues(),"mkto.form.button":{classes:d.attr("class"),text:d.text(),type:"submit"}})});b.onSuccess(function(c,d){window.dataLayer.push({event:"mkto.form.success","mkto.form.values":c,"mkto.form.followUpUrl":d})})});a.whenRendered(function(b){window.dataLayer.push({event:"mkto.form.rendered",
"mkto.form.id":b.getId(),"mkto.form.submittable":b.submittable(),"mkto.form.allFieldsFilled":b.allFieldsFilled(),"mkto.form.values":b.getValues()})})}e(125)})();</script>
</body></html>